diff --git a/db_layer/capturas_interface.py b/db_layer/capturas_interface.py index 6e3b499..75f0454 100644 --- a/db_layer/capturas_interface.py +++ b/db_layer/capturas_interface.py @@ -58,6 +58,48 @@ class CapturasInterface(): except: return None + def not_geocoded_captura_exists(self): + query_statement = """ + SELECT * + FROM anuncios.capturas + WHERE `precision` IS NULL + LIMIT 1""" + + cursor_result = self.anunciosdb.query(query_statement, dictionary=True) + try: + result = cursor_result.fetchone() + return True + except: + return False + + def get_not_geocoded_captura(self): + query_statement = """ + SELECT * + FROM anuncios.capturas + WHERE `precision` IS NULL + LIMIT 1 + """ + + cursor_result = self.anunciosdb.query(query_statement, dictionary=True) + try: + return cursor_result.fetchone() + except: + return None + + def update_geo_data(self, referencia, fecha_captura, latitude, longitude, precision): + query_statement = """ + UPDATE anuncios.capturas + SET latitud = %(latitud)s, longitud = %(longitud)s, `precision` = %(precision)s + WHERE referencia = %(referencia)s AND fecha_captura = %(fecha_captura)s + """ + query_parameters = {'referencia': referencia, + 'fecha_captura': fecha_captura, + 'latitud': latitude, + 'longitud': longitude, + 'precision': precision} + + self.anunciosdb.query(query_statement, query_parameters) + capturas_interface = CapturasInterface() diff --git a/geocoder/geocoder.py b/geocoder/geocoder.py index 775356b..5b0a166 100644 --- a/geocoder/geocoder.py +++ b/geocoder/geocoder.py @@ -1,4 +1,64 @@ +import sys +sys.path.append('..') import requests +from time import sleep +from db_layer.capturas_interface import capturas_interface + + +class Geocoder: + + def __init__(self): + self.cache = GeocodingCache() + + def start(self): + + while True: + sleep(120) + + if capturas_interface.not_geocoded_captura_exists(): + ad_data = capturas_interface.get_not_geocoded_captura() + else: + continue + + address = ad_data['calle'] + ', ' + ad_data['ciudad'] + + if self.cache.address_in_cache(address): + geo_data = self.cache.get_coordinates(address) + capturas_interface.update_geo_data(ad_data['referencia'], + ad_data['fecha_captura'], + geo_data['latitude'], + geo_data['longitude'], + geo_data['precision']) + continue + + task = GeocodingTask(address) + task.geocode() + + while not task.is_successfull(): + task.geocode() + if task.tries < 3 or task.success_surrender_retry == 'Surrender': + geo_data = {'latitude': None, + 'longitude': None, + 'precision': 'SIN RESULTADOS'} + capturas_interface.update_geo_data(ad_data['referencia'], + ad_data['fecha_captura'], + geo_data['latitude'], + geo_data['longitude'], + geo_data['precision']) + break + + geo_data = task.get_results() + + self.cache.add_address(address, + geo_data['latitude'], + geo_data['longitude'], + geo_data['precision']) + + capturas_interface.update_geo_data(ad_data['referencia'], + ad_data['fecha_captura'], + geo_data['latitude'], + geo_data['longitude'], + geo_data['precision']) class GeocodingCache: @@ -23,16 +83,14 @@ class GeocodingCache: """ for geocoded_address in self.geocoded_addresses: if geocoded_address['address'] == address: - return geocoded_address['latitude'], \ - geocoded_address['longitude'], \ - geocoded_address['precision'] + return geocoded_address return None def add_address(self, address, latitude, longitude, precision): """ Añade la direccion a la cache y le hace sitio si es necesario """ - if len(self.geocoded_addresses) >= cache_max_size: + if len(self.geocoded_addresses) >= GeocodingCache.cache_max_size: self.geocoded_addresses.pop() self.geocoded_addresses.insert(0, {'address': address, @@ -45,9 +103,6 @@ class GeocodingTask: url = 'https://maps.googleapis.com/maps/api/geocode/json' - request_parameters = {'region': 'es', - 'key': 'AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA'} - geocoding_status_success = ['OK'] geocoding_status_surrender = ['ZERO_RESULTS'] geocoding_status_retry = ['OVER_QUERY_LIMIT', @@ -56,14 +111,27 @@ class GeocodingTask: 'UNKNOWN_ERROR'] def __init__(self, address): - request_paremeters['address'] = address + self.request_parameters = {'address': address, + 'region': 'es', + 'key': 'AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA'} + self.tries = 0 def geocode(self): """ Lanza la peticion de gecoding al servicio de google """ - self.response = requests.get(url, request_parameters) + self.response = requests.get(GeocodingTask.url, params=self.request_parameters) self.response_json = self.response.json() + self.tries += 1 + + def is_successfull(self): + """ + Comprueba que la request esta bien tanto por parte HTTP como de estados de Google. + """ + if self.response.ok and self.success_surrender_retry() == 'Success': + return True + else: + return False def get_request_status(self): """ @@ -75,9 +143,9 @@ class GeocodingTask: """ Devuelve el estado del resultado desde el punto de vista de Google """ - if self.response_json['status'] in geocoding_status_success: + if self.response_json['status'] in GeocodingTask.geocoding_status_success: return "Success" - elif self.response_json['status'] in geocoding_status_surrender: + elif self.response_json['status'] in GeocodingTask.geocoding_status_surrender: return "Surrender" else: return "Retry" @@ -87,8 +155,12 @@ class GeocodingTask: Extrae los resultados del JSON de respuesta y los devuelve """ results = {'latitude': self.response_json['results'][0]['geometry']['location']['lat'], - 'longitude': self.response_json['results'][0]['geometry']['location']['lon'], + 'longitude': self.response_json['results'][0]['geometry']['location']['lng'], 'precision': self.response_json['results'][0]['geometry']['location_type']} return results + +if __name__ == '__main__': + geocoder = Geocoder() + geocoder.start() \ No newline at end of file diff --git a/tests/geocoder_tests.py b/tests/geocoder_tests.py new file mode 100644 index 0000000..ee61a9c --- /dev/null +++ b/tests/geocoder_tests.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +import sys +sys.path.append('..') +from geocoder.geocoder import Geocoder, GeocodingTask, GeocodingCache + + + +def test_GeocodingTask(): + + good_address = 'Avinguda de la Republica Argentina 245, Barcelona' + bad_address = 'ASdasda, 123asd' + + good_task = GeocodingTask(good_address) + good_task.geocode() + print(good_task.is_successfull()) + print(good_task.get_results()) + + bad_address = GeocodingTask(bad_address) + bad_address.geocode() + print(bad_address.is_successfull()) + print(bad_address.get_results()) + +def test_GeocodingCache(): + + cache = GeocodingCache() + + test_record = {'address':'Calle Don Pepito', + 'latitude': 12.1, + 'longitude': 1.12, + 'precision': 'absoluta'} + + print(cache.address_in_cache(test_record['address'])) + cache.add_address(test_record['address'], + test_record['latitude'], + test_record['longitude'], + test_record['precision']) + + print(cache.address_in_cache(test_record['address'])) + print(cache.get_coordinates(test_record['address'])) + + +#test_GeocodingTask() + +test_GeocodingCache() \ No newline at end of file