Merge branch 'dev' into testing

# Conflicts:
#	.idea/workspace.xml

db_layer/capturas_interface.py

@@ -58,6 +58,48 @@ class CapturasInterface():
         except:
             return None
 
+    def not_geocoded_captura_exists(self):
+        query_statement = """
+        SELECT *
+        FROM anuncios.capturas
+        WHERE `precision` IS NULL
+        LIMIT 1"""
+
+        cursor_result = self.anunciosdb.query(query_statement, dictionary=True)
+        try:
+            result = cursor_result.fetchone()
+            return True
+        except:
+            return False
+
+    def get_not_geocoded_captura(self):
+        query_statement = """
+        SELECT *
+        FROM anuncios.capturas
+        WHERE `precision` IS NULL
+        LIMIT 1
+        """
+
+        cursor_result = self.anunciosdb.query(query_statement, dictionary=True)
+        try:
+            return cursor_result.fetchone()
+        except:
+            return None
+
+    def update_geo_data(self, referencia, fecha_captura, latitude, longitude, precision):
+        query_statement = """
+        UPDATE anuncios.capturas
+        SET latitud = %(latitud)s, longitud = %(longitud)s, `precision` = %(precision)s
+        WHERE referencia = %(referencia)s AND fecha_captura = %(fecha_captura)s
+        """
+        query_parameters = {'referencia': referencia,
+                            'fecha_captura': fecha_captura,
+                            'latitud': latitude,
+                            'longitud': longitude,
+                            'precision': precision}
+
+        self.anunciosdb.query(query_statement, query_parameters)
+
 
 capturas_interface = CapturasInterface()
 

geocoder/geocoder.py

@@ -1,4 +1,64 @@
+import sys
+sys.path.append('..')
 import requests
+from time import sleep
+from db_layer.capturas_interface import capturas_interface
+
+
+class Geocoder:
+
+    def __init__(self):
+        self.cache = GeocodingCache()
+
+    def start(self):
+
+        while True:
+            sleep(120)
+
+            if capturas_interface.not_geocoded_captura_exists():
+                ad_data = capturas_interface.get_not_geocoded_captura()
+            else:
+                continue
+
+            address = ad_data['calle'] + ', ' + ad_data['ciudad']
+
+            if self.cache.address_in_cache(address):
+                geo_data = self.cache.get_coordinates(address)
+                capturas_interface.update_geo_data(ad_data['referencia'],
+                                                   ad_data['fecha_captura'],
+                                                   geo_data['latitude'],
+                                                   geo_data['longitude'],
+                                                   geo_data['precision'])
+                continue
+
+            task = GeocodingTask(address)
+            task.geocode()
+
+            while not task.is_successfull():
+                task.geocode()
+                if task.tries < 3 or task.success_surrender_retry == 'Surrender':
+                    geo_data = {'latitude': None,
+                                'longitude': None,
+                                'precision': 'SIN RESULTADOS'}
+                    capturas_interface.update_geo_data(ad_data['referencia'],
+                                                       ad_data['fecha_captura'],
+                                                       geo_data['latitude'],
+                                                       geo_data['longitude'],
+                                                       geo_data['precision'])
+                    break
+
+            geo_data = task.get_results()
+
+            self.cache.add_address(address,
+                                   geo_data['latitude'],
+                                   geo_data['longitude'],
+                                   geo_data['precision'])
+
+            capturas_interface.update_geo_data(ad_data['referencia'],
+                                               ad_data['fecha_captura'],
+                                               geo_data['latitude'],
+                                               geo_data['longitude'],
+                                               geo_data['precision'])
 
 
 class GeocodingCache:
@@ -23,16 +83,14 @@ class GeocodingCache:
         """
         for geocoded_address in self.geocoded_addresses:
             if geocoded_address['address'] == address:
-                return geocoded_address['latitude'], \
+                return geocoded_address
-                       geocoded_address['longitude'], \
-                       geocoded_address['precision']
         return None
 
     def add_address(self, address, latitude, longitude, precision):
         """
         Añade la direccion a la cache y le hace sitio si es necesario
         """
-        if len(self.geocoded_addresses) >= cache_max_size:
+        if len(self.geocoded_addresses) >= GeocodingCache.cache_max_size:
             self.geocoded_addresses.pop()
 
         self.geocoded_addresses.insert(0, {'address': address,
@@ -45,9 +103,6 @@ class GeocodingTask:
 
     url = 'https://maps.googleapis.com/maps/api/geocode/json'
 
-    request_parameters = {'region': 'es',
-                          'key': 'AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA'}
-
     geocoding_status_success = ['OK']
     geocoding_status_surrender = ['ZERO_RESULTS']
     geocoding_status_retry = ['OVER_QUERY_LIMIT',
@@ -56,14 +111,27 @@ class GeocodingTask:
                                    'UNKNOWN_ERROR']
 
     def __init__(self, address):
-        request_paremeters['address'] = address
+        self.request_parameters = {'address': address,
+                                   'region': 'es',
+                                   'key': 'AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA'}
+        self.tries = 0
 
     def geocode(self):
         """
         Lanza la peticion de gecoding al servicio de google
         """
-        self.response = requests.get(url, request_parameters)
+        self.response = requests.get(GeocodingTask.url, params=self.request_parameters)
         self.response_json = self.response.json()
+        self.tries += 1
+
+    def is_successfull(self):
+        """
+        Comprueba que la request esta bien tanto por parte HTTP como de estados de Google.
+        """
+        if self.response.ok and self.success_surrender_retry() == 'Success':
+            return True
+        else:
+            return False
 
     def get_request_status(self):
         """
@@ -75,9 +143,9 @@ class GeocodingTask:
         """
         Devuelve el estado del resultado desde el punto de vista de Google
         """
-        if self.response_json['status'] in geocoding_status_success:
+        if self.response_json['status'] in GeocodingTask.geocoding_status_success:
             return "Success"
-        elif self.response_json['status'] in geocoding_status_surrender:
+        elif self.response_json['status'] in GeocodingTask.geocoding_status_surrender:
             return "Surrender"
         else:
             return "Retry"
@@ -87,8 +155,12 @@ class GeocodingTask:
         Extrae los resultados del JSON de respuesta y los devuelve
         """
         results = {'latitude': self.response_json['results'][0]['geometry']['location']['lat'],
-                   'longitude': self.response_json['results'][0]['geometry']['location']['lon'],
+                   'longitude': self.response_json['results'][0]['geometry']['location']['lng'],
                    'precision': self.response_json['results'][0]['geometry']['location_type']}
 
         return results
 
+
+if __name__ == '__main__':
+    geocoder = Geocoder()
+    geocoder.start()

tests/geocoder_tests.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+import sys
+sys.path.append('..')
+from geocoder.geocoder import Geocoder, GeocodingTask, GeocodingCache
+
+
+
+def test_GeocodingTask():
+
+    good_address = 'Avinguda de la Republica Argentina 245, Barcelona'
+    bad_address = 'ASdasda, 123asd'
+
+    good_task = GeocodingTask(good_address)
+    good_task.geocode()
+    print(good_task.is_successfull())
+    print(good_task.get_results())
+
+    bad_address = GeocodingTask(bad_address)
+    bad_address.geocode()
+    print(bad_address.is_successfull())
+    print(bad_address.get_results())
+
+def test_GeocodingCache():
+
+    cache = GeocodingCache()
+
+    test_record = {'address':'Calle Don Pepito',
+                   'latitude': 12.1,
+                   'longitude': 1.12,
+                   'precision': 'absoluta'}
+
+    print(cache.address_in_cache(test_record['address']))
+    cache.add_address(test_record['address'],
+                      test_record['latitude'],
+                      test_record['longitude'],
+                      test_record['precision'])
+
+    print(cache.address_in_cache(test_record['address']))
+    print(cache.get_coordinates(test_record['address']))
+
+
+#test_GeocodingTask()
+
+test_GeocodingCache()