counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added logging to geocoder.py

Browse source
This commit is contained in:
pablo 2020-03-26 11:30:09 +01:00
parent a9242b2f3a
commit cdbb6b5325
1 changed files with 91 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

149
geocoder/geocoder.py
View file

@ -1,18 +1,22 @@
import sys import sys
sys.path.append('..')
sys.path.append("..")
import requests import requests
from time import sleep from time import sleep
from db_layer.capturas_interface import capturas_interface from db_layer.capturas_interface import capturas_interface
from core.config import google_api_key, geocoder_delay from core.config import google_api_key, geocoder_delay
from core import my_logger
import logging
class Geocoder: class Geocoder:
def __init__(self): def __init__(self):
self.cache = GeocodingCache() self.cache = GeocodingCache()
def start(self): def start(self):
logging.info("Starting geocoder.")
while True: while True:
sleep(geocoder_delay) sleep(geocoder_delay)
@ -22,56 +26,71 @@ class Geocoder:
continue continue
try: try:
address = ad_data['calle'] + ', ' + ad_data['ciudad'] address = ad_data["calle"] + ", " + ad_data["ciudad"]
except TypeError: except TypeError:
geo_data = {'latitude': None, geo_data = {
'longitude': None, "latitude": None,
'precision': 'SIN RESULTADOS'} "longitude": None,
capturas_interface.update_geo_data(ad_data['referencia'], "precision": "SIN RESULTADOS",
ad_data['fecha_captura'], }
geo_data['latitude'], capturas_interface.update_geo_data(
geo_data['longitude'], ad_data["referencia"],
geo_data['precision']) ad_data["fecha_captura"],
geo_data["latitude"],
geo_data["longitude"],
geo_data["precision"],
)
continue continue
if self.cache.address_in_cache(address): if self.cache.address_in_cache(address):
geo_data = self.cache.get_coordinates(address) geo_data = self.cache.get_coordinates(address)
capturas_interface.update_geo_data(ad_data['referencia'], capturas_interface.update_geo_data(
ad_data['fecha_captura'], ad_data["referencia"],
geo_data['latitude'], ad_data["fecha_captura"],
geo_data['longitude'], geo_data["latitude"],
geo_data['precision']) geo_data["longitude"],
geo_data["precision"],
)
continue continue
task = GeocodingTask(address) task = GeocodingTask(address)
task.geocode() task.geocode()
while not task.is_successfull(): while not task.is_successfull():
logging.warning("Unsuccessful request.")
task.geocode() task.geocode()
if task.tries > 3 or task.success_surrender_retry == 'Surrender': if task.tries > 3 or task.success_surrender_retry == "Surrender":
geo_data = {'latitude': None, geo_data = {
'longitude': None, "latitude": None,
'precision': 'SIN RESULTADOS'} "longitude": None,
capturas_interface.update_geo_data(ad_data['referencia'], "precision": "SIN RESULTADOS",
ad_data['fecha_captura'], }
geo_data['latitude'], capturas_interface.update_geo_data(
geo_data['longitude'], ad_data["referencia"],
geo_data['precision']) ad_data["fecha_captura"],
geo_data["latitude"],
geo_data["longitude"],
geo_data["precision"],
)
break break
if not (task.tries > 3 or task.success_surrender_retry == 'Surrender'): if not (task.tries > 3 or task.success_surrender_retry == "Surrender"):
geo_data = task.get_results() geo_data = task.get_results()
self.cache.add_address(address, self.cache.add_address(
geo_data['latitude'], address,
geo_data['longitude'], geo_data["latitude"],
geo_data['precision']) geo_data["longitude"],
geo_data["precision"],
)
capturas_interface.update_geo_data(ad_data['referencia'], capturas_interface.update_geo_data(
ad_data['fecha_captura'], ad_data["referencia"],
geo_data['latitude'], ad_data["fecha_captura"],
geo_data['longitude'], geo_data["latitude"],
geo_data['precision']) geo_data["longitude"],
geo_data["precision"],
)
class GeocodingCache: class GeocodingCache:
@ -86,7 +105,7 @@ class GeocodingCache:
Comprueba si la direccion ya esta en la cache Comprueba si la direccion ya esta en la cache
""" """
for geocoded_address in self.geocoded_addresses: for geocoded_address in self.geocoded_addresses:
if geocoded_address['address'] == address: if geocoded_address["address"] == address:
return True return True
return False return False
@ -95,7 +114,7 @@ class GeocodingCache:
Recupera los datos asociados a la direccion Recupera los datos asociados a la direccion
""" """
for geocoded_address in self.geocoded_addresses: for geocoded_address in self.geocoded_addresses:
if geocoded_address['address'] == address: if geocoded_address["address"] == address:
return geocoded_address return geocoded_address
return None return None
@ -105,28 +124,38 @@ class GeocodingCache:
""" """
if len(self.geocoded_addresses) >= GeocodingCache.cache_max_size: if len(self.geocoded_addresses) >= GeocodingCache.cache_max_size:
self.geocoded_addresses.pop() self.geocoded_addresses.pop()
logging.info("Geocoder cache has reached max size. Making some room...")
self.geocoded_addresses.insert(0, {'address': address, self.geocoded_addresses.insert(
'latitude': latitude, 0,
'longitude': longitude, {
'precision': precision}) "address": address,
"latitude": latitude,
"longitude": longitude,
"precision": precision,
},
)
class GeocodingTask: class GeocodingTask:
url = 'https://maps.googleapis.com/maps/api/geocode/json' url = "https://maps.googleapis.com/maps/api/geocode/json"
geocoding_status_success = ['OK'] geocoding_status_success = ["OK"]
geocoding_status_surrender = ['ZERO_RESULTS'] geocoding_status_surrender = ["ZERO_RESULTS"]
geocoding_status_retry = ['OVER_QUERY_LIMIT', geocoding_status_retry = [
'REQUEST_DENIED', "OVER_QUERY_LIMIT",
'INVALID_REQUEST', "REQUEST_DENIED",
'UNKNOWN_ERROR'] "INVALID_REQUEST",
"UNKNOWN_ERROR",
]
def __init__(self, address): def __init__(self, address):
self.request_parameters = {'address': address, self.request_parameters = {
'region': 'es', "address": address,
'key': google_api_key} "region": "es",
"key": google_api_key,
}
self.tries = 0 self.tries = 0
def geocode(self): def geocode(self):
@ -141,7 +170,7 @@ class GeocodingTask:
""" """
Comprueba que la request esta bien tanto por parte HTTP como de estados de Google. Comprueba que la request esta bien tanto por parte HTTP como de estados de Google.
""" """
if self.response.ok and self.success_surrender_retry() == 'Success': if self.response.ok and self.success_surrender_retry() == "Success":
return True return True
else: else:
return False return False
@ -156,9 +185,9 @@ class GeocodingTask:
""" """
Devuelve el estado del resultado desde el punto de vista de Google Devuelve el estado del resultado desde el punto de vista de Google
""" """
if self.response_json['status'] in GeocodingTask.geocoding_status_success: if self.response_json["status"] in GeocodingTask.geocoding_status_success:
return "Success" return "Success"
elif self.response_json['status'] in GeocodingTask.geocoding_status_surrender: elif self.response_json["status"] in GeocodingTask.geocoding_status_surrender:
return "Surrender" return "Surrender"
else: else:
return "Retry" return "Retry"
@ -167,13 +196,17 @@ class GeocodingTask:
""" """
Extrae los resultados del JSON de respuesta y los devuelve Extrae los resultados del JSON de respuesta y los devuelve
""" """
results = {'latitude': self.response_json['results'][0]['geometry']['location']['lat'], results = {
'longitude': self.response_json['results'][0]['geometry']['location']['lng'], "latitude": self.response_json["results"][0]["geometry"]["location"]["lat"],
'precision': self.response_json['results'][0]['geometry']['location_type']} "longitude": self.response_json["results"][0]["geometry"]["location"][
"lng"
],
"precision": self.response_json["results"][0]["geometry"]["location_type"],
}
return results return results
if __name__ == '__main__': if __name__ == "__main__":
geocoder = Geocoder() geocoder = Geocoder()
geocoder.start() geocoder.start()