counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' into testing

Browse source 
# Conflicts:
#	.idea/workspace.xml
This commit is contained in:
pablomartincalvo 2018-10-19 19:18:14 +02:00
commit c40d39e558
3 changed files with 170 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

42
db_layer/capturas_interface.py
View file

@ -58,6 +58,48 @@ class CapturasInterface():
except:
return None
def not_geocoded_captura_exists(self):
query_statement = """
SELECT *
FROM anuncios.capturas
WHERE `precision` IS NULL
LIMIT 1"""
cursor_result = self.anunciosdb.query(query_statement, dictionary=True)
try:
result = cursor_result.fetchone()
return True
except:
return False
def get_not_geocoded_captura(self):
query_statement = """
SELECT *
FROM anuncios.capturas
WHERE `precision` IS NULL
LIMIT 1
"""
cursor_result = self.anunciosdb.query(query_statement, dictionary=True)
try:
return cursor_result.fetchone()
except:
return None
def update_geo_data(self, referencia, fecha_captura, latitude, longitude, precision):
query_statement = """
UPDATE anuncios.capturas
SET latitud = %(latitud)s, longitud = %(longitud)s, `precision` = %(precision)s
WHERE referencia = %(referencia)s AND fecha_captura = %(fecha_captura)s
"""
query_parameters = {'referencia': referencia,
'fecha_captura': fecha_captura,
'latitud': latitude,
'longitud': longitude,
'precision': precision}
self.anunciosdb.query(query_statement, query_parameters)
capturas_interface = CapturasInterface()

96
geocoder/geocoder.py
View file

@ -1,4 +1,64 @@
import sys
sys.path.append('..')
import requests
from time import sleep
from db_layer.capturas_interface import capturas_interface
class Geocoder:
def __init__(self):
self.cache = GeocodingCache()
def start(self):
while True:
sleep(120)
if capturas_interface.not_geocoded_captura_exists():
ad_data = capturas_interface.get_not_geocoded_captura()
else:
continue
address = ad_data['calle'] + ', ' + ad_data['ciudad']
if self.cache.address_in_cache(address):
geo_data = self.cache.get_coordinates(address)
capturas_interface.update_geo_data(ad_data['referencia'],
ad_data['fecha_captura'],
geo_data['latitude'],
geo_data['longitude'],
geo_data['precision'])
continue
task = GeocodingTask(address)
task.geocode()
while not task.is_successfull():
task.geocode()
if task.tries < 3 or task.success_surrender_retry == 'Surrender':
geo_data = {'latitude': None,
'longitude': None,
'precision': 'SIN RESULTADOS'}
capturas_interface.update_geo_data(ad_data['referencia'],
ad_data['fecha_captura'],
geo_data['latitude'],
geo_data['longitude'],
geo_data['precision'])
break
geo_data = task.get_results()
self.cache.add_address(address,
geo_data['latitude'],
geo_data['longitude'],
geo_data['precision'])
capturas_interface.update_geo_data(ad_data['referencia'],
ad_data['fecha_captura'],
geo_data['latitude'],
geo_data['longitude'],
geo_data['precision'])
class GeocodingCache:
@ -23,16 +83,14 @@ class GeocodingCache:
"""
for geocoded_address in self.geocoded_addresses:
if geocoded_address['address'] == address:
return geocoded_address['latitude'], \
geocoded_address['longitude'], \
geocoded_address['precision']
return geocoded_address
return None
def add_address(self, address, latitude, longitude, precision):
"""
Añade la direccion a la cache y le hace sitio si es necesario
"""
if len(self.geocoded_addresses) >= cache_max_size:
if len(self.geocoded_addresses) >= GeocodingCache.cache_max_size:
self.geocoded_addresses.pop()
self.geocoded_addresses.insert(0, {'address': address,
@ -45,9 +103,6 @@ class GeocodingTask:
url = 'https://maps.googleapis.com/maps/api/geocode/json'
request_parameters = {'region': 'es',
'key': 'AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA'}
geocoding_status_success = ['OK']
geocoding_status_surrender = ['ZERO_RESULTS']
geocoding_status_retry = ['OVER_QUERY_LIMIT',
@ -56,14 +111,27 @@ class GeocodingTask:
'UNKNOWN_ERROR']
def __init__(self, address):
request_paremeters['address'] = address
self.request_parameters = {'address': address,
'region': 'es',
'key': 'AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA'}
self.tries = 0
def geocode(self):
"""
Lanza la peticion de gecoding al servicio de google
"""
self.response = requests.get(url, request_parameters)
self.response = requests.get(GeocodingTask.url, params=self.request_parameters)
self.response_json = self.response.json()
self.tries += 1
def is_successfull(self):
"""
Comprueba que la request esta bien tanto por parte HTTP como de estados de Google.
"""
if self.response.ok and self.success_surrender_retry() == 'Success':
return True
else:
return False
def get_request_status(self):
"""
@ -75,9 +143,9 @@ class GeocodingTask:
"""
Devuelve el estado del resultado desde el punto de vista de Google
"""
if self.response_json['status'] in geocoding_status_success:
if self.response_json['status'] in GeocodingTask.geocoding_status_success:
return "Success"
elif self.response_json['status'] in geocoding_status_surrender:
elif self.response_json['status'] in GeocodingTask.geocoding_status_surrender:
return "Surrender"
else:
return "Retry"
@ -87,8 +155,12 @@ class GeocodingTask:
Extrae los resultados del JSON de respuesta y los devuelve
"""
results = {'latitude': self.response_json['results'][0]['geometry']['location']['lat'],
'longitude': self.response_json['results'][0]['geometry']['location']['lon'],
'longitude': self.response_json['results'][0]['geometry']['location']['lng'],
'precision': self.response_json['results'][0]['geometry']['location_type']}
return results
if __name__ == '__main__':
geocoder = Geocoder()
geocoder.start()

44
tests/geocoder_tests.py Normal file
View file

@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
import sys
sys.path.append('..')
from geocoder.geocoder import Geocoder, GeocodingTask, GeocodingCache
def test_GeocodingTask():
good_address = 'Avinguda de la Republica Argentina 245, Barcelona'
bad_address = 'ASdasda, 123asd'
good_task = GeocodingTask(good_address)
good_task.geocode()
print(good_task.is_successfull())
print(good_task.get_results())
bad_address = GeocodingTask(bad_address)
bad_address.geocode()
print(bad_address.is_successfull())
print(bad_address.get_results())
def test_GeocodingCache():
cache = GeocodingCache()
test_record = {'address':'Calle Don Pepito',
'latitude': 12.1,
'longitude': 1.12,
'precision': 'absoluta'}
print(cache.address_in_cache(test_record['address']))
cache.add_address(test_record['address'],
test_record['latitude'],
test_record['longitude'],
test_record['precision'])
print(cache.address_in_cache(test_record['address']))
print(cache.get_coordinates(test_record['address']))
#test_GeocodingTask()
test_GeocodingCache()