counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Corregidos pequeños errores y type tras primer test del servicio.

Browse source
This commit is contained in:
pablomartincalvo 2018-09-22 23:17:49 +02:00
parent 1f372b85b6
commit 80d65b7a7c
3 changed files with 68 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

22
explorer/explorer.py
View file

@ -19,6 +19,8 @@ class Explorer():
working_hours = {'start': datetime.time(9, 0, 0),
'end': datetime.time(18, 0, 0)}
monthly_capture_target = 1000
ad_types = {'1': 'alquiler',
'2': 'venta'}
def __init__(self):
try:
@ -42,12 +44,8 @@ class Explorer():
if not self.database_is_up():
alert_master("SQL DOWN", "El explorer informa de que SQL esta caida. Actividad detenida")
self.stop(self)
if not self.queue_is_up():
alert_master("REDIS DOWN", "El explorer informa de que REDIS esta caido. Actividad detenida")
self.stop(self)
current_task = ExploringTask(self.compose_listing_url)
current_task = ExploringTask(self.compose_listing_url())
current_task.explore()
if current_task.status == 'Referencias ready':
@ -55,7 +53,7 @@ class Explorer():
for referencia in referencias:
create_capturing_task(referencia, self.tasksdb)
current_task._update_status(self, "Sent to queue")
current_task._update_status("Sent to queue")
continue
@ -169,12 +167,12 @@ class Explorer():
Genera URLs de manera aleatoria
:return:
"""
raiz = 'https://www.idealista.com/'
tipo = randint(1,2)
ciudad = 'barcelona'
numero = randint(1,30)
url = raiz + tipo + '-garajes/' + ciudad + '-' + ciudad + '/' + \
'pagina-' + numero + '.htm'
root = 'https://www.idealista.com/'
type = ad_type[str(randint(1,2))]
city = 'barcelona'
page_number = str(randint(1,30))
url = root + type + '-garajes/' + city + '-' + city + '/' + \
'pagina-' + page_number + '.htm'
return url