counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Retoques menores en capturer y explorer. De vuelta al testing.

Browse source
This commit is contained in:
pablomartincalvo 2018-10-14 18:41:12 +02:00
parent 6342a95f40
commit e0216060b9
4 changed files with 190 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

16
explorer/explorer.py
View file

@ -17,7 +17,7 @@ class Explorer():
sleep_time_no_work = 60
sleep_time_no_service = 600
working_hours = {'start': datetime.time(9, 0, 0),
'end': datetime.time(18, 0, 0)}
'end': datetime.time(21, 0, 0)}
monthly_capture_target = 1000
ad_types = {'1': 'alquiler',
'2': 'venta'}
@ -38,6 +38,7 @@ class Explorer():
while True:
if not self.there_is_work():
print('{}: Waiting. No work'.format(datetime.datetime.now()))
sleep(Explorer.sleep_time_no_work)
continue
@ -47,17 +48,16 @@ class Explorer():
current_task = ExploringTask(self.compose_listing_url())
current_task.explore()
print('{}: Exploring done'.format(datetime.datetime.now()))
if current_task.status == 'Referencias ready':
referencias = current_task.get_referencias()
for referencia in referencias:
capturing_interface.create_capturing_task(referencia)
current_task._update_status("Sent to queue")
capturing_interface.create_capturing_task(referencia, current_task.id)
current_task._update_status("Sent to queue")
continue
self.stop()
def stop(self):
#TODO Detener el servicio
@ -160,7 +160,7 @@ class Explorer():
"""
cursor_result = self.tasksdb.query(query_statement)
return cursor_result.row_count
return cursor_result.fetchone()[0]
def compose_listing_url(self):
"""
@ -168,7 +168,7 @@ class Explorer():
:return:
"""
root = 'https://www.idealista.com/'
type = ad_type[str(randint(1,2))]
type = Explorer.ad_types[str(randint(1,2))]
city = 'barcelona'
page_number = str(randint(1,30))
url = root + type + '-garajes/' + city + '-' + city + '/' + \