Retoques menores en capturer y explorer. De vuelta al testing.

This commit is contained in:
pablomartincalvo 2018-10-14 18:41:12 +02:00
parent 6342a95f40
commit e0216060b9
4 changed files with 190 additions and 68 deletions

View file

@ -3,6 +3,7 @@ sys.path.append('..')
from time import sleep
from bs4 import BeautifulSoup
import re
import datetime
from db_layer.capturing_tasks_interface import capturing_interface
from db_layer.capturas_interface import capturas_interface
from core.scrapping_utils import UrlAttack
@ -11,13 +12,15 @@ from core.scrapping_utils import UrlAttack
class Capturer:
sleep_time_no_work = 60
minimum_seconds_between_tries = 120
working_hours = {'start': datetime.time(9, 0, 0),
'end': datetime.time(21, 0, 0)}
def start(self):
while True:
if (capturing_interface.get_pending_task() is None
or capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries):
or capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries
or not self.in_working_hours()):
sleep(Capturer.sleep_time_no_work)
continue
@ -32,6 +35,10 @@ class Capturer:
capturas_interface.insert_captura(ad_data)
task._update_status('Captura inserted')
def in_working_hours(self):
return Capturer.working_hours['start'] <= datetime.datetime.now().time() <= Capturer.working_hours['end']
class CapturingTask:
sleep_time_failed_request = 60