Retoques menores en capturer y explorer. De vuelta al testing.
This commit is contained in:
parent
6342a95f40
commit
e0216060b9
4 changed files with 190 additions and 68 deletions
|
|
@ -3,6 +3,7 @@ sys.path.append('..')
|
|||
from time import sleep
|
||||
from bs4 import BeautifulSoup
|
||||
import re
|
||||
import datetime
|
||||
from db_layer.capturing_tasks_interface import capturing_interface
|
||||
from db_layer.capturas_interface import capturas_interface
|
||||
from core.scrapping_utils import UrlAttack
|
||||
|
|
@ -11,13 +12,15 @@ from core.scrapping_utils import UrlAttack
|
|||
class Capturer:
|
||||
sleep_time_no_work = 60
|
||||
minimum_seconds_between_tries = 120
|
||||
|
||||
working_hours = {'start': datetime.time(9, 0, 0),
|
||||
'end': datetime.time(21, 0, 0)}
|
||||
def start(self):
|
||||
|
||||
while True:
|
||||
|
||||
if (capturing_interface.get_pending_task() is None
|
||||
or capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries):
|
||||
or capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries
|
||||
or not self.in_working_hours()):
|
||||
sleep(Capturer.sleep_time_no_work)
|
||||
continue
|
||||
|
||||
|
|
@ -32,6 +35,10 @@ class Capturer:
|
|||
capturas_interface.insert_captura(ad_data)
|
||||
task._update_status('Captura inserted')
|
||||
|
||||
def in_working_hours(self):
|
||||
return Capturer.working_hours['start'] <= datetime.datetime.now().time() <= Capturer.working_hours['end']
|
||||
|
||||
|
||||
|
||||
class CapturingTask:
|
||||
sleep_time_failed_request = 60
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue