diff --git a/capturer/capturer.py b/capturer/capturer.py index 769eaed..e37f032 100644 --- a/capturer/capturer.py +++ b/capturer/capturer.py @@ -8,21 +8,18 @@ import datetime from db_layer.capturing_tasks_interface import capturing_interface from db_layer.capturas_interface import capturas_interface from core.scrapping_utils import UrlAttack +from core.config import working_hours, minimum_seconds_between_tries from refresher.refresher import Refresher class Capturer: sleep_time_no_work = 60 - minimum_seconds_between_tries = 120 - working_hours = {'start': datetime.time(9, 0, 0), - 'end': datetime.time(21, 0, 0)} - def start(self): while True: if (capturing_interface.get_pending_task() is None - or capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries + or capturing_interface.seconds_since_last_try() < minimum_seconds_between_tries or not self.in_working_hours()): sleep(Capturer.sleep_time_no_work) continue @@ -39,7 +36,7 @@ class Capturer: task._update_status('Captura inserted') def in_working_hours(self): - return Capturer.working_hours['start'] <= datetime.datetime.now().time() <= Capturer.working_hours['end'] + return working_hours['start'] <= datetime.datetime.now().time() <= working_hours['end'] class CapturingTask: diff --git a/core/config.py b/core/config.py index 23bcf80..594c282 100644 --- a/core/config.py +++ b/core/config.py @@ -23,5 +23,8 @@ except KeyError: working_hours = {'start': datetime.time(9, 0, 0), 'end': datetime.time(21, 0, 0)} -monthly_new_ads_target = 1000 +monthly_new_ads_target = 1200 google_api_key = 'AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA' +minimum_seconds_between_tries = 45 +geocoder_delay = 30 +refresher_delay = 10 \ No newline at end of file diff --git a/explorer/explorer.py b/explorer/explorer.py index 96be3ab..4c65350 100644 --- a/explorer/explorer.py +++ b/explorer/explorer.py @@ -226,7 +226,7 @@ class ExploringTask: Lanza una advertencia si no es así. """ soup = BeautifulSoup(html, 'html5lib') - ads = soup.find_all(class_ = "item") + ads = soup.find_all(class_="item") pattern = "^[0-9]{3,20}$" for ad in ads: diff --git a/geocoder/geocoder.py b/geocoder/geocoder.py index a8ecab4..86baf5b 100644 --- a/geocoder/geocoder.py +++ b/geocoder/geocoder.py @@ -3,7 +3,7 @@ sys.path.append('..') import requests from time import sleep from db_layer.capturas_interface import capturas_interface -from core.config import google_api_key +from core.config import google_api_key, geocoder_delay class Geocoder: @@ -13,7 +13,7 @@ class Geocoder: def start(self): while True: - sleep(120) + sleep(geocoder_delay) if capturas_interface.not_geocoded_captura_exists(): ad_data = capturas_interface.get_not_geocoded_captura() diff --git a/refresher/refresher.py b/refresher/refresher.py index 8aba225..2389442 100644 --- a/refresher/refresher.py +++ b/refresher/refresher.py @@ -3,14 +3,14 @@ sys.path.append('..') from time import sleep from db_layer.capturas_interface import capturas_interface from db_layer.capturing_tasks_interface import capturing_interface - +from core.config import refresher_delay class Refresher: def start(self): while True: - sleep(60) + sleep(refresher_delay) if capturas_interface.old_ads_exist(): old_ad = capturas_interface.get_old_ad()