Merge branch 'dev'

capturer/capturer.py

@@ -8,21 +8,18 @@ import datetime
 from db_layer.capturing_tasks_interface import capturing_interface
 from db_layer.capturas_interface import capturas_interface
 from core.scrapping_utils import UrlAttack
+from core.config import working_hours, minimum_seconds_between_tries
 from refresher.refresher import Refresher
 
 
 class Capturer:
     sleep_time_no_work = 60
-    minimum_seconds_between_tries = 120
-    working_hours = {'start': datetime.time(9, 0, 0),
-                     'end': datetime.time(21, 0, 0)}
-
     def start(self):
 
         while True:
 
             if (capturing_interface.get_pending_task() is None
-                    or capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries
+                    or capturing_interface.seconds_since_last_try() < minimum_seconds_between_tries
                     or not self.in_working_hours()):
                 sleep(Capturer.sleep_time_no_work)
                 continue
@@ -39,7 +36,7 @@ class Capturer:
             task._update_status('Captura inserted')
 
     def in_working_hours(self):
-        return Capturer.working_hours['start'] <= datetime.datetime.now().time() <= Capturer.working_hours['end']
+        return working_hours['start'] <= datetime.datetime.now().time() <= working_hours['end']
 
 
 class CapturingTask:

core/config.py

@@ -23,5 +23,8 @@ except KeyError:
 
 working_hours = {'start': datetime.time(9, 0, 0),
                  'end': datetime.time(21, 0, 0)}
-monthly_new_ads_target = 1000
+monthly_new_ads_target = 1200
 google_api_key = 'AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA'
+minimum_seconds_between_tries = 45
+geocoder_delay = 30
+refresher_delay = 10

explorer/explorer.py

@@ -226,7 +226,7 @@ class ExploringTask:
         Lanza una advertencia si no es así.
         """
         soup = BeautifulSoup(html, 'html5lib')
-        ads = soup.find_all(class_ = "item")
+        ads = soup.find_all(class_="item")
         pattern = "^[0-9]{3,20}$"
                 
         for ad in ads:

geocoder/geocoder.py

@@ -3,7 +3,7 @@ sys.path.append('..')
 import requests
 from time import sleep
 from db_layer.capturas_interface import capturas_interface
-from core.config import google_api_key
+from core.config import google_api_key, geocoder_delay
 
 class Geocoder:
 
@@ -13,7 +13,7 @@ class Geocoder:
     def start(self):
 
         while True:
-            sleep(120)
+            sleep(geocoder_delay)
 
             if capturas_interface.not_geocoded_captura_exists():
                 ad_data = capturas_interface.get_not_geocoded_captura()

refresher/refresher.py

@@ -3,14 +3,14 @@ sys.path.append('..')
 from time import sleep
 from db_layer.capturas_interface import capturas_interface
 from db_layer.capturing_tasks_interface import capturing_interface
-
+from core.config import refresher_delay
 
 class Refresher:
 
     def start(self):
 
         while True:
-            sleep(60)
+            sleep(refresher_delay)
 
             if capturas_interface.old_ads_exist():
                 old_ad = capturas_interface.get_old_ad()