diff --git a/core/config.py b/core/config.py index 71af021..684a914 100644 --- a/core/config.py +++ b/core/config.py @@ -33,11 +33,15 @@ except KeyError: current_db_parameters = databases["default"] working_hours = {"start": datetime.time(9, 0, 0), "end": datetime.time(23, 0, 0)} -monthly_new_ads_target = 6000 #TODO back to 1200 +monthly_new_ads_target = 6000 # TODO back to 1200 google_api_key = "AIzaSyCnKj0WnsxVZcaoxeAYkuRw3cKRNGiISYA" + def seconds_generator(): - return max(random.randint(90, 120) + random.normalvariate(0, 120), random.randint(10, 20)) + return max( + random.randint(90, 120) + random.normalvariate(0, 120), random.randint(10, 20) + ) + minimum_seconds_between_tries = seconds_generator geocoder_delay = 10 diff --git a/core/scrapping_utils.py b/core/scrapping_utils.py index 13f4e4c..69382c2 100644 --- a/core/scrapping_utils.py +++ b/core/scrapping_utils.py @@ -75,6 +75,13 @@ class UrlAttack: except Exception as e: self.success = False + if random.randrange(0, 100) < 2: + self.change_identity() + + def change_identity(self): + UrlAttack.headers = headers_random_generator() + UrlAttack.session = requests.Session() + def get_response(self): return self.response diff --git a/geocoder/geocoder.py b/geocoder/geocoder.py index b20aa58..76eb7fe 100644 --- a/geocoder/geocoder.py +++ b/geocoder/geocoder.py @@ -35,7 +35,7 @@ class Geocoder: } capturas_interface.update_geo_data( ad_data["referencia"], - ad_data["fecha_captura"], + ad_data[b"fecha_captura"], geo_data["latitude"], geo_data["longitude"], geo_data["precision"], @@ -46,7 +46,7 @@ class Geocoder: geo_data = self.cache.get_coordinates(address) capturas_interface.update_geo_data( ad_data["referencia"], - ad_data["fecha_captura"], + ad_data[b"fecha_captura"], geo_data["latitude"], geo_data["longitude"], geo_data["precision"], @@ -67,7 +67,7 @@ class Geocoder: } capturas_interface.update_geo_data( ad_data["referencia"], - ad_data["fecha_captura"], + ad_data[b"fecha_captura"], geo_data["latitude"], geo_data["longitude"], geo_data["precision"], @@ -86,7 +86,7 @@ class Geocoder: capturas_interface.update_geo_data( ad_data["referencia"], - ad_data["fecha_captura"], + ad_data[b"fecha_captura"], geo_data["latitude"], geo_data["longitude"], geo_data["precision"],