278 lines
8.7 KiB
Python
278 lines
8.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
import sys
|
|
sys.path.append('..')
|
|
import uuid
|
|
from datetime import datetime
|
|
from time import sleep
|
|
from bs4 import BeautifulSoup
|
|
import re
|
|
from random import randint
|
|
from core.mysql_wrapper import get_anunciosdb, get_tasksdb
|
|
from core.scrapping_utils import UrlAttack
|
|
import core.alerts
|
|
|
|
class Explorer():
|
|
|
|
sleep_time_no_work = 60
|
|
sleep_time_no_service = 600
|
|
working_hours = {start: datetime.time(9, 0, 0),
|
|
end: datetime.time(18, 0, 0)}
|
|
monthly_capture_target = 1000
|
|
|
|
def __init__(self):
|
|
try:
|
|
self.anunciosdb = get_anunciosdb()
|
|
except:
|
|
print("Could not connect to anuncios DB")
|
|
|
|
self.max_db_retries = 3
|
|
self.db_retries = 0
|
|
self.max_queue_retries = 3
|
|
self.queue_retries = 0
|
|
|
|
def start(self):
|
|
|
|
while True:
|
|
if not self.there_is_work():
|
|
sleep(sleep_time_no_work)
|
|
continue
|
|
|
|
if not self.database_is_up():
|
|
break
|
|
|
|
if not self.queue_is_up():
|
|
break
|
|
|
|
current_task = ExploringTask(self.compose_listing_url)
|
|
current_task.explore()
|
|
|
|
continue
|
|
|
|
self.stop()
|
|
|
|
|
|
def stop(self):
|
|
#TODO
|
|
#Detener el servicio
|
|
pass
|
|
|
|
def there_is_work(self):
|
|
"""
|
|
Funcion que agrupa las condiciones que se deben cumplir para poder trabajar
|
|
"""
|
|
if not self.in_working_hours():
|
|
return False
|
|
|
|
if self.get_referencias_acquired_today() >= self.get_max_referencias_for_today():
|
|
return False
|
|
|
|
if self.get_tasks_created_today() >= self.get_max_tasks_today():
|
|
return False
|
|
|
|
return True
|
|
|
|
def database_is_up(self):
|
|
while self.db_retries <= self.max_db_retries:
|
|
try:
|
|
self.anunciosdb.ping()
|
|
self.db_retries = 0
|
|
return True
|
|
except:
|
|
sleep(sleep_time_no_service)
|
|
self.db_retries = self.db_retries + 1
|
|
|
|
return False
|
|
|
|
def queue_is_up(self):
|
|
#TODO
|
|
while self.queue_retries <= self.max_queue_retries:
|
|
try:
|
|
#codigo que testea si redis esta vivo
|
|
self.queue_retries = 0
|
|
return True
|
|
except:
|
|
sleep(sleep_time_no_service)
|
|
self.queue_retries = self.queue_retries + 1
|
|
|
|
return False
|
|
|
|
def in_working_hours(self):
|
|
return working_hours['start'] <= datetime.now().time() <= working_hours['end']
|
|
|
|
def get_referencias_acquired_today(self):
|
|
"""
|
|
Cuenta cuantas nuevas referencias han aparecido en las ultimas 24 horas
|
|
"""
|
|
|
|
query_statement = """ SELECT count(referencias)
|
|
FROM primera_captura_full
|
|
WHERE fecha_captura >= now() - INTERVAL 1 DAY;
|
|
"""
|
|
|
|
cursor_result = self.anunciosdb.query(query_statement)
|
|
|
|
return cursor_result.fetchone()
|
|
|
|
def get_max_referencias_for_today(self):
|
|
"""
|
|
Calcula la cantidad objetivo para las ultimas 24 horas en base a la
|
|
diferencia con el objetivo mensual
|
|
"""
|
|
query_statement = """ SELECT count(referencias)
|
|
FROM primera_captura_full
|
|
WHERE fecha_captura >= now() - INTERVAL 30 DAY;
|
|
"""
|
|
cursor_result = self.anunciosdb.query(query_statement)
|
|
new_referencias_last_30 = cursor_result.fetchone()
|
|
|
|
deviation = (monthly_capture_target - new_referencias_last_30) / monthly_capture_target
|
|
max_referencias = (monthly_capture_target/30) * (1 + (deviation))
|
|
|
|
return max_referencias
|
|
|
|
def get_tasks_created_today(self):
|
|
"""
|
|
Mira en el task log cuantas tareas se han iniciado en las ultimas 24 horas
|
|
"""
|
|
query_statement = """ SELECT count(uuid)
|
|
FROM exploring_tasks
|
|
WHERE status = 'Attacked'
|
|
AND write_time >= now() - INTERVAL 1 DAY;
|
|
"""
|
|
cursor_result = self.tasksdb.query(query_statement)
|
|
tasks_created_today = cursor_result.fetchone()
|
|
|
|
return tasks_created_today
|
|
|
|
def get_max_tasks_today(self):
|
|
"""
|
|
Calcula el maximo diario de intentos en forma de tareas, en base al
|
|
maximo de capturas mas un multiplicador
|
|
"""
|
|
return (self.get_max_referencias_for_today() / 30) * 6
|
|
|
|
def compose_listing_url(self):
|
|
"""
|
|
Genera URLs de manera aleatoria
|
|
:return:
|
|
"""
|
|
raiz = 'https://www.idealista.com/'
|
|
tipo = randint(1,2)
|
|
ciudad = 'barcelona'
|
|
numero = randint(1,30)
|
|
url = raiz + tipo + '-garajes/' + ciudad + '-' + ciudad + '/' + \
|
|
'pagina-' + numero + '.htm'
|
|
|
|
return url
|
|
|
|
|
|
class ExploringTask():
|
|
|
|
def __init__(self, url):
|
|
self.anunciosdb = get_anunciosdb()
|
|
self.tasksdb = get_tasksdb()
|
|
self.target_url = url
|
|
self.id = str(uuid.uuid4())
|
|
self._update_status('Pending')
|
|
|
|
def _update_status(self, new_status):
|
|
self.status = new_status
|
|
self._log_in_tasksdb()
|
|
|
|
def explore(self):
|
|
attack = UrlAttack(self.target_url)
|
|
attack.attack()
|
|
self._update_status('Attacked')
|
|
|
|
if attack.success:
|
|
self._validate_referencias(attack.get_text())
|
|
self._extract_referencias(attack.get_text())
|
|
if self.referencias:
|
|
self._update_status('Referencias ready')
|
|
self._post_tasks_to_queue()
|
|
self._update_status('Sent to Queue')
|
|
elif self.there_are_referencias:
|
|
self._update_status('Failure - No new referencias in HTML')
|
|
else:
|
|
self._update_status('Failure - HTML with no referencias')
|
|
else:
|
|
self._update_status('Failure - Bad request')
|
|
|
|
|
|
def _log_in_tasksdb(self):
|
|
"""
|
|
Graba en la base de datos de tareas un registro con el UUID de la tarea,
|
|
un timestamp y el status
|
|
"""
|
|
|
|
query_statement = """INSERT INTO exploring_tasks_logs
|
|
(uuid, write_time, status)
|
|
VALUES (%(uuid)s, NOW(), %(status)s)"""
|
|
|
|
query_parameters = {'uuid': self.id,
|
|
'status': self.status}
|
|
|
|
self.tasksdb.query(query_statement, query_parameters)
|
|
|
|
def _validate_referencias(self, html):
|
|
"""
|
|
Comprueba que las etiquetas sigan el formato de un anuncio.
|
|
Lanza una advertencia si no es así.
|
|
"""
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
ads = soup.find_all(class_ = "item")
|
|
pattern = "^[0-9]{3,20}$"
|
|
|
|
for ad in ads:
|
|
if not re.match(pattern, ad["data-adid"]):
|
|
#TODO Levantar marron
|
|
pass
|
|
|
|
|
|
def _extract_referencias(self, html):
|
|
"""
|
|
Saca referencias de HTML, descarta las que ya exiten en la base de datos
|
|
de capturas, y guarda si han aparecido listings y si hay alguno nuevo
|
|
"""
|
|
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
ads = soup.find_all(class_ = "item")
|
|
self.there_are_referencias = bool(ads)
|
|
self.referencias = []
|
|
for ad in ads:
|
|
if self._is_new_listing(ad["data-adid"]):
|
|
self.referencias.append(ad["data-adid"])
|
|
|
|
|
|
def _is_new_listing(self, referencia):
|
|
"""
|
|
Comprueba si el listing ya existe en la base de datos de anuncios
|
|
"""
|
|
query_statement = """SELECT count(referencia)
|
|
FROM capturas
|
|
WHERE referencia = %s"""
|
|
query_params = (referencia,)
|
|
cursor_result = self.anunciosdb.query(query_statement, query_params)
|
|
|
|
result = cursor_result.fetchone()
|
|
if result[0] > 0:
|
|
return False
|
|
else:
|
|
return True
|
|
|
|
def _post_tasks_to_queue(self):
|
|
#TODO Mandar las referencias a redis
|
|
pass
|
|
|
|
|
|
def testear_exploring_task():
|
|
url = 'https://www.idealista.com/venta-garajes/barcelona-barcelona/'
|
|
task = ExploringTask(url)
|
|
task.explore()
|
|
|
|
print(task.referencias)
|
|
|
|
|
|
testear_exploring_task()
|
|
|
|
|