counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Mas avances en el explorer

Browse source
This commit is contained in:
pablomartincalvo 2018-08-14 20:02:40 +02:00
parent 7e8daab6ce
commit e466986721
3 changed files with 60 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

2
core/mysql_wrapper.py
View file

@ -30,8 +30,6 @@ class DatabaseWrapper():
def disconnect(self): def disconnect(self):
if self.connection.is_connected(): if self.connection.is_connected():
self.connection.disconnect() self.connection.disconnect()
else:
print("Connection was not active.")
def ping(self): def ping(self):
self.connect() self.connect()

4
core/scrapping_utils.py
View file

@ -20,8 +20,8 @@ class UrlAttack():
def attack(self): def attack(self):
self.has_been_attacked = True self.has_been_attacked = True
try: try:
self.response = requests.get(self.url, headers = headers, self.response = requests.get(self.url, headers = self.headers,
timeout = timeout) timeout = self.timeout)
if self.response.ok: if self.response.ok:
self.success = True self.success = True
except Exception: except Exception:

103
explorer/explorer.py
View file

@ -17,17 +17,19 @@ class Explorer():
def __init__(self): def __init__(self):
try: try:
self.anunciosdb = get_anunciosdb() self.anunciosdb = get_anunciosdb()
except: except:
print("Could not connect to DB") print("Could not connect to anuncios DB")
try:
self.task_log_db = #get_task_log_db()
except:
print("Could not connect to task log DB")
self.max_db_retries = 3 self.max_db_retries = 3
self.db_retries = 0 self.db_retries = 0
self.max_queue_retries = 3 self.max_queue_retries = 3
self.queue_retries = 0 self.queue_retries = 0
def start(self): def start(self):
#Arrancar el servicio #Arrancar el servicio
@ -43,18 +45,18 @@ class Explorer():
break break
current_task = ExploringTask(self.compose_listing_url) current_task = ExploringTask(self.compose_listing_url)
if current_task.is_ready_to_explore: if current_task.is_ready_to_explore:
current_task.explore() current_task.explore()
else: else:
#Qué? break
if current_task.status == 'Success': if current_task.status == 'referencias ready':
current_referencias = current_task.get_referencias() current_referencias = current_task.get_referencias()
for referencia in current_referencias:
self.post_task_to_queue(referencia)
current_task.update_status('Sent to queue')
continue
for referencia in current
self.stop() self.stop()
@ -66,6 +68,8 @@ class Explorer():
def there_is_work(self): def there_is_work(self):
#TODO #TODO
#Comprueba si hay trabajo por hacer #Comprueba si hay trabajo por hacer
#Mirando en la bd de tasks cuantas se han hecho ultimamente, mensualmente
#etc.
def database_is_up(self): def database_is_up(self):
while self.db_retries <= self.max_db_retries: while self.db_retries <= self.max_db_retries:
@ -94,6 +98,7 @@ class Explorer():
return False return False
def compose_listing_url(self): def compose_listing_url(self):
#TODO
#Decide que url hay que componer y la compone #Decide que url hay que componer y la compone
raiz = 'https://www.idealista.com/' raiz = 'https://www.idealista.com/'
tipo = #Logica random tipo = #Logica random
@ -105,32 +110,36 @@ class Explorer():
return url return url
def post_task_to_queue(self, referencia): def post_task_to_queue(self, referencia):
#TODO
#Manda la task a la cola redis
class ExploringTask(): class ExploringTask():
def __init__(self, url): def __init__(self, url):
self.creation_date = Datetime.Datetime.now()
self.target_url = url self.target_url = url
self.id = str(uuid.uuid4()) self.id = str(uuid.uuid4())
self.status = 'Pending' self.update_status('Pending')
try: try:
self.anunciosdb = get_anunciosdb() self.anunciosdb = get_anunciosdb()
except: except:
self.anunciosdb = None self.anunciosdb = None
print("Could not connect to DB") self.update_status('Unable to connect to anuncios DB')
try: try:
#TODO #TODO
#Pendiente de implementar wraper para MongoDB #Pendiente de implementar wraper para MongoDB
#self.task_log_db = None #self.task_log_db =
except: except:
#self.task_log_db = None self.update_status('Unable to connect to task log DB')
print("Could not connect to MongoDB") #self.task_log_db = None
def update_status(self, new_status):
self.status = new_status
self._log_in_taskdb()
self.log()
def is_ready_to_explore(self): def is_ready_to_explore(self):
if self.anunciosdb is not None and self.task_log_db is not None: if self.anunciosdb is not None and self.task_log_db is not None:
@ -139,47 +148,51 @@ class ExploringTask():
return False return False
def explore(self): def explore(self):
self.attack_date = Datetime.Datetime.now()
attack = UrlAttack(self.url) attack = UrlAttack(self.url)
attack.attack() attack.attack()
if attack.success: if attack.success:
self._extract_referencias(attack.get_text())
self.extract_listings(attack.get_text()) if self.new_listings:
if self.listings_ok: self.update_status('referencias ready')
self.status = 'Success'
else: else:
self.status = 'Failure - No listings in HTML' self.update_status('Failure - No listings in HTML')
else: else:
self.status = 'Failure - Bad request self.update_status('Failure - Bad request')
self.log()
def get_referencias(self): def get_referencias(self):
return self.referencias return self.referencias
def log_in_taskdb(self): def _log_in_taskdb(self):
#TODO #TODO
#Funcion que grabe estado y demas en una mongodb o argo azin #Funcion que grabe estado y demas en una mongodb o argo azin
def extract_referencias(self, html): def _extract_referencias(self, html):
#TODO """
#Coge el html de una pagina de listado y pilla las referencias. Saca referencias de HTML, descarta las que ya exiten en la base de datos
#Si no cae ninguna, se considera un fail de capturas, y guarda si han aparecido listings y si hay alguno nuevo
#Registra la propiedad listings_ok """
if self.status == 'Success':
soup = BeautifulSoup(self.html, 'html5lib') soup = BeautifulSoup(self.html, 'html5lib')
ads = sopa.find_all(class_ = "item") ads = sopa.find_all(class_ = "item")
self.referencias = [] self.referencias = []
for ad in ads:
if self._is_new_listing(ad["data-adid"]):
for ad in ads: self.referencias.append(ad["data-adid"])
if self.is_new_listing(ad["data-adid"]): self.new_listings = bool(self.referencias)
self.referencias.append(ad["data-adid"])
def is_new_listing(self, referencia): def _is_new_listing(self, referencia):
#TODO #TODO
#Comprobar contra base de datos si la referencia existe en base de datos #Comprobar contra base de datos si la referencia existe en base de datos
query_statement = """SELECT count(referencia)
FROM capturas
WHERE referencia = %s"""
query_params = (referencia,)
cursor_result = self.anunciosdb.query(query_statement, query_params)
result = cursor_result.fetchone()
if result[0] > 0:
return False
else:
return True