counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactorizado Explorer para que el Explorer, y no el exploring_task, se encargue de postear las tareas de captura.

Browse source 
Creado una funcion independiente de creacion de capturas para que sea compartida entre todos aquellos servicios que las creen.
This commit is contained in:
pablomartincalvo 2018-09-21 18:19:33 +02:00
parent b77a4752b8
commit 1f372b85b6
4 changed files with 286 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

297
.idea/workspace.xml generated
View file

@ -2,6 +2,8 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/capturer/__init__.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/explorer/explorer.py" beforeDir="false" afterPath="$PROJECT_DIR$/explorer/explorer.py" afterDir="false" />
</list>
@ -16,32 +18,37 @@
<usages-collector id="statistics.lifecycle.project">
<counts>
<entry key="project.open.time.0" value="1" />
<entry key="project.open.time.13" value="1" />
<entry key="project.open.time.13" value="2" />
<entry key="project.open.time.14" value="1" />
<entry key="project.opened" value="3" />
<entry key="project.opened" value="4" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="py" value="5" />
<entry key="py" value="10" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="Python" value="5" />
<entry key="Python" value="10" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.edit">
<counts>
<entry key="Python Console" value="1519" />
<entry key="py" value="1383" />
<entry key="txt" value="183" />
<entry key="py" value="2871" />
<entry key="txt" value="214" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="PLAIN_TEXT" value="183" />
<entry key="Python" value="2902" />
<entry key="PLAIN_TEXT" value="214" />
<entry key="Python" value="4390" />
</counts>
</usages-collector>
<usages-collector id="statistics.vcs.git.usages">
<counts>
<entry key="git.branch.create.new" value="1" />
</counts>
</usages-collector>
</session>
@ -50,47 +57,116 @@
<favorites_list name="Drogon" />
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="90">
<caret line="6" column="29" lean-forward="true" selection-start-line="6" selection-start-column="29" selection-end-line="6" selection-end-column="29" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="105">
<caret line="7" column="34" lean-forward="true" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding>
<element signature="e#0#46#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="525">
<caret line="35" column="37" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="391">
<caret line="36" column="22" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
</state>
</provider>
</entry>
</file>
</leaf>
<splitter split-orientation="horizontal" split-proportion="0.5">
<split-first>
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="444">
<caret line="162" column="8" selection-start-line="162" selection-start-column="8" selection-end-line="162" selection-end-column="8" />
<folding>
<element signature="e#418#504#0" />
<element signature="e#572#861#0" />
<element signature="e#589#664#0" />
<element signature="e#970#1029#0" />
<element signature="e#1101#1222#0" />
<element signature="e#1287#1412#0" />
<element signature="e#1592#1756#0" />
<element signature="e#1632#1699#0" />
<element signature="e#1632#1657#0" />
<element signature="e#2298#2629#0" />
<element signature="e#2356#2591#0" />
<element signature="e#2377#2463#0" />
<element signature="e#2500#2591#0" />
<element signature="e#3125#3230#0" />
<element signature="e#3290#3736#0" />
<element signature="e#3290#3385#1" />
<element signature="e#3421#3612#0" />
<element signature="e#3795#4502#0" />
<element signature="e#3795#3930#1" />
<element signature="e#3957#4149#0" />
<element signature="e#4551#4649#1" />
<element signature="e#4676#4912#0" />
<element signature="e#5111#5313#0" />
<element signature="e#5111#5250#1" />
<element signature="e#5358#5712#0" />
<element signature="e#5358#5430#1" />
<element signature="e#5392#8966#0" />
<element signature="e#5784#5960#0" />
<element signature="e#6018#6073#0" />
<element signature="e#6878#7377#0" />
<element signature="e#6878#7007#1" />
<element signature="e#7043#7201#0" />
<element signature="e#7238#7306#0" />
<element signature="e#7438#8094#0" />
<element signature="e#7438#7565#1" />
<element signature="e#7744#8094#0" />
<element signature="e#7803#8094#0" />
<element signature="e#7880#8047#0" />
<element signature="e#8146#8320#1" />
<element signature="e#8702#9196#0" />
<element signature="e#8702#8791#1" />
<element signature="e#8818#8942#0" />
<marker date="1537546530445" expanded="true" signature="5242:5702" ph="..." />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="345">
<caret line="23" lean-forward="true" selection-start-line="23" selection-end-line="23" />
<folding>
<marker date="1537545955912" expanded="true" signature="135:323" ph="..." />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="105">
<caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding>
<element signature="e#0#46#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="270">
<caret line="36" column="22" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
</state>
</provider>
</entry>
</file>
</leaf>
</split-first>
<split-second>
<leaf>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1106">
<caret line="274" column="44" selection-start-line="274" selection-start-column="44" selection-end-line="274" selection-end-column="44" />
<folding>
<element signature="e#342#5712#0" />
<marker date="1537546530445" expanded="true" signature="5242:5702" ph="..." />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</split-second>
</splitter>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
@ -111,6 +187,11 @@
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
<option name="RECENT_BRANCH_BY_REPOSITORY">
<map>
<entry key="$PROJECT_DIR$" value="master" />
</map>
</option>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
@ -119,6 +200,9 @@
<option value="$PROJECT_DIR$/explorer/test_explorer.py" />
<option value="$PROJECT_DIR$/core/mysql_wrapper.py" />
<option value="$PROJECT_DIR$/core/alerts.py" />
<option value="$PROJECT_DIR$/core/task.py" />
<option value="$PROJECT_DIR$/capturer/__init__.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" />
<option value="$PROJECT_DIR$/explorer/explorer.py" />
</list>
</option>
@ -133,6 +217,7 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
@ -140,6 +225,11 @@
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="capturer" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
@ -158,7 +248,6 @@
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
@ -223,7 +312,14 @@
<option name="project" value="LOCAL" />
<updated>1536513741650</updated>
</task>
<option name="localTasksCounter" value="2" />
<task id="LOCAL-00002" summary="Pequeños detalles en Explorer.">
<created>1536514972249</created>
<option name="number" value="00002" />
<option name="presentableId" value="LOCAL-00002" />
<option name="project" value="LOCAL" />
<updated>1536514972249</updated>
</task>
<option name="localTasksCounter" value="3" />
<servers />
</component>
<component name="TodoView" selected-index="1">
@ -239,17 +335,18 @@
<frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.4984326" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" sideWeight="0.5015674" side_tool="true" visible="true" weight="0.14918292" />
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.4937304" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" sideWeight="0.5062696" side_tool="true" visible="true" weight="0.14918292" />
<window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
<window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32983193" />
<window_info active="true" anchor="bottom" id="Version Control" order="7" sideWeight="0.49973643" visible="true" weight="0.32983193" />
<window_info active="true" anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" visible="true" weight="0.32983193" />
<window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.49973643" weight="0.269958" />
<window_info anchor="bottom" id="Terminal" order="8" weight="0.32983193" />
<window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.5007907" side_tool="true" weight="0.32983193" />
<window_info anchor="bottom" id="Python Console" order="10" sideWeight="0.49920928" weight="0.32983193" />
@ -290,14 +387,15 @@
<component name="VcsManagerConfiguration">
<MESSAGE value="Correcciones en wrapper_mysql y avance en metodos de explorer. Iniciado modulo de alertas." />
<MESSAGE value="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo." />
<option name="LAST_COMMIT_MESSAGE" value="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo." />
<MESSAGE value="Pequeños detalles en Explorer." />
<option name="LAST_COMMIT_MESSAGE" value="Pequeños detalles en Explorer." />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/explorer/test_explorer.py" />
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<entry file="file:///usr/lib/python3/dist-packages/IPython/core/interactiveshell.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="391">
<caret line="36" column="22" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
<state relative-caret-position="195">
<caret line="142" column="4" selection-start-line="142" selection-start-column="4" selection-end-line="142" selection-end-column="4" />
</state>
</provider>
</entry>
@ -308,27 +406,88 @@
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/IPython/core/interactiveshell.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="195">
<caret line="142" column="4" selection-start-line="142" selection-start-column="4" selection-end-line="142" selection-end-column="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="105">
<caret line="7" column="34" lean-forward="true" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding>
<element signature="e#0#46#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/task.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/capturer/__init__.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="345">
<caret line="23" lean-forward="true" selection-start-line="23" selection-end-line="23" />
<folding>
<marker date="1537545955912" expanded="true" signature="135:323" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="270">
<caret line="36" column="22" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="90">
<caret line="6" column="29" lean-forward="true" selection-start-line="6" selection-start-column="29" selection-end-line="6" selection-end-column="29" />
<state relative-caret-position="444">
<caret line="162" column="8" selection-start-line="162" selection-start-column="8" selection-end-line="162" selection-end-column="8" />
<folding>
<element signature="e#418#504#0" />
<element signature="e#572#861#0" />
<element signature="e#589#664#0" />
<element signature="e#970#1029#0" />
<element signature="e#1101#1222#0" />
<element signature="e#1287#1412#0" />
<element signature="e#1592#1756#0" />
<element signature="e#1632#1699#0" />
<element signature="e#1632#1657#0" />
<element signature="e#2298#2629#0" />
<element signature="e#2356#2591#0" />
<element signature="e#2377#2463#0" />
<element signature="e#2500#2591#0" />
<element signature="e#3125#3230#0" />
<element signature="e#3290#3736#0" />
<element signature="e#3290#3385#1" />
<element signature="e#3421#3612#0" />
<element signature="e#3795#4502#0" />
<element signature="e#3795#3930#1" />
<element signature="e#3957#4149#0" />
<element signature="e#4551#4649#1" />
<element signature="e#4676#4912#0" />
<element signature="e#5111#5313#0" />
<element signature="e#5111#5250#1" />
<element signature="e#5358#5712#0" />
<element signature="e#5358#5430#1" />
<element signature="e#5392#8966#0" />
<element signature="e#5784#5960#0" />
<element signature="e#6018#6073#0" />
<element signature="e#6878#7377#0" />
<element signature="e#6878#7007#1" />
<element signature="e#7043#7201#0" />
<element signature="e#7238#7306#0" />
<element signature="e#7438#8094#0" />
<element signature="e#7438#7565#1" />
<element signature="e#7744#8094#0" />
<element signature="e#7803#8094#0" />
<element signature="e#7880#8047#0" />
<element signature="e#8146#8320#1" />
<element signature="e#8702#9196#0" />
<element signature="e#8702#8791#1" />
<element signature="e#8818#8942#0" />
<marker date="1537546530445" expanded="true" signature="5242:5702" ph="..." />
</folding>
</state>
</provider>
</entry>

0
capturer/__init__.py Normal file
View file

23
capturer/capturer.py Normal file
View file

@ -0,0 +1,23 @@
import uuid
ads_root = 'https://www.idealista.com/inmueble/'
def create_capturing_task(referencia, db_wrapper, uuid_exploring=None):
query_parameters = {'url': ads_root + referencia,
'uuid': str(uuid.uuid4()),
'status': 'Pending'}
if uuid_exploring is None:
query_statement = """INSERT INTO capturing_tasks_logs
(uuid, write_time, status, url)
VALUES (%(uuid)s, NOW(), %(status)s, url)"""
else:
query_statement = """INSERT INTO capturing_tasks_logs
(uuid, write_time, status, url, fk_uuid_exploring)
VALUES (%(uuid)s, NOW(), %(status)s, url, %(uuid_exploring))s"""
query_parameters['uuid_exploring'] = uuid_exploring
db_wrapper.query(query_statement, query_parameters)

60
explorer/explorer.py
View file

@ -10,6 +10,7 @@ from random import randint
from core.mysql_wrapper import get_anunciosdb, get_tasksdb
from core.scrapping_utils import UrlAttack
from core.alerts import alert_master
from capturer.capturer import create_capturing_task
class Explorer():
@ -48,7 +49,14 @@ class Explorer():
current_task = ExploringTask(self.compose_listing_url)
current_task.explore()
if current_task.status == 'Referencias ready':
referencias = current_task.get_referencias()
for referencia in referencias:
create_capturing_task(referencia, self.tasksdb)
current_task._update_status(self, "Sent to queue")
continue
self.stop()
@ -59,10 +67,12 @@ class Explorer():
pass
def there_is_work(self):
#TODO Añadir que no se trabaja si se ha lanzado tarea en los ultimos 10 minutos
"""
Funcion que agrupa las condiciones que se deben cumplir para poder trabajar
"""
if self.check_if_recent_task():
return False
if not self.in_working_hours():
return False
@ -84,19 +94,6 @@ class Explorer():
sleep(Explorer.sleep_time_no_service)
self.db_retries = self.db_retries + 1
return False
def queue_is_up(self):
#TODO Comprobar que Redis esta vivo
while self.queue_retries <= self.max_queue_retries:
try:
#codigo que testea si redis esta vivo
self.queue_retries = 0
return True
except:
sleep(Explorer.sleep_time_no_service)
self.queue_retries = self.queue_retries + 1
return False
def in_working_hours(self):
@ -154,6 +151,19 @@ class Explorer():
"""
return (self.get_max_referencias_for_today() / 30) * 6
def check_if_recent_task(self):
"""
Mira si se ha creado alguna tarea recientemente
"""
query_statement = """ SELECT count(uuid)
FROM exploring_tasks_logs
WHERE status = 'Attacked'
AND write_time >= now() - INTERVAL 10 MINUTE
"""
cursor_result = self.tasksdb.query(query_statement)
return cursor_result.row_count
def compose_listing_url(self):
"""
Genera URLs de manera aleatoria
@ -169,7 +179,7 @@ class Explorer():
return url
class ExploringTask():
class ExploringTask:
def __init__(self, url):
self.anunciosdb = get_anunciosdb()
@ -192,8 +202,6 @@ class ExploringTask():
self._extract_referencias(attack.get_text())
if self.referencias:
self._update_status('Referencias ready')
self._post_tasks_to_queue()
self._update_status('Sent to Queue')
elif self.there_are_referencias:
self._update_status('Failure - No new referencias in HTML')
else:
@ -246,8 +254,7 @@ class ExploringTask():
for ad in ads:
if self._is_new_listing(ad["data-adid"]):
self.referencias.append(ad["data-adid"])
def _is_new_listing(self, referencia):
"""
Comprueba si el listing ya existe en la base de datos de anuncios
@ -263,12 +270,15 @@ class ExploringTask():
return False
else:
return True
def _post_tasks_to_queue(self):
#TODO Mandar las referencias a redis
pass
def get_referencias(self):
"""
Devuelve las referencias, si las hay
"""
if self.referencias:
return self.referencias
else:
return None