counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Pequeños detalles en Explorer.

Browse source
This commit is contained in:
pablomartincalvo 2018-09-09 19:42:52 +02:00
parent 7a795d1fb8
commit b77a4752b8
2 changed files with 129 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

161
.idea/workspace.xml generated
View file

@ -1,12 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="Correcciones en wrapper_mysql y avance en metodos de explorer. Iniciado modulo de alertas."> <list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/core/alerts.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/core/mysql_wrapper.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/mysql_wrapper.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/core/scrapping_utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/scrapping_utils.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/explorer/explorer.py" beforeDir="false" afterPath="$PROJECT_DIR$/explorer/explorer.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/explorer/explorer.py" beforeDir="false" afterPath="$PROJECT_DIR$/explorer/explorer.py" afterDir="false" />
</list> </list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
@ -21,40 +17,45 @@
<counts> <counts>
<entry key="project.open.time.0" value="1" /> <entry key="project.open.time.0" value="1" />
<entry key="project.open.time.13" value="1" /> <entry key="project.open.time.13" value="1" />
<entry key="project.opened" value="2" /> <entry key="project.open.time.14" value="1" />
<entry key="project.opened" value="3" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.extensions.open"> <usages-collector id="statistics.file.extensions.open">
<counts> <counts>
<entry key="py" value="4" /> <entry key="py" value="5" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.open"> <usages-collector id="statistics.file.types.open">
<counts> <counts>
<entry key="Python" value="4" /> <entry key="Python" value="5" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.extensions.edit"> <usages-collector id="statistics.file.extensions.edit">
<counts> <counts>
<entry key="py" value="634" /> <entry key="Python Console" value="1519" />
<entry key="txt" value="92" /> <entry key="py" value="1383" />
<entry key="txt" value="183" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.edit"> <usages-collector id="statistics.file.types.edit">
<counts> <counts>
<entry key="PLAIN_TEXT" value="92" /> <entry key="PLAIN_TEXT" value="183" />
<entry key="Python" value="634" /> <entry key="Python" value="2902" />
</counts> </counts>
</usages-collector> </usages-collector>
</session> </session>
</component> </component>
<component name="FavoritesManager">
<favorites_list name="Drogon" />
</component>
<component name="FileEditorManager"> <component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300"> <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/explorer/explorer.py"> <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="150"> <state relative-caret-position="90">
<caret line="11" column="18" lean-forward="true" selection-start-line="11" selection-start-column="18" selection-end-line="11" selection-end-column="18" /> <caret line="6" column="29" lean-forward="true" selection-start-line="6" selection-start-column="29" selection-end-line="6" selection-end-column="29" />
</state> </state>
</provider> </provider>
</entry> </entry>
@ -62,11 +63,10 @@
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/alerts.py"> <entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="45"> <state relative-caret-position="105">
<caret line="3" lean-forward="true" selection-start-line="2" selection-start-column="14" selection-end-line="3" /> <caret line="7" column="34" lean-forward="true" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding> <folding>
<element signature="e#0#46#0" expanded="true" /> <element signature="e#0#46#0" expanded="true" />
<marker date="1535650379609" expanded="true" signature="219:256" ph="..." />
</folding> </folding>
</state> </state>
</provider> </provider>
@ -76,16 +76,16 @@
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py"> <entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="525"> <state relative-caret-position="525">
<caret line="35" column="37" lean-forward="true" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" /> <caret line="35" column="37" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" />
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="true"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py"> <entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="391"> <state relative-caret-position="391">
<caret line="36" column="22" lean-forward="true" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" /> <caret line="36" column="22" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
</state> </state>
</provider> </provider>
</entry> </entry>
@ -102,6 +102,11 @@
<component name="FindInProjectRecents"> <component name="FindInProjectRecents">
<findStrings> <findStrings>
<find>queue_retries</find> <find>queue_retries</find>
<find>month</find>
<find>start</find>
<find>datetime</find>
<find>task</find>
<find>exploring_tasks</find>
</findStrings> </findStrings>
</component> </component>
<component name="Git.Settings"> <component name="Git.Settings">
@ -112,9 +117,9 @@
<list> <list>
<option value="$PROJECT_DIR$/core/scrapping_utils.py" /> <option value="$PROJECT_DIR$/core/scrapping_utils.py" />
<option value="$PROJECT_DIR$/explorer/test_explorer.py" /> <option value="$PROJECT_DIR$/explorer/test_explorer.py" />
<option value="$PROJECT_DIR$/explorer/explorer.py" />
<option value="$PROJECT_DIR$/core/alerts.py" />
<option value="$PROJECT_DIR$/core/mysql_wrapper.py" /> <option value="$PROJECT_DIR$/core/mysql_wrapper.py" />
<option value="$PROJECT_DIR$/core/alerts.py" />
<option value="$PROJECT_DIR$/explorer/explorer.py" />
</list> </list>
</option> </option>
</component> </component>
@ -157,6 +162,7 @@
</panes> </panes>
</component> </component>
<component name="PropertiesComponent"> <component name="PropertiesComponent">
<property name="TODO_SCOPE" value="All Places" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" /> <property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component> </component>
<component name="RunDashboard"> <component name="RunDashboard">
@ -171,6 +177,34 @@
</list> </list>
</option> </option>
</component> </component>
<component name="RunManager">
<configuration name="alerts" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/core" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/core/alerts.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Python.alerts" />
</list>
</recent_temporary>
</component>
<component name="SvnConfiguration"> <component name="SvnConfiguration">
<configuration /> <configuration />
</component> </component>
@ -182,9 +216,17 @@
<option name="presentableId" value="Default" /> <option name="presentableId" value="Default" />
<updated>1534095188685</updated> <updated>1534095188685</updated>
</task> </task>
<task id="LOCAL-00001" summary="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo.">
<created>1536513741650</created>
<option name="number" value="00001" />
<option name="presentableId" value="LOCAL-00001" />
<option name="project" value="LOCAL" />
<updated>1536513741650</updated>
</task>
<option name="localTasksCounter" value="2" />
<servers /> <servers />
</component> </component>
<component name="TodoView" selected-index="4"> <component name="TodoView" selected-index="1">
<todo-panel id="selected-file"> <todo-panel id="selected-file">
<is-autoscroll-to-source value="true" /> <is-autoscroll-to-source value="true" />
</todo-panel> </todo-panel>
@ -197,69 +239,96 @@
<frame x="0" y="-2" width="1920" height="1082" extended-state="6" /> <frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
<editor active="true" /> <editor active="true" />
<layout> <layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.14918292" /> <window_info content_ui="combo" id="Project" order="0" sideWeight="0.4984326" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" /> <window_info id="Structure" order="1" sideWeight="0.5015674" side_tool="true" visible="true" weight="0.14918292" />
<window_info id="Favorites" order="2" side_tool="true" /> <window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
<window_info anchor="bottom" id="Message" order="0" /> <window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" /> <window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" /> <window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" /> <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" /> <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" /> <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info active="true" anchor="bottom" id="TODO" order="6" visible="true" weight="0.32983193" /> <window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32983193" />
<window_info anchor="bottom" id="Version Control" order="7" weight="0.32983193" /> <window_info active="true" anchor="bottom" id="Version Control" order="7" sideWeight="0.49973643" visible="true" weight="0.32983193" />
<window_info anchor="bottom" id="Terminal" order="8" weight="0.32983193" /> <window_info anchor="bottom" id="Terminal" order="8" weight="0.32983193" />
<window_info anchor="bottom" id="Event Log" order="9" side_tool="true" /> <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.5007907" side_tool="true" weight="0.32983193" />
<window_info anchor="bottom" id="Python Console" order="10" weight="0.32983193" /> <window_info anchor="bottom" id="Python Console" order="10" sideWeight="0.49920928" weight="0.32983193" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" /> <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" /> <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" /> <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout> </layout>
</component> </component>
<component name="Vcs.Log.Tabs.Properties">
<option name="TAB_STATES">
<map>
<entry key="MAIN">
<value>
<State>
<option name="RECENTLY_FILTERED_USER_GROUPS">
<collection />
</option>
<option name="RECENTLY_FILTERED_BRANCH_GROUPS">
<collection />
</option>
<option name="COLUMN_ORDER">
<list>
<option value="0" />
<option value="1" />
<option value="2" />
<option value="3" />
</list>
</option>
</State>
</value>
</entry>
</map>
</option>
</component>
<component name="VcsContentAnnotationSettings"> <component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" /> <option name="myLimit" value="2678400000" />
</component> </component>
<component name="VcsManagerConfiguration"> <component name="VcsManagerConfiguration">
<MESSAGE value="Correcciones en wrapper_mysql y avance en metodos de explorer. Iniciado modulo de alertas." /> <MESSAGE value="Correcciones en wrapper_mysql y avance en metodos de explorer. Iniciado modulo de alertas." />
<option name="LAST_COMMIT_MESSAGE" value="Correcciones en wrapper_mysql y avance en metodos de explorer. Iniciado modulo de alertas." /> <MESSAGE value="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo." />
<option name="LAST_COMMIT_MESSAGE" value="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo." />
</component> </component>
<component name="editorHistoryManager"> <component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/explorer/test_explorer.py"> <entry file="file://$PROJECT_DIR$/explorer/test_explorer.py" />
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="90"> <state relative-caret-position="391">
<caret line="6" lean-forward="true" selection-start-line="6" selection-end-line="6" /> <caret line="36" column="22" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py"> <entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="525"> <state relative-caret-position="525">
<caret line="35" column="37" lean-forward="true" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" /> <caret line="35" column="37" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/explorer/explorer.py"> <entry file="file:///usr/lib/python3/dist-packages/IPython/core/interactiveshell.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="150"> <state relative-caret-position="195">
<caret line="11" column="18" lean-forward="true" selection-start-line="11" selection-start-column="18" selection-end-line="11" selection-end-column="18" /> <caret line="142" column="4" selection-start-line="142" selection-start-column="4" selection-end-line="142" selection-end-column="4" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/core/alerts.py"> <entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="45"> <state relative-caret-position="105">
<caret line="3" lean-forward="true" selection-start-line="2" selection-start-column="14" selection-end-line="3" /> <caret line="7" column="34" lean-forward="true" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding> <folding>
<element signature="e#0#46#0" expanded="true" /> <element signature="e#0#46#0" expanded="true" />
<marker date="1535650379609" expanded="true" signature="219:256" ph="..." />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py"> <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="391"> <state relative-caret-position="90">
<caret line="36" column="22" lean-forward="true" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" /> <caret line="6" column="29" lean-forward="true" selection-start-line="6" selection-start-column="29" selection-end-line="6" selection-end-column="29" />
</state> </state>
</provider> </provider>
</entry> </entry>

18
explorer/explorer.py
View file

@ -22,6 +22,7 @@ class Explorer():
def __init__(self): def __init__(self):
try: try:
self.anunciosdb = get_anunciosdb() self.anunciosdb = get_anunciosdb()
self.tasksdb = get_tasksdb()
except: except:
print("Could not connect to anuncios DB") print("Could not connect to anuncios DB")
@ -58,6 +59,7 @@ class Explorer():
pass pass
def there_is_work(self): def there_is_work(self):
#TODO Añadir que no se trabaja si se ha lanzado tarea en los ultimos 10 minutos
""" """
Funcion que agrupa las condiciones que se deben cumplir para poder trabajar Funcion que agrupa las condiciones que se deben cumplir para poder trabajar
""" """
@ -98,36 +100,36 @@ class Explorer():
return False return False
def in_working_hours(self): def in_working_hours(self):
return Explorer.working_hours['start'] <= datetime.now().time() <= Explorer.working_hours['end'] return Explorer.working_hours['start'] <= datetime.datetime.now().time() <= Explorer.working_hours['end']
def get_referencias_acquired_today(self): def get_referencias_acquired_today(self):
""" """
Cuenta cuantas nuevas referencias han aparecido en las ultimas 24 horas Cuenta cuantas nuevas referencias han aparecido en las ultimas 24 horas
""" """
query_statement = """ SELECT count(referencias) query_statement = """ SELECT count(referencia)
FROM primera_captura_full FROM primera_captura_full
WHERE fecha_captura >= now() - INTERVAL 1 DAY; WHERE fecha_captura >= now() - INTERVAL 1 DAY;
""" """
cursor_result = self.anunciosdb.query(query_statement) cursor_result = self.anunciosdb.query(query_statement)
return cursor_result.fetchone() return cursor_result.fetchone()[0]
def get_max_referencias_for_today(self): def get_max_referencias_for_today(self):
""" """
Calcula la cantidad objetivo para las ultimas 24 horas en base a la Calcula la cantidad objetivo para las ultimas 24 horas en base a la
diferencia con el objetivo mensual diferencia con el objetivo mensual
""" """
query_statement = """ SELECT count(referencias) query_statement = """ SELECT count(referencia)
FROM primera_captura_full FROM primera_captura_full
WHERE fecha_captura >= now() - INTERVAL 30 DAY; WHERE fecha_captura >= now() - INTERVAL 30 DAY;
""" """
cursor_result = self.anunciosdb.query(query_statement) cursor_result = self.anunciosdb.query(query_statement)
new_referencias_last_30 = cursor_result.fetchone() new_referencias_last_30 = cursor_result.fetchone()[0]
deviation = (Explorer.monthly_capture_target - new_referencias_last_30) / Explorer.monthly_capture_target deviation = (Explorer.monthly_capture_target - new_referencias_last_30) / Explorer.monthly_capture_target
max_referencias = (Explorer.monthly_capture_target/30) * (1 + (deviation)) max_referencias = (Explorer.monthly_capture_target/30) * (1 + deviation)
return max_referencias return max_referencias
@ -136,12 +138,12 @@ class Explorer():
Mira en el task log cuantas tareas se han iniciado en las ultimas 24 horas Mira en el task log cuantas tareas se han iniciado en las ultimas 24 horas
""" """
query_statement = """ SELECT count(uuid) query_statement = """ SELECT count(uuid)
FROM exploring_tasks FROM exploring_tasks_logs
WHERE status = 'Attacked' WHERE status = 'Attacked'
AND write_time >= now() - INTERVAL 1 DAY; AND write_time >= now() - INTERVAL 1 DAY;
""" """
cursor_result = self.tasksdb.query(query_statement) cursor_result = self.tasksdb.query(query_statement)
tasks_created_today = cursor_result.fetchone() tasks_created_today = cursor_result.fetchone()[0]
return tasks_created_today return tasks_created_today