counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Pequeños detalles en Explorer.

Browse source
This commit is contained in:
pablomartincalvo 2018-09-09 19:42:52 +02:00
parent 7a795d1fb8
commit b77a4752b8
2 changed files with 129 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

161
.idea/workspace.xml generated
View file

@ -1,12 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="Correcciones en wrapper_mysql y avance en metodos de explorer. Iniciado modulo de alertas.">
<change afterPath="$PROJECT_DIR$/core/alerts.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/core/mysql_wrapper.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/mysql_wrapper.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/core/scrapping_utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/scrapping_utils.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/explorer/explorer.py" beforeDir="false" afterPath="$PROJECT_DIR$/explorer/explorer.py" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
@ -21,40 +17,45 @@
<counts>
<entry key="project.open.time.0" value="1" />
<entry key="project.open.time.13" value="1" />
<entry key="project.opened" value="2" />
<entry key="project.open.time.14" value="1" />
<entry key="project.opened" value="3" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="py" value="4" />
<entry key="py" value="5" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="Python" value="4" />
<entry key="Python" value="5" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.edit">
<counts>
<entry key="py" value="634" />
<entry key="txt" value="92" />
<entry key="Python Console" value="1519" />
<entry key="py" value="1383" />
<entry key="txt" value="183" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="PLAIN_TEXT" value="92" />
<entry key="Python" value="634" />
<entry key="PLAIN_TEXT" value="183" />
<entry key="Python" value="2902" />
</counts>
</usages-collector>
</session>
</component>
<component name="FavoritesManager">
<favorites_list name="Drogon" />
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="150">
<caret line="11" column="18" lean-forward="true" selection-start-line="11" selection-start-column="18" selection-end-line="11" selection-end-column="18" />
<state relative-caret-position="90">
<caret line="6" column="29" lean-forward="true" selection-start-line="6" selection-start-column="29" selection-end-line="6" selection-end-column="29" />
</state>
</provider>
</entry>
@ -62,11 +63,10 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="45">
<caret line="3" lean-forward="true" selection-start-line="2" selection-start-column="14" selection-end-line="3" />
<state relative-caret-position="105">
<caret line="7" column="34" lean-forward="true" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding>
<element signature="e#0#46#0" expanded="true" />
<marker date="1535650379609" expanded="true" signature="219:256" ph="..." />
</folding>
</state>
</provider>
@ -76,16 +76,16 @@
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="525">
<caret line="35" column="37" lean-forward="true" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" />
<caret line="35" column="37" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="391">
<caret line="36" column="22" lean-forward="true" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
<caret line="36" column="22" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
</state>
</provider>
</entry>
@ -102,6 +102,11 @@
<component name="FindInProjectRecents">
<findStrings>
<find>queue_retries</find>
<find>month</find>
<find>start</find>
<find>datetime</find>
<find>task</find>
<find>exploring_tasks</find>
</findStrings>
</component>
<component name="Git.Settings">
@ -112,9 +117,9 @@
<list>
<option value="$PROJECT_DIR$/core/scrapping_utils.py" />
<option value="$PROJECT_DIR$/explorer/test_explorer.py" />
<option value="$PROJECT_DIR$/explorer/explorer.py" />
<option value="$PROJECT_DIR$/core/alerts.py" />
<option value="$PROJECT_DIR$/core/mysql_wrapper.py" />
<option value="$PROJECT_DIR$/core/alerts.py" />
<option value="$PROJECT_DIR$/explorer/explorer.py" />
</list>
</option>
</component>
@ -157,6 +162,7 @@
</panes>
</component>
<component name="PropertiesComponent">
<property name="TODO_SCOPE" value="All Places" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RunDashboard">
@ -171,6 +177,34 @@
</list>
</option>
</component>
<component name="RunManager">
<configuration name="alerts" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/core" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/core/alerts.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Python.alerts" />
</list>
</recent_temporary>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
@ -182,9 +216,17 @@
<option name="presentableId" value="Default" />
<updated>1534095188685</updated>
</task>
<task id="LOCAL-00001" summary="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo.">
<created>1536513741650</created>
<option name="number" value="00001" />
<option name="presentableId" value="LOCAL-00001" />
<option name="project" value="LOCAL" />
<updated>1536513741650</updated>
</task>
<option name="localTasksCounter" value="2" />
<servers />
</component>
<component name="TodoView" selected-index="4">
<component name="TodoView" selected-index="1">
<todo-panel id="selected-file">
<is-autoscroll-to-source value="true" />
</todo-panel>
@ -197,69 +239,96 @@
<frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.4984326" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" sideWeight="0.5015674" side_tool="true" visible="true" weight="0.14918292" />
<window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info active="true" anchor="bottom" id="TODO" order="6" visible="true" weight="0.32983193" />
<window_info anchor="bottom" id="Version Control" order="7" weight="0.32983193" />
<window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32983193" />
<window_info active="true" anchor="bottom" id="Version Control" order="7" sideWeight="0.49973643" visible="true" weight="0.32983193" />
<window_info anchor="bottom" id="Terminal" order="8" weight="0.32983193" />
<window_info anchor="bottom" id="Event Log" order="9" side_tool="true" />
<window_info anchor="bottom" id="Python Console" order="10" weight="0.32983193" />
<window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.5007907" side_tool="true" weight="0.32983193" />
<window_info anchor="bottom" id="Python Console" order="10" sideWeight="0.49920928" weight="0.32983193" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout>
</component>
<component name="Vcs.Log.Tabs.Properties">
<option name="TAB_STATES">
<map>
<entry key="MAIN">
<value>
<State>
<option name="RECENTLY_FILTERED_USER_GROUPS">
<collection />
</option>
<option name="RECENTLY_FILTERED_BRANCH_GROUPS">
<collection />
</option>
<option name="COLUMN_ORDER">
<list>
<option value="0" />
<option value="1" />
<option value="2" />
<option value="3" />
</list>
</option>
</State>
</value>
</entry>
</map>
</option>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value="Correcciones en wrapper_mysql y avance en metodos de explorer. Iniciado modulo de alertas." />
<option name="LAST_COMMIT_MESSAGE" value="Correcciones en wrapper_mysql y avance en metodos de explorer. Iniciado modulo de alertas." />
<MESSAGE value="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo." />
<option name="LAST_COMMIT_MESSAGE" value="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo." />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/explorer/test_explorer.py">
<entry file="file://$PROJECT_DIR$/explorer/test_explorer.py" />
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="90">
<caret line="6" lean-forward="true" selection-start-line="6" selection-end-line="6" />
<state relative-caret-position="391">
<caret line="36" column="22" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="525">
<caret line="35" column="37" lean-forward="true" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" />
<caret line="35" column="37" selection-start-line="35" selection-start-column="37" selection-end-line="35" selection-end-column="37" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<entry file="file:///usr/lib/python3/dist-packages/IPython/core/interactiveshell.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="150">
<caret line="11" column="18" lean-forward="true" selection-start-line="11" selection-start-column="18" selection-end-line="11" selection-end-column="18" />
<state relative-caret-position="195">
<caret line="142" column="4" selection-start-line="142" selection-start-column="4" selection-end-line="142" selection-end-column="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="45">
<caret line="3" lean-forward="true" selection-start-line="2" selection-start-column="14" selection-end-line="3" />
<state relative-caret-position="105">
<caret line="7" column="34" lean-forward="true" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding>
<element signature="e#0#46#0" expanded="true" />
<marker date="1535650379609" expanded="true" signature="219:256" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="391">
<caret line="36" column="22" lean-forward="true" selection-start-line="36" selection-start-column="22" selection-end-line="36" selection-end-column="22" />
<state relative-caret-position="90">
<caret line="6" column="29" lean-forward="true" selection-start-line="6" selection-start-column="29" selection-end-line="6" selection-end-column="29" />
</state>
</provider>
</entry>

26
explorer/explorer.py
View file

@ -18,10 +18,11 @@ class Explorer():
working_hours = {'start': datetime.time(9, 0, 0),
'end': datetime.time(18, 0, 0)}
monthly_capture_target = 1000
def __init__(self):
try:
self.anunciosdb = get_anunciosdb()
self.tasksdb = get_tasksdb()
except:
print("Could not connect to anuncios DB")
@ -58,6 +59,7 @@ class Explorer():
pass
def there_is_work(self):
#TODO Añadir que no se trabaja si se ha lanzado tarea en los ultimos 10 minutos
"""
Funcion que agrupa las condiciones que se deben cumplir para poder trabajar
"""
@ -98,50 +100,50 @@ class Explorer():
return False
def in_working_hours(self):
return Explorer.working_hours['start'] <= datetime.now().time() <= Explorer.working_hours['end']
return Explorer.working_hours['start'] <= datetime.datetime.now().time() <= Explorer.working_hours['end']
def get_referencias_acquired_today(self):
"""
Cuenta cuantas nuevas referencias han aparecido en las ultimas 24 horas
"""
query_statement = """ SELECT count(referencias)
query_statement = """ SELECT count(referencia)
FROM primera_captura_full
WHERE fecha_captura >= now() - INTERVAL 1 DAY;
"""
cursor_result = self.anunciosdb.query(query_statement)
return cursor_result.fetchone()
return cursor_result.fetchone()[0]
def get_max_referencias_for_today(self):
"""
Calcula la cantidad objetivo para las ultimas 24 horas en base a la
diferencia con el objetivo mensual
"""
query_statement = """ SELECT count(referencias)
query_statement = """ SELECT count(referencia)
FROM primera_captura_full
WHERE fecha_captura >= now() - INTERVAL 30 DAY;
"""
cursor_result = self.anunciosdb.query(query_statement)
new_referencias_last_30 = cursor_result.fetchone()
new_referencias_last_30 = cursor_result.fetchone()[0]
deviation = (Explorer.monthly_capture_target - new_referencias_last_30) / Explorer.monthly_capture_target
max_referencias = (Explorer.monthly_capture_target/30) * (1 + (deviation))
max_referencias = (Explorer.monthly_capture_target/30) * (1 + deviation)
return max_referencias
def get_tasks_created_today(self):
"""
Mira en el task log cuantas tareas se han iniciado en las ultimas 24 horas
"""
query_statement = """ SELECT count(uuid)
FROM exploring_tasks
FROM exploring_tasks_logs
WHERE status = 'Attacked'
AND write_time >= now() - INTERVAL 1 DAY;
"""
cursor_result = self.tasksdb.query(query_statement)
tasks_created_today = cursor_result.fetchone()
tasks_created_today = cursor_result.fetchone()[0]
return tasks_created_today