Retoques menores en refresher.

This commit is contained in:
pablomartincalvo 2018-10-18 22:36:10 +02:00
parent 3fdbc5f766
commit 238b0f5305
3 changed files with 163 additions and 148 deletions

296
.idea/workspace.xml generated
View file

@ -2,9 +2,7 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/tests/refresher_tests.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/db_layer/capturas_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturas_interface.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/refresher/refresher.py" beforeDir="false" afterPath="$PROJECT_DIR$/refresher/refresher.py" afterDir="false" />
</list>
@ -32,12 +30,12 @@
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="py" value="49" />
<entry key="py" value="52" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="Python" value="46" />
<entry key="Python" value="49" />
<entry key="Scratch" value="3" />
</counts>
</usages-collector>
@ -46,15 +44,15 @@
<entry key="Python Console" value="1579" />
<entry key="capturer" value="862" />
<entry key="dummy" value="14" />
<entry key="py" value="17003" />
<entry key="py" value="17120" />
<entry key="scratch_1" value="489" />
<entry key="txt" value="1009" />
<entry key="txt" value="1114" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="PLAIN_TEXT" value="1023" />
<entry key="Python" value="19661" />
<entry key="PLAIN_TEXT" value="1128" />
<entry key="Python" value="19778" />
<entry key="Scratch" value="272" />
</counts>
</usages-collector>
@ -85,32 +83,33 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret selection-end-line="3" selection-end-column="53" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="660">
<caret line="55" column="20" selection-start-line="55" selection-start-column="20" selection-end-line="55" selection-end-column="20" />
<folding>
<marker date="1539717431052" expanded="true" signature="1271:1277" ph="..." />
<marker date="1539717431052" expanded="true" signature="1298:1307" ph="..." />
</folding>
<state relative-caret-position="195">
<caret line="13" column="28" lean-forward="true" selection-start-line="13" selection-start-column="28" selection-end-line="13" selection-end-column="28" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="370">
<caret line="52" column="15" lean-forward="true" selection-start-line="52" selection-start-column="15" selection-end-line="52" selection-end-column="15" />
<folding>
<marker date="1539889996231" expanded="true" signature="1271:1277" ph="..." />
<marker date="1539889996231" expanded="true" signature="1298:1307" ph="..." />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/tests/refresher_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="68" column="14" selection-start-line="68" selection-start-column="14" selection-end-line="68" selection-end-column="14" />
<state relative-caret-position="165">
<caret line="258" column="30" lean-forward="true" selection-start-line="258" selection-start-column="30" selection-end-line="258" selection-end-column="30" />
<folding>
<element signature="e#123#18418#0" />
<element signature="e#18441#18448#0" />
</folding>
</state>
</provider>
@ -119,17 +118,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="435">
<caret line="39" selection-start-line="39" selection-end-line="42" selection-end-column="23" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="18720">
<caret line="1254" selection-start-line="1254" selection-end-line="1254" />
<state relative-caret-position="495">
<caret line="34" column="58" lean-forward="true" selection-start-line="34" selection-start-column="58" selection-end-line="34" selection-end-column="58" />
</state>
</provider>
</entry>
@ -159,7 +149,7 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="690">
<state relative-caret-position="390">
<caret line="47" selection-start-line="47" selection-end-line="47" />
<folding>
<element signature="e#24#46#0" expanded="true" />
@ -171,8 +161,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="450">
<caret line="36" selection-start-line="36" selection-end-line="36" />
<state relative-caret-position="400">
<caret line="292" selection-start-line="292" selection-end-line="294" selection-end-column="20" />
</state>
</provider>
</entry>
@ -180,13 +170,13 @@
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="16" column="24" selection-start-line="16" selection-start-column="24" selection-end-line="16" selection-end-column="24" />
<state relative-caret-position="210">
<caret line="14" selection-start-line="14" selection-end-line="14" />
<folding>
<element signature="e#0#22#0" expanded="true" />
<marker date="1539717431060" expanded="true" signature="166:167" ph="..." />
<marker date="1539717431060" expanded="true" signature="166:172" ph="..." />
<marker date="1539717431060" expanded="true" signature="470:471" ph="..." />
<marker date="1539889821910" expanded="true" signature="167:168" ph="..." />
<marker date="1539889821910" expanded="true" signature="167:173" ph="..." />
<marker date="1539889821910" expanded="true" signature="475:476" ph="..." />
</folding>
</state>
</provider>
@ -247,9 +237,9 @@
<option value="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" />
<option value="$PROJECT_DIR$/tests/refresher_tests.py" />
<option value="$PROJECT_DIR$/refresher/refresher.py" />
<option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
<option value="$PROJECT_DIR$/tests/refresher_tests.py" />
</list>
</option>
</component>
@ -327,7 +317,7 @@
</list>
</option>
</component>
<component name="RunManager" selected="Python.refresher_tests">
<component name="RunManager" selected="Python.refresher">
<configuration name="capturer" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" />
@ -349,27 +339,6 @@
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="capturer_tests" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/tests" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/tests/capturer_tests.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="explorer" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" />
@ -391,6 +360,27 @@
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="refresher" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/refresher" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/refresher/refresher.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="refresher_tests" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" />
@ -433,17 +423,17 @@
<list>
<item itemvalue="Python.scratch_1" />
<item itemvalue="Python.capturer" />
<item itemvalue="Python.capturer_tests" />
<item itemvalue="Python.explorer" />
<item itemvalue="Python.refresher_tests" />
<item itemvalue="Python.refresher" />
</list>
<recent_temporary>
<list>
<item itemvalue="Python.refresher" />
<item itemvalue="Python.refresher_tests" />
<item itemvalue="Python.scratch_1" />
<item itemvalue="Python.capturer" />
<item itemvalue="Python.explorer" />
<item itemvalue="Python.capturer_tests" />
</list>
</recent_temporary>
</component>
@ -549,7 +539,14 @@
<option name="project" value="LOCAL" />
<updated>1539537857221</updated>
</task>
<option name="localTasksCounter" value="14" />
<task id="LOCAL-00014" summary="Finalizado y testeado localmente refresher, listo para testear en&#10;entorno.">
<created>1539717575817</created>
<option name="number" value="00014" />
<option name="presentableId" value="LOCAL-00014" />
<option name="project" value="LOCAL" />
<updated>1539717575817</updated>
</task>
<option name="localTasksCounter" value="15" />
<servers />
</component>
<component name="TodoView" selected-index="1">
@ -563,16 +560,15 @@
</component>
<component name="ToolWindowManager">
<frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.48076922" weight="0.14918292" />
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.48076922" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" sideWeight="0.5192308" side_tool="true" weight="0.14918292" />
<window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
<window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.32983193" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.39915967" />
<window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32878152" />
@ -648,7 +644,8 @@
<MESSAGE value="Puesto mains en explorer y capturer." />
<MESSAGE value="Puesto bien mains en explorer y capturer." />
<MESSAGE value="Iniciado refresher." />
<option name="LAST_COMMIT_MESSAGE" value="Iniciado refresher." />
<MESSAGE value="Finalizado y testeado localmente refresher, listo para testear en&#10;entorno." />
<option name="LAST_COMMIT_MESSAGE" value="Finalizado y testeado localmente refresher, listo para testear en&#10;entorno." />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
@ -658,6 +655,11 @@
<line>38</line>
<option name="timeStamp" value="7" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/refresher/refresher.py</url>
<line>14</line>
<option name="timeStamp" value="13" />
</line-breakpoint>
</breakpoints>
<breakpoints-dialog>
<breakpoints-dialog />
@ -680,6 +682,11 @@
</expression>
</expressions>
<expressions id="evaluateExpression">
<expression>
<expression-string>capturas_interface.old_ads_exist()</expression-string>
<language-id>Python</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
<expression>
<expression-string>cursor_result.fetchone()</expression-string>
<language-id>Python</language-id>
@ -720,11 +727,6 @@
<language-id>Python</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
<expression>
<expression-string>self.get_max_tasks_today()</expression-string>
<language-id>Python</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
</expressions>
</component>
<component name="editorHistoryManager">
@ -817,13 +819,6 @@
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/apport_python_hook.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="211">
<caret line="46" selection-start-line="46" selection-end-line="46" />
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="18720">
@ -838,16 +833,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="690">
<caret line="47" selection-start-line="47" selection-end-line="47" />
<folding>
<element signature="e#24#46#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="600">
@ -855,51 +840,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="450">
<caret line="36" selection-start-line="36" selection-end-line="36" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="435">
<caret line="39" selection-start-line="39" selection-end-line="42" selection-end-column="23" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret selection-end-line="3" selection-end-column="53" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="16" column="24" selection-start-line="16" selection-start-column="24" selection-end-line="16" selection-end-column="24" />
<folding>
<element signature="e#0#22#0" expanded="true" />
<marker date="1539717431060" expanded="true" signature="166:167" ph="..." />
<marker date="1539717431060" expanded="true" signature="166:172" ph="..." />
<marker date="1539717431060" expanded="true" signature="470:471" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="660">
<caret line="55" column="20" selection-start-line="55" selection-start-column="20" selection-end-line="55" selection-end-column="20" />
<folding>
<marker date="1539717431052" expanded="true" signature="1271:1277" ph="..." />
<marker date="1539717431052" expanded="true" signature="1298:1307" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
<provider selected="true" editor-type-id="text-editor">
<state>
@ -909,10 +849,80 @@
</entry>
<entry file="file://$PROJECT_DIR$/tests/refresher_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="68" column="14" selection-start-line="68" selection-start-column="14" selection-end-line="68" selection-end-column="14" />
<state relative-caret-position="165">
<caret line="258" column="30" lean-forward="true" selection-start-line="258" selection-start-column="30" selection-end-line="258" selection-end-column="30" />
<folding>
<element signature="e#123#18418#0" />
<element signature="e#18441#18448#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="195">
<caret line="13" column="28" lean-forward="true" selection-start-line="13" selection-start-column="28" selection-end-line="13" selection-end-column="28" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="390">
<caret line="47" selection-start-line="47" selection-end-line="47" />
<folding>
<element signature="e#24#46#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="400">
<caret line="292" selection-start-line="292" selection-end-line="294" selection-end-column="20" />
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/apport_python_hook.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="316">
<caret line="46" selection-start-line="46" selection-end-line="46" />
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/mysql/connector/network.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="181">
<caret line="241" selection-start-line="241" selection-end-line="241" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="495">
<caret line="34" column="58" lean-forward="true" selection-start-line="34" selection-start-column="58" selection-end-line="34" selection-end-column="58" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="370">
<caret line="52" column="15" lean-forward="true" selection-start-line="52" selection-start-column="15" selection-end-line="52" selection-end-column="15" />
<folding>
<marker date="1539889996231" expanded="true" signature="1271:1277" ph="..." />
<marker date="1539889996231" expanded="true" signature="1298:1307" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="210">
<caret line="14" selection-start-line="14" selection-end-line="14" />
<folding>
<element signature="e#0#22#0" expanded="true" />
<marker date="1539889821910" expanded="true" signature="167:168" ph="..." />
<marker date="1539889821910" expanded="true" signature="167:173" ph="..." />
<marker date="1539889821910" expanded="true" signature="475:476" ph="..." />
</folding>
</state>
</provider>

View file

@ -49,17 +49,16 @@ class CapturasInterface():
ON da.ad_url LIKE CONCAT('%', uc.referencia, '%')
WHERE uc.fecha_captura < (NOW() - INTERVAL 10 day)
AND da.ad_url is null
ORDER BY RAND()
LIMIT 1
"""
cursor_result = self.anunciosdb.query(query_statement, dictionary=True)
try:
return c
return cursor_result.fetchone()
except:
return None
capturas_interface = CapturasInterface()

View file

@ -2,16 +2,17 @@ from time import sleep
from db_layer.capturas_interface import capturas_interface
from db_layer.capturing_tasks_interface import capturing_interface
class Refresher:
def start(self):
while True:
sleep(30)
sleep(1)
if capturas_interface.old_ads_exist():
old_ad = capturas_interface.get_old_ad()
capturing_interface.create_capturing_task(old_ad['referencia'])
capturing_interface.create_capturing_task(str(old_ad['referencia']))
@staticmethod
def dead_ad_checker(html):
@ -26,6 +27,11 @@ class Refresher:
return False
if __name__ == '__main__':
refresher = Refresher()
refresher.start()