counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Iniciado refresher.

Browse source
This commit is contained in:
pablomartincalvo 2018-10-14 19:24:17 +02:00
parent e0216060b9
commit 8f5a120096
4 changed files with 164 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

189
.idea/workspace.xml generated
View file

@ -1,11 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="Puesto bien mains en explorer y capturer.">
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/refresher/refresher.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/explorer/explorer.py" beforeDir="false" afterPath="$PROJECT_DIR$/explorer/explorer.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/db_layer/capturas_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturas_interface.py" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
@ -30,12 +30,12 @@
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="py" value="46" />
<entry key="py" value="47" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="Python" value="43" />
<entry key="Python" value="44" />
<entry key="Scratch" value="3" />
</counts>
</usages-collector>
@ -44,7 +44,7 @@
<entry key="Python Console" value="1579" />
<entry key="capturer" value="862" />
<entry key="dummy" value="14" />
<entry key="py" value="15563" />
<entry key="py" value="16307" />
<entry key="scratch_1" value="489" />
<entry key="txt" value="990" />
</counts>
@ -52,7 +52,7 @@
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="PLAIN_TEXT" value="1004" />
<entry key="Python" value="18323" />
<entry key="Python" value="19067" />
<entry key="Scratch" value="170" />
</counts>
</usages-collector>
@ -71,18 +71,66 @@
<splitter split-orientation="horizontal" split-proportion="0.5">
<split-first>
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<file pinned="false" current-in-tab="false">
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
<state relative-caret-position="210">
<caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
<provider selected="true" editor-type-id="text-editor" />
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="435">
<caret line="29" column="37" lean-forward="true" selection-start-line="29" selection-start-column="37" selection-end-line="29" selection-end-column="37" />
<folding>
<marker date="1539537667973" expanded="true" signature="646:655" ph="..." />
<marker date="1539537667973" expanded="true" signature="646:729" ph="..." />
<marker date="1539537667973" expanded="true" signature="750:759" ph="..." />
<marker date="1539537667973" expanded="true" signature="864:873" ph="..." />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
<folding>
<element signature="e#0#22#0" expanded="true" />
<marker date="1539537667985" expanded="true" signature="166:167" ph="..." />
<marker date="1539537667985" expanded="true" signature="166:172" ph="..." />
<marker date="1539537667985" expanded="true" signature="459:468" ph="..." />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="45">
<caret line="4" lean-forward="true" selection-start-line="4" selection-end-line="4" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="18720">
<caret line="1254" selection-start-line="1254" selection-end-line="1254" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
@ -101,20 +149,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="401">
<caret line="283" selection-start-line="283" selection-end-line="283" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="265">
<caret line="66" column="36" lean-forward="true" selection-start-line="66" selection-start-column="36" selection-end-line="66" selection-end-column="36" />
<folding>
<element signature="e#0#11#0" expanded="true" />
</folding>
<state relative-caret-position="330">
<caret line="55" column="20" selection-start-line="55" selection-start-column="20" selection-end-line="55" selection-end-column="90" />
</state>
</provider>
</entry>
@ -122,7 +158,7 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="263">
<state relative-caret-position="300">
<caret line="47" selection-start-line="47" selection-end-line="47" />
<folding>
<element signature="e#24#46#0" expanded="true" />
@ -131,14 +167,11 @@
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/alerts.py">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="105">
<caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding>
<element signature="e#0#46#0" expanded="true" />
</folding>
<state relative-caret-position="420">
<caret line="34" lean-forward="true" selection-start-line="34" selection-end-line="34" />
</state>
</provider>
</entry>
@ -167,6 +200,9 @@
<find>re.</find>
</findStrings>
</component>
<component name="Git.Rebase.Settings">
<option name="ONTO" value="refs/remotes/origin/testing" />
</component>
<component name="Git.Settings">
<option name="UPDATE_TYPE" value="REBASE" />
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
@ -192,10 +228,11 @@
<option value="$PROJECT_DIR$/tests/capturing_tests.py" />
<option value="$PROJECT_DIR$/tests/capturer_tests.py" />
<option value="$PROJECT_DIR$/core/mysql_wrapper.py" />
<option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
<option value="$PROJECT_DIR$/explorer/explorer.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" />
<option value="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" />
<option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" />
<option value="$PROJECT_DIR$/refresher/refresher.py" />
</list>
</option>
</component>
@ -219,7 +256,7 @@
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="capturer" type="462c0819:PsiDirectoryNode" />
<item name="tests" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
@ -497,14 +534,14 @@
<frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.48251748" visible="true" weight="0.14918292" />
<window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.48251748" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" sideWeight="0.5174825" side_tool="true" visible="true" weight="0.14918292" />
<window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
<window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.32983193" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
<window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" />
<window_info anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32878152" />
@ -658,9 +695,6 @@
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="225">
@ -736,25 +770,11 @@
</entry>
<entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="203">
<state relative-caret-position="18720">
<caret line="1254" selection-start-line="1254" selection-end-line="1254" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" selection-start-line="20" selection-end-line="20" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="225">
<caret line="15" column="18" lean-forward="true" selection-start-line="15" selection-start-column="18" selection-end-line="15" selection-end-column="18" />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/_pydev_imps/_pydev_execfile.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
@ -769,9 +789,33 @@
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="45">
<caret line="4" lean-forward="true" selection-start-line="4" selection-end-line="4" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="210">
<caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="330">
<caret line="55" column="20" selection-start-line="55" selection-start-column="20" selection-end-line="55" selection-end-column="90" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="263">
<state relative-caret-position="300">
<caret line="47" selection-start-line="47" selection-end-line="47" />
<folding>
<element signature="e#24#46#0" expanded="true" />
@ -779,27 +823,36 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="401">
<caret line="283" selection-start-line="283" selection-end-line="283" />
<state relative-caret-position="435">
<caret line="29" column="37" lean-forward="true" selection-start-line="29" selection-start-column="37" selection-end-line="29" selection-end-column="37" />
<folding>
<marker date="1539537667973" expanded="true" signature="646:655" ph="..." />
<marker date="1539537667973" expanded="true" signature="646:729" ph="..." />
<marker date="1539537667973" expanded="true" signature="750:759" ph="..." />
<marker date="1539537667973" expanded="true" signature="864:873" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="265">
<caret line="66" column="36" lean-forward="true" selection-start-line="66" selection-start-column="36" selection-end-line="66" selection-end-column="36" />
<state relative-caret-position="285">
<caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
<folding>
<element signature="e#0#11#0" expanded="true" />
<element signature="e#0#22#0" expanded="true" />
<marker date="1539537667985" expanded="true" signature="166:167" ph="..." />
<marker date="1539537667985" expanded="true" signature="166:172" ph="..." />
<marker date="1539537667985" expanded="true" signature="459:468" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
<state relative-caret-position="420">
<caret line="34" lean-forward="true" selection-start-line="34" selection-end-line="34" />
</state>
</provider>
</entry>

3
capturer/capturer.py
View file

@ -7,7 +7,7 @@ import datetime
from db_layer.capturing_tasks_interface import capturing_interface
from db_layer.capturas_interface import capturas_interface
from core.scrapping_utils import UrlAttack
from refresher.refresher import Refresher
class Capturer:
sleep_time_no_work = 60
@ -30,6 +30,7 @@ class Capturer:
if task.status == 'Data ready':
ad_data = task.get_ad_data()
else:
Refresher.dead_ad_checker(task.html)
continue
capturas_interface.insert_captura(ad_data)

11
db_layer/capturas_interface.py
View file

@ -20,6 +20,17 @@ class CapturasInterface():
self.anunciosdb.query(query_statement, query_parameters)
def old_ads_exist(self):
#TODO Mira si hay algun anuncio que este viejete y necesite refresco
def get_old_ad(self):
#TODO Recuperar anuncio que este viejete y que necesite refresco
def mark_dead_ad(self, referencia):
#TODO Marca un anuncio como muerto
capturas_interface = CapturasInterface()

30
refresher/refresher.py Normal file
View file

@ -0,0 +1,30 @@
from time import sleep
from db_layer.capturas_interface import capturas_interface
from db_layer.capturing_tasks_interface import capturing_interface
class Refresher:
def start(self):
while True:
sleep(30)
if capturas_interface.old_ads_exist():
old_ad = capturas_interface.get_old_ad()
capturing_interface.create_capturing_task(old_ad['referencia'])
def dead_ad_checker(self, html):
#TODO Comprueba si el anuncio esta muerto y, si es asi, lo marca en la tabla de capturas
if blabla:
capturas_interface.mark_dead_ad(referencia)