From 8f5a120096e0259fb220069f1cc6c7fb90be88de Mon Sep 17 00:00:00 2001
From: pablomartincalvo <pablomartincalvo@gmail.com>
Date: Sun, 14 Oct 2018 19:24:17 +0200
Subject: [PATCH] Iniciado refresher.

---
 .idea/workspace.xml            | 189 +++++++++++++++++++++------------
 capturer/capturer.py           |   3 +-
 db_layer/capturas_interface.py |  11 ++
 refresher/refresher.py         |  30 ++++++
 4 files changed, 164 insertions(+), 69 deletions(-)
 create mode 100644 refresher/refresher.py

diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 832f210..6fa0537 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -1,11 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ChangeListManager">
-    <list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="Puesto bien mains en explorer y capturer.">
+    <list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
+      <change afterPath="$PROJECT_DIR$/refresher/refresher.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/explorer/explorer.py" beforeDir="false" afterPath="$PROJECT_DIR$/explorer/explorer.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/db_layer/capturas_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturas_interface.py" afterDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
@@ -30,12 +30,12 @@
       </usages-collector>
       <usages-collector id="statistics.file.extensions.open">
         <counts>
-          <entry key="py" value="46" />
+          <entry key="py" value="47" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.types.open">
         <counts>
-          <entry key="Python" value="43" />
+          <entry key="Python" value="44" />
           <entry key="Scratch" value="3" />
         </counts>
       </usages-collector>
@@ -44,7 +44,7 @@
           <entry key="Python Console" value="1579" />
           <entry key="capturer" value="862" />
           <entry key="dummy" value="14" />
-          <entry key="py" value="15563" />
+          <entry key="py" value="16307" />
           <entry key="scratch_1" value="489" />
           <entry key="txt" value="990" />
         </counts>
@@ -52,7 +52,7 @@
       <usages-collector id="statistics.file.types.edit">
         <counts>
           <entry key="PLAIN_TEXT" value="1004" />
-          <entry key="Python" value="18323" />
+          <entry key="Python" value="19067" />
           <entry key="Scratch" value="170" />
         </counts>
       </usages-collector>
@@ -71,18 +71,66 @@
     <splitter split-orientation="horizontal" split-proportion="0.5">
       <split-first>
         <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
-          <file pinned="false" current-in-tab="true">
-            <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
+          <file pinned="false" current-in-tab="false">
+            <entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
+              <provider selected="true" editor-type-id="text-editor" />
+            </entry>
+          </file>
+          <file pinned="false" current-in-tab="false">
+            <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="285">
-                  <caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
+                <state relative-caret-position="210">
+                  <caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
                 </state>
               </provider>
             </entry>
           </file>
           <file pinned="false" current-in-tab="false">
-            <entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
-              <provider selected="true" editor-type-id="text-editor" />
+            <entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="435">
+                  <caret line="29" column="37" lean-forward="true" selection-start-line="29" selection-start-column="37" selection-end-line="29" selection-end-column="37" />
+                  <folding>
+                    <marker date="1539537667973" expanded="true" signature="646:655" ph="..." />
+                    <marker date="1539537667973" expanded="true" signature="646:729" ph="..." />
+                    <marker date="1539537667973" expanded="true" signature="750:759" ph="..." />
+                    <marker date="1539537667973" expanded="true" signature="864:873" ph="..." />
+                  </folding>
+                </state>
+              </provider>
+            </entry>
+          </file>
+          <file pinned="false" current-in-tab="true">
+            <entry file="file://$PROJECT_DIR$/refresher/refresher.py">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="285">
+                  <caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
+                  <folding>
+                    <element signature="e#0#22#0" expanded="true" />
+                    <marker date="1539537667985" expanded="true" signature="166:167" ph="..." />
+                    <marker date="1539537667985" expanded="true" signature="166:172" ph="..." />
+                    <marker date="1539537667985" expanded="true" signature="459:468" ph="..." />
+                  </folding>
+                </state>
+              </provider>
+            </entry>
+          </file>
+          <file pinned="false" current-in-tab="false">
+            <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="45">
+                  <caret line="4" lean-forward="true" selection-start-line="4" selection-end-line="4" />
+                </state>
+              </provider>
+            </entry>
+          </file>
+          <file pinned="false" current-in-tab="false">
+            <entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="18720">
+                  <caret line="1254" selection-start-line="1254" selection-end-line="1254" />
+                </state>
+              </provider>
             </entry>
           </file>
           <file pinned="false" current-in-tab="false">
@@ -101,20 +149,8 @@
           <file pinned="false" current-in-tab="false">
             <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="401">
-                  <caret line="283" selection-start-line="283" selection-end-line="283" />
-                </state>
-              </provider>
-            </entry>
-          </file>
-          <file pinned="false" current-in-tab="true">
-            <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
-              <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="265">
-                  <caret line="66" column="36" lean-forward="true" selection-start-line="66" selection-start-column="36" selection-end-line="66" selection-end-column="36" />
-                  <folding>
-                    <element signature="e#0#11#0" expanded="true" />
-                  </folding>
+                <state relative-caret-position="330">
+                  <caret line="55" column="20" selection-start-line="55" selection-start-column="20" selection-end-line="55" selection-end-column="90" />
                 </state>
               </provider>
             </entry>
@@ -122,7 +158,7 @@
           <file pinned="false" current-in-tab="false">
             <entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="263">
+                <state relative-caret-position="300">
                   <caret line="47" selection-start-line="47" selection-end-line="47" />
                   <folding>
                     <element signature="e#24#46#0" expanded="true" />
@@ -131,14 +167,11 @@
               </provider>
             </entry>
           </file>
-          <file pinned="false" current-in-tab="false">
-            <entry file="file://$PROJECT_DIR$/core/alerts.py">
+          <file pinned="false" current-in-tab="true">
+            <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="105">
-                  <caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
-                  <folding>
-                    <element signature="e#0#46#0" expanded="true" />
-                  </folding>
+                <state relative-caret-position="420">
+                  <caret line="34" lean-forward="true" selection-start-line="34" selection-end-line="34" />
                 </state>
               </provider>
             </entry>
@@ -167,6 +200,9 @@
       <find>re.</find>
     </findStrings>
   </component>
+  <component name="Git.Rebase.Settings">
+    <option name="ONTO" value="refs/remotes/origin/testing" />
+  </component>
   <component name="Git.Settings">
     <option name="UPDATE_TYPE" value="REBASE" />
     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
@@ -192,10 +228,11 @@
         <option value="$PROJECT_DIR$/tests/capturing_tests.py" />
         <option value="$PROJECT_DIR$/tests/capturer_tests.py" />
         <option value="$PROJECT_DIR$/core/mysql_wrapper.py" />
-        <option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
         <option value="$PROJECT_DIR$/explorer/explorer.py" />
-        <option value="$PROJECT_DIR$/capturer/capturer.py" />
         <option value="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" />
+        <option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
+        <option value="$PROJECT_DIR$/capturer/capturer.py" />
+        <option value="$PROJECT_DIR$/refresher/refresher.py" />
       </list>
     </option>
   </component>
@@ -219,7 +256,7 @@
             <path>
               <item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
               <item name="Drogon" type="462c0819:PsiDirectoryNode" />
-              <item name="capturer" type="462c0819:PsiDirectoryNode" />
+              <item name="tests" type="462c0819:PsiDirectoryNode" />
             </path>
             <path>
               <item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
@@ -497,14 +534,14 @@
     <frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
     <editor active="true" />
     <layout>
-      <window_info content_ui="combo" id="Project" order="0" sideWeight="0.48251748" visible="true" weight="0.14918292" />
+      <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.48251748" visible="true" weight="0.14918292" />
       <window_info id="Structure" order="1" sideWeight="0.5174825" side_tool="true" visible="true" weight="0.14918292" />
       <window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
       <window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
       <window_info anchor="bottom" id="Message" order="0" />
       <window_info anchor="bottom" id="Find" order="1" weight="0.32983193" />
       <window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
-      <window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" />
+      <window_info anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" />
       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
       <window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32878152" />
@@ -658,9 +695,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
-      <provider selected="true" editor-type-id="text-editor" />
-    </entry>
     <entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="225">
@@ -736,25 +770,11 @@
     </entry>
     <entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="203">
+        <state relative-caret-position="18720">
           <caret line="1254" selection-start-line="1254" selection-end-line="1254" />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="300">
-          <caret line="20" selection-start-line="20" selection-end-line="20" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="225">
-          <caret line="15" column="18" lean-forward="true" selection-start-line="15" selection-start-column="18" selection-end-line="15" selection-end-column="18" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/_pydev_imps/_pydev_execfile.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="255">
@@ -769,9 +789,33 @@
         </state>
       </provider>
     </entry>
+    <entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
+    <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="45">
+          <caret line="4" lean-forward="true" selection-start-line="4" selection-end-line="4" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="210">
+          <caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="330">
+          <caret line="55" column="20" selection-start-line="55" selection-start-column="20" selection-end-line="55" selection-end-column="90" />
+        </state>
+      </provider>
+    </entry>
     <entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="263">
+        <state relative-caret-position="300">
           <caret line="47" selection-start-line="47" selection-end-line="47" />
           <folding>
             <element signature="e#24#46#0" expanded="true" />
@@ -779,27 +823,36 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
+    <entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="401">
-          <caret line="283" selection-start-line="283" selection-end-line="283" />
+        <state relative-caret-position="435">
+          <caret line="29" column="37" lean-forward="true" selection-start-line="29" selection-start-column="37" selection-end-line="29" selection-end-column="37" />
+          <folding>
+            <marker date="1539537667973" expanded="true" signature="646:655" ph="..." />
+            <marker date="1539537667973" expanded="true" signature="646:729" ph="..." />
+            <marker date="1539537667973" expanded="true" signature="750:759" ph="..." />
+            <marker date="1539537667973" expanded="true" signature="864:873" ph="..." />
+          </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
+    <entry file="file://$PROJECT_DIR$/refresher/refresher.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="265">
-          <caret line="66" column="36" lean-forward="true" selection-start-line="66" selection-start-column="36" selection-end-line="66" selection-end-column="36" />
+        <state relative-caret-position="285">
+          <caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
           <folding>
-            <element signature="e#0#11#0" expanded="true" />
+            <element signature="e#0#22#0" expanded="true" />
+            <marker date="1539537667985" expanded="true" signature="166:167" ph="..." />
+            <marker date="1539537667985" expanded="true" signature="166:172" ph="..." />
+            <marker date="1539537667985" expanded="true" signature="459:468" ph="..." />
           </folding>
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="285">
-          <caret line="19" lean-forward="true" selection-start-line="19" selection-end-line="19" />
+        <state relative-caret-position="420">
+          <caret line="34" lean-forward="true" selection-start-line="34" selection-end-line="34" />
         </state>
       </provider>
     </entry>
diff --git a/capturer/capturer.py b/capturer/capturer.py
index 480f370..4017701 100644
--- a/capturer/capturer.py
+++ b/capturer/capturer.py
@@ -7,7 +7,7 @@ import datetime
 from db_layer.capturing_tasks_interface import capturing_interface
 from db_layer.capturas_interface import capturas_interface
 from core.scrapping_utils import UrlAttack
-
+from refresher.refresher import Refresher
 
 class Capturer:
     sleep_time_no_work = 60
@@ -30,6 +30,7 @@ class Capturer:
             if task.status == 'Data ready':
                 ad_data = task.get_ad_data()
             else:
+                Refresher.dead_ad_checker(task.html)
                 continue
 
             capturas_interface.insert_captura(ad_data)
diff --git a/db_layer/capturas_interface.py b/db_layer/capturas_interface.py
index 45d336a..6fcdb2b 100644
--- a/db_layer/capturas_interface.py
+++ b/db_layer/capturas_interface.py
@@ -20,6 +20,17 @@ class CapturasInterface():
 
         self.anunciosdb.query(query_statement, query_parameters)
 
+    def old_ads_exist(self):
+        #TODO Mira si hay algun anuncio que este viejete y necesite refresco
+
+    def get_old_ad(self):
+        #TODO Recuperar anuncio que este viejete y que necesite refresco
+
+    def mark_dead_ad(self, referencia):
+        #TODO Marca un anuncio como muerto
+
+
+
 
 capturas_interface = CapturasInterface()
 
diff --git a/refresher/refresher.py b/refresher/refresher.py
new file mode 100644
index 0000000..a976adb
--- /dev/null
+++ b/refresher/refresher.py
@@ -0,0 +1,30 @@
+from time import sleep
+from db_layer.capturas_interface import capturas_interface
+from db_layer.capturing_tasks_interface import capturing_interface
+
+class Refresher:
+
+    def start(self):
+
+        while True:
+            sleep(30)
+
+            if capturas_interface.old_ads_exist():
+                old_ad = capturas_interface.get_old_ad()
+                capturing_interface.create_capturing_task(old_ad['referencia'])
+
+
+    def dead_ad_checker(self, html):
+        #TODO Comprueba si el anuncio esta muerto y, si es asi, lo marca en la tabla de capturas
+
+
+        if blabla:
+            capturas_interface.mark_dead_ad(referencia)
+
+
+
+
+
+
+
+