From 4edff2e0c1a0d549ba004ed2781d493d81b827a0 Mon Sep 17 00:00:00 2001
From: pablomartincalvo <pablomartincalvo@gmail.com>
Date: Sat, 13 Oct 2018 17:45:42 +0200
Subject: [PATCH] Siguen los retoques. Tests locales exitosos, listo para mover
 a entorno de pruebas y dejar corriendo junto a explorer.

---
 .idea/workspace.xml                   | 212 ++++++++++++++++++--------
 capturer/capturer.py                  |   8 +-
 core/mysql_wrapper.py                 |   3 +-
 db_layer/capturas_interface.py        |   9 +-
 db_layer/capturing_tasks_interface.py |  14 +-
 requirements.txt                      |   2 +
 tests/capturer_tests.py               |  10 +-
 7 files changed, 177 insertions(+), 81 deletions(-)
 create mode 100644 requirements.txt

diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index bb6d346..dddca3f 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -2,13 +2,13 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
-      <change afterPath="$PROJECT_DIR$/tests/capturer_tests.py" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/requirements.txt" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/core/mysql_wrapper.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/mysql_wrapper.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/mysql/Dockerfile" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/Dockerfile" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/mysql/capturas_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturas_interface.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/mysql/capturing_tasks_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/db_layer/capturas_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturas_interface.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/tests/capturer_tests.py" beforeDir="false" afterPath="$PROJECT_DIR$/tests/capturer_tests.py" afterDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
@@ -33,29 +33,29 @@
       </usages-collector>
       <usages-collector id="statistics.file.extensions.open">
         <counts>
-          <entry key="py" value="39" />
+          <entry key="py" value="43" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.types.open">
         <counts>
-          <entry key="Python" value="36" />
+          <entry key="Python" value="40" />
           <entry key="Scratch" value="3" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.extensions.edit">
         <counts>
           <entry key="Python Console" value="1555" />
-          <entry key="capturer" value="843" />
+          <entry key="capturer" value="862" />
           <entry key="dummy" value="14" />
-          <entry key="py" value="14798" />
+          <entry key="py" value="15119" />
           <entry key="scratch_1" value="489" />
-          <entry key="txt" value="880" />
+          <entry key="txt" value="948" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.types.edit">
         <counts>
-          <entry key="PLAIN_TEXT" value="894" />
-          <entry key="Python" value="17515" />
+          <entry key="PLAIN_TEXT" value="962" />
+          <entry key="Python" value="17855" />
           <entry key="Scratch" value="170" />
         </counts>
       </usages-collector>
@@ -82,10 +82,11 @@
           <file pinned="false" current-in-tab="true">
             <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="270">
-                  <caret line="18" lean-forward="true" selection-start-line="18" selection-end-line="18" />
+                <state relative-caret-position="225">
+                  <caret line="15" column="18" lean-forward="true" selection-start-line="15" selection-start-column="18" selection-end-line="15" selection-end-column="18" />
                   <folding>
-                    <marker date="1539387124485" expanded="true" signature="129:134" ph="..." />
+                    <marker date="1539438542796" expanded="true" signature="139:144" ph="..." />
+                    <marker date="1539438542796" expanded="true" signature="509:514" ph="..." />
                   </folding>
                 </state>
               </provider>
@@ -94,8 +95,8 @@
           <file pinned="false" current-in-tab="false">
             <entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="405">
-                  <caret line="27" selection-start-line="27" selection-end-line="27" />
+                <state relative-caret-position="300">
+                  <caret line="20" selection-start-line="20" selection-end-line="20" />
                 </state>
               </provider>
             </entry>
@@ -103,8 +104,17 @@
           <file pinned="false" current-in-tab="false">
             <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="360">
-                  <caret line="24" lean-forward="true" selection-start-line="24" selection-end-line="24" />
+                <state relative-caret-position="930">
+                  <caret line="62" column="15" selection-start-line="62" selection-start-column="15" selection-end-line="62" selection-end-column="15" />
+                </state>
+              </provider>
+            </entry>
+          </file>
+          <file pinned="false" current-in-tab="false">
+            <entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
+              <provider selected="true" editor-type-id="text-editor">
+                <state relative-caret-position="203">
+                  <caret line="1254" selection-start-line="1254" selection-end-line="1254" />
                 </state>
               </provider>
             </entry>
@@ -143,8 +153,8 @@
           <file pinned="false" current-in-tab="false">
             <entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="435">
-                  <caret line="56" column="55" lean-forward="true" selection-start-line="56" selection-start-column="55" selection-end-line="56" selection-end-column="55" />
+                <state relative-caret-position="203">
+                  <caret line="47" selection-start-line="47" selection-end-line="47" />
                   <folding>
                     <element signature="e#24#46#0" expanded="true" />
                   </folding>
@@ -152,7 +162,7 @@
               </provider>
             </entry>
           </file>
-          <file pinned="false" current-in-tab="true">
+          <file pinned="false" current-in-tab="false">
             <entry file="file://$PROJECT_DIR$/core/alerts.py">
               <provider selected="true" editor-type-id="text-editor">
                 <state relative-caret-position="390">
@@ -164,11 +174,11 @@
               </provider>
             </entry>
           </file>
-          <file pinned="false" current-in-tab="false">
+          <file pinned="false" current-in-tab="true">
             <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="173">
-                  <caret line="166" selection-start-line="166" selection-end-line="166" />
+                <state relative-caret-position="300">
+                  <caret line="20" selection-start-line="20" selection-end-line="20" />
                 </state>
               </provider>
             </entry>
@@ -222,9 +232,10 @@
         <option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py" />
         <option value="$PROJECT_DIR$/tests/capturing_tests.py" />
         <option value="$PROJECT_DIR$/tests/capturer_tests.py" />
-        <option value="$PROJECT_DIR$/capturer/capturer.py" />
         <option value="$PROJECT_DIR$/core/mysql_wrapper.py" />
+        <option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
         <option value="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" />
+        <option value="$PROJECT_DIR$/capturer/capturer.py" />
       </list>
     </option>
   </component>
@@ -255,6 +266,11 @@
               <item name="Drogon" type="462c0819:PsiDirectoryNode" />
               <item name="core" type="462c0819:PsiDirectoryNode" />
             </path>
+            <path>
+              <item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
+              <item name="Drogon" type="462c0819:PsiDirectoryNode" />
+              <item name="db_layer" type="462c0819:PsiDirectoryNode" />
+            </path>
             <path>
               <item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
               <item name="Drogon" type="462c0819:PsiDirectoryNode" />
@@ -467,7 +483,14 @@
       <option name="project" value="LOCAL" />
       <updated>1538514864935</updated>
     </task>
-    <option name="localTasksCounter" value="9" />
+    <task id="LOCAL-00009" summary="Testeos en desarrollo del sistema de capturas. Pequeños retoques.">
+      <created>1539389338572</created>
+      <option name="number" value="00009" />
+      <option name="presentableId" value="LOCAL-00009" />
+      <option name="project" value="LOCAL" />
+      <updated>1539389338572</updated>
+    </task>
+    <option name="localTasksCounter" value="10" />
     <servers />
   </component>
   <component name="TodoView" selected-index="1">
@@ -483,14 +506,14 @@
     <frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
     <editor active="true" />
     <layout>
-      <window_info content_ui="combo" id="Project" order="0" sideWeight="0.48251748" visible="true" weight="0.14918292" />
+      <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.48251748" visible="true" weight="0.14918292" />
       <window_info id="Structure" order="1" sideWeight="0.5174825" side_tool="true" visible="true" weight="0.14918292" />
       <window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
       <window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
       <window_info anchor="bottom" id="Message" order="0" />
       <window_info anchor="bottom" id="Find" order="1" weight="0.32983193" />
       <window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
-      <window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" />
+      <window_info anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" />
       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
       <window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32878152" />
@@ -542,7 +565,41 @@
     <MESSAGE value="Avanzado en desarrollo de capturing task." />
     <MESSAGE value="Avanzado en desarrollo de capturing task. Creado clase GeocodingTask" />
     <MESSAGE value="Creado cache de Geocoding. Avanzado en Geocoding Task. Decido mover parte de la gestion del geocoding al capturer." />
-    <option name="LAST_COMMIT_MESSAGE" value="Creado cache de Geocoding. Avanzado en Geocoding Task. Decido mover parte de la gestion del geocoding al capturer." />
+    <MESSAGE value="Testeos en desarrollo del sistema de capturas. Pequeños retoques." />
+    <option name="LAST_COMMIT_MESSAGE" value="Testeos en desarrollo del sistema de capturas. Pequeños retoques." />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/capturer/capturer.py</url>
+          <line>16</line>
+          <option name="timeStamp" value="6" />
+        </line-breakpoint>
+      </breakpoints>
+      <breakpoints-dialog>
+        <breakpoints-dialog />
+      </breakpoints-dialog>
+    </breakpoint-manager>
+  </component>
+  <component name="debuggerHistoryManager">
+    <expressions id="evaluateExpression">
+      <expression>
+        <expression-string>capturing_interface.seconds_since_last_try()</expression-string>
+        <language-id>Python</language-id>
+        <evaluation-mode>EXPRESSION</evaluation-mode>
+      </expression>
+      <expression>
+        <expression-string>cursor.fetchone()</expression-string>
+        <language-id>Python</language-id>
+        <evaluation-mode>EXPRESSION</evaluation-mode>
+      </expression>
+      <expression>
+        <expression-string>cursor.fetchone()[0]</expression-string>
+        <language-id>Python</language-id>
+        <evaluation-mode>EXPRESSION</evaluation-mode>
+      </expression>
+    </expressions>
   </component>
   <component name="editorHistoryManager">
     <entry file="file://$PROJECT_DIR$/explorer/test_explorer.py" />
@@ -588,16 +645,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="435">
-          <caret line="56" column="55" lean-forward="true" selection-start-line="56" selection-start-column="55" selection-end-line="56" selection-end-column="55" />
-          <folding>
-            <element signature="e#24#46#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file:///usr/lib/python3.6/socket.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="421">
@@ -605,20 +652,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="405">
-          <caret line="27" selection-start-line="27" selection-end-line="27" />
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="360">
-          <caret line="24" lean-forward="true" selection-start-line="24" selection-end-line="24" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file:///usr/lib/python3.6/_collections_abc.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="91">
@@ -626,13 +659,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="173">
-          <caret line="166" selection-start-line="166" selection-end-line="166" />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$USER_HOME$/.local/lib/python3.6/site-packages/bs4/__init__.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="361">
@@ -664,12 +690,72 @@
         </state>
       </provider>
     </entry>
+    <entry file="file:///usr/lib/python3/dist-packages/html5lib/html5parser.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="496">
+          <caret line="423" selection-start-line="423" selection-end-line="423" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/_pydevd_bundle/pydevd_breakpoints.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="218">
+          <caret line="103" column="34" selection-start-line="103" selection-start-column="34" selection-end-line="103" selection-end-column="34" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file:///usr/lib/python3/dist-packages/apport_python_hook.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="211">
+          <caret line="48" selection-start-line="48" selection-end-line="48" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="203">
+          <caret line="1254" selection-start-line="1254" selection-end-line="1254" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="203">
+          <caret line="47" selection-start-line="47" selection-end-line="47" />
+          <folding>
+            <element signature="e#24#46#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="300">
+          <caret line="20" selection-start-line="20" selection-end-line="20" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="930">
+          <caret line="62" column="15" selection-start-line="62" selection-start-column="15" selection-end-line="62" selection-end-column="15" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="300">
+          <caret line="20" selection-start-line="20" selection-end-line="20" />
+        </state>
+      </provider>
+    </entry>
     <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="270">
-          <caret line="18" lean-forward="true" selection-start-line="18" selection-end-line="18" />
+        <state relative-caret-position="225">
+          <caret line="15" column="18" lean-forward="true" selection-start-line="15" selection-start-column="18" selection-end-line="15" selection-end-column="18" />
           <folding>
-            <marker date="1539387124485" expanded="true" signature="129:134" ph="..." />
+            <marker date="1539438542796" expanded="true" signature="139:144" ph="..." />
+            <marker date="1539438542796" expanded="true" signature="509:514" ph="..." />
           </folding>
         </state>
       </provider>
diff --git a/capturer/capturer.py b/capturer/capturer.py
index 870d41b..52a03f4 100644
--- a/capturer/capturer.py
+++ b/capturer/capturer.py
@@ -17,7 +17,7 @@ class Capturer:
         while True:
 
             if (capturing_interface.get_pending_task() is None
-                    and capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries):
+                    or capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries):
                 sleep(Capturer.sleep_time_no_work)
                 continue
 
@@ -30,6 +30,7 @@ class Capturer:
                 continue
 
             capturas_interface.insert_captura(ad_data)
+            task._update_status('Captura inserted')
 
 
 class CapturingTask:
@@ -64,6 +65,7 @@ class CapturingTask:
 
                 self._extract_data()
                 self._check_data()
+                return
 
             else:
                 self.request_failures += 1
@@ -82,7 +84,7 @@ class CapturingTask:
             self._update_status('Invalid value fields')
             return
 
-        if not self.parser.fields_missing():
+        if self.parser.fields_missing():
             self._update_status('Fields missing')
             return
 
@@ -107,7 +109,7 @@ class AdHtmlParser:
                 'value': None},
             'tamano_categorico': {
                 'found': False,
-                'optional': False,
+                'optional': True,
                 'value': None},
             'm2': {
                 'found': False,
diff --git a/core/mysql_wrapper.py b/core/mysql_wrapper.py
index f6945ae..9beeda7 100644
--- a/core/mysql_wrapper.py
+++ b/core/mysql_wrapper.py
@@ -54,8 +54,7 @@ class DatabaseWrapper():
                                              {}
                                           """.format(query_statement,
                                                      query_parameters,
-                                                     e)
-                                                     )
+                                                     e))
 
         else:
             raise Exception("Could not connect to the database.")
diff --git a/db_layer/capturas_interface.py b/db_layer/capturas_interface.py
index de4b98f..45d336a 100644
--- a/db_layer/capturas_interface.py
+++ b/db_layer/capturas_interface.py
@@ -10,13 +10,13 @@ class CapturasInterface():
     def insert_captura(self, ad_data):
 
         columns = ', '.join(ad_data.keys())
-        placeholders_string = ', '.join('%s' * len(ad_data))
+        placeholders_string = ', '.join(['%s'] * len(ad_data))
 
         query_statement = """ INSERT INTO capturas
-                              (%s)
-                              VALUES(%s)""".format(columns, placeholders_string)
+                              ( fecha_captura, {} )
+                              VALUES( NOW(), {} )""".format(columns, placeholders_string)
 
-        query_parameters = ad_data.values()
+        query_parameters = tuple([v for v in ad_data.values()])
 
         self.anunciosdb.query(query_statement, query_parameters)
 
@@ -24,4 +24,3 @@ class CapturasInterface():
 capturas_interface = CapturasInterface()
 
 
-
diff --git a/db_layer/capturing_tasks_interface.py b/db_layer/capturing_tasks_interface.py
index b77e18c..34d91be 100644
--- a/db_layer/capturing_tasks_interface.py
+++ b/db_layer/capturing_tasks_interface.py
@@ -59,16 +59,18 @@ class CapturingTasksInterface:
         self.tasksdb.query(query_statement, query_parameters)
 
     def seconds_since_last_try(self):
-        query_statement = """SELECT TIME_TO_SEC(TIME_DIFF(now(), write_time))
-                             FROM capturing_task_logs
-                             WHERE status = 'Pending'
+        query_statement = """SELECT TIME_TO_SEC(TIMEDIFF(now(), write_time))
+                             FROM capturing_tasks_logs
+                             WHERE status = 'Loading'
                              ORDER BY write_time
                              LIMIT 1
                             """
 
-        cursor = self.taskdb.query(query_statement)
-
-        return cursor.fetchone()[0]
+        cursor = self.tasksdb.query(query_statement)
 
+        try:
+            return cursor.fetchone()[0]
+        except:
+            return 999
 
 capturing_interface = CapturingTasksInterface()
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..f3afe7a
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,2 @@
+bs4 == 4.6.3
+mysql-connector = 2.1.6
diff --git a/tests/capturer_tests.py b/tests/capturer_tests.py
index 8c23f37..3380123 100644
--- a/tests/capturer_tests.py
+++ b/tests/capturer_tests.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import sys
 sys.path.append('..')
-from capturer.capturer import CapturingTask
+from capturer.capturer import CapturingTask, Capturer
 
 
 
@@ -17,5 +17,11 @@ def test_CapturingTask():
     print(task.get_ad_data())
 
 
+def test_Capturer():
+    capturer = Capturer()
+    capturer.start()
 
-test_CapturingTask()
\ No newline at end of file
+
+#  test_CapturingTask()
+
+test_Capturer()
\ No newline at end of file