From 7b47b1e76607eaf2dda958ea30a6b0cdecb7c339 Mon Sep 17 00:00:00 2001
From: pablomartincalvo <pablomartincalvo@gmail.com>
Date: Sat, 22 Sep 2018 23:56:01 +0200
Subject: [PATCH] Iniciadas clases de capturing_task y scraptargetfield.

---
 .idea/workspace.xml  | 120 +++++++++++++++++++++++--------------------
 capturer/capturer.py | 100 +++++++++++++++++++++++++++++++++++-
 explorer/explorer.py |   2 +-
 3 files changed, 165 insertions(+), 57 deletions(-)

diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 088cc1c..cc0d79c 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -26,25 +26,25 @@
       </usages-collector>
       <usages-collector id="statistics.file.extensions.open">
         <counts>
-          <entry key="py" value="10" />
+          <entry key="py" value="14" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.types.open">
         <counts>
-          <entry key="Python" value="10" />
+          <entry key="Python" value="14" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.extensions.edit">
         <counts>
           <entry key="Python Console" value="1519" />
-          <entry key="py" value="3070" />
-          <entry key="txt" value="472" />
+          <entry key="py" value="4904" />
+          <entry key="txt" value="569" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.file.types.edit">
         <counts>
-          <entry key="PLAIN_TEXT" value="472" />
-          <entry key="Python" value="4589" />
+          <entry key="PLAIN_TEXT" value="569" />
+          <entry key="Python" value="6423" />
         </counts>
       </usages-collector>
       <usages-collector id="statistics.vcs.git.usages">
@@ -61,34 +61,33 @@
     <splitter split-orientation="horizontal" split-proportion="0.5">
       <split-first>
         <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
-          <file pinned="false" current-in-tab="false">
-            <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
+          <file pinned="false" current-in-tab="true">
+            <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="2117">
-                  <caret line="176" column="18" lean-forward="true" selection-start-line="176" selection-start-column="18" selection-end-line="176" selection-end-column="18" />
+                <state relative-caret-position="285">
+                  <caret line="102" column="32" lean-forward="true" selection-start-line="102" selection-start-column="32" selection-end-line="102" selection-end-column="32" />
                   <folding>
-                    <marker date="1537650937299" expanded="true" signature="5092:5544" ph="..." />
+                    <element signature="e#1322#1377#0" />
+                    <element signature="e#1418#1920#0" />
+                    <marker date="1537653289723" expanded="true" signature="1082:1083" ph="..." />
+                    <marker date="1537653289723" expanded="true" signature="2990:3080" ph="..." />
                   </folding>
                 </state>
               </provider>
             </entry>
           </file>
+        </leaf>
+      </split-first>
+      <split-second>
+        <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
           <file pinned="false" current-in-tab="true">
-            <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
+            <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="315">
-                  <caret line="21" lean-forward="true" selection-start-line="21" selection-end-line="21" />
-                </state>
-              </provider>
-            </entry>
-          </file>
-          <file pinned="false" current-in-tab="false">
-            <entry file="file://$PROJECT_DIR$/core/alerts.py">
-              <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="75">
-                  <caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
+                <state relative-caret-position="260">
+                  <caret line="52" column="29" selection-start-line="52" selection-start-column="29" selection-end-line="52" selection-end-column="29" />
                   <folding>
-                    <element signature="e#0#46#0" expanded="true" />
+                    <element signature="e#6100#9639#0" />
+                    <marker date="1537653289735" expanded="true" signature="5088:5540" ph="..." />
                   </folding>
                 </state>
               </provider>
@@ -103,17 +102,13 @@
               </provider>
             </entry>
           </file>
-        </leaf>
-      </split-first>
-      <split-second>
-        <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
-          <file pinned="false" current-in-tab="true">
-            <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
+          <file pinned="false" current-in-tab="false">
+            <entry file="file://$PROJECT_DIR$/core/alerts.py">
               <provider selected="true" editor-type-id="text-editor">
-                <state relative-caret-position="317">
-                  <caret line="273" column="44" selection-start-line="273" selection-start-column="44" selection-end-line="273" selection-end-column="44" />
+                <state relative-caret-position="105">
+                  <caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
                   <folding>
-                    <marker date="1537650937299" expanded="true" signature="5092:5544" ph="..." />
+                    <element signature="e#0#46#0" expanded="true" />
                   </folding>
                 </state>
               </provider>
@@ -157,8 +152,8 @@
         <option value="$PROJECT_DIR$/core/alerts.py" />
         <option value="$PROJECT_DIR$/core/task.py" />
         <option value="$PROJECT_DIR$/capturer/__init__.py" />
-        <option value="$PROJECT_DIR$/explorer/explorer.py" />
         <option value="$PROJECT_DIR$/capturer/capturer.py" />
+        <option value="$PROJECT_DIR$/explorer/explorer.py" />
       </list>
     </option>
   </component>
@@ -281,7 +276,14 @@
       <option name="project" value="LOCAL" />
       <updated>1537546774036</updated>
     </task>
-    <option name="localTasksCounter" value="4" />
+    <task id="LOCAL-00004" summary="Corregidos pequeños errores y type tras primer test del servicio.">
+      <created>1537651070019</created>
+      <option name="number" value="00004" />
+      <option name="presentableId" value="LOCAL-00004" />
+      <option name="project" value="LOCAL" />
+      <updated>1537651070019</updated>
+    </task>
+    <option name="localTasksCounter" value="5" />
     <servers />
   </component>
   <component name="TodoView" selected-index="1">
@@ -297,17 +299,17 @@
     <frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
     <editor active="true" />
     <layout>
-      <window_info content_ui="combo" id="Project" order="0" sideWeight="0.4905956" weight="0.14918292" />
-      <window_info id="Structure" order="1" sideWeight="0.50940436" side_tool="true" visible="true" weight="0.14918292" />
+      <window_info content_ui="combo" id="Project" order="0" sideWeight="0.48982784" visible="true" weight="0.14918292" />
+      <window_info id="Structure" order="1" sideWeight="0.5101721" side_tool="true" visible="true" weight="0.14918292" />
       <window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
       <window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
       <window_info anchor="bottom" id="Message" order="0" />
-      <window_info active="true" anchor="bottom" id="Find" order="1" visible="true" weight="0.32983193" />
+      <window_info anchor="bottom" id="Find" order="1" weight="0.32983193" />
       <window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
       <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
-      <window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32914045" />
+      <window_info active="true" anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" visible="true" weight="0.32878152" />
       <window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.49973643" weight="0.269958" />
       <window_info anchor="bottom" id="Terminal" order="8" weight="0.32983193" />
       <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.5007907" side_tool="true" weight="0.32983193" />
@@ -351,7 +353,8 @@
     <MESSAGE value="Finalizado modulo de alertas. Testeado clase ExploringTask a fondo." />
     <MESSAGE value="Pequeños detalles en Explorer." />
     <MESSAGE value="Refactorizado Explorer para que el Explorer, y no el exploring_task, se encargue de postear las tareas de captura.&#10;&#10;Creado una funcion independiente de creacion de capturas para que sea compartida entre todos aquellos servicios que las creen." />
-    <option name="LAST_COMMIT_MESSAGE" value="Refactorizado Explorer para que el Explorer, y no el exploring_task, se encargue de postear las tareas de captura.&#10;&#10;Creado una funcion independiente de creacion de capturas para que sea compartida entre todos aquellos servicios que las creen." />
+    <MESSAGE value="Corregidos pequeños errores y type tras primer test del servicio." />
+    <option name="LAST_COMMIT_MESSAGE" value="Corregidos pequeños errores y type tras primer test del servicio." />
   </component>
   <component name="editorHistoryManager">
     <entry file="file://$PROJECT_DIR$/explorer/test_explorer.py" />
@@ -373,16 +376,6 @@
     <entry file="file://$PROJECT_DIR$/capturer/__init__.py">
       <provider selected="true" editor-type-id="text-editor" />
     </entry>
-    <entry file="file://$PROJECT_DIR$/core/alerts.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="75">
-          <caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
-          <folding>
-            <element signature="e#0#46#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="540">
@@ -390,20 +383,37 @@
         </state>
       </provider>
     </entry>
+    <entry file="file://$PROJECT_DIR$/core/alerts.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="105">
+          <caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
+          <folding>
+            <element signature="e#0#46#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
     <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="2117">
-          <caret line="176" column="18" lean-forward="true" selection-start-line="176" selection-start-column="18" selection-end-line="176" selection-end-column="18" />
+        <state relative-caret-position="260">
+          <caret line="52" column="29" selection-start-line="52" selection-start-column="29" selection-end-line="52" selection-end-column="29" />
           <folding>
-            <marker date="1537650937299" expanded="true" signature="5092:5544" ph="..." />
+            <element signature="e#6100#9639#0" />
+            <marker date="1537653289735" expanded="true" signature="5088:5540" ph="..." />
           </folding>
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="315">
-          <caret line="21" lean-forward="true" selection-start-line="21" selection-end-line="21" />
+        <state relative-caret-position="285">
+          <caret line="102" column="32" lean-forward="true" selection-start-line="102" selection-start-column="32" selection-end-line="102" selection-end-column="32" />
+          <folding>
+            <element signature="e#1322#1377#0" />
+            <element signature="e#1418#1920#0" />
+            <marker date="1537653289723" expanded="true" signature="1082:1083" ph="..." />
+            <marker date="1537653289723" expanded="true" signature="2990:3080" ph="..." />
+          </folding>
         </state>
       </provider>
     </entry>
diff --git a/capturer/capturer.py b/capturer/capturer.py
index 01f614a..a2f126d 100644
--- a/capturer/capturer.py
+++ b/capturer/capturer.py
@@ -1,7 +1,13 @@
+import sys
+sys.path.append('..')
 import uuid
+from core.mysql_wrapper import get_anunciosdb, get_tasksdb
+from core.scrapping_utils import UrlAttack
 
 ads_root = 'https://www.idealista.com/inmueble/'
 
+#TODO Crear la lista de campos
+
 
 def create_capturing_task(referencia, db_wrapper, uuid_exploring=None):
 
@@ -19,6 +25,98 @@ def create_capturing_task(referencia, db_wrapper, uuid_exploring=None):
                             (uuid,  write_time, status, url, fk_uuid_exploring)
                             VALUES (%(uuid)s, NOW(), %(status)s, %(ad_url)s, %(uuid_exploring)s)"""
 
-
     db_wrapper.query(query_statement, query_parameters)
 
+class CapturingTask:
+
+    def __init__(self, parameters):
+        self.uuid = parameters['uuid']
+        self.status = 'Loading'
+        self.ad_url = parameters['ad_url']
+        self.uuid_exploring = parameters['uuid_exploring']
+
+        self.tasksdb = get_tasksdb()
+
+        self._log_in_tasksdb()
+
+    def _update_status(self, new_status):
+        self.status = new_status
+        self._log_in_tasksdb()
+
+    def _log_in_tasksdb(self):
+        """
+        Graba en la base de datos de tareas un registro con el UUID de la tarea,
+        un timestamp y el status
+        """
+
+        query_statement = """INSERT INTO capturing_tasks_logs
+                            (uuid, write_time, status, ad_url, fk_uuid_exploring)
+                            VALUES (%(uuid)s, NOW(), %(status)s, %(ad_url)s, %(fk_uuid_exploring)s)"""
+
+        query_parameters = {'uuid': self.uuid,
+                            'status': self.status,
+                            'ad_url': self.ad_url,
+                            'fk_uuid_exploring': self.uuid_exploring}
+
+        self.tasksdb.query(query_statement, query_parameters)
+
+    def capture(self):
+        """
+        Metodo principal que contiene el flujo de captura
+        """
+        #TODO Desarrollar flujo de captura
+
+
+    def _html_is_valid(selfself, html=self.html):
+        """
+        Lee el HTML y aplica normas de validación del contenido
+        """
+
+        #TODO Comprobar si HTML es pagina de bloqueo
+
+
+        #TODO Check de longitud
+        pass
+
+    def _fields_not_present(self, field_list, html=self.html):
+        """
+        Lee el HTML y devuelve los campos que no esten presentes
+        """
+        fields_not_present = []
+        for field_parameters in field_list:
+            field = ScrapTargetField(field_parameters)
+        if not field.exists(html):
+            fields_not_present.append(field.name)
+
+        return fields_not_present
+
+
+class ScrapTargetField:
+
+    def __init__(self, target_parameters):
+        self.name = target_parameters['name']
+        self.search_method = target_parameters['search_method']
+        self.validation_method = target_parameters['validation_method']
+
+    def exists(self, html):
+        """
+        Busca el dato en un HTML
+        """
+        if self.search_method(html) is None:
+            return False
+        else:
+            return True
+
+    def validate_value(self, dato):
+        """
+        Comprueba el valor y valida con la norma respectiva que sea lo esperado
+        """
+        return self.validation_method(dato)
+
+    def get_value(self, html):
+        """
+        Busca en un HTML el dato
+        """
+        return self.search_method(html)
+
+
diff --git a/explorer/explorer.py b/explorer/explorer.py
index 414419d..1e2559b 100644
--- a/explorer/explorer.py
+++ b/explorer/explorer.py
@@ -43,7 +43,7 @@ class Explorer():
             
             if not self.database_is_up():
                 alert_master("SQL DOWN", "El explorer informa de que SQL esta caida. Actividad detenida")
-                self.stop(self)
+                self.stop()
 
             current_task = ExploringTask(self.compose_listing_url())
             current_task.explore()