counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Siguen los retoques. Tests locales exitosos, listo para mover a entorno

Browse source 
de pruebas y dejar corriendo junto a explorer.
This commit is contained in:
pablomartincalvo 2018-10-13 17:45:42 +02:00
parent 6a0baf4de6
commit 4edff2e0c1
7 changed files with 177 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

212
.idea/workspace.xml generated
View file

@ -2,13 +2,13 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment=""> <list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/tests/capturer_tests.py" afterDir="false" /> <change afterPath="$PROJECT_DIR$/requirements.txt" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/core/mysql_wrapper.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/mysql_wrapper.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/core/mysql_wrapper.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/mysql_wrapper.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/mysql/Dockerfile" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/Dockerfile" afterDir="false" /> <change beforePath="$PROJECT_DIR$/db_layer/capturas_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturas_interface.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/mysql/capturas_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturas_interface.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/mysql/capturing_tasks_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/tests/capturer_tests.py" beforeDir="false" afterPath="$PROJECT_DIR$/tests/capturer_tests.py" afterDir="false" />
</list> </list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
@ -33,29 +33,29 @@
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.extensions.open"> <usages-collector id="statistics.file.extensions.open">
<counts> <counts>
<entry key="py" value="39" /> <entry key="py" value="43" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.open"> <usages-collector id="statistics.file.types.open">
<counts> <counts>
<entry key="Python" value="36" /> <entry key="Python" value="40" />
<entry key="Scratch" value="3" /> <entry key="Scratch" value="3" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.extensions.edit"> <usages-collector id="statistics.file.extensions.edit">
<counts> <counts>
<entry key="Python Console" value="1555" /> <entry key="Python Console" value="1555" />
<entry key="capturer" value="843" /> <entry key="capturer" value="862" />
<entry key="dummy" value="14" /> <entry key="dummy" value="14" />
<entry key="py" value="14798" /> <entry key="py" value="15119" />
<entry key="scratch_1" value="489" /> <entry key="scratch_1" value="489" />
<entry key="txt" value="880" /> <entry key="txt" value="948" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.edit"> <usages-collector id="statistics.file.types.edit">
<counts> <counts>
<entry key="PLAIN_TEXT" value="894" /> <entry key="PLAIN_TEXT" value="962" />
<entry key="Python" value="17515" /> <entry key="Python" value="17855" />
<entry key="Scratch" value="170" /> <entry key="Scratch" value="170" />
</counts> </counts>
</usages-collector> </usages-collector>
@ -82,10 +82,11 @@
<file pinned="false" current-in-tab="true"> <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py"> <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="270"> <state relative-caret-position="225">
<caret line="18" lean-forward="true" selection-start-line="18" selection-end-line="18" /> <caret line="15" column="18" lean-forward="true" selection-start-line="15" selection-start-column="18" selection-end-line="15" selection-end-column="18" />
<folding> <folding>
<marker date="1539387124485" expanded="true" signature="129:134" ph="..." /> <marker date="1539438542796" expanded="true" signature="139:144" ph="..." />
<marker date="1539438542796" expanded="true" signature="509:514" ph="..." />
</folding> </folding>
</state> </state>
</provider> </provider>
@ -94,8 +95,8 @@
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py"> <entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="405"> <state relative-caret-position="300">
<caret line="27" selection-start-line="27" selection-end-line="27" /> <caret line="20" selection-start-line="20" selection-end-line="20" />
</state> </state>
</provider> </provider>
</entry> </entry>
@ -103,8 +104,17 @@
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py"> <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="360"> <state relative-caret-position="930">
<caret line="24" lean-forward="true" selection-start-line="24" selection-end-line="24" /> <caret line="62" column="15" selection-start-line="62" selection-start-column="15" selection-end-line="62" selection-end-column="15" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="203">
<caret line="1254" selection-start-line="1254" selection-end-line="1254" />
</state> </state>
</provider> </provider>
</entry> </entry>
@ -143,8 +153,8 @@
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py"> <entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="435"> <state relative-caret-position="203">
<caret line="56" column="55" lean-forward="true" selection-start-line="56" selection-start-column="55" selection-end-line="56" selection-end-column="55" /> <caret line="47" selection-start-line="47" selection-end-line="47" />
<folding> <folding>
<element signature="e#24#46#0" expanded="true" /> <element signature="e#24#46#0" expanded="true" />
</folding> </folding>
@ -152,7 +162,7 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="true"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/alerts.py"> <entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="390"> <state relative-caret-position="390">
@ -164,11 +174,11 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/capturer/capturer.py"> <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="173"> <state relative-caret-position="300">
<caret line="166" selection-start-line="166" selection-end-line="166" /> <caret line="20" selection-start-line="20" selection-end-line="20" />
</state> </state>
</provider> </provider>
</entry> </entry>
@ -222,9 +232,10 @@
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py" /> <option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py" />
<option value="$PROJECT_DIR$/tests/capturing_tests.py" /> <option value="$PROJECT_DIR$/tests/capturing_tests.py" />
<option value="$PROJECT_DIR$/tests/capturer_tests.py" /> <option value="$PROJECT_DIR$/tests/capturer_tests.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" />
<option value="$PROJECT_DIR$/core/mysql_wrapper.py" /> <option value="$PROJECT_DIR$/core/mysql_wrapper.py" />
<option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
<option value="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" /> <option value="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" />
</list> </list>
</option> </option>
</component> </component>
@ -255,6 +266,11 @@
<item name="Drogon" type="462c0819:PsiDirectoryNode" /> <item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="core" type="462c0819:PsiDirectoryNode" /> <item name="core" type="462c0819:PsiDirectoryNode" />
</path> </path>
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="db_layer" type="462c0819:PsiDirectoryNode" />
</path>
<path> <path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" /> <item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" /> <item name="Drogon" type="462c0819:PsiDirectoryNode" />
@ -467,7 +483,14 @@
<option name="project" value="LOCAL" /> <option name="project" value="LOCAL" />
<updated>1538514864935</updated> <updated>1538514864935</updated>
</task> </task>
<option name="localTasksCounter" value="9" /> <task id="LOCAL-00009" summary="Testeos en desarrollo del sistema de capturas. Pequeños retoques.">
<created>1539389338572</created>
<option name="number" value="00009" />
<option name="presentableId" value="LOCAL-00009" />
<option name="project" value="LOCAL" />
<updated>1539389338572</updated>
</task>
<option name="localTasksCounter" value="10" />
<servers /> <servers />
</component> </component>
<component name="TodoView" selected-index="1"> <component name="TodoView" selected-index="1">
@ -483,14 +506,14 @@
<frame x="0" y="-2" width="1920" height="1082" extended-state="6" /> <frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
<editor active="true" /> <editor active="true" />
<layout> <layout>
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.48251748" visible="true" weight="0.14918292" /> <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.48251748" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" sideWeight="0.5174825" side_tool="true" visible="true" weight="0.14918292" /> <window_info id="Structure" order="1" sideWeight="0.5174825" side_tool="true" visible="true" weight="0.14918292" />
<window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" /> <window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
<window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" /> <window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
<window_info anchor="bottom" id="Message" order="0" /> <window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.32983193" /> <window_info anchor="bottom" id="Find" order="1" weight="0.32983193" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32983193" /> <window_info anchor="bottom" id="Run" order="2" weight="0.32983193" />
<window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" /> <window_info anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" /> <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" /> <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32878152" /> <window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32878152" />
@ -542,7 +565,41 @@
<MESSAGE value="Avanzado en desarrollo de capturing task." /> <MESSAGE value="Avanzado en desarrollo de capturing task." />
<MESSAGE value="Avanzado en desarrollo de capturing task. Creado clase GeocodingTask" /> <MESSAGE value="Avanzado en desarrollo de capturing task. Creado clase GeocodingTask" />
<MESSAGE value="Creado cache de Geocoding. Avanzado en Geocoding Task. Decido mover parte de la gestion del geocoding al capturer." /> <MESSAGE value="Creado cache de Geocoding. Avanzado en Geocoding Task. Decido mover parte de la gestion del geocoding al capturer." />
<option name="LAST_COMMIT_MESSAGE" value="Creado cache de Geocoding. Avanzado en Geocoding Task. Decido mover parte de la gestion del geocoding al capturer." /> <MESSAGE value="Testeos en desarrollo del sistema de capturas. Pequeños retoques." />
<option name="LAST_COMMIT_MESSAGE" value="Testeos en desarrollo del sistema de capturas. Pequeños retoques." />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints>
<line-breakpoint suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/capturer/capturer.py</url>
<line>16</line>
<option name="timeStamp" value="6" />
</line-breakpoint>
</breakpoints>
<breakpoints-dialog>
<breakpoints-dialog />
</breakpoints-dialog>
</breakpoint-manager>
</component>
<component name="debuggerHistoryManager">
<expressions id="evaluateExpression">
<expression>
<expression-string>capturing_interface.seconds_since_last_try()</expression-string>
<language-id>Python</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
<expression>
<expression-string>cursor.fetchone()</expression-string>
<language-id>Python</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
<expression>
<expression-string>cursor.fetchone()[0]</expression-string>
<language-id>Python</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
</expressions>
</component> </component>
<component name="editorHistoryManager"> <component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/explorer/test_explorer.py" /> <entry file="file://$PROJECT_DIR$/explorer/test_explorer.py" />
@ -588,16 +645,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="435">
<caret line="56" column="55" lean-forward="true" selection-start-line="56" selection-start-column="55" selection-end-line="56" selection-end-column="55" />
<folding>
<element signature="e#24#46#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3.6/socket.py"> <entry file="file:///usr/lib/python3.6/socket.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="421"> <state relative-caret-position="421">
@ -605,20 +652,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="405">
<caret line="27" selection-start-line="27" selection-end-line="27" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="360">
<caret line="24" lean-forward="true" selection-start-line="24" selection-end-line="24" />
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3.6/_collections_abc.py"> <entry file="file:///usr/lib/python3.6/_collections_abc.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="91"> <state relative-caret-position="91">
@ -626,13 +659,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="173">
<caret line="166" selection-start-line="166" selection-end-line="166" />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.local/lib/python3.6/site-packages/bs4/__init__.py"> <entry file="file://$USER_HOME$/.local/lib/python3.6/site-packages/bs4/__init__.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="361"> <state relative-caret-position="361">
@ -664,12 +690,72 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file:///usr/lib/python3/dist-packages/html5lib/html5parser.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="496">
<caret line="423" selection-start-line="423" selection-end-line="423" />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/_pydevd_bundle/pydevd_breakpoints.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="218">
<caret line="103" column="34" selection-start-line="103" selection-start-column="34" selection-end-line="103" selection-end-column="34" />
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/apport_python_hook.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="211">
<caret line="48" selection-start-line="48" selection-end-line="48" />
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="203">
<caret line="1254" selection-start-line="1254" selection-end-line="1254" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="203">
<caret line="47" selection-start-line="47" selection-end-line="47" />
<folding>
<element signature="e#24#46#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" selection-start-line="20" selection-end-line="20" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="930">
<caret line="62" column="15" selection-start-line="62" selection-start-column="15" selection-end-line="62" selection-end-column="15" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" selection-start-line="20" selection-end-line="20" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py"> <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="270"> <state relative-caret-position="225">
<caret line="18" lean-forward="true" selection-start-line="18" selection-end-line="18" /> <caret line="15" column="18" lean-forward="true" selection-start-line="15" selection-start-column="18" selection-end-line="15" selection-end-column="18" />
<folding> <folding>
<marker date="1539387124485" expanded="true" signature="129:134" ph="..." /> <marker date="1539438542796" expanded="true" signature="139:144" ph="..." />
<marker date="1539438542796" expanded="true" signature="509:514" ph="..." />
</folding> </folding>
</state> </state>
</provider> </provider>

8
capturer/capturer.py
View file

@ -17,7 +17,7 @@ class Capturer:
while True: while True:
if (capturing_interface.get_pending_task() is None if (capturing_interface.get_pending_task() is None
and capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries): or capturing_interface.seconds_since_last_try() < Capturer.minimum_seconds_between_tries):
sleep(Capturer.sleep_time_no_work) sleep(Capturer.sleep_time_no_work)
continue continue
@ -30,6 +30,7 @@ class Capturer:
continue continue
capturas_interface.insert_captura(ad_data) capturas_interface.insert_captura(ad_data)
task._update_status('Captura inserted')
class CapturingTask: class CapturingTask:
@ -64,6 +65,7 @@ class CapturingTask:
self._extract_data() self._extract_data()
self._check_data() self._check_data()
return
else: else:
self.request_failures += 1 self.request_failures += 1
@ -82,7 +84,7 @@ class CapturingTask:
self._update_status('Invalid value fields') self._update_status('Invalid value fields')
return return
if not self.parser.fields_missing(): if self.parser.fields_missing():
self._update_status('Fields missing') self._update_status('Fields missing')
return return
@ -107,7 +109,7 @@ class AdHtmlParser:
'value': None}, 'value': None},
'tamano_categorico': { 'tamano_categorico': {
'found': False, 'found': False,
'optional': False, 'optional': True,
'value': None}, 'value': None},
'm2': { 'm2': {
'found': False, 'found': False,

3
core/mysql_wrapper.py
View file

@ -54,8 +54,7 @@ class DatabaseWrapper():
{} {}
""".format(query_statement, """.format(query_statement,
query_parameters, query_parameters,
e) e))
)
else: else:
raise Exception("Could not connect to the database.") raise Exception("Could not connect to the database.")

9
db_layer/capturas_interface.py
View file

@ -10,13 +10,13 @@ class CapturasInterface():
def insert_captura(self, ad_data): def insert_captura(self, ad_data):
columns = ', '.join(ad_data.keys()) columns = ', '.join(ad_data.keys())
placeholders_string = ', '.join('%s' * len(ad_data)) placeholders_string = ', '.join(['%s'] * len(ad_data))
query_statement = """ INSERT INTO capturas query_statement = """ INSERT INTO capturas
(%s) ( fecha_captura, {} )
VALUES(%s)""".format(columns, placeholders_string) VALUES( NOW(), {} )""".format(columns, placeholders_string)
query_parameters = ad_data.values() query_parameters = tuple([v for v in ad_data.values()])
self.anunciosdb.query(query_statement, query_parameters) self.anunciosdb.query(query_statement, query_parameters)
@ -24,4 +24,3 @@ class CapturasInterface():
capturas_interface = CapturasInterface() capturas_interface = CapturasInterface()

14
db_layer/capturing_tasks_interface.py
View file

@ -59,16 +59,18 @@ class CapturingTasksInterface:
self.tasksdb.query(query_statement, query_parameters) self.tasksdb.query(query_statement, query_parameters)
def seconds_since_last_try(self): def seconds_since_last_try(self):
query_statement = """SELECT TIME_TO_SEC(TIME_DIFF(now(), write_time)) query_statement = """SELECT TIME_TO_SEC(TIMEDIFF(now(), write_time))
FROM capturing_task_logs FROM capturing_tasks_logs
WHERE status = 'Pending' WHERE status = 'Loading'
ORDER BY write_time ORDER BY write_time
LIMIT 1 LIMIT 1
""" """
cursor = self.taskdb.query(query_statement) cursor = self.tasksdb.query(query_statement)
return cursor.fetchone()[0]
try:
return cursor.fetchone()[0]
except:
return 999
capturing_interface = CapturingTasksInterface() capturing_interface = CapturingTasksInterface()

2
requirements.txt Normal file
View file

@ -0,0 +1,2 @@
bs4 == 4.6.3
mysql-connector = 2.1.6

10
tests/capturer_tests.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import sys import sys
sys.path.append('..') sys.path.append('..')
from capturer.capturer import CapturingTask from capturer.capturer import CapturingTask, Capturer
@ -17,5 +17,11 @@ def test_CapturingTask():
print(task.get_ad_data()) print(task.get_ad_data())
def test_Capturer():
capturer = Capturer()
capturer.start()
test_CapturingTask()
# test_CapturingTask()
test_Capturer()