counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Correciones para deteccion de anuncios dados de baja.

Browse source 
Mejoras en script de deployment.
This commit is contained in:
pablomartincalvo 2018-11-16 18:20:50 +01:00
parent ee8df1b635
commit df07497125
7 changed files with 285 additions and 224 deletions
Download patch file Download diff file Expand all files Collapse all files

461
.idea/workspace.xml generated
View file

@ -2,11 +2,13 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment=""> <list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/tests/scrapping_utils_tests.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/core/config.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/config.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/core/config.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/config.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/db_layer/db_init_scripts/1_20181028dump.sql" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/db_init_scripts/1_20181028dump.sql" afterDir="false" /> <change beforePath="$PROJECT_DIR$/core/scrapping_utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/scrapping_utils.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/tests/capturer_tests.py" beforeDir="false" afterPath="$PROJECT_DIR$/tests/capturer_tests.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/deployer.sh" beforeDir="false" afterPath="$PROJECT_DIR$/deployer.sh" afterDir="false" />
<change beforePath="$PROJECT_DIR$/tests/refresher_tests.py" beforeDir="false" afterPath="$PROJECT_DIR$/tests/refresher_tests.py" afterDir="false" />
</list> </list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
@ -24,18 +26,18 @@
<entry key="project.open.time.13" value="3" /> <entry key="project.open.time.13" value="3" />
<entry key="project.open.time.14" value="3" /> <entry key="project.open.time.14" value="3" />
<entry key="project.open.time.17" value="2" /> <entry key="project.open.time.17" value="2" />
<entry key="project.open.time.18" value="2" /> <entry key="project.open.time.18" value="3" />
<entry key="project.open.time.2" value="1" /> <entry key="project.open.time.2" value="1" />
<entry key="project.open.time.21" value="1" /> <entry key="project.open.time.21" value="1" />
<entry key="project.open.time.24" value="1" /> <entry key="project.open.time.24" value="1" />
<entry key="project.open.time.26" value="1" /> <entry key="project.open.time.26" value="1" />
<entry key="project.opened" value="16" /> <entry key="project.opened" value="17" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.extensions.open"> <usages-collector id="statistics.file.extensions.open">
<counts> <counts>
<entry key="Dockerfile" value="11" /> <entry key="Dockerfile" value="11" />
<entry key="py" value="122" /> <entry key="py" value="129" />
<entry key="sh" value="7" /> <entry key="sh" value="7" />
<entry key="sql" value="6" /> <entry key="sql" value="6" />
<entry key="txt" value="6" /> <entry key="txt" value="6" />
@ -46,7 +48,7 @@
<entry key="Bash" value="5" /> <entry key="Bash" value="5" />
<entry key="DBN-SQL" value="3" /> <entry key="DBN-SQL" value="3" />
<entry key="PLAIN_TEXT" value="19" /> <entry key="PLAIN_TEXT" value="19" />
<entry key="Python" value="116" /> <entry key="Python" value="123" />
<entry key="SQL" value="3" /> <entry key="SQL" value="3" />
<entry key="Scratch" value="6" /> <entry key="Scratch" value="6" />
</counts> </counts>
@ -57,19 +59,19 @@
<entry key="Python Console" value="1651" /> <entry key="Python Console" value="1651" />
<entry key="capturer" value="862" /> <entry key="capturer" value="862" />
<entry key="dummy" value="14" /> <entry key="dummy" value="14" />
<entry key="py" value="19636" /> <entry key="py" value="19771" />
<entry key="scratch_1" value="489" /> <entry key="scratch_1" value="489" />
<entry key="sh" value="1633" /> <entry key="sh" value="1677" />
<entry key="sql" value="63" /> <entry key="sql" value="63" />
<entry key="txt" value="1315" /> <entry key="txt" value="1315" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.edit"> <usages-collector id="statistics.file.types.edit">
<counts> <counts>
<entry key="Bash" value="454" /> <entry key="Bash" value="498" />
<entry key="DBN-SQL" value="63" /> <entry key="DBN-SQL" value="63" />
<entry key="PLAIN_TEXT" value="2617" /> <entry key="PLAIN_TEXT" value="2617" />
<entry key="Python" value="22366" /> <entry key="Python" value="22501" />
<entry key="Scratch" value="272" /> <entry key="Scratch" value="272" />
</counts> </counts>
</usages-collector> </usages-collector>
@ -85,92 +87,130 @@
<favorites_list name="Drogon" /> <favorites_list name="Drogon" />
</component> </component>
<component name="FileEditorManager"> <component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300"> <splitter split-orientation="horizontal" split-proportion="0.5">
<file pinned="false" current-in-tab="false"> <split-first>
<entry file="file://$PROJECT_DIR$/db_layer/Dockerfile"> <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<provider selected="true" editor-type-id="text-editor"> <file pinned="false" current-in-tab="false">
<state relative-caret-position="60"> <entry file="file://$PROJECT_DIR$/db_layer/Dockerfile">
<caret line="4" column="50" selection-start-line="4" selection-start-column="50" selection-end-line="4" selection-end-column="50" /> <provider selected="true" editor-type-id="text-editor">
</state> <state relative-caret-position="60">
</provider> <caret line="4" selection-start-line="4" selection-end-line="4" />
</entry> </state>
</file> </provider>
<file pinned="false" current-in-tab="false"> </entry>
<entry file="file://$PROJECT_DIR$/core/config.py"> </file>
<provider selected="true" editor-type-id="text-editor"> <file pinned="false" current-in-tab="true">
<state relative-caret-position="165"> <entry file="file://$PROJECT_DIR$/deployer.sh">
<caret line="11" column="30" selection-start-line="11" selection-start-column="30" selection-end-line="11" selection-end-column="30" /> <provider selected="true" editor-type-id="text-editor">
<folding> <state relative-caret-position="705">
<element signature="e#0#22#0" expanded="true" /> <caret line="47" lean-forward="true" selection-start-line="47" selection-end-line="47" />
</folding> </state>
</state> </provider>
</provider> </entry>
</entry> </file>
</file> <file pinned="false" current-in-tab="false">
<file pinned="false" current-in-tab="false"> <entry file="file://$PROJECT_DIR$/core/config.py">
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py"> <provider selected="true" editor-type-id="text-editor">
<provider selected="true" editor-type-id="text-editor"> <state relative-caret-position="285">
<state relative-caret-position="300"> <caret line="19" column="22" lean-forward="true" selection-start-line="19" selection-start-column="22" selection-end-line="19" selection-end-column="22" />
<caret line="20" selection-start-line="20" selection-end-line="20" /> <folding>
</state> <element signature="e#0#22#0" expanded="true" />
</provider> </folding>
</entry> </state>
</file> </provider>
<file pinned="false" current-in-tab="false"> </entry>
<entry file="file://$PROJECT_DIR$/explorer/explorer.py"> </file>
<provider selected="true" editor-type-id="text-editor"> <file pinned="false" current-in-tab="false">
<state relative-caret-position="298"> <entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<caret line="74" column="89" lean-forward="true" selection-start-line="74" selection-start-column="89" selection-end-line="74" selection-end-column="89" /> <provider selected="true" editor-type-id="text-editor">
</state> <state relative-caret-position="285">
</provider> <caret line="85" column="53" lean-forward="true" selection-start-line="85" selection-start-column="53" selection-end-line="85" selection-end-column="53" />
</entry> </state>
</file> </provider>
<file pinned="false" current-in-tab="false"> </entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py"> </file>
<provider selected="true" editor-type-id="text-editor"> <file pinned="false" current-in-tab="false">
<state relative-caret-position="233"> <entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<caret line="194" column="16" lean-forward="true" selection-start-line="194" selection-start-column="16" selection-end-line="194" selection-end-column="16" /> <provider selected="true" editor-type-id="text-editor">
</state> <state relative-caret-position="332">
</provider> <caret line="29" column="27" lean-forward="true" selection-start-line="29" selection-start-column="27" selection-end-line="29" selection-end-column="27" />
</entry> </state>
</file> </provider>
<file pinned="false" current-in-tab="true"> </entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py"> </file>
<provider selected="true" editor-type-id="text-editor"> <file pinned="false" current-in-tab="false">
<state relative-caret-position="120"> <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<caret line="8" selection-start-line="8" selection-end-line="17" selection-end-column="57" /> <provider selected="true" editor-type-id="text-editor">
</state> <state relative-caret-position="210">
</provider> <caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
</entry> </state>
</file> </provider>
<file pinned="false" current-in-tab="false"> </entry>
<entry file="file://$PROJECT_DIR$/db_layer/db_init_scripts/1_20181028dump.sql"> </file>
<provider selected="true" editor-type-id="text-editor"> </leaf>
<state relative-caret-position="276"> </split-first>
<caret line="180" column="45" selection-start-line="180" selection-start-column="45" selection-end-line="180" selection-end-column="45" /> <split-second>
</state> <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
</provider> <file pinned="false" current-in-tab="false">
</entry> <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
</file> <provider selected="true" editor-type-id="text-editor">
<file pinned="false" current-in-tab="false"> <state relative-caret-position="480">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py"> <caret line="32" column="30" selection-start-line="32" selection-start-column="30" selection-end-line="32" selection-end-column="30" />
<provider selected="true" editor-type-id="text-editor"> <folding>
<state relative-caret-position="285"> <element signature="e#0#11#0" expanded="true" />
<caret line="47" selection-start-line="47" selection-end-line="47" /> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/geocoder/geocoder.py"> <entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1950"> <state relative-caret-position="390">
<caret line="130" column="11" selection-start-line="130" selection-start-column="11" selection-end-line="130" selection-end-column="11" /> <caret line="32" lean-forward="true" selection-start-line="32" selection-end-line="32" />
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
</leaf> <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/tests/scrapping_utils_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
<caret line="9" lean-forward="true" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="490">
<caret line="45" selection-start-line="45" selection-end-line="45" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2400">
<caret line="160" column="15" selection-start-line="160" selection-start-column="15" selection-end-line="160" selection-end-column="39" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/tests/refresher_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret selection-end-column="33" />
</state>
</provider>
</entry>
</file>
</leaf>
</split-second>
</splitter>
</component> </component>
<component name="FileTemplateManagerImpl"> <component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES"> <option name="RECENT_TEMPLATES">
@ -217,9 +257,7 @@
<option value="$PROJECT_DIR$/tests/capturing_tests.py" /> <option value="$PROJECT_DIR$/tests/capturing_tests.py" />
<option value="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" /> <option value="$PROJECT_DIR$/db_layer/capturing_tasks_interface.py" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py" /> <option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py" />
<option value="$PROJECT_DIR$/tests/refresher_tests.py" />
<option value="$PROJECT_DIR$/refresher/refresher.py" /> <option value="$PROJECT_DIR$/refresher/refresher.py" />
<option value="$PROJECT_DIR$/core/scrapping_utils.py" />
<option value="$PROJECT_DIR$/db_layer/capturas_interface.py" /> <option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
<option value="$PROJECT_DIR$/db_layer/Dockerfile" /> <option value="$PROJECT_DIR$/db_layer/Dockerfile" />
<option value="$PROJECT_DIR$/requirements.txt" /> <option value="$PROJECT_DIR$/requirements.txt" />
@ -230,13 +268,16 @@
<option value="$PROJECT_DIR$/geocoder/geocoder.py" /> <option value="$PROJECT_DIR$/geocoder/geocoder.py" />
<option value="$PROJECT_DIR$/explorer/explorer.py" /> <option value="$PROJECT_DIR$/explorer/explorer.py" />
<option value="$PROJECT_DIR$/db_layer/db_init_scripts/users.sql" /> <option value="$PROJECT_DIR$/db_layer/db_init_scripts/users.sql" />
<option value="$PROJECT_DIR$/deployer.sh" />
<option value="$PROJECT_DIR$/core/alerts.py" /> <option value="$PROJECT_DIR$/core/alerts.py" />
<option value="$PROJECT_DIR$/core/mysql_wrapper.py" /> <option value="$PROJECT_DIR$/core/mysql_wrapper.py" />
<option value="$PROJECT_DIR$/db_layer/db_init_scripts/1_20181028dump.sql" /> <option value="$PROJECT_DIR$/db_layer/db_init_scripts/1_20181028dump.sql" />
<option value="$PROJECT_DIR$/core/config.py" />
<option value="$PROJECT_DIR$/tests/capturer_tests.py" /> <option value="$PROJECT_DIR$/tests/capturer_tests.py" />
<option value="$PROJECT_DIR$/core/config.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" /> <option value="$PROJECT_DIR$/capturer/capturer.py" />
<option value="$PROJECT_DIR$/tests/refresher_tests.py" />
<option value="$PROJECT_DIR$/tests/scrapping_utils_tests.py" />
<option value="$PROJECT_DIR$/core/scrapping_utils.py" />
<option value="$PROJECT_DIR$/deployer.sh" />
</list> </list>
</option> </option>
</component> </component>
@ -250,6 +291,7 @@
<foldersAlwaysOnTop value="true" /> <foldersAlwaysOnTop value="true" />
</navigator> </navigator>
<panes> <panes>
<pane id="Scope" />
<pane id="ProjectPane"> <pane id="ProjectPane">
<subPane> <subPane>
<expand> <expand>
@ -265,18 +307,7 @@
<path> <path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" /> <item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" /> <item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="db_layer" type="462c0819:PsiDirectoryNode" /> <item name="core" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="db_layer" type="462c0819:PsiDirectoryNode" />
<item name="db_init_scripts" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="explorer" type="462c0819:PsiDirectoryNode" />
</path> </path>
<path> <path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" /> <item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
@ -310,7 +341,6 @@
<select /> <select />
</subPane> </subPane>
</pane> </pane>
<pane id="Scope" />
</panes> </panes>
</component> </component>
<component name="PropertiesComponent"> <component name="PropertiesComponent">
@ -342,7 +372,7 @@
</list> </list>
</option> </option>
</component> </component>
<component name="RunManager" selected="Python.capturer_tests"> <component name="RunManager" selected="Python.scrapping_utils_tests">
<configuration name="alerts" type="PythonConfigurationType" factoryName="Python" temporary="true"> <configuration name="alerts" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" /> <module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" /> <option name="INTERPRETER_OPTIONS" value="" />
@ -406,7 +436,7 @@
<option name="INPUT_FILE" value="" /> <option name="INPUT_FILE" value="" />
<method v="2" /> <method v="2" />
</configuration> </configuration>
<configuration name="geocoder" type="PythonConfigurationType" factoryName="Python" temporary="true"> <configuration name="refresher_tests" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" /> <module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" /> <option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" /> <option name="PARENT_ENVS" value="true" />
@ -414,11 +444,11 @@
<env name="PYTHONUNBUFFERED" value="1" /> <env name="PYTHONUNBUFFERED" value="1" />
</envs> </envs>
<option name="SDK_HOME" value="" /> <option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/geocoder" /> <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/tests" />
<option name="IS_MODULE_SDK" value="true" /> <option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" /> <option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" /> <option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/geocoder/geocoder.py" /> <option name="SCRIPT_NAME" value="$PROJECT_DIR$/tests/refresher_tests.py" />
<option name="PARAMETERS" value="" /> <option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" /> <option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" /> <option name="EMULATE_TERMINAL" value="false" />
@ -427,7 +457,7 @@
<option name="INPUT_FILE" value="" /> <option name="INPUT_FILE" value="" />
<method v="2" /> <method v="2" />
</configuration> </configuration>
<configuration name="mysql_wrapper" type="PythonConfigurationType" factoryName="Python" temporary="true"> <configuration name="scrapping_utils_tests" type="PythonConfigurationType" factoryName="Python" temporary="true">
<module name="Drogon" /> <module name="Drogon" />
<option name="INTERPRETER_OPTIONS" value="" /> <option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" /> <option name="PARENT_ENVS" value="true" />
@ -435,11 +465,11 @@
<env name="PYTHONUNBUFFERED" value="1" /> <env name="PYTHONUNBUFFERED" value="1" />
</envs> </envs>
<option name="SDK_HOME" value="" /> <option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/core" /> <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/tests" />
<option name="IS_MODULE_SDK" value="true" /> <option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" /> <option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" /> <option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/core/mysql_wrapper.py" /> <option name="SCRIPT_NAME" value="$PROJECT_DIR$/tests/scrapping_utils_tests.py" />
<option name="PARAMETERS" value="" /> <option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" /> <option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" /> <option name="EMULATE_TERMINAL" value="false" />
@ -450,18 +480,18 @@
</configuration> </configuration>
<list> <list>
<item itemvalue="Python.capturer_tests" /> <item itemvalue="Python.capturer_tests" />
<item itemvalue="Python.geocoder" />
<item itemvalue="Python.mysql_wrapper" />
<item itemvalue="Python.explorer" /> <item itemvalue="Python.explorer" />
<item itemvalue="Python.alerts" /> <item itemvalue="Python.alerts" />
<item itemvalue="Python.refresher_tests" />
<item itemvalue="Python.scrapping_utils_tests" />
</list> </list>
<recent_temporary> <recent_temporary>
<list> <list>
<item itemvalue="Python.scrapping_utils_tests" />
<item itemvalue="Python.refresher_tests" />
<item itemvalue="Python.capturer_tests" /> <item itemvalue="Python.capturer_tests" />
<item itemvalue="Python.alerts" /> <item itemvalue="Python.alerts" />
<item itemvalue="Python.explorer" /> <item itemvalue="Python.explorer" />
<item itemvalue="Python.mysql_wrapper" />
<item itemvalue="Python.geocoder" />
</list> </list>
</recent_temporary> </recent_temporary>
</component> </component>
@ -646,15 +676,15 @@
<frame x="0" y="-2" width="1920" height="1082" extended-state="6" /> <frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
<editor active="true" /> <editor active="true" />
<layout> <layout>
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.48076922" visible="true" weight="0.14918292" /> <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.47962382" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" sideWeight="0.5192308" side_tool="true" weight="0.14918292" /> <window_info id="Structure" order="1" sideWeight="0.52037615" side_tool="true" visible="true" weight="0.14918292" />
<window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" /> <window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
<window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" /> <window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
<window_info id="DB Browser" order="4" weight="0.32999474" /> <window_info id="DB Browser" order="4" weight="0.32999474" />
<window_info anchor="bottom" id="Message" order="0" /> <window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.32983193" /> <window_info anchor="bottom" id="Find" order="1" weight="0.32983193" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32983193" /> <window_info anchor="bottom" id="Run" order="2" visible="true" weight="0.32983193" />
<window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.39915967" /> <window_info anchor="bottom" id="Debug" order="3" weight="0.39915967" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" /> <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" /> <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32878152" /> <window_info anchor="bottom" id="TODO" order="6" sideWeight="0.49973643" weight="0.32878152" />
@ -746,6 +776,16 @@
<line>68</line> <line>68</line>
<option name="timeStamp" value="16" /> <option name="timeStamp" value="16" />
</line-breakpoint> </line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/tests/refresher_tests.py</url>
<line>257</line>
<option name="timeStamp" value="17" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/tests/scrapping_utils_tests.py</url>
<line>14</line>
<option name="timeStamp" value="18" />
</line-breakpoint>
</breakpoints> </breakpoints>
<breakpoints-dialog> <breakpoints-dialog>
<breakpoints-dialog /> <breakpoints-dialog />
@ -898,13 +938,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/tests/refresher_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="796">
<caret line="258" column="30" selection-start-line="258" selection-start-column="30" selection-end-line="258" selection-end-column="30" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/db_init_scripts/tables.sql" /> <entry file="file://$PROJECT_DIR$/db_layer/db_init_scripts/tables.sql" />
<entry file="file://$PROJECT_DIR$/db_layer/db_init_scripts/views.sql" /> <entry file="file://$PROJECT_DIR$/db_layer/db_init_scripts/views.sql" />
<entry file="file://$PROJECT_DIR$/tests/geocoder_tests.py"> <entry file="file://$PROJECT_DIR$/tests/geocoder_tests.py">
@ -934,20 +967,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="613">
<caret line="41" column="23" selection-start-line="41" selection-start-column="23" selection-end-line="41" selection-end-column="23" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="405">
<caret line="27" column="27" selection-start-line="27" selection-start-column="27" selection-end-line="27" selection-end-column="27" />
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py"> <entry file="file:///usr/lib/python3/dist-packages/mysql/connector/cursor.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="263"> <state relative-caret-position="263">
@ -962,37 +981,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="480">
<caret line="32" column="30" selection-start-line="32" selection-start-column="30" selection-end-line="32" selection-end-column="30" />
<folding>
<element signature="e#0#11#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/deployer.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="420">
<caret line="28" column="60" selection-start-line="28" selection-start-column="60" selection-end-line="28" selection-end-column="60" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/Dockerfile">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="60">
<caret line="4" column="50" selection-start-line="4" selection-start-column="50" selection-end-line="4" selection-end-column="50" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/geocoder/geocoder.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1950">
<caret line="130" column="11" selection-start-line="130" selection-start-column="11" selection-end-line="130" selection-end-column="11" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/db_init_scripts/1_20181028dump.sql"> <entry file="file://$PROJECT_DIR$/db_layer/db_init_scripts/1_20181028dump.sql">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="276"> <state relative-caret-position="276">
@ -1000,16 +988,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/core/config.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="11" column="30" selection-start-line="11" selection-start-column="30" selection-end-line="11" selection-end-column="30" />
<folding>
<element signature="e#0#22#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file:///usr/lib/python3/dist-packages/apport_python_hook.py"> <entry file="file:///usr/lib/python3/dist-packages/apport_python_hook.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="428"> <state relative-caret-position="428">
@ -1017,38 +995,107 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py"> <entry file="file://$PROJECT_DIR$/db_layer/Dockerfile">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300"> <state relative-caret-position="60">
<caret line="20" selection-start-line="20" selection-end-line="20" /> <caret line="4" selection-start-line="4" selection-end-line="4" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py"> <entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285"> <state relative-caret-position="480">
<caret line="47" selection-start-line="47" selection-end-line="47" /> <caret line="32" column="30" selection-start-line="32" selection-start-column="30" selection-end-line="32" selection-end-column="30" />
</state> <folding>
</provider> <element signature="e#0#11#0" expanded="true" />
</entry> </folding>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="233">
<caret line="194" column="16" lean-forward="true" selection-start-line="194" selection-start-column="16" selection-end-line="194" selection-end-column="16" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/explorer/explorer.py"> <entry file="file://$PROJECT_DIR$/explorer/explorer.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="298"> <state relative-caret-position="2400">
<caret line="74" column="89" lean-forward="true" selection-start-line="74" selection-start-column="89" selection-end-line="74" selection-end-column="89" /> <caret line="160" column="15" selection-start-line="160" selection-start-column="15" selection-end-line="160" selection-end-column="39" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="810">
<caret line="87" column="11" lean-forward="true" selection-start-line="87" selection-start-column="11" selection-end-line="87" selection-end-column="11" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/config.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="19" column="22" lean-forward="true" selection-start-line="19" selection-start-column="22" selection-end-line="19" selection-end-column="22" />
<folding>
<element signature="e#0#22#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/geocoder/geocoder.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="556">
<caret line="164" column="11" selection-start-line="164" selection-start-column="11" selection-end-line="164" selection-end-column="11" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/refresher_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret selection-end-column="33" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="85" column="53" lean-forward="true" selection-start-line="85" selection-start-column="53" selection-end-line="85" selection-end-column="53" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/mysql_wrapper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="490">
<caret line="45" selection-start-line="45" selection-end-line="45" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="390">
<caret line="32" lean-forward="true" selection-start-line="32" selection-end-line="32" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/scrapping_utils_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
<caret line="9" lean-forward="true" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="332">
<caret line="29" column="27" lean-forward="true" selection-start-line="29" selection-start-column="27" selection-end-line="29" selection-end-column="27" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py"> <entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120"> <state relative-caret-position="210">
<caret line="8" selection-start-line="8" selection-end-line="17" selection-end-column="57" /> <caret line="14" lean-forward="true" selection-start-line="14" selection-end-line="14" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/deployer.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="705">
<caret line="47" lean-forward="true" selection-start-line="47" selection-end-line="47" />
</state> </state>
</provider> </provider>
</entry> </entry>

2
capturer/capturer.py
View file

@ -81,8 +81,8 @@ class CapturingTask:
self._update_status('Dead ad') self._update_status('Dead ad')
return return
else: else:
self.request_failures += 1
self._update_status('Fail {}'.format(self.request_failures)) self._update_status('Fail {}'.format(self.request_failures))
self.request_failures += 1
sleep(CapturingTask.sleep_time_failed_request) sleep(CapturingTask.sleep_time_failed_request)
continue continue

2
core/config.py
View file

@ -17,7 +17,7 @@ databases = {'dev':{'host': '185.166.215.170 ',
try: try:
current_db = environ['DROGON_ENV'] current_db = environ['DROGON_ENV']
except KeyError: except KeyError:
current_db = 'dev' current_db = 'tst'
try: try:
current_db_parameters = databases[current_db] current_db_parameters = databases[current_db]

6
core/scrapping_utils.py
View file

@ -28,12 +28,10 @@ class UrlAttack():
self.success = False self.success = False
def get_response(self): def get_response(self):
if self.success: return self.response
return self.response
def get_text(self): def get_text(self):
if self.success: return self.response.text
return self.response.text
def get_status_code(self): def get_status_code(self):
try: try:

5
deployer.sh
View file

@ -6,10 +6,11 @@ set -x
echo "export DROGON_ENV=tst" | tee -a /etc/profile.d/drogon_variables.sh echo "export DROGON_ENV=tst" | tee -a /etc/profile.d/drogon_variables.sh
echo "export DB_SERVICE_NAME=tst_drogon_db" | tee -a /etc/profile.d/drogon_variables.sh echo "export DB_SERVICE_NAME=tst_drogon_db" | tee -a /etc/profile.d/drogon_variables.sh
echo "export NETWORK_NAME=tst_drogon_network" | tee -a /etc/profile.d/drogon_variables.sh echo "export NETWORK_NAME=tst_drogon_network" | tee -a /etc/profile.d/drogon_variables.sh
echo "export GIT_BRANCH=testing" | tee -a /etc/profile.d/drogon_variables.sh
DROGON_ENV=tst DROGON_ENV=tst
DB_SERVICE_NAME=tst_drogon_db DB_SERVICE_NAME=tst_drogon_db
NETWORK_NAME=tst_drogon_network NETWORK_NAME=tst_drogon_network
GIT_BRANCH=testing
############################################################### ###############################################################
################# INSTALACION DE HERRAMIENTAS ################# ################# INSTALACION DE HERRAMIENTAS #################
@ -43,7 +44,7 @@ fi
# Hacer clone desde remote # Hacer clone desde remote
git clone git@gitlab.com:pablomartincalvo/Drogon.git /opt/Drogon git clone git@gitlab.com:pablomartincalvo/Drogon.git /opt/Drogon
cd /opt/Drogon cd /opt/Drogon
git checkout testing git checkout ${GIT_BRANCH}
################# DB ################# DB

18
tests/refresher_tests.py
View file

File diff suppressed because one or more lines are too long

15
tests/scrapping_utils_tests.py Normal file
View file

@ -0,0 +1,15 @@
from core.scrapping_utils import *
def UrlAttack_test(url):
attack = UrlAttack(url)
attack.attack()
print(attack.get_status_code())
print(attack.get_response())
UrlAttack_test('https://www.idealista.com/inmueble/40402493/')