counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Retoques menores en geocoder y capturer por problemas de tipos.

Browse source
This commit is contained in:
pablomartincalvo 2018-10-20 15:58:37 +02:00
parent c40d39e558
commit 600ff889be
4 changed files with 88 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

139
.idea/workspace.xml generated
View file

@ -4,9 +4,8 @@
<list default="true" id="6e2fbba0-85ff-42d6-8e70-e4cdef1000c8" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/capturer/capturer.py" beforeDir="false" afterPath="$PROJECT_DIR$/capturer/capturer.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/db_layer/capturas_interface.py" beforeDir="false" afterPath="$PROJECT_DIR$/db_layer/capturas_interface.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/core/scrapping_utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/core/scrapping_utils.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/geocoder/geocoder.py" beforeDir="false" afterPath="$PROJECT_DIR$/geocoder/geocoder.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/refresher/refresher.py" beforeDir="false" afterPath="$PROJECT_DIR$/refresher/refresher.py" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
@ -25,19 +24,20 @@
<entry key="project.open.time.14" value="3" />
<entry key="project.open.time.17" value="1" />
<entry key="project.open.time.18" value="1" />
<entry key="project.open.time.2" value="1" />
<entry key="project.open.time.21" value="1" />
<entry key="project.open.time.26" value="1" />
<entry key="project.opened" value="11" />
<entry key="project.opened" value="12" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="py" value="73" />
<entry key="py" value="74" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="Python" value="68" />
<entry key="Python" value="69" />
<entry key="Scratch" value="5" />
</counts>
</usages-collector>
@ -46,7 +46,7 @@
<entry key="Python Console" value="1597" />
<entry key="capturer" value="862" />
<entry key="dummy" value="14" />
<entry key="py" value="18394" />
<entry key="py" value="18558" />
<entry key="scratch_1" value="489" />
<entry key="txt" value="1114" />
</counts>
@ -54,7 +54,7 @@
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="PLAIN_TEXT" value="1128" />
<entry key="Python" value="21070" />
<entry key="Python" value="21234" />
<entry key="Scratch" value="272" />
</counts>
</usages-collector>
@ -73,23 +73,29 @@
<splitter split-orientation="horizontal" split-proportion="0.5">
<split-first>
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="465">
<caret line="57" lean-forward="true" selection-start-line="57" selection-end-line="57" />
<state relative-caret-position="435">
<caret line="79" column="48" selection-start-line="79" selection-start-column="48" selection-end-line="79" selection-end-column="48" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="527">
<caret line="38" column="40" selection-start-line="38" selection-start-column="40" selection-end-line="38" selection-end-column="40" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-194">
<caret line="45" column="45" lean-forward="true" selection-start-line="45" selection-start-column="45" selection-end-line="45" selection-end-column="45" />
<folding>
<marker date="1539964895728" expanded="true" signature="2060:2402" ph="..." />
</folding>
<state relative-caret-position="15">
<caret line="45" column="45" selection-start-line="45" selection-start-column="45" selection-end-line="45" selection-end-column="45" />
</state>
</provider>
</entry>
@ -106,13 +112,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/geocoder/geocoder.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="615">
<caret line="41" selection-start-line="41" selection-end-line="41" />
<folding>
<marker date="1539964895724" expanded="true" signature="150:194" ph="..." />
<marker date="1539964895724" expanded="true" signature="210:220" ph="..." />
<marker date="1539964895724" expanded="true" signature="210:429" ph="..." />
</folding>
<state relative-caret-position="495">
<caret line="33" column="24" selection-start-line="33" selection-start-column="24" selection-end-line="33" selection-end-column="24" />
</state>
</provider>
</entry>
@ -134,7 +135,7 @@
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" lean-forward="true" selection-start-line="17" selection-end-line="17" />
<caret line="17" selection-start-line="17" selection-end-line="17" />
</state>
</provider>
</entry>
@ -151,7 +152,7 @@
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="405">
<state relative-caret-position="225">
<caret line="27" column="27" lean-forward="true" selection-start-line="27" selection-start-column="27" selection-end-line="27" selection-end-column="27" />
</state>
</provider>
@ -172,7 +173,7 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="75">
<state relative-caret-position="105">
<caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding>
<element signature="e#0#46#0" expanded="true" />
@ -220,7 +221,6 @@
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/core/scrapping_utils.py" />
<option value="$PROJECT_DIR$/explorer/test_explorer.py" />
<option value="$PROJECT_DIR$/core/alerts.py" />
<option value="$PROJECT_DIR$/core/task.py" />
@ -236,9 +236,10 @@
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_1.py" />
<option value="$PROJECT_DIR$/tests/refresher_tests.py" />
<option value="$PROJECT_DIR$/db_layer/capturas_interface.py" />
<option value="$PROJECT_DIR$/geocoder/geocoder.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" />
<option value="$PROJECT_DIR$/refresher/refresher.py" />
<option value="$PROJECT_DIR$/geocoder/geocoder.py" />
<option value="$PROJECT_DIR$/core/scrapping_utils.py" />
<option value="$PROJECT_DIR$/capturer/capturer.py" />
</list>
</option>
</component>
@ -252,6 +253,7 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
@ -264,6 +266,11 @@
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="capturer" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
<item name="core" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="Drogon" type="b2602c69:ProjectViewProjectNode" />
<item name="Drogon" type="462c0819:PsiDirectoryNode" />
@ -306,13 +313,13 @@
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="TODO_SCOPE" value="All Places" />
<property name="com.intellij.ide.scratch.LRUPopupBuilder$1/New Scratch File" value="Python" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="RecentsManager">
<key name="MoveFile.RECENT_KEYS">
@ -579,7 +586,7 @@
<frame x="0" y="-2" width="1920" height="1082" extended-state="6" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" sideWeight="0.48076922" visible="true" weight="0.14918292" />
<window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.48076922" visible="true" weight="0.14918292" />
<window_info id="Structure" order="1" sideWeight="0.5192308" side_tool="true" weight="0.14918292" />
<window_info id="Favorites" order="2" sideWeight="0.5015674" side_tool="true" weight="0.14918292" />
<window_info active="true" id="Repositories" order="3" sideWeight="0.49529782" visible="true" weight="0.32999474" />
@ -593,7 +600,7 @@
<window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.49973643" weight="0.269958" />
<window_info anchor="bottom" id="Terminal" order="8" weight="0.32983193" />
<window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.5007907" side_tool="true" weight="0.32983193" />
<window_info active="true" anchor="bottom" id="Python Console" order="10" sideWeight="0.49920928" visible="true" weight="0.32983193" />
<window_info anchor="bottom" id="Python Console" order="10" sideWeight="0.49920928" visible="true" weight="0.32983193" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
@ -755,13 +762,6 @@
<entry file="file://$PROJECT_DIR$/capturer/__init__.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="225">
<caret line="15" column="2" selection-start-line="15" selection-start-column="2" selection-end-line="15" selection-end-column="2" />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/pydevd.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="336">
@ -860,15 +860,10 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/geocoder/geocoder.py">
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="615">
<caret line="41" selection-start-line="41" selection-end-line="41" />
<folding>
<marker date="1539964895724" expanded="true" signature="150:194" ph="..." />
<marker date="1539964895724" expanded="true" signature="210:220" ph="..." />
<marker date="1539964895724" expanded="true" signature="210:429" ph="..." />
</folding>
<state relative-caret-position="15">
<caret line="45" column="45" selection-start-line="45" selection-start-column="45" selection-end-line="45" selection-end-column="45" />
</state>
</provider>
</entry>
@ -879,6 +874,13 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" selection-start-line="17" selection-end-line="17" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturing_tasks_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="15">
@ -898,7 +900,7 @@
</entry>
<entry file="file://$PROJECT_DIR$/core/alerts.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="75">
<state relative-caret-position="105">
<caret line="7" column="34" selection-start-line="7" selection-start-column="34" selection-end-line="7" selection-end-column="34" />
<folding>
<element signature="e#0#46#0" expanded="true" />
@ -906,34 +908,31 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tests/capturer_tests.py">
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" lean-forward="true" selection-start-line="17" selection-end-line="17" />
<state relative-caret-position="225">
<caret line="27" column="27" lean-forward="true" selection-start-line="27" selection-start-column="27" selection-end-line="27" selection-end-column="27" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/geocoder/geocoder.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="495">
<caret line="33" column="24" selection-start-line="33" selection-start-column="24" selection-end-line="33" selection-end-column="24" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/core/scrapping_utils.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="527">
<caret line="38" column="40" selection-start-line="38" selection-start-column="40" selection-end-line="38" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/capturer/capturer.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="465">
<caret line="57" lean-forward="true" selection-start-line="57" selection-end-line="57" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/refresher/refresher.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="405">
<caret line="27" column="27" lean-forward="true" selection-start-line="27" selection-start-column="27" selection-end-line="27" selection-end-column="27" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/db_layer/capturas_interface.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-194">
<caret line="45" column="45" lean-forward="true" selection-start-line="45" selection-start-column="45" selection-end-line="45" selection-end-column="45" />
<folding>
<marker date="1539964895728" expanded="true" signature="2060:2402" ph="..." />
</folding>
<state relative-caret-position="435">
<caret line="79" column="48" selection-start-line="79" selection-start-column="48" selection-end-line="79" selection-end-column="48" />
</state>
</provider>
</entry>

6
capturer/capturer.py
View file

@ -32,9 +32,6 @@ class Capturer:
if task.status == 'Data ready':
ad_data = task.get_ad_data()
elif Refresher.dead_ad_checker(task.html):
task._update_status('Dead ad')
continue
else:
continue
@ -80,6 +77,9 @@ class CapturingTask:
self._check_data()
return
elif attack.get_status_code() == 301:
self._update_status('Dead ad')
return
else:
self.request_failures += 1
self._update_status('Fail {}'.format(self.request_failures))

3
core/scrapping_utils.py
View file

@ -34,3 +34,6 @@ class UrlAttack():
def get_text(self):
if self.success:
return self.response.text
def get_status_code(self):
return self.response.status_code

11
geocoder/geocoder.py
View file

@ -20,7 +20,18 @@ class Geocoder:
else:
continue
try:
address = ad_data['calle'] + ', ' + ad_data['ciudad']
except TypeError:
geo_data = {'latitude': None,
'longitude': None,
'precision': 'SIN RESULTADOS'}
capturas_interface.update_geo_data(ad_data['referencia'],
ad_data['fecha_captura'],
geo_data['latitude'],
geo_data['longitude'],
geo_data['precision'])
continue
if self.cache.address_in_cache(address):
geo_data = self.cache.get_coordinates(address)