counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Minor fixes.

Browse source
This commit is contained in:
pablo 2021-01-03 20:05:34 +01:00
parent e34a34acaf
commit 007f458cd5
3 changed files with 140 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

6
core/parsing_utils.py
View file

@ -258,7 +258,7 @@ class TipoAnuncioFieldInstructions(BaseTargetFieldInstructions):
if "venta" in soup.find("title").text:
self.value = 1
self.found = True
if "alquiler" in soup.find("title").text:
if "Alquiler" in soup.find("title").text:
self.value = 2
self.found = True
@ -542,11 +542,11 @@ class ParsingFlow:
if (field.found or field.is_optional) and field.valid:
continue
this_field_issues = {}
if not field.found:
if not field.found and not field.is_optional:
this_field_issues["found"] = "Not found"
if field.search_issue:
this_field_issues["search_issue"] = field.search_issue
if not field.valid:
if not field.valid and field.valid is not None:
this_field_issues["validity"] = "Not valid"
this_field_issues["value"] = field.value

5
core/scrapping_utils.py
View file

@ -95,7 +95,10 @@ class UrlAttack:
except Exception as e:
self.success = False
if random.randrange(0, 100) < UrlAttack.identity_change_probability:
if (
not self.success
or random.randrange(0, 100) < UrlAttack.identity_change_probability
):
self._change_identity()
def _change_identity(self) -> None: