counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Modified input format of instructions for ParsingFlowGenerator. Previous dict wouldn't allow for more than one SecondaryFeaturesFieldInstructions class pointer.

Browse source
This commit is contained in:
pablo 2020-12-31 19:02:09 +01:00
parent 2b249063e0
commit def858ef6a
2 changed files with 19 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

15
core/parsing_utils.py
View file

@ -1,4 +1,4 @@
from typing import Union, Iterable, Dict, Callable, Type from typing import Union, Iterable, Dict, Callable, Type, Tuple
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
@ -563,18 +563,23 @@ class ParsingFlowGenerator:
def __init__( def __init__(
self, self,
parsing_flow_class: Type[ParsingFlow], parsing_flow_class: Type[ParsingFlow],
instructions_to_attach_with_params: Dict[ instructions_to_attach_with_params: Union[
Type[BaseTargetFieldInstructions], dict Tuple[Type[BaseTargetFieldInstructions], Dict],
Tuple[Tuple[Type[BaseTargetFieldInstructions], Dict]],
], ],
) -> None: ) -> None:
""" """
Set the flow class and group of instructions to use when creating new Set the flow class and group of instructions to use when creating new
instances of the flow class. instances of the flow class.
:param parsing_flow_class: the flow class to instantiate :param parsing_flow_class: the flow class to instantiate
:param instructions_to_attach_with_params: a key-value pair of field :param instructions_to_attach_with_params: one or more pair of field
instructions class and the paramteres to use when instantiating them instructions class and the paramteres to use when instantiating them
""" """
self._parsing_flow_class = parsing_flow_class self._parsing_flow_class = parsing_flow_class
if not isinstance(instructions_to_attach_with_params, tuple):
instructions_to_attach_with_params = tuple(
instructions_to_attach_with_params
)
self._instructions_to_attach_with_params = instructions_to_attach_with_params self._instructions_to_attach_with_params = instructions_to_attach_with_params
def get_new_flow(self) -> ParsingFlow: def get_new_flow(self) -> ParsingFlow:
@ -584,7 +589,7 @@ class ParsingFlowGenerator:
""" """
new_parsing_flow = self._parsing_flow_class() new_parsing_flow = self._parsing_flow_class()
for instruction, params in self._instructions_to_attach_with_params.items(): for instruction, params in self._instructions_to_attach_with_params:
new_parsing_flow.add_instructions(instruction(**params)) new_parsing_flow.add_instructions(instruction(**params))
return new_parsing_flow return new_parsing_flow

18
tests/parsing_utils_test.py
View file

@ -2730,15 +2730,15 @@ def test_parsing_flow_fails_for_unrelated_html(unrelated_html):
def test_parsing_flow_generator_returns_proper_flows(): def test_parsing_flow_generator_returns_proper_flows():
four_instructions_with_params = { four_instructions_with_params = (
ReferenciaFieldInstructions: {}, (ReferenciaFieldInstructions, {}),
PrecioFieldInstructions: {}, (PrecioFieldInstructions, {}),
TamanoCategoricoFieldInstructions: {}, (TamanoCategoricoFieldInstructions, {}),
SecondaryFeaturesFieldInstructions: { (
"field_name": "personal", SecondaryFeaturesFieldInstructions,
"search_keyword": "Personal", {"field_name": "personal", "search_keyword": "Personal"},
}, ),
} )
parsing_flow_generator = ParsingFlowGenerator( parsing_flow_generator = ParsingFlowGenerator(
parsing_flow_class=ParsingFlow, parsing_flow_class=ParsingFlow,