counterweight/drogon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Actualizados headers de los ataques a URL por cambios en idealsita.

Browse source
This commit is contained in:
pablomartincalvo 2018-12-30 19:28:05 +01:00
parent ed32b15bc1
commit 38984822a7
2 changed files with 11 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

12
core/scrapping_utils.py
View file

@ -3,12 +3,16 @@ import requests
class UrlAttack(): class UrlAttack():
headers = {'Upgrade-Insecure-Requests': "1", headers = {'Upgrade-Insecure-Requests': "1",
'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
'Accept-Encoding': "gzip, deflate, br", 'Accept-Encoding': "gzip, deflate, br",
'Accept-Language': "en-US,en;q=0.9", 'Accept-Language': "en-US,en;q=0.5",
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/67.0.3396.99 Chrome/67.0.3396.99 Safari/537.36'} 'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Host': 'www.idealista.com',
'Pragma': 'no-cache',
'user-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'}
timeout = 20 timeout = 20

6
tests/capturer_tests.py
View file

@ -7,7 +7,7 @@ from db_layer.capturas_interface import capturas_interface
def test_CapturingTask(): def test_CapturingTask():
parameters = {'uuid': 'testie test', parameters = {'uuid': 'testie test',
'ad_url': 'https://www.idealista.com/inmueble/39627481', 'ad_url': 'https://www.idealista.com/inmueble/28252032',
'fk_uuid_exploring': None, 'fk_uuid_exploring': None,
'status': 'Pending'} 'status': 'Pending'}
@ -225,8 +225,8 @@ var configTwoSteps = {
parser._validate() parser._validate()
test_AdHtmlParser() #test_AdHtmlParser()
#test_CapturingTask() test_CapturingTask()
#test_Capturer() #test_Capturer()