I crawled a website, the data is to send an asynchronous request load belt to the server, I imitated headers, parameters are not wrong, using requests can get a normal response, when scrapy is not good
def parse_histical_data(self, response):
html = BeautifulSoup(response.body, "lxml")
patterm = re.compile(r"smlId: [0-9]*", re.MULTILINE|re.UNICODE)
script = html.find("script", text=patterm).text
smlId_text = patterm.search(script).group()
smlId = smlId_text.split(" ")[1]
curr_id = response.meta["pair_id"]
header=html.select("-sharpleftColumn > div.instrumentHeader > h2")[0].string
st_date = "01/01/2001"
end_date = "05/07/2050"
interval_sec = "Daily"
sort_col = "date"
sort_ord = "DESC"
action = "historical_data"
data = {"smlID": smlId, "curr_id": curr_id, "header": header, "st_date": st_date, "end_state": end_date,
"interval_sec": interval_sec, "sort_col": sort_col, "sort_ord": sort_ord, "action": action}
head = self.download_headers.copy()
request = FormRequest(self.his_url, callback=self.parse_histical_data,
headers=head, formdata=data)
yield request
request with URL" https://www.investing.com/ins.", and return 400 with headers and data,scrapy exactly the same.