has been distributed recently, but both yield and return can return data when crawling. Why?
ask for advice
tags.append (tag ["name"])
item["tags"] = ",".join(tags)
item["fullName"] = companyData["fullName"]
item["scale"] = companyData["scale"]
asTs, asEncryptedTs = execjs.compile(self.js_read).call("p")
kw_url = "https://rong.36kr.com/n/api/search/company?asEncryptedTs={}&asTs={}&kw={}&sortField=MATCH_RATE".format(
asEncryptedTs, asTs, item["name"])
retunr scrapy.Request(
url=kw_url,
callback=self.get_kw_info,
dont_filter=True,
meta={"id": response.meta["id"], "item": item},
)
def get_kw_info (self, response):
item = response.meta["item"]
pageData = json.loads(response.text)["data"]["pageData"]["data"]
industryStr = pageData[0]["industryStr"]
item["industryStr"] = industryStr
asTs, asEncryptedTs = execjs.compile(self.js_read).call("p")
finance_link = "https://rong.36kr.com/n/api/company/{}/finance?asEncryptedTs={}&asTs={}".format(
response.meta["id"], asEncryptedTs, asTs)
retun scrapy.Request(
url=finance_link,
callback=self.get_finance_info,
dont_filter=True,
meta={"id": response.meta["id"], "item": item})
does this code always return data from both yield and return? Why is that?