from scrapy.http.headers import Headers
from Espider.tools.get_cookies import get_cookies
import pymongo
class united_middleware():
def __init__(self, mongodbHost, mongodbPort, mongodbName):
self.mongodbHost = mongodbHost
self.mongodbPort = mongodbPort
self.mongodbName = mongodbName
@classmethod
def from_crawler(cls, crawler):
return cls(mongodbHost=crawler.settings.get("MONGODB_HOST"), mongodbPort=crawler.settings.get("MONGODB_PORT"),
mongodbName=crawler.settings.get("MONGODB_DBNAME"))
def process_request(self, request, spider):
self.mongodb = pymongo.MongoClient(self.mongodbHost, self.mongodbPort)
self.totalcollection = self.mongodb[spider.name + "_cookie"]
self.cookies_str = list(self.totalcollection.find())
print("")
print(self.cookies_str)
self.headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"cookie": self.cookies_str}
if "finance?asEncryptedTs" in request.url or "member?asEncryptedTs" in request.url:
request.headers = Headers(self.headers)
else:
request.cookies = get_cookies(self.cookies_str)
request.headers.setdefault("User-Agent",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5")
Collection" object is not callable. If you meant to call the "find" method on a "Database" object it is failing because no such method exist??
python
topic description
sources of topics and their own ideas
related codes
/ / Please paste the code text below (do not replace the code with pictures)