def process_item (self, item, spider):
url = item["file_url"]
name = item["name"]
result = self.post.aggregate(
[
{"$group": {"_id": {"url": url, "name": name}}}
]
)
if result:
pass
else:
self.post.insert({"url": url, "name": name})
return item
def file_path(self, request, response=None, info=None):
return request.meta.get("filename", "")
def get_media_requests(self, item, info):
file_url = item["file_url"]
meta = {"filename": item["name"]}
yield Request(url=file_url, meta=meta)
here I inherit a pipeline and use the key combination to determine whether to repeat it and then download it, otherwise I won"t download it. How to write this?