this is the case now. I use mongodb in scrapy to remove duplicates
def process_item (self, item, spider):
url = item["file_url"]
name = item["name"]
result = self.post.aggregate(
[
{"$group": {"_id": {"url": url, "name": name}}}
]
)
if result:
pass
else:
self.post.insert({"url": url, "name": name})
return item
def file_path (self, request, response=None, info=None):
return request.meta.get("filename", "")
def get_media_requests (self, item, info):
file_url = item["file_url"]
meta = {"filename": item["name"]}
yield Request(url=file_url, meta=meta)how can I call the download after the rejudgment?