-sharp!/usr/bin/python3
import sys
from scrapy.cmdline import execute
class main(): def run(self): argv = sys.argv argv.append("-s") argv.append("LOG_FILE=logs/" + argv[2] + ".log") argv.append("-s") argv.append("LOG_FORMAT=%(asctime)s [" + argv[2] + "][%(name)s] %(levelname)s: %(message)s") print(argv[3]) print(argv) sys.exit(execute(argv))
if name == "main": main().run()
main
supervisor
-sharp!/usr/bin/python3 import sys import datetime from main import main import time from scrapy.cmdline import execute
if name == "main": argv = sys.argv while 1: startTime = time.time() main().run() endTime = time.time() time.sleep(argv[3]) if argv[3] < endTime - startTime: logerror_path = "logs/logerror.log"
this is a supervisor class, which calls the main class, but here I run the following
shenjianlin@newdev:~/my_project/Espider/Espider(shenwei/spider_12_20)$ python3 supervisor.py crawl traveldaily ["supervisor.py", "crawl", "traveldaily", "-s", "LOG_FILE=logs/traveldaily.log", "-s", "LOG_FORMAT=%(asctime)s [traveldaily][%(name)s] %(levelname)s: %(message)s"] shenjianlin@newdev:~/my_project/Espider/Espider(shenwei/spider_12_20)$ python3 supervisor.py crawl traveldaily 120 ["supervisor.py", "crawl", "traveldaily", "120", "-s", "LOG_FILE=logs/traveldaily.log", "-s", "LOG_FORMAT=%(asctime)s [traveldaily][%(name)s] %(levelname)s: %(message)s"] Usage
scrapy crawl [options] <spider></spider>
crawl: error: running "scrapy crawl" with more than one spider is no longer supported shenjianlin@newdev:~/my_project/Espider/Espider(shenwei/spider_12_20)$ python3 supervisor.py crawl traveldaily 120 120 ["supervisor.py", "crawl", "traveldaily", "120", "-s", "LOG_FILE=logs/traveldaily.log", "-s", "LOG_FORMAT=%(asctime)s [traveldaily][%(name)s] %(levelname)s: %(message)s"] Usage
scrapy crawl [options] <spider></spider>
crawl: error: running "scrapy crawl" with more than one spider is no longer supported
main sleep -sharp-sharp-sharp