from scrapy.utils.project import get_project_settings
from Espider.spiders.chinafof import chinafof
from Espider.spiders.ifenxi import ifenxi
from Espider.spiders.vcbeat import vcbeat
from Espider.spiders.zdreview import zdreview
from scrapy.crawler import CrawlerProcess
from multiprocessing import Process
def run_crawl ():
process=CrawlerProcess(get_project_settings())
process.crawl(chinafof)
process.crawl(ifenxi)
process.crawl(vcbeat)
process.crawl(zdreview)
process.start()
if _ _ name__=="__main__":
[Process(target=run_crawl).start() for _ in range(4)]
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 86, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 122, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 85, in callWithContext
return func(*args,**kw)
--- <exception caught here> ---
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/posixbase.py", line 619, in _doReadOrWrite
why = selectable.doWrite()
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1186, in doWrite
"doWrite called on a %s" % reflect.qual(self.__class__))
builtins.RuntimeError: doWrite called on a twisted.internet.tcp.Port
2018-12-02 17:25:54 [twisted] CRITICAL: Unhandled Error
Traceback (most recent call last):
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 103, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 86, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 122, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 85, in callWithContext
return func(*args,**kw)
--- <exception caught here> ---
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/posixbase.py", line 619, in _doReadOrWrite
why = selectable.doWrite()
File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1186, in doWrite
"doWrite called on a %s" % reflect.qual(self.__class__))
builtins.RuntimeError: doWrite called on a twisted.internet.tcp.Port