The problem of setting proxy in middlewares in scrapy

from fake_useragent import UserAgent


class RandomUserAgent(object):
    def __init__(self, crawler):
        super(RandomUserAgent, self).__init__()
        self.ua = UserAgent(verify_ssl=False)
        self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random")

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def process_request(self, request):
        def get_ua():
            return getattr(self.ua, self.ua_type)

        request.headers.setdefault("User_Agent", get_ua())


class RandomProxy(object):
    def __init__(self, ip):
        super(RandomProxy, self).__init__()
        self.iplist = ip

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            iplist=crawler.settings.get("PROXY")
        )

    def process_request(self, request, spider):
        proxy = self.ip
        request.meta["proxy"] = proxy

I always report an error when I write an agent on middleware in scrapy

uring handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/fake_useragent/utils.py", line 67, in get
    context=context,
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 526, in open
    response = self._open(req, data)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 544, in _open
    "_open", req)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 504, in _call_chain
    result = func(*args)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 1361, in https_open
    context=self._context, check_hostname=self._check_hostname)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 1320, in do_open
    raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 65] No route to host>
2018-10-09 14:52:25 [fake_useragent] DEBUG: Sleeping for 0.1 seconds

  self._send_output(message_body, encode_chunked=encode_chunked)

File "/ usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1026, in _ send_output

self.send(msg)

File "/ usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/htt

p/client.py", line 964, in send
    self.connect()
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1392, in connect
    super().connect()
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 936, in connect
    (self.host,self.port), self.timeout, self.source_address)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/socket.py", line 724, in create_connection
    raise err
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/socket.py", line 713, in create_connection
    sock.connect(sa)
OSError: [Errno 65] No route to host

Python

Aug.06,2021