pip install -U fake-useragent

• 如果以上方法不成功，则需要fake_useragent_0.1.11.json文件来处理了
• 参考[820]fake_useragent.errors.FakeUserAgentError: Maximum amount of retries reached_周小董-CSDN博客
• 下载链接UA.rarfake_useragent解决报错文件-Proxy文档类资源-CSDN下载
• 将fake_useragent_0.1.11.json文件复制到项目中,使用UserAgent类进行读取

• 
  from fake_useragent import UserAgent
  
  
  # 配置headers
  class RandomUserAgentMidddlware(object):
      # 随机更换user-agent
      def __init__(self, crawler):
          super(RandomUserAgentMidddlware, self).__init__()
          self.ua = UserAgent(path="spiders/fake_useragent_0.1.11.json")
          # 从配置文件读取随机类型
          self.ua_type = crawler.settings.get('RANDOM_UA_TYPE', 'random')
  
      @classmethod
      def from_crawler(cls, crawler):
          return cls(crawler)
  
      def process_request(self, request, spider):
          # 通过配置文件的随机类型进行调用
          def get_ua():
              return getattr(self.ua, self.ua_type)
  
          request.headers.setdefault('User-Agent', get_ua())