Scrapy - 在下载器中间件中设置随机User-Agent

项目目录

• 在middlewares.py中
• 编写随机User-Agent的逻辑

import random


class RandomUserAgent:
    def get_ua(self):
        a = random.randint(55, 62)
        c = random.randint(0, 3200)
        d = random.randint(0, 150)
        os_type = [
            '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)',
            '(Macintosh; Intel Mac OS X 10_12_6)'
        ]
        chrome_version = f'Chrome/{a}.0.{c}.{d}'
        ua = ' '.join(
            ['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36', '(KHTML, like Gecko)', chrome_version,
             'Safari/537.36'])
        return ua

    def process_request(self, request, spider):
        request.headers['User-Agent'] = self.get_ua()

• 在settings.py中
• 开启上面写的中间件

DOWNLOADER_MIDDLEWARES = {
    'scrapy_test.middlewares.RandomUserAgent': 100,
}