引言
在爬虫技术领域，处理动态加载的网页内容常常是一项挑战，尤其是对于那些通过用户滚动或其他交互动态加载更多内容的网站。本文将介绍如何结合使用Selenium和Scrapy来有效处理这类网页。

初探Selenium与Scrapy的结合
首先，我们探索如何使用Selenium在Scrapy中间件中处理动态加载内容的网页。关键在于模拟用户滚动行为，以加载并捕获所有内容。

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
from scrapy.http import HtmlResponse
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import random,time
from fake_useragent import UserAgent
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
from scrapy.utils.project import get_project_settings


class SeleniumMiddleware:
    @classmethod
    def from_crawler(cls, crawler):
        middleware = cls()
        crawler.signals.connect(middleware.spider_closed, signal=signals.spider_closed)
        return middleware

    def __init__(self):
        options = Options()
        # options.add_argument('--headless')  # 启用无头模式
        # options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36')
        # 创建UserAgent对象
        # ua = UserAgent()
        # settings=get_project_settings() #获取settings配置，设置需要的信息
        # 生成随机User-Agent 没有用这里是因为这里有可能会产生手机端的访问方式
        # user_agent = ua.random
        # user_agent = random.choice(settings["USER_AGENTS"])
        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
        # print("user_agent:",user_agent)
        options.add_argument('--disable-blink-features=AutomationControlled')#关闭自动控制blink特征
        options.add_argument(f"user-agent={user_agent}")
        options.add_experimental_option('excludeSwitches', ['enable-automation'])
        self.driver = webdriver.Chrome(options=options)

    def spider_closed(self, spider):
        self.driver.quit()

    def process_request(self, request, spider):
        self.driver.get(request.url)
        # 等待页面初步加载完成
        time.sleep(3)  # 示例等待时间，可能需要根据实际页面调整
        # 找到以游客模式进入的按钮
        try:
            element = self.driver.find_element(By.ID,"loginContainer")
            child_element = self.driver.find_elements(By.CLASS_NAME,'css-txolmk-DivGuestModeContainer')
            child_element[0].click()
        except Exception as e:
            print("以游客模式进入的按钮没有找到")

        time.sleep(3)
        try:
            flush_element = self.driver.find_elements(By.CLASS_NAME,'css-z9i4la-Button-StyledButton')
            flush_element[0].click()
        except Exception as e:
            print("刷新按钮没有找到")
        time.sleep(6)
        
        xunhuan = True
        temp_height = 0
        while xunhuan:
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(2)
            # 获取当前滚动条距离顶部的距离
            check_height = self.driver.execute_script("return document.documentElement.scrollTop || window.pageYOffset || document.body.scrollTop;")
            if check_height == temp_height:
                print("到底了")
                xunhuan = False
            else:
                temp_height = check_height
      

        body = self.driver.page_source.encode('utf-8')
        
        return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)
    
 

完整的Scrapy爬虫实例
下面是一个使用Selenium和Scrapy爬取网易新闻的示例。

import scrapy,json,html,base64
import datetime,hashlib
from tiktokSelenium.items import TiktokseleniumItem

class TiktokSpider(scrapy.Spider):
    name = "tiktok"
    # allowed_domains = ["httpbin.org"]
    # start_urls = ["https://httpbin.org/user-agent"]
    allowed_domains = ["tiktok.com"]
    start_urls = [
        "https://www.tiktok.com/@espn"
    ]

    def __init__(self):
        # settings=get_project_settings() #获取settings配置，设置需要的信息
        # self.tik_accounts = settings['TIK_ACCOUNTS']
        # 获取今天的爬取
        # self.fenrundate = time.strftime("%Y%m%d")
        # 获取今天的爬取
        self.fenrundate = self.get_yesterday_day()

    def parse(self, response):
        print("==================response.text=======================")
        # print(response.text)
        print(len(response.xpath('//div[@class="css-x6y88p-DivItemContainerV2 e19c29qe8"]')))
        author_url = response.url
        author_name = author_url.split("@")[-1]
        
        for sel in response.xpath('//div[@class="css-x6y88p-DivItemContainerV2 e19c29qe8"]'):
            link = sel.xpath('div/div/div/a')[0]
            # 获取视频的链接地址
            href = sel.xpath('div/div/div/a/@href').extract_first()
            # 视频id
            vid = href.split("/")[-1]
            vclick = link.xpath('div/div/strong[@class="video-count css-dirst9-StrongVideoCount e148ts222"]/text()').extract_first()
            # vclick = videoCount[0].text
            title = html.escape(sel.xpath('div[2]/div/@aria-label').extract_first())
            # continue
            uqc_arr = [title,vclick]
            cvideo_item = TiktokseleniumItem()
            # cvideo_item = {}
            cvideo_item['author_url'] = author_url
            cvideo_item['author_name'] = author_name
            cvideo_item['video_id'] = vid
            cvideo_item['video_url'] = href
            cvideo_item['video_title'] = title
            cvideo_item['video_hits'] = vclick
            
            cvideo_item['date'] = self.fenrundate
            cvideo_item['video_real_hits'] = self.convert_to_real_hits(vclick)
            # print(cvideo_item)
            cvideo_item['unique_key'] = self.str_md5("_".join(uqc_arr))
            yield cvideo_item     
            # print(cvideo_item)

    # 获取昨天的日期 
    def get_yesterday_day(self):
        today = datetime.date.today()
        yesterday = today - datetime.timedelta(days=1)
        yesterday2 = str(yesterday)
        return yesterday2.replace("-","")

    # 点击量转化为数字
    def convert_to_real_hits(self,strs):
        lastItem = strs[-1]
        
        if lastItem in ['K','M','B','k','m','b']:
            strs = eval(strs[0:-1])
            
            if lastItem == 'K' or lastItem == 'k':
                strs = strs * 1000
            elif lastItem == 'M' or lastItem == 'm':
                strs = strs * 1000 * 1000
            else:
                strs = strs * 1000 * 1000 * 1000
        return int(strs)
    
    def str_md5(self,strs):
        m = hashlib.md5()
        bs = base64.b64encode(strs.encode("utf-8"))
        m.update(bs)
        str_md5 = m.hexdigest()
        return str_md5