破解豆瓣Ajax动态加载：Python爬取完整长评论和短评

在互联网数据采集领域，动态加载内容一直是爬虫开发者需要应对的重要挑战。豆瓣作为中国知名的文化内容社区，其评论系统采用了Ajax动态加载技术，传统的简单爬虫难以获取完整数据。本文将深入分析豆瓣的Ajax加载机制，并提供完整的Python解决方案。

1. 豆瓣评论加载机制分析

豆瓣电影页面的评论系统采用了典型的"渐进式加载"设计。初始页面只包含少量评论，当用户滚动到页面底部时，会通过Ajax请求加载更多内容。这种设计不仅提升了页面初始加载速度，也为反爬虫提供了一定保护。

通过浏览器开发者工具分析网络请求，我们可以发现：

短评接口：**https://movie.douban.com/subject/{movie_id}/comments?start={start}&limit=20&status=P&sort=new_score**
长评接口：**https://movie.douban.com/subject/{movie_id}/reviews?start={start}**

这些接口返回的是结构化数据，相比解析HTML更容易提取信息。

2. 技术选型与环境准备

本项目主要使用以下Python库：

**requests**：发送HTTP请求
**json**：解析返回的JSON数据
**time**：添加请求延迟
**pandas**：数据存储和处理（可选）

3. 实现豆瓣评论爬虫

3.1 获取短评数据

短评接口返回的是HTML片段，我们需要从中提取数据：

import requests
from bs4 import BeautifulSoup
import time
import random
import json
import csv# 代理信息配置
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"# 构建代理字典
proxies = {"http": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}","https": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
}def get_short_comments(movie_id, max_count=200):"""获取豆瓣电影短评:param movie_id: 豆瓣电影ID:param max_count: 最大获取数量:return: 短评列表"""comments = []start = 0limit = 20# 请求头模拟浏览器行为headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8','Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2','Accept-Encoding': 'gzip, deflate, br','Connection': 'keep-alive','Upgrade-Insecure-Requests': '1',}while start < max_count:url = f'https://movie.douban.com/subject/{movie_id}/comments'params = {'start': start,'limit': limit,'status': 'P','sort': 'new_score'}try:# 添加代理参数response = requests.get(url, params=params, headers=headers, proxies=proxies,  # 添加代理timeout=10)if response.status_code != 200:print(f"请求失败，状态码：{response.status_code}")breaksoup = BeautifulSoup(response.text, 'html.parser')comment_items = soup.select('.comment-item')if not comment_items:print("未找到评论内容，可能已获取所有评论或遇到反爬虫")breakfor item in comment_items:try:# 提取用户信息user = item.select_one('.comment-info a')['title']# 提取评分rating_class = item.select_one('.comment-info .rating')rating = 0if rating_class:rating = int(rating_class['class'][0][-2])/10# 提取评论时间comment_time = item.select_one('.comment-time')['title']# 提取评论内容content = item.select_one('.comment-content').text.strip()comments.append({'user': user,'rating': rating,'time': comment_time,'content': content})except Exception as e:print(f"解析单条评论时出错：{e}")continueprint(f"已获取 {len(comments)} 条短评")# 随机延迟，避免请求过于频繁time.sleep(random.uniform(1, 2))start += limitexcept requests.exceptions.ProxyError as e:print(f"代理连接错误：{e}")breakexcept requests.exceptions.ConnectTimeout as e:print(f"连接超时：{e}")time.sleep(5)  # 超时后等待更长时间continueexcept requests.exceptions.RequestException as e:print(f"网络请求异常：{e}")time.sleep(3)continueexcept Exception as e:print(f"获取短评时出错：{e}")breakreturn comments# 其他函数也需要添加代理支持
def get_long_comments(movie_id, max_count=100):"""获取豆瓣电影长评（需要添加代理支持）"""# 实现代码与get_short_comments类似，需要添加proxies参数passdef get_full_review(review_url):"""获取完整长评内容（需要添加代理支持）"""# 实现代码需要添加proxies参数pass# 使用示例
if __name__ == "__main__":# 测试代理连接try:test_response = requests.get("http://httpbin.org/ip", proxies=proxies, timeout=10)print("代理连接测试成功")print(f"当前代理IP: {test_response.json()['origin']}")except Exception as e:print(f"代理连接测试失败: {e}")# 获取短评movie_id = "1292052"  # 肖申克的救赎comments = get_short_comments(movie_id, max_count=100)print(f"成功获取 {len(comments)} 条评论")

3.2 获取长评数据

长评接口返回的是JSON格式数据，处理起来更加方便：

def get_long_comments(movie_id, max_count=100):"""获取豆瓣电影长评:param movie_id: 豆瓣电影ID:param max_count: 最大获取数量:return: 长评列表"""reviews = []start = 0headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36','Accept': 'application/json, text/plain, */*','Referer': f'https://movie.douban.com/subject/{movie_id}/','X-Requested-With': 'XMLHttpRequest',}while start < max_count:url = f'https://movie.douban.com/subject/{movie_id}/reviews'params = {'start': start}try:response = requests.get(url, params=params, headers=headers, timeout=10)if response.status_code != 200:print(f"请求失败，状态码：{response.status_code}")break# 豆瓣长评页面返回的是HTML，需要解析soup = BeautifulSoup(response.text, 'html.parser')review_items = soup.select('.review-item')if not review_items:print("未找到长评内容，可能已获取所有长评或遇到反爬虫")breakfor item in review_items:try:# 提取标题title = item.select_one('.title-link').text.strip()# 提取作者author = item.select_one('.name').text.strip()# 提取评分rating_ele = item.select_one('.main-title-rating')rating = 0if rating_ele:rating_class = rating_ele['class'][1]rating = int(rating_class[-2])/10# 提取发布时间pub_time = item.select_one('.main-meta').text.strip()# 提取内容摘要content_short = item.select_one('.short-content').text.strip()# 获取完整长评需要进入详情页review_link = item.select_one('.title-link')['href']full_content = get_full_review(review_link)reviews.append({'title': title,'author': author,'rating': rating,'time': pub_time,'content_short': content_short,'content_full': full_content,'link': review_link})except Exception as e:print(f"解析单条长评时出错：{e}")continueprint(f"已获取 {len(reviews)} 条长评")# 随机延迟time.sleep(random.uniform(1, 3))start += len(review_items)except Exception as e:print(f"获取长评时出错：{e}")breakreturn reviewsdef get_full_review(review_url):"""获取完整长评内容:param review_url: 长评详情页URL:return: 完整内容"""headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',}try:response = requests.get(review_url, headers=headers, timeout=10)if response.status_code != 200:return "获取失败"soup = BeautifulSoup(response.text, 'html.parser')# 尝试查找长评内容content = soup.select_one('.review-content')if content:# 移除无关元素for elem in content.select('.spoiler-tip, .hidden'):elem.decompose()return content.text.strip()return "内容解析失败"except Exception as e:print(f"获取完整长评内容时出错：{e}")return "请求失败"

3.3 数据存储功能

def save_to_csv(data, filename):"""将数据保存为CSV文件:param data: 数据列表:param filename: 文件名"""if not data:print("无数据可保存")returnkeys = data[0].keys()with open(filename, 'w', newline='', encoding='utf-8-sig') as f:writer = csv.DictWriter(f, fieldnames=keys)writer.writeheader()writer.writerows(data)print(f"数据已保存至 {filename}")def save_to_json(data, filename):"""将数据保存为JSON文件:param data: 数据列表:param filename: 文件名"""with open(filename, 'w', encoding='utf-8') as f:json.dump(data, f, ensure_ascii=False, indent=4)print(f"数据已保存至 {filename}")