turbohung/vibecoding
1
1"""2抖音视频解析核心模块 (Playwright版本)3使用浏览器自动化绕过反爬，支持短链接和长链接4"""5 6import re7import json8import asyncio9from typing import Optional, Dict, List, Tuple10from urllib.parse import urlparse, parse_qs11from playwright.async_api import async_playwright12 13# ============================================================14# 正则模式 — 覆盖所有已知抖音链接格式15# ============================================================16SHORT_PATTERN = re.compile(r'v\.douyin\.com/([A-Za-z0-9_\-]+)/?')17LONG_PATTERN = re.compile(r'(?:www\.)?douyin\.com/video/(\d+)')18NOTE_PATTERN = re.compile(r'(?:www\.)?douyin\.com/note/(\d+)')19USER_VIDEO_PATTERN = re.compile(r'(?:www\.)?douyin\.com/user/\S*?video/(\d+)')20USER_MODAL_PATTERN = re.compile(r'(?:www\.)?douyin\.com/user/\S*\?.*modal_id=(\d+)')21SHARE_PATTERN = re.compile(r'(?:www\.)?douyin\.com/share/video/(\d+)')22IES_SHARE_PATTERN = re.compile(r'(?:www\.)?iesdouyin\.com/share/video/(\d+)')23SEARCH_MODAL_PATTERN = re.compile(r'douyin\.com/search/\S*\?.*modal_id=(\d+)')24DISCOVER_MODAL_PATTERN = re.compile(r'douyin\.com/discover\?.*modal_id=(\d+)')25FOLLOW_PATTERN = re.compile(r'douyin\.com/follow\?.*modal_id=(\d+)')26RAW_VIDEO_ID_PATTERN = re.compile(r'^(\d{15,20})$')27RENDER_DATA_PATTERN = re.compile(28 r'<script\s+id="RENDER_DATA"\s+type="application/json">(.*?)</script>',29 re.DOTALL30)31 32# 所有提取ID的正则（按优先级排列）33ALL_ID_PATTERNS: List[Tuple[str, re.Pattern]] = [34 ('long_video', LONG_PATTERN),35 ('note', NOTE_PATTERN),36 ('ies_share', IES_SHARE_PATTERN),37 ('share', SHARE_PATTERN),38 ('user_video', USER_VIDEO_PATTERN),39 ('user_modal', USER_MODAL_PATTERN),40 ('search_modal', SEARCH_MODAL_PATTERN),41 ('discover_modal', DISCOVER_MODAL_PATTERN),42 ('follow_modal', FOLLOW_PATTERN),43]44 45# 链接类型中文提示46LINK_TYPE_LABELS = {47 'short': '短链接（需浏览器跟随重定向）',48 'long_video': '视频链接',49 'note': '图文笔记',50 'share': '分享链接',51 'ies_share': '企业号分享链接',52 'user_video': '用户主页视频',53 'user_modal': '用户主页弹窗',54 'search_modal': '搜索结果弹窗',55 'discover_modal': '发现页弹窗',56 'follow_modal': '关注页弹窗',57 'modal_page': '精选/发现页（modal_id）',58 'raw_id': '裸视频ID',59 'unknown': '无法识别的格式',60}61 62# 查询参数中可能包含视频ID的key63VIDEO_ID_QUERY_KEYS = ['video_id', 'aweme_id', 'item_id', 'modal_id', 'id']64 65# 从混合文本中提取抖音URL66# 匹配带或不带 http 前缀的douyin链接，含路径中的特殊字符67DOUYIN_URL_IN_TEXT = re.compile(68 r'(?:https?://)?(?:v\.|www\.)?(?:ies)?douyin\.com/[A-Za-z0-9/?=&_\-\.%~@!#]+'69)70# 纯文本提取（不用 http 前缀）71DOUYIN_URL_BARE = re.compile(72 r'((?:v\.|www\.)?(?:ies)?douyin\.com/[A-Za-z0-9/?=&_\-\.%~@!#]+)'73)74# 检测是否包含抖音域名（用于兜底验证）75DOUYIN_DOMAIN = re.compile(r'(?:v\.|www\.)?(?:ies)?douyin\.com')76 77 78class DouyinParser:79 """抖音链接解析器 — 基于Playwright浏览器自动化"""80 81 # ---- 类属性（外部可引用） ----82 SHORT_PATTERN = SHORT_PATTERN83 LONG_PATTERN = LONG_PATTERN84 NOTE_PATTERN = NOTE_PATTERN85 USER_VIDEO_PATTERN = USER_VIDEO_PATTERN86 USER_MODAL_PATTERN = USER_MODAL_PATTERN87 SHARE_PATTERN = SHARE_PATTERN88 IES_SHARE_PATTERN = IES_SHARE_PATTERN89 RAW_VIDEO_ID_PATTERN = RAW_VIDEO_ID_PATTERN90 91 # ================================================================92 # 浏览器生命周期93 # ================================================================94 95 @classmethod96 async def _launch_browser(cls):97 """每次请求独立启动浏览器（避免事件循环冲突）"""98 playwright = await async_playwright().start()99 browser = await playwright.chromium.launch(100 headless=True,101 args=[102 '--no-sandbox',103 '--disable-setuid-sandbox',104 '--disable-blink-features=AutomationControlled',105 ],106 )107 return playwright, browser108 109 @classmethod110 async def _launch_context(cls, playwright, browser):111 """创建带反检测的浏览器上下文"""112 context = await browser.new_context(113 user_agent=(114 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '115 'AppleWebKit/537.36 (KHTML, like Gecko) '116 'Chrome/131.0.0.0 Safari/537.36'117 ),118 viewport={'width': 1440, 'height': 900},119 locale='zh-CN',120 extra_http_headers={121 'Accept-Language': 'zh-CN,zh;q=0.9',122 },123 )124 # Hide automation traces125 await context.add_init_script('''126 Object.defineProperty(navigator, 'webdriver', {get: () => false});127 Object.defineProperty(navigator, 'plugins', {get: () => [1,2,3,4,5]});128 Object.defineProperty(navigator, 'languages', {get: () => ['zh-CN','zh','en']});129 window.chrome = {runtime: {}};130 ''')131 return context132 133 # ================================================================134 # 链接类型检测 & ID提取135 # ================================================================136 137 @classmethod138 def extract_douyin_url(cls, text: str) -> Optional[str]:139 """140 从混合文本中提取抖音链接。141 142 处理从抖音App复制的原始分享文本，如：143 "0.79 o@d.Ag 05/14 DUY:/ # 大模型 https://v.douyin.com/xxx/ 复制此链接..."144 145 返回干净链接，没有则返回 None。146 """147 text = text.strip()148 149 # 如果整个文本就是一个 URL，直接返回150 if text.startswith('http://') or text.startswith('https://'):151 url = text.split()[0] if ' ' in text else text152 url = url.rstrip('.,;:!?。，；：！？)')153 if 'douyin.com' in url or 'iesdouyin.com' in url:154 return url155 return None156 157 # 处理裸域名（v.douyin.com/xxx 无协议前缀）158 if text.startswith('v.douyin.com/') or text.startswith('douyin.com/') or text.startswith('www.douyin.com/'):159 url = 'https://' + text.split()[0].rstrip('.,;:!?。，；：！？)')160 return url161 162 # 从文本中提取抖音 URL163 urls = DOUYIN_URL_IN_TEXT.findall(text)164 if not urls:165 return None166 167 # 返回第一个匹配（大概率是要看的那个）168 url = urls[0].rstrip('.,;:!?。，；：！？)')169 return url170 171 @classmethod172 def detect_link_type(cls, url: str) -> str:173 """174 检测链接类型，返回类型标识符。175 176 支持的类型：177 short — v.douyin.com 短链178 long_video — douyin.com/video/123179 note — douyin.com/note/123180 share — douyin.com/share/video/123181 ies_share — iesdouyin.com/share/video/123182 user_video — douyin.com/user/xxx/video/123183 user_modal — douyin.com/user/xxx?modal_id=123184 modal_page — jingxuan/discover/search?modal_id=123185 raw_id — 纯数字视频ID186 unknown — 无法识别187 """188 url = url.strip()189 190 # 从混合文本中提取URL（处理从App复制的分享文案）191 if not url.startswith('http') and not url.startswith('v.douyin'):192 extracted = cls.extract_douyin_url(url)193 if extracted:194 url = extracted195 196 if RAW_VIDEO_ID_PATTERN.match(url):197 return 'raw_id'198 199 if SHORT_PATTERN.search(url):200 return 'short'201 202 for link_type, pattern in ALL_ID_PATTERNS:203 if pattern.search(url):204 return link_type205 206 # 查询参数兜底（jingxuan / discover / search 等）207 try:208 # 确保有 scheme 才能 urlparse209 if '://' not in url:210 url = 'https://' + url211 parsed = urlparse(url)212 params = parse_qs(parsed.query)213 for key in VIDEO_ID_QUERY_KEYS:214 if key in params and params[key][0].isdigit():215 return 'modal_page'216 except Exception:217 pass218 219 # 最后尝试：是否包含抖音域名（可能是未知的新链接格式，但仍可尝试解析）220 if DOUYIN_DOMAIN.search(url):221 url = url if '://' in url else 'https://' + url222 # 尝试跟随看看能不能得到video_id223 return 'short' # 当作短链接处理，尝试跟随重定向224 225 return 'unknown'226 227 @classmethod228 def link_type_label(cls, link_type: str) -> str:229 """链接类型 → 中文描述"""230 return LINK_TYPE_LABELS.get(link_type, link_type)231 232 @classmethod233 def extract_video_id(cls, url: str) -> Optional[str]:234 """235 从各种格式的抖音链接中提取视频ID（同步，不解析短链）。236 237 支持：238 - 混合文本（如: "复制打开抖音 https://v.douyin.com/xxx/ ..."）239 - 裸ID：7628983823787627689240 - douyin.com/video/{id}241 - douyin.com/note/{id}242 - douyin.com/share/video/{id}243 - iesdouyin.com/share/video/{id}244 - douyin.com/user/xxx/video/{id}245 - douyin.com/user/xxx?modal_id={id}246 - douyin.com/jingxuan?modal_id={id}247 - URL查询参数 video_id / aweme_id / item_id / modal_id248 """249 url = url.strip()250 251 # 混合文本 → 先提取URL252 if not url.startswith('http') and not RAW_VIDEO_ID_PATTERN.match(url):253 extracted = cls.extract_douyin_url(url)254 if extracted:255 url = extracted256 257 # 裸视频ID（纯数字）258 m = RAW_VIDEO_ID_PATTERN.match(url)259 if m:260 return m.group(1)261 262 # 正则匹配263 for _type, pattern in ALL_ID_PATTERNS:264 match = pattern.search(url)265 if match:266 return match.group(1)267 268 # URL 查询参数兜底269 try:270 parsed = urlparse(url)271 params = parse_qs(parsed.query)272 for key in VIDEO_ID_QUERY_KEYS:273 if key in params:274 val = params[key][0]275 if val.isdigit():276 return val277 except Exception:278 pass279 280 return None281 282 @classmethod283 async def parse_url(cls, url: str) -> Optional[str]:284 """285 统一入口：自动检测链接类型 + 提取视频ID。286 - 混合文本自动识别（如从App复制的分享文案）287 - 短链自动用 Playwright 跟随重定向288 返回 video_id，解析失败返回 None。289 """290 url = url.strip()291 292 # 如果不是纯URL/纯ID → 从混合文本中提取抖音链接293 if not url.startswith('http') and not url.startswith('v.douyin') and not RAW_VIDEO_ID_PATTERN.match(url):294 extracted = cls.extract_douyin_url(url)295 if extracted:296 url = extracted297 298 # 确保有 scheme（裸域名补 https://）299 if DOUYIN_DOMAIN.search(url) and '://' not in url:300 url = 'https://' + url301 302 link_type = cls.detect_link_type(url)303 304 if link_type == 'raw_id':305 return url306 307 if link_type == 'short':308 return await cls._resolve_short_link(url)309 310 return cls.extract_video_id(url)311 312 @classmethod313 async def _resolve_short_link(cls, url: str) -> Optional[str]:314 """Playwright 跟随短链重定向，提取 video_id。兜底：从页面 RENDER_DATA 提取。"""315 pw, browser = await cls._launch_browser()316 try:317 context = await browser.new_context(318 user_agent=(319 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) '320 'AppleWebKit/605.1.15 (KHTML, like Gecko) '321 'Version/16.0 Mobile/15E148 Safari/604.1'322 ),323 viewport={'width': 390, 'height': 844},324 locale='zh-CN',325 )326 page = await context.new_page()327 328 video_id = None329 330 async def on_response(resp):331 nonlocal video_id332 if resp.status in (301, 302, 200, 307, 308):333 final_url = resp.url334 for _type, pattern in ALL_ID_PATTERNS:335 match = pattern.search(final_url)336 if match:337 video_id = match.group(1)338 return339 # 也检查查询参数340 try:341 from urllib.parse import urlparse as up, parse_qs as pq342 params = pq(up(final_url).query)343 for key in VIDEO_ID_QUERY_KEYS:344 if key in params and params[key][0].isdigit():345 video_id = params[key][0]346 return347 except Exception:348 pass349 350 page.on('response', on_response)351 await page.goto(url, wait_until='domcontentloaded', timeout=20000)352 await page.wait_for_timeout(3000)353 354 # 如果响应拦截没拿到，尝试从最终 URL 提取355 if not video_id:356 final_url = page.url357 for _type, pattern in ALL_ID_PATTERNS:358 match = pattern.search(final_url)359 if match:360 video_id = match.group(1)361 break362 # 尝试从查询参数提取363 if not video_id:364 try:365 from urllib.parse import urlparse as up, parse_qs as pq366 params = pq(up(final_url).query)367 for key in VIDEO_ID_QUERY_KEYS:368 if key in params and params[key][0].isdigit():369 video_id = params[key][0]370 break371 except Exception:372 pass373 374 # 兜底：从页面 RENDER_DATA 提取375 if not video_id:376 try:377 content = await page.content()378 match = RENDER_DATA_PATTERN.search(content)379 if match:380 import json381 data = json.loads(match.group(1))382 video_id = cls._extract_id_from_render_data(data)383 except Exception:384 pass385 386 await context.close()387 finally:388 await browser.close()389 await pw.stop()390 391 return video_id392 393 @classmethod394 def _extract_id_from_render_data(cls, data: dict) -> Optional[str]:395 """从 RENDER_DATA JSON 中递归提取 aweme_id / video_id。"""396 if isinstance(data, dict):397 for key in ('aweme_id', 'video_id', 'item_id', 'modal_id'):398 val = data.get(key)399 if val and isinstance(val, (int, str)) and str(val).isdigit():400 return str(val)401 for v in data.values():402 result = cls._extract_id_from_render_data(v)403 if result:404 return result405 elif isinstance(data, list):406 for item in data:407 result = cls._extract_id_from_render_data(item)408 if result:409 return result410 return None411 412 # 旧接口兼容413 resolve_short_link = _resolve_short_link414 415 # ================================================================416 # 视频信息获取417 # ================================================================418 419 @classmethod420 async def get_video_info(cls, video_id: str) -> Optional[Dict]:421 """422 通过Playwright浏览器获取视频信息。423 利用页面自身JS发起的API请求获取数据（绕过加密验证）。424 """425 pw, browser = await cls._launch_browser()426 try:427 context = await cls._launch_context(pw, browser)428 page = await context.new_page()429 430 video_data = None431 432 async def on_response(resp):433 nonlocal video_data434 url = resp.url435 436 # 拦截 aweme/detail API437 if 'aweme/v1/web/aweme/detail' in url or (438 'aweme/v2' in url and 'detail' in url439 ):440 try:441 body = await resp.json()442 detail = body.get('aweme_detail', {})443 if detail and detail.get('aweme_id'):444 video_data = detail445 except Exception:446 pass447 448 # feed API 备选449 if not video_data and (450 'aweme/v1/web/aweme/post' in url or 'general/feed' in url451 ):452 try:453 body = await resp.json()454 items = body.get('aweme_list', []) or body.get('data', [])455 for item in items:456 if isinstance(item, dict) and item.get('aweme_info'):457 item = item['aweme_info']458 if item.get('aweme_id') == video_id:459 video_data = item460 break461 except Exception:462 pass463 464 page.on('response', on_response)465 466 # 直接访问视频页面（跳过首页，减少被拦截概率）467 video_url = f'https://www.douyin.com/video/{video_id}'468 try:469 await page.goto(video_url, wait_until='domcontentloaded', timeout=25000)470 except Exception:471 # 超时也继续，可能API已经在响应中472 pass473 await page.wait_for_timeout(6000)474 475 # RENDER_DATA 兜底476 if not video_data:477 html = await page.content()478 match = RENDER_DATA_PATTERN.search(html)479 if match:480 try:481 from urllib.parse import unquote482 decoded = unquote(match.group(1))483 data = json.loads(decoded)484 video_data = cls._extract_from_render_data(data)485 except Exception:486 pass487 488 # page.evaluate 最终兜底489 if not video_data:490 try:491 video_data = await page.evaluate(492 '''() => {493 try {494 const app = document.querySelector('#RENDER_DATA');495 if (app) {496 const data = JSON.parse(497 decodeURIComponent(app.textContent)498 );499 function findVideo(obj, depth) {500 if (depth > 10 || !obj || typeof obj !== 'object')501 return null;502 if (obj.video && obj.video.playAddr) return obj;503 if (obj.video && obj.video.play_addr) return obj;504 for (const key in obj) {505 const r = findVideo(obj[key], depth + 1);506 if (r) return r;507 }508 return null;509 }510 return findVideo(data, 0);511 }512 } catch(e) {}513 return null;514 }'''515 )516 except Exception:517 pass518 519 except Exception as e:520 print(f"Playwright error: {e}")521 finally:522 await context.close()523 await browser.close()524 await pw.stop()525 526 return video_data527 528 @classmethod529 def _extract_from_render_data(cls, data: dict, depth: int = 0) -> Optional[Dict]:530 """从RENDER_DATA递归提取视频信息"""531 if depth > 10 or not isinstance(data, dict):532 return None533 if 'video' in data and isinstance(data['video'], dict):534 if 'play_addr' in data['video'] or 'playAddr' in data['video']:535 return data536 for key, value in data.items():537 if isinstance(value, dict):538 result = cls._extract_from_render_data(value, depth + 1)539 if result:540 return result541 elif isinstance(value, list):542 for item in value:543 if isinstance(item, dict):544 result = cls._extract_from_render_data(item, depth + 1)545 if result:546 return result547 return None548 549 # ================================================================550 # 清晰度解析551 # ================================================================552 553 @classmethod554 def parse_quality_options(cls, video_data: Dict) -> List[Dict]:555 """解析视频清晰度选项"""556 options = []557 video_info = video_data.get('video', {})558 559 bit_rates = video_info.get('bit_rate', [])560 if bit_rates:561 for br in bit_rates:562 gear_name = br.get('gear_name', '')563 bit_rate_val = br.get('bit_rate', 0)564 play_addr = br.get('play_addr', {}) or br.get('playAddr', {})565 url_list = play_addr.get('url_list', []) or play_addr.get('urlList', [])566 567 if not url_list:568 continue569 570 label = cls._quality_label(gear_name, bit_rate_val)571 options.append({572 'label': label,573 'gear_name': gear_name,574 'bit_rate': bit_rate_val,575 'url': url_list[0],576 'url_list': url_list,577 'width': play_addr.get('width', 0),578 'height': play_addr.get('height', 0),579 })580 581 if not options:582 play_addr = video_info.get('play_addr', {}) or video_info.get('playAddr', {})583 url_list = play_addr.get('url_list', []) or play_addr.get('urlList', [])584 if url_list:585 options.append({586 'label': '默认清晰度',587 'gear_name': 'default',588 'bit_rate': video_info.get('bit_rate', 0),589 'url': url_list[0],590 'url_list': url_list,591 'width': play_addr.get('width', 0),592 'height': play_addr.get('height', 0),593 })594 595 # 去重 + 按码率降序596 seen = set()597 unique = []598 for opt in options:599 if opt['gear_name'] not in seen:600 seen.add(opt['gear_name'])601 unique.append(opt)602 unique.sort(key=lambda x: x['bit_rate'], reverse=True)603 return unique604 605 # ================================================================606 # 字幕提取607 # ================================================================608 609 @classmethod610 def extract_captions(cls, video_data: Dict) -> Optional[str]:611 """提取视频字幕/文案"""612 captions = []613 desc = video_data.get('desc', '')614 if desc:615 captions.append(f"【描述】{desc}")616 617 stickers = video_data.get('interaction_stickers') or []618 for sticker in stickers:619 text = sticker.get('text_content', '') or sticker.get('text', '')620 if text:621 captions.append(f"【贴纸】{text}")622 623 text_extra = video_data.get('text_extra') or []624 for extra in text_extra:625 tag = extra.get('hashtag_name', '') or extra.get('tag', '')626 if tag:627 captions.append(f"#话题# {tag}")628 629 music = video_data.get('music') or {}630 if music:631 music_title = music.get('title', '') or music.get('author', '')632 if music_title:633 captions.append(f"【音乐】{music_title}")634 635 author = video_data.get('author') or {}636 if author:637 nickname = author.get('nickname', '')638 if nickname:639 captions.append(f"【作者】{nickname}")640 641 return '\n'.join(captions) if captions else None642 643 # ================================================================644 # 内部工具645 # ================================================================646 647 @classmethod648 def _quality_label(cls, gear_name: str, bit_rate: int) -> str:649 """生成清晰度标签"""650 label_map = {651 '720_1': '720P 标清',652 '720_2': '720P 高清',653 '1080_1': '1080P 标清',654 '1080_2': '1080P 高清',655 '1080_4': '1080P 超清',656 '1080_8': '1080P 极清',657 'adapt_lowest_1': '自适应_最低',658 'adapt_low_1': '自适应_低',659 'adapt_medium_1': '自适应_中',660 'adapt_high_1': '自适应_高',661 'adapt_highest_1': '自适应_最高',662 'default': '默认清晰度',663 }664 if gear_name in label_map:665 return label_map[gear_name]666 if bit_rate > 8_000_000:667 return f"超清 ({bit_rate // 1_000_000}Mbps)"668 if bit_rate > 4_000_000:669 return f"高清 ({bit_rate // 1_000_000}Mbps)"670 if bit_rate > 1_000_000:671 return f"标清 ({bit_rate // 1_000_000}Mbps)"672 return gear_name or "未知清晰度"673 674 675# ================================================================676# 评论抓取器677# ================================================================678 679class CommentFetcher:680 """抖音评论抓取器 — Playwright 页面内 fetch 调用 API（自动带签名）"""681 682 @classmethod683 def fetch_comments_sync(684 cls, video_id: str, max_comments: int = 3000685 ) -> Dict:686 """同步抓取评论 — 在 Playwright 页面上下文中调用 fetch，自动翻页直到达到上限"""687 from playwright.sync_api import sync_playwright688 689 result = {690 'video_id': video_id,691 'hot_comments': [],692 'latest_comments': [],693 'total_fetched': 0,694 'cursor': 0,695 'has_more': False,696 }697 698 with sync_playwright() as pw:699 browser = pw.chromium.launch(700 headless=True,701 args=['--no-sandbox', '--disable-setuid-sandbox'],702 )703 try:704 context = browser.new_context(705 user_agent=(706 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '707 'AppleWebKit/537.36 (KHTML, like Gecko) '708 'Chrome/131.0.0.0 Safari/537.36'709 ),710 viewport={'width': 1440, 'height': 900},711 locale='zh-CN',712 )713 page = context.new_page()714 715 # 访问首页建立 session716 page.goto(717 'https://www.douyin.com/', wait_until='domcontentloaded', timeout=20000718 )719 page.wait_for_timeout(3000)720 721 # 访问视频页面722 page.goto(723 f'https://www.douyin.com/video/{video_id}',724 wait_until='domcontentloaded', timeout=20000,725 )726 page.wait_for_timeout(5000)727 728 # 滚动触发页面 JS 环境完全激活729 for _ in range(3):730 page.evaluate('window.scrollTo(0, document.body.scrollHeight)')731 page.wait_for_timeout(2000)732 733 all_hot = []734 all_latest = []735 cursor = 0736 has_more = True737 738 while len(all_hot) + len(all_latest) < max_comments:739 js_code = f'''740 async () => {{741 const url = 'https://www.douyin.com/aweme/v1/web/comment/list/'742 + '?aweme_id={video_id}&cursor={cursor}&count=50&item_type=0';743 const resp = await fetch(url, {{ credentials: 'include' }});744 const text = await resp.text();745 return text;746 }}747 '''748 try:749 raw_text = page.evaluate(js_code)750 data = json.loads(raw_text)751 except Exception as e:752 print(f"[CommentFetcher] page eval error: {e}", flush=True)753 break754 755 raw = data.get('comments', []) or []756 if not raw:757 break758 759 for c in raw:760 item = cls._normalize_comment(c)761 if not item['content']:762 continue763 if c.get('is_hot') or c.get('is_hot_comment'):764 all_hot.append(item)765 else:766 all_latest.append(item)767 768 cursor = data.get('cursor', 0)769 has_more = data.get('has_more', 0) == 1770 if not has_more:771 break772 773 context.close()774 finally:775 browser.close()776 777 result['hot_comments'] = all_hot[:max_comments]778 result['latest_comments'] = all_latest[:max_comments]779 result['total_fetched'] = len(all_hot) + len(all_latest)780 if all_latest:781 result['cursor'] = all_latest[-1].get('create_time', 0)782 result['has_more'] = has_more783 784 print(f"[CommentFetcher] total: {result['total_fetched']} comments for {video_id}",785 flush=True)786 return result787 788 @staticmethod789 def _normalize_comment(c: Dict) -> Dict:790 """统一评论字段格式"""791 user = c.get('user', {}) or {}792 return {793 'cid': c.get('cid', ''),794 'nickname': user.get('nickname', '匿名'),795 'avatar': user.get('avatar_thumb', {}).get('url_list', [''])[0] or '',796 'content': (c.get('text', '') or c.get('content', '')).strip(),797 'digg_count': c.get('digg_count', 0) or 0,798 'reply_count': c.get('reply_comment_total', 0) or 0,799 'create_time': c.get('create_time', 0) or 0,800 'is_hot': bool(801 c.get('is_hot') or802 c.get('is_hot_comment') or803 c.get('label_type') or804 (c.get('digg_count', 0) >= 100)805 ),806 }807