From cba214e93eda35a59d9ac6e0b7c19ee47ab017bc Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Oct 2026 00:15:20 +0800 Subject: [PATCH] feat(sites): extract douban IDs from list rows (audiences) (ID-3) Co-Authored-By: Claude Opus 5.5 --- .../src/sites/definitions/audiences.json | 10 +++++++ packages/adapters/src/sites/spider/map.ts | 1 + packages/adapters/src/sites/spider/spider.ts | 10 +++++++ .../test/differential/list-ids.test.ts | 30 +++++++++++++++++++ .../movieclaw-overrides/audiences.json | 12 +++++++- 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 packages/adapters/test/differential/list-ids.test.ts diff --git a/packages/adapters/src/sites/definitions/audiences.json b/packages/adapters/src/sites/definitions/audiences.json index b657b49e..3a8c83d8 100644 --- a/packages/adapters/src/sites/definitions/audiences.json +++ b/packages/adapters/src/sites/definitions/audiences.json @@ -152,6 +152,16 @@ } ] }, + "doubanid": { + "selector": "a[href*=\"douban.com/subject/\"]", + "attribute": "href", + "filters": [ + { + "name": "re_search", + "args": ["subject/(\\d+)", 1] + } + ] + }, "freedate": { "selector": "img[class^=\"pro_\"]", "attribute": "title", diff --git a/packages/adapters/src/sites/spider/map.ts b/packages/adapters/src/sites/spider/map.ts index 1215b9b1..0fab73cc 100644 --- a/packages/adapters/src/sites/spider/map.ts +++ b/packages/adapters/src/sites/spider/map.ts @@ -78,5 +78,6 @@ export function spiderTorrentToRelease( }; if (t.description) release.subtitle = t.description; if (t.imdbid) release.imdbId = t.imdbid; + if (t.doubanid) release.doubanId = t.doubanid; return release; } diff --git a/packages/adapters/src/sites/spider/spider.ts b/packages/adapters/src/sites/spider/spider.ts index d197da06..ece36689 100644 --- a/packages/adapters/src/sites/spider/spider.ts +++ b/packages/adapters/src/sites/spider/spider.ts @@ -67,6 +67,8 @@ export interface SpiderTorrent { seeders?: number; size?: number; imdbid?: string; + /** UMT 扩展(ID-3,MP 没有):列表行里的豆瓣 subject id,字段 `doubanid` 与 imdbid 同样的选择器写法 */ + doubanid?: string; downloadvolumefactor?: number; uploadvolumefactor?: number; pubdate?: string; @@ -476,6 +478,7 @@ export class SiteSpider { this.getSeeders(row); this.getSize(row); this.getImdbid(row); + this.getDoubanid(row); this.getVolumeFactor(row, "downloadvolumefactor"); this.getVolumeFactor(row, "uploadvolumefactor"); this.getPubdate(row); @@ -659,6 +662,13 @@ export class SiteSpider { } } + private getDoubanid(row: Sel) { + if (!("doubanid" in this.fields)) return; + const selector = this.fields.doubanid ?? {}; + const v = this.filterText(this.safeQuery(row, selector), selector.filters); + if (v != null && /^\d{4,10}$/.test(String(v))) this.info.doubanid = String(v); + } + private getImdbid(row: Sel) { if (!("imdbid" in this.fields)) return; const selector = this.fields.imdbid ?? {}; diff --git a/packages/adapters/test/differential/list-ids.test.ts b/packages/adapters/test/differential/list-ids.test.ts new file mode 100644 index 00000000..7d3b187c --- /dev/null +++ b/packages/adapters/test/differential/list-ids.test.ts @@ -0,0 +1,30 @@ +/** + * ID-3 列表页外部 ID:观众列表行右侧的评分徽标带 IMDb / 豆瓣链接(DET-2 列表样本 rl-audiences-list,脱敏)。 + * imdbid 是 MP 字段;doubanid 是 UMT 扩展字段(MP SiteSpider 不读,索引差分只比 MP 字段,不受影响)。 + */ +import { describe, expect, it } from "vitest"; +import { audiencesDefinition, SiteSpider } from "../../src/sites/index.ts"; +import { spiderTorrentToRelease } from "../../src/sites/spider/map.ts"; +import { readInput } from "./diff-util.ts"; + +describe("观众列表页:IMDb / 豆瓣 ID(真实样本)", () => { + const html = readInput("inputs/indexer/rl-audiences-list.html.txt"); + const rows = new SiteSpider(audiencesDefinition, {}).parse(html).torrents; + const byId = (id: string) => rows.find((r) => r.page_url?.includes(`id=${id}&`)); + + it("有豆瓣徽标的行都抽到豆瓣 ID,有 IMDb 徽标的抽到 IMDb", () => { + expect(byId("725858")).toMatchObject({ imdbid: "tt25399154", doubanid: "35322132" }); + expect(byId("726249")).toMatchObject({ imdbid: "tt0039689", doubanid: "1302338" }); + // 剧集行只有豆瓣(IMDb 在详情页里) + const prophet = byId("727077")!; + expect(prophet.doubanid).toBe("37297001"); + expect(prophet.imdbid).toBeUndefined(); + expect(rows.filter((r) => r.doubanid).length).toBe(10); + }); + + it("映射到 SiteRelease.doubanId / imdbId", () => { + const r = spiderTorrentToRelease(byId("727077")!, audiencesDefinition, "2026-09-30T00:00:00.000Z"); + expect(r).toMatchObject({ siteReleaseId: "727077", doubanId: "37297001" }); + expect(r?.imdbId).toBeUndefined(); + }); +}); diff --git a/scripts/mp-sync/movieclaw-overrides/audiences.json b/scripts/mp-sync/movieclaw-overrides/audiences.json index 23510f09..9dfd2012 100644 --- a/scripts/mp-sync/movieclaw-overrides/audiences.json +++ b/scripts/mp-sync/movieclaw-overrides/audiences.json @@ -1,5 +1,5 @@ { - "$comment": "W1-3b 手写观众定义(按 MP indexer schema;仅对 testkit mock 验证)。深合并到 MovieClaw v0.27.0 转换结果之上,本文件的值优先,MovieClaw 值只补空。 umt.detail.fields=null:保留 W1-3b rowhead 详情模型(已对 mock 验证),不采用 MovieClaw 默认逐字段选择器。 DET-2(2026-09-30 真实账号核对列表页 1 次 + 详情页 2 次):列表副标题取 span.torrent-subtitle-text(span.torrent-subtitle-line 里先是标签);详情 h1 促销是 font.free / font.halfdown 等 + 「剩余时间」span[title](umt.detail.promo);做种 / 下载 / 完成数在基本信息行右侧 #seeders / #leechers / a[title=完成](没有「同伴」行);种子 Hash 在 #copyTorrentHashLink[data-hash];上传者在 span.torrent-uploader-wrap(匿名为「匿名」);简介开头的站点声明 #ad_torrentdetail 去掉;文件清单不在页面上(点击「N个文件」后由 common.js 的 toggleFileList AJAX 加载,端点未核对),暂不另发请求。", + "$comment": "W1-3b 手写观众定义(按 MP indexer schema;仅对 testkit mock 验证)。深合并到 MovieClaw v0.27.0 转换结果之上,本文件的值优先,MovieClaw 值只补空。 umt.detail.fields=null:保留 W1-3b rowhead 详情模型(已对 mock 验证),不采用 MovieClaw 默认逐字段选择器。 DET-2(2026-09-30 真实账号核对列表页 1 次 + 详情页 2 次):列表副标题取 span.torrent-subtitle-text(span.torrent-subtitle-line 里先是标签);详情 h1 促销是 font.free / font.halfdown 等 + 「剩余时间」span[title](umt.detail.promo);做种 / 下载 / 完成数在基本信息行右侧 #seeders / #leechers / a[title=完成](没有「同伴」行);种子 Hash 在 #copyTorrentHashLink[data-hash];上传者在 span.torrent-uploader-wrap(匿名为「匿名」);简介开头的站点声明 #ad_torrentdetail 去掉;文件清单不在页面上(点击「N个文件」后由 common.js 的 toggleFileList AJAX 加载,端点未核对),暂不另发请求。 ID-3:列表行的豆瓣链接(评分徽标 a[href*=douban.com/subject/],DET-2 列表样本 rl-audiences-list 核对)抽成 torrents.fields.doubanid(UMT 扩展字段,MP 不读)。", "id": "audiences", "name": "观众", "domain": "https://audiences.me/", @@ -84,6 +84,16 @@ } ] }, + "doubanid": { + "selector": "a[href*=\"douban.com/subject/\"]", + "attribute": "href", + "filters": [ + { + "name": "re_search", + "args": ["subject/(\\d+)", 1] + } + ] + }, "date_elapsed": { "selector": "td:nth-child(4) > span" },