fix(sites): 按真实列表页修正 OurBits / 彩虹岛 / 憨憨 解析
2026-10-02 用真实账号各抓一次列表页与用户页(只读)核对: - OurBits:标题列嵌套 table.torrentname 有 5 个 td,td:nth-child(n) 先命中嵌套格子, 大小全为 0、发布时间 N/A;改为只匹配行的直接子单元格。副标题原取成标签块,改为标签之后的正文; 新增标签与列表页 IMDb / 豆瓣 ID(label.imdb_rate / douban_rate 的 data 属性) - 彩虹岛:font.subtitle 去掉前面的 div.tag 与末尾徽标,标签单独取 - 憨憨:新增列表标签(span.tag,不含促销) - 纯数字 IMDb ID 补成 tt + 7 位(imdbIdOf) - 用户页核对:下载量为 0 是站点实际显示(VIP),解析正确;上传量四站与页面一致 - 脱敏样例 sites-2026.ts + 定义测试;差分登记 23 条有意偏离(新增 labels、OurBits 副标题) Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
cac1833f53
commit
64e1dfb1d2
12 files changed
+327
-16
No files matched your search
@@ -149,7 +149,8 @@
|
||||
}
|
||||
},
|
||||
"description": {
|
||||
"selector": "font.subtitle"
|
||||
"selector": "font.subtitle",
|
||||
"remove": "div"
|
||||
},
|
||||
"category": {
|
||||
"selector": "a[href^='?cat=']",
|
||||
@@ -170,6 +171,9 @@
|
||||
"args": ["\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
"labels": {
|
||||
"selector": "font.subtitle div.tag"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
import {
|
||||
CHDBITS_LIST_PAGE,
|
||||
CHDBITS_LIST_PAGE_2026,
|
||||
CHDBITS_LOGIN_PAGE,
|
||||
HHANCLUB_DETAIL_PAGE,
|
||||
HHANCLUB_LIST_PAGE,
|
||||
HHANCLUB_LIST_PAGE_2026,
|
||||
HHANCLUB_LIST_PAGE_TAGS,
|
||||
OURBITS_LIST_PAGE,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getSiteDefinition, listSiteDefinitions } from "../registry.ts";
|
||||
import { parseDetailPage } from "../spider/detail.ts";
|
||||
import { spiderTorrentToRelease } from "../spider/map.ts";
|
||||
import { imdbIdOf, spiderTorrentToRelease } from "../spider/map.ts";
|
||||
import type { IndexerDefinition } from "../spider/schema.ts";
|
||||
import { SiteSpider } from "../spider/spider.ts";
|
||||
import {
|
||||
@@ -198,3 +201,44 @@ describe("CHDBits 定义(标准 NexusPHP)", () => {
|
||||
expect(r.torrents).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("2026-10 真实列表页结构(脱敏样例)", () => {
|
||||
const parse = (key: string, html: string) => {
|
||||
const def = BUILTIN_SITE_DEFINITIONS[key] as IndexerDefinition;
|
||||
return new SiteSpider(def, {}).parse(html).torrents.map((t) => spiderTorrentToRelease(t, def, OBSERVED)!);
|
||||
};
|
||||
|
||||
it("OurBits:嵌套 torrentname 不干扰列号;副标题在标签之后;标签与列表页 IMDb / 豆瓣 ID", () => {
|
||||
const [a] = parse("ourbits", OURBITS_LIST_PAGE);
|
||||
expect(a).toMatchObject({
|
||||
siteReleaseId: "900001",
|
||||
sizeBytes: Math.round(38.44 * 1024 ** 3),
|
||||
seeders: 174,
|
||||
leechers: 17,
|
||||
snatched: 334,
|
||||
subtitle: "示例电影[简繁双语字幕]",
|
||||
tags: ["官方", "中字"],
|
||||
imdbId: "tt0299977",
|
||||
doubanId: "1306123",
|
||||
});
|
||||
});
|
||||
|
||||
it("CHDBits:副标题去掉前面的标签块和末尾徽标,标签单独取", () => {
|
||||
const [a] = parse("chdbits", CHDBITS_LIST_PAGE_2026);
|
||||
expect(a!.subtitle).toBe("示例劫案 [DIY UHD 原盘]");
|
||||
expect(a!.tags).toEqual(["官方", "DIY", "中字"]);
|
||||
});
|
||||
|
||||
it("HHClub:标签取 span.tag,不混入促销", () => {
|
||||
const [a] = parse("hhanclub", HHANCLUB_LIST_PAGE_TAGS);
|
||||
expect(a!.tags).toEqual(["官方", "中字"]);
|
||||
});
|
||||
|
||||
it("IMDb ID 规范化:纯数字补 tt + 7 位,0 与非法值丢弃", () => {
|
||||
expect(imdbIdOf("299977")).toBe("tt0299977");
|
||||
expect(imdbIdOf("35522483")).toBe("tt35522483");
|
||||
expect(imdbIdOf("tt0120907")).toBe("tt0120907");
|
||||
expect(imdbIdOf("0")).toBeUndefined();
|
||||
expect(imdbIdOf("N/A")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -181,6 +181,9 @@
|
||||
},
|
||||
"hr": {
|
||||
"selector": "div.torrent-info-hr"
|
||||
},
|
||||
"labels": {
|
||||
"selector": "div.torrent-title span.tag"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
"ref": "v0.27.0",
|
||||
"commit": "c79bb1b0",
|
||||
"license": "MIT",
|
||||
"file": "src/movieclaw_tracker/sites/configs/ourbits.yaml"
|
||||
"file": "src/movieclaw_tracker/sites/configs/ourbits.yaml",
|
||||
"overrides": "scripts/mp-sync/movieclaw-overrides/ourbits.json"
|
||||
},
|
||||
"domain": "https://ourbits.club/",
|
||||
"encoding": "UTF-8",
|
||||
@@ -105,20 +106,20 @@
|
||||
"attribute": "href"
|
||||
},
|
||||
"imdbid": {
|
||||
"selector": "a[href*=\"imdb.com/title/tt\"]",
|
||||
"attribute": "href",
|
||||
"selector": "label.imdb_rate",
|
||||
"attribute": "data-imdbid",
|
||||
"filters": [
|
||||
{
|
||||
"name": "re_search",
|
||||
"args": ["tt\\d+", 0]
|
||||
"args": ["[1-9]\\d*", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
"date_elapsed": {
|
||||
"selector": "td:nth-child(4)"
|
||||
"selector": "> td:nth-child(4)"
|
||||
},
|
||||
"date_added": {
|
||||
"selector": "td:nth-child(4) > span",
|
||||
"selector": "> td:nth-child(4) > span",
|
||||
"attribute": "title"
|
||||
},
|
||||
"date": {
|
||||
@@ -131,16 +132,16 @@
|
||||
]
|
||||
},
|
||||
"size": {
|
||||
"selector": "td:nth-child(5)"
|
||||
"selector": "> td:nth-child(5)"
|
||||
},
|
||||
"seeders": {
|
||||
"selector": "td:nth-child(6)"
|
||||
"selector": "> td:nth-child(6)"
|
||||
},
|
||||
"leechers": {
|
||||
"selector": "td:nth-child(7)"
|
||||
"selector": "> td:nth-child(7)"
|
||||
},
|
||||
"grabs": {
|
||||
"selector": "td:nth-child(8)"
|
||||
"selector": "> td:nth-child(8)"
|
||||
},
|
||||
"downloadvolumefactor": {
|
||||
"case": {
|
||||
@@ -161,7 +162,8 @@
|
||||
}
|
||||
},
|
||||
"description": {
|
||||
"selector": "td.embedded > div"
|
||||
"selector": "table.torrentname td.embedded",
|
||||
"remove": "a, b, img, div, br, font"
|
||||
},
|
||||
"category": {
|
||||
"selector": "a[href^='?cat=']",
|
||||
@@ -175,6 +177,19 @@
|
||||
},
|
||||
"hr": {
|
||||
"selector": "img.hitandrun"
|
||||
},
|
||||
"labels": {
|
||||
"selector": "span.tag"
|
||||
},
|
||||
"doubanid": {
|
||||
"selector": "label.douban_rate",
|
||||
"attribute": "data-doubanid",
|
||||
"filters": [
|
||||
{
|
||||
"name": "re_search",
|
||||
"args": ["[1-9]\\d*", 0]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -45,6 +45,14 @@ export function categoryName(def: IndexerDefinition, id: string | undefined): st
|
||||
}
|
||||
|
||||
/** SpiderTorrent(MP TorrentInfo 字段)→ SiteRelease。raw 保存完整 MP 字段(enclosure 脱敏)。 */
|
||||
/** 站点给的 IMDb ID:`tt0299977` 原样;纯数字(OurBits 列表页 data-imdbid)补成 `tt` + 至少 7 位;其他丢弃 */
|
||||
export function imdbIdOf(v: string | undefined): string | undefined {
|
||||
const s = v?.trim();
|
||||
if (!s) return undefined;
|
||||
if (/^tt\d{7,}$/.test(s)) return s;
|
||||
return /^\d+$/.test(s) && Number(s) > 0 ? `tt${s.padStart(7, "0")}` : undefined;
|
||||
}
|
||||
|
||||
export function spiderTorrentToRelease(
|
||||
t: SpiderTorrent,
|
||||
def: IndexerDefinition,
|
||||
@@ -77,7 +85,8 @@ export function spiderTorrentToRelease(
|
||||
observedAt,
|
||||
};
|
||||
if (t.description) release.subtitle = t.description;
|
||||
if (t.imdbid) release.imdbId = t.imdbid;
|
||||
const imdb = imdbIdOf(t.imdbid);
|
||||
if (imdb) release.imdbId = imdb;
|
||||
if (t.doubanid) release.doubanId = t.doubanid;
|
||||
return release;
|
||||
}
|
||||
@@ -7333,6 +7333,98 @@
|
||||
"tjupt@tk-chdbits-list#1.pubdate": {
|
||||
"kind": "mp_bug",
|
||||
"note": "tjupt 定义 date_added 取单元格文本(\"2天\"/\"26天\"相对时间),MP normalize_datetime 用 dateparser 以当前时刻换算成绝对时间(生成时冻结 now=2026-09-27T12:00),结果随抓取时刻漂移;本项目 normalizeDatetime 只接受绝对时间,相对时间不写 pubdate(与 MP __is_relative_pubdate_text 的意图一致)。"
|
||||
},
|
||||
"chdbits@tk-audiences-list#0.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"chdbits@tk-audiences-list#1.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"chdbits@tk-audiences-list#2.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"chdbits@tk-audiences-list#3.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"chdbits@tk-audiences-list#4.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"chdbits@tk-chdbits-list#0.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"chdbits@tk-chdbits-list#1.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"hhanclub@tk-hhanclub-list#0.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"hhanclub@tk-hhanclub-list#1.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#0.description": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 OurBits 覆盖层改了副标题选择器(原定义 td.embedded > div 取到的是标签块,真实页面副标题在标签之后);MP 参照仍用原定义,在通用样例上为 null,本项目取到正文。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#0.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#1.description": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 OurBits 覆盖层改了副标题选择器(原定义 td.embedded > div 取到的是标签块,真实页面副标题在标签之后);MP 参照仍用原定义,在通用样例上为 null,本项目取到正文。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#1.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#2.description": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 OurBits 覆盖层改了副标题选择器(原定义 td.embedded > div 取到的是标签块,真实页面副标题在标签之后);MP 参照仍用原定义,在通用样例上为 null,本项目取到正文。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#2.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#3.description": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 OurBits 覆盖层改了副标题选择器(原定义 td.embedded > div 取到的是标签块,真实页面副标题在标签之后);MP 参照仍用原定义,在通用样例上为 null,本项目取到正文。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#3.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#4.description": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 OurBits 覆盖层改了副标题选择器(原定义 td.embedded > div 取到的是标签块,真实页面副标题在标签之后);MP 参照仍用原定义,在通用样例上为 null,本项目取到正文。"
|
||||
},
|
||||
"ourbits@tk-audiences-list#4.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"ourbits@tk-chdbits-list#0.description": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 OurBits 覆盖层改了副标题选择器(原定义 td.embedded > div 取到的是标签块,真实页面副标题在标签之后);MP 参照仍用原定义,在通用样例上为 null,本项目取到正文。"
|
||||
},
|
||||
"ourbits@tk-chdbits-list#0.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
},
|
||||
"ourbits@tk-chdbits-list#1.description": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 OurBits 覆盖层改了副标题选择器(原定义 td.embedded > div 取到的是标签块,真实页面副标题在标签之后);MP 参照仍用原定义,在通用样例上为 null,本项目取到正文。"
|
||||
},
|
||||
"ourbits@tk-chdbits-list#1.labels": {
|
||||
"kind": "intentional",
|
||||
"note": "2026-10-02 按真实列表页给 OurBits / 彩虹岛 / 憨憨 覆盖层新增 labels 字段(MovieClaw 原定义没有,MP 参照因此为 undefined);样例里没有标签时本项目输出空数组。"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user