fix(sites): integrate shared download access and register synced adapters

This commit is contained in:
root committed 2026-11-06 00:15:15 +08:00
1 parent 6dab134617
commit 36f3ece5b5
20 files changed
+175 -38

No files matched your search

@@ -9,12 +9,10 @@ import {
fetchEpisodeGroup,
fetchRichDetail,
fetchRichSeason,
getSiteAdapter,
getSiteDefinition,
groupSeasonAsRichSeason,
type ProbeResult,
qbAuthConfig,
type SecretResolver,
SiteError,
type TmdbRichDetail,
type TmdbRichSeasonDetail,
@@ -200,20 +198,8 @@ export function createPorts(
const siteMeta = (raw._site ?? {}) as Record<string, unknown>;
const defId = [str(siteMeta.adapterSiteId), site.key].find((k) => k && getSiteDefinition(k));
if (!defId) throw new SiteError(site.id, "unsupported", `站点 ${site.key} 没有可用的适配器定义`);
const accounts = await ctx.db
.selectFrom("site_account")
.select(["auth_kind", "secret_ref"])
.where("site_id", "=", site.id)
.where("enabled", "=", true)
.execute();
const secrets: SecretResolver = {
async resolve({ kind }) {
const want = kind === "apiKey" ? "api_key" : kind;
const acc = accounts.find((a) => a.auth_kind === want);
return (await resolveSecret(acc?.secret_ref ?? null)) ?? null;
},
};
const adapter = getSiteAdapter(defId, { secrets, baseUrl: site.base_url });
// 与搜索/详情共用站点出口、账号会话、UA 和跨进程限流;API 基址仍取站点配置。
const adapter = rt().siteAdapter({ ...site, key: defId });
const t = await adapter.downloadTarget(release.site_release_id);
if (t.kind === "torrent")
return { kind: "file", bytes: t.bytes, ...(t.fileName ? { filename: t.fileName } : {}) };
@@ -199,6 +199,13 @@ export class SourcesRuntime {
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
headers.set("user-agent", await pick);
}
const credentials = init?.credentials ?? (input instanceof Request ? input.credentials : undefined);
if (credentials === "omit") {
// 签名下载不复用账号 Cookie 或浏览器 profile,包括 cf_clearance。
headers.delete("cookie");
headers.delete("authorization");
return base(input, { ...init, headers, credentials });
}
if (!siteId) return base(input, { ...init, headers });
const url = input instanceof Request ? input.url : String(input);
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
@@ -413,10 +413,11 @@ export async function refreshSiteAccount(
const profile = (def.umt as { profile?: Record<string, string> }).profile;
const vipLevels = (def.umt as { vipLevels?: string[] }).vipLevels;
// 馒头 API 基址与适配器一致:站点配置 > 定义里的 apiBaseUrl;不再由站点地址推 api.<根域>(IP / 本地地址会拼成非法 URL)
const siteConfig = (site.config ?? {}) as { apiBaseUrl?: string; userdataPath?: string };
const apiUrl =
def.umt.kind === "mteam"
? (site.config as { apiBaseUrl?: string })?.apiBaseUrl || mteamApiBase(def as MteamDefinition)
: null;
? siteConfig.apiBaseUrl || mteamApiBase(def as MteamDefinition)
: siteConfig.apiBaseUrl || ("api_url" in def ? def.api_url : null);
const fetchUserData = async (
cred: { cookie?: string | null; apikey?: string | null },
@@ -425,6 +426,7 @@ export async function refreshSiteAccount(
const info = await refreshSiteUserData({
name: site.name,
url: site.base_url,
...(siteConfig.userdataPath ? { userdataPath: siteConfig.userdataPath } : {}),
schema: (def as { schema?: string }).schema ?? null,
public: true,
cookie: cred.cookie ?? null,
@@ -0,0 +1,22 @@
import { expect, it, vi } from "vitest";
import { SourcesRuntime } from "../src/modules/sources-identity/runtime.ts";
it("签名下载的 credentials omit 不复用账号或浏览器Cookie", async () => {
let sent = new Headers();
const rt = new SourcesRuntime({} as never, {
fetch: async (_input, init) => {
sent = new Headers(init?.headers);
return new Response("subtitle bytes");
},
});
const browser = vi.spyOn(rt.cloudflare, "handle");
const fetch = rt.siteFetch({ id: "original-site", proxy: false }, undefined, "Site User Agent");
const headers = { cookie: "account-cookie", authorization: "account-token" };
const url = "https://signed-cdn.example/file?credential=ephemeral";
const response = await fetch(url, { headers, credentials: "omit" });
expect(await response.text()).toBe("subtitle bytes");
expect(sent.has("cookie")).toBe(false);
expect(sent.has("authorization")).toBe(false);
expect(sent.get("user-agent")).toBe("Site User Agent");
expect(browser).not.toHaveBeenCalled();
});
@@ -1,6 +1,7 @@
import { TaskRegistry } from "@umt/tasks";
import { createMockMteamApi, type MockMteamApi } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { createPorts } from "../../src/modules/acquisition-organize/ports.ts";
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
@@ -50,6 +51,27 @@ describe.skipIf(!up)("M-Team 快照与任务注册", () => {
expect(JSON.stringify(rows)).not.toContain(api.apiKey);
});
it("下载种子沿用配置的 API 基址,并与站点访问共用 PG 限流桶", async () => {
const site = await h.db
.updateTable("site")
.set({ rate_limit: JSON.stringify({ minIntervalMs: 10, burst: 1 }) })
.where("id", "=", siteId)
.returningAll()
.executeTakeFirstOrThrow();
const release = await h.db
.selectFrom("source_release")
.selectAll()
.where("site_id", "=", siteId)
.executeTakeFirstOrThrow();
const before = api.issuedTokens;
const source = await createPorts(h.ctx).torrentSource({ release, site });
expect(source.kind).toBe("file");
expect(api.issuedTokens).toBe(before + 1);
expect(
await h.db.selectFrom("rate_limit_bucket").select("key").where("key", "=", `site:${siteId}`).executeTakeFirst(),
).toBeTruthy();
});
it("key 失效:账号标失效 + 待处理,快照结果 failed", async () => {
api.setFailMode("banned_key");
const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
+1
View File
@@ -23,6 +23,7 @@ const UNIT = [
"test/mapping-compat.test.ts",
"test/modules.test.ts",
"test/sse.test.ts",
"test/site-fetch-credentials.test.ts",
"test/static.test.ts",
"test/system-resources.test.ts",
];
@@ -49,7 +49,7 @@ export const SUBTITLE_LANGUAGE_LABEL: Record<SubtitleLanguage, string> = {
ko: "韩文",
unknown: "未知",
};
const SOURCE_LABEL = { upload: "上传", subhd: "SubHD", zimuku: "字幕库" } as const;
const SOURCE_LABEL = { upload: "上传", subhd: "SubHD", zimuku: "字幕库", mteam: "馒头" } as const;
const SKIP = "__skip__";
const ACCEPT = ".srt,.ass,.ssa,.vtt,.sub,.sbv,.zip";
@@ -32,9 +32,9 @@ const PUBLIC = ["anibt", "animegarden", "bangumimoe", "mikan", "nyaa"];
describe("站点定义:加载时 zod 校验", () => {
const ids = Object.keys(MOVIECLAW_DEFINITION_JSON).sort();
it("MovieClaw v0.27.0 全部 23 站转换结果 + 公开 BT 站均在注册表中", () => {
it("MovieClaw v0.27.0 全部 23 站、公开 BT 站与手写 Milkie 定义均在注册表中", () => {
expect(ids).toHaveLength(23);
expect(listSiteDefinitions().map((d) => d.id)).toEqual([...ids, ...PUBLIC].sort());
expect(listSiteDefinitions().map((d) => d.id)).toEqual([...ids, ...PUBLIC, "milkie"].sort());
});
it.each(ids)("%s:schema 校验通过,provenance 指向 MovieClaw v0.27.0 MIT", (id) => {
@@ -8,6 +8,7 @@ import type { MteamDefinition } from "../mteam/definition.ts";
import type { IndexerDefinition } from "../spider/schema.ts";
import type { SpecializedSiteDefinition } from "../spiders/types.ts";
import { MOVIECLAW_DEFINITION_JSON } from "./manifest.generated.ts";
import milkieJson from "./milkie.json" with { type: "json" };
import { PUBLIC_SITE_DEFINITION_JSON } from "./public.ts";
import { parseIndexerDefinition, parseMteamDefinition, parseSpecializedSiteDefinition } from "./schema.ts";
@@ -27,13 +28,12 @@ export function loadIndexerDefinition(json: unknown): IndexerDefinition {
return parseIndexerDefinition(json);
}
/** 随产品发布的全部定义(已校验),键为站点 id:MovieClaw 转换的 PT 站 + 手写的公开 BT 站 */
/** 随产品发布的全部定义(已校验),键为站点 id:MovieClaw 转换站、手写 API 站与公开 BT 站 */
export const BUILTIN_SITE_DEFINITIONS: Readonly<Record<string, SiteDefinition>> = Object.freeze(
Object.fromEntries(
Object.entries({ ...MOVIECLAW_DEFINITION_JSON, ...PUBLIC_SITE_DEFINITION_JSON }).map(([id, json]) => [
id,
loadSiteDefinition(json),
]),
Object.entries({ ...MOVIECLAW_DEFINITION_JSON, ...PUBLIC_SITE_DEFINITION_JSON, milkie: milkieJson }).map(
([id, json]) => [id, loadSiteDefinition(json)],
),
),
);
@@ -0,0 +1,25 @@
{
"id": "milkie",
"name": "Milkie",
"domain": "https://milkie.cc/",
"parser": "Milkie",
"enabled": false,
"provenance": {
"source": "handwritten",
"ref": "v3.2.4",
"commit": "3b156a545f14fea0285cd3ed3ea4aa49d97faa57",
"license": "GPL-3.0-only",
"file": "app/modules/indexer/spider/milkie.py",
"note": "按公开 API 行为重写;凭据由 SecretResolver 注入,未做真实站点验收"
},
"umt": {
"version": "2026-11-05",
"kind": "api",
"authSupported": ["apikey"],
"displayName": "Milkie",
"timezone": "+00:00",
"rateLimit": { "minIntervalMs": 2000, "burst": 2 },
"capabilities": { "list": true, "search": true, "detail": false, "download": true, "rss": false },
"categoryNames": { "1": "电影", "2": "电视剧", "3": "音乐" }
}
}
@@ -216,6 +216,8 @@ export const IndexerDefinitionSchema = z.strictObject({
export const MteamDefinitionSchema = z.strictObject({
...BaseDefinitionShape,
parser: z.literal("mTorrent"),
api_domain: z.string().min(1).optional(),
api_host: z.string().min(1).optional(),
timeout: z.number().positive(),
umt: z.strictObject({
version: z.string().min(1),
@@ -242,6 +244,7 @@ export const SpecializedSiteDefinitionSchema = z.strictObject({
"TNodeSpider",
"TorrentLeech",
"Yema",
"Milkie",
"Nyaa",
"Mikan",
"AniBT",
@@ -251,6 +254,8 @@ export const SpecializedSiteDefinitionSchema = z.strictObject({
public: z.boolean().optional(),
category: CategoryConfigSchema.optional(),
api_url: z.string().url().optional(),
api_domain: z.string().min(1).optional(),
api_host: z.string().min(1).optional(),
umt: z.strictObject({
version: z.string().min(1),
kind: z.literal("api"),
@@ -258,6 +263,7 @@ export const SpecializedSiteDefinitionSchema = z.strictObject({
timezone: TimezoneSchema,
rateLimit: RateLimitSchema,
capabilities: CapabilitiesSchema,
authSupported: z.array(z.enum(["cookie", "credential", "apikey"])).optional(),
categoryNames: z.record(z.string(), z.string()).optional(),
apiBaseUrl: z.string().url().optional(),
}),
+3
View File
@@ -150,6 +150,9 @@ export {
type SitePromotion,
type SiteRelease,
type SiteReleaseDetail,
type SiteSubtitle,
type SiteSubtitleFile,
type SiteSubtitlePage,
staticSecretResolver,
} from "./types.ts";
export * from "./user-parsers/index.ts";
+6 -1
View File
@@ -186,7 +186,12 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps
throw new SiteError(def.id, "rate_limited", `genDlToken ${id}:未返回下载链接(可能未认证或达到下载频率限制)`);
}
// MP:涉及解析地址的不使用 Cookie 下载;直链含一次性签名,只在内存中使用
const res = await siteFetch(http, url, { method: "GET", ...(opts.signal ? { signal: opts.signal } : {}) });
const res = await siteFetch(http, url, {
method: "GET",
credentials: "omit",
referrerPolicy: "no-referrer",
...(opts.signal ? { signal: opts.signal } : {}),
});
if (!res.ok) {
await res.body?.cancel().catch(() => {});
throw new SiteError(
@@ -17,11 +17,20 @@ afterEach(async () => {
api = undefined;
});
function adapterFor(m: MockMteamApi, key: string | null = m.apiKey, limiter?: SiteRateLimiter): SiteAdapter {
function adapterFor(
m: MockMteamApi,
key: string | null = m.apiKey,
limiter?: SiteRateLimiter,
requests?: RequestInit[],
): SiteAdapter {
return createMteamAdapter(mteamDefinition, {
baseUrl: m.baseUrl,
secrets: staticSecretResolver(key ? { mteam: { apiKey: key } } : {}),
limiter: limiter ?? createSiteRateLimiter({ minIntervalMs: 0 }),
fetch: (input, init) => {
requests?.push(init ?? {});
return globalThis.fetch(input, init);
},
});
}
@@ -129,13 +138,15 @@ describe("M-Team API 适配器(mock)", () => {
it("下载令牌:现场 genDlToken → 一次性直链 → .torrent,infohash 校验", async () => {
api = await createMockMteamApi();
const a = adapterFor(api);
const requests: RequestInit[] = [];
const a = adapterFor(api, api.apiKey, undefined, requests);
const t = await a.downloadTarget("912233");
expect(api.issuedTokens).toBe(1);
expect(api.requests.map((r) => r.path.split("?")[0])).toEqual(["/api/torrent/genDlToken", "/api/rss/dlv2"]);
// MP:genDlToken 的 id 走 query 参数;直链下载不带 x-api-key
expect(api.requests[0]!.path).toBe("/api/torrent/genDlToken?id=912233");
expect(api.requests[1]!.headers["x-api-key"]).toBeUndefined();
expect(requests[1]).toMatchObject({ credentials: "omit", referrerPolicy: "no-referrer" });
expect(t.kind).toBe("torrent");
if (t.kind === "torrent") expect(t.infoHash).toBe(api.infoHashOf("912233"));
// 每次下载都重新签发令牌
@@ -162,15 +173,27 @@ describe("M-Team API 适配器(mock)", () => {
it("非 JSON 维护页 → site_maintenance;业务限流/HTTP 429 → rate_limited 并顺延限流器", async () => {
api = await createMockMteamApi();
const limiter = createSiteRateLimiter({ minIntervalMs: 0 });
let clock = 0;
const waits: number[] = [];
const limiter = createSiteRateLimiter({
minIntervalMs: 0,
now: () => clock,
sleep: async (ms) => {
waits.push(ms);
clock += ms;
},
});
const a = adapterFor(api, api.apiKey, limiter);
api.setFailMode("non_json");
await expectSiteError(a.list(1), "site_maintenance");
api.setFailMode("rate_limit");
await expectSiteError(a.list(1), "rate_limited");
const business = await expectSiteError(a.list(1), "rate_limited");
expect(business.retryAfterMs).toBe(3600_000);
expect(limiter.tryAcquire()).toBe(false);
api.setFailMode("http_429");
const e = await expectSiteError(a.list(1), "rate_limited");
expect(e.retryAfterMs).toBe(5000);
expect(waits).toEqual([3600_000]);
expect(limiter.tryAcquire()).toBe(false);
});
@@ -17,7 +17,7 @@ import { inspectTorrentIsolated } from "../../isolation.ts";
import { redactText } from "../../rss/redact.ts";
import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts";
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts";
import { buildFailureSample, redactPageHtml, type SiteRequestKind } from "../failure-sample.ts";
import { readBody, readText, siteFetch } from "../http.ts";
import {
type DownloadTarget,
@@ -155,7 +155,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
const rule = challengeRuleOfUnrecognizedPage(page.html);
const title = redactText(loadHtml(page.html)("title").first().text().replace(/\s+/g, " ").trim()).slice(0, 120);
const title = redactPageHtml(loadHtml(page.html)("title").first().text()).replace(/\s+/g, " ").trim().slice(0, 120);
const message = rule
? `${what}:${PAGE_LABEL[kind]}需要人机验证(Cloudflare 等),${challengeOutcomeText("unhandled")}`
: `${parseMessage}(HTTP ${page.status}${title ? `,页面标题:${title}` : ""};页面样本见站点页技术信息)`;
@@ -1,7 +1,7 @@
import { createMockFetch } from "@umt/testkit";
import { describe, expect, it } from "vitest";
import { createSiteRateLimiter } from "../rate-limit.ts";
import { staticSecretResolver } from "../types.ts";
import { SiteError, staticSecretResolver } from "../types.ts";
import { createSpiderSiteAdapter } from "./adapter.ts";
import type { IndexerDefinition } from "./schema.ts";
import { SiteSpider } from "./spider.ts";
@@ -78,13 +78,19 @@ describe("通用索引XML与本地分页", () => {
it("无法识别的200页面错误包含安全标题", async () => {
const mock = createMockFetch({
"/search": { body: "<html><head><title>Search temporarily unavailable</title></head><body>Later</body></html>" },
"/search": {
body: "<html><head><title>Search temporarily unavailable passkey=private-key session=session-secret</title></head><body>Later</body></html>",
},
});
const adapter = createSpiderSiteAdapter(def, {
fetch: mock.fetch,
secrets: staticSecretResolver({}),
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
});
await expect(adapter.search("special")).rejects.toThrow("HTTP 200,页面标题:Search temporarily unavailable");
const error = await adapter.search("special").catch((e: unknown) => e);
expect(error).toBeInstanceOf(SiteError);
expect((error as SiteError).message).toContain("HTTP 200,页面标题:Search temporarily unavailable");
expect((error as SiteError).message).not.toMatch(/private-key|session-secret/);
expect((error as SiteError).sample?.message).not.toMatch(/private-key|session-secret/);
});
});
@@ -2,7 +2,7 @@
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/chain/download/submission.py @ e6820a8
* @mp-source app/foundation/temporal.py @ e6820a8
* @mp-source app/foundation/temporal.py @ 3b156a5
*
* 专用 spider 共用 IO 与工具:
* - `submission._resolve_indirect_download_url` → `resolveIndirectDownloadUrl`(配置对象在内存中构造,不再 base64 编进 enclosure)
@@ -58,6 +58,7 @@ export interface SpecializedSiteDefinition {
timezone: string;
rateLimit: { minIntervalMs: number; burst: number };
capabilities: SiteCapabilities;
authSupported?: Array<"cookie" | "credential" | "apikey">;
categoryNames?: Record<string, string>;
/** 覆盖 API 基址(测试指向 mock,或站点换域) */
apiBaseUrl?: string;
+28
View File
@@ -170,6 +170,31 @@ export interface SiteCallOptions {
signal?: AbortSignal;
}
/** API 站字幕结果只保存稳定 ID;下载签名在 downloadSubtitle 时现场获取。 */
export interface SiteSubtitle {
id: string;
title: string;
fileName: string | null;
language: string | null;
sizeBytes: number | null;
uploadedAt: string | null;
downloads: number | null;
uploader: string | null;
torrentId: string | null;
detailUrl: string | null;
}
export interface SiteSubtitlePage {
items: SiteSubtitle[];
page: number;
totalPages: number | null;
}
export interface SiteSubtitleFile {
bytes: Uint8Array;
fileName: string | null;
}
export interface SiteAdapter {
readonly id: string;
readonly kind: SiteKind;
@@ -182,6 +207,9 @@ export interface SiteAdapter {
detail(siteReleaseId: string, opts?: SiteCallOptions): Promise<SiteReleaseDetail>;
/** 下载前现场取目标:会复核登录态/令牌,校验返回确为 .torrent */
downloadTarget(siteReleaseId: string, opts?: SiteCallOptions): Promise<DownloadTarget>;
searchSubtitles?(query: string, opts?: SiteCallOptions & { page?: number }): Promise<SiteSubtitlePage>;
listSubtitles?(torrentId: string, opts?: SiteCallOptions): Promise<SiteSubtitle[]>;
downloadSubtitle?(subtitleId: string, opts?: SiteCallOptions): Promise<SiteSubtitleFile>;
/** 不抛错;把失败归类为状态 */
healthCheck(opts?: SiteCallOptions): Promise<SiteHealth>;
}
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/__init__.py @ e6820a8
* @mp-source app/modules/indexer/__init__.py @ 3b156a5
*
* 站点用户信息解析器注册表与刷新入口。
* - `IndexerModule.__init__` 里 `ModuleHelper.load('app.modules.indexer.parser', filter=有 schema)` → `USER_PARSER_CLASSES`