fix(sites): integrate shared download access and register synced adapters
This commit is contained in:
1 parent
6dab134617
commit
36f3ece5b5
20 files changed
+175
-38
No files matched your search
@@ -9,12 +9,10 @@ import {
|
|||||||
fetchEpisodeGroup,
|
fetchEpisodeGroup,
|
||||||
fetchRichDetail,
|
fetchRichDetail,
|
||||||
fetchRichSeason,
|
fetchRichSeason,
|
||||||
getSiteAdapter,
|
|
||||||
getSiteDefinition,
|
getSiteDefinition,
|
||||||
groupSeasonAsRichSeason,
|
groupSeasonAsRichSeason,
|
||||||
type ProbeResult,
|
type ProbeResult,
|
||||||
qbAuthConfig,
|
qbAuthConfig,
|
||||||
type SecretResolver,
|
|
||||||
SiteError,
|
SiteError,
|
||||||
type TmdbRichDetail,
|
type TmdbRichDetail,
|
||||||
type TmdbRichSeasonDetail,
|
type TmdbRichSeasonDetail,
|
||||||
@@ -200,20 +198,8 @@ export function createPorts(
|
|||||||
const siteMeta = (raw._site ?? {}) as Record<string, unknown>;
|
const siteMeta = (raw._site ?? {}) as Record<string, unknown>;
|
||||||
const defId = [str(siteMeta.adapterSiteId), site.key].find((k) => k && getSiteDefinition(k));
|
const defId = [str(siteMeta.adapterSiteId), site.key].find((k) => k && getSiteDefinition(k));
|
||||||
if (!defId) throw new SiteError(site.id, "unsupported", `站点 ${site.key} 没有可用的适配器定义`);
|
if (!defId) throw new SiteError(site.id, "unsupported", `站点 ${site.key} 没有可用的适配器定义`);
|
||||||
const accounts = await ctx.db
|
// 与搜索/详情共用站点出口、账号会话、UA 和跨进程限流;API 基址仍取站点配置。
|
||||||
.selectFrom("site_account")
|
const adapter = rt().siteAdapter({ ...site, key: defId });
|
||||||
.select(["auth_kind", "secret_ref"])
|
|
||||||
.where("site_id", "=", site.id)
|
|
||||||
.where("enabled", "=", true)
|
|
||||||
.execute();
|
|
||||||
const secrets: SecretResolver = {
|
|
||||||
async resolve({ kind }) {
|
|
||||||
const want = kind === "apiKey" ? "api_key" : kind;
|
|
||||||
const acc = accounts.find((a) => a.auth_kind === want);
|
|
||||||
return (await resolveSecret(acc?.secret_ref ?? null)) ?? null;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const adapter = getSiteAdapter(defId, { secrets, baseUrl: site.base_url });
|
|
||||||
const t = await adapter.downloadTarget(release.site_release_id);
|
const t = await adapter.downloadTarget(release.site_release_id);
|
||||||
if (t.kind === "torrent")
|
if (t.kind === "torrent")
|
||||||
return { kind: "file", bytes: t.bytes, ...(t.fileName ? { filename: t.fileName } : {}) };
|
return { kind: "file", bytes: t.bytes, ...(t.fileName ? { filename: t.fileName } : {}) };
|
||||||
|
|||||||
@@ -199,6 +199,13 @@ export class SourcesRuntime {
|
|||||||
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
const pick = typeof ua === "function" ? ua() : ua !== undefined ? Promise.resolve(ua) : this.defaultUserAgent();
|
||||||
headers.set("user-agent", await pick);
|
headers.set("user-agent", await pick);
|
||||||
}
|
}
|
||||||
|
const credentials = init?.credentials ?? (input instanceof Request ? input.credentials : undefined);
|
||||||
|
if (credentials === "omit") {
|
||||||
|
// 签名下载不复用账号 Cookie 或浏览器 profile,包括 cf_clearance。
|
||||||
|
headers.delete("cookie");
|
||||||
|
headers.delete("authorization");
|
||||||
|
return base(input, { ...init, headers, credentials });
|
||||||
|
}
|
||||||
if (!siteId) return base(input, { ...init, headers });
|
if (!siteId) return base(input, { ...init, headers });
|
||||||
const url = input instanceof Request ? input.url : String(input);
|
const url = input instanceof Request ? input.url : String(input);
|
||||||
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
|
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
|
||||||
|
|||||||
@@ -413,10 +413,11 @@ export async function refreshSiteAccount(
|
|||||||
const profile = (def.umt as { profile?: Record<string, string> }).profile;
|
const profile = (def.umt as { profile?: Record<string, string> }).profile;
|
||||||
const vipLevels = (def.umt as { vipLevels?: string[] }).vipLevels;
|
const vipLevels = (def.umt as { vipLevels?: string[] }).vipLevels;
|
||||||
// 馒头 API 基址与适配器一致:站点配置 > 定义里的 apiBaseUrl;不再由站点地址推 api.<根域>(IP / 本地地址会拼成非法 URL)
|
// 馒头 API 基址与适配器一致:站点配置 > 定义里的 apiBaseUrl;不再由站点地址推 api.<根域>(IP / 本地地址会拼成非法 URL)
|
||||||
|
const siteConfig = (site.config ?? {}) as { apiBaseUrl?: string; userdataPath?: string };
|
||||||
const apiUrl =
|
const apiUrl =
|
||||||
def.umt.kind === "mteam"
|
def.umt.kind === "mteam"
|
||||||
? (site.config as { apiBaseUrl?: string })?.apiBaseUrl || mteamApiBase(def as MteamDefinition)
|
? siteConfig.apiBaseUrl || mteamApiBase(def as MteamDefinition)
|
||||||
: null;
|
: siteConfig.apiBaseUrl || ("api_url" in def ? def.api_url : null);
|
||||||
|
|
||||||
const fetchUserData = async (
|
const fetchUserData = async (
|
||||||
cred: { cookie?: string | null; apikey?: string | null },
|
cred: { cookie?: string | null; apikey?: string | null },
|
||||||
@@ -425,6 +426,7 @@ export async function refreshSiteAccount(
|
|||||||
const info = await refreshSiteUserData({
|
const info = await refreshSiteUserData({
|
||||||
name: site.name,
|
name: site.name,
|
||||||
url: site.base_url,
|
url: site.base_url,
|
||||||
|
...(siteConfig.userdataPath ? { userdataPath: siteConfig.userdataPath } : {}),
|
||||||
schema: (def as { schema?: string }).schema ?? null,
|
schema: (def as { schema?: string }).schema ?? null,
|
||||||
public: true,
|
public: true,
|
||||||
cookie: cred.cookie ?? null,
|
cookie: cred.cookie ?? null,
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
import { expect, it, vi } from "vitest";
|
||||||
|
import { SourcesRuntime } from "../src/modules/sources-identity/runtime.ts";
|
||||||
|
|
||||||
|
it("签名下载的 credentials omit 不复用账号或浏览器Cookie", async () => {
|
||||||
|
let sent = new Headers();
|
||||||
|
const rt = new SourcesRuntime({} as never, {
|
||||||
|
fetch: async (_input, init) => {
|
||||||
|
sent = new Headers(init?.headers);
|
||||||
|
return new Response("subtitle bytes");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const browser = vi.spyOn(rt.cloudflare, "handle");
|
||||||
|
const fetch = rt.siteFetch({ id: "original-site", proxy: false }, undefined, "Site User Agent");
|
||||||
|
const headers = { cookie: "account-cookie", authorization: "account-token" };
|
||||||
|
const url = "https://signed-cdn.example/file?credential=ephemeral";
|
||||||
|
const response = await fetch(url, { headers, credentials: "omit" });
|
||||||
|
expect(await response.text()).toBe("subtitle bytes");
|
||||||
|
expect(sent.has("cookie")).toBe(false);
|
||||||
|
expect(sent.has("authorization")).toBe(false);
|
||||||
|
expect(sent.get("user-agent")).toBe("Site User Agent");
|
||||||
|
expect(browser).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import { TaskRegistry } from "@umt/tasks";
|
import { TaskRegistry } from "@umt/tasks";
|
||||||
import { createMockMteamApi, type MockMteamApi } from "@umt/testkit";
|
import { createMockMteamApi, type MockMteamApi } from "@umt/testkit";
|
||||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import { createPorts } from "../../src/modules/acquisition-organize/ports.ts";
|
||||||
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
||||||
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
||||||
|
|
||||||
@@ -50,6 +51,27 @@ describe.skipIf(!up)("M-Team 快照与任务注册", () => {
|
|||||||
expect(JSON.stringify(rows)).not.toContain(api.apiKey);
|
expect(JSON.stringify(rows)).not.toContain(api.apiKey);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("下载种子沿用配置的 API 基址,并与站点访问共用 PG 限流桶", async () => {
|
||||||
|
const site = await h.db
|
||||||
|
.updateTable("site")
|
||||||
|
.set({ rate_limit: JSON.stringify({ minIntervalMs: 10, burst: 1 }) })
|
||||||
|
.where("id", "=", siteId)
|
||||||
|
.returningAll()
|
||||||
|
.executeTakeFirstOrThrow();
|
||||||
|
const release = await h.db
|
||||||
|
.selectFrom("source_release")
|
||||||
|
.selectAll()
|
||||||
|
.where("site_id", "=", siteId)
|
||||||
|
.executeTakeFirstOrThrow();
|
||||||
|
const before = api.issuedTokens;
|
||||||
|
const source = await createPorts(h.ctx).torrentSource({ release, site });
|
||||||
|
expect(source.kind).toBe("file");
|
||||||
|
expect(api.issuedTokens).toBe(before + 1);
|
||||||
|
expect(
|
||||||
|
await h.db.selectFrom("rate_limit_bucket").select("key").where("key", "=", `site:${siteId}`).executeTakeFirst(),
|
||||||
|
).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
it("key 失效:账号标失效 + 待处理,快照结果 failed", async () => {
|
it("key 失效:账号标失效 + 待处理,快照结果 failed", async () => {
|
||||||
api.setFailMode("banned_key");
|
api.setFailMode("banned_key");
|
||||||
const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
|
const r = await snapshotSite(h.rt, { siteId, kind: "manual" });
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ const UNIT = [
|
|||||||
"test/mapping-compat.test.ts",
|
"test/mapping-compat.test.ts",
|
||||||
"test/modules.test.ts",
|
"test/modules.test.ts",
|
||||||
"test/sse.test.ts",
|
"test/sse.test.ts",
|
||||||
|
"test/site-fetch-credentials.test.ts",
|
||||||
"test/static.test.ts",
|
"test/static.test.ts",
|
||||||
"test/system-resources.test.ts",
|
"test/system-resources.test.ts",
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ export const SUBTITLE_LANGUAGE_LABEL: Record<SubtitleLanguage, string> = {
|
|||||||
ko: "韩文",
|
ko: "韩文",
|
||||||
unknown: "未知",
|
unknown: "未知",
|
||||||
};
|
};
|
||||||
const SOURCE_LABEL = { upload: "上传", subhd: "SubHD", zimuku: "字幕库" } as const;
|
const SOURCE_LABEL = { upload: "上传", subhd: "SubHD", zimuku: "字幕库", mteam: "馒头" } as const;
|
||||||
const SKIP = "__skip__";
|
const SKIP = "__skip__";
|
||||||
const ACCEPT = ".srt,.ass,.ssa,.vtt,.sub,.sbv,.zip";
|
const ACCEPT = ".srt,.ass,.ssa,.vtt,.sub,.sbv,.zip";
|
||||||
|
|
||||||
|
|||||||
@@ -32,9 +32,9 @@ const PUBLIC = ["anibt", "animegarden", "bangumimoe", "mikan", "nyaa"];
|
|||||||
describe("站点定义:加载时 zod 校验", () => {
|
describe("站点定义:加载时 zod 校验", () => {
|
||||||
const ids = Object.keys(MOVIECLAW_DEFINITION_JSON).sort();
|
const ids = Object.keys(MOVIECLAW_DEFINITION_JSON).sort();
|
||||||
|
|
||||||
it("MovieClaw v0.27.0 全部 23 站转换结果 + 公开 BT 站均在注册表中", () => {
|
it("MovieClaw v0.27.0 全部 23 站、公开 BT 站与手写 Milkie 定义均在注册表中", () => {
|
||||||
expect(ids).toHaveLength(23);
|
expect(ids).toHaveLength(23);
|
||||||
expect(listSiteDefinitions().map((d) => d.id)).toEqual([...ids, ...PUBLIC].sort());
|
expect(listSiteDefinitions().map((d) => d.id)).toEqual([...ids, ...PUBLIC, "milkie"].sort());
|
||||||
});
|
});
|
||||||
|
|
||||||
it.each(ids)("%s:schema 校验通过,provenance 指向 MovieClaw v0.27.0 MIT", (id) => {
|
it.each(ids)("%s:schema 校验通过,provenance 指向 MovieClaw v0.27.0 MIT", (id) => {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import type { MteamDefinition } from "../mteam/definition.ts";
|
|||||||
import type { IndexerDefinition } from "../spider/schema.ts";
|
import type { IndexerDefinition } from "../spider/schema.ts";
|
||||||
import type { SpecializedSiteDefinition } from "../spiders/types.ts";
|
import type { SpecializedSiteDefinition } from "../spiders/types.ts";
|
||||||
import { MOVIECLAW_DEFINITION_JSON } from "./manifest.generated.ts";
|
import { MOVIECLAW_DEFINITION_JSON } from "./manifest.generated.ts";
|
||||||
|
import milkieJson from "./milkie.json" with { type: "json" };
|
||||||
import { PUBLIC_SITE_DEFINITION_JSON } from "./public.ts";
|
import { PUBLIC_SITE_DEFINITION_JSON } from "./public.ts";
|
||||||
import { parseIndexerDefinition, parseMteamDefinition, parseSpecializedSiteDefinition } from "./schema.ts";
|
import { parseIndexerDefinition, parseMteamDefinition, parseSpecializedSiteDefinition } from "./schema.ts";
|
||||||
|
|
||||||
@@ -27,13 +28,12 @@ export function loadIndexerDefinition(json: unknown): IndexerDefinition {
|
|||||||
return parseIndexerDefinition(json);
|
return parseIndexerDefinition(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 随产品发布的全部定义(已校验),键为站点 id:MovieClaw 转换的 PT 站 + 手写的公开 BT 站 */
|
/** 随产品发布的全部定义(已校验),键为站点 id:MovieClaw 转换站、手写 API 站与公开 BT 站 */
|
||||||
export const BUILTIN_SITE_DEFINITIONS: Readonly<Record<string, SiteDefinition>> = Object.freeze(
|
export const BUILTIN_SITE_DEFINITIONS: Readonly<Record<string, SiteDefinition>> = Object.freeze(
|
||||||
Object.fromEntries(
|
Object.fromEntries(
|
||||||
Object.entries({ ...MOVIECLAW_DEFINITION_JSON, ...PUBLIC_SITE_DEFINITION_JSON }).map(([id, json]) => [
|
Object.entries({ ...MOVIECLAW_DEFINITION_JSON, ...PUBLIC_SITE_DEFINITION_JSON, milkie: milkieJson }).map(
|
||||||
id,
|
([id, json]) => [id, loadSiteDefinition(json)],
|
||||||
loadSiteDefinition(json),
|
),
|
||||||
]),
|
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"id": "milkie",
|
||||||
|
"name": "Milkie",
|
||||||
|
"domain": "https://milkie.cc/",
|
||||||
|
"parser": "Milkie",
|
||||||
|
"enabled": false,
|
||||||
|
"provenance": {
|
||||||
|
"source": "handwritten",
|
||||||
|
"ref": "v3.2.4",
|
||||||
|
"commit": "3b156a545f14fea0285cd3ed3ea4aa49d97faa57",
|
||||||
|
"license": "GPL-3.0-only",
|
||||||
|
"file": "app/modules/indexer/spider/milkie.py",
|
||||||
|
"note": "按公开 API 行为重写;凭据由 SecretResolver 注入,未做真实站点验收"
|
||||||
|
},
|
||||||
|
"umt": {
|
||||||
|
"version": "2026-11-05",
|
||||||
|
"kind": "api",
|
||||||
|
"authSupported": ["apikey"],
|
||||||
|
"displayName": "Milkie",
|
||||||
|
"timezone": "+00:00",
|
||||||
|
"rateLimit": { "minIntervalMs": 2000, "burst": 2 },
|
||||||
|
"capabilities": { "list": true, "search": true, "detail": false, "download": true, "rss": false },
|
||||||
|
"categoryNames": { "1": "电影", "2": "电视剧", "3": "音乐" }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -216,6 +216,8 @@ export const IndexerDefinitionSchema = z.strictObject({
|
|||||||
export const MteamDefinitionSchema = z.strictObject({
|
export const MteamDefinitionSchema = z.strictObject({
|
||||||
...BaseDefinitionShape,
|
...BaseDefinitionShape,
|
||||||
parser: z.literal("mTorrent"),
|
parser: z.literal("mTorrent"),
|
||||||
|
api_domain: z.string().min(1).optional(),
|
||||||
|
api_host: z.string().min(1).optional(),
|
||||||
timeout: z.number().positive(),
|
timeout: z.number().positive(),
|
||||||
umt: z.strictObject({
|
umt: z.strictObject({
|
||||||
version: z.string().min(1),
|
version: z.string().min(1),
|
||||||
@@ -242,6 +244,7 @@ export const SpecializedSiteDefinitionSchema = z.strictObject({
|
|||||||
"TNodeSpider",
|
"TNodeSpider",
|
||||||
"TorrentLeech",
|
"TorrentLeech",
|
||||||
"Yema",
|
"Yema",
|
||||||
|
"Milkie",
|
||||||
"Nyaa",
|
"Nyaa",
|
||||||
"Mikan",
|
"Mikan",
|
||||||
"AniBT",
|
"AniBT",
|
||||||
@@ -251,6 +254,8 @@ export const SpecializedSiteDefinitionSchema = z.strictObject({
|
|||||||
public: z.boolean().optional(),
|
public: z.boolean().optional(),
|
||||||
category: CategoryConfigSchema.optional(),
|
category: CategoryConfigSchema.optional(),
|
||||||
api_url: z.string().url().optional(),
|
api_url: z.string().url().optional(),
|
||||||
|
api_domain: z.string().min(1).optional(),
|
||||||
|
api_host: z.string().min(1).optional(),
|
||||||
umt: z.strictObject({
|
umt: z.strictObject({
|
||||||
version: z.string().min(1),
|
version: z.string().min(1),
|
||||||
kind: z.literal("api"),
|
kind: z.literal("api"),
|
||||||
@@ -258,6 +263,7 @@ export const SpecializedSiteDefinitionSchema = z.strictObject({
|
|||||||
timezone: TimezoneSchema,
|
timezone: TimezoneSchema,
|
||||||
rateLimit: RateLimitSchema,
|
rateLimit: RateLimitSchema,
|
||||||
capabilities: CapabilitiesSchema,
|
capabilities: CapabilitiesSchema,
|
||||||
|
authSupported: z.array(z.enum(["cookie", "credential", "apikey"])).optional(),
|
||||||
categoryNames: z.record(z.string(), z.string()).optional(),
|
categoryNames: z.record(z.string(), z.string()).optional(),
|
||||||
apiBaseUrl: z.string().url().optional(),
|
apiBaseUrl: z.string().url().optional(),
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -150,6 +150,9 @@ export {
|
|||||||
type SitePromotion,
|
type SitePromotion,
|
||||||
type SiteRelease,
|
type SiteRelease,
|
||||||
type SiteReleaseDetail,
|
type SiteReleaseDetail,
|
||||||
|
type SiteSubtitle,
|
||||||
|
type SiteSubtitleFile,
|
||||||
|
type SiteSubtitlePage,
|
||||||
staticSecretResolver,
|
staticSecretResolver,
|
||||||
} from "./types.ts";
|
} from "./types.ts";
|
||||||
export * from "./user-parsers/index.ts";
|
export * from "./user-parsers/index.ts";
|
||||||
|
|||||||
@@ -186,7 +186,12 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps
|
|||||||
throw new SiteError(def.id, "rate_limited", `genDlToken ${id}:未返回下载链接(可能未认证或达到下载频率限制)`);
|
throw new SiteError(def.id, "rate_limited", `genDlToken ${id}:未返回下载链接(可能未认证或达到下载频率限制)`);
|
||||||
}
|
}
|
||||||
// MP:涉及解析地址的不使用 Cookie 下载;直链含一次性签名,只在内存中使用
|
// MP:涉及解析地址的不使用 Cookie 下载;直链含一次性签名,只在内存中使用
|
||||||
const res = await siteFetch(http, url, { method: "GET", ...(opts.signal ? { signal: opts.signal } : {}) });
|
const res = await siteFetch(http, url, {
|
||||||
|
method: "GET",
|
||||||
|
credentials: "omit",
|
||||||
|
referrerPolicy: "no-referrer",
|
||||||
|
...(opts.signal ? { signal: opts.signal } : {}),
|
||||||
|
});
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
await res.body?.cancel().catch(() => {});
|
await res.body?.cancel().catch(() => {});
|
||||||
throw new SiteError(
|
throw new SiteError(
|
||||||
|
|||||||
@@ -17,11 +17,20 @@ afterEach(async () => {
|
|||||||
api = undefined;
|
api = undefined;
|
||||||
});
|
});
|
||||||
|
|
||||||
function adapterFor(m: MockMteamApi, key: string | null = m.apiKey, limiter?: SiteRateLimiter): SiteAdapter {
|
function adapterFor(
|
||||||
|
m: MockMteamApi,
|
||||||
|
key: string | null = m.apiKey,
|
||||||
|
limiter?: SiteRateLimiter,
|
||||||
|
requests?: RequestInit[],
|
||||||
|
): SiteAdapter {
|
||||||
return createMteamAdapter(mteamDefinition, {
|
return createMteamAdapter(mteamDefinition, {
|
||||||
baseUrl: m.baseUrl,
|
baseUrl: m.baseUrl,
|
||||||
secrets: staticSecretResolver(key ? { mteam: { apiKey: key } } : {}),
|
secrets: staticSecretResolver(key ? { mteam: { apiKey: key } } : {}),
|
||||||
limiter: limiter ?? createSiteRateLimiter({ minIntervalMs: 0 }),
|
limiter: limiter ?? createSiteRateLimiter({ minIntervalMs: 0 }),
|
||||||
|
fetch: (input, init) => {
|
||||||
|
requests?.push(init ?? {});
|
||||||
|
return globalThis.fetch(input, init);
|
||||||
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,13 +138,15 @@ describe("M-Team API 适配器(mock)", () => {
|
|||||||
|
|
||||||
it("下载令牌:现场 genDlToken → 一次性直链 → .torrent,infohash 校验", async () => {
|
it("下载令牌:现场 genDlToken → 一次性直链 → .torrent,infohash 校验", async () => {
|
||||||
api = await createMockMteamApi();
|
api = await createMockMteamApi();
|
||||||
const a = adapterFor(api);
|
const requests: RequestInit[] = [];
|
||||||
|
const a = adapterFor(api, api.apiKey, undefined, requests);
|
||||||
const t = await a.downloadTarget("912233");
|
const t = await a.downloadTarget("912233");
|
||||||
expect(api.issuedTokens).toBe(1);
|
expect(api.issuedTokens).toBe(1);
|
||||||
expect(api.requests.map((r) => r.path.split("?")[0])).toEqual(["/api/torrent/genDlToken", "/api/rss/dlv2"]);
|
expect(api.requests.map((r) => r.path.split("?")[0])).toEqual(["/api/torrent/genDlToken", "/api/rss/dlv2"]);
|
||||||
// MP:genDlToken 的 id 走 query 参数;直链下载不带 x-api-key
|
// MP:genDlToken 的 id 走 query 参数;直链下载不带 x-api-key
|
||||||
expect(api.requests[0]!.path).toBe("/api/torrent/genDlToken?id=912233");
|
expect(api.requests[0]!.path).toBe("/api/torrent/genDlToken?id=912233");
|
||||||
expect(api.requests[1]!.headers["x-api-key"]).toBeUndefined();
|
expect(api.requests[1]!.headers["x-api-key"]).toBeUndefined();
|
||||||
|
expect(requests[1]).toMatchObject({ credentials: "omit", referrerPolicy: "no-referrer" });
|
||||||
expect(t.kind).toBe("torrent");
|
expect(t.kind).toBe("torrent");
|
||||||
if (t.kind === "torrent") expect(t.infoHash).toBe(api.infoHashOf("912233"));
|
if (t.kind === "torrent") expect(t.infoHash).toBe(api.infoHashOf("912233"));
|
||||||
// 每次下载都重新签发令牌
|
// 每次下载都重新签发令牌
|
||||||
@@ -162,15 +173,27 @@ describe("M-Team API 适配器(mock)", () => {
|
|||||||
|
|
||||||
it("非 JSON 维护页 → site_maintenance;业务限流/HTTP 429 → rate_limited 并顺延限流器", async () => {
|
it("非 JSON 维护页 → site_maintenance;业务限流/HTTP 429 → rate_limited 并顺延限流器", async () => {
|
||||||
api = await createMockMteamApi();
|
api = await createMockMteamApi();
|
||||||
const limiter = createSiteRateLimiter({ minIntervalMs: 0 });
|
let clock = 0;
|
||||||
|
const waits: number[] = [];
|
||||||
|
const limiter = createSiteRateLimiter({
|
||||||
|
minIntervalMs: 0,
|
||||||
|
now: () => clock,
|
||||||
|
sleep: async (ms) => {
|
||||||
|
waits.push(ms);
|
||||||
|
clock += ms;
|
||||||
|
},
|
||||||
|
});
|
||||||
const a = adapterFor(api, api.apiKey, limiter);
|
const a = adapterFor(api, api.apiKey, limiter);
|
||||||
api.setFailMode("non_json");
|
api.setFailMode("non_json");
|
||||||
await expectSiteError(a.list(1), "site_maintenance");
|
await expectSiteError(a.list(1), "site_maintenance");
|
||||||
api.setFailMode("rate_limit");
|
api.setFailMode("rate_limit");
|
||||||
await expectSiteError(a.list(1), "rate_limited");
|
const business = await expectSiteError(a.list(1), "rate_limited");
|
||||||
|
expect(business.retryAfterMs).toBe(3600_000);
|
||||||
|
expect(limiter.tryAcquire()).toBe(false);
|
||||||
api.setFailMode("http_429");
|
api.setFailMode("http_429");
|
||||||
const e = await expectSiteError(a.list(1), "rate_limited");
|
const e = await expectSiteError(a.list(1), "rate_limited");
|
||||||
expect(e.retryAfterMs).toBe(5000);
|
expect(e.retryAfterMs).toBe(5000);
|
||||||
|
expect(waits).toEqual([3600_000]);
|
||||||
expect(limiter.tryAcquire()).toBe(false);
|
expect(limiter.tryAcquire()).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import { inspectTorrentIsolated } from "../../isolation.ts";
|
|||||||
import { redactText } from "../../rss/redact.ts";
|
import { redactText } from "../../rss/redact.ts";
|
||||||
import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts";
|
import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts";
|
||||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||||||
import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts";
|
import { buildFailureSample, redactPageHtml, type SiteRequestKind } from "../failure-sample.ts";
|
||||||
import { readBody, readText, siteFetch } from "../http.ts";
|
import { readBody, readText, siteFetch } from "../http.ts";
|
||||||
import {
|
import {
|
||||||
type DownloadTarget,
|
type DownloadTarget,
|
||||||
@@ -155,7 +155,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
|||||||
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
|
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
|
||||||
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
|
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
|
||||||
const rule = challengeRuleOfUnrecognizedPage(page.html);
|
const rule = challengeRuleOfUnrecognizedPage(page.html);
|
||||||
const title = redactText(loadHtml(page.html)("title").first().text().replace(/\s+/g, " ").trim()).slice(0, 120);
|
const title = redactPageHtml(loadHtml(page.html)("title").first().text()).replace(/\s+/g, " ").trim().slice(0, 120);
|
||||||
const message = rule
|
const message = rule
|
||||||
? `${what}:${PAGE_LABEL[kind]}需要人机验证(Cloudflare 等),${challengeOutcomeText("unhandled")}`
|
? `${what}:${PAGE_LABEL[kind]}需要人机验证(Cloudflare 等),${challengeOutcomeText("unhandled")}`
|
||||||
: `${parseMessage}(HTTP ${page.status}${title ? `,页面标题:${title}` : ""};页面样本见站点页技术信息)`;
|
: `${parseMessage}(HTTP ${page.status}${title ? `,页面标题:${title}` : ""};页面样本见站点页技术信息)`;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { createMockFetch } from "@umt/testkit";
|
import { createMockFetch } from "@umt/testkit";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { createSiteRateLimiter } from "../rate-limit.ts";
|
import { createSiteRateLimiter } from "../rate-limit.ts";
|
||||||
import { staticSecretResolver } from "../types.ts";
|
import { SiteError, staticSecretResolver } from "../types.ts";
|
||||||
import { createSpiderSiteAdapter } from "./adapter.ts";
|
import { createSpiderSiteAdapter } from "./adapter.ts";
|
||||||
import type { IndexerDefinition } from "./schema.ts";
|
import type { IndexerDefinition } from "./schema.ts";
|
||||||
import { SiteSpider } from "./spider.ts";
|
import { SiteSpider } from "./spider.ts";
|
||||||
@@ -78,13 +78,19 @@ describe("通用索引XML与本地分页", () => {
|
|||||||
|
|
||||||
it("无法识别的200页面错误包含安全标题", async () => {
|
it("无法识别的200页面错误包含安全标题", async () => {
|
||||||
const mock = createMockFetch({
|
const mock = createMockFetch({
|
||||||
"/search": { body: "<html><head><title>Search temporarily unavailable</title></head><body>Later</body></html>" },
|
"/search": {
|
||||||
|
body: "<html><head><title>Search temporarily unavailable passkey=private-key session=session-secret</title></head><body>Later</body></html>",
|
||||||
|
},
|
||||||
});
|
});
|
||||||
const adapter = createSpiderSiteAdapter(def, {
|
const adapter = createSpiderSiteAdapter(def, {
|
||||||
fetch: mock.fetch,
|
fetch: mock.fetch,
|
||||||
secrets: staticSecretResolver({}),
|
secrets: staticSecretResolver({}),
|
||||||
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
|
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
|
||||||
});
|
});
|
||||||
await expect(adapter.search("special")).rejects.toThrow("HTTP 200,页面标题:Search temporarily unavailable");
|
const error = await adapter.search("special").catch((e: unknown) => e);
|
||||||
|
expect(error).toBeInstanceOf(SiteError);
|
||||||
|
expect((error as SiteError).message).toContain("HTTP 200,页面标题:Search temporarily unavailable");
|
||||||
|
expect((error as SiteError).message).not.toMatch(/private-key|session-secret/);
|
||||||
|
expect((error as SiteError).sample?.message).not.toMatch(/private-key|session-secret/);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
* SPDX-License-Identifier: GPL-3.0-only
|
* SPDX-License-Identifier: GPL-3.0-only
|
||||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||||
* @mp-source app/chain/download/submission.py @ e6820a8
|
* @mp-source app/chain/download/submission.py @ e6820a8
|
||||||
* @mp-source app/foundation/temporal.py @ e6820a8
|
* @mp-source app/foundation/temporal.py @ 3b156a5
|
||||||
*
|
*
|
||||||
* 专用 spider 共用 IO 与工具:
|
* 专用 spider 共用 IO 与工具:
|
||||||
* - `submission._resolve_indirect_download_url` → `resolveIndirectDownloadUrl`(配置对象在内存中构造,不再 base64 编进 enclosure)
|
* - `submission._resolve_indirect_download_url` → `resolveIndirectDownloadUrl`(配置对象在内存中构造,不再 base64 编进 enclosure)
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ export interface SpecializedSiteDefinition {
|
|||||||
timezone: string;
|
timezone: string;
|
||||||
rateLimit: { minIntervalMs: number; burst: number };
|
rateLimit: { minIntervalMs: number; burst: number };
|
||||||
capabilities: SiteCapabilities;
|
capabilities: SiteCapabilities;
|
||||||
|
authSupported?: Array<"cookie" | "credential" | "apikey">;
|
||||||
categoryNames?: Record<string, string>;
|
categoryNames?: Record<string, string>;
|
||||||
/** 覆盖 API 基址(测试指向 mock,或站点换域) */
|
/** 覆盖 API 基址(测试指向 mock,或站点换域) */
|
||||||
apiBaseUrl?: string;
|
apiBaseUrl?: string;
|
||||||
|
|||||||
@@ -170,6 +170,31 @@ export interface SiteCallOptions {
|
|||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** API 站字幕结果只保存稳定 ID;下载签名在 downloadSubtitle 时现场获取。 */
|
||||||
|
export interface SiteSubtitle {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
fileName: string | null;
|
||||||
|
language: string | null;
|
||||||
|
sizeBytes: number | null;
|
||||||
|
uploadedAt: string | null;
|
||||||
|
downloads: number | null;
|
||||||
|
uploader: string | null;
|
||||||
|
torrentId: string | null;
|
||||||
|
detailUrl: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SiteSubtitlePage {
|
||||||
|
items: SiteSubtitle[];
|
||||||
|
page: number;
|
||||||
|
totalPages: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SiteSubtitleFile {
|
||||||
|
bytes: Uint8Array;
|
||||||
|
fileName: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
export interface SiteAdapter {
|
export interface SiteAdapter {
|
||||||
readonly id: string;
|
readonly id: string;
|
||||||
readonly kind: SiteKind;
|
readonly kind: SiteKind;
|
||||||
@@ -182,6 +207,9 @@ export interface SiteAdapter {
|
|||||||
detail(siteReleaseId: string, opts?: SiteCallOptions): Promise<SiteReleaseDetail>;
|
detail(siteReleaseId: string, opts?: SiteCallOptions): Promise<SiteReleaseDetail>;
|
||||||
/** 下载前现场取目标:会复核登录态/令牌,校验返回确为 .torrent */
|
/** 下载前现场取目标:会复核登录态/令牌,校验返回确为 .torrent */
|
||||||
downloadTarget(siteReleaseId: string, opts?: SiteCallOptions): Promise<DownloadTarget>;
|
downloadTarget(siteReleaseId: string, opts?: SiteCallOptions): Promise<DownloadTarget>;
|
||||||
|
searchSubtitles?(query: string, opts?: SiteCallOptions & { page?: number }): Promise<SiteSubtitlePage>;
|
||||||
|
listSubtitles?(torrentId: string, opts?: SiteCallOptions): Promise<SiteSubtitle[]>;
|
||||||
|
downloadSubtitle?(subtitleId: string, opts?: SiteCallOptions): Promise<SiteSubtitleFile>;
|
||||||
/** 不抛错;把失败归类为状态 */
|
/** 不抛错;把失败归类为状态 */
|
||||||
healthCheck(opts?: SiteCallOptions): Promise<SiteHealth>;
|
healthCheck(opts?: SiteCallOptions): Promise<SiteHealth>;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
/**
|
/**
|
||||||
* SPDX-License-Identifier: GPL-3.0-only
|
* SPDX-License-Identifier: GPL-3.0-only
|
||||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||||
* @mp-source app/modules/indexer/__init__.py @ e6820a8
|
* @mp-source app/modules/indexer/__init__.py @ 3b156a5
|
||||||
*
|
*
|
||||||
* 站点用户信息解析器注册表与刷新入口。
|
* 站点用户信息解析器注册表与刷新入口。
|
||||||
* - `IndexerModule.__init__` 里 `ModuleHelper.load('app.modules.indexer.parser', filter=有 schema)` → `USER_PARSER_CLASSES`
|
* - `IndexerModule.__init__` 里 `ModuleHelper.load('app.modules.indexer.parser', filter=有 schema)` → `USER_PARSER_CLASSES`
|
||||||
|
|||||||
Reference in new issue
Block a user