From edf732f37b3bb125b21657c4a73d3fc9a9407bcd Mon Sep 17 00:00:00 2001 From: jyxjjj <773933146@qq.com> Date: Sat, 19 Sep 2026 23:59:44 +0800 Subject: [PATCH] fix(storage): reduce worker CPU usage - Derive one HKDF-backed AES key per config operation and migrate legacy envelopes on save - Await OneDrive token persistence and fold admin refresh updates into a single config write - Remove redundant driver initialization and add config encryption regression coverage Co-authored-by: Codex <267193182+codex@users.noreply.github.com> Signed-off-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> --- src/backend/drivers/onedrive/driver.ts | 4 +- src/backend/drivers/onedrive/util.ts | 4 +- src/backend/internal/model/db.ts | 88 +++++++++++++-------- src/backend/internal/model/db_cache.test.ts | 28 +++++++ src/backend/internal/op/storage.ts | 62 ++++++++++----- src/backend/pkg/crypto.ts | 73 ++++++++++++++++- src/backend/pkg/crypto_security.test.ts | 18 ++++- src/backend/server/admin.ts | 11 ++- 8 files changed, 225 insertions(+), 63 deletions(-) diff --git a/src/backend/drivers/onedrive/driver.ts b/src/backend/drivers/onedrive/driver.ts index f5f83f8a..1f648844 100644 --- a/src/backend/drivers/onedrive/driver.ts +++ b/src/backend/drivers/onedrive/driver.ts @@ -25,11 +25,11 @@ export class Onedrive implements StorageDriver { order_direction: string = "asc" accessToken: string = "" - onTokenUpdate?: (token: string) => void + onTokenUpdate?: (token: string) => void | Promise constructor( addition?: Partial, - onTokenUpdate?: (token: string) => void, + onTokenUpdate?: (token: string) => void | Promise, ) { if (addition) { if (addition.root_folder_path !== undefined) diff --git a/src/backend/drivers/onedrive/util.ts b/src/backend/drivers/onedrive/util.ts index 03e012a3..23c93383 100644 --- a/src/backend/drivers/onedrive/util.ts +++ b/src/backend/drivers/onedrive/util.ts @@ -21,7 +21,7 @@ export async function refreshToken( } d.accessToken = data.access_token d.refresh_token = data.refresh_token - ;(d as any).onTokenUpdate?.(d.refresh_token) + await (d as any).onTokenUpdate?.(d.refresh_token) return } @@ -51,7 +51,7 @@ export async function refreshToken( } d.refresh_token = data.refresh_token d.accessToken = data.access_token - ;(d as any).onTokenUpdate?.(d.refresh_token) + await (d as any).onTokenUpdate?.(d.refresh_token) } export async function requestApi( diff --git a/src/backend/internal/model/db.ts b/src/backend/internal/model/db.ts index 39c784ad..04008902 100644 --- a/src/backend/internal/model/db.ts +++ b/src/backend/internal/model/db.ts @@ -1,4 +1,9 @@ -import { encrypt, decrypt } from "../../pkg/crypto" +import { + decrypt, + decryptConfigValue, + deriveConfigEncryptionKey, + encryptConfigValue, +} from "../../pkg/crypto" import { generateSecret, readPersistedSecret, @@ -1272,7 +1277,15 @@ export const getDb = async (envCtx?: any) => { // 保持既有部署（无密钥）向后兼容。已存在的明文数据不带前缀，unseal 时原样 // 返回，不会因升级而丢失。 // ============================================================ -const ENCRYPTION_PREFIX = "enc:v1:" +const LEGACY_ENCRYPTION_PREFIX = "enc:v1:" +const ENCRYPTION_PREFIX = "enc:v2:" + +function isSealedValue(value: string): boolean { + return ( + value.startsWith(ENCRYPTION_PREFIX) || + value.startsWith(LEGACY_ENCRYPTION_PREFIX) + ) +} const SENSITIVE_SETTING_KEYS = new Set([ "token", @@ -1557,16 +1570,26 @@ export async function ensureEncryptionSecret( } } -async function sealValue(value: string, key: string): Promise { +async function sealValue(value: string, key: CryptoKey): Promise { if (!value) return value - if (value.startsWith(ENCRYPTION_PREFIX)) return value // idempotent - return ENCRYPTION_PREFIX + (await encrypt(value, key)) + if (isSealedValue(value)) return value // idempotent + return ENCRYPTION_PREFIX + (await encryptConfigValue(value, key)) } -async function unsealValue(value: string, key: string): Promise { - if (!value || !value.startsWith(ENCRYPTION_PREFIX)) return value +async function unsealValue( + value: string, + secret: string, + configKey: () => Promise, +): Promise { + if (!value || !isSealedValue(value)) return value try { - return await decrypt(value.slice(ENCRYPTION_PREFIX.length), key) + if (value.startsWith(ENCRYPTION_PREFIX)) { + return await decryptConfigValue( + value.slice(ENCRYPTION_PREFIX.length), + await configKey(), + ) + } + return await decrypt(value.slice(LEGACY_ENCRYPTION_PREFIX.length), secret) } catch (e) { console.warn( "[DB] Failed to decrypt a sealed secret (wrong JWT_SECRET?):", @@ -1579,6 +1602,8 @@ async function unsealValue(value: string, key: string): Promise { async function sealDb(data: any, key: string | null): Promise { if (!key || !data) return data const copy = JSON.parse(JSON.stringify(data)) + // Derive once per save. All fields still receive independent random GCM IVs. + const configKey = await deriveConfigEncryptionKey(key) // 1. 加密存储配置中的 addition 字段（网盘凭据） for (const s of copy.storages || []) { @@ -1586,14 +1611,14 @@ async function sealDb(data: any, key: string | null): Promise { const str = typeof s.addition === "string" ? s.addition : JSON.stringify(s.addition) if (str && str !== "{}") { - s.addition = await sealValue(str, key) + s.addition = await sealValue(str, configKey) } } // 2. 加密敏感的系统设置 for (const st of copy.settings || []) { if (st && SENSITIVE_SETTING_KEYS.has(st.key) && st.value) { - st.value = await sealValue(String(st.value), key) + st.value = await sealValue(String(st.value), configKey) } } @@ -1601,11 +1626,11 @@ async function sealDb(data: any, key: string | null): Promise { for (const u of copy.users || []) { // OTP 密钥 if (u && u.otp_secret) { - u.otp_secret = await sealValue(String(u.otp_secret), key) + u.otp_secret = await sealValue(String(u.otp_secret), configKey) } // 密码二次加密（defense-in-depth，即使已哈希也加密存储） if (u && u.password) { - u.password = await sealValue(String(u.password), key) + u.password = await sealValue(String(u.password), configKey) } } @@ -1615,39 +1640,40 @@ async function sealDb(data: any, key: string | null): Promise { /** * 解密并发上限。 * - * 解密是 WebCrypto + PBKDF2（10 万次迭代）的异步重活：串行会让墙钟随字段数 - * 线性增长，而一次性全部并发又会在字段极多时造成 CPU/内存峰值。16 是兼顾 - * serverless 延迟与峰值的折中值。 + * v2 密文只需一次快速密钥派生；旧 v1 密文仍需 PBKDF2（10 万次迭代），并在 + * 下次保存时自动迁移。限制并发可避免旧数据字段很多时产生 CPU/内存峰值。 */ const UNSEAL_CONCURRENCY = 16 async function unsealDb(data: any, key: string | null): Promise { if (!key || !data) return + // v2 的派生结果在本次加载中共享；纯 v1 数据不会做这次派生。 + let configKey: Promise | null = null + const getConfigKey = () => (configKey ||= deriveConfigEncryptionKey(key)) + // 并行解密（带并发上限）： // - // 原先三类字段（storage/setting/user）各自串行 await，字段一多就是「N 次 - // await 叠加」；而 decrypt 走 WebCrypto + PBKDF2（10 万次迭代），是真正的 - // 异步重活，且该函数在一次请求内会被调用多次（历史缺陷下更是数十次）， - // 是加载变慢的主要贡献之一。 + // 原先三类字段（storage/setting/user）各自串行 await，旧 v1 字段一多就是 + // 「N 次 PBKDF2（10 万次迭代）」叠加，且该函数在一次请求内会被调用多次 + // （历史缺陷下更是数十次），是加载变慢的主要贡献之一。 // // 这里先**同步收集 thunk**（不在收集阶段就把解密全部发起），再按 // UNSEAL_CONCURRENCY 分批 await：既拿到并行带来的墙钟收益，又避免字段极多 // （如数千用户）时一次性并发过多造成 CPU/内存峰值。 const tasks: Array<() => Promise> = [] - // 1. 解密存储配置 for (const s of data.storages || []) { if ( s && typeof s.addition === "string" && - s.addition.startsWith(ENCRYPTION_PREFIX) + isSealedValue(s.addition) ) { const target = s const cipher = target.addition tasks.push(async () => { - target.addition = await unsealValue(cipher, key) + target.addition = await unsealValue(cipher, key, getConfigKey) }) } } @@ -1658,12 +1684,12 @@ async function unsealDb(data: any, key: string | null): Promise { st && SENSITIVE_SETTING_KEYS.has(st.key) && typeof st.value === "string" && - st.value.startsWith(ENCRYPTION_PREFIX) + isSealedValue(st.value) ) { const target = st const cipher = target.value tasks.push(async () => { - target.value = await unsealValue(cipher, key) + target.value = await unsealValue(cipher, key, getConfigKey) }) } } @@ -1672,25 +1698,19 @@ async function unsealDb(data: any, key: string | null): Promise { for (const u of data.users || []) { if (!u) continue // OTP 密钥 - if ( - typeof u.otp_secret === "string" && - u.otp_secret.startsWith(ENCRYPTION_PREFIX) - ) { + if (typeof u.otp_secret === "string" && isSealedValue(u.otp_secret)) { const target = u const cipher = target.otp_secret tasks.push(async () => { - target.otp_secret = await unsealValue(cipher, key) + target.otp_secret = await unsealValue(cipher, key, getConfigKey) }) } // 密码解密 - if ( - typeof u.password === "string" && - u.password.startsWith(ENCRYPTION_PREFIX) - ) { + if (typeof u.password === "string" && isSealedValue(u.password)) { const target = u const cipher = target.password tasks.push(async () => { - target.password = await unsealValue(cipher, key) + target.password = await unsealValue(cipher, key, getConfigKey) }) } } diff --git a/src/backend/internal/model/db_cache.test.ts b/src/backend/internal/model/db_cache.test.ts index e88c8c2b..93015c34 100644 --- a/src/backend/internal/model/db_cache.test.ts +++ b/src/backend/internal/model/db_cache.test.ts @@ -129,6 +129,34 @@ test("getDb: saveDb 后无参读取可观察到最新写入（写后读一致） assert.equal(getData().settings[0].value, "Changed") }) +test("saveDb: sensitive fields use the low-CPU v2 envelope and remain readable", async () => { + __resetDbCacheForTest() + const { backend, getData } = createCountingBackend({ + ...SAMPLE, + users: [{ id: 1, username: "admin", password: "password-hash" }], + storages: [{ id: 1, addition: '{"refresh_token":"secret"}' }], + }) + __setStoreBackendLoaderForTest(async () => backend) + + const env = { + DB_DRIVER: "counting", + JWT_SECRET: "test-config-encryption-secret", + } + const plain = await getDb(env) + await saveDb(plain, env) + + const persisted = getData() + assert.match(persisted.storages[0].addition, /^enc:v2:/) + assert.match(persisted.users[0].password, /^enc:v2:/) + + __resetDbCacheForTest() + const reloadedBackend = createCountingBackend(persisted).backend + __setStoreBackendLoaderForTest(async () => reloadedBackend) + const reloaded = await getDb(env) + assert.equal(reloaded.storages[0].addition, plain.storages[0].addition) + assert.equal(reloaded.users[0].password, plain.users[0].password) +}) + test("getDb: 五个无参 getter 复用同一份缓存快照", async () => { __resetDbCacheForTest() const { backend, stats } = createCountingBackend({ diff --git a/src/backend/internal/op/storage.ts b/src/backend/internal/op/storage.ts index 96b620be..bdc17615 100644 --- a/src/backend/internal/op/storage.ts +++ b/src/backend/internal/op/storage.ts @@ -122,6 +122,7 @@ async function getFTPDriver(storageConfig: any): Promise { const driverCache = new Map() const driverInitCache = new Map>() const cookiePersistenceCache = new Map>() +const deferredTokenPersistence = new WeakSet() // M-8：驱动实例缓存容量上限。storage.modified 变化会产生新 key，旧条目若不清理， // 长生命周期 isolate 中会无限累积。超出上限时按插入顺序淘汰最旧条目。 @@ -140,6 +141,10 @@ export interface StorageRequestContext { env?: any // ESA/Cloudflare env，用于请求级缓存复用 } +export interface GetDriverOptions { + deferTokenPersistence?: boolean +} + export async function getOrCreateDriver( cache: Map>, key: string, @@ -224,6 +229,11 @@ async function createDriver( : st.addition || {} stAddition.refresh_token = refreshToken st.addition = JSON.stringify(stAddition) + // The admin update path persists updatedStorage after driver init. + // Keep that object in sync so its final save cannot restore the old + // refresh token over the rotated token saved here. + storageConfig.addition = st.addition + if (deferredTokenPersistence.has(storageConfig)) return await saveDb(db) } catch (e) { console.warn("[Onedrive] failed to persist refresh token:", e) @@ -1175,29 +1185,43 @@ async function createDriver( export async function getDriver( driverName: string, storageConfig?: any, + options: GetDriverOptions = {}, ): Promise { - const normDriver = (driverName || "").toLowerCase().replace(/[^a-z0-9]/g, "") - if (normDriver === "local") { - return createDriver(driverName, storageConfig) - } + const deferTokenPersistence = Boolean( + options.deferTokenPersistence && + storageConfig && + typeof storageConfig === "object", + ) + if (deferTokenPersistence) deferredTokenPersistence.add(storageConfig) - if (!storageConfig) { - throw new Error( - "failed get driver: storage config not found for driver " + driverName, - ) - } + try { + const normDriver = (driverName || "") + .toLowerCase() + .replace(/[^a-z0-9]/g, "") + if (normDriver === "local") { + return createDriver(driverName, storageConfig) + } - const cacheKey = `${storageConfig.id}_${storageConfig.modified}` - const cached = driverCache.get(cacheKey) - if (cached) return cached + if (!storageConfig) { + throw new Error( + "failed get driver: storage config not found for driver " + driverName, + ) + } - return getOrCreateDriver(driverInitCache, cacheKey, async () => { - const ready = driverCache.get(cacheKey) - if (ready) return ready - const driver = await createDriver(driverName, storageConfig) - setDriverCache(cacheKey, driver) - return driver - }) + const cacheKey = `${storageConfig.id}_${storageConfig.modified}` + const cached = driverCache.get(cacheKey) + if (cached) return cached + + return getOrCreateDriver(driverInitCache, cacheKey, async () => { + const ready = driverCache.get(cacheKey) + if (ready) return ready + const driver = await createDriver(driverName, storageConfig) + setDriverCache(cacheKey, driver) + return driver + }) + } finally { + if (deferTokenPersistence) deferredTokenPersistence.delete(storageConfig) + } } function isCloud189Driver(driverName: string): boolean { diff --git a/src/backend/pkg/crypto.ts b/src/backend/pkg/crypto.ts index 9ad62dc4..726b45c1 100644 --- a/src/backend/pkg/crypto.ts +++ b/src/backend/pkg/crypto.ts @@ -222,7 +222,7 @@ export async function decrypt( legacyKdfWarned = true console.warn( "[Crypto] Decrypting legacy weak-KDF format. It will be re-encrypted " + - "with the strong PBKDF2 format on the next config save.", + "with the current config format on the next save.", ) } } else { @@ -240,6 +240,77 @@ export async function decrypt( return new TextDecoder().decode(plainBuf) } +// ─── Low-CPU config encryption helpers ────────────────────────────────────── + +/** + * Derive the AES key used by the versioned config-encryption envelope. + * + * Config encryption is keyed by JWT_SECRET or by a randomly generated secret + * persisted during setup. Running a password-strengthening KDF separately for + * every encrypted field made a single config load exceed the CPU allowance of + * Cloudflare Workers. HKDF keeps this key independent from other JWT_SECRET + * uses while allowing all fields in one load or save to reuse one CryptoKey. + * + * The legacy PBKDF2 envelope remains supported by decrypt() above. Callers can + * therefore migrate existing values when they next persist the config. + */ +export async function deriveConfigEncryptionKey( + secret: string, +): Promise { + const material = await crypto.subtle.importKey( + "raw", + toBytes(secret), + "HKDF", + false, + ["deriveKey"], + ) + return crypto.subtle.deriveKey( + { + name: "HKDF", + hash: "SHA-256", + salt: toBytes("openlist-config-encryption-v2"), + info: toBytes("AES-256-GCM"), + }, + material, + { name: "AES-GCM", length: 256 }, + false, + ["encrypt", "decrypt"], + ) +} + +/** Encrypt one config field with an already-derived AES key. */ +export async function encryptConfigValue( + data: string, + key: CryptoKey, +): Promise { + const iv = crypto.getRandomValues(new Uint8Array(12)) + const cipherBuf = await crypto.subtle.encrypt( + { name: "AES-GCM", iv }, + key, + toBytes(data), + ) + return `${hexEncode(iv.buffer)}:${hexEncode(cipherBuf)}` +} + +/** Decrypt one config field encrypted by encryptConfigValue(). */ +export async function decryptConfigValue( + encryptedData: string, + key: CryptoKey, +): Promise { + const parts = encryptedData.split(":") + if (parts.length !== 2) { + throw new Error("Invalid config encrypted data format") + } + const iv = fromHex(parts[0]) + const cipherBuf = fromHex(parts[1]) + const plainBuf = await crypto.subtle.decrypt( + { name: "AES-GCM", iv: iv as any }, + key, + cipherBuf as any, + ) + return new TextDecoder().decode(plainBuf) +} + /** Generate a random hex string of given length */ export function randomString(length: number): string { const bytes = crypto.getRandomValues(new Uint8Array(Math.ceil(length / 2))) diff --git a/src/backend/pkg/crypto_security.test.ts b/src/backend/pkg/crypto_security.test.ts index a9baf5a1..7a796337 100644 --- a/src/backend/pkg/crypto_security.test.ts +++ b/src/backend/pkg/crypto_security.test.ts @@ -1,7 +1,13 @@ import test from "node:test" import assert from "node:assert/strict" import { Hono } from "hono" -import { encrypt, decrypt } from "./crypto" +import { + decrypt, + decryptConfigValue, + deriveConfigEncryptionKey, + encrypt, + encryptConfigValue, +} from "./crypto" import { hashPasswordSHA256, authRouter, getOrInitUsers } from "../server/auth" import { saveDb, getDb } from "../internal/model/db" @@ -58,6 +64,16 @@ test("AES decrypt backwards compatibility with 2-segment legacy format", async ( ) }) +test("config encryption reuses one derived key with independent IVs", async () => { + const key = await deriveConfigEncryptionKey("my-test-secret-key-123456") + const first = await encryptConfigValue("first-secret", key) + const second = await encryptConfigValue("second-secret", key) + + assert.notEqual(first.split(":")[0], second.split(":")[0]) + assert.equal(await decryptConfigValue(first, key), "first-secret") + assert.equal(await decryptConfigValue(second, key), "second-secret") +}) + test("StaticHash produces consistent 64-char sha256 output (Go StaticHash)", async () => { const hash = await hashPasswordSHA256("admin") assert.equal(typeof hash, "string") diff --git a/src/backend/server/admin.ts b/src/backend/server/admin.ts index 4e9479a1..411a8713 100644 --- a/src/backend/server/admin.ts +++ b/src/backend/server/admin.ts @@ -398,8 +398,12 @@ adminRouter.post("/storage/update", async (c) => { } if (!updatedStorage.disabled) { try { - const driver = await getDriver(updatedStorage.driver, updatedStorage) - await driver.init?.() + // getDriver creates and initializes a cache miss. Calling init again + // refreshes rotating credentials twice and persists the whole config + // twice before the final update below. + await getDriver(updatedStorage.driver, updatedStorage, { + deferTokenPersistence: true, + }) updatedStorage.status = "work" } catch (e: any) { updatedStorage.status = e.message || String(e) @@ -442,8 +446,7 @@ adminRouter.post("/storage/enable", async (c) => { // 重新加载时会再次尝试。 ;(async () => { try { - const driver = await getDriver(s.driver, s) - await driver.init?.() + await getDriver(s.driver, s) const db2 = await getDb(c.env) const st = db2.storages.find((x: any) => x.id === id) if (st && !st.disabled) {