liuyu/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-06-06 05:51:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(media): use typed auth for no-auth media providers

Browse Source
This commit is contained in:
Peter Steinberger
2026-05-31 12:56:38 +01:00
committed by GitHub
parent f59113cfd3
commit 242eab9d20
9 changed files with 321 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
docs/plugins/sdk-provider-plugins.md
View File

@@ -662,18 +662,18 @@ API key auth, and dynamic model resolution.
```
Local or self-hosted media providers that intentionally do not require
credentials can expose `resolveSyntheticAuth` and return a non-secret
marker. OpenClaw still keeps the normal auth gate for providers that do
not explicitly opt in.
credentials can expose `resolveAuth` and return `kind: "none"`.
OpenClaw still keeps the normal auth gate for providers that do not
explicitly opt in. Existing providers can keep reading `req.apiKey`;
new providers should prefer `req.auth`.
```typescript
api.registerMediaUnderstandingProvider({
id: "local-audio",
capabilities: ["audio"],
resolveSyntheticAuth: () => ({
apiKey: "custom-local",
source: "local-audio plugin synthetic auth",
mode: "api-key",
resolveAuth: () => ({
kind: "none",
source: "local-audio plugin no-auth",
}),
transcribeAudio: async (req) => ({ text: "Transcript..." }),
});

23
src/agents/model-auth-runtime-shared.ts
View File

@@ -12,6 +12,27 @@ export type ResolvedProviderAuth = {
mode: "api-key" | "oauth" | "token" | "aws-sdk";
};
export type ProviderAuthErrorCode = "missing-api-key" | "missing-provider-auth";
export class ProviderAuthError extends Error {
readonly code: ProviderAuthErrorCode;
readonly provider: string;
constructor(code: ProviderAuthErrorCode, provider: string, message: string) {
super(message);
this.name = "ProviderAuthError";
this.code = code;
this.provider = provider;
}
}
export function isProviderAuthError(
err: unknown,
code?: ProviderAuthErrorCode,
): err is ProviderAuthError {
return err instanceof ProviderAuthError && (!code || err.code === code);
}
export function resolveAwsSdkEnvVarName(env: NodeJS.ProcessEnv = process.env): string | undefined {
if (env[AWS_BEARER_ENV]?.trim()) {
return AWS_BEARER_ENV;
@@ -34,5 +55,5 @@ export function requireApiKey(auth: ResolvedProviderAuth, provider: string): str
if (key) {
return key;
}
throw new Error(formatMissingAuthError(auth, provider));
throw new ProviderAuthError("missing-api-key", provider, formatMissingAuthError(auth, provider));
}

8
src/agents/model-auth.ts
View File

@@ -48,7 +48,7 @@ import {
isNonSecretApiKeyMarker,
NON_ENV_SECRETREF_MARKER,
} from "./model-auth-markers.js";
import type { ResolvedProviderAuth } from "./model-auth-runtime-shared.js";
import { ProviderAuthError, type ResolvedProviderAuth } from "./model-auth-runtime-shared.js";
import { normalizeProviderId } from "./model-selection.js";
export {
@@ -58,6 +58,8 @@ export {
} from "./auth-profiles.js";
export {
formatMissingAuthError,
isProviderAuthError,
ProviderAuthError,
requireApiKey,
resolveAwsSdkEnvVarName,
} from "./model-auth-runtime-shared.js";
@@ -1264,7 +1266,9 @@ export async function resolveApiKeyForProvider(params: {
const authStorePath = resolveAuthStorePathForDisplay(agentDir);
const resolvedAgentDir = path.dirname(authStorePath);
throw new Error(
throw new ProviderAuthError(
"missing-provider-auth",
provider,
[
`No API key found for provider "${provider}".`,
`Auth store: ${authStorePath} (agentDir: ${resolvedAgentDir}).`,

41
src/media-understanding/openai-compatible-audio.test.ts
View File

@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { CUSTOM_LOCAL_AUTH_MARKER } from "../agents/model-auth-markers.js";
import { VERSION } from "../version.js";
import {
createRequestCaptureJsonFetch,
@@ -72,6 +73,46 @@ describe("transcribeOpenAiCompatibleAudio", () => {
expect((file as File).name).toBe("voice-note.m4a");
});
it("omits bearer auth for explicit no-auth requests", async () => {
const { fetchFn, getRequest } = createRequestCaptureJsonFetch({ text: "ok" });
await transcribeOpenAiCompatibleAudio({
buffer: Buffer.from("audio"),
fileName: "note.mp3",
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
auth: { kind: "none", source: "local provider" },
timeoutMs: 1000,
fetchFn,
provider: "local-audio",
baseUrl: "https://audio.example.com/v1",
defaultBaseUrl: "https://audio.example.com/v1",
defaultModel: "whisper-local",
});
const headers = new Headers(getRequest().init?.headers);
expect(headers.get("authorization")).toBeNull();
});
it("uses typed api-key auth for bearer headers", async () => {
const { fetchFn, getRequest } = createRequestCaptureJsonFetch({ text: "ok" });
await transcribeOpenAiCompatibleAudio({
buffer: Buffer.from("audio"),
fileName: "note.mp3",
apiKey: "legacy-key",
auth: { kind: "api-key", apiKey: "typed-key", source: "test" },
timeoutMs: 1000,
fetchFn,
provider: "local-audio",
baseUrl: "https://audio.example.com/v1",
defaultBaseUrl: "https://audio.example.com/v1",
defaultModel: "whisper-local",
});
const headers = new Headers(getRequest().init?.headers);
expect(headers.get("authorization")).toBe("Bearer typed-key");
});
it("wraps malformed transcription JSON with a stable provider error", async () => {
const fetchFn = vi.fn<typeof fetch>().mockResolvedValueOnce(new Response("{ nope"));

11
src/media-understanding/openai-compatible-audio.ts
View File

@@ -23,15 +23,20 @@ export async function transcribeOpenAiCompatibleAudio(
params: OpenAiCompatibleAudioParams,
): Promise<AudioTranscriptionResult> {
const fetchFn = params.fetchFn ?? fetch;
const apiKey = params.auth?.kind === "api-key" ? params.auth.apiKey : params.apiKey;
const defaultHeaders =
params.auth?.kind === "none" || !apiKey
? undefined
: {
authorization: `Bearer ${apiKey}`,
};
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: params.baseUrl,
defaultBaseUrl: params.defaultBaseUrl,
headers: params.headers,
request: params.request,
defaultHeaders: {
authorization: `Bearer ${params.apiKey}`,
},
defaultHeaders,
provider: params.provider,
api: "openai-audio-transcriptions",
capability: "audio",

205
src/media-understanding/runner.entries.ts
View File

@@ -9,6 +9,7 @@ import {
collectProviderApiKeysForExecution,
executeWithApiKeyRotation,
} from "../agents/api-key-rotation.js";
import { CUSTOM_LOCAL_AUTH_MARKER } from "../agents/model-auth-markers.js";
import {
mergeModelProviderRequestOverrides,
sanitizeConfiguredModelProviderRequest,
@@ -16,7 +17,7 @@ import {
} from "../agents/provider-request-config.js";
import type { MsgContext } from "../auto-reply/templating.js";
import { applyTemplate } from "../auto-reply/templating.js";
import type { OpenClawConfig } from "../config/types.js";
import type { ModelProviderConfig, OpenClawConfig } from "../config/types.js";
import type {
MediaUnderstandingConfig,
MediaUnderstandingModelConfig,
@@ -54,10 +55,12 @@ import { estimateBase64Size, resolveVideoMaxBase64Bytes } from "./video.js";
export type ProviderRegistry = Map<string, MediaUnderstandingProvider>;
type ResolveApiKeyForProvider = typeof import("../agents/model-auth.js").resolveApiKeyForProvider;
type RequireApiKey = typeof import("../agents/model-auth.js").requireApiKey;
type IsProviderAuthError = typeof import("../agents/model-auth.js").isProviderAuthError;
let cachedModelAuth: {
resolveApiKeyForProvider: ResolveApiKeyForProvider;
requireApiKey: RequireApiKey;
isProviderAuthError: IsProviderAuthError;
} | null = null;
async function loadModelAuth() {
@@ -425,13 +428,18 @@ function resolveMediaRequestOverrides(config: MediaUnderstandingConfig | undefin
};
}
function isMissingProviderApiKeyError(err: unknown, providerId: string): boolean {
const message = err instanceof Error ? err.message : String(err);
return (
message.includes(`No API key found for provider "${providerId}"`) ||
message.includes(`No API key resolved for provider "${providerId}"`)
);
}
type ProviderExecutionAuth =
| {
kind: "api-key";
apiKeys: string[];
source?: string;
providerConfig?: ModelProviderConfig;
}
| {
kind: "none";
source: string;
providerConfig?: ModelProviderConfig;
};
async function resolveProviderExecutionAuth(params: {
providerId: string;
@@ -440,7 +448,7 @@ async function resolveProviderExecutionAuth(params: {
entry: MediaUnderstandingModelConfig;
agentDir?: string;
workspaceDir?: string;
}) {
}): Promise<ProviderExecutionAuth> {
const providerConfig = params.cfg.models?.providers?.[params.providerId];
const literalApiKey = resolveLiteralProviderApiKey({
cfg: params.cfg,
@@ -448,22 +456,60 @@ async function resolveProviderExecutionAuth(params: {
});
if (literalApiKey) {
return {
kind: "api-key",
apiKeys: collectProviderApiKeysForExecution({
provider: params.providerId,
primaryApiKey: literalApiKey,
}),
source: `models.providers.${params.providerId}.apiKey`,
providerConfig,
};
}
const resolveMediaProviderSyntheticAuth = () => {
const syntheticAuth = params.provider?.resolveSyntheticAuth?.({
const resolveMediaProviderAuth = (): ProviderExecutionAuth | undefined => {
const context = {
config: params.cfg,
provider: params.providerId,
providerConfig,
});
return syntheticAuth?.apiKey?.trim() || undefined;
};
const providerAuth = params.provider?.resolveAuth?.(context);
if (!providerAuth) {
const syntheticAuth = params.provider?.resolveSyntheticAuth?.(context);
const syntheticApiKey = syntheticAuth?.apiKey.trim();
const syntheticSource = syntheticAuth?.source;
return syntheticApiKey
? {
kind: "api-key",
apiKeys: collectProviderApiKeysForExecution({
provider: params.providerId,
primaryApiKey: syntheticApiKey,
}),
source: syntheticSource,
providerConfig,
}
: undefined;
}
if (providerAuth.kind === "none") {
return {
kind: "none",
source: providerAuth.source,
providerConfig,
};
}
const apiKey = providerAuth.apiKey.trim();
if (!apiKey) {
return undefined;
}
return {
kind: "api-key",
apiKeys: collectProviderApiKeysForExecution({
provider: params.providerId,
primaryApiKey: apiKey,
}),
source: providerAuth.source,
providerConfig,
};
};
const { requireApiKey, resolveApiKeyForProvider } = await loadModelAuth();
const { isProviderAuthError, requireApiKey, resolveApiKeyForProvider } = await loadModelAuth();
try {
const auth = await resolveApiKeyForProvider({
provider: params.providerId,
@@ -473,26 +519,26 @@ async function resolveProviderExecutionAuth(params: {
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
});
const apiKey = requireApiKey(auth, params.providerId);
return {
kind: "api-key",
apiKeys: collectProviderApiKeysForExecution({
provider: params.providerId,
primaryApiKey: requireApiKey(auth, params.providerId),
primaryApiKey: apiKey,
}),
source: auth.source,
providerConfig,
};
} catch (err) {
if (!isMissingProviderApiKeyError(err, params.providerId)) {
if (
!isProviderAuthError(err, "missing-provider-auth") &&
!isProviderAuthError(err, "missing-api-key")
) {
throw err;
}
const syntheticApiKey = resolveMediaProviderSyntheticAuth();
if (syntheticApiKey) {
return {
apiKeys: collectProviderApiKeysForExecution({
provider: params.providerId,
primaryApiKey: syntheticApiKey,
}),
providerConfig,
};
const mediaAuth = resolveMediaProviderAuth();
if (mediaAuth) {
return mediaAuth;
}
throw err;
}
@@ -507,7 +553,7 @@ async function resolveProviderExecutionContext(params: {
agentDir?: string;
workspaceDir?: string;
}) {
const { apiKeys, providerConfig } = await resolveProviderExecutionAuth({
const auth = await resolveProviderExecutionAuth({
providerId: params.providerId,
provider: params.provider,
cfg: params.cfg,
@@ -515,6 +561,7 @@ async function resolveProviderExecutionContext(params: {
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
});
const providerConfig = auth.providerConfig;
const baseUrl = params.entry.baseUrl ?? params.config?.baseUrl ?? providerConfig?.baseUrl;
const mergedHeaders = {
...sanitizeProviderHeaders(providerConfig?.headers as Record<string, unknown> | undefined),
@@ -527,7 +574,7 @@ async function resolveProviderExecutionContext(params: {
sanitizeConfiguredProviderRequest(params.config?.request),
sanitizeConfiguredProviderRequest(params.entry.request),
);
return { apiKeys, baseUrl, headers, request };
return { auth, baseUrl, headers, request };
}
export function formatDecisionSummary(decision: MediaUnderstandingDecision): string {
@@ -686,7 +733,7 @@ export async function runProviderEntry(params: {
timeoutMs,
});
assertMinAudioSize({ size: media.size, attachmentIndex: params.attachmentIndex });
const { apiKeys, baseUrl, headers, request } = await resolveProviderExecutionContext({
const { auth, baseUrl, headers, request } = await resolveProviderExecutionContext({
providerId,
provider,
cfg,
@@ -709,31 +756,39 @@ export async function runProviderEntry(params: {
workspaceDir: params.workspaceDir,
}) ||
entry.model;
const result = await executeWithApiKeyRotation({
provider: providerId,
apiKeys,
transientRetry: providerOperationRetryConfig("read"),
execute: async (apiKey) =>
transcribeAudio({
buffer: media.buffer,
fileName: media.fileName,
mime: media.mime,
apiKey,
baseUrl,
headers,
request,
model,
language:
requestOverrides.language ??
entry.language ??
params.config?.language ??
cfg.tools?.media?.audio?.language,
prompt: requestOverrides.prompt ?? prompt,
query: providerQuery,
timeoutMs,
fetchFn,
}),
const authSource = auth.source ?? `provider:${providerId}`;
const buildRequest = (requestAuth: { kind: "api-key"; apiKey: string } | { kind: "none" }) => ({
buffer: media.buffer,
fileName: media.fileName,
mime: media.mime,
apiKey: requestAuth.kind === "api-key" ? requestAuth.apiKey : CUSTOM_LOCAL_AUTH_MARKER,
auth:
requestAuth.kind === "api-key"
? { kind: "api-key" as const, apiKey: requestAuth.apiKey, source: auth.source }
: { kind: "none" as const, source: authSource },
baseUrl,
headers,
request,
model,
language:
requestOverrides.language ??
entry.language ??
params.config?.language ??
cfg.tools?.media?.audio?.language,
prompt: requestOverrides.prompt ?? prompt,
query: providerQuery,
timeoutMs,
fetchFn,
});
const result =
auth.kind === "api-key"
? await executeWithApiKeyRotation({
provider: providerId,
apiKeys: auth.apiKeys,
transientRetry: providerOperationRetryConfig("read"),
execute: async (apiKey) => transcribeAudio(buildRequest({ kind: "api-key", apiKey })),
})
: await transcribeAudio(buildRequest({ kind: "none" }));
return {
kind: "audio.transcription",
attachmentIndex: params.attachmentIndex,
@@ -760,7 +815,7 @@ export async function runProviderEntry(params: {
`Video attachment ${params.attachmentIndex + 1} base64 payload ${estimatedBase64Bytes} exceeds ${maxBase64Bytes}`,
);
}
const { apiKeys, baseUrl, headers, request } = await resolveProviderExecutionContext({
const { auth, baseUrl, headers, request } = await resolveProviderExecutionContext({
providerId,
provider,
cfg,
@@ -769,25 +824,33 @@ export async function runProviderEntry(params: {
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
});
const result = await executeWithApiKeyRotation({
provider: providerId,
apiKeys,
transientRetry: providerOperationRetryConfig("read"),
execute: (apiKey) =>
describeVideo({
buffer: media.buffer,
fileName: media.fileName,
mime: media.mime,
apiKey,
baseUrl,
headers,
request,
model: entry.model,
prompt,
timeoutMs,
fetchFn,
}),
const authSource = auth.source ?? `provider:${providerId}`;
const buildRequest = (requestAuth: { kind: "api-key"; apiKey: string } | { kind: "none" }) => ({
buffer: media.buffer,
fileName: media.fileName,
mime: media.mime,
apiKey: requestAuth.kind === "api-key" ? requestAuth.apiKey : CUSTOM_LOCAL_AUTH_MARKER,
auth:
requestAuth.kind === "api-key"
? { kind: "api-key" as const, apiKey: requestAuth.apiKey, source: auth.source }
: { kind: "none" as const, source: authSource },
baseUrl,
headers,
request,
model: entry.model,
prompt,
timeoutMs,
fetchFn,
});
const result =
auth.kind === "api-key"
? await executeWithApiKeyRotation({
provider: providerId,
apiKeys: auth.apiKeys,
transientRetry: providerOperationRetryConfig("read"),
execute: (apiKey) => describeVideo(buildRequest({ kind: "api-key", apiKey })),
})
: await describeVideo(buildRequest({ kind: "none" }));
return {
kind: "video.description",
attachmentIndex: params.attachmentIndex,

121
src/media-understanding/runner.local-no-auth.test.ts
View File

@@ -146,7 +146,7 @@ describe("runCapability local no-auth audio providers", () => {
});
});
it("regression #74644: plugin-only local no-auth audio provider can use synthetic auth", async () => {
it("regression #74644: plugin-only local no-auth audio provider can use no-auth", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await withAudioFixture(
@@ -167,10 +167,9 @@ describe("runCapability local no-auth audio providers", () => {
agentDir,
providerRegistry: buildProviderRegistry({
"local-audio": createAudioProvider("local-audio", transcribeAudio, {
resolveSyntheticAuth: () => ({
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
source: "local-audio plugin synthetic auth",
mode: "api-key",
resolveAuth: () => ({
kind: "none",
source: "local-audio plugin no-auth",
}),
}),
}),
@@ -186,13 +185,17 @@ describe("runCapability local no-auth audio providers", () => {
expect(result.outputs[0]?.text).toBe("plugin local ok");
expect(transcribeAudio).toHaveBeenCalledTimes(1);
expect(transcribeAudio.mock.calls[0]?.[0].apiKey).toBe(CUSTOM_LOCAL_AUTH_MARKER);
expect(transcribeAudio.mock.calls[0]?.[0].auth).toEqual({
kind: "none",
source: "local-audio plugin no-auth",
});
},
);
});
});
});
it("prefers resolver env credentials over plugin-only media synthetic auth", async () => {
it("prefers resolver env credentials over plugin-only media no-auth", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync({ ...AUTH_ENV, OPENAI_API_KEY: "env-openai-audio-key" }, async () => {
await withAudioFixture("openclaw-openai-audio-env-key", async ({ ctx, media, cache }) => {
@@ -211,10 +214,9 @@ describe("runCapability local no-auth audio providers", () => {
agentDir,
providerRegistry: buildProviderRegistry({
openai: createAudioProvider("openai", transcribeAudio, {
resolveSyntheticAuth: () => ({
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
source: "openai plugin synthetic auth",
mode: "api-key",
resolveAuth: () => ({
kind: "none",
source: "openai plugin no-auth",
}),
}),
}),
@@ -229,7 +231,7 @@ describe("runCapability local no-auth audio providers", () => {
});
});
it("prefers stored auth profile credentials over plugin-only media synthetic auth", async () => {
it("prefers stored auth profile credentials over plugin-only media no-auth", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await fs.writeFile(
@@ -263,10 +265,9 @@ describe("runCapability local no-auth audio providers", () => {
agentDir,
providerRegistry: buildProviderRegistry({
"local-audio": createAudioProvider("local-audio", transcribeAudio, {
resolveSyntheticAuth: () => ({
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
source: "local-audio plugin synthetic auth",
mode: "api-key",
resolveAuth: () => ({
kind: "none",
source: "local-audio plugin no-auth",
}),
}),
}),
@@ -322,7 +323,7 @@ describe("runCapability local no-auth audio providers", () => {
});
});
it("prefers literal configured provider apiKey over media synthetic auth hook", async () => {
it("prefers literal configured provider apiKey over media no-auth hook", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await withAudioFixture(
@@ -350,10 +351,9 @@ describe("runCapability local no-auth audio providers", () => {
agentDir,
providerRegistry: buildProviderRegistry({
"local-audio": createAudioProvider("local-audio", transcribeAudio, {
resolveSyntheticAuth: () => ({
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
source: "local-audio plugin synthetic auth",
mode: "api-key",
resolveAuth: () => ({
kind: "none",
source: "local-audio plugin no-auth",
}),
}),
}),
@@ -368,7 +368,47 @@ describe("runCapability local no-auth audio providers", () => {
});
});
it("does not allow plugin-only media provider without explicit synthetic auth", async () => {
it("allows a media auth hook to provide an api key after normal auth misses", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await withAudioFixture("openclaw-local-audio-hook-key", async ({ ctx, media, cache }) => {
const transcribeAudio = vi.fn(async (req: AudioTranscriptionRequest) => ({
text: `hook:${req.apiKey}`,
model: req.model,
}));
const cfg = createAudioCfg({ provider: "local-audio", model: "whisper-local" });
const result = await runCapability({
capability: "audio",
cfg,
ctx,
attachments: cache,
media,
agentDir,
providerRegistry: buildProviderRegistry({
"local-audio": createAudioProvider("local-audio", transcribeAudio, {
resolveAuth: () => ({
kind: "api-key",
apiKey: "hook-key",
source: "local-audio media auth hook",
}),
}),
}),
});
expect(result.decision.outcome).toBe("success");
expect(result.outputs[0]?.text).toBe("hook:hook-key");
expect(transcribeAudio.mock.calls[0]?.[0].auth).toEqual({
kind: "api-key",
apiKey: "hook-key",
source: "local-audio media auth hook",
});
});
});
});
});
it("does not allow plugin-only media provider without explicit no-auth", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await withAudioFixture("openclaw-local-audio-no-hook", async ({ ctx, media, cache }) => {
@@ -400,7 +440,7 @@ describe("runCapability local no-auth audio providers", () => {
});
});
it("does not allow plugin-only media provider when synthetic auth hook returns null", async () => {
it("does not allow plugin-only media provider when no-auth hook returns null", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await withAudioFixture("openclaw-local-audio-null-hook", async ({ ctx, media, cache }) => {
@@ -419,7 +459,7 @@ describe("runCapability local no-auth audio providers", () => {
agentDir,
providerRegistry: buildProviderRegistry({
"local-audio": createAudioProvider("local-audio", transcribeAudio, {
resolveSyntheticAuth: () => null,
resolveAuth: () => null,
}),
}),
});
@@ -434,7 +474,7 @@ describe("runCapability local no-auth audio providers", () => {
});
});
it("does not let plugin-only synthetic auth override an explicit missing profile", async () => {
it("does not let plugin-only no-auth override an explicit missing profile", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await withAudioFixture(
@@ -459,10 +499,9 @@ describe("runCapability local no-auth audio providers", () => {
agentDir,
providerRegistry: buildProviderRegistry({
"local-audio": createAudioProvider("local-audio", transcribeAudio, {
resolveSyntheticAuth: () => ({
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
source: "local-audio plugin synthetic auth",
mode: "api-key",
resolveAuth: () => ({
kind: "none",
source: "local-audio plugin no-auth",
}),
}),
}),
@@ -479,7 +518,7 @@ describe("runCapability local no-auth audio providers", () => {
});
});
it("does not let media synthetic auth override an explicit missing profile", async () => {
it("does not let media no-auth override an explicit missing profile", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await withAudioFixture(
@@ -509,10 +548,9 @@ describe("runCapability local no-auth audio providers", () => {
agentDir,
providerRegistry: buildProviderRegistry({
"local-audio": createAudioProvider("local-audio", transcribeAudio, {
resolveSyntheticAuth: () => ({
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
source: "local-audio plugin synthetic auth",
mode: "api-key",
resolveAuth: () => ({
kind: "none",
source: "local-audio plugin no-auth",
}),
}),
}),
@@ -529,14 +567,14 @@ describe("runCapability local no-auth audio providers", () => {
});
});
it("allows explicit synthetic auth for plugin-only no-auth video provider", async () => {
it("allows explicit no-auth for plugin-only no-auth video provider", async () => {
await withIsolatedAgentDir(async (agentDir) => {
await withEnvAsync(AUTH_ENV, async () => {
await withVideoFixture(
"openclaw-local-video-plugin-only",
async ({ ctx, media, cache }) => {
const describeVideo = vi.fn(async (req: VideoDescriptionRequest) => ({
text: `video:${req.apiKey}`,
text: `video:${req.auth?.kind}`,
model: req.model,
}));
const cfg = createVideoCfg({ provider: "local-video", model: "video-local" });
@@ -550,19 +588,22 @@ describe("runCapability local no-auth audio providers", () => {
agentDir,
providerRegistry: buildProviderRegistry({
"local-video": createVideoProvider("local-video", describeVideo, {
resolveSyntheticAuth: () => ({
apiKey: CUSTOM_LOCAL_AUTH_MARKER,
source: "local-video plugin synthetic auth",
mode: "api-key",
resolveAuth: () => ({
kind: "none",
source: "local-video plugin no-auth",
}),
}),
}),
});
expect(result.decision.outcome).toBe("success");
expect(result.outputs[0]?.text).toBe(`video:${CUSTOM_LOCAL_AUTH_MARKER}`);
expect(result.outputs[0]?.text).toBe("video:none");
expect(describeVideo).toHaveBeenCalledTimes(1);
expect(describeVideo.mock.calls[0]?.[0].apiKey).toBe(CUSTOM_LOCAL_AUTH_MARKER);
expect(describeVideo.mock.calls[0]?.[0].auth).toEqual({
kind: "none",
source: "local-video plugin no-auth",
});
},
);
});

20
src/media-understanding/types.ts
View File

@@ -84,11 +84,17 @@ type MediaUnderstandingProviderRequestTransportOverrides = {
allowPrivateNetwork?: boolean;
};
export type MediaUnderstandingProviderRequestAuth =
| { kind: "api-key"; apiKey: string; source?: string }
| { kind: "none"; source: string };
export type AudioTranscriptionRequest = {
buffer: Buffer;
fileName: string;
mime?: string;
/** Compatibility field for existing providers; prefer auth.kind/apiKey. */
apiKey: string;
auth?: MediaUnderstandingProviderRequestAuth;
baseUrl?: string;
headers?: Record<string, string>;
request?: MediaUnderstandingProviderRequestTransportOverrides;
@@ -109,7 +115,9 @@ export type VideoDescriptionRequest = {
buffer: Buffer;
fileName: string;
mime?: string;
/** Compatibility field for existing providers; prefer auth.kind/apiKey. */
apiKey: string;
auth?: MediaUnderstandingProviderRequestAuth;
baseUrl?: string;
headers?: Record<string, string>;
request?: MediaUnderstandingProviderRequestTransportOverrides;
@@ -218,12 +226,16 @@ export type MediaUnderstandingDocumentModelDefaults = {
image?: string | false;
};
export type MediaUnderstandingProviderSyntheticAuthContext = {
export type MediaUnderstandingProviderAuthContext = {
config?: OpenClawConfig;
provider: string;
providerConfig?: ModelProviderConfig;
};
export type MediaUnderstandingProviderAuthResult =
| { kind: "none"; source: string }
| { kind: "api-key"; apiKey: string; source: string; mode?: "api-key" };
export type MediaUnderstandingProviderSyntheticAuthResult = {
apiKey: string;
source: string;
@@ -237,8 +249,12 @@ export type MediaUnderstandingProvider = {
autoPriority?: Partial<Record<MediaUnderstandingCapability, number>>;
nativeDocumentInputs?: Array<"pdf">;
documentModels?: Partial<Record<"pdf", MediaUnderstandingDocumentModelDefaults>>;
resolveAuth?: (
ctx: MediaUnderstandingProviderAuthContext,
) => MediaUnderstandingProviderAuthResult | null | undefined;
/** @deprecated Use resolveAuth. */
resolveSyntheticAuth?: (
ctx: MediaUnderstandingProviderSyntheticAuthContext,
ctx: MediaUnderstandingProviderAuthContext,
) => MediaUnderstandingProviderSyntheticAuthResult | null | undefined;
transcribeAudio?: (req: AudioTranscriptionRequest) => Promise<AudioTranscriptionResult>;
describeVideo?: (req: VideoDescriptionRequest) => Promise<VideoDescriptionResult>;

4
src/plugin-sdk/media-understanding.ts
View File

@@ -9,6 +9,10 @@ export type {
ImagesDescriptionRequest,
ImagesDescriptionResult,
MediaUnderstandingProvider,
MediaUnderstandingProviderAuthContext,
MediaUnderstandingProviderAuthResult,
MediaUnderstandingProviderRequestAuth,
MediaUnderstandingProviderSyntheticAuthResult,
StructuredExtractionImageInput,
StructuredExtractionInput,
StructuredExtractionRequest,