mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-06 05:51:15 +08:00
docs: document media and sdk package facades
This commit is contained in:
@@ -1,12 +1,17 @@
|
||||
import { uniqueTrimmedStrings } from "./string.js";
|
||||
|
||||
// Shared media-generation catalog contracts and static entry synthesis.
|
||||
|
||||
/** Catalog kind for generated media model entries. */
|
||||
export type MediaGenerationCatalogKind =
|
||||
| "image_generation"
|
||||
| "video_generation"
|
||||
| "music_generation";
|
||||
|
||||
/** Source for a media generation catalog entry. */
|
||||
export type MediaGenerationCatalogSource = "static" | "live" | "cache" | "configured";
|
||||
|
||||
/** Media generation model catalog entry. */
|
||||
export type MediaGenerationCatalogEntry<TCapabilities = unknown> = {
|
||||
kind: MediaGenerationCatalogKind;
|
||||
provider: string;
|
||||
@@ -24,6 +29,7 @@ export type MediaGenerationCatalogEntry<TCapabilities = unknown> = {
|
||||
warnings?: readonly string[];
|
||||
};
|
||||
|
||||
/** Provider metadata used to synthesize static media generation catalog entries. */
|
||||
export type MediaGenerationCatalogProvider<TCapabilities = unknown> = {
|
||||
id: string;
|
||||
aliases?: readonly string[];
|
||||
@@ -33,10 +39,12 @@ export type MediaGenerationCatalogProvider<TCapabilities = unknown> = {
|
||||
capabilities: TCapabilities;
|
||||
};
|
||||
|
||||
/** Return unique configured models with default model first when present. */
|
||||
function uniqueModels(provider: { defaultModel?: string; models?: readonly string[] }): string[] {
|
||||
return uniqueTrimmedStrings([provider.defaultModel, ...(provider.models ?? [])]);
|
||||
}
|
||||
|
||||
/** Synthesize static catalog entries from provider metadata. */
|
||||
export function synthesizeMediaGenerationCatalogEntries<TCapabilities>(params: {
|
||||
kind: MediaGenerationCatalogKind;
|
||||
provider: MediaGenerationCatalogProvider<TCapabilities>;
|
||||
@@ -63,6 +71,7 @@ export function synthesizeMediaGenerationCatalogEntries<TCapabilities>(params: {
|
||||
});
|
||||
}
|
||||
|
||||
/** Return unique model ids exposed by a media generation provider. */
|
||||
export function listMediaGenerationProviderModels(provider: {
|
||||
defaultModel?: string;
|
||||
models?: readonly string[];
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
// Public barrel for media-generation shared model refs and catalog helpers.
|
||||
|
||||
export * from "./capability-model-ref.js";
|
||||
export * from "./catalog.js";
|
||||
export * from "./model-ref.js";
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Shared string normalization helpers for media-generation packages.
|
||||
|
||||
/** Normalize optional strings, returning undefined for non-strings or empty values. */
|
||||
export function normalizeOptionalString(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
@@ -6,6 +9,7 @@ export function normalizeOptionalString(value: unknown): string | undefined {
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
/** Return unique trimmed strings while preserving first-seen order. */
|
||||
export function uniqueTrimmedStrings(values: readonly unknown[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Active media-understanding model selection contract.
|
||||
|
||||
/** Provider/model pair selected for one media-understanding request. */
|
||||
export type ActiveMediaModel = {
|
||||
provider: string;
|
||||
model?: string;
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import type { MediaUnderstandingCapability } from "./types.js";
|
||||
|
||||
// Shared defaults for media-understanding limits, prompts, and concurrency.
|
||||
|
||||
const MB = 1024 * 1024;
|
||||
|
||||
/** Default max response characters for bounded text outputs. */
|
||||
export const DEFAULT_MAX_CHARS = 500;
|
||||
/** Default max response characters by capability. */
|
||||
export const DEFAULT_MAX_CHARS_BY_CAPABILITY: Record<
|
||||
MediaUnderstandingCapability,
|
||||
number | undefined
|
||||
@@ -11,22 +15,29 @@ export const DEFAULT_MAX_CHARS_BY_CAPABILITY: Record<
|
||||
audio: undefined,
|
||||
video: DEFAULT_MAX_CHARS,
|
||||
};
|
||||
/** Default input byte limits by capability. */
|
||||
export const DEFAULT_MAX_BYTES: Record<MediaUnderstandingCapability, number> = {
|
||||
image: 10 * MB,
|
||||
audio: 20 * MB,
|
||||
video: 50 * MB,
|
||||
};
|
||||
/** Default request timeout by capability. */
|
||||
export const DEFAULT_TIMEOUT_SECONDS: Record<MediaUnderstandingCapability, number> = {
|
||||
image: 60,
|
||||
audio: 60,
|
||||
video: 120,
|
||||
};
|
||||
/** Default prompts by capability. */
|
||||
export const DEFAULT_PROMPT: Record<MediaUnderstandingCapability, string> = {
|
||||
image: "Describe the image.",
|
||||
audio: "Transcribe the audio.",
|
||||
video: "Describe the video.",
|
||||
};
|
||||
/** Upper bound for base64-expanded video payloads. */
|
||||
export const DEFAULT_VIDEO_MAX_BASE64_BYTES = 70 * MB;
|
||||
/** CLI output buffer used by provider child processes. */
|
||||
export const CLI_OUTPUT_MAX_BUFFER = 5 * MB;
|
||||
/** Default parallel media-understanding request count. */
|
||||
export const DEFAULT_MEDIA_CONCURRENCY = 2;
|
||||
/** Minimum bytes for audio files before transcription is attempted. */
|
||||
export const MIN_AUDIO_FILE_BYTES = 1024;
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Media-understanding skip error used for non-fatal attachment omissions.
|
||||
|
||||
/** Reason a media-understanding attachment was skipped. */
|
||||
type MediaUnderstandingSkipReason =
|
||||
| "maxBytes"
|
||||
| "timeout"
|
||||
@@ -6,6 +9,7 @@ type MediaUnderstandingSkipReason =
|
||||
| "blocked"
|
||||
| "tooSmall";
|
||||
|
||||
/** Error used when a media attachment should be skipped without failing the whole request. */
|
||||
export class MediaUnderstandingSkipError extends Error {
|
||||
readonly reason: MediaUnderstandingSkipReason;
|
||||
|
||||
@@ -16,6 +20,7 @@ export class MediaUnderstandingSkipError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** Narrow unknown errors to media-understanding skip errors. */
|
||||
export function isMediaUnderstandingSkipError(err: unknown): err is MediaUnderstandingSkipError {
|
||||
return err instanceof MediaUnderstandingSkipError;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
// Public barrel for media-understanding shared contracts and helpers.
|
||||
|
||||
export * from "./active-model.js";
|
||||
export * from "./defaults.js";
|
||||
export * from "./errors.js";
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// OpenAI-compatible video request/response helpers.
|
||||
|
||||
/** Minimal OpenAI-compatible video response payload shape. */
|
||||
export type OpenAiCompatibleVideoPayload = {
|
||||
choices?: Array<{
|
||||
message?: {
|
||||
@@ -7,6 +10,7 @@ export type OpenAiCompatibleVideoPayload = {
|
||||
}>;
|
||||
};
|
||||
|
||||
/** Trim optional strings, falling back when empty. */
|
||||
export function resolveMediaUnderstandingString(
|
||||
value: string | undefined,
|
||||
fallback: string,
|
||||
@@ -15,6 +19,7 @@ export function resolveMediaUnderstandingString(
|
||||
return trimmed || fallback;
|
||||
}
|
||||
|
||||
/** Coerce text from OpenAI-compatible content or reasoning fields. */
|
||||
export function coerceOpenAiCompatibleVideoText(
|
||||
payload: OpenAiCompatibleVideoPayload,
|
||||
): string | null {
|
||||
@@ -40,6 +45,7 @@ export function coerceOpenAiCompatibleVideoText(
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Build an OpenAI-compatible request body with an inline data URL video. */
|
||||
export function buildOpenAiCompatibleVideoRequestBody(params: {
|
||||
model: string;
|
||||
prompt: string;
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Output extractors for media-understanding provider CLI responses.
|
||||
|
||||
/** Parse the last JSON object in a noisy provider output string. */
|
||||
function extractLastJsonObject(raw: string): unknown {
|
||||
const trimmed = raw.trim();
|
||||
const start = trimmed.lastIndexOf("{");
|
||||
@@ -12,6 +15,7 @@ function extractLastJsonObject(raw: string): unknown {
|
||||
}
|
||||
}
|
||||
|
||||
/** Extract Gemini CLI-style response text from the last JSON object in output. */
|
||||
export function extractGeminiResponse(raw: string): string | null {
|
||||
const payload = extractLastJsonObject(raw);
|
||||
if (!payload || typeof payload !== "object") {
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
// Provider id normalization for media-understanding config and execution.
|
||||
|
||||
/** Normalize a provider id for comparison. */
|
||||
function normalizeProviderId(provider: string): string {
|
||||
return provider.trim().toLowerCase();
|
||||
}
|
||||
|
||||
/** Normalize provider aliases to canonical config provider ids. */
|
||||
export function normalizeMediaProviderId(id: string): string {
|
||||
const normalized = normalizeProviderId(id);
|
||||
if (normalized === "gemini") {
|
||||
@@ -16,6 +20,7 @@ export function normalizeMediaProviderId(id: string): string {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/** Normalize provider ids while preserving execution-specific regional aliases. */
|
||||
export function normalizeMediaExecutionProviderId(id: string): string {
|
||||
const normalized = normalizeProviderId(id);
|
||||
if (normalized === "minimax-cn" || normalized === "minimax-portal-cn") {
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import type { MediaUnderstandingCapability, MediaUnderstandingProvider } from "./types.js";
|
||||
|
||||
// Capability checks for media-understanding provider objects.
|
||||
|
||||
/** Return true when a provider exposes the method for a media capability. */
|
||||
export function providerSupportsCapability(
|
||||
provider: MediaUnderstandingProvider | undefined,
|
||||
capability: MediaUnderstandingCapability,
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
// Shared media-understanding provider, attachment, output, and capability contracts.
|
||||
|
||||
/** Kind of media-understanding output produced for an attachment. */
|
||||
export type MediaUnderstandingKind =
|
||||
| "audio.transcription"
|
||||
| "video.description"
|
||||
| "image.description";
|
||||
|
||||
/** Capability exposed by a media-understanding provider. */
|
||||
export type MediaUnderstandingCapability = "image" | "audio" | "video";
|
||||
|
||||
/** Capability registry keyed by provider id. */
|
||||
export type MediaUnderstandingCapabilityRegistry = Map<
|
||||
string,
|
||||
{
|
||||
@@ -12,6 +17,7 @@ export type MediaUnderstandingCapabilityRegistry = Map<
|
||||
}
|
||||
>;
|
||||
|
||||
/** Media attachment passed to understanding providers. */
|
||||
export type MediaAttachment = {
|
||||
path?: string;
|
||||
url?: string;
|
||||
@@ -20,6 +26,7 @@ export type MediaAttachment = {
|
||||
alreadyTranscribed?: boolean;
|
||||
};
|
||||
|
||||
/** Normalized text output produced by media understanding. */
|
||||
export type MediaUnderstandingOutput = {
|
||||
kind: MediaUnderstandingKind;
|
||||
attachmentIndex: number;
|
||||
@@ -28,6 +35,7 @@ export type MediaUnderstandingOutput = {
|
||||
model?: string;
|
||||
};
|
||||
|
||||
/** Provider shape used for capability discovery and dispatch. */
|
||||
export type MediaUnderstandingProvider = {
|
||||
id: string;
|
||||
capabilities?: MediaUnderstandingCapability[];
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
import { DEFAULT_VIDEO_MAX_BASE64_BYTES } from "./defaults.js";
|
||||
|
||||
// Video payload size helpers for base64-expanded request bodies.
|
||||
|
||||
/** Estimate base64 size for a byte count. */
|
||||
export function estimateBase64Size(bytes: number): number {
|
||||
return Math.ceil(bytes / 3) * 4;
|
||||
}
|
||||
|
||||
/** Resolve video base64 byte limit from raw byte limit and global cap. */
|
||||
export function resolveVideoMaxBase64Bytes(maxBytes: number): number {
|
||||
const expanded = Math.floor(maxBytes * (4 / 3));
|
||||
return Math.min(expanded, DEFAULT_VIDEO_MAX_BASE64_BYTES);
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for browser config contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/browser-config.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for config runtime helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/config-runtime.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for exec approval runtime helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/exec-approvals-runtime.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for gateway method runtime helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/gateway-method-runtime.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for plugin entry contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/plugin-entry.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for plugin runtime helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/plugin-runtime.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider auth runtime helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-auth-runtime.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider auth contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-auth.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider entry contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-entry.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider HTTP helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-http.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider model shared helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-model-shared.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider model type contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-model-types.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider onboarding contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-onboard.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider stream shared helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-stream-shared.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for provider tool contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-tools.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for web search provider config contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-web-search-config-contract.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for web search provider contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/provider-web-search.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for runtime doctor contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/runtime-doctor.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for runtime environment helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/runtime-env.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for secret input contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/secret-input.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for security runtime helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/security-runtime.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for plugin SDK testing helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/testing.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for text runtime helpers.
|
||||
|
||||
export * from "../../../src/plugin-sdk/text-runtime.js";
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
// Public package facade for video generation provider contracts.
|
||||
|
||||
export * from "../../../src/plugin-sdk/video-generation.js";
|
||||
|
||||
Reference in New Issue
Block a user