mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-24 16:23:54 +08:00
Compare commits
1 Commits
main
...
codex/web-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea4f19882a |
@@ -1,4 +1,4 @@
|
||||
9246475f5771612a5fd12de38b153783c4a4cbb8b2682a5c40115916661c90f2 config-baseline.json
|
||||
6349131baaa1828f2a071f42e4d7b17c8966c59b6588c8a4c1a32ea5ea4dcd5e config-baseline.core.json
|
||||
86a0119a199543a450bc2489959d028ba2331e3fb86b8a6e601da5fc04e6e3da config-baseline.json
|
||||
09591715d30d5d08cdd8c3cebe43f240920eb30e05e06d0f1f9ab7192ddae28f config-baseline.core.json
|
||||
671979e86e4c4f59415d0a20879e838f9bbd883b3d29eeb02cb5131db8d187fe config-baseline.channel.json
|
||||
94529978588d6e3776a86780b22cf9ff46a6f9957f2f178d3829403fad451ca7 config-baseline.plugin.json
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
f7247b5bbfe3f96bffffd25a8be2f89b37999e36731f34a159ae21ded1cedd05 plugin-sdk-api-baseline.json
|
||||
ce88a53dadc194ceccc63f50146aee03a1a425f551117da826a21519d5bf80db plugin-sdk-api-baseline.jsonl
|
||||
95f2562304eebefd432c7694a90b860e4611f989e77bd3214b7c2cbeabba1882 plugin-sdk-api-baseline.json
|
||||
5d2c93807dae6e142616d82b0718964326ce46389bf81288972bbf664af64ae7 plugin-sdk-api-baseline.jsonl
|
||||
|
||||
@@ -93,6 +93,7 @@ content.
|
||||
ssrfPolicy: {
|
||||
allowRfc2544BenchmarkRange: true, // opt-in for trusted fake-IP proxies using 198.18.0.0/15
|
||||
allowIpv6UniqueLocalRange: true, // opt-in for trusted fake-IP proxies using fc00::/7
|
||||
hostnameAllowlist: ["example.com", "*.example.org"], // optional exact/subdomain-only restriction
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -160,6 +161,10 @@ Current runtime behavior:
|
||||
official Firecrawl plugin; third-party external fetch plugins stay excluded.
|
||||
- If Readability is disabled, `web_fetch` skips straight to the selected
|
||||
provider fallback. If no provider is available, it fails closed.
|
||||
- When `ssrfPolicy.hostnameAllowlist` is configured, hosted/provider fallbacks
|
||||
are disabled because their redirect chain cannot be enforced by OpenClaw's
|
||||
local network guard. The direct fetch and local extraction paths remain
|
||||
available.
|
||||
|
||||
## Trusted env proxy
|
||||
|
||||
@@ -183,6 +188,11 @@ outbound policy after DNS resolution.
|
||||
- Response body is capped at `maxResponseBytes` before parsing; oversized
|
||||
responses are truncated with a warning
|
||||
- Private/internal hostnames are blocked
|
||||
- `tools.web.fetch.ssrfPolicy.hostnameAllowlist` restricts both the initial URL
|
||||
and every redirect to exact hostnames or subdomain-only wildcard patterns.
|
||||
For example, `*.example.com` matches `cdn.example.com`, but not `example.com`.
|
||||
Entries are case-insensitive, trailing dots are normalized, and an empty or
|
||||
malformed list or any catch-all entry is rejected.
|
||||
- `tools.web.fetch.ssrfPolicy.allowRfc2544BenchmarkRange` and
|
||||
`tools.web.fetch.ssrfPolicy.allowIpv6UniqueLocalRange` are narrow opt-ins
|
||||
for trusted fake-IP proxy stacks; leave them unset unless your proxy owns
|
||||
|
||||
@@ -121,46 +121,6 @@ describe("compileMemoryWikiVault", () => {
|
||||
).resolves.toContain('"text":"Alpha is the canonical source page."');
|
||||
});
|
||||
|
||||
it("discovers pages in nested subdirectories during compile", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot(),
|
||||
initialize: true,
|
||||
});
|
||||
|
||||
await fs.mkdir(path.join(rootDir, "sources", "sub"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "top.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.top", title: "Top Source" },
|
||||
body: "# Top Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "sub", "nested.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.nested", title: "Nested Source" },
|
||||
body: "# Nested Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await compileMemoryWikiVault(config);
|
||||
|
||||
expect(result.pageCounts.source).toBe(2);
|
||||
// Root index should link to both
|
||||
await expect(fs.readFile(path.join(rootDir, "index.md"), "utf8")).resolves.toContain(
|
||||
"[Top Source](sources/top.md)",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "index.md"), "utf8")).resolves.toContain(
|
||||
"[Nested Source](sources/sub/nested.md)",
|
||||
);
|
||||
// Sources index should link to nested file
|
||||
await expect(fs.readFile(path.join(rootDir, "sources", "index.md"), "utf8")).resolves.toContain(
|
||||
"[Nested Source](sub/nested.md)",
|
||||
);
|
||||
});
|
||||
|
||||
it("renders native directory index links relative to each generated index", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot(),
|
||||
|
||||
@@ -364,15 +364,10 @@ export type RefreshMemoryWikiIndexesResult = {
|
||||
|
||||
async function collectMarkdownFiles(rootDir: string, relativeDir: string): Promise<string[]> {
|
||||
const dirPath = path.join(rootDir, relativeDir);
|
||||
const entries = await fs
|
||||
.readdir(dirPath, { withFileTypes: true, recursive: true })
|
||||
.catch(() => []);
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true }).catch(() => []);
|
||||
return entries
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
|
||||
.map((entry) => {
|
||||
const absPath = path.join(entry.parentPath ?? dirPath, entry.name);
|
||||
return path.relative(rootDir, absPath).split(path.sep).join("/");
|
||||
})
|
||||
.map((entry) => path.join(relativeDir, entry.name))
|
||||
.filter((relativePath) => path.basename(relativePath) !== "index.md")
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
@@ -1067,35 +1067,6 @@ describe("searchMemoryWiki", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("discovers pages in nested subdirectories", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
});
|
||||
await fs.mkdir(path.join(rootDir, "sources", "sub"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "top.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.top", title: "Top Source" },
|
||||
body: "# Top Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "sub", "nested.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.nested", title: "Nested Source" },
|
||||
body: "# Nested Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const results = await searchMemoryWiki({ config, query: "Source" });
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
const paths = results.map((r) => r.path).toSorted();
|
||||
expect(paths).toEqual(["sources/sub/nested.md", "sources/top.md"]);
|
||||
});
|
||||
|
||||
it("drops gateway-style owner-qualified session hits that collide with the scoped store", async () => {
|
||||
const { config } = await createQueryVault({
|
||||
initialize: true,
|
||||
|
||||
@@ -245,17 +245,12 @@ async function listWikiMarkdownFiles(rootDir: string): Promise<string[]> {
|
||||
await Promise.all(
|
||||
QUERY_DIRS.map(async (relativeDir) => {
|
||||
const dirPath = path.join(rootDir, relativeDir);
|
||||
const entries = await fs
|
||||
.readdir(dirPath, { withFileTypes: true, recursive: true })
|
||||
.catch(() => []);
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true }).catch(() => []);
|
||||
return entries
|
||||
.filter(
|
||||
(entry) => entry.isFile() && entry.name.endsWith(".md") && entry.name !== "index.md",
|
||||
)
|
||||
.map((entry) => {
|
||||
const absPath = path.join(entry.parentPath ?? dirPath, entry.name);
|
||||
return path.relative(rootDir, absPath).split(path.sep).join("/");
|
||||
});
|
||||
.map((entry) => path.join(relativeDir, entry.name));
|
||||
}),
|
||||
)
|
||||
).flat();
|
||||
|
||||
@@ -92,39 +92,6 @@ describe("resolveMemoryWikiStatus", () => {
|
||||
expect(status.warnings.map((warning) => warning.code)).toContain("bridge-artifacts-missing");
|
||||
});
|
||||
|
||||
it("discovers pages in nested subdirectories", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-nested-",
|
||||
initialize: true,
|
||||
});
|
||||
|
||||
await fs.mkdir(path.join(rootDir, "sources", "sub"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "top.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.top", title: "Top Source" },
|
||||
body: "# Top Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "sub", "nested.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.nested", title: "Nested Source" },
|
||||
body: "# Nested Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const status = await resolveMemoryWikiStatus(config, {
|
||||
pathExists: async () => true,
|
||||
resolveCommand: async () => null,
|
||||
});
|
||||
|
||||
expect(status.pageCounts.source).toBe(2);
|
||||
expect(status.sourceCounts.native).toBe(2);
|
||||
});
|
||||
|
||||
it("counts source provenance from the vault", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-status-",
|
||||
|
||||
@@ -87,28 +87,26 @@ async function collectVaultCounts(vaultPath: string): Promise<{
|
||||
};
|
||||
const dirs = ["entities", "concepts", "sources", "syntheses", "reports"] as const;
|
||||
for (const dir of dirs) {
|
||||
const dirPath = path.join(vaultPath, dir);
|
||||
const entries = await fs
|
||||
.readdir(dirPath, { withFileTypes: true, recursive: true })
|
||||
.readdir(path.join(vaultPath, dir), { withFileTypes: true })
|
||||
.catch(() => []);
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
|
||||
continue;
|
||||
}
|
||||
const absolutePath = path.join(entry.parentPath ?? dirPath, entry.name);
|
||||
const relativeToVault = path.relative(vaultPath, absolutePath).split(path.sep).join("/");
|
||||
const kind = inferWikiPageKind(relativeToVault);
|
||||
const kind = inferWikiPageKind(path.join(dir, entry.name));
|
||||
if (kind) {
|
||||
pageCounts[kind] += 1;
|
||||
}
|
||||
if (dir === "sources") {
|
||||
const absolutePath = path.join(vaultPath, dir, entry.name);
|
||||
const raw = await fs.readFile(absolutePath, "utf8").catch(() => null);
|
||||
if (!raw) {
|
||||
continue;
|
||||
}
|
||||
const page = toWikiPageSummary({
|
||||
absolutePath,
|
||||
relativePath: relativeToVault,
|
||||
relativePath: path.join(dir, entry.name),
|
||||
raw,
|
||||
});
|
||||
if (!page) {
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
// Error-format helper tests cover the non-Error cause stringifier contract.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { stringifyNonErrorCause } from "./error-format.js";
|
||||
|
||||
describe("stringifyNonErrorCause", () => {
|
||||
it("returns a string for values JSON.stringify serializes to undefined", () => {
|
||||
// JSON.stringify(fn|symbol|undefined) is undefined; the `string`-typed helper must not leak it.
|
||||
expect(stringifyNonErrorCause(() => {})).toBe("[object Function]");
|
||||
expect(stringifyNonErrorCause(Symbol("x"))).toBe("[object Symbol]");
|
||||
expect(stringifyNonErrorCause(undefined)).toBe("[object Undefined]");
|
||||
});
|
||||
|
||||
it("stringifies ordinary scalar and object causes", () => {
|
||||
expect(stringifyNonErrorCause({ a: 1 })).toBe('{"a":1}');
|
||||
expect(stringifyNonErrorCause("hi")).toBe("hi");
|
||||
expect(stringifyNonErrorCause(42)).toBe("42");
|
||||
expect(stringifyNonErrorCause(null)).toBe("null");
|
||||
});
|
||||
});
|
||||
@@ -75,9 +75,7 @@ export function stringifyNonErrorCause(value: unknown): string {
|
||||
return String(value);
|
||||
}
|
||||
try {
|
||||
// JSON.stringify returns undefined (not a string) for functions/symbols/undefined; fall back to
|
||||
// a tag string so this `string`-typed helper never leaks undefined (matches src/infra/errors.ts).
|
||||
return JSON.stringify(value) ?? Object.prototype.toString.call(value);
|
||||
return JSON.stringify(value);
|
||||
} catch {
|
||||
return Object.prototype.toString.call(value);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import http from "node:http";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
|
||||
import { createBundleMcpJsonSchemaValidator } from "./agent-bundle-mcp-runtime.js";
|
||||
import { cleanupBundleMcpHarness } from "./agent-bundle-mcp-test-harness.js";
|
||||
import {
|
||||
@@ -27,8 +26,6 @@ vi.mock("./embedded-agent-mcp.js", () => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
type RuntimeFactoryOptions = NonNullable<
|
||||
Parameters<typeof testing.createSessionMcpRuntimeManager>[0]
|
||||
>;
|
||||
@@ -40,12 +37,10 @@ async function writeListToolsMcpServer(params: {
|
||||
filePath: string;
|
||||
logPath: string;
|
||||
delayMs?: number;
|
||||
initializeDelayMs?: number;
|
||||
hang?: boolean;
|
||||
inputSchema?: unknown;
|
||||
tools?: Array<{ name: string; description?: string; inputSchema?: unknown }>;
|
||||
capabilities?: Record<string, unknown>;
|
||||
notifyListChangedOnInitialized?: boolean;
|
||||
listToolsMethodNotFound?: boolean;
|
||||
callToolIsError?: boolean;
|
||||
callToolJsonRpcError?: boolean;
|
||||
@@ -58,10 +53,8 @@ import fs from "node:fs/promises";
|
||||
|
||||
const logPath = ${JSON.stringify(params.logPath)};
|
||||
const delayMs = ${params.delayMs ?? 0};
|
||||
const initializeDelayMs = ${params.initializeDelayMs ?? 0};
|
||||
const hang = ${params.hang === true};
|
||||
const capabilities = ${JSON.stringify(params.capabilities ?? { tools: {} })};
|
||||
const notifyListChangedOnInitialized = ${params.notifyListChangedOnInitialized === true};
|
||||
const listToolsMethodNotFound = ${params.listToolsMethodNotFound === true};
|
||||
const tools = ${JSON.stringify(
|
||||
params.tools ?? [
|
||||
@@ -91,7 +84,7 @@ function handle(message) {
|
||||
}
|
||||
log("recv " + String(message.method ?? "unknown"));
|
||||
if (message.method === "initialize") {
|
||||
const response = {
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: {
|
||||
@@ -99,19 +92,10 @@ function handle(message) {
|
||||
capabilities,
|
||||
serverInfo: { name: "test-list-tools", version: "1.0.0" },
|
||||
},
|
||||
};
|
||||
if (initializeDelayMs > 0) {
|
||||
setTimeout(() => send(response), initializeDelayMs);
|
||||
} else {
|
||||
send(response);
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (message.method === "notifications/initialized") {
|
||||
if (notifyListChangedOnInitialized) {
|
||||
log("notify tools/list_changed");
|
||||
send({ jsonrpc: "2.0", method: "notifications/tools/list_changed" });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (message.method === "tools/list") {
|
||||
@@ -297,7 +281,6 @@ function makeRuntime(
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
cleanupTempDirs(tempDirs);
|
||||
await cleanupBundleMcpHarness();
|
||||
});
|
||||
|
||||
@@ -2052,529 +2035,4 @@ process.stdin.on("end", () => {
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it(
|
||||
"parallelizes MCP server catalog loading across multiple slow servers",
|
||||
{ timeout: LIST_TOOLS_TEST_DEADLINE_MS },
|
||||
async () => {
|
||||
const tempDir = makeTempDir(tempDirs, "bundle-mcp-parallel-");
|
||||
const delays = [200, 400, 600];
|
||||
const serverPaths = delays.map((delay, i) => {
|
||||
const serverPath = path.join(tempDir, `slow-server-${i}.mjs`);
|
||||
const logPath = path.join(tempDir, `server-${i}.log`);
|
||||
return { serverPath, logPath, delay, serverName: `slowServer${i}` };
|
||||
});
|
||||
|
||||
await Promise.all(
|
||||
serverPaths.map(({ serverPath, logPath, delay }) =>
|
||||
writeListToolsMcpServer({ filePath: serverPath, logPath, delayMs: delay }),
|
||||
),
|
||||
);
|
||||
|
||||
testing.setBundleMcpCatalogListTimeoutMsForTest(4_000);
|
||||
|
||||
const runtime = await getOrCreateSessionMcpRuntime({
|
||||
sessionId: "session-parallel-catalog-test",
|
||||
sessionKey: "agent:test:session-parallel-catalog-test",
|
||||
workspaceDir: "/workspace",
|
||||
cfg: {
|
||||
mcp: {
|
||||
servers: Object.fromEntries(
|
||||
serverPaths.map(({ serverName, serverPath }) => [
|
||||
serverName,
|
||||
{
|
||||
command: process.execPath,
|
||||
args: [serverPath],
|
||||
connectionTimeoutMs: 2_000,
|
||||
},
|
||||
]),
|
||||
),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const sumDelays = delays.reduce((a, b) => a + b, 0);
|
||||
const maxDelay = Math.max(...delays);
|
||||
const parallelBudgetMs = maxDelay + 500;
|
||||
|
||||
const t0 = performance.now();
|
||||
const catalog = await runtime.getCatalog();
|
||||
const wallTime = performance.now() - t0;
|
||||
|
||||
// Must have successfully connected to all servers
|
||||
expect(Object.keys(catalog.servers)).toHaveLength(delays.length);
|
||||
expect(catalog.tools.map((t) => t.toolName)).toEqual([
|
||||
"slow_tool",
|
||||
"slow_tool",
|
||||
"slow_tool",
|
||||
]);
|
||||
|
||||
// Sequential listing would have to wait roughly sumDelays before overhead;
|
||||
// parallel listing should stay near the slowest server plus launch overhead.
|
||||
expect(wallTime).toBeLessThan(parallelBudgetMs);
|
||||
expect(parallelBudgetMs).toBeLessThan(sumDelays);
|
||||
|
||||
expect(wallTime).toBeGreaterThanOrEqual(maxDelay * 0.7);
|
||||
} finally {
|
||||
await runtime.dispose();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it(
|
||||
"awaits in-progress MCP session connections after catalog invalidation",
|
||||
{ timeout: LIST_TOOLS_TEST_DEADLINE_MS },
|
||||
async () => {
|
||||
const tempDir = makeTempDir(tempDirs, "bundle-mcp-inflight-connect-");
|
||||
const invalidatingServer = {
|
||||
serverName: "invalidatingServer",
|
||||
serverPath: path.join(tempDir, "invalidating-server.mjs"),
|
||||
logPath: path.join(tempDir, "invalidating-server.log"),
|
||||
};
|
||||
const slowConnectServer = {
|
||||
serverName: "slowConnectServer",
|
||||
serverPath: path.join(tempDir, "slow-connect-server.mjs"),
|
||||
logPath: path.join(tempDir, "slow-connect-server.log"),
|
||||
};
|
||||
|
||||
await writeListToolsMcpServer({
|
||||
filePath: invalidatingServer.serverPath,
|
||||
logPath: invalidatingServer.logPath,
|
||||
capabilities: { tools: { listChanged: true } },
|
||||
notifyListChangedOnInitialized: true,
|
||||
});
|
||||
await writeListToolsMcpServer({
|
||||
filePath: slowConnectServer.serverPath,
|
||||
logPath: slowConnectServer.logPath,
|
||||
initializeDelayMs: 500,
|
||||
});
|
||||
|
||||
testing.setBundleMcpCatalogListTimeoutMsForTest(4_000);
|
||||
|
||||
const runtime = await getOrCreateSessionMcpRuntime({
|
||||
sessionId: "session-inflight-connect-test",
|
||||
sessionKey: "agent:test:session-inflight-connect-test",
|
||||
workspaceDir: "/workspace",
|
||||
cfg: {
|
||||
mcp: {
|
||||
servers: Object.fromEntries(
|
||||
[invalidatingServer, slowConnectServer].map(({ serverName, serverPath }) => [
|
||||
serverName,
|
||||
{
|
||||
command: process.execPath,
|
||||
args: [serverPath],
|
||||
connectionTimeoutMs: 2_000,
|
||||
},
|
||||
]),
|
||||
),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const firstCatalog = runtime.getCatalog();
|
||||
await waitForFileText(
|
||||
invalidatingServer.logPath,
|
||||
"notify tools/list_changed",
|
||||
LIST_TOOLS_SERVER_LOG_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
const secondCatalog = await runtime.getCatalog();
|
||||
await firstCatalog;
|
||||
|
||||
expect(Object.keys(secondCatalog.servers).toSorted()).toEqual([
|
||||
invalidatingServer.serverName,
|
||||
slowConnectServer.serverName,
|
||||
]);
|
||||
expect(secondCatalog.diagnostics ?? []).toEqual([]);
|
||||
} finally {
|
||||
await runtime.dispose();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it(
|
||||
"retires timed-out shared MCP sessions before later catalog retries",
|
||||
{ timeout: 8_000 },
|
||||
async () => {
|
||||
const tempDir = makeTempDir(tempDirs, "bundle-mcp-timeout-retire-");
|
||||
const triggerServerPath = path.join(tempDir, "trigger-server.mjs");
|
||||
const triggerLogPath = path.join(tempDir, "trigger.log");
|
||||
const slowServerPath = path.join(tempDir, "slow-server.mjs");
|
||||
const slowLogPath = path.join(tempDir, "slow.log");
|
||||
const firstConnectMarkerPath = path.join(tempDir, "first-connect.marker");
|
||||
|
||||
await writeExecutable(
|
||||
triggerServerPath,
|
||||
`#!/usr/bin/env node
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
const logPath = ${JSON.stringify(triggerLogPath)};
|
||||
let buffer = "";
|
||||
function log(line) {
|
||||
void fs.appendFile(logPath, line + "\\n", "utf8").catch(() => {});
|
||||
}
|
||||
function send(message) {
|
||||
process.stdout.write(JSON.stringify(message) + "\\n");
|
||||
}
|
||||
function handle(message) {
|
||||
if (!message || typeof message !== "object") {
|
||||
return;
|
||||
}
|
||||
log("recv " + String(message.method ?? "unknown"));
|
||||
if (message.method === "initialize") {
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: {
|
||||
protocolVersion: message.params?.protocolVersion ?? "2025-03-26",
|
||||
capabilities: { tools: { listChanged: true } },
|
||||
serverInfo: { name: "timeout-trigger", version: "1.0.0" },
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (message.method === "notifications/initialized") {
|
||||
send({ jsonrpc: "2.0", method: "notifications/tools/list_changed" });
|
||||
log("sent initial tools/list_changed");
|
||||
return;
|
||||
}
|
||||
if (message.method === "tools/list") {
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: {
|
||||
tools: [{ name: "poke", inputSchema: { type: "object", properties: {} } }],
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (message.method === "tools/call") {
|
||||
send({ jsonrpc: "2.0", method: "notifications/tools/list_changed" });
|
||||
log("sent call tools/list_changed");
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: { isError: false, content: [{ type: "text", text: "poked" }] },
|
||||
});
|
||||
}
|
||||
}
|
||||
process.stdin.setEncoding("utf8");
|
||||
function shutdown() {
|
||||
process.exit(0);
|
||||
}
|
||||
process.stdin.on("data", (chunk) => {
|
||||
buffer += chunk;
|
||||
while (true) {
|
||||
const newline = buffer.indexOf("\\n");
|
||||
if (newline < 0) {
|
||||
return;
|
||||
}
|
||||
const line = buffer.slice(0, newline).replace(/\\r$/, "");
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (line.trim()) {
|
||||
handle(JSON.parse(line));
|
||||
}
|
||||
}
|
||||
});
|
||||
process.stdin.on("end", shutdown);
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);`,
|
||||
);
|
||||
|
||||
await writeExecutable(
|
||||
slowServerPath,
|
||||
`#!/usr/bin/env node
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
const logPath = ${JSON.stringify(slowLogPath)};
|
||||
const markerPath = ${JSON.stringify(firstConnectMarkerPath)};
|
||||
let buffer = "";
|
||||
function log(line) {
|
||||
void fs.appendFile(logPath, line + "\\n", "utf8").catch(() => {});
|
||||
}
|
||||
function send(message) {
|
||||
process.stdout.write(JSON.stringify(message) + "\\n");
|
||||
}
|
||||
async function isFirstConnect() {
|
||||
try {
|
||||
const handle = await fs.open(markerPath, "wx");
|
||||
await handle.close();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
async function handle(message) {
|
||||
if (!message || typeof message !== "object") {
|
||||
return;
|
||||
}
|
||||
log("recv " + String(message.method ?? "unknown"));
|
||||
if (message.method === "initialize") {
|
||||
const response = {
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: {
|
||||
protocolVersion: message.params?.protocolVersion ?? "2025-03-26",
|
||||
capabilities: { tools: {} },
|
||||
serverInfo: { name: "timeout-slow", version: "1.0.0" },
|
||||
},
|
||||
};
|
||||
if (await isFirstConnect()) {
|
||||
log("slow first initialize");
|
||||
setTimeout(() => send(response), 600);
|
||||
} else {
|
||||
log("fast retry initialize");
|
||||
send(response);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (message.method === "tools/list") {
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: {
|
||||
tools: [{ name: "slow_tool", inputSchema: { type: "object", properties: {} } }],
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
process.stdin.setEncoding("utf8");
|
||||
function shutdown() {
|
||||
process.exit(0);
|
||||
}
|
||||
process.stdin.on("data", (chunk) => {
|
||||
buffer += chunk;
|
||||
while (true) {
|
||||
const newline = buffer.indexOf("\\n");
|
||||
if (newline < 0) {
|
||||
return;
|
||||
}
|
||||
const line = buffer.slice(0, newline).replace(/\\r$/, "");
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (line.trim()) {
|
||||
void handle(JSON.parse(line));
|
||||
}
|
||||
}
|
||||
});
|
||||
process.stdin.on("end", shutdown);
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);`,
|
||||
);
|
||||
|
||||
const runtime = await getOrCreateSessionMcpRuntime({
|
||||
sessionId: "session-timeout-retire-test",
|
||||
sessionKey: "agent:test:session-timeout-retire-test",
|
||||
workspaceDir: "/workspace",
|
||||
cfg: {
|
||||
mcp: {
|
||||
servers: {
|
||||
trigger: {
|
||||
command: process.execPath,
|
||||
args: [triggerServerPath],
|
||||
connectionTimeoutMs: 2_000,
|
||||
},
|
||||
slow: {
|
||||
command: process.execPath,
|
||||
args: [slowServerPath],
|
||||
connectionTimeoutMs: 150,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const firstCatalog = runtime.getCatalog();
|
||||
await waitForFileText(
|
||||
triggerLogPath,
|
||||
"sent initial tools/list_changed",
|
||||
LIST_TOOLS_SERVER_LOG_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
const secondCatalog = await runtime.getCatalog();
|
||||
await firstCatalog;
|
||||
|
||||
expect(secondCatalog.servers.trigger).toBeDefined();
|
||||
expect(secondCatalog.diagnostics?.some((diag) => diag.serverName === "slow")).toBe(true);
|
||||
await waitForFileText(
|
||||
slowLogPath,
|
||||
"slow first initialize",
|
||||
LIST_TOOLS_SERVER_LOG_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
await expect(runtime.callTool("trigger", "poke", {})).resolves.toMatchObject({
|
||||
content: [{ type: "text", text: "poked" }],
|
||||
isError: false,
|
||||
});
|
||||
await waitForFileText(
|
||||
triggerLogPath,
|
||||
"sent call tools/list_changed",
|
||||
LIST_TOOLS_SERVER_LOG_TIMEOUT_MS,
|
||||
);
|
||||
await waitForPredicate(
|
||||
() => runtime.peekCatalog() === null,
|
||||
"manual list_changed to retry timed-out server",
|
||||
LIST_TOOLS_SERVER_LOG_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
const retriedCatalog = await runtime.getCatalog();
|
||||
|
||||
expect(retriedCatalog.diagnostics?.some((diag) => diag.serverName === "slow")).not.toBe(
|
||||
true,
|
||||
);
|
||||
expect(retriedCatalog.servers.slow).toBeDefined();
|
||||
expect(retriedCatalog.tools.map((tool) => tool.toolName).toSorted()).toEqual([
|
||||
"poke",
|
||||
"slow_tool",
|
||||
]);
|
||||
await waitForFileText(
|
||||
slowLogPath,
|
||||
"fast retry initialize",
|
||||
LIST_TOOLS_SERVER_LOG_TIMEOUT_MS,
|
||||
);
|
||||
} finally {
|
||||
await runtime.dispose();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it(
|
||||
"does not dispose sessions shared with a newer catalog generation",
|
||||
{ timeout: LIST_TOOLS_TEST_DEADLINE_MS },
|
||||
async () => {
|
||||
const tempDir = makeTempDir(tempDirs, "bundle-mcp-overlap-generation-");
|
||||
const serverPath = path.join(tempDir, "overlap-server.mjs");
|
||||
const logPath = path.join(tempDir, "server.log");
|
||||
|
||||
await writeExecutable(
|
||||
serverPath,
|
||||
`#!/usr/bin/env node
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
const logPath = ${JSON.stringify(logPath)};
|
||||
let buffer = "";
|
||||
let listCount = 0;
|
||||
function log(line) {
|
||||
void fs.appendFile(logPath, line + "\\n", "utf8").catch(() => {});
|
||||
}
|
||||
function send(message) {
|
||||
process.stdout.write(JSON.stringify(message) + "\\n");
|
||||
}
|
||||
function handle(message) {
|
||||
if (!message || typeof message !== "object") {
|
||||
return;
|
||||
}
|
||||
log("recv " + String(message.method ?? "unknown"));
|
||||
if (message.method === "initialize") {
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: {
|
||||
protocolVersion: message.params?.protocolVersion ?? "2025-03-26",
|
||||
capabilities: { tools: { listChanged: true } },
|
||||
serverInfo: { name: "overlap-generation", version: "1.0.0" },
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (message.method === "notifications/initialized") {
|
||||
send({ jsonrpc: "2.0", method: "notifications/tools/list_changed" });
|
||||
log("sent tools/list_changed");
|
||||
return;
|
||||
}
|
||||
if (message.method === "tools/list") {
|
||||
listCount += 1;
|
||||
const currentList = listCount;
|
||||
log("tools/list " + currentList);
|
||||
if (currentList === 1) {
|
||||
setTimeout(() => {
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: {
|
||||
tools: [{ name: "ok_tool", inputSchema: [] }],
|
||||
},
|
||||
});
|
||||
}, 100);
|
||||
return;
|
||||
}
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: {
|
||||
tools: [{ name: "ok_tool", inputSchema: { type: "object", properties: {} } }],
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (message.method === "tools/call") {
|
||||
send({
|
||||
jsonrpc: "2.0",
|
||||
id: message.id,
|
||||
result: { isError: false, content: [{ type: "text", text: "still connected" }] },
|
||||
});
|
||||
}
|
||||
}
|
||||
process.stdin.setEncoding("utf8");
|
||||
function shutdown() {
|
||||
process.exit(0);
|
||||
}
|
||||
process.stdin.on("data", (chunk) => {
|
||||
buffer += chunk;
|
||||
while (true) {
|
||||
const newline = buffer.indexOf("\\n");
|
||||
if (newline < 0) {
|
||||
return;
|
||||
}
|
||||
const line = buffer.slice(0, newline).replace(/\\r$/, "");
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (line.trim()) {
|
||||
handle(JSON.parse(line));
|
||||
}
|
||||
}
|
||||
});
|
||||
process.stdin.on("end", shutdown);
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);`,
|
||||
);
|
||||
|
||||
const runtime = await getOrCreateSessionMcpRuntime({
|
||||
sessionId: "session-overlap-generation-test",
|
||||
sessionKey: "agent:test:session-overlap-generation-test",
|
||||
workspaceDir: "/workspace",
|
||||
cfg: {
|
||||
mcp: {
|
||||
servers: {
|
||||
overlap: {
|
||||
command: process.execPath,
|
||||
args: [serverPath],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const firstCatalog = runtime.getCatalog();
|
||||
await waitForFileText(logPath, "sent tools/list_changed", LIST_TOOLS_SERVER_LOG_TIMEOUT_MS);
|
||||
await waitForFileText(logPath, "tools/list 1", LIST_TOOLS_SERVER_LOG_TIMEOUT_MS);
|
||||
|
||||
const secondCatalog = await runtime.getCatalog();
|
||||
const firstCatalogResult = await firstCatalog;
|
||||
|
||||
expect(firstCatalogResult.diagnostics?.[0]?.serverName).toBe("overlap");
|
||||
expect(secondCatalog.diagnostics ?? []).toEqual([]);
|
||||
expect(secondCatalog.tools.map((tool) => tool.toolName)).toEqual(["ok_tool"]);
|
||||
|
||||
await expect(runtime.callTool("overlap", "ok_tool", {})).resolves.toMatchObject({
|
||||
content: [{ type: "text", text: "still connected" }],
|
||||
isError: false,
|
||||
});
|
||||
} finally {
|
||||
await runtime.dispose();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -23,7 +23,6 @@ import {
|
||||
findJsonSchemaShapeError,
|
||||
normalizeJsonSchemaForTypeBox,
|
||||
} from "../shared/json-schema-defaults.js";
|
||||
import { runTasksWithConcurrency } from "../utils/run-with-concurrency.js";
|
||||
import { sanitizeServerName } from "./agent-bundle-mcp-names.js";
|
||||
import type {
|
||||
McpCatalogTool,
|
||||
@@ -44,11 +43,6 @@ type BundleMcpSession = {
|
||||
transportType: "stdio" | "sse" | "streamable-http";
|
||||
requestTimeoutMs: number;
|
||||
supportsParallelToolCalls: boolean;
|
||||
connected: boolean;
|
||||
retiring: boolean;
|
||||
catalogUseCount: number;
|
||||
sharedAcrossCatalogGenerations: boolean;
|
||||
connectPromise?: Promise<void>;
|
||||
detachStderr?: () => void;
|
||||
};
|
||||
|
||||
@@ -65,7 +59,6 @@ const SESSION_MCP_RUNTIME_SWEEP_INTERVAL_MS = 60 * 1000;
|
||||
const BUNDLE_MCP_FAILURE_THRESHOLD = 3;
|
||||
const BUNDLE_MCP_FAILURE_COOLDOWN_MS = 60_000;
|
||||
const BUNDLE_MCP_CATALOG_LIST_TIMEOUT_MS = 1_500;
|
||||
const BUNDLE_MCP_CATALOG_CONNECT_CONCURRENCY = 6;
|
||||
const BUNDLE_MCP_METADATA_TEXT_LIMIT = 1_200;
|
||||
let bundleMcpCatalogListTimeoutMs: number | undefined;
|
||||
|
||||
@@ -540,41 +533,6 @@ export function createSessionMcpRuntime(params: {
|
||||
throw createDisposedError(params.sessionId);
|
||||
}
|
||||
};
|
||||
const ensureSessionConnected = async (
|
||||
session: BundleMcpSession,
|
||||
connectionTimeoutMs: number,
|
||||
): Promise<void> => {
|
||||
if (session.retiring) {
|
||||
throw new Error(`bundle-mcp server "${session.serverName}" is retiring`);
|
||||
}
|
||||
if (session.connected) {
|
||||
return;
|
||||
}
|
||||
session.connectPromise ??= connectWithTimeout(
|
||||
session.client,
|
||||
session.transport,
|
||||
connectionTimeoutMs,
|
||||
)
|
||||
.then(() => {
|
||||
session.connected = true;
|
||||
})
|
||||
.finally(() => {
|
||||
session.connectPromise = undefined;
|
||||
});
|
||||
await session.connectPromise;
|
||||
};
|
||||
const retireSessionIfCurrent = async (
|
||||
serverName: string,
|
||||
session: BundleMcpSession,
|
||||
): Promise<boolean> => {
|
||||
if (sessions.get(serverName) !== session) {
|
||||
return false;
|
||||
}
|
||||
session.retiring = true;
|
||||
sessions.delete(serverName);
|
||||
await disposeSession(session);
|
||||
return true;
|
||||
};
|
||||
|
||||
const getCatalog = async (): Promise<McpToolCatalog> => {
|
||||
failIfDisposed();
|
||||
@@ -601,13 +559,6 @@ export function createSessionMcpRuntime(params: {
|
||||
const usedServerNames = new Set<string>();
|
||||
|
||||
try {
|
||||
// Pre-compute safe server names sequentially (synchronous, fast — no I/O)
|
||||
const preparedEntries: Array<{
|
||||
serverName: string;
|
||||
rawServer: (typeof loaded.mcpServers)[string];
|
||||
resolved: NonNullable<ReturnType<typeof resolveMcpTransport>>;
|
||||
safeServerName: string;
|
||||
}> = [];
|
||||
for (const [serverName, rawServer] of Object.entries(loaded.mcpServers)) {
|
||||
failIfDisposed();
|
||||
const resolved = resolveMcpTransport(serverName, rawServer);
|
||||
@@ -620,209 +571,137 @@ export function createSessionMcpRuntime(params: {
|
||||
`bundle-mcp: server key "${serverName}" registered as "${safeServerName}" for provider-safe tool names.`,
|
||||
);
|
||||
}
|
||||
preparedEntries.push({ serverName, rawServer, resolved, safeServerName });
|
||||
}
|
||||
|
||||
// Bounded fan-out keeps common 4-5 server setups parallel without letting
|
||||
// large configs spawn/connect every MCP transport at once.
|
||||
type ServerResult = {
|
||||
serverName: string;
|
||||
serverEntry: McpServerCatalog | null;
|
||||
toolEntries: McpCatalogTool[];
|
||||
diagnostics: McpToolCatalogDiagnostic[];
|
||||
};
|
||||
|
||||
const tasks = preparedEntries.map(
|
||||
({ serverName, rawServer, resolved, safeServerName }) =>
|
||||
async (): Promise<ServerResult> => {
|
||||
failIfDisposed();
|
||||
|
||||
let session = sessions.get(serverName);
|
||||
if (session?.retiring) {
|
||||
session = undefined;
|
||||
}
|
||||
const reusedSession = Boolean(session);
|
||||
if (!session) {
|
||||
const client = new Client(
|
||||
{
|
||||
name: "openclaw-bundle-mcp",
|
||||
version: "0.0.0",
|
||||
},
|
||||
{
|
||||
jsonSchemaValidator: createBundleMcpJsonSchemaValidator(),
|
||||
listChanged: {
|
||||
tools: {
|
||||
autoRefresh: false,
|
||||
debounceMs: 0,
|
||||
onChanged: (error) => {
|
||||
if (error) {
|
||||
logWarn(
|
||||
`bundle-mcp: failed to refresh changed tool list for server "${serverName}": ${redactErrorUrls(error)}`,
|
||||
);
|
||||
}
|
||||
catalogInvalidationGeneration += 1;
|
||||
catalog = null;
|
||||
catalogInFlight = undefined;
|
||||
},
|
||||
},
|
||||
let session = sessions.get(serverName);
|
||||
const reusedSession = Boolean(session);
|
||||
let connected = Boolean(session);
|
||||
if (!session) {
|
||||
const client = new Client(
|
||||
{
|
||||
name: "openclaw-bundle-mcp",
|
||||
version: "0.0.0",
|
||||
},
|
||||
{
|
||||
jsonSchemaValidator: createBundleMcpJsonSchemaValidator(),
|
||||
listChanged: {
|
||||
tools: {
|
||||
autoRefresh: false,
|
||||
debounceMs: 0,
|
||||
onChanged: (error) => {
|
||||
if (error) {
|
||||
logWarn(
|
||||
`bundle-mcp: failed to refresh changed tool list for server "${serverName}": ${redactErrorUrls(error)}`,
|
||||
);
|
||||
}
|
||||
catalogInvalidationGeneration += 1;
|
||||
catalog = null;
|
||||
catalogInFlight = undefined;
|
||||
},
|
||||
},
|
||||
);
|
||||
session = {
|
||||
serverName,
|
||||
client,
|
||||
transport: resolved.transport,
|
||||
transportType: resolved.transportType,
|
||||
requestTimeoutMs: resolved.requestTimeoutMs,
|
||||
supportsParallelToolCalls: resolved.supportsParallelToolCalls,
|
||||
connected: false,
|
||||
retiring: false,
|
||||
catalogUseCount: 0,
|
||||
sharedAcrossCatalogGenerations: false,
|
||||
detachStderr: resolved.detachStderr,
|
||||
};
|
||||
sessions.set(serverName, session);
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
session = {
|
||||
serverName,
|
||||
client,
|
||||
transport: resolved.transport,
|
||||
transportType: resolved.transportType,
|
||||
requestTimeoutMs: resolved.requestTimeoutMs,
|
||||
supportsParallelToolCalls: resolved.supportsParallelToolCalls,
|
||||
detachStderr: resolved.detachStderr,
|
||||
};
|
||||
sessions.set(serverName, session);
|
||||
}
|
||||
|
||||
if (session.catalogUseCount === 0) {
|
||||
session.sharedAcrossCatalogGenerations = false;
|
||||
}
|
||||
if (reusedSession && session.catalogUseCount > 0) {
|
||||
session.sharedAcrossCatalogGenerations = true;
|
||||
}
|
||||
session.catalogUseCount += 1;
|
||||
let connectedForCatalog = false;
|
||||
try {
|
||||
failIfDisposed();
|
||||
await ensureSessionConnected(session, resolved.connectionTimeoutMs);
|
||||
connectedForCatalog = true;
|
||||
failIfDisposed();
|
||||
const capabilities = summarizeServerCapabilities(
|
||||
session.client.getServerCapabilities(),
|
||||
);
|
||||
const listedTools = await listAllToolsBestEffort({
|
||||
client: session.client,
|
||||
timeoutMs: getCatalogListTimeoutMs(rawServer, resolved.requestTimeoutMs),
|
||||
suppressUnsupported: Boolean(
|
||||
!capabilities.tools && (capabilities.resources || capabilities.prompts),
|
||||
),
|
||||
});
|
||||
failIfDisposed();
|
||||
const selection = getMcpToolSelection(rawServer);
|
||||
const exposedTools = listedTools.filter((tool) =>
|
||||
shouldExposeMcpTool(selection, tool.name.trim()),
|
||||
);
|
||||
const serverEntry: McpServerCatalog = {
|
||||
serverName,
|
||||
safeServerName,
|
||||
launchSummary: resolved.description,
|
||||
toolCount: exposedTools.length,
|
||||
requestTimeoutMs: resolved.requestTimeoutMs,
|
||||
supportsParallelToolCalls: resolved.supportsParallelToolCalls,
|
||||
...(capabilities.resources ? { resources: capabilities.resources } : {}),
|
||||
...(capabilities.prompts ? { prompts: capabilities.prompts } : {}),
|
||||
...(capabilities.tools
|
||||
? {
|
||||
tools: {
|
||||
...capabilities.tools,
|
||||
...(exposedTools.length !== listedTools.length
|
||||
? { filteredCount: listedTools.length - exposedTools.length }
|
||||
: {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(selection.include || selection.exclude
|
||||
? {
|
||||
toolFilter: {
|
||||
...(selection.include ? { include: [...selection.include] } : {}),
|
||||
...(selection.exclude ? { exclude: [...selection.exclude] } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const toolEntries: McpCatalogTool[] = [];
|
||||
for (const tool of exposedTools) {
|
||||
const toolName = tool.name.trim();
|
||||
if (!toolName) {
|
||||
continue;
|
||||
try {
|
||||
failIfDisposed();
|
||||
if (!connected) {
|
||||
await connectWithTimeout(
|
||||
session.client,
|
||||
session.transport,
|
||||
resolved.connectionTimeoutMs,
|
||||
);
|
||||
connected = true;
|
||||
}
|
||||
failIfDisposed();
|
||||
const capabilities = summarizeServerCapabilities(
|
||||
session.client.getServerCapabilities(),
|
||||
);
|
||||
const listedTools = await listAllToolsBestEffort({
|
||||
client: session.client,
|
||||
timeoutMs: getCatalogListTimeoutMs(rawServer, resolved.requestTimeoutMs),
|
||||
suppressUnsupported: Boolean(
|
||||
!capabilities.tools && (capabilities.resources || capabilities.prompts),
|
||||
),
|
||||
});
|
||||
failIfDisposed();
|
||||
const selection = getMcpToolSelection(rawServer);
|
||||
const exposedTools = listedTools.filter((tool) =>
|
||||
shouldExposeMcpTool(selection, tool.name.trim()),
|
||||
);
|
||||
servers[serverName] = {
|
||||
serverName,
|
||||
safeServerName,
|
||||
launchSummary: resolved.description,
|
||||
toolCount: exposedTools.length,
|
||||
requestTimeoutMs: resolved.requestTimeoutMs,
|
||||
supportsParallelToolCalls: resolved.supportsParallelToolCalls,
|
||||
...(capabilities.resources ? { resources: capabilities.resources } : {}),
|
||||
...(capabilities.prompts ? { prompts: capabilities.prompts } : {}),
|
||||
...(capabilities.tools
|
||||
? {
|
||||
tools: {
|
||||
...capabilities.tools,
|
||||
...(exposedTools.length !== listedTools.length
|
||||
? { filteredCount: listedTools.length - exposedTools.length }
|
||||
: {}),
|
||||
},
|
||||
}
|
||||
toolEntries.push({
|
||||
serverName,
|
||||
safeServerName,
|
||||
toolName,
|
||||
title: tool.title,
|
||||
description: sanitizeMcpMetadataText(tool.description),
|
||||
inputSchema: tool.inputSchema,
|
||||
fallbackDescription: `Provided by bundle MCP server "${serverName}" (${resolved.description}).`,
|
||||
});
|
||||
}
|
||||
return {
|
||||
serverName,
|
||||
serverEntry,
|
||||
toolEntries,
|
||||
diagnostics: [] as McpToolCatalogDiagnostic[],
|
||||
};
|
||||
} catch (error) {
|
||||
const message = redactErrorUrls(error);
|
||||
if (!disposed) {
|
||||
const action = reusedSession ? "refresh" : "start";
|
||||
logWarn(
|
||||
`bundle-mcp: failed to ${action} server "${serverName}" (${resolved.description}): ${message}`,
|
||||
);
|
||||
}
|
||||
const diags: McpToolCatalogDiagnostic[] = [
|
||||
{
|
||||
serverName,
|
||||
safeServerName,
|
||||
launchSummary: resolved.description,
|
||||
message,
|
||||
},
|
||||
];
|
||||
const sharedWithNewerGeneration =
|
||||
session.sharedAcrossCatalogGenerations || session.catalogUseCount > 1;
|
||||
if (!connectedForCatalog && !session.connected) {
|
||||
// Timed-out connects can still leave the SDK client bound to a
|
||||
// transport. Delete before async close so future catalogs start fresh.
|
||||
await retireSessionIfCurrent(serverName, session);
|
||||
} else if (!reusedSession && !sharedWithNewerGeneration) {
|
||||
// Catalog invalidation can overlap generations; an older failed
|
||||
// generation must not dispose a session a newer one already reused.
|
||||
await retireSessionIfCurrent(serverName, session);
|
||||
}
|
||||
failIfDisposed();
|
||||
return {
|
||||
serverName,
|
||||
serverEntry: null,
|
||||
toolEntries: [],
|
||||
diagnostics: diags,
|
||||
} as ServerResult;
|
||||
} finally {
|
||||
session.catalogUseCount -= 1;
|
||||
if (session.catalogUseCount === 0) {
|
||||
session.sharedAcrossCatalogGenerations = false;
|
||||
}
|
||||
: {}),
|
||||
...(selection.include || selection.exclude
|
||||
? {
|
||||
toolFilter: {
|
||||
...(selection.include ? { include: [...selection.include] } : {}),
|
||||
...(selection.exclude ? { exclude: [...selection.exclude] } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
for (const tool of exposedTools) {
|
||||
const toolName = tool.name.trim();
|
||||
if (!toolName) {
|
||||
continue;
|
||||
}
|
||||
},
|
||||
);
|
||||
const { results, firstError, hasError } = await runTasksWithConcurrency({
|
||||
tasks,
|
||||
limit: BUNDLE_MCP_CATALOG_CONNECT_CONCURRENCY,
|
||||
errorMode: "continue",
|
||||
});
|
||||
if (hasError) {
|
||||
throw firstError;
|
||||
}
|
||||
|
||||
for (const result of results) {
|
||||
if (!result) {
|
||||
continue;
|
||||
tools.push({
|
||||
serverName,
|
||||
safeServerName,
|
||||
toolName,
|
||||
title: tool.title,
|
||||
description: sanitizeMcpMetadataText(tool.description),
|
||||
inputSchema: tool.inputSchema,
|
||||
fallbackDescription: `Provided by bundle MCP server "${serverName}" (${resolved.description}).`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const message = redactErrorUrls(error);
|
||||
if (!disposed) {
|
||||
const action = reusedSession ? "refresh" : "start";
|
||||
logWarn(
|
||||
`bundle-mcp: failed to ${action} server "${serverName}" (${resolved.description}): ${message}`,
|
||||
);
|
||||
}
|
||||
diagnostics.push({
|
||||
serverName,
|
||||
safeServerName,
|
||||
launchSummary: resolved.description,
|
||||
message,
|
||||
});
|
||||
if (!reusedSession) {
|
||||
await disposeSession(session);
|
||||
sessions.delete(serverName);
|
||||
}
|
||||
failIfDisposed();
|
||||
}
|
||||
const { serverEntry, toolEntries, diagnostics: serverDiags } = result;
|
||||
if (serverEntry) {
|
||||
servers[result.serverName] = serverEntry;
|
||||
}
|
||||
tools.push(...toolEntries);
|
||||
diagnostics.push(...serverDiags);
|
||||
}
|
||||
|
||||
failIfDisposed();
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
resolveMediaToolInboundRoots,
|
||||
resolveCapabilityModelConfigForTool,
|
||||
resolveMediaToolLocalRoots,
|
||||
resolveRemoteMediaSsrfPolicy,
|
||||
resolveModelFromRegistry,
|
||||
} from "./media-tool-shared.js";
|
||||
|
||||
@@ -123,6 +124,25 @@ describe("resolveMediaToolLocalRoots", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveRemoteMediaSsrfPolicy", () => {
|
||||
it("does not forward web_fetch hostname restrictions to generation providers", () => {
|
||||
expect(
|
||||
resolveRemoteMediaSsrfPolicy({
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
ssrfPolicy: {
|
||||
allowRfc2544BenchmarkRange: true,
|
||||
hostnameAllowlist: ["web-fetch-only.example"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toEqual({ allowRfc2544BenchmarkRange: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveModelFromRegistry", () => {
|
||||
it("normalizes provider and model refs before registry lookup", () => {
|
||||
const foundModel = { provider: "ollama", id: "qwen3.5:397b-cloud" };
|
||||
|
||||
@@ -16,10 +16,10 @@ import {
|
||||
} from "../../../packages/media-generation-core/src/capability-model-ref.js";
|
||||
import type { AgentModelConfig } from "../../config/types.agents-shared.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||
import type { Model } from "../../llm/types.js";
|
||||
import { resolveChannelInboundAttachmentRootsForChannel } from "../../media/channel-inbound-roots.js";
|
||||
import { getDefaultLocalRoots } from "../../media/local-media-access.js";
|
||||
import { projectRemoteMediaSsrfPolicy } from "../../media/remote-media-ssrf-policy.js";
|
||||
import { readSnakeCaseParamRaw } from "../../param-key.js";
|
||||
import { loadCapabilityManifestSnapshot } from "../../plugins/capability-provider-runtime.js";
|
||||
import { listAvailableManifestContractValues } from "../../plugins/manifest-contract-eligibility.js";
|
||||
@@ -126,13 +126,9 @@ export function readGenerationTimeoutMs(args: Record<string, unknown>): number |
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the shared remote-media SSRF policy used by media tools that fetch URLs.
|
||||
*/
|
||||
export function resolveRemoteMediaSsrfPolicy(
|
||||
cfg: OpenClawConfig | undefined,
|
||||
): SsrFPolicy | undefined {
|
||||
return cfg?.tools?.web?.fetch?.ssrfPolicy;
|
||||
/** Resolves the remote-media subset of the web_fetch SSRF policy. */
|
||||
export function resolveRemoteMediaSsrfPolicy(cfg: OpenClawConfig | undefined) {
|
||||
return projectRemoteMediaSsrfPolicy(cfg);
|
||||
}
|
||||
|
||||
function applyAgentDefaultModelConfig(
|
||||
|
||||
@@ -298,4 +298,61 @@ describe("web_fetch provider fallback normalization", () => {
|
||||
expect(secondDetails.text).toContain("perplexity-fetch fallback body");
|
||||
expect(secondDetails.cached).toBeUndefined();
|
||||
});
|
||||
|
||||
it("suppresses provider fallbacks and their cache entries under a hostname allowlist", async () => {
|
||||
global.fetch = withFetchPreconnect(
|
||||
vi.fn(async () => {
|
||||
throw new Error("network failed");
|
||||
}),
|
||||
);
|
||||
resolveWebFetchDefinitionMock.mockReturnValue({
|
||||
provider: { id: "firecrawl" },
|
||||
definition: {
|
||||
description: "firecrawl",
|
||||
parameters: {},
|
||||
execute: async () => ({ text: "provider-only content" }),
|
||||
},
|
||||
});
|
||||
const url = "https://restricted-fallback.example/page";
|
||||
const permissiveTool = createWebFetchTool({
|
||||
config: {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
provider: "firecrawl",
|
||||
cacheTtlMinutes: 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
sandboxed: false,
|
||||
});
|
||||
|
||||
const permissive = await permissiveTool?.execute?.("permissive-fallback", { url });
|
||||
expect((permissive?.details as { text?: string } | undefined)?.text).toContain(
|
||||
"provider-only content",
|
||||
);
|
||||
|
||||
const restrictiveTool = createWebFetchTool({
|
||||
config: {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
provider: "firecrawl",
|
||||
cacheTtlMinutes: 10,
|
||||
ssrfPolicy: {
|
||||
hostnameAllowlist: ["restricted-fallback.example"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
sandboxed: false,
|
||||
});
|
||||
|
||||
await expect(restrictiveTool?.execute?.("restricted-fallback", { url })).rejects.toThrow(
|
||||
"network failed",
|
||||
);
|
||||
expect(resolveWebFetchDefinitionMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,6 +9,12 @@ import "./web-fetch.test-mocks.js";
|
||||
|
||||
const lookupMock = vi.fn();
|
||||
const resolvePinnedHostname = ssrf.resolvePinnedHostname;
|
||||
const logWarnMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../../logger.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../../logger.js")>("../../logger.js");
|
||||
return { ...actual, logWarn: logWarnMock };
|
||||
});
|
||||
|
||||
function redirectResponse(location: string): Response {
|
||||
return {
|
||||
@@ -50,7 +56,11 @@ function firstFetchUrl(fetchSpy: ReturnType<typeof setMockFetch>): string {
|
||||
function createWebFetchToolForTest(params?: {
|
||||
firecrawlApiKey?: string;
|
||||
useTrustedEnvProxy?: boolean;
|
||||
ssrfPolicy?: { allowRfc2544BenchmarkRange?: boolean; allowIpv6UniqueLocalRange?: boolean };
|
||||
ssrfPolicy?: {
|
||||
allowRfc2544BenchmarkRange?: boolean;
|
||||
allowIpv6UniqueLocalRange?: boolean;
|
||||
hostnameAllowlist?: string[];
|
||||
};
|
||||
cacheTtlMinutes?: number;
|
||||
}) {
|
||||
return createWebFetchTool({
|
||||
@@ -103,6 +113,7 @@ describe("web_fetch SSRF protection", () => {
|
||||
afterEach(() => {
|
||||
global.fetch = priorFetch;
|
||||
lookupMock.mockClear();
|
||||
logWarnMock.mockClear();
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
@@ -161,6 +172,137 @@ describe("web_fetch SSRF protection", () => {
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("supports exact and subdomain-only wildcard hostname allowlist entries", async () => {
|
||||
lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
const fetchSpy = setMockFetch().mockResolvedValue(textResponse("allowed"));
|
||||
const tool = createWebFetchToolForTest({
|
||||
ssrfPolicy: {
|
||||
hostnameAllowlist: [" ALLOWED.EXAMPLE. ", "*.trusted.example."],
|
||||
},
|
||||
});
|
||||
|
||||
await tool?.execute?.("exact", { url: "https://allowed.example/page" });
|
||||
await tool?.execute?.("wildcard", { url: "https://cdn.trusted.example/page" });
|
||||
await expectBlockedUrl(tool, "https://trusted.example/page", /allowlist/i);
|
||||
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("blocks an initial hostname outside the configured allowlist", async () => {
|
||||
lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
const fetchSpy = setMockFetch();
|
||||
const tool = createWebFetchToolForTest({
|
||||
ssrfPolicy: { hostnameAllowlist: ["allowed.example"] },
|
||||
});
|
||||
|
||||
await expectBlockedUrl(tool, "https://blocked.example/page", /allowlist/i);
|
||||
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(lookupMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs an initial allowlist block once without URL secrets", async () => {
|
||||
const fetchSpy = setMockFetch();
|
||||
const tool = createWebFetchToolForTest({
|
||||
ssrfPolicy: { hostnameAllowlist: ["allowed.example"] },
|
||||
});
|
||||
|
||||
await expectBlockedUrl(
|
||||
tool,
|
||||
"https://user:password@blocked.example/private/path?token=secret#fragment",
|
||||
/allowlist/i,
|
||||
);
|
||||
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(logWarnMock).toHaveBeenCalledTimes(1);
|
||||
const warning = String(logWarnMock.mock.calls[0]?.[0] ?? "");
|
||||
expect(warning).toContain(
|
||||
"security: blocked URL fetch (url-fetch) targetOrigin=https://blocked.example",
|
||||
);
|
||||
expect(warning).not.toContain("user");
|
||||
expect(warning).not.toContain("password");
|
||||
expect(warning).not.toContain("/private/path");
|
||||
expect(warning).not.toContain("token=secret");
|
||||
expect(warning).not.toContain("#fragment");
|
||||
});
|
||||
|
||||
it("re-applies the hostname allowlist to every redirect", async () => {
|
||||
lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
const fetchSpy = setMockFetch().mockResolvedValueOnce(
|
||||
redirectResponse("https://blocked.example/secret"),
|
||||
);
|
||||
const tool = createWebFetchToolForTest({
|
||||
ssrfPolicy: { hostnameAllowlist: ["allowed.example"] },
|
||||
});
|
||||
|
||||
await expectBlockedUrl(tool, "https://allowed.example/start", /allowlist/i);
|
||||
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
expect(lookupMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not let a permissive cache entry bypass a later hostname allowlist", async () => {
|
||||
lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
const fetchSpy = setMockFetch().mockResolvedValue(textResponse("cached body"));
|
||||
const url = "https://cache-policy.example/page";
|
||||
|
||||
const permissiveTool = createWebFetchToolForTest({ cacheTtlMinutes: 1 });
|
||||
await permissiveTool?.execute?.("permissive", { url });
|
||||
|
||||
const restrictiveTool = createWebFetchToolForTest({
|
||||
cacheTtlMinutes: 1,
|
||||
ssrfPolicy: { hostnameAllowlist: ["allowed.example"] },
|
||||
});
|
||||
await expectBlockedUrl(restrictiveTool, url, /allowlist/i);
|
||||
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("isolates cache entries across different restrictive hostname policies", async () => {
|
||||
lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
const fetchSpy = setMockFetch()
|
||||
.mockResolvedValueOnce(textResponse("exact policy"))
|
||||
.mockResolvedValueOnce(textResponse("wildcard policy"));
|
||||
const url = "https://cache.allowed.example/page";
|
||||
|
||||
const exactTool = createWebFetchToolForTest({
|
||||
cacheTtlMinutes: 1,
|
||||
ssrfPolicy: { hostnameAllowlist: ["cache.allowed.example"] },
|
||||
});
|
||||
await exactTool?.execute?.("exact-policy", { url });
|
||||
|
||||
const wildcardTool = createWebFetchToolForTest({
|
||||
cacheTtlMinutes: 1,
|
||||
ssrfPolicy: { hostnameAllowlist: ["*.allowed.example"] },
|
||||
});
|
||||
const result = await wildcardTool?.execute?.("wildcard-policy", { url });
|
||||
|
||||
expect((result?.details as { cached?: boolean } | undefined)?.cached).toBeUndefined();
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("revalidates the hostname policy before returning a cache hit", async () => {
|
||||
lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
const fetchSpy = setMockFetch().mockResolvedValue(textResponse("cached policy body"));
|
||||
const assertPolicySpy = vi.spyOn(ssrf, "assertHostnameAllowedWithPolicy");
|
||||
const tool = createWebFetchToolForTest({
|
||||
cacheTtlMinutes: 1,
|
||||
ssrfPolicy: { hostnameAllowlist: ["cache-hit.example"] },
|
||||
});
|
||||
|
||||
await tool?.execute?.("prime-cache", { url: "https://cache-hit.example/page" });
|
||||
assertPolicySpy.mockClear();
|
||||
const cached = await tool?.execute?.("read-cache", {
|
||||
url: "https://cache-hit.example/page",
|
||||
});
|
||||
|
||||
expect((cached?.details as { cached?: boolean } | undefined)?.cached).toBe(true);
|
||||
expect(assertPolicySpy).toHaveBeenCalledExactlyOnceWith("cache-hit.example", {
|
||||
hostnameAllowlist: ["cache-hit.example"],
|
||||
});
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("allows public hosts", async () => {
|
||||
lookupMock.mockResolvedValue([{ address: "93.184.216.34", family: 4 }]);
|
||||
|
||||
|
||||
@@ -9,15 +9,21 @@ import {
|
||||
normalizeOptionalString,
|
||||
} from "@openclaw/normalization-core/string-coerce";
|
||||
import { Type } from "typebox";
|
||||
import { resolveWebProviderConfig } from "../../../packages/web-content-core/src/provider-runtime-shared.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { SsrFBlockedError, type LookupFn, type SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||
import {
|
||||
assertHostnameAllowedWithPolicy,
|
||||
normalizeHostnameAllowlist,
|
||||
SsrFBlockedError,
|
||||
type LookupFn,
|
||||
type SsrFPolicy,
|
||||
} from "../../infra/net/ssrf.js";
|
||||
import { logDebug } from "../../logger.js";
|
||||
import type { RuntimeWebFetchMetadata } from "../../secrets/runtime-web-tools.types.js";
|
||||
import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js";
|
||||
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
|
||||
import { isRecord } from "../../utils.js";
|
||||
import { extractReadableContent } from "../../web-fetch/content-extractors.runtime.js";
|
||||
import { resolveWebProviderConfig } from "../../../packages/web-content-core/src/provider-runtime-shared.js";
|
||||
import { stringEnum } from "../schema/string-enum.js";
|
||||
import { setToolTerminalPresentation } from "../tool-terminal-presentation.js";
|
||||
import type { AnyAgentTool } from "./common.js";
|
||||
@@ -135,6 +141,28 @@ function resolveFetchUseTrustedEnvProxy(fetch?: WebFetchConfig): boolean {
|
||||
return fetch?.useTrustedEnvProxy === true;
|
||||
}
|
||||
|
||||
function resolveFetchSsrfPolicy(fetch?: WebFetchConfig): SsrFPolicy | undefined {
|
||||
const configured = fetch?.ssrfPolicy;
|
||||
if (!configured) {
|
||||
return undefined;
|
||||
}
|
||||
const hostnameAllowlist =
|
||||
configured.hostnameAllowlist === undefined
|
||||
? undefined
|
||||
: normalizeHostnameAllowlist(configured.hostnameAllowlist);
|
||||
if (configured.hostnameAllowlist !== undefined && hostnameAllowlist?.length === 0) {
|
||||
throw new Error(
|
||||
"tools.web.fetch.ssrfPolicy.hostnameAllowlist must contain at least one specific hostname",
|
||||
);
|
||||
}
|
||||
const policy: SsrFPolicy = {
|
||||
...(configured.allowRfc2544BenchmarkRange === true ? { allowRfc2544BenchmarkRange: true } : {}),
|
||||
...(configured.allowIpv6UniqueLocalRange === true ? { allowIpv6UniqueLocalRange: true } : {}),
|
||||
...(hostnameAllowlist ? { hostnameAllowlist } : {}),
|
||||
};
|
||||
return Object.keys(policy).length > 0 ? policy : undefined;
|
||||
}
|
||||
|
||||
function resolveFetchMaxCharsCap(fetch?: WebFetchConfig): number {
|
||||
const raw =
|
||||
fetch && "maxCharsCap" in fetch && typeof fetch.maxCharsCap === "number"
|
||||
@@ -328,10 +356,7 @@ type WebFetchRuntimeParams = {
|
||||
readabilityEnabled: boolean;
|
||||
config?: OpenClawConfig;
|
||||
useTrustedEnvProxy: boolean;
|
||||
ssrfPolicy?: {
|
||||
allowRfc2544BenchmarkRange?: boolean;
|
||||
allowIpv6UniqueLocalRange?: boolean;
|
||||
};
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
providerCacheKey?: string;
|
||||
lookupFn?: LookupFn;
|
||||
signal?: AbortSignal;
|
||||
@@ -452,6 +477,11 @@ async function maybeFetchProviderWebFetchPayload(
|
||||
tookMs: number;
|
||||
},
|
||||
): Promise<Record<string, unknown> | null> {
|
||||
// Hosted providers own their redirect chain, so they cannot preserve a
|
||||
// caller's restrictive hostname policy. Keep restricted fetches local.
|
||||
if (params.ssrfPolicy?.hostnameAllowlist?.length) {
|
||||
return null;
|
||||
}
|
||||
const providerFallback = await params.resolveProviderFallback();
|
||||
if (!providerFallback) {
|
||||
return null;
|
||||
@@ -474,23 +504,24 @@ async function maybeFetchProviderWebFetchPayload(
|
||||
}
|
||||
|
||||
async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string, unknown>> {
|
||||
const allowRfc2544BenchmarkRange = params.ssrfPolicy?.allowRfc2544BenchmarkRange === true;
|
||||
const allowIpv6UniqueLocalRange = params.ssrfPolicy?.allowIpv6UniqueLocalRange === true;
|
||||
const useTrustedEnvProxy = params.useTrustedEnvProxy;
|
||||
const ssrfPolicy: SsrFPolicy | undefined =
|
||||
allowRfc2544BenchmarkRange || allowIpv6UniqueLocalRange
|
||||
? {
|
||||
...(allowRfc2544BenchmarkRange ? { allowRfc2544BenchmarkRange } : {}),
|
||||
...(allowIpv6UniqueLocalRange ? { allowIpv6UniqueLocalRange } : {}),
|
||||
}
|
||||
: undefined;
|
||||
const ssrfPolicy = params.ssrfPolicy;
|
||||
const cachePolicy = {
|
||||
allowRfc2544BenchmarkRange: ssrfPolicy?.allowRfc2544BenchmarkRange === true,
|
||||
allowIpv6UniqueLocalRange: ssrfPolicy?.allowIpv6UniqueLocalRange === true,
|
||||
hostnameAllowlist: [...(ssrfPolicy?.hostnameAllowlist ?? [])].toSorted(),
|
||||
};
|
||||
const cacheKey = normalizeCacheKey(
|
||||
`fetch:${params.url}:${params.extractMode}:${params.maxChars}${params.providerCacheKey ? `:provider:${params.providerCacheKey}` : ""}${allowRfc2544BenchmarkRange ? ":allow-rfc2544" : ""}${allowIpv6UniqueLocalRange ? ":allow-ipv6-ula" : ""}${useTrustedEnvProxy ? ":trusted-env-proxy" : ""}`,
|
||||
JSON.stringify({
|
||||
kind: "fetch",
|
||||
url: params.url,
|
||||
extractMode: params.extractMode,
|
||||
maxChars: params.maxChars,
|
||||
provider: params.providerCacheKey ?? null,
|
||||
ssrfPolicy: cachePolicy,
|
||||
useTrustedEnvProxy,
|
||||
}),
|
||||
);
|
||||
const cached = readCache(FETCH_CACHE, cacheKey);
|
||||
if (cached) {
|
||||
return { ...cached.value, cached: true };
|
||||
}
|
||||
|
||||
let parsedUrl: URL;
|
||||
try {
|
||||
@@ -501,6 +532,13 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
|
||||
if (!["http:", "https:"].includes(parsedUrl.protocol)) {
|
||||
throw new Error("Invalid URL: must be http or https");
|
||||
}
|
||||
const cached = readCache(FETCH_CACHE, cacheKey);
|
||||
if (cached) {
|
||||
// Cache hits skip the guarded network path, so apply its hostname policy
|
||||
// before returning cached content. Misses use the guard's audited check.
|
||||
assertHostnameAllowedWithPolicy(parsedUrl.hostname, ssrfPolicy);
|
||||
return { ...cached.value, cached: true };
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
let res: Response;
|
||||
@@ -634,8 +672,9 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
|
||||
title = basic.title;
|
||||
extractor = "raw-html";
|
||||
} else {
|
||||
const providerLabel =
|
||||
(await params.resolveProviderFallback())?.provider.label ?? "provider fallback";
|
||||
const providerLabel = params.ssrfPolicy?.hostnameAllowlist?.length
|
||||
? "provider fallback"
|
||||
: ((await params.resolveProviderFallback())?.provider.label ?? "provider fallback");
|
||||
throw new Error(
|
||||
`Web fetch extraction failed: Readability, ${providerLabel}, and basic HTML cleanup returned no content.`,
|
||||
);
|
||||
@@ -795,7 +834,7 @@ export function createWebFetchTool(options?: {
|
||||
readabilityEnabled,
|
||||
config,
|
||||
useTrustedEnvProxy: resolveFetchUseTrustedEnvProxy(executionFetch),
|
||||
ssrfPolicy: executionFetch?.ssrfPolicy,
|
||||
ssrfPolicy: resolveFetchSsrfPolicy(executionFetch),
|
||||
...(providerCacheKey ? { providerCacheKey } : {}),
|
||||
lookupFn: options?.lookupFn,
|
||||
signal,
|
||||
|
||||
@@ -948,6 +948,8 @@ export const FIELD_HELP: Record<string, string> = {
|
||||
"Allow RFC 2544 benchmark-range IPs (198.18.0.0/15) for fake-IP proxy compatibility such as Clash or Surge.",
|
||||
"tools.web.fetch.ssrfPolicy.allowIpv6UniqueLocalRange":
|
||||
"Allow IPv6 Unique Local Addresses (fc00::/7) for trusted fake-IP proxy compatibility such as sing-box, Clash, or Surge.",
|
||||
"tools.web.fetch.ssrfPolicy.hostnameAllowlist":
|
||||
"Restrict web_fetch to exact hostnames and subdomain-only wildcard patterns such as *.example.com. The initial URL and every redirect must match.",
|
||||
models:
|
||||
"Model catalog root for provider definitions, merge/replace behavior, and optional Bedrock discovery integration. Keep provider definitions explicit and validated before relying on production failover paths.",
|
||||
"models.mode":
|
||||
|
||||
@@ -342,6 +342,7 @@ export const FIELD_LABELS: Record<string, string> = {
|
||||
"tools.web.fetch.ssrfPolicy.allowRfc2544BenchmarkRange":
|
||||
"Web Fetch Allow RFC 2544 Benchmark Range",
|
||||
"tools.web.fetch.ssrfPolicy.allowIpv6UniqueLocalRange": "Web Fetch Allow IPv6 Unique Local Range",
|
||||
"tools.web.fetch.ssrfPolicy.hostnameAllowlist": "Web Fetch Hostname Allowlist",
|
||||
"gateway.controlUi.basePath": "Control UI Base Path",
|
||||
"gateway.controlUi.root": "Control UI Assets Root",
|
||||
"gateway.controlUi.embedSandbox": "Control UI Embed Sandbox Mode",
|
||||
|
||||
@@ -847,6 +847,7 @@ describe("config schema", () => {
|
||||
ssrfPolicy: {
|
||||
allowRfc2544BenchmarkRange: true,
|
||||
allowIpv6UniqueLocalRange: true,
|
||||
hostnameAllowlist: [" API.EXAMPLE.COM. ", "*.Assets.Example.com.", "api.example.com"],
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -855,9 +856,30 @@ describe("config schema", () => {
|
||||
expect(parsed?.web?.fetch?.ssrfPolicy).toEqual({
|
||||
allowRfc2544BenchmarkRange: true,
|
||||
allowIpv6UniqueLocalRange: true,
|
||||
hostnameAllowlist: ["api.example.com", "*.assets.example.com"],
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "empty", hostnameAllowlist: [] },
|
||||
{ name: "blank", hostnameAllowlist: [""] },
|
||||
{ name: "catch-all", hostnameAllowlist: ["*"] },
|
||||
{ name: "suffix-star", hostnameAllowlist: ["example.*"] },
|
||||
{ name: "nested-star", hostnameAllowlist: ["*.*.example.com"] },
|
||||
{ name: "URL-like", hostnameAllowlist: ["https://example.com"] },
|
||||
{ name: "wildcard path", hostnameAllowlist: ["*.example.com/path"] },
|
||||
])("rejects a $name web fetch hostname allowlist", ({ hostnameAllowlist }) => {
|
||||
const parsed = ToolsSchema.safeParse({
|
||||
web: {
|
||||
fetch: {
|
||||
ssrfPolicy: { hostnameAllowlist },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(parsed.success).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts web fetch trusted env proxy opt-in in the runtime zod schema", () => {
|
||||
const parsed = ToolsSchema.parse({
|
||||
web: {
|
||||
|
||||
@@ -680,6 +680,8 @@ export type ToolsConfig = {
|
||||
allowRfc2544BenchmarkRange?: boolean;
|
||||
/** Allow IPv6 Unique Local Addresses (fc00::/7) for trusted fake-IP proxy compatibility. */
|
||||
allowIpv6UniqueLocalRange?: boolean;
|
||||
/** Restrict initial and redirected requests to exact hostnames or subdomain-only `*.example.com` patterns. */
|
||||
hostnameAllowlist?: string[];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
@@ -10,6 +10,7 @@ import { splitSandboxBindSpec } from "../agents/sandbox/bind-spec.js";
|
||||
import { isSandboxHostPathAbsolute } from "../agents/sandbox/host-paths.js";
|
||||
import { getBlockedNetworkModeReason } from "../agents/sandbox/network-mode.js";
|
||||
import { parseDurationMs } from "../cli/parse-duration.js";
|
||||
import { normalizeHostnameAllowlist } from "../infra/net/ssrf.js";
|
||||
import { isBlockedObjectKey } from "../infra/prototype-keys.js";
|
||||
import { LEGACY_WEB_SEARCH_PROVIDER_CONFIG_KEYS } from "./web-search-legacy-provider-keys.js";
|
||||
import { AgentModelSchema, AgentToolModelSchema } from "./zod-schema.agent-model.js";
|
||||
@@ -433,6 +434,52 @@ const ToolsWebSearchSchema = z
|
||||
)
|
||||
.optional();
|
||||
|
||||
function normalizeWebFetchHostnameAllowlistPattern(value: string): string | undefined {
|
||||
const normalized = normalizeHostnameAllowlist([value])[0];
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
const wildcard = normalized.startsWith("*.");
|
||||
if (normalized.includes("*") && (!wildcard || normalized.slice(2).includes("*"))) {
|
||||
return undefined;
|
||||
}
|
||||
const hostname = wildcard ? normalized.slice(2) : normalized;
|
||||
if (
|
||||
!hostname ||
|
||||
hostname.startsWith(".") ||
|
||||
hostname.endsWith(".") ||
|
||||
hostname.includes("..") ||
|
||||
(wildcard && hostname.includes(":"))
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const authority = hostname.includes(":") ? `[${hostname}]` : hostname;
|
||||
const parsed = new URL(`https://${authority}`);
|
||||
return normalizeHostnameAllowlist([parsed.hostname])[0] === hostname ? normalized : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const WebFetchHostnameAllowlistSchema = z
|
||||
.array(z.string())
|
||||
.min(1, "tools.web.fetch.ssrfPolicy.hostnameAllowlist must not be empty")
|
||||
.superRefine((values, ctx) => {
|
||||
for (const [index, value] of values.entries()) {
|
||||
if (normalizeWebFetchHostnameAllowlistPattern(value)) {
|
||||
continue;
|
||||
}
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: [index],
|
||||
message:
|
||||
"hostname allowlist entries must be exact hostnames or subdomain-only patterns such as '*.example.com'",
|
||||
});
|
||||
}
|
||||
})
|
||||
.transform((values) => normalizeHostnameAllowlist(values));
|
||||
|
||||
const ToolsWebFetchSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
@@ -450,6 +497,7 @@ const ToolsWebFetchSchema = z
|
||||
.object({
|
||||
allowRfc2544BenchmarkRange: z.boolean().optional(),
|
||||
allowIpv6UniqueLocalRange: z.boolean().optional(),
|
||||
hostnameAllowlist: WebFetchHostnameAllowlistSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
|
||||
@@ -434,6 +434,14 @@ describe("applyMediaUnderstanding", () => {
|
||||
};
|
||||
const cfg: OpenClawConfig = {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
ssrfPolicy: {
|
||||
allowRfc2544BenchmarkRange: true,
|
||||
hostnameAllowlist: ["web-fetch-only.example"],
|
||||
},
|
||||
},
|
||||
},
|
||||
media: {
|
||||
audio: {
|
||||
enabled: true,
|
||||
@@ -460,6 +468,11 @@ describe("applyMediaUnderstanding", () => {
|
||||
});
|
||||
|
||||
expect(result.appliedAudio).toBe(true);
|
||||
expect(mockedReadRemoteMediaBuffer).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
ssrfPolicy: { allowRfc2544BenchmarkRange: true },
|
||||
}),
|
||||
);
|
||||
expect(ctx.Transcript).toBe("remote transcript");
|
||||
expect(ctx.Body).toBe("[Audio]\nTranscript:\nremote transcript");
|
||||
});
|
||||
|
||||
@@ -5,19 +5,20 @@ import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "@openclaw/normalization-core/string-coerce";
|
||||
import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
|
||||
import type { MsgContext } from "../auto-reply/templating.js";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||
import { renderFileContextBlock } from "../media/file-context.js";
|
||||
import { extractFileContentFromSource, normalizeMimeType } from "../media/input-files.js";
|
||||
import { wrapExternalContent } from "../security/external-content.js";
|
||||
import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
|
||||
import {
|
||||
extractMediaUserText,
|
||||
formatAudioTranscripts,
|
||||
formatMediaUnderstandingBody,
|
||||
} from "../../packages/media-understanding-common/src/format.js";
|
||||
import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
|
||||
import type { MsgContext } from "../auto-reply/templating.js";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||
import { renderFileContextBlock } from "../media/file-context.js";
|
||||
import { extractFileContentFromSource, normalizeMimeType } from "../media/input-files.js";
|
||||
import { projectRemoteMediaSsrfPolicy } from "../media/remote-media-ssrf-policy.js";
|
||||
import { wrapExternalContent } from "../security/external-content.js";
|
||||
import { resolveAttachmentKind } from "./attachments.js";
|
||||
import { runWithConcurrency } from "./concurrency.js";
|
||||
import { DEFAULT_ECHO_TRANSCRIPT_FORMAT, sendTranscriptEcho } from "./echo-transcript.js";
|
||||
@@ -539,7 +540,7 @@ export async function applyMediaUnderstanding(params: {
|
||||
ctx,
|
||||
workspaceDir: params.workspaceDir,
|
||||
}),
|
||||
ssrfPolicy: cfg.tools?.web?.fetch?.ssrfPolicy,
|
||||
ssrfPolicy: projectRemoteMediaSsrfPolicy(cfg),
|
||||
workspaceDir: mediaWorkspaceDir,
|
||||
});
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
// Audio transcription runner executes the configured media-understanding audio
|
||||
// pipeline and extracts the first transcript output.
|
||||
import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
|
||||
import type { MsgContext } from "../auto-reply/templating.js";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
|
||||
import { projectRemoteMediaSsrfPolicy } from "../media/remote-media-ssrf-policy.js";
|
||||
import {
|
||||
buildProviderRegistry,
|
||||
createMediaAttachmentCache,
|
||||
@@ -29,7 +30,7 @@ export async function runAudioTranscription(params: {
|
||||
const providerRegistry = buildProviderRegistry(params.providers, params.cfg);
|
||||
const cache = createMediaAttachmentCache(attachments, {
|
||||
...(params.localPathRoots ? { localPathRoots: params.localPathRoots } : {}),
|
||||
ssrfPolicy: params.cfg.tools?.web?.fetch?.ssrfPolicy,
|
||||
ssrfPolicy: projectRemoteMediaSsrfPolicy(params.cfg),
|
||||
});
|
||||
|
||||
try {
|
||||
|
||||
@@ -585,7 +585,19 @@ describe("media-understanding runtime", () => {
|
||||
provider: "zai",
|
||||
model: "glm-4.6v",
|
||||
prompt: "Describe it",
|
||||
cfg: {} as OpenClawConfig,
|
||||
cfg: {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
ssrfPolicy: {
|
||||
allowRfc2544BenchmarkRange: true,
|
||||
allowIpv6UniqueLocalRange: true,
|
||||
hostnameAllowlist: ["web-fetch-only.example"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
agentDir: "/tmp/agent",
|
||||
timeoutMs: 45_000,
|
||||
}),
|
||||
@@ -598,7 +610,12 @@ describe("media-understanding runtime", () => {
|
||||
});
|
||||
expect(mocks.createMediaAttachmentCache).toHaveBeenCalledWith(
|
||||
[{ index: 0, url: "https://httpbin.org/image/png", mime: "image/png" }],
|
||||
{ ssrfPolicy: undefined },
|
||||
{
|
||||
ssrfPolicy: {
|
||||
allowRfc2544BenchmarkRange: true,
|
||||
allowIpv6UniqueLocalRange: true,
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(mocks.getBuffer).toHaveBeenCalledWith({
|
||||
attachmentIndex: 0,
|
||||
|
||||
@@ -4,6 +4,7 @@ import path from "node:path";
|
||||
import { kindFromMime, mimeTypeFromFilePath } from "@openclaw/media-core/mime";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import { readLocalFileSafely } from "../infra/fs-safe.js";
|
||||
import { projectRemoteMediaSsrfPolicy } from "../media/remote-media-ssrf-policy.js";
|
||||
import { DEFAULT_MAX_BYTES } from "./defaults.constants.js";
|
||||
import { normalizeImageDescriptionInput } from "./image-input-normalize.js";
|
||||
import { describeImageWithModel } from "./image-runtime.js";
|
||||
@@ -192,7 +193,7 @@ export async function runMediaUnderstandingFile(
|
||||
const providerRegistry = buildProviderRegistry(undefined, cfg);
|
||||
const cache = createMediaAttachmentCache(attachments, {
|
||||
localPathRoots: params.mediaUrl ? undefined : resolveFileLocalRoots(params.filePath),
|
||||
ssrfPolicy: cfg.tools?.web?.fetch?.ssrfPolicy,
|
||||
ssrfPolicy: projectRemoteMediaSsrfPolicy(cfg),
|
||||
});
|
||||
|
||||
try {
|
||||
@@ -295,7 +296,7 @@ async function readImageDescriptionInput(params: {
|
||||
buildFileContext({ ...params, capability: "image" }),
|
||||
);
|
||||
const cache = createMediaAttachmentCache(attachments, {
|
||||
ssrfPolicy: params.cfg.tools?.web?.fetch?.ssrfPolicy,
|
||||
ssrfPolicy: projectRemoteMediaSsrfPolicy(params.cfg),
|
||||
});
|
||||
try {
|
||||
const media = await cache.getBuffer({
|
||||
|
||||
40
src/media/remote-media-ssrf-policy.test.ts
Normal file
40
src/media/remote-media-ssrf-policy.test.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { projectRemoteMediaSsrfPolicy } from "./remote-media-ssrf-policy.js";
|
||||
|
||||
describe("projectRemoteMediaSsrfPolicy", () => {
|
||||
it("keeps fake-IP compatibility flags without forwarding web_fetch host restrictions", () => {
|
||||
const cfg: OpenClawConfig = {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
ssrfPolicy: {
|
||||
allowRfc2544BenchmarkRange: true,
|
||||
allowIpv6UniqueLocalRange: false,
|
||||
hostnameAllowlist: ["content.example.com"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(projectRemoteMediaSsrfPolicy(cfg)).toEqual({
|
||||
allowRfc2544BenchmarkRange: true,
|
||||
allowIpv6UniqueLocalRange: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns no media policy when web_fetch only restricts hostnames", () => {
|
||||
const cfg: OpenClawConfig = {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
ssrfPolicy: { hostnameAllowlist: ["content.example.com"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(projectRemoteMediaSsrfPolicy(cfg)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
29
src/media/remote-media-ssrf-policy.ts
Normal file
29
src/media/remote-media-ssrf-policy.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { SsrFPolicy } from "../infra/net/ssrf.js";
|
||||
|
||||
type RemoteMediaSsrfPolicy = Pick<
|
||||
SsrFPolicy,
|
||||
"allowRfc2544BenchmarkRange" | "allowIpv6UniqueLocalRange"
|
||||
>;
|
||||
|
||||
/**
|
||||
* Projects web_fetch SSRF compatibility flags onto remote media/provider reads.
|
||||
* Hostname restrictions are scoped to web_fetch URLs and must not constrain provider APIs or CDNs.
|
||||
*/
|
||||
export function projectRemoteMediaSsrfPolicy(
|
||||
cfg: OpenClawConfig | undefined,
|
||||
): RemoteMediaSsrfPolicy | undefined {
|
||||
const policy = cfg?.tools?.web?.fetch?.ssrfPolicy;
|
||||
if (!policy) {
|
||||
return undefined;
|
||||
}
|
||||
const projected: RemoteMediaSsrfPolicy = {
|
||||
...(policy.allowRfc2544BenchmarkRange !== undefined
|
||||
? { allowRfc2544BenchmarkRange: policy.allowRfc2544BenchmarkRange }
|
||||
: {}),
|
||||
...(policy.allowIpv6UniqueLocalRange !== undefined
|
||||
? { allowIpv6UniqueLocalRange: policy.allowIpv6UniqueLocalRange }
|
||||
: {}),
|
||||
};
|
||||
return Object.keys(projected).length > 0 ? projected : undefined;
|
||||
}
|
||||
@@ -926,7 +926,6 @@ describe("test-projects args", () => {
|
||||
config: "test/vitest/vitest.agents.config.ts",
|
||||
forwardedArgs: [],
|
||||
includePatterns: [
|
||||
"src/agents/agent-bundle-mcp-runtime.test.ts",
|
||||
"src/agents/models-config.file-mode.test.ts",
|
||||
"src/agents/sandbox/ssh.test.ts",
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user