openclaw/src/cron/service/ops.test.ts

import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import { runOpenClawStateWriteTransaction } from "../../state/openclaw-state-db.js";
import * as detachedTaskRuntime from "../../tasks/detached-task-runtime.js";
import { findTaskByRunId, resetTaskRegistryForTests } from "../../tasks/task-registry.js";
import { formatTaskStatusDetail } from "../../tasks/task-status.js";
import { setupCronServiceSuite, writeCronStoreSnapshot } from "../service.test-harness.js";
import { loadCronStore, loadCronStoreWithConfigJobs } from "../store.js";
import type { CronJob } from "../types.js";
import { add, run, start, stop, update } from "./ops.js";
import { createCronServiceState } from "./state.js";
import { runMissedJobs } from "./timer.js";

const { logger, makeStorePath } = setupCronServiceSuite({
  prefix: "cron-service-ops-seam",
});

function withStateDirForStorePath(storePath: string) {
  const stateRoot = path.dirname(path.dirname(storePath));
  const originalStateDir = process.env.OPENCLAW_STATE_DIR;
  process.env.OPENCLAW_STATE_DIR = stateRoot;
  resetTaskRegistryForTests();
  return () => {
    if (originalStateDir === undefined) {
      delete process.env.OPENCLAW_STATE_DIR;
    } else {
      process.env.OPENCLAW_STATE_DIR = originalStateDir;
    }
    resetTaskRegistryForTests();
  };
}

function createTimedOutIsolatedCronState(params: { storePath: string; now: number }) {
  return createCronServiceState({
    storePath: params.storePath,
    cronEnabled: true,
    log: logger,
    nowMs: () => params.now,
    enqueueSystemEvent: vi.fn(),
    requestHeartbeat: vi.fn(),
    runIsolatedAgentJob: vi.fn(async () => {
      throw new Error("cron: job execution timed out");
    }),
  });
}

function createOkIsolatedCronState(params: { storePath: string; now: number; summary?: string }) {
  return createCronServiceState({
    storePath: params.storePath,
    cronEnabled: true,
    log: logger,
    nowMs: () => params.now,
    enqueueSystemEvent: vi.fn(),
    requestHeartbeat: vi.fn(),
    runIsolatedAgentJob: vi.fn(async () => ({
      status: "ok" as const,
      ...(params.summary === undefined ? {} : { summary: params.summary }),
    })),
  });
}

function createInterruptedMainJob(now: number): CronJob {
  return {
    id: "startup-interrupted",
    name: "startup interrupted",
    enabled: true,
    createdAtMs: now - 86_400_000,
    updatedAtMs: now - 30 * 60_000,
    schedule: { kind: "cron", expr: "0 * * * *", tz: "UTC" },
    sessionTarget: "main",
    wakeMode: "next-heartbeat",
    payload: { kind: "systemEvent", text: "should not replay on startup" },
    state: {
      nextRunAtMs: now - 60_000,
      runningAtMs: now - 30 * 60_000,
      lastFailureNotificationDelivered: true,
      lastFailureNotificationDeliveryStatus: "delivered",
    },
  };
}

function createDueIsolatedJob(now: number): CronJob {
  return {
    id: "isolated-timeout",
    name: "isolated timeout",
    enabled: true,
    createdAtMs: now - 60_000,
    updatedAtMs: now - 60_000,
    schedule: { kind: "every", everyMs: 60_000, anchorMs: now - 60_000 },
    sessionTarget: "isolated",
    wakeMode: "next-heartbeat",
    payload: { kind: "agentTurn", message: "do work" },
    sessionKey: "agent:main:main",
    state: { nextRunAtMs: now - 1 },
  };
}

async function writeDueIsolatedJobSnapshot(storePath: string, now: number) {
  await writeCronStoreSnapshot({
    storePath,
    jobs: [createDueIsolatedJob(now)],
  });
}

async function writeLegacyCronArraySnapshot(storePath: string, jobs: CronJob[]) {
  await fs.mkdir(path.dirname(storePath), { recursive: true });
  await fs.writeFile(storePath, JSON.stringify(jobs, null, 2), "utf-8");
}

function insertCronJobRow(storePath: string, job: CronJob) {
  runOpenClawStateWriteTransaction(({ db }) => {
    db.prepare(
      `INSERT INTO cron_jobs (
        store_key, job_id, name, enabled, created_at_ms, schedule_kind,
        at, every_ms, anchor_ms, schedule_expr, session_target, wake_mode, payload_kind,
        payload_message, delivery_mode, delivery_to, job_json, state_json, updated_at
      ) VALUES (
        $storeKey, $jobId, $name, $enabled, $createdAtMs, $scheduleKind,
        $at, $everyMs, $anchorMs, $scheduleExpr, $sessionTarget, $wakeMode, $payloadKind,
        $payloadMessage, $deliveryMode, $deliveryTo, $jobJson, $stateJson, $updatedAt
      )`,
    ).run({
      $storeKey: path.resolve(storePath),
      $jobId: job.id,
      $name: job.name,
      $enabled: job.enabled ? 1 : 0,
      $createdAtMs: job.createdAtMs,
      $scheduleKind: job.schedule.kind,
      $at: job.schedule.kind === "at" ? job.schedule.at : null,
      $everyMs: job.schedule.kind === "every" ? job.schedule.everyMs : null,
      $anchorMs: job.schedule.kind === "every" ? (job.schedule.anchorMs ?? null) : null,
      $scheduleExpr: job.schedule.kind === "cron" ? job.schedule.expr : null,
      $sessionTarget: job.sessionTarget,
      $wakeMode: job.wakeMode,
      $payloadKind: job.payload.kind,
      $payloadMessage: "message" in job.payload ? job.payload.message : null,
      $deliveryMode: job.delivery ? (job.delivery.mode ?? "announce") : null,
      $deliveryTo: job.delivery?.to ?? null,
      $jobJson: JSON.stringify(job),
      $stateJson: JSON.stringify(job.state),
      $updatedAt: job.updatedAtMs,
    });
  });
}

async function expectDueIsolatedManualRunProgresses(storePath: string, now: number) {
  const state = createOkIsolatedCronState({ storePath, now, summary: "done" });

  await expect(run(state, "isolated-timeout")).resolves.toEqual({ ok: true, ran: true });

  const persisted = (await loadCronStore(storePath)) as {
    jobs: CronJob[];
  };
  expect(persisted.jobs[0]?.state.runningAtMs).toBeUndefined();
  expect(persisted.jobs[0]?.state.lastStatus).toBe("ok");
}

function expectWarnedJob(params: { field: "jobId" | "jobStatus"; value: string; message: string }) {
  const warnCalls = logger.warn.mock.calls as unknown as Array<[Record<string, unknown>, string]>;
  const warning = warnCalls.find(
    ([metadata, message]) => metadata[params.field] === params.value && message === params.message,
  );
  expect(warning?.[0][params.field]).toBe(params.value);
  expect(warning?.[1]).toBe(params.message);
}

function expectTaskRun(params: {
  runId: string;
  runtime: string;
  status: string;
  sourceId: string;
  progressSummary?: string;
}) {
  const task = findTaskByRunId(params.runId);
  expect(task?.runtime).toBe(params.runtime);
  expect(task?.status).toBe(params.status);
  expect(task?.sourceId).toBe(params.sourceId);
  if (params.progressSummary !== undefined) {
    expect(task?.progressSummary).toBe(params.progressSummary);
  }
}

function createMissedIsolatedJob(now: number): CronJob {
  return {
    id: "startup-timeout",
    name: "startup timeout",
    enabled: true,
    createdAtMs: now - 86_400_000,
    updatedAtMs: now - 30 * 60_000,
    schedule: { kind: "cron", expr: "0 * * * *", tz: "UTC" },
    sessionTarget: "isolated",
    wakeMode: "next-heartbeat",
    payload: { kind: "agentTurn", message: "should timeout" },
    sessionKey: "agent:main:main",
    state: {
      nextRunAtMs: now - 60_000,
    },
  };
}

describe("cron service ops seam coverage", () => {
  it("keeps core add paths on SQLite and leaves legacy JSON for doctor migration", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-05-20T08:00:00.000Z");
    const legacyJobs: CronJob[] = [
      {
        id: "legacy-alpha",
        name: "legacy alpha",
        enabled: true,
        createdAtMs: now - 120_000,
        updatedAtMs: now - 120_000,
        schedule: { kind: "every", everyMs: 3_600_000 },
        sessionTarget: "main",
        wakeMode: "next-heartbeat",
        payload: { kind: "systemEvent", text: "alpha" },
        state: { nextRunAtMs: now + 3_600_000 },
      },
      {
        id: "legacy-beta",
        name: "legacy beta",
        enabled: true,
        createdAtMs: now - 60_000,
        updatedAtMs: now - 60_000,
        schedule: { kind: "every", everyMs: 7_200_000 },
        sessionTarget: "main",
        wakeMode: "next-heartbeat",
        payload: { kind: "systemEvent", text: "beta" },
        state: { nextRunAtMs: now + 7_200_000 },
      },
    ];
    await writeLegacyCronArraySnapshot(storePath, legacyJobs);
    const state = createOkIsolatedCronState({ storePath, now });

    const newJob = await add(state, {
      name: "new after upgrade",
      enabled: true,
      schedule: { kind: "every", everyMs: 10_800_000 },
      sessionTarget: "main",
      wakeMode: "next-heartbeat",
      payload: { kind: "systemEvent", text: "new" },
    });
    if (state.timer) {
      clearTimeout(state.timer);
    }

    const loaded = await loadCronStore(storePath);

    expect(loaded.jobs.map((job) => job.id)).toEqual([newJob.id]);
    expect(await fs.stat(storePath)).toBeTruthy();
    await expect(fs.stat(`${storePath}.migrated`)).rejects.toMatchObject({ code: "ENOENT" });
  });

  it("leaves legacy notify fallback for doctor instead of migrating during startup", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-05-20T09:00:00.000Z");
    const legacyJob = {
      id: "legacy-notify",
      name: "legacy notify",
      enabled: true,
      createdAtMs: now - 60_000,
      updatedAtMs: now - 60_000,
      schedule: { kind: "every", everyMs: 3_600_000, anchorMs: now },
      sessionTarget: "isolated",
      wakeMode: "next-heartbeat",
      payload: { kind: "agentTurn", message: "do work" },
      delivery: { to: "telegram:chat-1" },
      notify: true,
      state: { nextRunAtMs: now + 3_600_000 },
    } as CronJob & { notify: true };
    insertCronJobRow(storePath, legacyJob);
    const state = createCronServiceState({
      storePath,
      cronEnabled: true,
      cronConfig: { webhook: "https://example.invalid/cron" },
      log: logger,
      nowMs: () => now,
      enqueueSystemEvent: vi.fn(),
      requestHeartbeat: vi.fn(),
      runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
    });

    await start(state);
    if (state.timer) {
      clearTimeout(state.timer);
    }

    const loaded = await loadCronStoreWithConfigJobs(storePath);
    const persisted = loaded.store.jobs[0] as CronJob & { notify?: unknown };
    expect(persisted.notify).toBeUndefined();
    expect(persisted.delivery).toEqual({
      mode: "announce",
      to: "telegram:chat-1",
    });
    expect(loaded.configJobs[0]?.notify).toBe(true);
    expect(logger.info).not.toHaveBeenCalledWith(
      { storePath },
      "cron: migrated legacy notify fallback jobs before scheduler startup",
    );
    expect(logger.warn).not.toHaveBeenCalledWith(
      expect.objectContaining({ storePath }),
      "cron: legacy notify fallback jobs need cron.webhook before migration",
    );
  });

  it("start marks interrupted running jobs failed, persists, and arms the timer", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-03-23T12:00:00.000Z");
    const enqueueSystemEvent = vi.fn();
    const requestHeartbeat = vi.fn();
    const timeoutSpy = vi.spyOn(globalThis, "setTimeout");

    await writeCronStoreSnapshot({
      storePath,
      jobs: [createInterruptedMainJob(now)],
    });

    const state = createCronServiceState({
      storePath,
      cronEnabled: true,
      log: logger,
      nowMs: () => now,
      enqueueSystemEvent,
      requestHeartbeat,
      runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
    });

    await start(state);

    expectWarnedJob({
      field: "jobId",
      value: "startup-interrupted",
      message: "cron: marking interrupted running job failed on startup",
    });
    expect(enqueueSystemEvent).not.toHaveBeenCalled();
    expect(requestHeartbeat).not.toHaveBeenCalled();
    if (state.timer === undefined) {
      throw new Error("Expected cron service timer");
    }

    const persisted = (await loadCronStore(storePath)) as {
      jobs: CronJob[];
    };
    const job = persisted.jobs[0];
    if (!job) {
      throw new Error("expected persisted cron job");
    }
    expect(job.state.runningAtMs).toBeUndefined();
    expect(job.state.lastStatus).toBe("error");
    expect(job.state.lastRunStatus).toBe("error");
    expect(job.state.lastRunAtMs).toBe(now - 30 * 60_000);
    expect(job.state.lastError).toBe("cron: job interrupted by gateway restart");
    expect(job.state.lastFailureNotificationDelivered).toBeUndefined();
    expect(job.state.lastFailureNotificationDeliveryStatus).toBe("not-requested");
    expect(job.state.lastFailureNotificationDeliveryError).toBeUndefined();
    expect((job.state.nextRunAtMs ?? 0) > now).toBe(true);

    const delays = timeoutSpy.mock.calls
      .map(([, delay]) => delay)
      .filter((delay): delay is number => typeof delay === "number");
    const positiveDelays = delays.filter((delay) => delay > 0);
    expect(positiveDelays.length).toBeGreaterThan(0);

    timeoutSpy.mockRestore();
    stop(state);
  });

  it("start persists load-time updatedAtMs repairs to the state sidecar only", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-04-09T08:00:00.000Z");
    const createdAtMs = now - 86_400_000;
    const nextRunAtMs = Date.parse("2026-04-10T09:00:00.000Z");
    const jobId = "future-sidecar-repair";
    await writeCronStoreSnapshot({
      storePath,
      jobs: [
        {
          id: jobId,
          name: "future sidecar repair",
          enabled: true,
          createdAtMs,
          updatedAtMs: createdAtMs,
          schedule: { kind: "cron", expr: "0 9 * * *", tz: "UTC" },
          sessionTarget: "main",
          wakeMode: "next-heartbeat",
          payload: { kind: "systemEvent", text: "daily" },
          state: { nextRunAtMs },
        },
      ],
    });
    const state = createCronServiceState({
      storePath,
      cronEnabled: true,
      log: logger,
      nowMs: () => now,
      enqueueSystemEvent: vi.fn(),
      requestHeartbeat: vi.fn(),
      runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
    });

    try {
      await start(state);

      const persisted = await loadCronStore(storePath);
      const job = persisted.jobs.find((entry) => entry.id === jobId);

      await expect(fs.stat(`${storePath}.migrated`)).rejects.toMatchObject({ code: "ENOENT" });
      expect(job?.updatedAtMs).toBe(createdAtMs);
      expect(job?.state?.nextRunAtMs).toBe(nextRunAtMs);
    } finally {
      stop(state);
    }
  });

  it("keeps manual acknowledgement IDs separate from recoverable task run IDs", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-03-23T12:00:00.000Z");
    const restoreStateDir = withStateDirForStorePath(storePath);

    try {
      await writeDueIsolatedJobSnapshot(storePath, now);

      const state = createOkIsolatedCronState({ storePath, now, summary: "done" });
      const manualRunId = `manual:isolated-timeout:${now}:1`;

      await expect(
        run(state, "isolated-timeout", "force", { runId: manualRunId }),
      ).resolves.toEqual({
        ok: true,
        ran: true,
      });

      expectTaskRun({
        runId: `cron:isolated-timeout:${now}`,
        runtime: "cron",
        status: "succeeded",
        sourceId: "isolated-timeout",
        progressSummary: "Running cron job.",
      });
      expect(findTaskByRunId(manualRunId)).toBeUndefined();
    } finally {
      restoreStateDir();
    }
  });

  it("records timed out manual runs as timed_out in the shared task registry", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-03-23T12:00:00.000Z");
    const restoreStateDir = withStateDirForStorePath(storePath);

    await writeDueIsolatedJobSnapshot(storePath, now);

    const state = createTimedOutIsolatedCronState({
      storePath,
      now,
    });

    await run(state, "isolated-timeout");

    expectTaskRun({
      runId: `cron:isolated-timeout:${now}`,
      runtime: "cron",
      status: "timed_out",
      sourceId: "isolated-timeout",
    });

    restoreStateDir();
  });

  it("keeps manual cron runs progressing when task ledger creation fails", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-03-23T12:00:00.000Z");

    await writeCronStoreSnapshot({
      storePath,
      jobs: [createDueIsolatedJob(now)],
    });

    const createTaskRecordSpy = vi
      .spyOn(detachedTaskRuntime, "createRunningTaskRun")
      .mockImplementation(() => {
        throw new Error("disk full");
      });

    await expectDueIsolatedManualRunProgresses(storePath, now);
    expectWarnedJob({
      field: "jobId",
      value: "isolated-timeout",
      message: "cron: failed to create task ledger record",
    });

    createTaskRecordSpy.mockRestore();
  });

  it("keeps manual cron cleanup progressing when task ledger updates fail", async () => {
    const { storePath } = await makeStorePath();
    const stateRoot = path.dirname(path.dirname(storePath));
    const now = Date.parse("2026-03-23T12:00:00.000Z");
    const originalStateDir = process.env.OPENCLAW_STATE_DIR;
    process.env.OPENCLAW_STATE_DIR = stateRoot;
    resetTaskRegistryForTests();

    await writeDueIsolatedJobSnapshot(storePath, now);

    const updateTaskRecordSpy = vi
      .spyOn(detachedTaskRuntime, "completeTaskRunByRunId")
      .mockImplementation(() => {
        throw new Error("disk full");
      });

    await expectDueIsolatedManualRunProgresses(storePath, now);
    expectWarnedJob({
      field: "jobStatus",
      value: "ok",
      message: "cron: failed to update task ledger record",
    });

    updateTaskRecordSpy.mockRestore();
    if (originalStateDir === undefined) {
      delete process.env.OPENCLAW_STATE_DIR;
    } else {
      process.env.OPENCLAW_STATE_DIR = originalStateDir;
    }
    resetTaskRegistryForTests();
  });

  it("non-schedule edit preserves nextRunAtMs (#63499)", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-04-09T08:00:00.000Z");
    const originalNextRunAtMs = Date.parse("2026-04-10T09:00:00.000Z");

    await writeCronStoreSnapshot({
      storePath,
      jobs: [
        {
          id: "daily-report",
          name: "daily report",
          enabled: true,
          createdAtMs: now - 86_400_000,
          updatedAtMs: now - 3_600_000,
          schedule: { kind: "cron", expr: "0 9 * * *", tz: "Asia/Shanghai" },
          sessionTarget: "main",
          wakeMode: "next-heartbeat",
          payload: { kind: "systemEvent", text: "daily" },
          state: { nextRunAtMs: originalNextRunAtMs },
        },
      ],
    });

    const state = createOkIsolatedCronState({ storePath, now });

    const updated = await update(state, "daily-report", { description: "edited" });

    expect(updated.description).toBe("edited");
    expect(updated.state.nextRunAtMs).toBe(originalNextRunAtMs);
  });

  it("repairs nextRunAtMs=0 on non-schedule edit (#63499)", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-04-09T08:00:00.000Z");

    await writeCronStoreSnapshot({
      storePath,
      jobs: [
        {
          id: "broken-job",
          name: "broken",
          enabled: true,
          createdAtMs: now - 86_400_000,
          updatedAtMs: now - 3_600_000,
          schedule: { kind: "cron", expr: "0 9 * * *", tz: "UTC" },
          sessionTarget: "main",
          wakeMode: "next-heartbeat",
          payload: { kind: "systemEvent", text: "test" },
          state: { nextRunAtMs: 0 },
        },
      ],
    });

    const state = createOkIsolatedCronState({ storePath, now });

    const updated = await update(state, "broken-job", { description: "fixed" });

    expect(updated.description).toBe("fixed");
    expect(updated.state.nextRunAtMs).toBeGreaterThan(0);
    expect(updated.state.nextRunAtMs).toBeGreaterThan(now);
  });

  it("records startup catch-up timeouts as timed_out in the shared task registry", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-03-23T12:00:00.000Z");
    const restoreStateDir = withStateDirForStorePath(storePath);

    try {
      await writeCronStoreSnapshot({
        storePath,
        jobs: [createMissedIsolatedJob(now)],
      });

      const state = createTimedOutIsolatedCronState({
        storePath,
        now,
      });

      await runMissedJobs(state);

      expectTaskRun({
        runId: `cron:startup-timeout:${now}`,
        runtime: "cron",
        status: "timed_out",
        sourceId: "startup-timeout",
        progressSummary: "Running cron job.",
      });
    } finally {
      restoreStateDir();
    }
  });

  it("seeds active manual cron task progress for status surfaces", async () => {
    const { storePath } = await makeStorePath();
    const now = Date.parse("2026-03-23T12:00:00.000Z");
    const restoreStateDir = withStateDirForStorePath(storePath);

    try {
      await writeDueIsolatedJobSnapshot(storePath, now);
      let resolveRun: ((value: { status: "ok"; summary: string }) => void) | undefined;
      const state = createCronServiceState({
        storePath,
        cronEnabled: true,
        log: logger,
        nowMs: () => now,
        enqueueSystemEvent: vi.fn(),
        requestHeartbeat: vi.fn(),
        runIsolatedAgentJob: vi.fn(
          () =>
            new Promise<{ status: "ok"; summary: string }>((resolve) => {
              resolveRun = resolve;
            }),
        ),
      });

      const manualRun = run(state, "isolated-timeout");
      await vi.waitFor(() => {
        expect(state.deps.runIsolatedAgentJob).toHaveBeenCalledTimes(1);
      });

      const task = findTaskByRunId(`cron:isolated-timeout:${now}`);
      if (!task) {
        throw new Error("expected active manual cron task ledger record");
      }
      expect(task.status).toBe("running");
      expect(task.progressSummary).toBe("Running cron job.");
      expect(formatTaskStatusDetail(task)).toBe("Running cron job.");

      resolveRun?.({ status: "ok", summary: "done" });
      await manualRun;
    } finally {
      restoreStateDir();
    }
  });
});