From cf570679f14d323aa14c626d176edf3ca11c8524 Mon Sep 17 00:00:00 2001 From: Marcus Castro Date: Fri, 5 Jun 2026 02:09:55 -0300 Subject: [PATCH] feat(whatsapp): expand qa driver message support --- .../whatsapp/src/auto-reply.test-harness.ts | 6 + extensions/whatsapp/src/inbound.test.ts | 10 + .../whatsapp/src/inbound/extract.test.ts | 11 + extensions/whatsapp/src/inbound/extract.ts | 16 + .../whatsapp/src/inbound/send-api.test.ts | 54 +++ extensions/whatsapp/src/inbound/send-api.ts | 53 +++ .../whatsapp/src/qa-driver.runtime.test.ts | 310 +++++++++++++++++- extensions/whatsapp/src/qa-driver.runtime.ts | 300 ++++++++++++++++- 8 files changed, 750 insertions(+), 10 deletions(-) diff --git a/extensions/whatsapp/src/auto-reply.test-harness.ts b/extensions/whatsapp/src/auto-reply.test-harness.ts index f077fdd507dc..21f72df84077 100644 --- a/extensions/whatsapp/src/auto-reply.test-harness.ts +++ b/extensions/whatsapp/src/auto-reply.test-harness.ts @@ -28,9 +28,12 @@ type MockWebListener = { close: () => Promise; onClose: Promise; signalClose: () => void; + sendContact: () => Promise; + sendLocation: () => Promise; sendMessage: () => Promise; sendPoll: () => Promise; sendReaction: () => Promise; + sendSticker: () => Promise; sendComposingTo: () => Promise; }; type UnknownMock = Mock<(...args: unknown[]) => unknown>; @@ -262,9 +265,12 @@ export function createMockWebListener(): MockWebListener { close: vi.fn(async () => undefined), onClose: new Promise(() => {}), signalClose: vi.fn(), + sendContact: vi.fn(async () => createAcceptedWhatsAppSendResult("text", "contact-1")), + sendLocation: vi.fn(async () => createAcceptedWhatsAppSendResult("text", "location-1")), sendMessage: vi.fn(async () => createAcceptedWhatsAppSendResult("text", "msg-1")), sendPoll: vi.fn(async () => createAcceptedWhatsAppSendResult("poll", "poll-1")), sendReaction: vi.fn(async () => createAcceptedWhatsAppSendResult("reaction", "reaction-1")), + sendSticker: vi.fn(async () => createAcceptedWhatsAppSendResult("media", "sticker-1")), sendComposingTo: vi.fn(async () => undefined), }; } diff --git a/extensions/whatsapp/src/inbound.test.ts b/extensions/whatsapp/src/inbound.test.ts index 9018b6c248a9..206d13dec3e5 100644 --- a/extensions/whatsapp/src/inbound.test.ts +++ b/extensions/whatsapp/src/inbound.test.ts @@ -29,6 +29,16 @@ describe("web inbound helpers", () => { expect(body).toBe("doc"); }); + it("extracts WhatsApp poll questions", () => { + const body = extractText({ + pollCreationMessage: { + name: " Reply after seeing this poll ", + options: [], + }, + } as unknown as import("baileys").proto.IMessage); + expect(body).toBe("Reply after seeing this poll"); + }); + it("extracts WhatsApp contact cards", () => { const body = extractText({ contactMessage: { diff --git a/extensions/whatsapp/src/inbound/extract.test.ts b/extensions/whatsapp/src/inbound/extract.test.ts index ceb3d534ab14..183210f3bd44 100644 --- a/extensions/whatsapp/src/inbound/extract.test.ts +++ b/extensions/whatsapp/src/inbound/extract.test.ts @@ -180,6 +180,17 @@ describe("hasInboundUserContent", () => { ).toBe(true); }); + it("returns true for poll creation messages", () => { + expect( + hasInboundUserContent({ + pollCreationMessage: { + name: "Pick one", + options: [], + }, + } as proto.IMessage), + ).toBe(true); + }); + it("returns true for buttons response (user button click)", () => { expect( hasInboundUserContent({ diff --git a/extensions/whatsapp/src/inbound/extract.ts b/extensions/whatsapp/src/inbound/extract.ts index 6c75df7895e3..abaa816893fa 100644 --- a/extensions/whatsapp/src/inbound/extract.ts +++ b/extensions/whatsapp/src/inbound/extract.ts @@ -31,6 +31,9 @@ const MESSAGE_CONTENT_KEYS = [ "liveLocationMessage", "contactMessage", "contactsArrayMessage", + "pollCreationMessage", + "pollCreationMessageV2", + "pollCreationMessageV3", "buttonsResponseMessage", "listResponseMessage", "templateButtonReplyMessage", @@ -265,6 +268,10 @@ export function extractText(rawMessage: proto.IMessage | undefined): string | un if (contactPlaceholder) { return contactPlaceholder; } + const pollQuestion = extractPollQuestion(message); + if (pollQuestion) { + return pollQuestion; + } return undefined; } @@ -305,6 +312,15 @@ function extractContactPlaceholder(rawMessage: proto.IMessage | undefined): stri return ``; } +function extractPollQuestion(rawMessage: proto.IMessage | undefined): string | undefined { + const message = unwrapMessage(rawMessage); + const question = + message?.pollCreationMessage?.name ?? + message?.pollCreationMessageV2?.name ?? + message?.pollCreationMessageV3?.name; + return question?.trim() || undefined; +} + export function extractContactContext( rawMessage: proto.IMessage | undefined, ): WhatsAppStructuredContactContext | undefined { diff --git a/extensions/whatsapp/src/inbound/send-api.test.ts b/extensions/whatsapp/src/inbound/send-api.test.ts index 4d172c6e67e4..593d17c3a1ea 100644 --- a/extensions/whatsapp/src/inbound/send-api.test.ts +++ b/extensions/whatsapp/src/inbound/send-api.test.ts @@ -426,6 +426,60 @@ describe("createWebSendApi", () => { }); }); + it("sends contacts for QA structured inbound coverage", async () => { + const res = await api.sendContact("+1555", { + displayName: "QA Contact", + vcard: "BEGIN:VCARD\nVERSION:3.0\nFN:QA Contact\nEND:VCARD", + }); + + expectFirstSendJid("1555@s.whatsapp.net"); + expect(requireSendContent().contacts).toEqual({ + displayName: "QA Contact", + contacts: [ + { + displayName: "QA Contact", + vcard: "BEGIN:VCARD\nVERSION:3.0\nFN:QA Contact\nEND:VCARD", + }, + ], + }); + expect(res.messageId).toBe("msg-1"); + expect(recordChannelActivity).toHaveBeenCalledWith({ + channel: "whatsapp", + accountId: "main", + direction: "outbound", + }); + }); + + it("sends locations for QA structured inbound coverage", async () => { + const res = await api.sendLocation("+1555", { + address: "1 Market St", + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + name: "QA Location", + }); + + expectFirstSendJid("1555@s.whatsapp.net"); + expect(requireSendContent().location).toEqual({ + address: "1 Market St", + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + name: "QA Location", + }); + expect(res.messageId).toBe("msg-1"); + }); + + it("sends stickers for QA structured inbound coverage", async () => { + const sticker = Buffer.from("webp"); + const res = await api.sendSticker("+1555", sticker, { mimetype: "image/webp" }); + + expectFirstSendJid("1555@s.whatsapp.net"); + expectSendContentFields(0, { + sticker, + mimetype: "image/webp", + }); + expect(res.messageId).toBe("msg-1"); + }); + it("sends reactions with participant JID normalization", async () => { const res = await api.sendReaction("+1555", "msg-2", "👍", false, "+1999"); expectFirstSendJid("1555@s.whatsapp.net"); diff --git a/extensions/whatsapp/src/inbound/send-api.ts b/extensions/whatsapp/src/inbound/send-api.ts index 64cd3b5b01f9..f0183fef11f8 100644 --- a/extensions/whatsapp/src/inbound/send-api.ts +++ b/extensions/whatsapp/src/inbound/send-api.ts @@ -171,6 +171,46 @@ export function createWebSendApi(params: { recordWhatsAppOutbound(params.defaultAccountId); return normalizeWhatsAppSendResult(result, "poll"); }, + sendContact: async ( + to: string, + contact: { displayName: string; vcard: string }, + ): Promise => { + const jid = resolveOutboundJid(to); + const result = await params.sock.sendMessage(jid, { + contacts: { + displayName: contact.displayName, + contacts: [ + { + displayName: contact.displayName, + vcard: contact.vcard, + }, + ], + }, + } as AnyMessageContent); + recordWhatsAppOutbound(params.defaultAccountId); + return normalizeWhatsAppSendResult(result, "text"); + }, + sendLocation: async ( + to: string, + location: { + address?: string; + degreesLatitude: number; + degreesLongitude: number; + name?: string; + }, + ): Promise => { + const jid = resolveOutboundJid(to); + const result = await params.sock.sendMessage(jid, { + location: { + degreesLatitude: location.degreesLatitude, + degreesLongitude: location.degreesLongitude, + name: location.name, + address: location.address, + }, + } as AnyMessageContent); + recordWhatsAppOutbound(params.defaultAccountId); + return normalizeWhatsAppSendResult(result, "text"); + }, sendReaction: async ( chatJid: string, messageId: string, @@ -201,5 +241,18 @@ export function createWebSendApi(params: { } await params.sock.sendPresenceUpdate("composing", jid); }, + sendSticker: async ( + to: string, + stickerBuffer: Buffer, + options?: { mimetype?: string }, + ): Promise => { + const jid = resolveOutboundJid(to); + const result = await params.sock.sendMessage(jid, { + sticker: stickerBuffer, + mimetype: options?.mimetype ?? "image/webp", + } as AnyMessageContent); + recordWhatsAppOutbound(params.defaultAccountId); + return normalizeWhatsAppSendResult(result, "media"); + }, } as const; } diff --git a/extensions/whatsapp/src/qa-driver.runtime.test.ts b/extensions/whatsapp/src/qa-driver.runtime.test.ts index 37434964aaa0..d343a5428570 100644 --- a/extensions/whatsapp/src/qa-driver.runtime.test.ts +++ b/extensions/whatsapp/src/qa-driver.runtime.test.ts @@ -7,7 +7,12 @@ import { startWhatsAppQaDriverSession } from "./qa-driver.runtime.js"; const mocks = vi.hoisted(() => ({ createWaSocket: vi.fn(), jidToE164: vi.fn(), + sendContact: vi.fn(), + sendLocation: vi.fn(), + sendPoll: vi.fn(), + sendReaction: vi.fn(), sendMessage: vi.fn(), + sendSticker: vi.fn(), waitForWaConnection: vi.fn(), })); @@ -22,7 +27,12 @@ vi.mock("./text-runtime.js", () => ({ vi.mock("./inbound/send-api.js", () => ({ createWebSendApi: () => ({ + sendContact: mocks.sendContact, + sendLocation: mocks.sendLocation, sendMessage: mocks.sendMessage, + sendPoll: mocks.sendPoll, + sendReaction: mocks.sendReaction, + sendSticker: mocks.sendSticker, }), })); @@ -36,11 +46,11 @@ function createMockSocket() { }; } -function incomingMessage(remoteJid: string, text: string): WAMessage { +function incomingMessage(remoteJid: string, text: string, id = "message-1"): WAMessage { return { key: { fromMe: false, - id: "message-1", + id, remoteJid, }, message: { @@ -49,6 +59,79 @@ function incomingMessage(remoteJid: string, text: string): WAMessage { } as WAMessage; } +function incomingImageMessage(remoteJid: string, text: string): WAMessage { + return { + key: { + fromMe: false, + id: "image-1", + remoteJid, + }, + message: { + imageMessage: { + caption: text, + mimetype: "image/png", + }, + }, + } as WAMessage; +} + +function incomingAudioMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "audio-1", + remoteJid, + }, + message: { + audioMessage: { + mimetype: "audio/ogg; codecs=opus", + }, + }, + } as WAMessage; +} + +function incomingReactionMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "reaction-1", + remoteJid, + }, + message: { + reactionMessage: { + text: "👍", + key: { + fromMe: true, + id: "driver-message-1", + participant: "15551234567@s.whatsapp.net", + }, + }, + }, + } as WAMessage; +} + +function incomingQuotedMessage(remoteJid: string): WAMessage { + return { + key: { + fromMe: false, + id: "quoted-reply-1", + remoteJid, + }, + message: { + extendedTextMessage: { + text: "reply body", + contextInfo: { + participant: "15551234567@s.whatsapp.net", + quotedMessage: { + conversation: "original body", + }, + stanzaId: "driver-message-1", + }, + }, + }, + } as WAMessage; +} + describe("startWhatsAppQaDriverSession", () => { afterEach(() => { vi.useRealTimers(); @@ -79,6 +162,7 @@ describe("startWhatsAppQaDriverSession", () => { { fromJid: "12345@lid", fromPhoneE164: "+15551234567", + kind: "text", messageId: "message-1", observedAt, text: "hello", @@ -88,6 +172,228 @@ describe("startWhatsAppQaDriverSession", () => { await session.close(); }); + it("does not satisfy a wait with messages observed before the lower bound", async () => { + vi.useFakeTimers(); + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + vi.setSystemTime(new Date("2026-06-04T23:42:32.036Z")); + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingMessage("12345@lid", "OpenClaw status stale", "stale-message")], + }); + + const observedAfter = new Date("2026-06-04T23:46:59.166Z"); + vi.setSystemTime(observedAfter); + const waited = session.waitForMessage({ + observedAfter, + timeoutMs: 1_000, + match: (message) => message.text.includes("OpenClaw status"), + }); + + vi.setSystemTime(new Date("2026-06-04T23:47:00.000Z")); + sock.ev.emit("messages.upsert", { + messages: [incomingMessage("12345@lid", "OpenClaw status fresh", "fresh-message")], + }); + + await expect(waited).resolves.toMatchObject({ + messageId: "fresh-message", + text: "OpenClaw status fresh", + }); + + await session.close(); + }); + + it("observes media messages without dropping their caption text", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingImageMessage("12345@lid", "image caption")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + hasMedia: true, + kind: "media", + mediaType: "image/png", + text: "image caption", + }); + + await session.close(); + }); + + it("observes audio media messages without requiring a text body", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingAudioMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + hasMedia: true, + kind: "media", + mediaType: "audio/ogg; codecs=opus", + text: "", + }); + + await session.close(); + }); + + it("observes reaction messages that have no text body", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingReactionMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + kind: "reaction", + reaction: { + emoji: "👍", + fromMe: true, + messageId: "driver-message-1", + participant: "15551234567@s.whatsapp.net", + }, + text: "", + }); + + await session.close(); + }); + + it("observes quoted reply context", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.jidToE164.mockReturnValue("+15551234567"); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + sock.ev.emit("messages.upsert", { + messages: [incomingQuotedMessage("12345@lid")], + }); + + expect(session.getObservedMessages()[0]).toMatchObject({ + kind: "text", + quoted: { + messageId: "driver-message-1", + participant: "15551234567@s.whatsapp.net", + text: "original body", + }, + text: "reply body", + }); + + await session.close(); + }); + + it("exposes structured send helpers over the web send API", async () => { + const sock = createMockSocket(); + mocks.createWaSocket.mockResolvedValue(sock); + mocks.waitForWaConnection.mockResolvedValue(undefined); + mocks.sendMessage.mockResolvedValue({ messageId: "send-1" }); + mocks.sendPoll.mockResolvedValue({ messageId: "poll-1" }); + mocks.sendReaction.mockResolvedValue({ messageId: "reaction-send-1" }); + mocks.sendContact.mockResolvedValue({ messageId: "contact-1" }); + mocks.sendLocation.mockResolvedValue({ messageId: "location-1" }); + mocks.sendSticker.mockResolvedValue({ messageId: "sticker-1" }); + + const session = await startWhatsAppQaDriverSession({ + authDir: "/tmp/openclaw-whatsapp-auth", + }); + + await expect( + session.sendMedia("15551234567", "caption", Buffer.from("png"), "image/png", { + fileName: "qa.png", + }), + ).resolves.toEqual({ messageId: "send-1" }); + await expect( + session.sendPoll("15551234567", { + question: "Pick one", + options: ["A", "B"], + }), + ).resolves.toEqual({ messageId: "poll-1" }); + await expect( + session.sendReaction("15551234567@s.whatsapp.net", "driver-message-1", "👍", { + fromMe: true, + }), + ).resolves.toEqual({ messageId: "reaction-send-1" }); + await expect( + session.sendContact("15551234567", { + displayName: "QA Contact", + vcard: "BEGIN:VCARD\nFN:QA Contact\nEND:VCARD", + }), + ).resolves.toEqual({ messageId: "contact-1" }); + await expect( + session.sendLocation("15551234567", { + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + name: "QA Location", + }), + ).resolves.toEqual({ messageId: "location-1" }); + await expect( + session.sendSticker("15551234567", Buffer.from("webp"), { mimetype: "image/webp" }), + ).resolves.toEqual({ messageId: "sticker-1" }); + + expect(mocks.sendMessage).toHaveBeenCalledWith( + "15551234567", + "caption", + Buffer.from("png"), + "image/png", + { fileName: "qa.png" }, + ); + expect(mocks.sendPoll).toHaveBeenCalledWith("15551234567", { + question: "Pick one", + options: ["A", "B"], + }); + expect(mocks.sendReaction).toHaveBeenCalledWith( + "15551234567@s.whatsapp.net", + "driver-message-1", + "👍", + true, + undefined, + ); + expect(mocks.sendContact).toHaveBeenCalledWith("15551234567", { + displayName: "QA Contact", + vcard: "BEGIN:VCARD\nFN:QA Contact\nEND:VCARD", + }); + expect(mocks.sendLocation).toHaveBeenCalledWith("15551234567", { + degreesLatitude: 37.7749, + degreesLongitude: -122.4194, + name: "QA Location", + }); + expect(mocks.sendSticker).toHaveBeenCalledWith("15551234567", Buffer.from("webp"), { + mimetype: "image/webp", + }); + + await session.close(); + }); + it("passes the connection timeout to the shared connection waiter", async () => { const sock = createMockSocket(); mocks.createWaSocket.mockResolvedValue(sock); diff --git a/extensions/whatsapp/src/qa-driver.runtime.ts b/extensions/whatsapp/src/qa-driver.runtime.ts index 0d4d1464e644..a22aa86b507d 100644 --- a/extensions/whatsapp/src/qa-driver.runtime.ts +++ b/extensions/whatsapp/src/qa-driver.runtime.ts @@ -1,24 +1,109 @@ // Whatsapp plugin module implements qa driver behavior. import type { WAMessage } from "baileys"; -import { extractText } from "./inbound/extract.js"; +import { extractContextInfo, extractText } from "./inbound/extract.js"; import { createWebSendApi } from "./inbound/send-api.js"; +import type { ActiveWebSendOptions } from "./inbound/types.js"; import { createWaSocket, waitForWaConnection } from "./session.js"; import { jidToE164 } from "./text-runtime.js"; +export type WhatsAppQaDriverObservedMessageKind = + | "media" + | "poll" + | "reaction" + | "text" + | "unknown"; + +export type WhatsAppQaDriverQuotedMessage = { + messageId?: string; + participant?: string; + text?: string; +}; + +export type WhatsAppQaDriverObservedReaction = { + emoji: string; + fromMe?: boolean; + messageId?: string; + participant?: string; +}; + +export type WhatsAppQaDriverObservedPoll = { + options: string[]; + question?: string; +}; + export type WhatsAppQaDriverObservedMessage = { fromJid?: string; fromPhoneE164?: string | null; + hasMedia?: boolean; + kind: WhatsAppQaDriverObservedMessageKind; + mediaFileName?: string; + mediaType?: string; messageId?: string; observedAt: string; + poll?: WhatsAppQaDriverObservedPoll; + quoted?: WhatsAppQaDriverQuotedMessage; + reaction?: WhatsAppQaDriverObservedReaction; text: string; }; +export type WhatsAppQaDriverSendTextOptions = Pick; + +export type WhatsAppQaDriverSendMediaOptions = Pick< + ActiveWebSendOptions, + "asDocument" | "fileName" | "gifPlayback" | "quotedMessageKey" +>; + +export type WhatsAppQaDriverSendReactionOptions = { + fromMe: boolean; + participant?: string; +}; + export type WhatsAppQaDriverSession = { close: () => Promise; getObservedMessages: () => WhatsAppQaDriverObservedMessage[]; - sendText: (to: string, text: string) => Promise<{ messageId?: string }>; + sendContact: ( + to: string, + contact: { displayName: string; vcard: string }, + ) => Promise<{ messageId?: string }>; + sendLocation: ( + to: string, + location: { + address?: string; + degreesLatitude: number; + degreesLongitude: number; + name?: string; + }, + ) => Promise<{ messageId?: string }>; + sendMedia: ( + to: string, + text: string, + mediaBuffer: Buffer, + mediaType: string, + options?: WhatsAppQaDriverSendMediaOptions, + ) => Promise<{ messageId?: string }>; + sendPoll: ( + to: string, + poll: { maxSelections?: number; options: string[]; question: string }, + ) => Promise<{ messageId?: string }>; + sendReaction: ( + chatJid: string, + messageId: string, + emoji: string, + options: WhatsAppQaDriverSendReactionOptions, + ) => Promise<{ messageId?: string }>; + sendSticker: ( + to: string, + stickerBuffer: Buffer, + options?: { mimetype?: string }, + ) => Promise<{ messageId?: string }>; + sendText: ( + to: string, + text: string, + options?: WhatsAppQaDriverSendTextOptions, + ) => Promise<{ messageId?: string }>; waitForMessage: (params: { match: (message: WhatsAppQaDriverObservedMessage) => boolean; + observedAfter?: Date; timeoutMs: number; }) => Promise; }; @@ -34,6 +119,138 @@ type Waiter = { timeout: NodeJS.Timeout; }; +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object"); +} + +function readString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function readBoolean(value: unknown): boolean | undefined { + return typeof value === "boolean" ? value : undefined; +} + +function findMessageSection( + message: unknown, + sectionNames: readonly string[], +): Record | undefined { + if (!isRecord(message)) { + return undefined; + } + const queue: Array<{ depth: number; value: Record }> = [ + { depth: 0, value: message }, + ]; + const seen = new Set>(); + while (queue.length > 0) { + const current = queue.shift(); + if (!current || seen.has(current.value)) { + continue; + } + seen.add(current.value); + for (const sectionName of sectionNames) { + const section = current.value[sectionName]; + if (isRecord(section)) { + return section; + } + } + if (current.depth >= 4) { + continue; + } + for (const wrapperName of [ + "botInvokeMessage", + "documentWithCaptionMessage", + "ephemeralMessage", + "groupMentionedMessage", + "viewOnceMessage", + "viewOnceMessageV2", + "viewOnceMessageV2Extension", + ]) { + const wrapper = current.value[wrapperName]; + if (isRecord(wrapper) && isRecord(wrapper.message)) { + queue.push({ depth: current.depth + 1, value: wrapper.message }); + } + } + } + return undefined; +} + +function readReaction(message: unknown): WhatsAppQaDriverObservedReaction | undefined { + const reaction = findMessageSection(message, ["reactionMessage"]); + if (!reaction) { + return undefined; + } + const emoji = readString(reaction.text) ?? ""; + const key = isRecord(reaction.key) ? reaction.key : undefined; + return { + emoji, + fromMe: readBoolean(key?.fromMe), + messageId: readString(key?.id), + participant: readString(key?.participant), + }; +} + +function readPoll(message: unknown): WhatsAppQaDriverObservedPoll | undefined { + const poll = findMessageSection(message, [ + "pollCreationMessage", + "pollCreationMessageV2", + "pollCreationMessageV3", + ]); + if (!poll) { + return undefined; + } + const rawOptions = Array.isArray(poll.options) ? poll.options : []; + const options = rawOptions + .map((option) => (isRecord(option) ? readString(option.optionName) : undefined)) + .filter((option): option is string => Boolean(option)); + return { + options, + question: readString(poll.name), + }; +} + +function readMedia(message: unknown): + | { + fileName?: string; + mediaType?: string; + } + | undefined { + const mediaSections = [ + ["imageMessage", "image"] as const, + ["videoMessage", "video"] as const, + ["audioMessage", "audio"] as const, + ["documentMessage", "document"] as const, + ["stickerMessage", "sticker"] as const, + ]; + for (const [sectionName, fallbackType] of mediaSections) { + const section = findMessageSection(message, [sectionName]); + if (!section) { + continue; + } + return { + fileName: readString(section.fileName), + mediaType: readString(section.mimetype) ?? fallbackType, + }; + } + return undefined; +} + +function readQuotedMessage(message: WAMessage): WhatsAppQaDriverQuotedMessage | undefined { + const contextInfo = extractContextInfo(message.message ?? undefined); + if (!contextInfo) { + return undefined; + } + const quotedText = extractText(contextInfo.quotedMessage ?? undefined); + if (!contextInfo.stanzaId && !contextInfo.participant && !quotedText) { + return undefined; + } + return { + messageId: contextInfo.stanzaId ?? undefined, + participant: contextInfo.participant ?? undefined, + text: quotedText, + }; +} + function normalizeObservedMessage( message: WAMessage, authDir: string, @@ -42,16 +259,36 @@ function normalizeObservedMessage( return null; } const text = extractText(message.message ?? undefined); - if (!text) { + const reaction = readReaction(message.message); + const poll = readPoll(message.message); + const media = readMedia(message.message); + const quoted = readQuotedMessage(message); + const kind: WhatsAppQaDriverObservedMessageKind = reaction + ? "reaction" + : poll + ? "poll" + : media + ? "media" + : text + ? "text" + : "unknown"; + if (!text && kind === "unknown") { return null; } const fromJid = message.key.remoteJid ?? undefined; return { fromJid, fromPhoneE164: fromJid ? jidToE164(fromJid, { authDir }) : null, + hasMedia: media ? true : undefined, + kind, + mediaFileName: media?.fileName, + mediaType: media?.mediaType, messageId: message.key.id ?? undefined, observedAt: new Date().toISOString(), - text, + poll, + quoted, + reaction, + text: text ?? "", }; } @@ -139,6 +376,7 @@ export async function startWhatsAppQaDriverSession(params: { const sendApi = createWebSendApi({ sock, defaultAccountId: "qa-driver", + authDir: params.authDir, }); return { @@ -148,20 +386,66 @@ export async function startWhatsAppQaDriverSession(params: { getObservedMessages() { return [...observedMessages]; }, - async sendText(to, text) { - const result = await sendApi.sendMessage(to, text); + async sendContact(to, contact) { + const result = await sendApi.sendContact(to, contact); + return { + messageId: result.messageId, + }; + }, + async sendLocation(to, location) { + const result = await sendApi.sendLocation(to, location); + return { + messageId: result.messageId, + }; + }, + async sendMedia(to, text, mediaBuffer, mediaType, options) { + const result = await sendApi.sendMessage(to, text, mediaBuffer, mediaType, options); + return { + messageId: result.messageId, + }; + }, + async sendPoll(to, poll) { + const result = await sendApi.sendPoll(to, poll); + return { + messageId: result.messageId, + }; + }, + async sendReaction(chatJid, messageId, emoji, options) { + const result = await sendApi.sendReaction( + chatJid, + messageId, + emoji, + options.fromMe, + options.participant, + ); + return { + messageId: result.messageId, + }; + }, + async sendSticker(to, stickerBuffer, options) { + const result = await sendApi.sendSticker(to, stickerBuffer, options); + return { + messageId: result.messageId, + }; + }, + async sendText(to, text, options) { + const result = await sendApi.sendMessage(to, text, undefined, undefined, options); return { messageId: result.messageId, }; }, async waitForMessage(paramsLocal) { - const existing = observedMessages.find(paramsLocal.match); + const predicate = (message: WhatsAppQaDriverObservedMessage) => + (!paramsLocal.observedAfter || + new Date(message.observedAt).getTime() >= paramsLocal.observedAfter.getTime()) && + paramsLocal.match(message); + const existing = observedMessages.find(predicate); if (existing) { return existing; } return await new Promise((resolve, reject) => { const waiter: Waiter = { - predicate: paramsLocal.match, + predicate, resolve, reject, timeout: setTimeout(() => {