Browser App

// server.mjs: serves the page and creates the stream token. Your API key stays here.
import express from "express";

const app = express();
app.use(express.static("public"));

app.post("/pearl-call", async (req, res) => {
  const response = await fetch(`https://api.nlpearl.ai/v2/Pearl/${process.env.NLPEARL_PEARL_ID}/Stream/Token`, {
    method: "POST",
    headers: { Authorization: `Bearer ${process.env.NLPEARL_API_KEY}`, "Content-Type": "application/json" },
    body: JSON.stringify({
      endUserId: "visitor-123",   // your own id for the visitor
      audioFormat: "opus_24000",  // HD Opus, coded in the page with WebCodecs
      callData: {},               // the variables your opening sentence uses, such as { firstName: "Dana" }
    }),
  });
  const body = await response.text();   // { callId, url, token, expiresAt, audio }, or why it was refused
  res.status(response.status).type("json").send(body || "{}");
});

app.listen(3000, () => console.log("http://localhost:3000"));

<!doctype html>
<html>
  <body>
    <button id="talk">Talk to Pearl</button>
    <button id="hang-up">Hang up</button>
    <script type="module" src="app.js"></script>
  </body>
</html>

// app.js: asks your server for a stream token, then streams the microphone to Pearl and plays its voice.
const talk = document.querySelector("#talk");
const hangUp = document.querySelector("#hang-up");

talk.onclick = async () => {
  talk.disabled = true;
  const response = await fetch("/pearl-call", { method: "POST" });
  const call = await response.json();   // { url, token, audio, ... }, or the reason it was refused
  if (!response.ok) {
    console.error("Create Stream Token failed:", response.status, call);
    talk.disabled = false;
    return;
  }

  const microphone = await navigator.mediaDevices.getUserMedia({
    audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
  });
  const context = new AudioContext({ sampleRate: 24000 });   // the browser resamples the microphone to 24 kHz
  await context.resume();
  await context.audioWorklet.addModule("pearl-audio.js");
  const audio = new AudioWorkletNode(context, "pearl-audio", { outputChannelCount: [1] });
  context.createMediaStreamSource(microphone).connect(audio).connect(context.destination);

  const ws = new WebSocket(`${call.url}?token=${call.token}`);
  ws.binaryType = "arraybuffer";
  let ready = false;

  // Opus both ways, with the browser's own codec (WebCodecs): one packet per 20 ms frame.
  const encoder = new AudioEncoder({
    output: (chunk) => {
      const packet = new Uint8Array(chunk.byteLength);
      chunk.copyTo(packet);
      if (ws.readyState === WebSocket.OPEN) ws.send(packet);
    },
    error: (error) => console.error("AudioEncoder", error),
  });
  encoder.configure({ codec: "opus", sampleRate: 24000, numberOfChannels: 1, bitrate: 32000 });
  const decoder = new AudioDecoder({
    output: (frame) => {
      const decoded = new Float32Array(frame.numberOfFrames);
      frame.copyTo(decoded, { planeIndex: 0, format: "f32-planar" });
      const step = frame.sampleRate / 24000;   // 2 in Chrome, which decodes Opus at 48 kHz
      frame.close();
      const samples = decoded.filter((_, i) => i % step === 0);
      audio.port.postMessage(samples, [samples.buffer]);   // to the worklet's jitter buffer
    },
    error: (error) => console.error("AudioDecoder", error),
  });
  decoder.configure({ codec: "opus", sampleRate: 24000, numberOfChannels: 1 });

  // The end user: one 20 ms frame every 20 ms from the worklet, coded to Opus
  let timestamp = 0;
  audio.port.onmessage = (event) => {
    if (!ready) return;
    const frame = new AudioData({ format: "f32", sampleRate: 24000, numberOfChannels: 1, numberOfFrames: event.data.length, timestamp, data: event.data });
    timestamp += 20000;
    encoder.encode(frame);
    frame.close();
  };

  // Pearl's voice: one Opus packet per message
  ws.onmessage = (message) => {
    if (typeof message.data !== "string") {
      decoder.decode(new EncodedAudioChunk({ type: "key", timestamp: 0, data: message.data }));
      return;
    }
    const event = JSON.parse(message.data);
    if (event.type === "ready") ready = true;
    if (event.type === "transfer") console.log("transfer to", event.target);
    if (event.type === "ended") console.log("ended:", event.reason);
  };

  ws.onclose = (event) => {
    console.log("closed", event.code);   // 1006 at once: the token was refused
    microphone.getTracks().forEach((track) => track.stop());
    encoder.close();
    decoder.close();
    context.close();
    talk.disabled = false;
  };

  hangUp.onclick = () => {
    if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "hangup" }));
  };
};

// pearl-audio.js: one AudioWorklet that records the microphone and plays Pearl's voice, both at 24 kHz.
const FRAME = 480;        // 20 ms at 24 kHz
const CUSHION = 1920;     // start playing with 80 ms buffered
const MAX_BEHIND = 7200;  // past 300 ms buffered, drop the oldest audio

class PearlAudio extends AudioWorkletProcessor {
  constructor() {
    super();
    this.mic = new Float32Array(FRAME);
    this.micLength = 0;
    this.queue = [];      // Pearl's audio, as Float32Array chunks
    this.offset = 0;      // read position in queue[0]
    this.queued = 0;      // samples not played yet
    this.playing = false;
    this.port.onmessage = (event) => {
      this.queue.push(event.data);
      this.queued += event.data.length;
    };
  }

  process(inputs, outputs) {
    // The microphone: 20 ms frames, in real time, silence included when muted.
    const input = inputs[0][0];
    if (input) {
      for (let i = 0; i < input.length; i++) {
        this.mic[this.micLength++] = input[i];
        if (this.micLength === FRAME) {
          this.port.postMessage(this.mic.slice());
          this.micLength = 0;
        }
      }
    }

    // Pearl's voice: played through a small jitter buffer.
    while (this.queued > MAX_BEHIND && this.queue.length > 1) {
      this.queued -= this.queue.shift().length - this.offset;
      this.offset = 0;
    }
    if (!this.playing && this.queued >= CUSHION) this.playing = true;
    const output = outputs[0][0];
    for (let i = 0; i < output.length; i++) {
      if (!this.playing || this.queue.length === 0) {
        output[i] = 0;
        this.playing = false;   // ran dry after a stall: buffer again before playing
        continue;
      }
      const chunk = this.queue[0];
      output[i] = chunk[this.offset++];
      this.queued--;
      if (this.offset === chunk.length) {
        this.queue.shift();
        this.offset = 0;
      }
    }
    return true;
  }
}

registerProcessor("pearl-audio", PearlAudio);