Browser App

// server.mjs: serves the page and creates the connection. Your API key stays here.
import express from "express";

const app = express();
app.use(express.static("public"));

app.post("/pearl-call", async (req, res) => {
  const response = await fetch(`https://api.nlpearl.ai/v2/Pearl/${process.env.NLPEARL_PEARL_ID}/WebSocket`, {
    method: "POST",
    headers: { Authorization: `Bearer ${process.env.NLPEARL_API_KEY}`, "Content-Type": "application/json" },
    body: JSON.stringify({ endUserId: "visitor-123", audioFormat: "pcm_24000" }),   // your own id for the visitor
  });
  res.status(response.status).json(await response.json());   // { callId, url, ticket, expiresAt, audio }
});

app.listen(3000, () => console.log("http://localhost:3000"));

<!doctype html>
<html>
  <body>
    <button id="talk">Talk to Pearl</button>
    <button id="hang-up">Hang up</button>
    <script type="module" src="app.js"></script>
  </body>
</html>

// app.js: asks your server for a connection, then streams the microphone to Pearl and plays its voice.
const talk = document.querySelector("#talk");
const hangUp = document.querySelector("#hang-up");

talk.onclick = async () => {
  talk.disabled = true;
  const call = await (await fetch("/pearl-call", { method: "POST" })).json();   // { url, ticket, audio, ... }

  const microphone = await navigator.mediaDevices.getUserMedia({
    audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
  });
  const context = new AudioContext({ sampleRate: 24000 });   // the browser resamples the microphone to 24 kHz
  await context.audioWorklet.addModule("pearl-audio.js");
  const audio = new AudioWorkletNode(context, "pearl-audio", { outputChannelCount: [1] });
  context.createMediaStreamSource(microphone).connect(audio).connect(context.destination);

  const ws = new WebSocket(`${call.url}?ticket=${call.ticket}`);
  ws.binaryType = "arraybuffer";
  let ready = false;

  // The end user: one 960-byte frame every 20 ms, from the worklet
  audio.port.onmessage = (event) => {
    if (ready && ws.readyState === WebSocket.OPEN) ws.send(event.data);
  };

  ws.onmessage = (message) => {
    if (typeof message.data !== "string") {
      // Pearl's voice: 20 ms of 16-bit PCM, to the worklet's jitter buffer
      const pcm = new Int16Array(message.data);
      const samples = Float32Array.from(pcm, (sample) => sample / 0x8000);
      audio.port.postMessage(samples, [samples.buffer]);
      return;
    }
    const event = JSON.parse(message.data);
    if (event.type === "ready") ready = true;
    if (event.type === "transfer") console.log("transfer to", event.target);
    if (event.type === "ended") console.log("ended:", event.reason);
  };

  ws.onclose = (event) => {
    console.log("closed", event.code);   // 1006 at once: the ticket was refused
    microphone.getTracks().forEach((track) => track.stop());
    context.close();
    talk.disabled = false;
  };

  hangUp.onclick = () => ws.send(JSON.stringify({ type: "hangup" }));
};

// pearl-audio.js: one AudioWorklet that records the microphone and plays Pearl's voice, both at 24 kHz.
const FRAME = 480;        // 20 ms at 24 kHz
const CUSHION = 1920;     // start playing with 80 ms buffered
const MAX_BEHIND = 7200;  // past 300 ms buffered, drop the oldest audio

class PearlAudio extends AudioWorkletProcessor {
  constructor() {
    super();
    this.mic = new Int16Array(FRAME);
    this.micLength = 0;
    this.queue = [];      // Pearl's audio, as Float32Array chunks
    this.offset = 0;      // read position in queue[0]
    this.queued = 0;      // samples not played yet
    this.playing = false;
    this.port.onmessage = (event) => {
      this.queue.push(event.data);
      this.queued += event.data.length;
    };
  }

  process(inputs, outputs) {
    // The microphone: 20 ms frames of 16-bit PCM, in real time, silence included when muted.
    const input = inputs[0][0];
    if (input) {
      for (let i = 0; i < input.length; i++) {
        this.mic[this.micLength++] = Math.max(-1, Math.min(1, input[i])) * 0x7fff;
        if (this.micLength === FRAME) {
          this.port.postMessage(this.mic.buffer.slice(0));
          this.micLength = 0;
        }
      }
    }

    // Pearl's voice: played through a small jitter buffer.
    while (this.queued > MAX_BEHIND && this.queue.length > 1) {
      this.queued -= this.queue.shift().length - this.offset;
      this.offset = 0;
    }
    if (!this.playing && this.queued >= CUSHION) this.playing = true;
    const output = outputs[0][0];
    for (let i = 0; i < output.length; i++) {
      if (!this.playing || this.queue.length === 0) {
        output[i] = 0;
        this.playing = false;   // ran dry after a stall: buffer again before playing
        continue;
      }
      const chunk = this.queue[0];
      output[i] = chunk[this.offset++];
      this.queued--;
      if (this.offset === chunk.length) {
        this.queue.shift();
        this.offset = 0;
      }
    }
    return true;
  }
}

registerProcessor("pearl-audio", PearlAudio);