Messages

// After session_ready, capture mic audio, convert to 16-bit PCM, and send.
function startMicrophone(ws) {
  navigator.mediaDevices.getUserMedia({
    audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true }
  }).then((stream) => {
    const ctx = new AudioContext({ sampleRate: 16000 });
    const source = ctx.createMediaStreamSource(stream);
    const node = ctx.createScriptProcessor(4096, 1, 1);
    node.onaudioprocess = (e) => {
      const f32 = e.inputBuffer.getChannelData(0);
      const pcm = new Int16Array(f32.length);
      for (let i = 0; i < f32.length; i++) {
        pcm[i] = Math.max(-1, Math.min(1, f32[i])) * 0x7fff; // float32 -> int16
      }
      if (ws.readyState === WebSocket.OPEN) ws.send(pcm.buffer); // binary frame
    };
    source.connect(node);
    node.connect(ctx.destination);
  });
}

import asyncio, json, websockets

async def stream(ws, pcm_source):
    # pcm_source yields raw pcm_s16le bytes (mono, 16 kHz).
    async for chunk in pcm_source:
        await ws.send(chunk)               # binary frame
    await ws.send(json.dumps({"type": "end_utterance"}))  # flush the tail
    await ws.send(json.dumps({"type": "close"}))          # end the session

ws.onmessage = (event) => {
  // Ignore any binary frames the server does not send in normal operation.
  if (typeof event.data !== "string") return;
  const msg = JSON.parse(event.data);

  switch (msg.type) {
    case "transcript":
      lastProcessedUntil = msg.processed_until;      // track for reconnection
      appendCaption(msg.seq, msg.segment, msg.text); // render
      break;
    case "ack":
      lastProcessedUntil = msg.processed_until;
      break;
    case "backpressure":
      throttleSendRate();                            // slow down
      break;
    case "status":
      console.log("status:", msg.state);
      break;
    case "error":
      console.error(msg.code, msg.message);
      if (msg.fatal) ws.close();
      break;
  }
};

async def receive(ws, state):
    async for message in ws:
        if isinstance(message, (bytes, bytearray)):
            continue
        msg = json.loads(message)
        t = msg.get("type")
        if t == "transcript":
            state["processed_until"] = msg["processed_until"]
            print(f"[{msg['segment']['start']:.2f}] {msg['text']}")
        elif t == "ack":
            state["processed_until"] = msg["processed_until"]
        elif t == "backpressure":
            state["throttle"] = True
        elif t == "status":
            print("status:", msg["state"])
        elif t == "error":
            print("error:", msg["code"], msg["message"])
            if msg.get("fatal"):
                break