Server part:
I have an ASR voice recognition AI model, it will generate data like below and use websocket to pass data to my server.
{ id: 1, segment: 1, data: "I am glad", final: false }
{ id: 2, segment: 1, data: "I am glad to": final: false }
{ id: 3, segment: 1, data: "I am glad to see you", final: true },
...
{ id: 50, segment: 3, data: "something other data...", }
On the other hand, my server has a bridge websocket to get and store ASR data then pass data to the clients.
Client Part:
When client get data from my websocket server, it would allow client to update the realtime data. So I use websocket to send back to the bridge websocket server. BTW, I can’t wait ASR segment to be final.
{ update: true, segment: 1, original: "I am glad", replaced: "I am v glad" }
{ update: true, segment: 1, original: "I am glad", replaced: "I am ve glad" }
{ update: true, segment: 1, original: "I am glad to", replaced: "I am ver glad to" }
My question is like how to design the bridge websocket server or there has a 3rd library I can use to handle this situation? Because when I use this pattern trying to update data, it doesn’t work correctly. The main reason is that I can’t handle ASR model generated time.
ex: I send update message to the server, but at the meantime ASR also send message to the client.
On my website, the sequence would be like
- I am glad
- I am v glad <== typing v send message to the server
- I am glad to <== because the ASR send back first, v character disappear ;(
- I am v glad to <== next render, it will replace
Below is my code:
let asrWS, timeout;
recordWS.on("connection", async (ws) => {
console.log("record socket connected");
ws.on("error", () => {});
ws.on("message", async (event) => {
const evt = JSON.parse(event);
if (evt.status) {
try {
if (evt.status === "start") {
sendStatus("pending");
const token = await login();
const { wsHost, ticket } = await getStreamInfo(token);
// connect to ASR websocket, store ASR data then broadcast to the clients
asrWS = await connectWS(wsHost, ticket, recordWS);
start(asrWS);
sendStatus("recording");
} else if (evt.status === "stop") {
stop();
if (asrWS) {
asrWS.close();
}
sendStatus("stop");
timeout = setTimeout(() => {
sendStatus("start");
}, 3000);
}
} catch (error) {
console.warn(error);
}
}
if (evt.update) {
// update ASR global data
setTranscripts(+evt.segment, evt.original, evt.replaced, "update");
console.log(getTranscripts());
}
});
ws.on("close", () => {
if (timeout) {
clearTimeout(timeout);
}
console.log("Disconnected record");
});
});
function connectWS(wsHost, ticket, recordWS) {
const webSocketUrl = `${wsHost}?ticket=${encodeURIComponent(
ticket
)}&modelName=${modelName}&rate=16000`;
const asrWS = new WebSocket(webSocketUrl, {
binaryType: "arraybuffer",
});
asrWS.onopen = () => {
console.log("open ASR server connection");
};
asrWS.onerror = (error) => {
console.error(`ASR server encountered error: ${error.message}`);
asrWS.close();
};
asrWS.onmessage = (event) => {
let d = JSON.parse(event.data);
// ...
const transcripts = getTranscripts();
const current = transcripts[+d.result.segment];
if (current) {
const re = new RegExp(current.original, "i");
d.t= d.t.replace(re, current.replaced);
}
d = JSON.stringify(d);
recordWS.clients.forEach((client) => {
client.send(d);
});
}
asrWS.onclose = (event) => {
console.log(`Close ASR server connection ${event.code}`);
};
return new Promise((resolve) => asrWS.once("open", () => resolve(asrWS)));
}