I’m trying to create a web extension using Web Audio API and Essentia.js.
Basically, the process I’m currently focused on is:
- User inputs an audio file from device;
- File is converted to the
Uint8Array(I can’t send it as anArrayBufferusing runtime message) and sent to the offscreen document with its metadata; - Offscreen creates two
AudioContexts: one for playback and one for analysis, after which it converts the contents to theArrayBufferand decodes it to create anAudioBuffer(both contexts decode it separately, basically copying); - Analysis context buffer is converted – its channels’ data (
Float32Array) is pushed to an array (usinggetChannelData), after that it’s sent to the processingWorkeropened just before that; Workerconverts the received data to mono and analyses it using the Essentia.js library, specificallyRhythmExtractor2013;Workersents the results of analysis back.
I’m using Worker for the processing task because the lib uses eval and/or new Function the use of which is forbidden – 'unsafe-eval' directive is strictly prohibited in the manifest.json. But I’ve found that this problem can be avoided using Web Workers.
I’m making the extension for Google Chrome and Manifest V3, so I don’t rule out that it’s just impossible to do on Chrome right now.
Initially, I wanted to make the extension for the Mozilla Firefox, but when I found out, that audio capturing using getDisplayMedia is NOT supported in Firefox I moved to Chrome. Currently, I don’t mind moving back, because the deadlines are pressing very hard and I’m not going to implement this functionality anyway. :)
Manifest:
{
"manifest_version": 3,
"default_locale": "en",
"name": "TimeKeeper", // also – MetroGnome
"version": "1.0",
"description": "Provides real-time beat detection and metronome from a playing audio.",
"action": {
"default_popup": "popup/popup.html#popup?=true"
// cool hack! to check if the script was opened by an extension popup,
// set "default_popup": "popup.html#popup", and then check with location.hash == "popup"
// (if string has a word "popup" in it – it will determine whether it is a popup
// (no one will open this page in such a way anyway))
},
"background": {
"service_worker": "background/background.js", // Google Chrome
// "scripts": ["background/background.js"], // Mozilla Firefox
"type": "module"
},
"content_scripts": [{
"matches": ["<all_urls>"],
"js": ["unsorted/content.js"]
}],
"content_security_policy": {
"extension_pages": "script-src 'self' 'wasm-unsafe-eval'; worker-src 'self' 'wasm-unsafe-eval'"
},
"permissions": [
"tabs",
"activeTab",
"offscreen",
"tabCapture",
"storage"
],
"host_permissions": [
"<all_urls>"
],
"web_accessible_resources": [
{
"resources": [
"lib/essentia/*",
"file-mode/*"
],
"matches": ["<all_urls>"]
}
]
}
Offscreen document:
if (typeof browser === "undefined")
var browser = chrome;
// ...
/*
There are 2 audio contexts: playback and analysis.
Playback context is responsible for:
- playing the audio;
- generating click-track;
- volume control.
Analysis context is responsible for:
- analysing different parts of the loaded song;
- giving the necessary data for the playback context.
*/
/** All parts of the playback audio context (nodes and clicker) */
let playback = {
/** @type {AudioContext} */
context: null,
/** @type {AudioBuffer} */
buffer: null,
/** @type {AudioBufferSourceNode} */
audioSource: null,
/** @type {IntervalClicker} */
clickSource: null,
/** @type {GainNode} */
audioGain: null,
/** @type {GainNode} */
clickGain: null
};
/** All parts of the analysis audio context (nodes) */
let analysis = {
/** @type {AudioContext} */
context: null,
/** @type {AudioBuffer} */
buffer: null,
/** @type {AudioBufferSourceNode} */
source: null,
/** @type {Worker} */
detector: null,
result: null
};
// ...
async function audioContextsSetup(file) {
if (playback.context || analysis.context)
throw new Error("Previous context wasn't closed");
audioContextsClose();
playback.context = new AudioContext();
analysis.context = new AudioContext();
try {
// Because they're different context's, the buffer have to be decoded twice...
playback.buffer = await playback.context.decodeAudioData(file.buffer.slice(0));
analysis.buffer = await analysis.context.decodeAudioData(file.buffer.slice(0));
file.meta.duration = playback.buffer.duration;
}
catch (error) {
return Result.failure({
description: "Failed decoding audio: " + error
})
}
playback.clickSource = new IntervalClicker(playback.context);
playback.audioGain = new GainNode(playback.context, { gain: state.songVolume});
playback.clickGain = new GainNode(playback.context, { gain: state.clickVolume});
analysis.detector = new Worker("processor.js", { type: "module" });
analysis.detector.onmessage = (e) => {
const msg = e.data;
switch (msg.type) {
case "message-error":
console.error("Message error occured: " + msg.description);
break;
case "analysis-log":
console.log("Analysis:", msg.description);
break;
case "analysis-result":
analysis.result = msg.result;
playbackStartClickerFrom(playback.context.currentTime);
break;
case "analysis-error":
console.error("Analysis error occured: " + msg.description);
break;
}
};
analysis.detector.onerror = (e) => {
console.error("Worker error occured:", e);
}
if (analysis.buffer.numberOfChannels > 2) {
return Result.failure({
description: "Audio data has more than 2 channels"
});
}
let channels = [];
for (let c = 0; c < analysis.buffer.numberOfChannels; c++) {
channels.push(analysis.buffer.getChannelData(c).buffer);
}
analysis.detector.postMessage({
type: "analysis-start",
audio: {
channels
}
}, [...channels]);
playback.audioGain.connect(playback.context.destination);
playback.clickSource.connect(playback.clickGain);
playback.clickGain.connect(playback.context.destination);
state.currentFile = file.meta;
state.songTimeLast = 0.;
console.log("Current file is:", state.currentFile);
return Result.success({ description: "Audio contexts setup successfully" });
}
Worker:
import Essentia from "../lib/essentia/essentia.js-core.es.js";
import { EssentiaWASM } from "../lib/essentia/essentia-wasm.web.js";
const essentia = new Essentia(EssentiaWASM);
let audio;
let result;
const messageHandlers = {
"analysis-start": (message) => {
if (message?.audio === undefined)
postMessage({
type: "analysis-fail",
description: "Can't analyse audio since no audio passed"
});
audio = message.audio;
postMessage({
type: "analysis-log",
description: "Audio acquired, starting analysis.."
});
try { analyse(); }
catch(error) {
postMessage({
type: "analysis-error",
description: error
});
}
postMessage({
type: "analysis-result",
result: {
bpm: result.bpm,
beats: result.beats
}
});
}
};
onmessage = (e) => {
const message = e.data;
console.log(message);
const handle = messageHandlers[message.type];
if (handle === undefined) {
postMessage({
type: "message-error",
description: "Unknown message type"
});
}
else handle(message);
};
function analyse() {
if (audio?.audioBuffer) analyseAudioBuffer();
else if (audio?.channels) analyseChannels();
else postMessage({
type: "analysis-error",
description: "Unknown structure of received audio data"
});
}
function analyseAudioBuffer() {
let signal;
signal = essentia.audioBufferToMonoSignal(audio.audioBuffer);
result = essentia.RhythmExtractor2013(signal);
}
function analyseChannels() {
let signal;
switch (audio.channels.length) {
case 1:
signal = essentia.arrayToVector(audio.channels[0]);
break;
case 2:
signal = essentia.MonoMixer(audio.channels[0], audio.channels[1]);
break;
default:
throw new Error("Analysis is supported only for 1 or 2 (mixed to mono) channel audio");
}
result = essentia.RhythmExtractor2013(signal);
}
I’ve now tried to test just the loading and found out that only essentia.js-core.es.*.js doesn’t fire a Worker error.
The WASM backends on the other hand ALL fire errors but ONLY the essentia-wasm.es.js fires an informative event. All other don’t have any description.
essentia-wasm.es.js error: Uncaught EvalError: Refused to evaluate a string as JavaScript because 'unsafe-eval' is not an allowed source of script in the following Content Security Policy directive: "script-src 'self' 'wasm-unsafe-eval' 'inline-speculation-rules' http://localhost:* http://127.0.0.1:*".
Please help!

