RSA-OAEP Encrypt: OperationError on File upload but not strings using SubtleCrypto

I am trying to allow the client to upload a file, have the file be encrypted with RSA-OAEP using a public key provided by the server, then have the file be uploaded and completely readable by the server.
However, my current code gives me OperationError: The operation failed for an operation-specific reason (on Safari and Firefox). Here is my code:

document.getElementById("input").addEventListener('change', async event => {
    if (event.target.files[0]) {
        const reader = new FileReader();
        reader.addEventListener('load', async (event2) => {
            const res = await fetch("/key");
            const exported = await res.text();
            const key = await importRSAPublicKey(exported);
            const arrayBuffer = event2.target.result;
            console.log(arrayBuffer);
            const encrypted = await encryptRSA(key, arrayBuffer); // Fails here at the call to window.crypto.subtle.encrypt()
            console.log(encrypted);
            await fetch(`/upload`, {method: "POST", body: encrypted});
        });
        reader.addEventListener('error', () => {
            reject(new Error("There was an error reading the inserted file as text."));
        })
        /**
         * @type {File}
         */
        const file = event.target.files[0];
        reader.readAsArrayBuffer(file);
    }
});

async function importRSAPublicKey(key) {
    return new Promise(async (resolve, reject) => {
        try {
            const imported = await window.crypto.subtle.importKey(
                "jwk",
                JSON.parse(atob(key)),
                {
                    name: "RSA-OAEP",
                    modulusLength: 4096,
                    publicExponent: new Uint8Array([1, 0, 1]),
                    hash: "SHA-256",
                },
                true,
                ["encrypt"]
            );
            return resolve(imported);
        } catch (error) {
            reject(error);
        }
    })
}

async function encryptRSA(key, data) {
    return new Promise(async (resolve, reject) => {
        try {       
            const encryptedData = await window.crypto.subtle.encrypt(
                { name: "RSA-OAEP" },
                key,
                data
            )
    
            const uintArray = new Uint8Array(encryptedData);
    
            const string = String.fromCharCode.apply(null, uintArray);
    
            const base64Data = btoa(string);
    
            return resolve(base64Data);
        } catch (error) {
            return reject(error);
        }
    });
}

Interestingly, the same operation, when preformed either on text or via similar code for AES-GCM on the same file, works perfectly. Here is my (working) code for the plaintext application:

document.getElementById("input").addEventListener('change', async event => {
    const res = await fetch("/key");
    const exported = await res.text();
    const key = await importRSAPublicKey(exported);
    const encrypted = await encryptRSA(key, new TextEncoder().encode("Test"));
    console.log(encrypted);
    await fetch(`/upload`, {method: "POST", body: encrypted});
});
// ... Helper functions from the previous codeblock unchanged

Why, since both times an ArrayBuffer is being passed into the function, does one work and the other not?

I would love it if someone could tell me a way I can encrypt these without using btoa() on the cleartext (which is a pain to convert back into actual binary from there).