I am quite new to mp4 file. But here is my working attempt to extract image frame given video url and a timestamp.

In reality the input url is an 8K 10hours 200GB video, so I can’t download it all, I can’t load it to memory, and this is an API call so it has to be fast.

Is there anything else I can optimize ?

My doubts:

• This line ffprobe -v error -select_streams v:0 -show_entries packet=pos,size,dts_time -read_intervals ${timestamp}%+5 -of csv=p=0 "${url}" I chose this clingy 5s, in which case would this fail ?

• Same line, I don’t know what’s going on under the hood of this ffprobe command, but I tested it with the big 8K video and it seems to complete fast. So is it safe to assume that the entire 200GB was not downloaded? An explanation of how this ffprobe command work would be appreciated

• Based on trial and error, I concluded that the intervals returned is parsable by ffmpeg only if the first frame until the timestamp is included. If I include only that the single frame interval, ffmpeg says it is an invalid file. (Makes sense cuz I don’t think I’ll get an image from a 4byte data.) However, how can I be sure that I am selecting the least number of intervals.

• Worse bottleneck: The function extractFrame takes 6seconds on the big video. It seems to read the entire video segment fetched (by the preceding subrange step). I couldn’t find a way to jump to the last frame without computing. Is this how MP4 work ? I read somewhere that MP4 computes the current frame based on the previous. If that is true, does it mean there is no way to compute a specific frame without reading everything since the last keyframe ?

• Finally, this ffmpeg line is fishy (I got it from SO Extract final frame from a video using ffmpeg) it says that it ovewrites the output at everyframe . Does it mean it is writing to the disk every time ? I experience severe degradation in performance when I used .png instead of .jpg. This tells me that the image is computed every
  frame. Can we compute only the final image at the very end ?

Here is the working code to optimize.

import path from "path";
import axios from "axios";
import ffmpeg from "fluent-ffmpeg";
import fs from "fs";
import {promisify} from 'util';
import {exec} from 'child_process';

const execPromise = promisify(exec);


// URL of the video and desired timestamp (in seconds)

const videoUrl = 'https://raw.githubusercontent.com/tolotrasamuel/assets/main/videoplayback.mp4';

console.log(videoUrl);
const timestamp = 30; // Example: 30 seconds into the video


// Function to get the byte range using ffprobe
const getByteRangeForTimestamp = async (url, timestamp) => {
    // Use ffprobe to get the offset and size of the frame at the given timestamp
    const command = `ffprobe -v error -select_streams v:0 -show_entries packet=pos,size,dts_time -read_intervals ${timestamp}%+5 -of csv=p=0 "${url}"`;
    console.log('Running command:', command);
    const {stdout} = await execPromise(command);


    // Parse the output
    const timeStamps = stdout.trim().split("n");
    const frames = timeStamps.map(ts => {
        const [dts_time, size, offset] = ts.split(',');
        const timeInt = parseFloat(dts_time);
        const offsetInt = parseInt(offset);
        const sizeInt = parseInt(size);
        return {dts_time: timeInt, size: sizeInt, offset: offsetInt};
    })

    if (frames.length === 0) {
        throw new Error('No frames found in the specified interval');
    }

    let closest;


    let i = 0
    while (i < frames.length) {
        if (i === frames.length) {
            throw new Error('No frames found in the specified 5s interval');
        }
        if (frames[i].dts_time >= timestamp) {
            const oldDiff = Math.abs(closest.dts_time - timestamp);
            const newDiff = Math.abs(frames[i].dts_time - timestamp);
            if (newDiff < oldDiff) {
                closest = frames[i];
            }
            break;
        }
        closest = frames[i];
        i++;
    }

    // I experimented with this, but it seems that the first frame is always at the beginning of a valid atom
    // anything after that will make the video unplayable
    const startByte = frames[0].offset;
    const endByte = closest.offset + closest.size - 1;

    const diff = Math.abs(closest.dts_time - timestamp);
    const size = endByte - startByte + 1;
    console.log("Start: ", startByte, "End: ", endByte, "Diff: ", diff, "Timestamp: ", timestamp, "Closest: ", closest.dts_time, "Size to fetch: ", size)


    const startTime = closest.dts_time - frames[0].dts_time;
    return {startByte, endByte, startTime};
};

// Download the specific segment
const downloadSegment = async (url, startByte, endByte, outputPath) => {
    console.log(`Downloading bytes ${startByte}-${endByte}...`);
    const response = await axios.get(url, {
        responseType: 'arraybuffer',
        headers: {
            Range: `bytes=${startByte}-${endByte}`,
        },
    });

    console.log('Segment downloaded!', response.data.length, "Written to: ", outputPath);
    fs.writeFileSync(outputPath, response.data);
};

// Extract frame from the segment
const extractFrameRaw = async (videoPath, timestamp, outputFramePath) => {


    const command = `ffmpeg -sseof -3 -i ${videoPath} -update 1 -q:v 1 ${outputFramePath} -y`;
    console.log('Running command:', command);
    const startTime = new Date().getTime();
    await execPromise(command);
    const endTime = new Date().getTime();
    console.log('Processing time:', endTime - startTime, 'ms');
    console.log('Frame extracted to:', outputFramePath);
};
const extractFrame = (videoPath, timestamp, outputFramePath) => {
    ffmpeg(videoPath)
        .inputOptions(['-sseof -5'])  // Seeks to 3 seconds before the end of the video
        .outputOptions([
            '-update 1', // Continuously update the output file with new frames
            '-q:v 1'     // Set the highest JPEG quality
        ])
        .output(outputFramePath)  // Set the output file path

        // log
        .on('start', function (commandLine) {
            console.log('Spawned Ffmpeg with command: ' + commandLine);
        })
        .on('progress', function (progress) {
            console.log('Processing: ' + progress.timemark + '% done', progress, 'frame: ', progress.frames);
        })
        .on('end', function () {
            console.log('Processing finished !');
        })
        .on('error', function (err, stdout, stderr) {
            console.error('Error:', err);
            console.error('ffmpeg stderr:', stderr);
        })
        .run();
};


const __dirname = path.resolve();

// Main function to orchestrate the process
(async () => {
    try {
        // ffmpeg.setFfmpegPath('/usr/local/bin/ffmpeg');
        const {startByte, endByte, startTime} = await getByteRangeForTimestamp(videoUrl, timestamp);
        const tmpVideoPath = path.resolve(__dirname, 'temp_video.mp4');
        const outputFramePath = path.resolve(__dirname, `frame_${timestamp}.jpg`);

        await downloadSegment(videoUrl, startByte, endByte, tmpVideoPath);
        await extractFrame(tmpVideoPath, startTime, outputFramePath);
    } catch (err) {
        console.error('Error:', err);
    }
})();