Slow rendering of image from cache

I try to dynamically prefetch the next image on mouse enter or after 400ms. The network tab shows that it works correctly. But even though the network tab says that it is cached and not making a request the first render of any image is slower than the next ones. Any ideas why?

function RouteComponent() {
    const images = {
        1: "https://images.unsplash.com/photo-1721332149274-586f2604884d?q=80&w=1936&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDF8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D",
        2: "https://plus.unsplash.com/premium_photo-1736798695947-2c7661f72bc8?q=80&w=1974&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D",
        3: "https://images.unsplash.com/photo-1735732519861-3b67d0aee297?q=80&w=1935&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D",
        4: "https://images.unsplash.com/photo-1735767976699-6096acda642d?q=80&w=1974&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D",
        5: "https://plus.unsplash.com/premium_photo-1732721750556-f5aef2460dfd?q=80&w=2070&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D",
    }

    const [page, setPage] = useState(1)
    const loadedImagesRef = useRef(new Set<number>([1]))

    const handleChange = (_event: React.ChangeEvent<unknown>, value: number) => {
        setPage(value) // Update the state with the new page
        console.log("Current page:", value) // Log the current page
        setTimeout(() => {
            fetchNewImage(value)
            console.log("fired")
        }, 400)
    }

    const fetchNewImage = (value: number) => {
        if (!images[(value + 1) as keyof typeof images]) {
            return
        }
        if (loadedImagesRef.current.has(value + 1)) {
            return
        }
        const img = new Image()
        img.src = images[(value + 1) as keyof typeof images]
    }

    const handleHover = () => {
        fetchNewImage(page)
    }

    return (
        <Box
            sx={{
                border: "1px solid red",
                height: "100vh",
                display: "flex",
                flexDirection: "column",
                justifyContent: "end",
                alignItems: "center",
                paddingBottom: "50px",
            }}
        >
            <Box
                sx={{
                    border: "1px solid red",
                }}
            >
                <img src={images[page as keyof typeof images]} alt="d" width="500px" />
            </Box>
            <Pagination count={5} page={page} onChange={handleChange} onMouseEnter={handleHover} />
        </Box>
    )
}