Prevent dragging image over container boundaries in React JS

In my react js application i am trying to create a zoomable image with the possibility to drag image in zoom mode.

import "./styles.css";
import React, { useState, useRef, useEffect } from "react";

export default function App() {
  const [isDragging, setIsDragging] = useState(false);
  const [zoomLevel, setZoomLevel] = useState(1);
  const [position, setPosition] = useState({ x: 0, y: 0 });
  const [dragStart, setDragStart] = useState({ x: 0, y: 0 });
  const imgRef = useRef(null);
  const containerRef = useRef(null);

  useEffect(() => {
    if (zoomLevel === 1) {
      setPosition({ x: 0, y: 0 }); // Reset position when zoom level is 1
    }
  }, [zoomLevel]);

  const handleWheel = (e) => {
    e.preventDefault();
    const zoomAmount = e.deltaY * -0.001;
    setZoomLevel((prevZoom) => Math.min(Math.max(prevZoom + zoomAmount, 1), 5));
  };

  const handleZoomIn = () => {
    setZoomLevel((prevZoom) => Math.min(prevZoom + 0.1, 5));
  };

  const handleZoomOut = () => {
    setZoomLevel((prevZoom) => Math.max(prevZoom - 0.1, 1));
  };

  const calculateBoundaries = () => {
    if (!imgRef.current || !containerRef.current) return;

    // Get bounding rects
    const containerRect = containerRef.current.getBoundingClientRect();
    const imageRect = imgRef.current.getBoundingClientRect();

    // Calculate boundaries
    const minX = containerRect.left - imageRect.width + position.x; // Allow dragging to the left edge
    const maxX = containerRect.right - position.x; // Prevent dragging past the right edge
    const minY = containerRect.top - imageRect.height + position.y; // Allow dragging to the top edge
    const maxY = containerRect.bottom - position.y; // Prevent dragging past the bottom edge

    return { minX, maxX, minY, maxY };
  };

  const handleMouseDown = (e) => {
    if (zoomLevel === 1) return; // Disable dragging if not zoomed
    e.preventDefault();
    setIsDragging(true);
    setDragStart({
      x: e.clientX - position.x,
      y: e.clientY - position.y,
    });
  };

  const handleMouseMove = (e) => {
    if (!isDragging) return;
    e.preventDefault();
    // Calculate the distance dragged

    const { minX, maxX, minY, maxY } = calculateBoundaries();
    const newX = e.clientX - dragStart.x;
    const newY = e.clientY - dragStart.y;

    // Apply boundaries to the new position
    const updatedX = Math.min(Math.max(newX, minX), maxX);
    const updatedY = Math.min(Math.max(newY, minY), maxY);

    setPosition({
      x: updatedX,
      y: updatedY,
    });

    // Change cursor based on position
    if (
      updatedX === minX ||
      updatedX === maxX ||
      updatedY === minY ||
      updatedY === maxY
    ) {
      document.body.style.cursor = "not-allowed"; // Change cursor to not-allowed if at boundary
    } else {
      document.body.style.cursor = isDragging ? "grabbing" : "grab"; // Default cursor
    }
  };

  const handleMouseUp = () => {
    setIsDragging(false);
    document.body.style.cursor = "default"; // Reset cursor when dragging stops
  };

  return (
    <div>
      <div className="zoom-controls">
        <button onClick={handleZoomIn}>Zoom In</button>
        <button onClick={handleZoomOut}>Zoom Out</button>
      </div>
      <div
        className="zoom-container"
        ref={containerRef}
        onWheel={handleWheel}
        onMouseDown={handleMouseDown}
        onMouseMove={handleMouseMove}
        onMouseUp={handleMouseUp}
        onMouseLeave={handleMouseUp}
      >
        <img
          ref={imgRef}
          src="https://asset.gecdesigns.com/img/wallpapers/beautiful-fantasy-wallpaper-ultra-hd-wallpaper-4k-sr10012418-1706506236698-cover.webp"
          alt="Zoomable"
          style={{
            transform: `translate(${position.x}px, ${position.y}px) scale(${zoomLevel})`,
            cursor:
              zoomLevel > 1 ? (isDragging ? "grabbing" : "grab") : "default",
          }}
          className="zoom-image"
        />
      </div>
    </div>
  );
}

.zoom-container {
  overflow: hidden;
  width: 100%;
  height: 400px;
  position: relative;
  border: 1px solid #ddd;
  background-repeat: no-repeat;
  background-position: cover;
  cursor: default; /* Default cursor when not interacting */
}

.zoom-image {
  transform-origin: center center;
  transition: transform 0.1s ease-out;
  background-position: contain;
  width: 100%;
  height: 400px;
}

The component works, zoom works fine. The only issue what i am trying to solve is to prevent dragging the image more than its boundaries.

Expected result:

When user will zoom and will start dragging the image in any direction then he should be stopped to drag more and to appear these white spaces between image and container like is now. So, as example if user will drag from top to bottom, the maximum allowed dragging effect should be when the user will drag until the image edge, not more..
demo: https://codesandbox.io/p/sandbox/pedantic-bose-ywk35n