Google Maps tiles not loading properly when Marker location significantly changes

I’m using @vis.gl/react-google-maps and React to create a Google Map. The parent component supplies a bunch of coordinates, which I apply to the map using new window.google.maps.LatLngBounds() and map.fitBounds() inside a useEffect. When the app sends a new set of coordinates, the useEffect creates a new instance of window.google.maps.LatLngBounds() and applies the bounds again.

For the most part this works fine, but if there is a significant change in location, say from Paris to somewhere in US, the tiles do not load properly.

Whatever I’ve searched on SO, the answers are quite old. There is also a workaround of adding the prop key={JSON.stringify(coordinateData)} that forces the map to re-render. But obviously that’s terribly inefficient, and I also lose the feature of the Map smoothly panning or zooming in to new locations.

import {
  AdvancedMarker,
  Map,
  useMap,
} from "@vis.gl/react-google-maps";

const MarkerWithInfoWindow = ({ activityData, position }) => {

return (
    <>
      <AdvancedMarker
        position={position}
        title={activityData.name}
      >
        <Image
           src={getActivityImgSrc(activityData)}
           width={36}
           height={36}
           alt={activityData.type}
        />
      </AdvancedMarker>
    </>
  );
};

export default function CustomMap({ displayData }) {
  const map = useMap();
  console.log('rerender map', displayData)

  useEffect(() => {
    if (!map || !displayData) return;


    const bounds = new window.google.maps.LatLngBounds();
    const { outdoorsData, travelData } = displayData;
    console.log(displayData);

    const yearsForFilter = new Set();

    for (var i = 0; i < outdoorsData.length; i++) {
      bounds.extend(
        new window.google.maps.LatLng(
          outdoorsData[i].coordinates.lat,
          outdoorsData[i].coordinates.lng
        )
      );
      yearsForFilter.add(new Date(outdoorsData[i].date).getFullYear());
    }
    for (var i = 0; i < travelData.length; i++) {
      for (var k = 0; k < travelData[i].coordinatesArray.length; k++) {
        bounds.extend(
           new window.google.maps.LatLng(
              travelData[i].coordinatesArray[k].lat,
              travelData[i].coordinatesArray[k].lng
           )
        );
      }
      if (travelData[i].startDate) {
         yearsForFilter.add(new Date(travelData[i].startDate).getFullYear());
      }
    }

    map.fitBounds(bounds);

  }, [map, displayData]);

return (
    <Map
        mapId="DEMO"
        defaultZoom={2}
        defaultCenter={{ lat: 5.145259, lng: -27.8719489 }}
        mapTypeControl={false}
        streetViewControl={false}
    // key={JSON.stringify(displayData)}
    >
        {displayData.outdoorsData.map((activityData, index) => {

            return (
                <MarkerWithInfoWindow
                    key={index}
                    activityData={activityData}
                    position={{
                        lat: activityData.coordinates.lat,
                        lng: activityData.coordinates.lng,
                    }}
                />
            );
        })}
        {displayData.travelData.map((activityData) => {
            return activityData.coordinatesArray.map((coordinatesObj, index) => {
                return (
                    <MarkerWithInfoWindow
                        key={index}
                        activityData={activityData}
                        position={{ lat: coordinatesObj.lat, lng: coordinatesObj.lng }}
                    />
                );
            });
        })}
    </Map>
);

};