Zoom D3 bar chart in React on one axis only, but still be able to pan on both axises

I’m trying to have a bar chart in React that should pan on only the y-axis, but I still want to be able to pan on both axis. Right now I have a solution where I can zoom on the y-axis only, but then I can’t pan on the x.

Here is a CodePen with what I have atm

And here’s the D3 part of that CodePen:

const BarChart = () => {
  const width = 1000,
    height = 800;
  const ref = useRef(null);

  const margin = { top: 20, right: 0, bottom: 30, left: 40 };
  const extent = [
    [margin.left, margin.top],
    [width - margin.right, height - margin.top]
  ];

  useEffect(() => {
    const svg = d3.select(ref.current);

    const xScale = d3
      .scaleBand()
      .range([margin.left, width - margin.right])
      .domain(data.map((d) => d.name))
      .padding(0.3);

    const yScale = d3
      .scaleLinear()
      .range([height - margin.bottom, margin.top])
      .domain([0, d3.max(data, (d) => d.value)])
      .nice();

    const xAxis = (g) =>
      g
        .attr('transform', `translate(0,${height - margin.bottom})`)
        .call(d3.axisBottom(xScale).tickSizeOuter(0));

    const yAxis = (g) =>
      g
        .attr('transform', `translate(${margin.left},0)`)
        .call(d3.axisLeft(yScale))
        .call((g_local) => g_local.select('.domain').remove());

    svg
      .append('g')
      .attr('class', 'bars')
      .selectAll('rect')
      .attr('fill', 'steelblue')
      .data(data)
      .enter()
      .append('rect')
      .attr('x', (d) => xScale(d.name))
      .attr('y', (d) => yScale(d.value))
      .attr('width', xScale.bandwidth())
      .attr('height', (d) => yScale(0) - yScale(d.value))
      .attr('fill', 'steelblue');

    svg.append('g').attr('class', 'x-axis').call(xAxis);

    svg.append('g').attr('class', 'y-axis').call(yAxis);

    function zoom() {
      function zoomed(event) {
        yScale.range(
          [height - margin.bottom, margin.top].map((d) =>
            event.transform.applyY(d)
          )
        );
        svg
          .selectAll('.bars rect')
          .attr('y', (d) => yScale(d.value))
          .attr('height', (d) => yScale(0) - yScale(d.value));
        svg.selectAll('.y-axis').call(yAxis);
      }

      svg.call(
        d3
          .zoom()
          .scaleExtent([1, 8])
          .translateExtent(extent)
          .extent(extent)
          .on('zoom', zoomed)
      );
    }

    d3.select(ref.current).call(zoom);
  }, [data, height, width]);

  return (
    <svg
      ref={ref}
      width={width}
      height={height}
      viewBox={`0 0 ${width - 400} ${height}`}
    ></svg>
  );
};