How to stop triggering the event listener callback function on the button click?

I have feature where as I scroll I automatically select the card that is in view port, also I have another feature where there are 2 buttons “Prev” and “Next” and when i click it will go to the next or prev card and bring that into the view port. But when i click my event listener callback is also getting triggered, I want to trigger my event listener only when I scroll, not when I click on “Prev” and “Next”
Below is my code where I have the Scroll event listener and the OnClick for Prev and Next.
I tried Using both ref and state to see the updated value inside handleScroll but first time I am getting the correct value and again for some reason it was getting triggered 2nd time again with false for `isButtonScrolling.current`, I suspect it was after due to scrollintoView that happened inside the store. I event tried to do event.stopPropagation() but that did not work as well.

import { Grid, LinearProgress, Typography } from '@mui/material';
import { makeStyles } from '@mui/styles';
import { useAppDispatch, useAppSelector } from '@store/StoreTypedHooks';
import PreviousButton from './PreviousButton';
import NextButton from './NextButton';
import useDeviceType from '@hooks/useDeviceType';
import { answeredQuestionCount, expandCollapse, totalQuestionCount } from '@store/respondStore/RespondStoreSelectors';
import { useEffect, useRef, useState } from 'react';
import { updateCurrentQuestionOnScroll, updatePreviousNextQuest } from '@store/respondStore/RespondStoreThunks';
import QuestionScrollObserever from './QuestionScrollObserver';
import { debounce } from '@utils/GeneralUtils';

export default function RespondPageHeader() {
  const classes = RespondPageStyles();
  const dispatch = useAppDispatch();
  const totalCount = useAppSelector(totalCardCount);
  const totalComputedCount = useAppSelector(computedCount);
  const expandCollapseOption = useAppSelector(expandCollapse);
  const progressValue = (totalComputedCount / totalCount) * 100;
  const { isMobileTab, isMobile } = useDeviceType();

  const isButtonScrolling = useRef(false);

  const isElementInViewport = (element: HTMLElement, containerRect: any) => {
    const rect = element.getBoundingClientRect();
    return (
      rect.left >= 0 &&
      rect.bottom <= containerRect.bottom &&
      rect.right <= containerRect.right
    );
  };

  const isContainerScrolledToEnd = (container: HTMLElement) => {
    return container.scrollHeight - container.scrollTop === container.clientHeight;
  };

  const getTopCardId = (scrollLock: boolean) => {
    if (scrollLock || expandCollapseOption !== 'EXPAND') {
      console.log('Skipping scroll processing - Button scroll active:', isButtonScrolling.current);
      return;
    }

    const container = document.getElementById('question_scroll_observer');
    if (!container) return;

    const cards = container.getElementsByClassName('card');
    const containerRect = container.getBoundingClientRect();

    let maxVisibility = 0;
    let selectedCardId = null;

    for (let card of cards) {
      const questionElement = card.querySelector('.question');

      if (!questionElement || !isElementInViewport(questionElement, containerRect)) {
        continue;
      }

      const cardRect = card.getBoundingClientRect();
      const cardVisibleHeight = Math.min(cardRect.bottom, containerRect.bottom) - Math.max(cardRect.top, containerRect.top);
      const visiblePercentage = cardVisibleHeight / cardRect.height;

      if (visiblePercentage > maxVisibility) {
        maxVisibility = visiblePercentage;
        selectedCardId = card.id.replace('question_card_', '');
      }
    }

    if (isContainerScrolledToEnd(container)) {
      const lastCard = cards[cards.length - 1];
      if (lastCard) {
        selectedCardId = lastCard.id.replace('question_card_', '');
      }
    }

    if (selectedCardId) {
      dispatch(updateCurrentQuestionOnScroll({ questionId: selectedCardId }));
    }
  };

  const handleScroll = () => {
    console.log('Scroll event fired - Button scroll active:', isButtonScrolling.current);
    if (!isButtonScrolling.current) {
      getTopCardId(isButtonScrolling.current);
    }
  };

  const goToPreviousNextSection = (type: string) => {

    isButtonScrolling.current = true;

    // Dispatch navigation action where I scroll to the card either next card or prev card based on the type I am getting in parameters and to scroll to the card I use scrollIntoView for that particular card id
    dispatch(updatePreviousNextQuest({ changeQuestion: type }));

    // Set timeout to reset ref
    const scrollTimeout = setTimeout(() => {
      isButtonScrolling.current = false;
      clearTimeout(scrollTimeout)
    }, 1000);
  };

  // Set up scroll event listener
  useEffect(() => {
    const container = document.getElementById('question_scroll_observer');

    if (container) {
      container.addEventListener('scroll', handleScroll);

      return () => {
        container.removeEventListener('scroll', handleScroll);
      };
    }
  }, []);

  return (
    <Grid className={`panel-color h-52 flex`}>
      <div className={`p-24 pl-10p pr-12p flex layout-row w-full items-center ${isMobileTab && 'justify-between'}`}>
        {isMobileTab && (
          <Grid className='pr-8p'>
            <PreviousButton onChange={() => goToPreviousNextSection("PREVIOUS")} />
          </Grid>
        )}
        <Grid className='pr-8p'>
          <Typography className='radiant-green-text text-base'>
            {`${totalCount - answeredCount} questions left`}
          </Typography>
        </Grid>
        {!isMobile && (
          <Grid className='layout-flex'>
            <LinearProgress
              value={progressValue}
              variant='determinate'
              className={classes.progressBarStyle}
            />
          </Grid>
        )}
        <Grid className='flex justify-between'>
          {!isMobileTab && (
            <PreviousButton onChange={() => goToPreviousNextSection("PREVIOUS")} />
          )}
          <NextButton onChange={() => goToPreviousNextSection("NEXT")} />
        </Grid>
      </div>
    </Grid>
  );
}