React: How do I prevent the usage of useRef and useState to keep track of a same value in order to prevent useEffect from triggering?

currently I am making a navbar that only shows when you scroll up, to prevent useEffect to run everytime when the visible state get changed, I had to use both a ref and a state that is synced together to do comparison in the useEffect, using ref and a state to keep track of a same value seems extremely fishy, is there another way of doing this? one that does not involve triggering useEffect from creating the event handlers everytime the state changes?

import React, { useState, useRef, useEffect } from 'react';
import Link from 'next/link';

const NavbarLink = ({ name, href }: { name: string, href: string }) => {
  return (
    <Link href={href}>
      <a>{ name }</a>
    </Link>
  );
}

const Navbar = () => {
  const scrollYRef = useRef(0);
  const visibleRef = useRef(true);
  const [visible, setVisible] = useState(true);

  useEffect(() => {
    const onScroll = (event: Event) => {
      event.preventDefault();
      if ((window.scrollY < scrollYRef.current) != visibleRef.current) {
        visibleRef.current = !visibleRef.current;
        setVisible(x => !x);
      }
      scrollYRef.current = window.scrollY;
    }

    window.addEventListener('scroll', onScroll);
    return () => {
      window.removeEventListener('scroll', onScroll);
    }
  }, []);

  return (
    <div className={`${!visible && '-translate-y-full'} fixed flex w-full h-32 font-bold text-white transition-all`}>
      <NavbarLink name="home" href='/'/>
    </div>
  );
}