React state with JSX elements is setting the last ref in ref array to null

I have a piano key component. When a key is pressed a visual is created, which is set in a state “visual” so that the component rerenders with the new visual. This visual is a div which is also assigned a ref so that I can animate it easily. I have a counter ref to key into the “visual” state and “visual_ref” ref. After 3 seconds, I delete both the ref and the visual for the div as it is offscreen. Deleting the ref works fine, but deleting the visual deletes the corresponding visual, but also sets the last visual in the visual_ref to null. I do not know why

visual_ref.current 3 Visuals created. visual_ref.current[counter] is correctly deleted, however set_visuals deletion of its JSX element sets the last index of visual_ref.current to null. This is not because the last ref was assigned to it

PianoKey Component

 const audio = useRef(null)

  let visual_refs = useRef([])

  let curr_animation = useRef([null, true])

  let [visuals, set_visuals] = useState([])
  let glowline = useRef(null)

  let counter = useRef(0)

  useEffect(() => {
    console.log(visual_refs.current)
    if (visuals[counter.current] && curr_animation.current[1]) {
      curr_animation.current = [attribute_animation(visual_refs.current[counter.current], 'height', '0', '300000px', 1000000), false]
    }
    
  }, [visuals])


  useEffect(() => {
    if (pressed) {
      audio.current.play()

      set_visuals(prev_state => {
        curr_animation.current[1] = true

        return ({
          ...prev_state,
          [counter.current]: (
          <div key={`${counter.current}`} ref={ref => visual_refs.current[counter.current] = ref} 
          className={`visualizer-instance ${color === 'black' ? 'black-visualizer': ''}`}></div>
          )
      })}) 
      
      attribute_animation(glowline.current, 'opacity', '0', '1', 600, 'cubic-bezier(0,.99,.26,.99)')

    } else if (!pressed && pressed !== null && counter.current in visual_refs.current && curr_animation.current[0]) {

        curr_animation.current[0].pause()
        attribute_animation(visual_refs.current[counter.current], 'bottom', '0', '300000px', 1000000)
        attribute_animation(glowline.current, 'opacity', '1', '0', 3000, 'cubic-bezier(.19,.98,.24,1.01)')

        let curr_counter = counter.current
        setTimeout(() => {
          delete visual_refs.current[curr_counter]
          // BELOW DELETES EXTRA VISUAL REF
          set_visuals(prev_state => {
            const new_state = {...prev_state}

            delete new_state[curr_counter]
            return new_state
          })
        }, 3000, curr_counter)
          
        counter.current += 1
    }
  }, [pressed])