Svelte list crossfade randomly misplaces elements

I have a list which I want to animate so that when an item is selected, it floats to top and the rest fly out. On reselecting the element it floats back to its place and others fly back in.

Using this crossfade demo I was able to produce a pretty solid version:

REPL

However, I noticed that the elements sometimes aren’t returned to their original position and only a subsequent re-animating might fix it. I am pretty sure this is because my crossfade function needs more sophisticated handling (or that I should use just regular transition instead?) but I am unsure how exactly.

<script>
  import { flip } from 'svelte/animate'
  import { writable } from 'svelte/store'
  import { crossfade, fly } from 'svelte/transition'

  const ACTIONS = [
    { key: 'blue' },
    { key: 'green' },
    { key: 'yellow' },
    { key: 'purple' },
    { key: 'orange' },
    { key: 'red' },
  ]
  const selectedAction = writable(null)
  let shownActions = ACTIONS.map(a => a)

  const [send, receive] = crossfade({
    fallback(node, _params) {
      return fly(node, { x: -800, duration: 600 })
    }
  })

  function selectAction(action) {
    if ($selectedAction === action) {
      shownActions = ACTIONS.map(a => a)
      selectedAction.update(v => {
        if (v === action || action === undefined) {
          return null
        } else {
          return action
        }
      })
    } else {
      shownActions = ACTIONS.filter(a => a.key === action)
      selectedAction.update(v => {
        if (v === action || action === undefined) {
          return null
        } else {
          return action
        }
      })
      window.scrollTo({
        top: 0,
        behavior: 'smooth'
      })
    }
  }
</script>

<ul>
{#each shownActions as action (action)}
  <li
    class={`${$$props.class || ''}`}
    animate:flip={{ duration: 600 }}
    in:receive={{ key: action.key }}
    out:send={{ key: action.key }}
  >
    <button on:click={e => selectAction(action.key)}>{action.key}</button>
  </li>
{/each}
</ul>

<style lang="scss">
    ul {
        list-style: none;
    }
    li {
        display: flex;
        margin-bottom: 1rem;
    }
    button {
        margin-right: 1rem;
        padding: 0.5rem 1rem;
    }
</style>