Remove element class with a matching id of radio button if unchecked

I’ve been trying to find an answer but it isn’t giving me any answers and all I found is for checkboxes I tried this but .removeClass doesn’t work plus I have to copy and paste every single thing. I know there’s a more compact way how do it but i don’t know how to

$("#group1").change(function() {
  if ($(this).is(":checked")) {
    $(".group1").addClass("red");
  } else {
    $(".group2").removeClass("orange");
  }
});

$("#group2").change(function() {
  if ($(this).is(":checked")) {
    $(".group2").addClass("orange");
  } else {
    $(".group2").removeClass("orange");
  }
});

div {
  width: 100px;
  height: 100px;
  border: 1px solid
}

.red {
  background-color: red
}

.orange {
  background-color: orange
}

<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>

<input type="radio" name="numbers" value="first" id="group1" />
<label for="id1">1</label>

<input type="radio" name="numbers" value="second" id="group2" />
<label for="id2">2</label>

<div class="group1"></div>
<div class="group2"></div>