Select2 unslected options

I am using select2 with pagination to load more data from db when user scroll the drop down, the issue i am facing is when i select all the records and then i unselect some record then some of them show in dropdown and some not.

$(field).select2({
      allowClear: true,
      placeholder: 'Select Records',
      width: 'resolve',
      closeOnSelect: false,
      ajax: {
        url: `/programs/${field.dataset.programId}/fetch_records`,
        type: 'POST',
        dataType: 'json',
        beforeSend: function(xhr) {
          xhr.setRequestHeader('X-CSRF-Token', $('meta[name="csrf-token"]').attr('content'));
        },
        data: function (params) {
          $(document).on('manual-append-end', function(event) {
            return;
          });

          var query = {
            term: params.term,
            page:  params.page || 1
          }

          return query;
        },
        processResults: function (data, params) {
          debugger
          params.page = params.page || 1;
          var dropdown = $('#select2-manual_scheduler_record_ids-results')
          var existingOptions = dropdown.find('li[aria-selected="false"]').toArray();
          var existingResults = existingOptions.map(function(option) {
            return {
              id: $(option).val(),
              text: $(option).text()
            };
          });
          var combinedResults = existingResults.concat(data.results);
          return {
            results: combinedResults,
            pagination: {
              more: dropdown.find('li[aria-selected="false"]').length < 1 ? data.pagination.more : false
            }
          };
        },
        cache: true
      },
      minimumInputLength: 0
    })

  $('.select2').on('select2:unselect', function(e) {
    debugger
    var optionElement = new Option(e.params.data.text, e.params.data.id, false, false);

    $(e.target).append(optionElement).trigger('change');

    // Trigger a custom event to signal that the manual append is complete
    $(e.target).trigger('manual-append-end');
  });

 $('.select2').on('select2:unselect', function(e) {
    var unselectedValue = e.params.data.id;

    var programId = $(this).data('program-id');
    $.ajax({
      url: `/programs/${programId}/fetch_records`,
      type: 'POST',
      dataType: 'json',
      data: { term: unselectedValue, bulk: true },
      beforeSend: function(xhr) {
        xhr.setRequestHeader('X-CSRF-Token', $('meta[name="csrf-token"]').attr('content'));
      },
      success: function(response) {
        var newOption = response.results.find(option => option.id == unselectedValue);
        if (newOption) {
          var optionElement = new Option(newOption.text, newOption.id, false, false);
          $(e.target).append(optionElement).trigger('change');
        }
      }
    });
  });

I have tried all this but nothing is working. It would be very helpfull if some give some solution, the above solutions i tried work but after that when i open the select box the created option disappear du to the execution of data part