RecyclerView SelectionTracker unable to keep track after navigating back to fragment

In my scenario, I have two fragments. In my first Fragment I have a RecyclerView with SelectionTracker. After selecting an item in my first fragment I can navigate to the second fragment. But from my second fragment if I navigate back to the first fragment using back button all my previous selection is lost. How can I prevent this? I am using Jetpack Navigation Component to handle Fragment navigation.

Here is my fragment

class SelectVideoForCourseFragment : Fragment(R.layout.fragment_select_video_for_course) {

    .....
    ..

    private val adapter by lazy { MediaListAdapter() }

    private lateinit var selectionTracker: SelectionTracker<String>

    override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
        super.onViewCreated(view, savedInstanceState)
        initializeListeners()
        initializeRecyclerView(savedInstanceState)
        fetchMedias()
    }

    override fun onSaveInstanceState(outState: Bundle) {
        super.onSaveInstanceState(outState)
        selectionTracker.onSaveInstanceState(outState)
    }

    private fun fetchMedias() {
        val mediaRepo = MediaRepo()
        mediaRepo.getVideos(contentResolver).observe(viewLifecycleOwner) {
            adapter.submitItems(it)
        }
    }

    private fun initializeRecyclerView(savedInstanceState: Bundle?) {

        binding.mediaList.layoutManager = GridLayoutManager(requireContext(), 3)
        binding.mediaList.addItemDecoration(GridSpacingItemDecoration(3, 5.dp, false))
        binding.mediaList.adapter = adapter

        selectionTracker = SelectionTracker.Builder(
            "tracker-select-video-for-course",
            binding.mediaList,
            MediaItemKeyProvider(adapter),
            MediaItemDetailsLookup(binding.mediaList),
            StorageStrategy.createStringStorage()
        ).withSelectionPredicate(
            SelectionPredicates.createSelectSingleAnything()
        ).build()
        if (savedInstanceState != null) {
            selectionTracker.onRestoreInstanceState(savedInstanceState)
        }
        adapter.tracker = selectionTracker
    }

    ......
    ...
}

My RecyclerView Adapter:

class MediaListAdapter: RecyclerView.Adapter<RecyclerView.ViewHolder>() {

    val differ = AsyncListDiffer(this, DIFF_CALLBACK_MEDIA)

    var tracker: SelectionTracker<String>? = null

    fun submitItems(updatedItems: List<Any>) {
        differ.submitList(updatedItems)
    }

    fun getSelections(): List<MediaEntity> {
        ....
        ..
    }

    override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): RecyclerView.ViewHolder {
        return if (viewType == 1) {
            val binding = ListItemImageBinding.inflate(
                LayoutInflater.from(parent.context),
                parent,
                false
            )
            ViewHolderImage(binding)
        } else {
            val binding = ListItemVideoBinding.inflate(
                LayoutInflater.from(parent.context),
                parent,
                false
            )
            ViewHolderVideo(binding)
        }
    }

    override fun onBindViewHolder(holder: RecyclerView.ViewHolder, position: Int) {
        if (holder is ViewHolderImage) {

            tracker?.let {
                val item = differ.currentList[position] as ImageEntity
                holder.bind(item , it.isSelected(item.uri.toString()))
            }

        } else {

            holder as ViewHolderVideo
            val item = differ.currentList[position] as VideoEntity

            tracker?.let {
                holder.bind(item, it.isSelected(item.uri.toString()))
            }
        }
    }

    override fun getItemCount(): Int = differ.currentList.size

    override fun getItemViewType(position: Int): Int {
        return if (differ.currentList[position] is ImageEntity) {
            1
        } else {
            2
        }
    }

    inner class ViewHolderImage(
        private val binding: ListItemImageBinding): RecyclerView.ViewHolder(binding.root
    ) {

        fun bind(item: ImageEntity, isSelected: Boolean) {
            .....
            ...
        }
        

        fun getItemDetails(): ItemDetailsLookup.ItemDetails<String> =
            object : ItemDetailsLookup.ItemDetails<String>() {
                override fun getPosition(): Int = absoluteAdapterPosition
                override fun getSelectionKey(): String {
                    val item = differ.currentList[absoluteAdapterPosition] as ImageEntity
                    return item.uri.toString()
                }
                override fun inSelectionHotspot(e: MotionEvent): Boolean = true
            }
    }

    inner class ViewHolderVideo(
        private val binding: ListItemVideoBinding,
    ): RecyclerView.ViewHolder(binding.root) {


        fun bind(item: VideoEntity, isSelected: Boolean) {
            .....
            ...
        }

        fun getItemDetails(): ItemDetailsLookup.ItemDetails<String> =
            object : ItemDetailsLookup.ItemDetails<String>() {
                override fun getPosition(): Int = absoluteAdapterPosition
                override fun getSelectionKey(): String {
                    val item = differ.currentList[absoluteAdapterPosition] as VideoEntity
                    return item.uri.toString()
                }

                override fun inSelectionHotspot(e: MotionEvent): Boolean = true
            }
    }

    companion object {
        val DIFF_CALLBACK_MEDIA = object: DiffUtil.ItemCallback<Any>() {
            .....
            ...
        }
    }

}