Using Intersection observer to highlight active link isn't consistent

I have a aside section with anchor links on the left and content on the right. What I want to do is to highlight the anchor link when the user scrolls to a particular text that's anchored.

I am using Intersection Observer for this feature. However, the problem is that on scroll it's all working fine, but when you click the anchor, the active class doesn't trigger properly. That's because the paragraphs that I am observing are all different sizes.

Some of them are taller, some smaller. So if I have 2 paragraphs that are small and right close to each other, the class doesn't trigger on a link which I clicked.

I think I'd need some dynamic thershold or margins but I am not sure how to do this.

For example, on this site it works really well even though the paragpraphs are close to each other: https://n26.com/en-eu/blog/bike-tours-europe#france-champagne-region

 async highlightAnchorOnScroll() {
      if (window.matchMedia('(min-width:768px)')) {
        await new Promise((resolve) =>
          setTimeout(() => {
            let observerOptions = {
              threshold: 0.5,
              rootMargin: '0px 0px -50% 0px',
            }

            const list = document.querySelectorAll('.b-anchor')

            this.$options.observer = new IntersectionObserver(
              observerCallback,
              observerOptions
            )
            const asideLinks = document.querySelectorAll('.aside ul li')

            function observerCallback(entries) {
              entries.forEach((entry, index) => {
               

                if (entry.isIntersecting) {
                  asideLinks.forEach((el) => {
                    if (
                      entry.target.previousSibling.children[0].innerText.trim() ===
                      el.innerText.trim()
                    ) {
                      el.classList.add('activeLink')
                    } else {
                      el.classList.remove('activeLink')
                    }
                  })
                }
              })
            }

            Array.from(list).forEach((element, index) => {
              this.$options.observer.observe(element.nextElementSibling)
            })
          }, 200)
        )
      }
    },

Code when clicking on anchor

 scrollToElement(refName) {
  const test = document.getElementById(refName)
  const y = window.pageYOffset + test.getBoundingClientRect().top - 200
  // document.getElementById(refName).getBoundingClientRect().top - 200

  window.scroll({
    top: y,
    behavior: 'smooth',
  })
},