Weird behavior of animating css using scroll event on legacy browsers except firefox

I'm working on a simple gallery using css grid and I'm trying to animate gaps between the items when the user scrolls down. But I don't know why I get this weird behavior on chrome and edge which is every item just shrinks by height when I try to scroll. It works fine in firefox and everything under the hood seems to work fine and i get no error whatsoever and I tried autoprefixer on my css but it didn't help in any way.

<div class="container" id="container">
    ...images goes here (sm,md,lg classes)
</div>

.container {
  display: grid;
  grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
  grid-auto-rows: minmax(10%, 20%);
  grid-auto-flow: dense;
  transition: all 1s;
  transition-delay: 0.1s;
  gap: 1rem;
  padding: 0 1rem;
}

img {
  display: block;
  width: 100%;
  height: 100%;
  object-fit: cover;
  border-radius: 2px;
}
.sm {
  grid-row: span 1;
}
.md {
  grid-row: span 5;
}
.lg {
  grid-row: span 6;
}

.scroll-animation {
  gap: 0.1rem;
  padding: 0 0.1rem;
}

let lastScrolled = 0; // Number of pixels the user has scrolled in the last event
// Basically we want to add "scroll-animation" to the container when the user scrolled down

// This state controlls event calls
let fired = false;

window.addEventListener('scroll', (e) => {
  const scrollable = document.documentElement.scrollHeight - window.innerHeight; // Numbers of pixels the user can scroll
  const scrolled = window.scrollY; // Number of pixels the user has scrolled

  if (fired === false) {
    fired = true;
    // If the user scrolls down the scrolled value increases and vice versa
    // So basically
    if (scrolled < lastScrolled) {
      // Then the user scrolled up
      console.log('GET OUT! YOU SCROLLED UP!');
      // But you should update the lastScrolled value nonetheless
      lastScrolled = scrolled;
      setTimeout(() => {
        // not the best kind of solution
        fired = false;
      }, 500);
      return; // And then get out!
    }

    // Before we continue
    // It gets weird when it reaches the bottom of page so i had to add this fix
    // Checks if the user scrolled all the way to the bottom of page and returns
    if (Math.ceil(scrollable) === scrolled) {
      console.log('STOP!');
      fired = false;
      return;
    }

    // And finally you need a setTimeOut function for this to work
    // Because you need to add and remove the "scroll-animation" class between some kind of time interval

    // Add class as soon as the scroll event starts
    container.classList.add('scroll-animation');

    // And remove it after the time interval
    setTimeout(() => {
      container.classList.remove('scroll-animation');
      fired = false;
      console.log('Scrolling has stopped.');
    }, 1050);
  }
  lastScrolled = scrolled; // The value gets updated in any case
});

Codepen