Javascript - scaled down div suddenly "moves away" from mouse cursor during dragging

const scaleDownBtn = document.getElementById('scaleDown')
scaleDownBtn.addEventListener('click', scaleDown)

function scaleDown(event) {
  const selfViewVideo = document.querySelector('.self-video-div')
  selfViewVideo.classList.add("resize-div")
}

const resetBtn = document.getElementById('reset')
resetBtn.addEventListener('click', reset)

function reset(event) {
  const selfViewVideo = document.querySelector('.self-video-div')
  selfViewVideo.classList.remove("resize-div")
}

// dragging implemented here
const selfViewVideoDiv = document.querySelector('.self-video-div')
selfViewVideoDiv.addEventListener('mousedown', dragDiv)

function dragDiv(event) {
  // we bind event handlers to the window element because when we drag an
  // element the mouse could actaully be slightly outside it. If we don't do this,
  // the drag would end immediately after the mouse would leave the div boundaries
  window.addEventListener("mousemove", mouseMove)
  window.addEventListener("mouseup", mouseUp)
  // 'this' points to element the dragDiv event handler was binded to.
  // i.e. the html div selfViewVideoDiv, which is the parent of the <video>
  // tag showing local video from the webcam
  const selfViewVideoDiv = this
  // mouse x and y coordinates when the user clicked on the selfViewVideoDiv 
  let prevX = event.clientX
  let prevY = event.clientY
  console.info(`mouseDown prevX: ${prevX}, prevY ${prevY}`)

  // real div dragging is implemented inside mouseMove
  function mouseMove(event) {
    // newX and newY are actually deltas, i.e. the difference between old x/y pair
    // and new x/y pair
    let newX = event.clientX - prevX
    let newY = event.clientY - prevY
    console.info(`mouseMove deltaX: ${newX}, deltaY ${newY}`)
    let rect = selfViewVideoDiv.getBoundingClientRect()
    console.info(`div X is actually: ${rect.left} `)
    console.info(`div Y is actually: ${rect.top} `)
    // we just need to assign the delta values to the selfVideoDiv because they are actually summed to the existing
    // distance between selfViewVideoDiv and its parent element, i.e. div with class "video-row".
    // e.g.: if we set style.left to 25px, this 25 px are added to the exisisting distance, they
    // are not absolute, ie. taken from the left limit of the viewport. This is due to the fact
    // we styled self-video-div with position-relative  
    selfViewVideoDiv.style.left = newX + "px"
    selfViewVideoDiv.style.top = newY + "px"

    rect = selfViewVideoDiv.getBoundingClientRect()
    console.info(`div X is now: ${rect.left} `)
    console.info(`div Y is now: ${rect.top} `)

  }

  // when end user lifts the mouse, we need to remove the event handler otherwise the div would 
  // be dragged along forever
  function mouseUp(event) {

    window.removeEventListener("mousemove", mouseMove)
    window.removeEventListener("mouseup", mouseUp)
  }

}

.remote-video-div {
  background-color: coral;
}

.self-video-div {
  background-color: green;
  transition: transform 1s;
}

.resize-div {
  transform: scale(.30);
  transform-origin: bottom right;
}

<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.1/dist/css/bootstrap.min.css" rel="stylesheet" />

<div class="container my-3">
  <div class="row position-relative video-row">
    <div class="col position-absolute remote-video-div">
      <video id="remote-video" autoplay="autoplay"></video>
    </div>
    <div class="col position-relative overlay border self-video-div">
      <video id="self-view-video" autoplay="autoplay" poster=""></video>
    </div>
  </div>
  <div class="row">
    <div class="col text-center">
      <button type="button" class="btn btn-outline-primary btn-lg my-3" id="scaleDown">
        <span>Scale down div</span>
      </button>
      <button type="button" class="btn btn-outline-primary btn-lg my-3" id="reset">
        <span>Reset</span>
      </button>
    </div>
  </div>
</div>
</div>