Is there a way calculate the ending position of an element before css transform matrix is applied?

The issue is that you're mixing two different coordinate systems:

• The client system (used by getBoundingClientRect())
• The local system (used by transform)

As stated in the linked article, the local system's origin is the element's center:

When a transform is applied to an object, it creates a local coordinate system. By default, the origin — the (0,0) point — of the local coordinate system lies at the object's center

Let's call [xc, yc] the element's center.

To transform a point (expressed in the client system), you need to do the following:

• Convert the coordinates from client to local system (by subtracting xc and yc)
• Apply the transform matrix
• Convert the resulting coordinates back from local to client (by adding xc and yc)

Here is a code that transforms the 4 box corners and computes the associated bounding box:

var matrix = [2, 1, -1, 2, 200, 400];

var box = document.querySelector('.box');
var startingRect = box.getBoundingClientRect();
var xc = startingRect.left + startingRect.width/2;
var yc = startingRect.top + startingRect.height/2;

const applyMatrix = (matrix, point) => [
  matrix[0] * point[0] + matrix[2] * point[1] + matrix[4],
  matrix[1] * point[0] + matrix[3] * point[1] + matrix[5]
];
const clientToLocal = (point) => [point[0] - xc, point[1] - yc];
const localToClient = (point) => [point[0] + xc, point[1] + yc];
const transformPoint = (point) => localToClient(applyMatrix(matrix, clientToLocal(point)));

function getBoundingBox(pt1, pt2, pt3, pt4)
{
    var x1 = Math.min(pt1[0], pt2[0], pt3[0], pt4[0]);
    var y1 = Math.min(pt1[1], pt2[1], pt3[1], pt4[1]);
    var x2 = Math.max(pt1[0], pt2[0], pt3[0], pt4[0]);
    var y2 = Math.max(pt1[1], pt2[1], pt3[1], pt4[1]);
    return {x: x1, y: y1, width: x2 - x1, height: y2 - y1};
}

var topLeft = [startingRect.left, startingRect.top];
var topRight = [startingRect.right, startingRect.top];
var bottomLeft = [startingRect.left, startingRect.bottom];
var bottomRight = [startingRect.right, startingRect.bottom];
var transformedBox = getBoundingBox(transformPoint(topLeft), transformPoint(topRight), transformPoint(bottomLeft), transformPoint(bottomRight));
console.log(transformedBox);

box.style.transform = 'matrix('+matrix.join()+')';
var endingRect = box.getBoundingClientRect();
console.log(endingRect);

Output:

{x: 158, y: 400, width: 150, height: 150}
DOMRect {x: 158, y: 400, width: 150, height: 150, top: 400, …}

As mentioned in the comments, you have to simply multiply the matrix (you can do just the 2 x 2 version, and then add the translation variables.) But as mentioned by Olivier, you need to first translate the coordinates, and then translate them back. This is easy enough by just calculating the center.

I would write this as a function from a rectangle and matrix (as a flat array of the six variables used for the CSS transforms) into another rectangle, either a simple {left, right, top, bottom} rectangle or if you want, a DomRect.

You can see it in this snippet (easier to see if you expand it with the "full page" link):

const transformRect = (rect, matrix) => {
  const {left, top, right, bottom} = rect
  const [a, b, c, d, tx, ty] = matrix
  const dx =  (left + right) / 2, dy = (top + bottom) / 2
  const newCorners = [[left, top], [right, top], [right, bottom], [left, bottom]]
    .map (([x, y]) => [
      a * (x - dx) + c * (y - dy) + tx + dx, 
      b * (x - dx) + d * (y - dy) + ty + dy
    ])
  const _left = Math .min (... newCorners .map (p => p [0]))
  const _right = Math .max (... newCorners .map (p => p [0]))
  const _top = Math .min (... newCorners .map (p => p [1]))
  const _bottom = Math .max (... newCorners .map (p => p [1]))

  return DOMRect.fromRect (
    {x: _left, y: _top, width: _right - _left, height: _bottom - _top}
  ) // or just
 // return {x: _left, y: _top, width: _right - _left, height: _bottom - _top}
}

const div = document .getElementById ('d2') 
const rect = div.getBoundingClientRect()
console .log ('Before:' , rect)

const matrix = [2, 1, -1, 2, 200, 400]
div.style.transform = `matrix(${matrix .join (', ')})`

console .log ('After:', transformRect (rect, matrix))

.box {
  height: 50px;
  width: 50px;
  background: green;
  color: white;
}
#d1 {
  background: #ccc;
  position: absolute;
  top: 8;
  left: 8;
}

<div id="d1" class="box">shadow</div>
<div id="d2" class="box">content</div>

...to figure out when the element has left the screen.

If you want to do something when element enters or exits view then Intersection Observer is a better choice.

View following snippet in full page mode.

let observer = new IntersectionObserver((entries) => {
  entries.forEach(entry => {
    if (entry.isIntersecting)
      log.textContent = entry.target.textContent;
  })
});

// set elements to observe
observer.observe(one);
observer.observe(two);

down.addEventListener('click', () => two.scrollIntoView({
  behavior: "smooth",
  block: "end",
  inline: "nearest"
}));

body {
  position: relative;
  height: calc(150vh + 25px);
}

p {
  position: sticky;
  top: 1rem;
}

p #log {
  background-color: yellow;
}

#down {
  position: fixed;
  top: 2rem;
  right: 2rem;
  background-color: skyblue;
  padding: .3rem;
  border-radius: 50%;
  cursor: pointer;
  user-select: none;
}

div {
  height: 50px;
  width: 50px;
  background-color: wheat;
  position: absolute;
  left: 50vw;
}

#one {
  background-color: rgb(250, 165, 165);
  top: 50vh;
  transform: translate(-50%, -50%);
}

#two {
  background-color: lime;
  top: 100%;
  transform: translateX(-50%);
}

<span id="down"></span>
<p>Element in view: <span id="log"></span></p>
<div id="one">One</div>
<div id="two">Two</div>

The API is configurable you can even see how much of the element is visible.