Sort recursive array of objects

I think you're missing sorting on b.Children. And you can compare a and b by using a.sortOrder - b.sortOrder

This is comparator function:

const comparator = (a: node, b: node) => {
    if (a.Children && a.Children.length > 1) {
        a.Children.sort(comparator)
    }
    if (b.Children && b.Children.length > 1) {
        b.Children.sort(comparator)
    }
    return a.sortOrder - b.sortOrder;
}

And make sorting nodes by:

nodes.sort(comparator)

You can see my full example here

As it is already mentioned it is a bad idea to have side effects in a comparator function, Try this code instead:

function compareSortOrder(a, b) {
    if (a.sortOrder < b.sortOrder) {
        return -1;
    }
    if (a.sortOrder > b.sortOrder) {
        return 1;
    }
    return 0;
}

function sort(array, compareSortOrder) {
    if (array && array.length > 1) {
        array.sort(compareSortOrder);
        for (const item of array) {
            sort(item.children, compareSortOrder);
        }
    }
}
sort(nodes, compareSortOrder)

Alternatively, you can try this, instead of mutating in the comparator

const nodes = [{
    sortOrder: 2,
    children: [{
      sortOrder: 2,
      children: []
    }, {
      sortOrder: 1,
      children: []
    }]
  },
  {
    sortOrder: 1,
    children: []
  }
]

function compareSortOrder(a, b) {
  return a.sortOrder - b.sortOrder;
}

function sort(nodes){
  for(const node of nodes){
    if(node.children.length){
      sort(node.children);
    }
  }
  nodes.sort(compareSortOrder);
}

sort(nodes);

console.log(nodes)

If you debugged your function, you would have seen an error

Error: Cannot read property 'compareSortOrder' of undefined

So, passing the function reference 'this.compareSortOrder' to the sort function results in an instance reference loss within your compareSortOrder function.

You can use the bind() function for this:

nodes.sort(this.compareSortOrder.bind(this));

private compareSortOrder(a, b) {
  if(a.children && a.children.length) {
    a.children.sort(this.compareSortOrder.bind(this));
  }

  if(b.children && b.children.length) {
      b.children.sort(this.compareSortOrder.bind(this));
  }
  // ...
}

Or provide an anonymous function to the sort function in order to preserve the this reference.

nodes.sort((n1, n2) => this.compareSortOrder(n1, n2));

private compareSortOrder(a, b) {
    if(a.children && a.children.length) {
        a.children.sort((n1, n2) => this.compareSortOrder(n1, n2));
    }

    if(b.children && b.children.length) {
        b.children.sort((n1, n2) => this.compareSortOrder(n1, n2));
    }
    // ...
}

One more thing, you should also consider the case where the sortOrder might be equal. Further, if you want to have a consistent sorting, you'ld have to use an additional sorting attribute e.g. id or similar

if (a.sortOrder === b.sortOrder) {
 return 0;
 // or 
 // return a.id > b.id ? 1 : -1; 
} else {
 return a.sortOrder > b.sortOrder ? 1 : -1;
}