Flutter: RouterDelegate build method called recursively

I get the build method of the RouterDelegate called recursively if I scroll the page for some reasons and this causes the UI to lag in any page, as if the page is being rebuilt from scratch every time I scroll it.

This is happening since I moved to Navigator 2

class AppRouterDelegate extends RouterDelegate<AppRoutePath>
    with ChangeNotifier, PopNavigatorRouterDelegateMixin<AppRoutePath> {

  final AppState appState;
  final UserState userState;

  @override
  final GlobalKey<NavigatorState> navigatorKey = GlobalKey<NavigatorState>();

  AppRouterDelegate(this.appState, this.userState);

  Page _buildPage(NutmegPage p, Widget w) {
    return MaterialPage(
        key: ValueKey(p.toString().split(".").last),
        name: p.toString().split(".").last,
        child: w);
  }

  List<Page> _buildPages() {
    List<NutmegPage> stack;
    if (!appState.loadingDone)
     stack = List<NutmegPage>.of([NutmegPage.LAUNCH]);
    else {
      stack = appState.stack;

      if (!userState.isLoggedIn() && stack.last == NutmegPage.CREATE_MATCH)
        stack.add(NutmegPage.LOGIN);
    }

    return stack.map((p) {
      switch (p) {
        case NutmegPage.LAUNCH: return _buildPage(p, LaunchWidget());
        case NutmegPage.CREATE_MATCH: return _buildPage(p, CreateMatch());
        case NutmegPage.LOGIN: return _buildPage(p, Login());
        case NutmegPage.UNKNOWN: return null;
      }
    }).toList();
  }

  @override
  Widget build(BuildContext context) {
    print("rebuilding stack: $appState");
    return Navigator(
      key: navigatorKey,
      pages: _buildPages(),
      onPopPage: (route, result) {
        print("popping " + route.settings.name);

        if (!route.didPop(result)) {
          return false;
        }
        appState.removeLastFromStack();
        return true;
      },
    );
  }
}

and state and configuration are

class AppState extends ChangeNotifier {

  bool loadingDone = false;
  List<NutmegPage> stack = List<NutmegPage>.of([NutmegPage.HOME]);
  String selectedMatch;

  void setStack(List<NutmegPage> stack, [String matchId]) {
   this.stack = stack;
   this.selectedMatch = matchId;
   print("notifying set stack");
   notifyListeners();
  }

  void addToStack(NutmegPage p, [String matchId]) {
    this.stack.add(p);
    this.selectedMatch = matchId;
    print("notifying add to stack");
    notifyListeners();
  }

  void removeLastFromStack() {
    this.stack.removeLast();
    print("notifying remove last from stack");
    notifyListeners();
  }

  void setLoadingDone() {
    loadingDone = true;
    notifyListeners();
  }
}


class AppRoutePath {
  final String matchId;
  final NutmegPage page;

  AppRoutePath(this.page, [this.matchId]);
}

I am currently not using URL syncing methods. I would first like to fix this behaviour.

In the logs I see for example many

rebuilding stack: AppState{stack: [NutmegPage.HOME, NutmegPage.USER], selectedMatch: null, loadingDone: true}

when I go to User page