Flutter firebase auth web not persistent

I have the following auth provider, and I can sign the user in and the app state changes. But when I do a sign in I notice that nothing is stored in the local storage and when I refresh the page the authStateChanges gives null for the user. I also just tried tho get the current user as a Future which also returns null. What am I doing wrong?

const String oneTimePasswordFunctionName = "requestOneTimePassword";
const String signInWithOneTimePasswordFunctionName = "signInWithOneTimePassword";

class FirebaseAuthProvider extends AuthProvider {
  final FirebaseAuth _firebaseAuth = FirebaseAuth.instance;
  final FirebaseFunctions _firebaseFunctions = FirebaseFunctions.instance;

  Stream<AuthUser> get user {
    return _firebaseAuth.authStateChanges().asyncMap((firebaseUser) {
      //firebaseUser is always null on refresh
      return firebaseUser == null ? AuthUser.empty : firebaseUser.toUser();
    });
  }

  Future<void> requestOneTimePassword({required String email}) async {
    final data = {"email": email};
    final cloudFunction = _firebaseFunctions.httpsCallable(oneTimePasswordFunctionName);
    await cloudFunction.call(data);
  }

  Future<void> signInWithOneTimePassword({required String email, required String oneTimePassword}) async {
    final data = {"email": email, "oneTimePassword": oneTimePassword};
    final cloudFunction = _firebaseFunctions.httpsCallable(signInWithOneTimePasswordFunctionName);
    final result = await cloudFunction.call(data);
    final jwt = result.data["data"]["jwt"] as String;
    if (jwt.isEmpty) return;
    await FirebaseAuth.instance.setPersistence(Persistence.LOCAL);
    await _firebaseAuth.signInWithCustomToken(jwt);
  }

  Future<void> signOut() async {
    try {
      await _firebaseAuth.signOut();
    } on Exception {
      throw SignOutException();
    }
  }
}

extension on User {
  FutureOr<AuthUser> toUser() async {
    final tokenResult = await this.getIdTokenResult(true);
    final claims = tokenResult.claims?.map<String, String>((key, value) => MapEntry(key, value.toString()));

    return AuthUser(
      id: uid,
      email: Email.dirty(email ?? ''),
      name: displayName ?? '',
      photoURL: Uri.parse(photoURL ?? ''),
      claims: claims ?? const {},
    );
  }
}

I really need it to work with the authStateChanges stream, because I want it also to work when the user does a sign out.

I am using the firebase auth emulator. This is my main

void main() async {
  WidgetsFlutterBinding.ensureInitialized();

  await FirebaseAuth.instance.useAuthEmulator("localhost", 9099);

  FirebaseFirestore.instance.settings = Settings(
    host: "localhost:8080",
    sslEnabled: false,
    persistenceEnabled: false,
  );

  FirebaseFunctions.instance.useFunctionsEmulator("localhost", 5001);

  Future<dynamic> Function() initializeFirebase = () async {
    if (Firebase.apps.length == 0) await Firebase.initializeApp();
  };

  await Future.wait([
    initializeFirebase(),
    EasyLocalization.ensureInitialized(),
  ]);

  runApp(_buildAppWithDependencies());
}

I am testing in the latest version of google chrome

Versions

flutter: 2.5.3 (stable)

firebase_core: ^1.10.0

firebase_auth: ^3.2.0