NextAuth refreshing or recognizing expired token from Azure AD

I am having some issues where I can correctly auth to Azure AD and make requests. Azure AD has a token expiration of 1 hour. Once the token expires, my GateKeeper is not recognizing that the token is expired. So the page is served, but any API requests 401

Right now I'm not bothered too much with figuring out if refresh tokens work, but just that I can't get NextAuth to recognize that the token is expired.

So, is this the right flow to be using and are there any obvious oversights when I am initially authorizing the client.

What I expect to happen is that a user is auth'd, once the token expires it tries to refresh the token, if it is unable to refresh it renders the SignIn component.

GateKeeper.js

//GateKeeper.js
// This component wraps any protected page from being shown unless a user is authenticated.
import { useSession, signIn, signOut } from 'next-auth/react'
import { useEffect } from 'react'
import SignIn from '@components/auth/SignIn'

const GateKeeper = (props) => {
  const { data: session, status } = useSession()

  useEffect(() => {
    if (session?.error === "RefreshAccessTokenError") {
      signIn(); // Force sign in to hopefully resolve error
    }
  }, [session, status]);

  // Important note. JS Date returns in miliseconds while the expiry is in seconds
  if (session && session.user && session.user.access_token_expires_at !== null) {
    if (+ new Date() > session.user.access_token_expires_at) {
      signOut()
    }
  }

  if (status === "loading") {
    return null
  }

  if (status === "unauthenticated") {
    return <SignIn />
  }

  if (session) {
    return (
      <>{props.children}</>
    )
  } else {
    return <SignIn />
  }
}

export default GateKeeper

[...nextauth].js

// [...nextauth].js
// Taken from the docs as well as the tutorial on how to implement refresh tokens.
// Note that in my instance, the id_token is actually used to auth the user to API
// Services
import NextAuth from "next-auth"
import AzureADProvider from "next-auth/providers/azure-ad"

async function refreshAccessToken(token) {
  try {
    const url =
      `https://login.microsoftonline.com/${process.env.AZURE_AD_TENANT_ID}/oauth2/v2.0/token` +
      new URLSearchParams({
        client_id: process.env.AZURE_AD_CLIENT_ID,
        client_secret: process.env.AZURE_AD_CLIENT_SECRET,
        scope: 'email openid profile User.Read offline_access',
        grant_type: 'refresh_token',
        refresh_token: token.refreshToken,
      })

    const response = await fetch(url, {
      headers: {
        "Content-Type": "application/x-www-form-urlencoded",
      },
      method: "POST",
    })

    const refreshedTokens = await response.json()

    if (!response.ok) {
      throw refreshedTokens
    }

    return {
      ...token,
      accessToken: refreshedTokens.id_token,
      accessTokenExpires: refreshedTokens.expires_at * 1000,
      refreshToken: refreshedTokens.refresh_token ?? token.refreshToken, // Fall back to old refresh token
    }
  } catch (error) {
    console.log(error)

    return {
      ...token,
      error: "RefreshAccessTokenError",
    }
  }
}

export default NextAuth({
  providers: [
    AzureADProvider({
      clientId: process.env.AZURE_AD_CLIENT_ID,
      clientSecret: process.env.AZURE_AD_CLIENT_SECRET,
      tenantId: process.env.AZURE_AD_TENANT_ID,
      maxAge: 60 * 60,
      authorization: { params: { scope: 'email openid profile User.Read offline_access' } }
    })
  ],
  callbacks: {
    async signIn({ user, account, profile, email, credentials }) {
      return true
    },
    async jwt({ token, user, account, profile, isNewUser }) {
      // Initial sign in
      if (account && user) {
        return {
          accessToken: account.id_token,
          accessTokenExpires: account.expires_at * 1000,
          refreshToken: account.refresh_token,
          user,
        }
      }

      // Return previous token if the access token has not expired yet
      if (Date.now() < token.accessTokenExpires) {
        return token
      }

      // Access token has expired, try to update it
      return refreshAccessToken(token)
    },
    async session({ session, token, user }) {
      session.user = token.user
      session.error = token.error
      session.user.access_token_expires_at = token.accessTokenExpires

      return session
    },
    async redirect({ url, baseUrl }) {
      return baseUrl
    }
  }
})