IdentityServer4 client - Refreshing access tokens on CookieAuthenticationEvents

I am trying to use refresh token when the access token expires. A similar so question is answered here. And a sample code to renew token by an action

And i end up with the following code in the startup.cs

app.UseCookieAuthentication(new CookieAuthenticationOptions
{
    AuthenticationScheme = "Cookies",
    //ExpireTimeSpan = TimeSpan.FromSeconds(100),
    AutomaticAuthenticate = true,
    AutomaticChallenge = true,
    Events = new CookieAuthenticationEvents()
    {
        OnValidatePrincipal = async x =>
        {
            if (x.Properties?.Items[".Token.expires_at"] == null) return;

            var logger = loggerFactory.CreateLogger(this.GetType());

            var now = DateTimeOffset.UtcNow;
            var tokenExpireTime = DateTime.Parse(x.Properties.Items[".Token.expires_at"]).ToUniversalTime();
            var timeElapsed = now.Subtract(x.Properties.IssuedUtc.Value);
            var timeRemaining = tokenExpireTime.Subtract(now.DateTime);

            if (timeElapsed > timeRemaining)
            {
                var httpContextAuthentication = x.HttpContext.Authentication;//Donot use the HttpContext.Authentication to retrieve anything, this cause recursive call to this event
                var oldAccessToken = await httpContextAuthentication.GetTokenAsync("access_token");
                var oldRefreshToken = await httpContextAuthentication.GetTokenAsync("refresh_token");
                logger.LogInformation($"Refresh token :{oldRefreshToken}, old access token:{oldAccessToken}");


                var disco = await DiscoveryClient.GetAsync(AuthorityServer);
                if (disco.IsError) throw new Exception(disco.Error);

                var tokenClient = new TokenClient(disco.TokenEndpoint, ApplicationId, "secret");
                var tokenResult = await tokenClient.RequestRefreshTokenAsync(oldRefreshToken);
                logger.LogInformation("Refresh token requested. " + tokenResult.ErrorDescription);


                if (!tokenResult.IsError)
                {

                    var oldIdToken = await httpContextAuthentication.GetTokenAsync("id_token");
                    var newAccessToken = tokenResult.AccessToken;
                    var newRefreshToken = tokenResult.RefreshToken;

                    var tokens = new List<AuthenticationToken>
                    {
                        new AuthenticationToken {Name = OpenIdConnectParameterNames.IdToken, Value = oldIdToken},
                        new AuthenticationToken {Name = OpenIdConnectParameterNames.AccessToken, Value = newAccessToken},
                        new AuthenticationToken {Name = OpenIdConnectParameterNames.RefreshToken, Value = newRefreshToken}
                    };

                    var expiresAt = DateTime.UtcNow + TimeSpan.FromSeconds(tokenResult.ExpiresIn);
                    tokens.Add(new AuthenticationToken { Name = "expires_at", Value = expiresAt.ToString("o", CultureInfo.InvariantCulture) });

                    var info = await httpContextAuthentication.GetAuthenticateInfoAsync("Cookies");
                    info.Properties.StoreTokens(tokens);
                    await httpContextAuthentication.SignInAsync("Cookies", info.Principal, info.Properties);

                }
                x.ShouldRenew = true;
            }
            else
            {
                logger.LogInformation("Not expired");
            }
        }

    }
});

The client setup is as follows

AllowAccessTokensViaBrowser = true,
RefreshTokenUsage = TokenUsage.ReUse,
RefreshTokenExpiration = TokenExpiration.Sliding,
AbsoluteRefreshTokenLifetime = 86400,    
AccessTokenLifetime = 10,
AllowOfflineAccess = true,
AccessTokenType = AccessTokenType.Reference

After successfully login, i am getting a 401 for every other request. And the log says

[Identity Server]2017-07-04 10:15:58.819 +01:00 [Debug] "TjpIkvHQi../cfivu6Nql5ADJJlZRuoJV1QI=" found in database: True

[Identity Server]2017-07-04 10:15:58.820 +01:00 [Debug] "reference_token" grant with value: "..9e64c1235c6675fcef617914911846fecd72f7b372" found in store, but has expired.

[Identity Server]2017-07-04 10:15:58.821 +01:00 [Error] Invalid reference token. "{ \"ValidateLifetime\": true,
\"AccessTokenType\": \"Reference\", \"TokenHandle\": \"..9e64c1235c6675fcef617914911846fecd72f7b372\" }"

[Identity Server]2017-07-04 10:15:58.822 +01:00 [Debug] Token is invalid.

[Identity Server]2017-07-04 10:15:58.822 +01:00 [Debug] Creating introspection response for inactive token.

[Identity Server]2017-07-04 10:15:58.822 +01:00 [Information] Success token introspection. Token status: "inactive", for API name: "api1"

Any help would by highly appreciated

UPDATE:

Basically, when the token expires i get a System.StackOverflowException on the following line

var tokenExpireTime = DateTime.Parse(x.Properties.Items[".Token.expires_at"]).ToUniversalTime();

UPDATE 2: Do not use HttpContext.Authentication to retrieve anything. Check my answer below to find the working implementaion