Spring OAuth 2 + JWT Inlcuding additional info JUST in access token

I am able to include additional information into the access token implementing my own TokenEnhancer, but such info is included twice. One in the encoded access_token, and the other in the auth server response.

Long story short! I request an access token with right credentials, and I get this response:

{
    "access_token" : "eyJhbGciOiJSUzI1NiJ9.eyJjbGllbnRJZCI6Ik1ZX0NVU1RPTV9JTkZPX0NMSUVOVCIsInVzZXJfbmFtZSI6IlVTRVIiLCJzY29wZSI6WyJGT08iXSwiZXhwIjoxNTA2MzkwOTM5LCJhdXRob3JpdGllcyI6WyJST0xFX1VTRVIiXSwianRpIjoiZjJkYWFkM2ItYzkzOC00ZjExLWI3ODctMzExZDdlNjYzYzhhIiwiY2xpZW50X2lkIjoid2ViX2FwcCJ9.IdgYRxwZGRPR97nxHpAcJXNWDTShQE1tsg9NsBwlOk8eDWE1B-mjfGTaKiyTO1-m9GBpXnxt2PaOV7AbdLsCZ5xLPUR0_5ehuNB6WCXLSkdac5xbw-rmNdJHTe9gLJizOZAKF6J-_Xo9OOQISKBqliY5vo5y0btqIw4CX6-ukYoWZmwHThwnAsEA_PqGuEXsbXMGz-vqJaSVpvJeEOBNL0KOh-cNxc0ft-rJ3snjPerN_efAiZdFkzxdCeuoGmZvSyHRjYR8kQ3ZqZ5MOunw9YuTvidL1IK5TODHQ2BjiCTpbgDlYx-Oh5UxcYNrPOhD-tBjRuuqDSz8K6ddpke4RQ",
    "token_type" : "bearer",
    "refresh_token" : "eyJhbGciOiJSUzI1NiJ9.eyJjbGllbnRJZCI6Ik1ZX0NVU1RPTV9JTkZPX0NMSUVOVCIsInVzZXJfbmFtZSI6IlVTRVIiLCJzY29wZSI6WyJGT08iXSwiYXRpIjoiZjJkYWFkM2ItYzkzOC00ZjExLWI3ODctMzExZDdlNjYzYzhhIiwiZXhwIjoxNTA4OTM5NzM5LCJhdXRob3JpdGllcyI6WyJST0xFX1VTRVIiXSwianRpIjoiOGU2Zjc0OTEtMmQ3MC00NTUwLThhMDgtZjk0YjkzYTVkYWZmIiwiY2xpZW50X2lkIjoid2ViX2FwcCJ9.MqwMrYrofu7pUQu2mF33__h6M4OWSRrQ-lc8JzTn0DkpJ6a3-yjnjjppZ9fs3KBz_lpRIO8jo--eId449rEjP4M3_9lDRSW9_HyBAvd57OtyUHa5SPM9prD6ReXGCyiIw2gO07euIf-Vp4UHsjoKK0MdtfMmFIWms1JMGFBmzBha8kqKaMxKzppGy-jVdP7384K9oovD20H-NubjScfoO2Crp1cTM-SXc-0v6kwB1qV-cI6HKXmbkoFhbH2bL_nRvXTkLYI-UvRNTNLHzqhcqztLTrszcWa2BjNU2IofsNByFS8BHTDV1vu0BqZA4kfNCJcFJ89tBDt2L8vfFkYezQ",
    "expires_in" : 43199,
    "scope" : "FOO",
    "clientId" : "MY_CUSTOM_INFO_CLIENT",
    "jti" : "f2daad3b-c938-4f11-b787-311d7e663c8a"
}

So I can see clientId included in the response... Now I copy my access_token and I decoded in: https://jwt.io/

And in the payload is included also the clientId...

My question is: How can I remove the addional info from the server response and leave it just in the tokens (access_token and refresh_token).

Please see below the code:

import java.util.Arrays;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.ClassPathResource;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.oauth2.config.annotation.web.configuration.AuthorizationServerConfigurerAdapter;
import org.springframework.security.oauth2.config.annotation.web.configuration.EnableAuthorizationServer;
import org.springframework.security.oauth2.config.annotation.web.configurers.AuthorizationServerEndpointsConfigurer;
import org.springframework.security.oauth2.provider.token.DefaultTokenServices;
import org.springframework.security.oauth2.provider.token.TokenEnhancer;
import org.springframework.security.oauth2.provider.token.TokenEnhancerChain;
import org.springframework.security.oauth2.provider.token.TokenStore;
import org.springframework.security.oauth2.provider.token.store.JwtAccessTokenConverter;
import org.springframework.security.oauth2.provider.token.store.JwtTokenStore;
import org.springframework.security.oauth2.provider.token.store.KeyStoreKeyFactory;

@Configuration
@EnableAuthorizationServer
public class OAuth2AuthorizationServerConfig extends AuthorizationServerConfigurerAdapter {

    @Autowired
    @Qualifier("authenticationManagerBean")
    private AuthenticationManager authenticationManager;

    @Override
    public void configure(AuthorizationServerEndpointsConfigurer endpoints) throws Exception {
        TokenEnhancerChain tokenEnhancerChain = new TokenEnhancerChain();
        tokenEnhancerChain.setTokenEnhancers(
                  Arrays.asList(tokenEnhancer(), accessTokenConverter()));
        endpoints
                .tokenStore(tokenStore())
                .authenticationManager(authenticationManager)
                .tokenEnhancer(tokenEnhancerChain);
    }
    @Override
    public void configure(ClientDetailsServiceConfigurer clients) throws Exception {

       clients.inMemory()
           .withClient("web_app")
           .secret("web_app123")
           .scopes("FOO")
           .autoApprove(true)
           .authorities("FOO_READ", "FOO_WRITE")
           .authorizedGrantTypes("refresh_token", "password");
 }
    @Bean
    public TokenStore tokenStore() {
        return new JwtTokenStore(accessTokenConverter());
    }

    @Bean
    public JwtAccessTokenConverter accessTokenConverter() {
        JwtAccessTokenConverter converter = new JwtAccessTokenConverter();
        KeyStoreKeyFactory keyStoreKeyFactory = 
          new KeyStoreKeyFactory(new ClassPathResource("mykey.jks"), "mykey123".toCharArray());
        converter.setKeyPair(keyStoreKeyFactory.getKeyPair("mykey"));
        return converter;
    }

    @Bean
    @Primary
    public DefaultTokenServices tokenServices() {
        DefaultTokenServices defaultTokenServices = new DefaultTokenServices();
        defaultTokenServices.setTokenStore(tokenStore());
        defaultTokenServices.setSupportRefreshToken(true);
        return defaultTokenServices;
    }

    @Bean
    public TokenEnhancer tokenEnhancer() {
        return new CustomTokenEnhancer();
    }
}

And my CustomTokenEnhancer:

import java.util.HashMap;
import java.util.Map;

import org.springframework.security.oauth2.common.DefaultOAuth2AccessToken;
import org.springframework.security.oauth2.common.OAuth2AccessToken;
import org.springframework.security.oauth2.provider.OAuth2Authentication;
import org.springframework.security.oauth2.provider.token.TokenEnhancer;

import com.mapflow.ms.security.service.UserDetailInfo;

public class CustomTokenEnhancer implements TokenEnhancer {

    @Override
    public OAuth2AccessToken enhance(OAuth2AccessToken accessToken,
            OAuth2Authentication authentication) {
        UserDetailInfo user = (UserDetailInfo) authentication.getPrincipal();
        final Map<String, Object> additionalInfo = new HashMap<String, Object>();

        additionalInfo.put("clientId", user.getClientId());

        ((DefaultOAuth2AccessToken) accessToken)
                .setAdditionalInformation(additionalInfo);

        return accessToken;
    }
}