WSO2 Service Providers not Populating in API Manager UI but it is Reading it from Database (RDS - postgres)

While configuring fresh installs of wso2 API-Manager and Identity server, we have encountered this issue. Identity providers added through the API UI will populate in the IS UI, but service providers added through the IS console do not appear on API.

The two servers are running on EC2 instances with all traffic opened between them and we have verified on a different installation that the certs and keystores being used should not be causing any problems. We have not seen any specific connection refused issues in the logs, either, and the tcpdump shows that the two servers are communicating back and forth with each other. Using the working installation as a reference, there also does not appear to be any discrepancies from the deployment.tomls.

Both EC2s share the same RDS postgres database, but what is interesting is that if you try to manually add the service provider in the API UI, it will say that a provider with the same name already exists (meaning that it is reaching out to the database and finding the providers created in IS but it's still not populating them in the list in API console).

We've really been stumped by this one so any help would be greatly appreciated. Even trace/debug logs have not been helpful, but just let me know of any screenshots that could aid in remedying this problem. Thank you.

Edits: WSO2 AM is version 3.1.0, WSO2 IS is version 5.10.0. Both shared_db and the am_dbs are shared across both instances

deployment.toml for IS

[server]
hostname = 
node_ip = "127.0.0.1"
base_path = "https://$ref{server.hostname}:${carbon.management.port}"
serverDetails = "WSO2 IS as KM 5.10.0"
mode = "single"
userAgent = "WSO2 IS as KM 5.10.0"

[super_admin]
username = 
password = 
create_admin_account = true

[user_store]
type = "database_unique_id"
[user_store.properties]
IsEmailUserName = false
MaxRoleNameListLength = 101
SharedGroupEnabled = false
UsernameJavaRegEx = "[a-zA-Z0-9@._\\+-|//]{3,50}$"
UsernameJavaScriptRegEx = "^[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,4}$"
UsernameWithEmailJavaScriptRegEx = "[a-zA-Z0-9@._+-|//]{3,50}$"
UserRolesCacheEnabled = false



[database.apim_db]
type = "postgre"
url = "jdbc:postgresql://wso2_am?autoReconnect=true&relaxAutoCommit=true&"
username = 
password = 
driver = "org.postgresql.Driver"

[database.shared_db]
type = "postgre"
url = "jdbc:postgresql://wso2_shared?autoReconnect=true&relaxAutoCommit=true&"
username = 
password = 
driver = "org.postgresql.Driver"

[database.config]
type = "postgre"
url = "jdbc:postgresql://wso2_shared?autoReconnect=true&relaxAutoCommit=true&"
username = 
password = 
driver = "org.postgresql.Driver"

[database.user]
type = "postgre"
url = "jdbc:postgresql://wso2_user?autoReconnect=true&relaxAutoCommit=true&"
username = 
password = 
driver = "org.postgresql.Driver"

[realm_manager]
data_source = "WSO2USER_DB"

[database.apim_db.pool_options]
 maxActive = "80"
 maxWait = "60000"
 minIdle = "5"
 testOnBorrow = true
 validationQuery="SELECT 1; COMMIT"
 validationInterval="30000"
 defaultAutoCommit=true
[database.shared_db.pool_options]
 maxActive = "80"
 maxWait = "60000"
 minIdle = "5"
 testOnBorrow = true
 validationQuery="SELECT 1; COMMIT"
 validationInterval="30000"
 defaultAutoCommit=true


[keystore.primary]
name = "wso2carbon.jks"
password = "wso2carbon"

[[event_listener]]
id = "mutual_tls_authenticator"
type = "org.wso2.carbon.identity.core.handler.AbstractIdentityHandler"
name = "org.wso2.carbon.identity.oauth2.token.handler.clientauth.mutualtls.MutualTLSClientAuthenticator"
order = "158"
enable = false
ssl_enabled_protocols = ["TLSv1.2"]

[[apim.throttling.url_group]]
traffic_manager_urls = ["tcp://localhost:9611"]
traffic_manager_auth_urls = ["ssl://localhost:9711"]
type = "loadbalance"
[admin_service.wsdl]
enable= true
[tenant_mgt]
enable_email_domain= true
[identity_mgt_account_suspension]
use_identity_claims = false

[[apim.gateway.environment]]
name = "Production and Sandbox"
type = "hybrid"
description = "This is a hybrid gateway that handles both production and sandbox token traffic."
service_url = 
username= 
password= 
display_in_api_console = true
ws_endpoint="ws://${carbon.local.ip}:9099"
https_endpoint=
[apim.oauth_config]
revoke_endpoint = 
[apim.key_manager]
service_url = 
username = 
password = 

[apim.jwt]
enable = true
encoding = "base64" # base64,base64url
generator_impl = "org.wso2.carbon.apimgt.keymgt.token.JWTGenerator"
claim_dialect = "http://wso2.org/claims"
header = "X-JWT-Assertion"
signing_algorithm = "SHA256withRSA"
enable_user_claims = true
claims_extractor_impl = "org.wso2.carbon.apimgt.impl.token.DefaultClaimsRetriever"
[transport.http.properties]
compression = "off"

[transport.https.properties]
compression = "off"

deployment.toml for APIM

[server]
hostname = 
node_ip = "127.0.0.1"
#offset=0
serverDetails = "WSO2 IS as KM 5.10.0"
mode = "single" #single or ha
base_path = "${carbon.protocol}://${carbon.host}:${carbon.management.port}"
#discard_empty_caches = false
server_role = "default"
userAgent = "WSO2 IS as KM 5.10.0"

[super_admin]
username = 
password = 
create_admin_account = true

[user_store]
type = "database_unique_id"

[database.apim_db]
type = "postgre"
url = "jdbc:postgresql://wso2_am?autoReconnect=true&relaxAutoCommit=true&"
username = 
password = 
driver = "org.postgresql.Driver"

[database.shared_db]
type = "postgre"
url = "jdbc:postgresql://wso2_shared?autoReconnect=true&relaxAutoCommit=true&"
username = 
password = 
driver = "org.postgresql.Driver"

[database.apim_db.pool_options]
maxActive = "80"
maxWait = "60000"
minIdle = "5"
testOnBorrow = true
validationQuery = "SELECT 1; COMMIT"
validationInterval = "30000"
defaultAutoCommit = true

[database.shared_db.pool_options]
maxActive = "80"
maxWait = "60000"
minIdle = "5"
testOnBorrow = true
validationQuery = "SELECT 1; COMMIT"
validationInterval = "30000"
defaultAutoCommit = true

[database.config]
type = "postgre"
url = "jdbc:postgresql://wso2_shared?autoReconnect=true&relaxAutoCommit=true&"
username = 
password = 
driver = "org.postgresql.Driver"

[database.user]
type = "postgre"
url = "jdbc:postgresql://wso2_user?autoReconnect=true&relaxAutoCommit=true&"
username = 
password = 
driver = "org.postgresql.Driver"

[keystore.tls]
file_name =  "wso2carbon.jks"
type =  "JKS"
password =  "wso2carbon"
alias =  "wso2carbon"
key_password =  "wso2carbon"

#[keystore.primary]
#file_name =  "wso2carbon.jks"
#type =  "JKS"
#password =  "wso2carbon"
#alias =  "wso2carbon"
#key_password =  "wso2carbon"

#[keystore.internal]
#file_name =  "wso2carbon.jks"
#type =  "JKS"
#password =  "wso2carbon"
#alias =  "wso2carbon"
#key_password =  "wso2carbon"

[[apim.gateway.environment]]
name = "Production and Sandbox"
type = "hybrid"
display_in_api_console = true
description = "This is a hybrid gateway that handles both production and sandbox token traffic."
show_as_token_endpoint_url = true
service_url = 
username= 
password= 
ws_endpoint = 
wss_endpoint = 
http_endpoint = 
https_endpoint = 

[apim.cache.gateway_token]
enable = false
#expiry_time = "900s"

#[apim.cache.resource]
#enable = true
#expiry_time = "900s"

#[apim.cache.km_token]
#enable = false
#expiry_time = "15m"

#[apim.cache.recent_apis]
#enable = false

#[apim.cache.scopes]
#enable = true

#[apim.cache.publisher_roles]
#enable = true

#[apim.cache.jwt_claim]
#enable = true
#expiry_time = "15m"

#[apim.cache.tags]
#expiry_time = "2m"

#[apim.analytics]
#enable = false
#store_api_url = "https://localhost:7444"
#username = "$ref{super_admin.username}"
#password = "$ref{super_admin.password}"
#event_publisher_type = "default"
#event_publisher_impl = "org.wso2.carbon.apimgt.usage.publisher.APIMgtUsageDataBridgeDataPublisher"
#publish_response_size = true

#[[apim.analytics.url_group]]
#analytics_url =["tcp://analytics1:7611","tcp://analytics2:7611"]
#analytics_auth_url =["ssl://analytics1:7711","ssl://analytics2:7711"]
#type = "loadbalance"

#[[apim.analytics.url_group]]
#analytics_url =["tcp://analytics1:7612","tcp://analytics2:7612"]
#analytics_auth_url =["ssl://analytics1:7712","ssl://analytics2:7712"]
#type = "failover"

[apim.key_manager]
service_url = 
username = 
password = 
#pool.init_idle_capacity = 50
#pool.max_idle = 100
#key_validation_handler_type = "default"
#key_validation_handler_type = "custom"
#key_validation_handler_impl = "org.wso2.carbon.apimgt.keymgt.handlers.DefaultKeyValidationHandler"

#[apim.idp]
#authorize_endpoint = "https://localhost:${mgt.transport.https.port}/oauth2/authorize"
#oidc_logout_endpoint = "https://localhost:${mgt.transport.https.port}/oidc/logout"

[apim.jwt]
enable = true
encoding = "base64" # base64,base64url
generator_impl = "org.wso2.carbon.apimgt.keymgt.token.JWTGenerator"
claim_dialect = "http://wso2.org/claims"
header = "X-JWT-Assertion"
signing_algorithm = "SHA256withRSA"
enable_user_claims = true
claims_extractor_impl = "org.wso2.carbon.apimgt.impl.token.DefaultClaimsRetriever"

[apim.oauth_config]
#enable_outbound_auth_header = false
#auth_header = "Authorization"
revoke_endpoint = 
#enable_token_encryption = false
#enable_token_hashing = false

[apim.devportal]
url = 
#enable_application_sharing = false
#if application_sharing_type, application_sharing_impl both defined priority goes to application_sharing_impl
#application_sharing_type = "default" #changed type, saml, default #todo: check the new config for rest api
#application_sharing_impl = "org.wso2.carbon.apimgt.impl.SAMLGroupIDExtractorImpl"
#display_multiple_versions = false
#display_deprecated_apis = false
#enable_comments = true
#enable_ratings = true
#enable_forum = true

[apim.cors]
allow_origins = "*"
allow_methods = ["GET","PUT","POST","DELETE","PATCH","OPTIONS"]
allow_headers = ["authorization","Access-Control-Allow-Origin","Content-Type","SOAPAction","apikey"]
allow_credentials = false

#[apim.throttling]
#enable_data_publishing = true
#enable_policy_deploy = true
#enable_blacklist_condition = true
#enable_persistence = true
#throttle_decision_endpoints = ["tcp://localhost:5672","tcp://localhost:5672"]

#[apim.throttling.blacklist_condition]
#start_delay = "5m"
#period = "1h"

#[apim.throttling.jms]
#start_delay = "5m"

#[apim.throttling.event_sync]
#hostName = "0.0.0.0"
#port = 11224

#[apim.throttling.event_management]
#hostName = "0.0.0.0"
#port = 10005

#[[apim.throttling.url_group]]
#traffic_manager_urls = ["tcp://localhost:9611","tcp://localhost:9611"]
#traffic_manager_auth_urls = ["ssl://localhost:9711","ssl://localhost:9711"]
#type = "loadbalance"

#[[apim.throttling.url_group]]
#traffic_manager_urls = ["tcp://localhost:9611","tcp://localhost:9611"]
#traffic_manager_auth_urls = ["ssl://localhost:9711","ssl://localhost:9711"]
#type = "failover"

#[apim.workflow]
#enable = false
#service_url = "https://localhost:9445/bpmn"
#username = "$ref{super_admin.username}"
#password = "$ref{super_admin.password}"
#callback_endpoint = "https://localhost:${mgt.transport.https.port}/api/am/admin/v0.16/workflows/update-workflow-status"
#token_endpoint = "https://localhost:${https.nio.port}/token"
#client_registration_endpoint = "https://localhost:${mgt.transport.https.port}/client-registration/v0.16/register"
#client_registration_username = "$ref{super_admin.username}"
#client_registration_password = "$ref{super_admin.password}"

#data bridge config
#[transport.receiver]
#type = "binary"
#worker_threads = 10
#session_timeout = "30m"
#keystore.file_name = "$ref{keystore.tls.file_name}"
#keystore.password = "$ref{keystore.tls.password}"
#tcp_port = 9611
#ssl_port = 9711
#ssl_receiver_thread_pool_size = 100
#tcp_receiver_thread_pool_size = 100
#ssl_enabled_protocols = ["TLSv1","TLSv1.1","TLSv1.2"]
#ciphers = ["SSL_RSA_WITH_RC4_128_MD5","SSL_RSA_WITH_RC4_128_SHA"]

#[apim.notification]
#from_address = "APIM.com"
#username = "APIM"
#password = "APIM+123"
#hostname = "localhost"
#port = 3025
#enable_start_tls = false
#enable_authentication = true

#[apim.token.revocation]
#notifier_impl = "org.wso2.carbon.apimgt.keymgt.events.TokenRevocationNotifierImpl"
#enable_realtime_notifier = true
#realtime_notifier.ttl = 5000
#enable_persistent_notifier = true
#persistent_notifier.hostname = "https://localhost:2379/v2/keys/jti/"
#persistent_notifier.ttl = 5000
#persistent_notifier.username = "root"
#persistent_notifier.password = "root"

[[event_handler]]
name="userPostSelfRegistration"
subscriptions=["POST_ADD_USER"]

[service_provider]
sp_name_regex = "^[\\sa-zA-Z0-9._-]*$"

[database.local]
url = "jdbc:h2:./repository/database/WSO2CARBON_DB;DB_CLOSE_ON_EXIT=FALSE"

[admin_service.local]
enable = true