Attempting to update a Spring-webflux java service to use BouncyCastle for FIPS compliant cryptography would startup, however every request to the service would end up failing with a stack trace like this:
[ctor-http-nio-5] o.b.jsse.provider.ProvTlsServer : Server ignored SNI (no matchers specified)
[ctor-http-nio-5] o.b.jsse.provider.ProvTlsServer : Server specified new session: 3897681a0667834f4be0c7c579976c84863b65c3cb5b572ab2064488b89069e7
[ctor-http-nio-5] o.b.jsse.provider.ProvTlsServer : Server (1.2) selected credentials for signature scheme 'ecdsa_secp521r1_sha512(0x603)' (keyType 'ECDHE_ECDSA'), with private key algorithm 'RSA'
[ctor-http-nio-5] o.b.jsse.provider.ProvTlsServer : Server selected cipher suite: TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
[ctor-http-nio-5] o.b.jsse.provider.ProvTlsServer : Server raised fatal(2) internal_error(80) alert: Failed to process record
java.lang.IllegalStateException: Invalid algorithm: {sha512(6),ecdsa(3)} at org.bouncycastle.tls.crypto.impl.jcajce.JcaTlsRSASigner.generateRawSignature(JcaTlsRSASigner.java:59)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.DefaultTlsCredentialedSigner.generateRawSignature(DefaultTlsCredentialedSigner.java:53)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.TlsUtils.generateServerKeyExchangeSignature(TlsUtils.java:2587)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.TlsECDHEKeyExchange.generateServerKeyExchange(TlsECDHEKeyExchange.java:79)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.TlsServerProtocol.handleHandshakeMessage(TlsServerProtocol.java:1082)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.TlsProtocol.processHandshakeQueue(TlsProtocol.java:695)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.TlsProtocol.processRecord(TlsProtocol.java:584)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.RecordStream.readFullRecord(RecordStream.java:207)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.TlsProtocol.safeReadFullRecord(TlsProtocol.java:890)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.TlsProtocol.offerInput(TlsProtocol.java:1282)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.tls.TlsProtocol.offerInput(TlsProtocol.java:1250)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at org.bouncycastle.jsse.provider.ProvSSLEngine.unwrap(ProvSSLEngine.java:464)
~[bctls-fips-1.0.12.2.jar:1.0.12.2] at java.base/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:637)
~[na:na] at io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:298)
~[netty-handler-4.1.66.Final.jar:4.1.66.Final] at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1344)
~[netty-handler-4.1.66.Final.jar:4.1.66.Final] at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1237)
~[netty-handler-4.1.66.Final.jar:4.1.66.Final] at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286)
~[netty-handler-4.1.66.Final.jar:4.1.66.Final] at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:507)
~[netty-codec-4.1.66.Final.jar:4.1.66.Final] at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:446)
~[netty-codec-4.1.66.Final.jar:4.1.66.Final] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276)
~[netty-codec-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:719)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:655)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:581)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
~[netty-transport-4.1.66.Final.jar:4.1.66.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
~[netty-common-4.1.66.Final.jar:4.1.66.Final] at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
~[netty-common-4.1.66.Final.jar:4.1.66.Final] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
~[netty-common-4.1.66.Final.jar:4.1.66.Final] at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na]