RuntimeException when closing CipherInputStream

Viewed 1844

I kind of got stuck with this exception:

java.lang.RuntimeException: error:0407806d:RSA routines:decrypt:DATA_LEN_NOT_EQUAL_TO_MOD_LEN
 at com.android.org.conscrypt.NativeCrypto.RSA_private_decrypt(Native Method)
 at com.android.org.conscrypt.OpenSSLCipherRSA.engineDoFinal(OpenSSLCipherRSA.java:274)
 at javax.crypto.Cipher.doFinal(Cipher.java:1440)
 at javax.crypto.CipherInputStream.close(CipherInputStream.java:190)
 ...

It is thrown when I close the CipherInputStream on Android Marshmallow. Everything seems to work with earlier Android Versions.

What does DATA_LEN_NOT_EQUAL_TO_MOD_LEN mean? Why does it seem to decrypt (call to RSA_private_decrypt) when it should free resource handles (close)?

UPDATE:

I managed to reproduce the error with some test code. It encrypts and decrypts "foobar". One time using the cipher directly and one time through a CipherInputStream (like it's done in the original app).

Everything works on android < 6 and the non-streaming code is even working on android 6. I was able to get the streaming code to work on android 6 when I changed the explicit cipher RSA/ECB/PKCS1Padding to generic RSA. But I would bet that it's there for a reason ;)

static final String RSA_ALGO = "RSA/ECB/PKCS1Padding";
//  static final String RSA_ALGO = "RSA";

private void _testCrypto2() throws Exception {
  KeyPairGenerator keyGen;
  KeyPair          keys;
  byte[]           encrypted;
  byte[]           decrypted;
  String           input;
  String           output;

  keyGen = KeyPairGenerator.getInstance("RSA");
  keyGen.initialize(2048);
  keys = keyGen.generateKeyPair();

  input = "foobar";

  // Plain crypto.
  encrypted = this.RSAEncrypt(input, keys.getPublic());
  output    = this.RSADecrypt(encrypted, keys.getPrivate());

  // Streaming crypto.
  encrypted = this.RSAEncryptStream(input, keys.getPublic());
  output    = this.RSADecryptStream(encrypted, keys.getPrivate());
}

public byte[] RSAEncrypt(final String plain, PublicKey _publicKey) throws Exception {
  byte[] encryptedBytes;
  Cipher cipher;

  cipher = Cipher.getInstance(RSA_ALGO);
  cipher.init(Cipher.ENCRYPT_MODE, _publicKey);
  encryptedBytes = cipher.doFinal(plain.getBytes());

  return encryptedBytes;
}

public String RSADecrypt(final byte[] encryptedBytes, PrivateKey _privateKey) throws Exception {
  Cipher cipher;
  byte[] decryptedBytes;
  String decrypted;

  cipher = Cipher.getInstance(RSA_ALGO);
  cipher.init(Cipher.DECRYPT_MODE, _privateKey);

  decryptedBytes = cipher.doFinal(encryptedBytes);
  decrypted      = new String(decryptedBytes);

  return decrypted;
}

public byte[] RSAEncryptStream(final String _plain, PublicKey _publicKey) throws Exception {
  Cipher                cipher;
  InputStream           in;
  ByteArrayOutputStream out;
  int                   numBytes;
  byte                  buffer[] = new byte[0xffff];

  in     = new ByteArrayInputStream(_plain.getBytes());
  out    = new ByteArrayOutputStream();
  cipher = Cipher.getInstance(RSA_ALGO);
  cipher.init(Cipher.ENCRYPT_MODE, _publicKey);

  try {
    in = new CipherInputStream(in, cipher);
    while ((numBytes = in.read(buffer)) != -1) {
      out.write(buffer, 0, numBytes);
    }
  }
  finally {
    in.close();
  }

  return out.toByteArray();
}

public String RSADecryptStream(final byte[] _encryptedBytes, PrivateKey _privateKey) throws Exception {
  Cipher                cipher;
  InputStream           in;
  ByteArrayOutputStream out;
  int                   numBytes;
  byte                  buffer[] = new byte[0xffff];

  in     = new ByteArrayInputStream(_encryptedBytes);
  out    = new ByteArrayOutputStream();  
  cipher = Cipher.getInstance(RSA_ALGO);
  cipher.init(Cipher.DECRYPT_MODE, _privateKey);

  try {
    in = new CipherInputStream(in, cipher);
    while ((numBytes = in.read(buffer)) != -1) {
      out.write(buffer, 0, numBytes);
    }
  }
  finally {
    in.close();
  }

  return new String(out.toByteArray());
}

However, it looks like there are two directions for a fix:

  • Getting rid of the streaming for RSA
  • Removing explicit RSA cipher instantiation

What do you think?

1 Answers
Related