Switch to SHA512 as default preferred hash algo (config.preferredHashAlgorithm)

SHA512 is usually faster than SHA256 on 64-bit platforms.

SHA-512 cannot be used with RSA/DSA keys shorter than 512-bits (which are insecure, but may
be used e.g. for testing), so error messages have been improved for this edge case.

src/config/config.js

@@ -26,7 +26,7 @@ export default {
    * @memberof module:config
    * @property {Integer} preferredHashAlgorithm Default hash algorithm {@link module:enums.hash}
    */
-  preferredHashAlgorithm: enums.hash.sha256,
+  preferredHashAlgorithm: enums.hash.sha512,
   /**
    * @memberof module:config
    * @property {Integer} preferredSymmetricAlgorithm Default encryption cipher {@link module:enums.symmetric}

src/crypto/public_key/rsa.js

@@ -26,6 +26,7 @@ import { uint8ArrayToB64, b64ToUint8Array } from '../../encoding/base64';
 import { emsaEncode, emeEncode, emeDecode } from '../pkcs1';
 import enums from '../../enums';
 import { bigIntToNumber, bigIntToUint8Array, bitLength, byteLength, mod, modExp, modInv, uint8ArrayToBigInt } from '../biginteger';
+import hash from '../hash';
 
 const webCrypto = util.getWebCrypto();
 const nodeCrypto = util.getNodeCrypto();
@@ -45,6 +46,14 @@ const _1n = BigInt(1);
  * @async
  */
 export async function sign(hashAlgo, data, n, e, d, p, q, u, hashed) {
+  if (hash.getHashByteLength(hashAlgo) >= n.length) {
+    // Throw here instead of `emsaEncode` below, to provide a clearer and consistent error
+    // e.g. if a 512-bit RSA key is used with a SHA-512 digest.
+    // The size limit is actually slightly different but here we only care about throwing
+    // on common key sizes.
+    throw new Error('Digest size cannot exceed key modulus size');
+  }
+
   if (data && !util.isStream(data)) {
     if (util.getWebCrypto()) {
       try {
@@ -264,9 +273,6 @@ async function bnSign(hashAlgo, n, d, hashed) {
   n = uint8ArrayToBigInt(n);
   const m = uint8ArrayToBigInt(emsaEncode(hashAlgo, hashed, byteLength(n)));
   d = uint8ArrayToBigInt(d);
-  if (m >= n) {
-    throw new Error('Message size cannot exceed modulus size');
-  }
   return bigIntToUint8Array(modExp(m, d, n), 'be', byteLength(n));
 }
 

test/crypto/rsa.js

@@ -121,6 +121,22 @@ export default () => describe('basic RSA cryptography', function () {
     expect(util.uint8ArrayToHex(signatureNative)).to.be.equal(util.uint8ArrayToHex(signatureBN));
   });
 
+  it('compare native crypto and bnSign: throw on key size shorter than digest size', async function() {
+    if (!detectNative()) { this.skip(); }
+
+    const bits = 512;
+    const hashName = 'sha512'; // digest too long for a 512-bit key
+    const { publicParams, privateParams } = await crypto.generateParams(openpgp.enums.publicKey.rsaSign, bits);
+    const { n, e, d, p, q, u } = { ...publicParams, ...privateParams };
+    const message = random.getRandomBytes(64);
+    const hashAlgo = openpgp.enums.write(openpgp.enums.hash, hashName);
+    const hashed = await crypto.hash.digest(hashAlgo, message);
+    enableNative();
+    await expect(crypto.publicKey.rsa.sign(hashAlgo, message, n, e, d, p, q, u, hashed)).to.be.rejectedWith(/Digest size cannot exceed key modulus size/);
+    disableNative();
+    await expect(crypto.publicKey.rsa.sign(hashAlgo, message, n, e, d, p, q, u, hashed)).to.be.rejectedWith(/Digest size cannot exceed key modulus size/);
+  });
+
   it('compare native crypto and bnVerify', async function() {
     if (!detectNative()) { this.skip(); }
 

test/general/key.js

@@ -2261,7 +2261,7 @@ function versionSpecificTests() {
         ]);
       }
       const hash = openpgp.enums.hash;
-      expect(selfSignature.preferredHashAlgorithms).to.eql([hash.sha256, hash.sha512, hash.sha3_256, hash.sha3_512]);
+      expect(selfSignature.preferredHashAlgorithms).to.eql([hash.sha512, hash.sha256, hash.sha3_256, hash.sha3_512]);
       const compr = openpgp.enums.compression;
       expect(selfSignature.preferredCompressionAlgorithms).to.eql([compr.uncompressed, compr.zlib, compr.zip]);
 
@@ -2495,7 +2495,7 @@ function versionSpecificTests() {
   });
 
   it('Generate RSA key - two subkeys with default values', async function() {
-    const rsaBits = 512;
+    const rsaBits = 1024;
     const minRSABits = openpgp.config.minRSABits;
     openpgp.config.minRSABits = rsaBits;
 
@@ -2601,7 +2601,7 @@ function versionSpecificTests() {
   });
 
   it('Generate key - override main RSA key options for subkey', async function() {
-    const rsaBits = 512;
+    const rsaBits = 1024;
     const minRSABits = openpgp.config.minRSABits;
     openpgp.config.minRSABits = rsaBits;
 

test/general/openpgp.js

@@ -1457,7 +1457,7 @@ VFBLG8uc9IiaKann/DYBAJcZNZHRSfpDoV2pUA5EAEi2MdjxkRysFQnYPRAu
 
     beforeEach(async function() {
       minRSABitsVal = openpgp.config.minRSABits;
-      openpgp.config.minRSABits = 512;
+      openpgp.config.minRSABits = 1024;
     });
 
     afterEach(function() {
@@ -3845,7 +3845,9 @@ XfA3pqV4mTzF
           signingKeys: privateKey_1337,
           detached: true,
           date: past,
-          format: 'binary'
+          format: 'binary',
+          // SHA-512 cannot be used with a 512-bit RSA key (digest too long)
+          config: { minRSABits: 512, preferredHashAlgorithm: openpgp.enums.hash.sha256 }
         };
         const verifyOpt = {
           message,