Switch to SHA512 as default preferred hash algo (config.preferredHashAlgorithm) (#1801)

This affects the preferences of newly generated keys, which by default will
have SHA512 as first hash algo preference.
SHA512 will also be used when signing, as long as the recipient keys declare
support for the algorithm.

src/config/config.js

@@ -26,7 +26,7 @@ export default {
    * @memberof module:config
    * @property {Integer} preferredHashAlgorithm Default hash algorithm {@link module:enums.hash}
    */
-  preferredHashAlgorithm: enums.hash.sha256,
+  preferredHashAlgorithm: enums.hash.sha512,
   /**
    * @memberof module:config
    * @property {Integer} preferredSymmetricAlgorithm Default encryption cipher {@link module:enums.symmetric}

src/crypto/public_key/rsa.js

@@ -26,6 +26,7 @@ import { uint8ArrayToB64, b64ToUint8Array } from '../../encoding/base64';
 import { emsaEncode, emeEncode, emeDecode } from '../pkcs1';
 import enums from '../../enums';
 import { bigIntToNumber, bigIntToUint8Array, bitLength, byteLength, mod, modExp, modInv, uint8ArrayToBigInt } from '../biginteger';
+import hash from '../hash';
 
 const webCrypto = util.getWebCrypto();
 const nodeCrypto = util.getNodeCrypto();
@@ -45,6 +46,14 @@ const _1n = BigInt(1);
  * @async
  */
 export async function sign(hashAlgo, data, n, e, d, p, q, u, hashed) {
+  if (hash.getHashByteLength(hashAlgo) >= n.length) {
+    // Throw here instead of `emsaEncode` below, to provide a clearer and consistent error
+    // e.g. if a 512-bit RSA key is used with a SHA-512 digest.
+    // The size limit is actually slightly different but here we only care about throwing
+    // on common key sizes.
+    throw new Error('Digest size cannot exceed key modulus size');
+  }
+
   if (data && !util.isStream(data)) {
     if (util.getWebCrypto()) {
       try {
@@ -264,9 +273,6 @@ async function bnSign(hashAlgo, n, d, hashed) {
   n = uint8ArrayToBigInt(n);
   const m = uint8ArrayToBigInt(emsaEncode(hashAlgo, hashed, byteLength(n)));
   d = uint8ArrayToBigInt(d);
-  if (m >= n) {
-    throw new Error('Message size cannot exceed modulus size');
-  }
   return bigIntToUint8Array(modExp(m, d, n), 'be', byteLength(n));
 }
 

test/crypto/rsa.js

@@ -121,6 +121,22 @@ export default () => describe('basic RSA cryptography', function () {
     expect(util.uint8ArrayToHex(signatureNative)).to.be.equal(util.uint8ArrayToHex(signatureBN));
   });
 
+  it('compare native crypto and bnSign: throw on key size shorter than digest size', async function() {
+    if (!detectNative()) { this.skip(); }
+
+    const bits = 512;
+    const hashName = 'sha512'; // digest too long for a 512-bit key
+    const { publicParams, privateParams } = await crypto.generateParams(openpgp.enums.publicKey.rsaSign, bits);
+    const { n, e, d, p, q, u } = { ...publicParams, ...privateParams };
+    const message = random.getRandomBytes(64);
+    const hashAlgo = openpgp.enums.write(openpgp.enums.hash, hashName);
+    const hashed = await crypto.hash.digest(hashAlgo, message);
+    enableNative();
+    await expect(crypto.publicKey.rsa.sign(hashAlgo, message, n, e, d, p, q, u, hashed)).to.be.rejectedWith(/Digest size cannot exceed key modulus size/);
+    disableNative();
+    await expect(crypto.publicKey.rsa.sign(hashAlgo, message, n, e, d, p, q, u, hashed)).to.be.rejectedWith(/Digest size cannot exceed key modulus size/);
+  });
+
   it('compare native crypto and bnVerify', async function() {
     if (!detectNative()) { this.skip(); }
 

test/general/key.js

@@ -2261,7 +2261,7 @@ function versionSpecificTests() {
         ]);
       }
       const hash = openpgp.enums.hash;
-      expect(selfSignature.preferredHashAlgorithms).to.eql([hash.sha256, hash.sha512, hash.sha3_256, hash.sha3_512]);
+      expect(selfSignature.preferredHashAlgorithms).to.eql([hash.sha512, hash.sha256, hash.sha3_256, hash.sha3_512]);
       const compr = openpgp.enums.compression;
       expect(selfSignature.preferredCompressionAlgorithms).to.eql([compr.uncompressed, compr.zlib, compr.zip]);
 
@@ -2495,7 +2495,7 @@ function versionSpecificTests() {
   });
 
   it('Generate RSA key - two subkeys with default values', async function() {
-    const rsaBits = 512;
+    const rsaBits = 1024;
     const minRSABits = openpgp.config.minRSABits;
     openpgp.config.minRSABits = rsaBits;
 
@@ -2601,7 +2601,7 @@ function versionSpecificTests() {
   });
 
   it('Generate key - override main RSA key options for subkey', async function() {
-    const rsaBits = 512;
+    const rsaBits = 1024;
     const minRSABits = openpgp.config.minRSABits;
     openpgp.config.minRSABits = rsaBits;
 

test/general/openpgp.js

@@ -1457,7 +1457,7 @@ VFBLG8uc9IiaKann/DYBAJcZNZHRSfpDoV2pUA5EAEi2MdjxkRysFQnYPRAu
 
     beforeEach(async function() {
       minRSABitsVal = openpgp.config.minRSABits;
-      openpgp.config.minRSABits = 512;
+      openpgp.config.minRSABits = 1024;
     });
 
     afterEach(function() {
@@ -3894,7 +3894,9 @@ XfA3pqV4mTzF
           signingKeys: privateKey_1337,
           detached: true,
           date: past,
-          format: 'binary'
+          format: 'binary',
+          // SHA-512 cannot be used with a 512-bit RSA key (digest too long)
+          config: { minRSABits: 512, preferredHashAlgorithm: openpgp.enums.hash.sha256 }
         };
         const verifyOpt = {
           message,