Publish Vulpes Preview 15 sources for desktop, Android and Tundra

Share Gaia assets and adapters, keep private profiles and development documents local, and update both public READMEs.

assets/webapps/sync.gaiamobile.org/shared/js/sync/bootstrap.js

@@ -0,0 +1,152 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+'use strict';
+
+/* global
+  DUMP,
+  ERROR_SYNC_APP_SYNC_IN_PROGRESS,
+  ERROR_SYNC_INVALID_REQUEST_OPTIONS,
+  IACHandler,
+  LazyLoader,
+  SyncEngine
+*/
+
+/* exported
+  Bootstrap,
+  DataAdapters
+*/
+
+var DataAdapters = {
+  // To be filled by js/adapters/*.js
+};
+
+if (!window.DUMP) {
+  window.DUMP = () => {};
+}
+
+const DEBUG = (msg, args) => {
+  DUMP(`[Sync] ${msg}`, args);
+};
+
+const Bootstrap = (() => {
+  var running = false;
+
+  const loadMainScripts = () => {
+    return LazyLoader.load([
+      'shared/js/sync/crypto/stringconversion.js',
+      'shared/js/sync/crypto/keyderivation.js',
+      'shared/js/sync/crypto/fxsyncwebcrypto.js',
+
+      'shared/js/sync/ext/kinto.min.js',
+      'shared/js/sync/engine.js'
+    ]);
+  };
+
+  const loadDataAdapterScript = (collectionName) => {
+    return LazyLoader.load(`js/adapters/${collectionName}.js`);
+  };
+
+  const loadErrorConstants = () => {
+    return LazyLoader.load('shared/js/sync/errors.js');
+  };
+
+  /**
+    * @param request {Object} Should contain the following fields:
+    *                         * URL {String} Like 'http://localhost:8000/v1/'.
+    *                         * assertion {String} A BrowserID assertion.
+    *                         * keys {Object} Like `{ kA: 'foo', kB: 'bar' }`.
+    *                         * collections {Object} Like `{ history: {} }`.
+    */
+  const handleSyncRequest = (request) => {
+    if ((typeof request !== 'object') ||
+        (typeof request.URL !== 'string') ||
+        (typeof request.assertion !== 'string') ||
+        (typeof request.keys !== 'object') ||
+        (typeof request.keys.kB !== 'string') ||
+        (typeof request.collections !== 'object')) {
+      return loadErrorConstants().then(() => {
+        throw new Error(ERROR_SYNC_INVALID_REQUEST_OPTIONS);
+      });
+    }
+
+    if (running) {
+      return loadErrorConstants().then(() => {
+        throw new Error(ERROR_SYNC_APP_SYNC_IN_PROGRESS);
+      });
+    }
+
+    running = true;
+    return loadMainScripts().then(() => {
+      const collectionNames = Object.keys(request.collections);
+      return Promise.all(collectionNames.map(collectionName => {
+        return loadDataAdapterScript(collectionName);
+      }));
+    }).then(() => {
+      var syncEngine = new SyncEngine({
+        kB: request.keys.kB,
+        URL: request.URL,
+        assertion: request.assertion,
+        adapters: DataAdapters
+      });
+
+      return syncEngine.syncNow(request.collections);
+    }).then(() => {
+      running = false;
+    }).catch(err => {
+      running = false;
+      throw err;
+    });
+  };
+
+  const sendPortMessage = message => {
+    var port = IACHandler.getPort('gaia::sync::request', this);
+    if (port) {
+      DEBUG('Port message', message);
+      port.postMessage(message);
+    } else {
+      console.error('No gaia::sync::request port');
+    }
+  };
+
+  window.addEventListener('iac-gaia::sync::request', event => {
+    if (!event || !event.detail || !event.detail.id) {
+      console.error('Wrong IAC request');
+      window.close();
+      return;
+    }
+
+    const request = event.detail;
+
+    DEBUG('Sync request', request);
+
+    switch (request.name) {
+      case 'sync':
+        handleSyncRequest(request).then(() => {
+          sendPortMessage({
+            id: request.id
+          });
+          window.close();
+        }).catch(error => {
+          DEBUG('Sync request error', error.message || error.name || error);
+          sendPortMessage({
+            id: request.id,
+            error: { message: error.message }
+          });
+          window.close();
+        });
+        break;
+      case 'cancel':
+        console.warn('Closing app');
+        window.close();
+        break;
+      default:
+        console.error('Unknown IAC request');
+        window.close();
+    }
+  });
+
+  // Expose Bootstrap for unit testing:
+  return { handleSyncRequest };
+})();

assets/webapps/sync.gaiamobile.org/shared/js/sync/crypto/fxsyncwebcrypto.js

@@ -0,0 +1,307 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+'use strict';
+
+/* global
+  crypto,
+  KeyDerivation,
+  StringConversion
+*/
+
+/* exported
+  FxSyncWebCrypto
+*/
+
+// WebCrypto-based client for Firefox Sync.
+
+const FxSyncWebCrypto = Object.freeze((() => {
+
+  const HKDF_INFO_STR = 'identity.mozilla.com/picl/v1/oldsync';
+
+  const importKeyBundle = (aesKeyAB, hmacKeyAB) => {
+    const pAes = crypto.subtle.importKey('raw', aesKeyAB,
+                                       { name: 'AES-CBC', length: 256 },
+                                       true, [ 'encrypt', 'decrypt' ]);
+    const pHmac =  crypto.subtle.importKey('raw', hmacKeyAB,
+                                         { name: 'HMAC', hash: 'SHA-256' },
+                                         true, [ 'sign', 'verify' ]);
+    return Promise.all([pAes, pHmac]).then(results => {
+      return Object.freeze({
+        aes: results[0],
+        hmac: results[1]
+      });
+    }, () => {
+      // err might be a TypeError if aesKeyAB or hmacKeyAB was no ArrayBuffer.
+      throw new Error('Could not import key bundle.');
+    });
+  };
+
+  const importKb = function(kBByteArray) {
+    // The number 64 here comes from
+    // (256 bits for AES + 256 bits for HMAC) / (8 bits per byte).
+    return KeyDerivation.hkdf(kBByteArray,
+                              StringConversion.stringToUtf8Uint8Array(
+                                  HKDF_INFO_STR),
+                              new Uint8Array(64), 64).then(output => {
+      const aesKeyAB = output.slice(0, 32).buffer;
+      const hmacKeyAB = output.slice(32).buffer;
+      return Object.freeze(importKeyBundle(aesKeyAB, hmacKeyAB));
+    });
+  };
+
+  const verifySyncKeys = function(
+      signedTextByteArray,
+      cryptoKeysHmacByteArray,
+      hmacKey) {
+    return crypto.subtle.verify({ name: 'HMAC', hash: 'AES-256' },
+        hmacKey, cryptoKeysHmacByteArray, signedTextByteArray);
+  };
+
+  const importSyncKeys = function(
+      cryptoKeysIVByteArray,
+      cryptoKeysCiphertextByteArray,
+      aesKey) {
+    return crypto.subtle.decrypt({ name: 'AES-CBC', iv: cryptoKeysIVByteArray },
+                                 aesKey,
+                                 cryptoKeysCiphertextByteArray)
+        .then(keyBundleAB => {
+      const cryptoKeysJSON = String.fromCharCode.apply(
+          null,
+          new Uint8Array(keyBundleAB));
+      var bulkKeyBundle;
+      try {
+        bulkKeyBundle = JSON.parse(cryptoKeysJSON);
+        return importKeyBundle(
+            StringConversion.base64StringToUint8Array(
+                bulkKeyBundle.default[0]),
+            StringConversion.base64StringToUint8Array(
+                bulkKeyBundle.default[1])).then(keyBundle => {
+          bulkKeyBundle.defaultAsKeyBundle = keyBundle;
+          return Object.freeze(bulkKeyBundle);
+        });
+      } catch(e) {
+        throw new Error('Deciphered crypto keys, but not JSON');
+      }
+    }, () => {
+      throw new Error(`Could not decrypt crypto keys using AES part of stretche\
+d kB key`);
+    });
+  };
+
+  const importFromStrings = (obj) => {
+    const ret = {};
+    try {
+      ret.ciphertext = StringConversion.
+        base64StringToUint8Array(obj.ciphertext);
+    } catch (e) {
+      throw new Error('Could not parse ciphertext as a base64 string');
+    }
+
+    // Intentionally using StringConversion.stringToUtf8Uint8Array
+    // instead of StringConversion.base64StringToUint8Array on the ciphertext
+    // here - see https://github.com/mozilla/firefox-ios/blob/ \
+    // 1cce59c8eac282e151568f1204ffbbcc27349eff/Sync/KeyBundle.swift#L178.
+    ret.hmacSignedText = StringConversion
+      .stringToUtf8Uint8Array(obj.ciphertext);
+
+    try {
+      ret.IV = StringConversion.base64StringToUint8Array(obj.IV);
+    } catch (e) {
+      throw new Error('Could not parse IV as a base64 string');
+    }
+    try {
+      ret.hmacSignature = StringConversion.hexStringToUint8Array(obj.hmac);
+    } catch (e) {
+      throw new Error('Could not parse hmac as a hex string');
+    }
+    return Object.freeze(ret);
+  };
+
+  const getBulkKeyBundle = (kBByteArray, cryptoKeys) => {
+    return importKb(kBByteArray).then(mainSyncKey => {
+      return verifySyncKeys(cryptoKeys.hmacSignedText,
+                                  cryptoKeys.hmacSignature,
+                                  mainSyncKey.hmac).then(verified => {
+        if (verified) {
+          return importSyncKeys(cryptoKeys.IV,
+                                cryptoKeys.ciphertext,
+                                mainSyncKey.aes);
+        }
+        throw new Error(new Error(`SyncKeys hmac could not be verified with cur\
+rent main key`));
+      });
+    });
+  };
+
+  /*
+   * setKeys - import kB and crypto/keys
+   *
+   * @param {String} kB Hex string with kB from FxA onepw protocol
+   * @param {Object} cryptoKeysStrings Object with:
+   *     - ciphertext {String} A Base64 String containing an AES-CBC ciphertext
+   *     - IV {String} A Base64 String containing the AES-CBC Initialization
+   *         Vector
+   *     - hmac {String} A Hex String containing the HMAC-SHA256 signature
+   * @returns {Promise} A promise that will resolve after importing kB,
+   *     decrypting cryptoKeys, and setting this.bulkKeyBundle.
+   */
+  const setKeys = function(kB, cryptoKeysStrings) {
+    var kBByteArray, cryptoKeys;
+
+    // Input checking.
+    try {
+      kBByteArray = StringConversion.hexStringToUint8Array(kB);
+    } catch (e) {
+      return Promise.reject(new Error('Could not parse kB as a hex string'));
+    }
+
+    try {
+      cryptoKeys = importFromStrings(cryptoKeysStrings);
+    } catch(err) {
+      return Promise.reject(err);
+    }
+
+    return getBulkKeyBundle(kBByteArray, cryptoKeys).then(bulkKeyBundle => {
+      this.bulkKeyBundle = bulkKeyBundle;
+    });
+  };
+
+  /*
+   * decrypt - verify and decrypt a Weave Basic Object
+   *
+   * @param {Object} payloadStrings Object with:
+   *     - ciphertext {String} A Base64 String containing an AES-CBC ciphertext
+   *     - IV {String} A Base64 String containing the AES-CBC Initialization
+   *         Vector
+   *     - hmac {String} A Hex String containing the HMAC-SHA256 signature
+   * @param {String} collectionName String The name of the Sync collection
+   *     (currently ignored, see
+   *     https://github.com/michielbdejong/fxsync-webcrypto/issues/19)
+   * @returns {Promise} A promise for the decrypted Weave Basic Object.
+   */
+  const decrypt = function(payloadStrings, collectionName) {
+    if (typeof payloadStrings !== 'object') {
+      return Promise.reject(new Error('PayloadStrings is not an object'));
+    }
+    if (typeof collectionName !== 'string') {
+      return Promise.reject(new Error('collectionName is not a string'));
+    }
+
+    var keyBundle;
+
+    try {
+      keyBundle = this.bulkKeyBundle.defaultAsKeyBundle;
+    } catch(e) {
+      return Promise.reject(new Error(`No key bundle found for \
+${collectionName} - did you call setKeys?`));
+    }
+
+    const payload = importFromStrings(payloadStrings);
+    return crypto.subtle.verify({ name: 'HMAC', hash: 'SHA-256' },
+                                keyBundle.hmac, payload.hmacSignature,
+                                payload.hmacSignedText).then(result => {
+      if (!result) {
+        throw new Error(`Record verification failed with current hmac key for \
+${collectionName}`);
+      }
+    }).then(() => {
+      return crypto.subtle.decrypt({ name: 'AES-CBC', iv: payload.IV },
+                                   keyBundle.aes,
+                                   payload.ciphertext)
+          .then(recordArrayBuffer => {
+        var recordObj;
+        // recordArrayBuffer which is the cleartext payload from FxSync contains
+        // UTF-8 JSON.
+        const recordJSON = StringConversion.utf8Uint8ArrayToString(
+            new Uint8Array(recordArrayBuffer));
+        try {
+          recordObj = JSON.parse(recordJSON);
+        } catch(e) {
+          throw new Error('Deciphered record, but not JSON');
+        }
+        return recordObj;
+      }, () => {
+        throw new Error(`Could not decrypt record using AES part of key bundle \
+for collection ${collectionName}`);
+      });
+    });
+  };
+
+  const encryptAndSign = (keyBundle, cleartext) => {
+    // Generate a random IV using the PRNG of the device.
+    var IV = new Uint8Array(16);
+    crypto.getRandomValues(IV);
+    return crypto.subtle.encrypt({ name: 'AES-CBC', iv: IV }, keyBundle.aes,
+                                 cleartext).then(ciphertext => {
+      const ciphertextB64 = StringConversion.arrayBufferToBase64String(
+          ciphertext);
+      return crypto.subtle.sign({ name: 'HMAC', hash: 'SHA-256' },
+                                keyBundle.hmac,
+                                StringConversion.stringToUtf8Uint8Array(
+                                    ciphertextB64)).
+          then(hmac => {
+        return {
+          hmac: StringConversion.arrayBufferToHexString(hmac),
+          ciphertext: ciphertextB64,
+          IV: StringConversion.uint8ArrayToBase64String(IV)
+        };
+      });
+    });
+  };
+
+  /*
+   * encrypt - encrypt and sign a record
+   *
+   * @param {Object} record Object The data to be JSON-stringified and stored
+   * @param {String} collectionName String The name of the Sync collection
+   *     (currently ignored, see
+   *     https://github.com/michielbdejong/fxsync-webcrypto/issues/19)
+   * @returns {Promise} A promise for an object with ciphertext, IV, and hmac.
+   */
+  const encrypt = function(record, collectionName) {
+    if (typeof record !== 'object') {
+      return Promise.reject(new Error('Record should be an object'));
+    }
+    if (typeof collectionName !== 'string') {
+      return Promise.reject(new Error('collectionName is not a string'));
+    }
+
+    var cleartextStr, keyBundle;
+
+    try {
+      cleartextStr = JSON.stringify(record);
+    } catch(e) {
+      return Promise.reject(new Error('Record cannot be JSON-stringified'));
+    }
+    const cleartext = StringConversion.stringToUtf8Uint8Array(cleartextStr);
+    try {
+      keyBundle = this.bulkKeyBundle.defaultAsKeyBundle;
+    } catch(e) {
+      return Promise.reject(new Error(`No key bundle found for \
+${collectionName} - did you call setKeys?`));
+    }
+    return encryptAndSign(keyBundle, cleartext);
+  };
+
+  const FxSyncWebCrypto = function() {
+    // Basic check for presence of WebCrypto.
+    if (!crypto || !crypto.subtle) {
+      throw new Error('This environment does not support WebCrypto');
+    }
+
+    this.bulkKeyBundle = null;
+  };
+
+  FxSyncWebCrypto.prototype = Object.freeze({
+    setKeys: setKeys,
+    decrypt: decrypt,
+    encrypt: encrypt
+  });
+
+  // Expose this for unit test:
+  FxSyncWebCrypto.importKeyBundle = Object.freeze(importKeyBundle);
+
+  return FxSyncWebCrypto;
+})());

assets/webapps/sync.gaiamobile.org/shared/js/sync/crypto/keyderivation.js

@@ -0,0 +1,97 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+'use strict';
+
+/* global
+  StringConversion
+*/
+
+/* exported
+  KeyDerivation
+*/
+
+const KeyDerivation = Object.freeze((() => {
+  // Hash length is 32 because only SHA256 is used at this moment.
+  const HASH_LENGTH = 32;
+  const subtle = window.crypto.subtle;
+
+  const concatU8Array = (buffer1, buffer2) => {
+    var aux = new Uint8Array(buffer1.byteLength + buffer2.byteLength);
+    aux.set(new Uint8Array(buffer1), 0);
+    aux.set(new Uint8Array(buffer2), buffer1.byteLength);
+    return aux;
+  };
+
+  const alg = {
+    name: 'HMAC',
+    hash: 'SHA-256'
+  };
+
+  const doImportKey = (rawKey) => {
+    return subtle.importKey('raw', rawKey, alg, false, ['sign']);
+  };
+
+  // Converts a ArrayBuffer into a ArrayBufferView (U8) if it's not that
+  // already.
+  const arrayBuffer2Uint8 = (buff) => {
+    return buff.buffer && buff || new Uint8Array(buff);
+  };
+
+  const doHMAC = (tbsData, hmacKey) => {
+    return subtle.sign(alg.name, hmacKey, tbsData).then(arrayBuffer2Uint8);
+  };
+
+  const bitSlice = (arr, start, end) => {
+    return (end !== undefined ? arr.subarray(start / 8, end / 8) :
+                                arr.subarray(start / 8));
+  };
+
+  const newEmptyArray = () => new Uint8Array(0);
+
+  return {
+    /**
+     * hkdf - The HMAC-based Key Derivation Function
+     *
+     * @param {bitArray} ikm Initial keying material
+     * @param {bitArray} info Key derivation data
+     * @param {bitArray} salt Salt
+     * @param {integer} length Length of the derived key in bytes
+     * @return promise object- It will resolve with `output` data
+     */
+    hkdf: (ikm, info, salt, length) => {
+      const numBlocks = Math.ceil(length / HASH_LENGTH);
+
+      const doHKDFRound = (roundNumber, prevDigest, prevOutput, hkdfKey) => {
+        // Do the data accumulating part of an HKDF round. Also, it
+        // checks if there are still more rounds left and fires the next
+        // Or just finishes the process calling the callback.
+        const addToOutput = (digest) => {
+          const output = prevOutput +
+              StringConversion.uint8ArrayToHexString(digest);
+
+          if (++roundNumber <= numBlocks) {
+            return doHKDFRound(roundNumber, digest, output, hkdfKey);
+          }
+          return new Promise(resolve => {
+            const truncated = bitSlice(
+                StringConversion.hexStringToUint8Array(output), 0, length * 8);
+            resolve(truncated);
+          });
+        };
+        const input = concatU8Array(
+          concatU8Array(prevDigest, info),
+          StringConversion.stringToUtf8Uint8Array(
+               String.fromCharCode(roundNumber)));
+        return doHMAC(input, hkdfKey).then(addToOutput);
+      };
+
+      return doImportKey(salt). // Imports the initial key.
+        then(doHMAC.bind(undefined, ikm)). // Generates the key deriving key.
+        then(doImportKey). // Imports the key deriving key.
+        then(doHKDFRound.bind(undefined, 1, newEmptyArray(), ''));
+      // Launches the first HKDF round.
+    }
+  };
+})());

assets/webapps/sync.gaiamobile.org/shared/js/sync/crypto/stringconversion.js

@@ -0,0 +1,108 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+'use strict';
+
+/* exported
+  StringConversion
+*/
+
+const StringConversion = Object.freeze((() => {
+
+  return {
+    rawStringToUint8Array(str) {
+      if (typeof str != 'string') {
+        throw new Error('Not a string');
+      }
+      const strLen = str.length;
+      var byteArray = new Uint8Array(strLen);
+      for (var i = 0; i < strLen; i++) {
+        byteArray[i] = str.charCodeAt(i);
+      }
+      return byteArray;
+    },
+
+    stringToUtf8Uint8Array(str) {
+      if (typeof str != 'string') {
+        throw new Error('Not a string');
+      }
+      var string = unescape(encodeURIComponent(str)),
+          charList = string.split(''),
+          uintArray = [];
+      for (var i = 0; i < charList.length; i++) {
+        uintArray.push(charList[i].charCodeAt(0));
+      }
+      return new Uint8Array(uintArray);
+    },
+
+    base64StringToUint8Array(base64) {
+      if (typeof base64 != 'string' || base64.length % 4 !== 0) {
+        throw new Error(`Number of base64 digits must be a multiple of 4 to con\
+vert to bytes`);
+      }
+      return StringConversion.rawStringToUint8Array(window.atob(base64));
+    },
+
+    hexStringToUint8Array(hexStr) {
+      if (typeof hexStr != 'string' || hexStr.length % 2 !== 0) {
+        throw new Error(`Must have an even number of hex digits to convert to b\
+ytes`);
+      }
+      const numBytes = hexStr.length / 2;
+      var byteArray = new Uint8Array(numBytes);
+      for (var i = 0; i < numBytes; i++) {
+        byteArray[i] = parseInt(hexStr.substr(i * 2, 2), 16);
+      }
+      return byteArray;
+    },
+
+    uint8ArrayToBase64String(bytes) {
+      if (!(bytes instanceof Uint8Array)) {
+        throw new Error('Not a Uint8Array');
+      }
+      var binary = '';
+      const len = bytes.byteLength;
+      for (var i=0; i<len; i++) {
+          binary += String.fromCharCode(bytes[i]);
+      }
+      return window.btoa(binary);
+    },
+
+    arrayBufferToBase64String(buffer) {
+      if (!(buffer instanceof ArrayBuffer)) {
+        throw new Error('Not an ArrayBuffer');
+      }
+      var bytes = new Uint8Array(buffer);
+      return StringConversion.uint8ArrayToBase64String(bytes);
+    },
+
+    utf8Uint8ArrayToString(array) {
+      if (!(array instanceof Uint8Array)) {
+        throw new Error('Not an Uint8Array');
+      }
+      var utf8_string = String.fromCharCode.apply(null, array);
+      return decodeURIComponent(escape(utf8_string));
+    },
+
+    uint8ArrayToHexString(bytes) {
+      if (!(bytes instanceof Uint8Array)) {
+        throw new Error('Not a Uint8Array');
+      }
+      var hex = '';
+      for (var i=0; i <bytes.length; ++i) {
+        const zeropad = (bytes[i] < 0x10) ? '0' : '';
+        hex += zeropad + bytes[i].toString(16);
+      }
+      return hex;
+    },
+
+    arrayBufferToHexString(buffer) {
+      if (!(buffer instanceof ArrayBuffer)) {
+        throw new Error('Not an ArrayBuffer');
+      }
+      var bytes = new Uint8Array(buffer);
+      return StringConversion.uint8ArrayToHexString(bytes);
+    }
+  };
+})());

assets/webapps/sync.gaiamobile.org/shared/js/sync/engine.js

@@ -0,0 +1,465 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * SyncEngine
+ *
+ * The SyncEngine constructor takes care of:
+ * - Setting up the kinto.js client.
+ * - Configuring kinto.js to connect to the remote Syncto server (which in turn
+ *   proxies all requests to the remote Firefox Sync server).
+ * - Creating kinto.js collections for the 'meta' and 'crypto' collections, with
+ *   the appropriate IdSchemas (the 'meta' collection only contains one record,
+ *   called 'global', and the 'crypto' collection only contains one record,
+ *   called 'keys').
+ * - Setting up the FxSyncWebCrypto instance to be used for cryptographically
+ *   encoding / decoding records, before uploading / after downloading them.
+ * - Registering the DataAdapters to be used on each collection (see below).
+ *
+ * The `syncNow` method takes care of:
+ * - If called for the first time, or if this failed the previous time:
+ *   - Retrieving 'meta/global' to check the storageVersion.
+ *   - Retrieving and decrypting 'crypto/keys' to obtain the bulk key bundle.
+ * - Making sure a kinto.js collection exists for each collection that needs
+ *   syncing.
+ * - These collections are set up with the correct IdSchema (typically
+ *   allowing only 9-character URL-safe Base64 strings, although for some
+ *   collections other strings are allowed, e.g. bookmarks also has id's like
+ *   'menu' and 'toolbar').
+ * - Adding a RemoteTransformer to each of these collections, so that outgoing
+ *   and incoming data is passed through FxSyncWebCrypto for its en-/decryption.
+ * - For each collection to be synced, the following steps occur:
+ *   - Download the latest remote data into the local decrypted copy, managed by
+ *     kinto.js.
+ *   - For each incoming conflict reported by kinto.js (if any), call the
+ *     `handleConflict` method on the DataAdapter registered for this
+ *     collection.
+ *   - When kinto.js has obtained a full decrypted copy of the remote data, call
+ *     the `update` method on the DataAdapter, passing it an interface to the
+ *     Kinto collection in question.
+ *   - When this method returns its promise, sync the Kinto collection a second
+ *     time, to make sure any changes made there by the DataAdapter are synced
+ *     up to the remote Firefox Sync server (again, through the remote Syncto
+ *     proxy server). The RemoteTransformer on the Kinto collection will take
+ *     care of encrypting the records before they get uploaded.
+ *   - Call the `handleConflict` method on the DataAdapter again for any
+ *     conflicts kinto.js may report during this second sync process.
+ *
+ */
+
+'use strict';
+
+/* global
+  crypto,
+  FxSyncWebCrypto,
+  Kinto
+*/
+
+/* exported
+  SyncEngine
+*/
+
+var SyncEngine = (function() {
+  const HTTP_TIMEOUT_SECONDS = 180;
+
+  var createFxSyncIdSchema = () => {
+    return {
+      generate: function() {
+        var bytes = new Uint8Array(9);
+        crypto.getRandomValues(bytes);
+        var binStr = '';
+        for (var i = 0; i < 9; i++) {
+            binStr += String.fromCharCode(bytes[i]);
+        }
+        // See https://docs.services.mozilla.com/storage/apis-1.5.html
+        return window.btoa(binStr).replace('+', '-').replace('/', '_');
+      },
+      validate: function(id) {
+        // FxSync id's "should" be 12 ASCII characters, representing 9 bytes of
+        // data in modified Base64 for URL variants, where the '+' and '/'
+        // characters of standard Base64 are respectively replaced by '-' and
+        // '_'. See https://docs.services.mozilla.com/storage/apis-1.5.html
+        // But in practice, they could be any string, see bug 1209906.
+        return (typeof id === 'string');
+      }
+    };
+  };
+
+  var createControlCollectionIdSchema = (keyName) => {
+    return {
+      generate: function() {
+        return keyName;
+      },
+      validate: function(id) {
+        return (id === keyName);
+      }
+    };
+  };
+
+  var createWebCryptoTransformer = (collectionName, fswc) => {
+    if (!fswc.bulkKeyBundle) {
+      throw new Error(
+          'Attempt to register Transformer with no bulk key bundle!');
+    }
+    return {
+      encode: function(record) {
+        return fswc.encrypt(record.payload, collectionName).then(
+            payloadEnc => {
+          return {
+            id: record.id,
+            payload: JSON.stringify(payloadEnc)
+          };
+        });
+      },
+      decode: function(record) {
+        // Allowing JSON.parse errors to bubble up to the errors list in the
+        // syncResults:
+        return fswc.decrypt(JSON.parse(record.payload), collectionName).
+            then(payloadDec => {
+          return {
+            id: record.id,
+            last_modified: record.last_modified,
+            payload: payloadDec
+          };
+        });
+      }
+    };
+  };
+
+  const generateXClientState = (kB) => {
+    var kBarray = [];
+    for(var i = 0; i < kB.length; i += 2){
+      kBarray.push(parseInt(kB.substring(i, i + 2), 16));
+    }
+    return window.crypto.subtle.digest({name: 'SHA-256'},
+                                       new Uint8Array(kBarray))
+    .then(hash => {
+      var bytes = new Uint8Array(hash).slice(0, 16);
+      var array = [];
+      bytes.forEach(byte => {
+        array.push((byte < 16 ? '0' : '') + byte.toString(16).toLowerCase());
+      });
+      return array.join('');
+    });
+  };
+
+  /**
+    * SyncEngine - Constructor.
+    * @param {Object} options Should contain the following fields:
+    *                         * URL - e.g. 'http://localhost:8000/v1/'
+    *                         * assertion - a BrowserID assertion (Base64)
+    *                         * kB - kB key from token server (Base64)
+    *                         * adapters - object whose keys are collection
+    *                                      names, and whose values are
+    *                                      DataAdapter instances.
+    * @returns {SyncEngine}
+    */
+  var SyncEngine = function(options) {
+    if (typeof options !== 'object') {
+      throw new Error('options should be an Object');
+    }
+    ['URL', 'assertion', 'kB'].forEach(field => {
+      if (typeof options[field] !== 'string') {
+        throw new Error(`options.${field} should be a String`);
+      }
+    });
+    if (typeof options.adapters !== 'object') {
+      throw new Error('options.adapters should be an Object');
+    }
+    for (var collectionName in options.adapters) {
+      if (typeof options.adapters[collectionName] !== 'object') {
+        throw new Error(`options.adapters.${collectionName} should be an Object\
+`);
+      }
+      ['update', 'handleConflict'].forEach(methodName => {
+        if (typeof options.adapters[collectionName][methodName] !==
+            'function') {
+          throw new Error(`options.adapters.${collectionName}.${methodName} sho\
+uld be a Function`);
+        }
+      });
+    }
+
+    ['kB', 'assertion', 'URL', 'adapters'].forEach(field => {
+      this[`_${field}`] = options[field];
+    });
+
+    this._collections = {};
+    this._controlCollections = {};
+    this._fswc = new FxSyncWebCrypto();
+    this._engines = null;
+    this._kinto = null;
+    this._xClientState = null;
+    this._haveUnsyncedConflicts = {};
+    this._ready = false;
+  };
+
+  function promiseWaterfall(self, tasks, executor) {
+    return tasks.reduce((reduced, current) => {
+      return reduced.then(() => {
+        return executor.apply(self, current);
+      });
+    }, Promise.resolve());
+  }
+
+  SyncEngine.prototype = {
+    _reset: function(collectionName, userid) {
+      var collectionsToReset = ['meta', 'crypto'];
+      if (collectionName) {
+        collectionsToReset.push(collectionName);
+      }
+      return Promise.all(collectionsToReset.map(collectionName => {
+        var coll = this._getCollection(collectionName);
+        return coll.clear().then(() => {
+          if (this._adapters[collectionName]) {
+            return this._adapters[collectionName].reset({ userid });
+          }
+        });
+      }));
+    },
+
+    _createKinto: function(kintoCredentials) {
+      var kinto = new Kinto({
+        bucket: kintoCredentials.xClientState,
+        remote: kintoCredentials.URL,
+        timeout: HTTP_TIMEOUT_SECONDS * 1000,
+        headers: {
+          'Authorization': 'BrowserID ' + kintoCredentials.assertion
+        }
+      });
+      var addControlCollection = (collectionName, keyName) => {
+        var idSchema = createControlCollectionIdSchema(keyName);
+        this._controlCollections[collectionName] =
+            kinto.collection(collectionName, { idSchema });
+      };
+      addControlCollection('meta', 'global');
+      addControlCollection('crypto', 'keys');
+      return kinto;
+    },
+
+    _getCollection: function(collectionName) {
+      if (['meta', 'crypto'].indexOf(collectionName) === -1) {
+        return this._collections[collectionName];
+      }
+      return this._controlCollections[collectionName];
+    },
+
+    _getItem: function(collectionName, itemName, syncIfNeeded) {
+      return this._getCollection(collectionName).get(itemName).catch(err => {
+        if (syncIfNeeded) {
+          return this._syncCollection(collectionName).then(() => {
+            return this._getItem(collectionName, itemName, false);
+          });
+        }
+        if (err.message === 'Record with id=global not found.') {
+          throw new SyncEngine.InvalidAccountError();
+        }
+        throw err;
+      });
+    },
+
+    _resolveConflicts: function(collectionName, conflicts) {
+      return Promise.all(conflicts.map(conflict => {
+        return this._adapters[collectionName].handleConflict(conflict)
+          .then(resolution =>
+              this._collections[collectionName].resolve(conflict, resolution))
+          .then(() => this._haveUnsyncedConflicts[collectionName] = true);
+      }));
+    },
+
+    _syncCollection: function(collectionName, collectionOptions) {
+      var collection = this._getCollection(collectionName);
+      // Let synchronization strategy default to 'manual', see
+      // http://kintojs.readthedocs.org \
+      //     /en/latest/api/#fetching-and-publishing-changes
+
+      var userid = collectionOptions && collectionOptions.userid;
+      return collection.sync({
+        fetchLimit: (collectionOptions && collectionOptions.limit) || undefined
+      }).then(syncResults => {
+        if (syncResults && syncResults.errors && syncResults.errors.length) {
+          var error = new Error('Errors in SyncResults');
+          error.data = syncResults;
+          throw error;
+        }
+        if (syncResults.conflicts.length) {
+          return this._resolveConflicts(collectionName, syncResults.conflicts);
+        }
+      }).catch(err => {
+        if (err instanceof Error && typeof err.response === 'object') {
+          if (err.response.status === 401) {
+            throw new SyncEngine.AuthError(err.message);
+          }
+          throw new SyncEngine.TryLaterError(err.message);
+        }
+        if (err.message === `HTTP 0; TypeError: NetworkError when attempting to\
+ fetch resource.`) {
+          throw new SyncEngine.TryLaterError('Syncto server unreachable',
+              this._kinto && this._kinto._options &&
+              this._kinto._options.remote);
+        }
+        return this._reset(collectionName, userid).then(
+          () => {
+          throw new SyncEngine.UnrecoverableError(err.message);
+        });
+      });
+    },
+
+    _checkMetaGlobal: function(metaGlobal) {
+      var payloadObj;
+      try {
+        payloadObj = JSON.parse(metaGlobal.data.payload);
+      } catch(e) {
+        return false;
+      }
+      this._engines = payloadObj.engines;
+
+      return (typeof payloadObj === 'object' &&
+          payloadObj.storageVersion === 5);
+    },
+
+    _initFxSyncWebCrypto: function(cryptoKeys) {
+      this._fswc = new FxSyncWebCrypto();
+      return this._fswc.setKeys(this._kB, cryptoKeys).then(() => {
+        this._createCollections();
+        this._ready = true;
+      }, err => {
+        if (err === 'SyncKeys hmac could not be verified with current main ' +
+            'key') {
+          this._reset().then(() => {
+            throw new SyncEngine.UnrecoverableError(err);
+          });
+        }
+        throw err;
+      });
+    },
+
+    _ensureReady: function() {
+      if (this._ready) {
+        return Promise.resolve();
+      }
+      return generateXClientState(this._kB).then(xClientState => {
+        this._kinto = this._createKinto({
+           URL: this._URL,
+           assertion: this._assertion,
+           xClientState
+         });
+         this._xClientState = xClientState;
+      }).then(() => {
+        return this._syncCollection('meta');
+      }).then(() => {
+        return this._getItem('meta', 'global');
+      }).then(metaGlobal => {
+        if (!this._checkMetaGlobal(metaGlobal)) {
+          throw new SyncEngine.UnrecoverableError(`Incompatible storage version\
+ or storage version not recognized.`);
+        }
+      });
+    },
+
+    _getBulkKeyBundle: function() {
+      return this._getItem('crypto', 'keys', true /* syncIfNeeded */)
+          .then((cryptoKeysRecord) => {
+        var cryptoKeys;
+        try {
+          cryptoKeys = JSON.parse(cryptoKeysRecord.data.payload);
+        } catch (e) {
+          throw new SyncEngine.UnrecoverableError(`Could not parse crypto/keys \
+payload as JSON`);
+        }
+        return cryptoKeys;
+      }).then((cryptoKeys) => {
+        return this._initFxSyncWebCrypto(cryptoKeys);
+      });
+    },
+
+    _createCollections: function() {
+      for (var collectionName in this._adapters) {
+        this._collections[collectionName] = this._kinto.collection(
+            collectionName, {
+              idSchema: createFxSyncIdSchema(collectionName),
+              remoteTransformers: [
+                createWebCryptoTransformer(collectionName, this._fswc)
+              ]
+            });
+      }
+    },
+
+    _updateCollection: function(collectionName, collectionOptions) {
+      if (this._engines && !this._engines[collectionName]) {
+        console.warn(`Collection ${collectionName} not present on this FxSync a\
+ccount`);
+        return Promise.resolve();
+      }
+      return this._syncCollection(collectionName, collectionOptions)
+          .then(() => {
+        return this._adapters[collectionName].update(
+            this._collections[collectionName], collectionOptions);
+      }).then(changed => {
+        if (!changed && !this._haveUnsyncedConflicts[collectionName]) {
+          return Promise.resolve();
+        }
+        return this._syncCollection(collectionName, collectionOptions)
+            .then(() => {
+          this._haveUnsyncedConflicts[collectionName] = false;
+        });
+      });
+    },
+
+    /**
+      * syncNow - Syncs collections up and down between device and server.
+      * @param {object} collectionOptions The options per collection. Can be:
+      *                                   {boolean} readonly (defaults to true)
+      *                                   {number} limit
+      * @returns {Promise}
+      */
+    syncNow: function(collectionOptions) {
+      if (typeof collectionOptions !== 'object') {
+        return Promise.reject(new Error(
+            'collectionOptions should be an object'));
+      }
+      return this._ensureReady().then(() => {
+        if (Object.keys(collectionOptions).length === 0) {
+          // This can happen if the system app just wanted to check if an
+          // FxSync account exists for the credentials derived from kB.
+          return Promise.resolve();
+        }
+        return this._getBulkKeyBundle().then(() => {
+          var tasks = [];
+          for (var collectionName in collectionOptions) {
+            collectionOptions[collectionName].userid = this._xClientState;
+            tasks.push([ collectionName, collectionOptions[collectionName] ]);
+          }
+          return promiseWaterfall(this, tasks, this._updateCollection);
+        });
+      });
+    }
+  };
+
+  SyncEngine.UnrecoverableError = function() {
+    console.error('[SyncEngine Unrecoverable]', arguments);
+    this.message = 'unrecoverable';
+  };
+  SyncEngine.UnrecoverableError.prototype = Object.create(Error.prototype);
+
+  SyncEngine.TryLaterError = function() {
+    console.error('[SyncEngine TryLater]', arguments);
+    this.message = 'try later';
+  };
+  SyncEngine.TryLaterError.prototype = Object.create(Error.prototype);
+
+  SyncEngine.AuthError = function() {
+    console.error('[SyncEngine Auth]', arguments);
+    this.message = 'unauthorized';
+  };
+  SyncEngine.AuthError.prototype = Object.create(Error.prototype);
+
+  SyncEngine.InvalidAccountError = function() {
+    console.error('[SyncEngine InvalidAccount]', arguments);
+    this.message = 'invalid account';
+  };
+  SyncEngine.InvalidAccountError.prototype = Object.create(Error.prototype);
+
+  return SyncEngine;
+})();

assets/webapps/sync.gaiamobile.org/shared/js/sync/errors.js

@@ -0,0 +1,86 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+'use strict';
+
+/* global ERROR_DIALOG_CLOSED_BY_USER */
+/* global ERROR_INVALID_SYNC_ACCOUNT */
+/* global ERROR_NO_KEY_FETCH_TOKEN */
+/* global ERROR_OFFLINE */
+/* global ERROR_SYNC_APP_TRY_LATER */
+/* global ERROR_UNKNOWN */
+
+/**
+ * These errors are shared by all Firefox Sync related modules. The key should
+ * be used as a constant and the value can be used as a l10n string key if
+ * needed.
+ */
+
+(function(exports) {
+
+  // Unrecoverable errors will cause the Sync state machine to transition to
+  // the 'disabled' state.
+  const unrecoverableErrors = {
+    // The user closed the FxA dialog during the sync enabling state.
+    ERROR_DIALOG_CLOSED_BY_USER: 'fxsync-error-dialog-closed-by-user',
+    // Cannot get Firefox Accounts assertion.
+    ERROR_GET_FXA_ASSERTION: 'fxsync-error-get-fxa-assertion',
+    // The user logged in with an invalid account. This is probably because the
+    // account has no crypto keys associated and we are unable to generate them
+    // so far.
+    ERROR_INVALID_SYNC_ACCOUNT: 'fxsync-error-invalid-account',
+    // The device is offline.
+    ERROR_OFFLINE: 'fxsync-error-offline',
+    // The current login is invalid as it does not contain the token to fetch
+    // sync keys.
+    ERROR_NO_KEY_FETCH_TOKEN: 'fxsync-error-no-key-fetch-token',
+    // Cannot register sync request.
+    ERROR_REQUEST_SYNC_REGISTRATION: 'fxsync-error-request-fxsync-registration',
+    // The request options passed to the Sync app failed basic DataType checks.
+    ERROR_SYNC_INVALID_REQUEST_OPTIONS: 'fxsync-error-invalid-request-options',
+    // Cannot perform sync request.
+    ERROR_SYNC_REQUEST: 'fxsync-error-request-failed',
+    ERROR_UNKNOWN: 'fxsync-error-unknown'
+  };
+
+  const recoverableErrors = {
+    // The app was killed while performing a sync operation.
+    ERROR_SYNC_APP_KILLED: 'fxsync-error-app-killed',
+    // The app is already performing a sync request.
+    ERROR_SYNC_APP_SYNC_IN_PROGRESS: 'fxsync-error-app-fxsync-in-progress',
+    // Error while trying to sync.
+    ERROR_SYNC_APP_GENERIC: 'fxsync-error-app-generic',
+    // The server cannot be reached.
+    ERROR_SYNC_APP_TRY_LATER: 'fxsync-error-app-try-later',
+    // A DataStore was updated by another app at the same time.
+    ERROR_SYNC_APP_RACE_CONDITION: 'fxsync-error-app-race-condition',
+    // The user is logged in with an unverified account.
+    ERROR_UNVERIFIED_ACCOUNT: 'fxsync-error-unverified-account'
+  };
+
+  exports.SyncUnrecoverableErrors = [];
+  Object.keys(unrecoverableErrors).forEach(key => {
+    exports[key] = unrecoverableErrors[key];
+    exports.SyncUnrecoverableErrors.push(unrecoverableErrors[key]);
+  });
+
+  exports.SyncRecoverableErrors = [];
+  Object.keys(recoverableErrors).forEach(key => {
+    exports[key] = recoverableErrors[key];
+    exports.SyncRecoverableErrors.push(recoverableErrors[key]);
+  });
+
+  // Map between external (FxA, Syncto, kinto.js) and Sync errors.
+  exports.SyncErrors = {
+    'invalid account': ERROR_INVALID_SYNC_ACCOUNT,
+    'No keyFetchToken': ERROR_NO_KEY_FETCH_TOKEN,
+    'OFFLINE': ERROR_OFFLINE,
+    'try later': ERROR_SYNC_APP_TRY_LATER,
+    'UI_ERROR': ERROR_DIALOG_CLOSED_BY_USER,
+    'unauthorized': ERROR_INVALID_SYNC_ACCOUNT,
+    'UNVERIFIED_ACCOUNT': ERROR_INVALID_SYNC_ACCOUNT,
+    'unrecoverable': ERROR_UNKNOWN
+  };
+
+}(window));