VulpesOS/assets/webapps/system.gaiamobile.org/js/advanced_telemetry.js

/**
 * This module is responsible for managing the AdvancedTelemetry features in
 * Gaia. Advanced Telemetry includes:
 *  1) Engineering Performance metrics on a per app basis that are collected
 *  by the HUD engine in gecko.
 *  2) Custom App Metrics.  Apps can define their custom metrics and have them
 *  collected in a histogram as well.  This API is defined via the
 *  AdvancedTelemetryHelper.
 *
 * Metrics are gathered in gecko and requested from AdvancedTelemetry Module
 * when it is time to ship the metrics to the server depending on the send
 * interval desired.
 */

/* global asyncStorage, SettingsListener, uuid, TelemetryRequest, Gzip,
   LazyLoader */

(function(exports) {
  'use strict';

  // This is the asyncStorage key we use to persist our device ID
  const DEVICE_ID_KEY = 'metrics.advanced_telemetry.deviceID.v4';

  // Constants for events we register/unreg for
  const IDLE = 'idle';
  const ACTIVE = 'active';
  const HUDMETRICS = 'advanced-telemetry-update';


  // Constant for debugging lines
  const AT_DEBUG_PREFIX = '[AdvancedTelemetry]';

  // This is the list of event types we register handlers for
  const EVENT_TYPES = [
    IDLE,
    ACTIVE,
    HUDMETRICS
  ];


  // This AdvancedTelemetry() constructor is the value we export from
  // this module. This constructor does no initialization itself: that
  // is all done in the start() instance method. See bootstrap.js for
  // the code that actually calls the constructor and the start method.
  function AdvancedTelemetry() {}

  // We use the acronym AT internally to save typing and make
  // references to class variables and constants more legible.
  const AT = AdvancedTelemetry;

  AT.BatchTiming = BatchTiming;

  // This the ping type in the v4 telemetry spec.  Exposed for unit testing.
  AT.REASON = 'advancedtelemetry';

  // asyncStorage key that stores the BatchTiming start time.
  AT.BATCH_KEY = 'metrics.advanced_telemetry.data.v4';

  // This is the asyncStorage key we use to persist our metrics
  // in case we need to merge them later.
  AT.METRICS_KEY = 'metrics.advanced_telemetry.metrics.v4';

  // Set to true to to enable debug output.  Not recommended to turn this
  // on except while doing deep debugging.
  AT.DEBUG = false;

  // LOGINFO method allows for debugging of initialization events and
  // packet sending events and is at a higher level then DEBUG.
  AT.LOGINFO = false;

  // Constants for BatchTiming Batch Type
  AT.NORMAL_BATCH = true;
  AT.RETRY_BATCH = false;

  //Constants for BatchTiming reset
  AT.RESET_BATCH = true;
  AT.EXISTING_BATCH = false;

  function debug(...args) {
    if (!AT.DEBUG) {
      return;
    }

    var iCount = 0;
    // Get the total length of all arguments passed
    args.forEach(function (value) {
      iCount += value.length;
    });
    // if the length is greater than what adb console can handle, chunk it up.
    if (iCount > 1000) {
      var longLine = '';
      args.forEach(function (value) {
        longLine += value;
      });
      // Chop it into chunks of what adb console can handle.
      var array = longLine.match(/.{1,1000}/g);
      // Output each chunk to the console
      array.forEach(function (value) {
        console.log(value);
      });
    } else {
      // Its < what ADB can handle so just output it normally.
      args.unshift(AT_DEBUG_PREFIX);
      console.log.apply(console, args);
    }
  }

  // This info level is more of a information level.  It's recommended to
  // turn this on just to see general events related to initializations,
  // and sending of payloads, etc.
  function loginfo(...args) {
    if (AT.LOGINFO) {
      args.unshift('[AdvancedTelemetryInfo]');
      console.log.apply(console, args);
    }
  }

  // This value is what turns advanced telemetry on/off
  AT.TELEMETRY_ENABLED_KEY = 'metrics.selectedMetrics.level';

  // Base URL for sending data reports
  // Needs to be formatted as:
  // /submit/telemetry/id/reason/appName/appUpdateChannel/appVersion/appBuildID
  AT.REPORT_URL = 'https://incoming.telemetry.mozilla.org/submit/telemetry';
  // How often do we try to send the reports
  // Can be overridden with metrics.advancedtelemetry.reportInterval setting.
  AT.REPORT_INTERVAL = 14 * 24 * 60 * 60 * 1000;  // Two Weeks

  // If the telemetry server does not respond within this amount of time
  // just give up and try again later.
  // Can be overridden with metrics.advancedtelemetry.reportTimeout setting.
  AT.REPORT_TIMEOUT = 20 * 1000;             // 20 seconds

  // How long do we wait before trying it again?
  AT.RETRY_INTERVAL = 60 * 60 * 1000;        // 1 hour

  // How much user idle time (in seconds, not ms) do we wait for before
  // persisting our data to asyncStorage or trying to transmit it.
  AT.IDLE_TIME = 5;                          // seconds

  // Telemetry payload version
  AT.TELEMETRY_VERSION = 4;

  // Interval to merge the metrics to guard against reboot
  AT.MERGE_INTERVAL = 60 * 60 * 1000;       // 1 hour

  // App name (static for Telemetry)
  AT.TELEMETRY_APP_NAME = 'FirefoxOS';


  AT.prototype.start = function start() {
    loginfo('Starting AdvancedTelemetry');
    this.advancedTelemetryEnabledListener =
      function advancedTelemetryEnabledListener(enabled) {
        if (enabled === 'Enhanced') {
          loginfo('Start Collecting');
          this.startCollecting();
        }
        else {
          loginfo('Stop Collecting');
          this.stopCollecting();
        }
      }.bind(this);

    SettingsListener.observe(AT.TELEMETRY_ENABLED_KEY,
      false, this.advancedTelemetryEnabledListener);
  };

  AT.prototype.stopCollecting = function stopCollecting() {
    this.collecting = false;
    EVENT_TYPES.forEach(function (type) {
      window.removeEventListener(type, self);
    });
  };

  AT.prototype.startCollecting = function startCollecting(done) {
    var self = this;
    // If we're already running there is nothing to start
    if (this.collecting) {
      return;
    }
    this.collecting = true;

    this.metrics = new BatchTiming(AT.RESET_BATCH);
    this.mergeTimeStart = Date.now();
    this.merge = false;

    getDeviceID();

    function getDeviceID() {
      asyncStorage.getItem(DEVICE_ID_KEY, function(value) {
        if (value) {
          self.deviceID = value;
        }
        else {
          // Our device id does not need to be unique, just probably unique.
          self.deviceID = uuid();
          asyncStorage.setItem(DEVICE_ID_KEY, self.deviceID);
        }

        // Move on to the next step in the startup process
        registerHandlers();
      });
    }

    function registerHandlers() {
      // Basic event handlers
      EVENT_TYPES.forEach(function (type) {
        window.addEventListener(type, self);
      });

      self.startBatch();

      self.idleObserver = {
        time: AT.IDLE_TIME,
        onidle: function() { self.handleEvent(new CustomEvent(IDLE)); },
        onactive: function() { self.handleEvent(new CustomEvent(ACTIVE)); }
      };

      // Register for idle events
      navigator.addIdleObserver(self.idleObserver);

      if (done) {
        loginfo('Initialization Complete');
        done();
      }
    }
  };

  // This method shuts everything down and is only exposed for unit testing.
  // Note that this is not the same as the stopCollecting() method which is
  // used to stop data collection but keep the module running.
  AT.prototype.stop = function stop() {
    this.stopCollecting();
    SettingsListener.unobserve(AT.TELEMETRY_ENABLED_KEY,
      this.advancedTelemetryEnabledListener);
  };

  // Reset (or initialize) the AdvancedTelemetry instance variables
  AT.prototype.reset = function() {
    this.collecting = false;
    this.metrics = null;
    this.deviceID = null;
    // Is the user idle? Updated in handleEvent() based on an idle observer
    this.idle = false;
    this.merge = false;
  };

  AT.prototype.handleEvent = function handleEvent(e) {
    switch (e.type) {
      case HUDMETRICS:
        this.handleGeckoMessage(e.detail);
        break;
      case IDLE:
        this.idle = true;
        // If we are idle and past the merge time, merge the metrics.
        if ((Date.now() - this.mergeTimeStart) > AT.MERGE_INTERVAL) {
          this.merge = true;
          this.mergeTimeStart = Date.now();
          this.getPayload();
        }
        break;
      case ACTIVE:
        this.idle = false;
        break;
      default:
        break;
    }
  };

  AT.prototype.handleGeckoMessage = function handleGeckoMessage(payload,
                                                                callback) {
    var self = this;
    // We always need to merge the Metrics when we get this.
    var promise = self.mergeMetrics(payload);
    promise.then(function(result) {
      // If it's a merge request mark it as completed.
      if (self.merge) {
        self.merge = false;
        self.clearPayload(false);
      }
      // if it's a request to transmit, transmit the merged metrics.
      else if (self.collecting) {
        self.transmit(result);
      }
      if (callback) {
        callback();
      }
    });
  };

  // Signal gecko to send the histograms with a telemetry message.
  AT.prototype.getPayload = function getPayload() {
    // TODO: Add this line below to AdvancedTelemetryHelper as an API.
    console.info('telemetry|MGMT|TIMETOSHIP');
  };

  // Signal gecko to clear out the histograms and start fresh.
  AT.prototype.clearPayload = function clearPayload(resetStorage) {
    // TODO: Add this line below to AdvancedTelemetryHelper as an API.
    console.info('telemetry|MGMT|CLEARMETRICS');
    if (resetStorage) {
      asyncStorage.setItem(AT.METRICS_KEY, null);
    }
  };

  // Start a timer to notify when to send the payload.
  AT.prototype.startBatch = function startBatch() {
    var self = this;
    this.timer = setTimeout(function() {
      self.getPayload();
    }, self.metrics.getInterval());
  };

  AT.prototype.stopBatch = function stopBatch() {
    clearTimeout(this.timer);
  };

  // Once we have the histogram payload, we can send it to the telemetry
  // server.  Once we receive 200 OK back from telemetry server, we can
  // start a fresh batch of metrics.
  AT.prototype.transmit = function transmit(payload) {
    var self = this;
    if (!this.collecting) {
      return;
    }

    var deviceInfoQuery = {
      'app.update.channel': 'unknown',
      'deviceinfo.platform_version': 'unknown',
      'deviceinfo.platform_build_id': 'unknown'
    };

    // Query the settings db for parameters for the URL
    TelemetryRequest.getSettings(deviceInfoQuery, (deviceResponse) => {
      // Note that this wrapper is using the new v4 Unified Telemetry format
      var wrapper = {
        type: AT.REASON,
        id: uuid(),
        creationDate: new Date().toISOString(),
        version: AT.TELEMETRY_VERSION,
        application: {
          architecture: 'arm',
          buildId: deviceResponse['deviceinfo.platform_build_id'],
          name: AT.TELEMETRY_APP_NAME,
          version: deviceResponse['deviceinfo.platform_version'],
          vendor: 'Mozilla',
          platformVersion: deviceResponse['deviceinfo.platform_version'],
          xpcomAbi: 'arm-gcc3',
          channel: deviceResponse['app.update.channel']
        },
        clientId: this.deviceID,
        payload: payload
      };

      debug(JSON.stringify(wrapper));

      // Build the wrapper for the telemetry version
      send(wrapper, deviceResponse);
    });

    function send(payload, deviceInfoQuery) {
      if (!navigator.onLine) {
        self.startRetryBatch();
        return;
      }

      self.request = new AdvancedTelemetryPing(payload, deviceInfoQuery,
        self.deviceID);

      // We don't actually have to do anything if the data is transmitted
      // successfully. We are already set up to collect the next batch of data.
      function onload() {
        loginfo('Transmitted Successfully.');
        AdvancedTelemetry.prototype.clearPayload(true);
        // Start a new batch.
        self.startNewBatch();
      }

      function retry(e) {
        loginfo('Advanced Telemetry metrics transmission failure:', e.type);
        // Start a retry batch.  Don't clear the payload yet.  It will continue
        // to accumulate until the retry interval expires.
        self.startRetryBatch();
      }
      self.request.packHistograms();
      self.request.send({
        timeout: AT.REPORT_TIMEOUT,
        onload: onload,
        onerror: retry,
        onabort: retry,
        ontimeout: retry
      });
    }
  };

  AT.prototype.startNewBatch = function startNewBatch() {
    this.metrics = new BatchTiming(AT.EXISTING_BATCH, AT.RETRY_BATCH);
    this.startBatch();
  };

  AT.prototype.startRetryBatch = function startRetryBatch() {
    this.metrics = new BatchTiming(AT.EXISTING_BATCH, AT.NORMAL_BATCH);
    this.startBatch();
  };

  // Check if there are existing metrics that need to be merged, if so,
  // merge them and return them for either storing or sending.
  AT.prototype.mergeMetrics = function mergeMetrics(payloadNew) {
    return new Promise(function(resolve) {
      // Get the old Payload for merging.
      asyncStorage.getItem(AT.METRICS_KEY, function(value) {
        var merg;
        if (value) {
          merg = mergePayloads(value, payloadNew);
        } else {
          merg = payloadNew;
        }
        saveMergedPayload(merg, function() {
          resolve(merg);
        });
      });

      function saveMergedPayload(payload, callback) {
        asyncStorage.setItem(AT.METRICS_KEY, payload, function() {
          callback();
        });
      }

      // Merge the new metrics into the old metrics and then transmit the
      // result to the server.
      function mergePayloads(payloadOld, payloadNew) {
        mergeHistogramType(payloadOld.keyedHistograms, payloadOld, payloadNew,
          'keyedHistograms');
        mergeHistogramType(payloadOld.addonHistograms, payloadOld, payloadNew,
          'addonHistograms');
        return payloadOld;
      }
    });
  };

  function mergeHistogram(oldHist, newHist) {
    oldHist.sum += newHist.sum;
    oldHist.log_sum += newHist.log_sum;
    oldHist.log_sum_squares += newHist.log_sum_squares;

    for (var i = 0; i < oldHist.counts.length; i++) {
      oldHist.counts[i] += newHist.counts[i];
    }
  }

  function mergeLists(oldList, newList) {
    var histMap = new Map();

    for (var keyOld in oldList) {
      histMap.set(keyOld, oldList[keyOld]);
    }

    for (var keyNew in newList) {
      if (histMap.has(keyNew)) {
        mergeHistogram(oldList[keyNew], newList[keyNew]);
      } else {
        oldList[keyNew] = newList[keyNew];
      }
    }
  }

  function mergeHistogramType(oldHist, payloadOld, payloadNew, histType) {
    var keyedOldMapKeys = new Map();

    for (var keyOld in oldHist) {
      keyedOldMapKeys.set(keyOld, oldHist[keyOld]);
    }

    var keyedNew = payloadNew[histType];
    var keyedOld = payloadOld[histType];
    for (var keyNew in keyedNew) {
      if (keyedOldMapKeys.has(keyNew)) {
        mergeLists(keyedOld[keyNew], keyedNew[keyNew]);
      } else {
        keyedOld[keyNew] = keyedNew[keyNew];
      }
    }
  }

  function AdvancedTelemetryPing(payload, deviceQuery) {
    if (!payload) {
      throw new Error('No arguments');
    }
    // clone so we don't put data into the object that was given to us
    this.packet = JSON.parse(JSON.stringify(payload));

    // URL format: /id/reason/appName/appVersion/appUpdateChannel/appBuildID?v=4
    var uriParts = [
      AT.REPORT_URL,
      encodeURIComponent(payload.id),
      encodeURIComponent(AT.REASON),
      encodeURIComponent(AT.TELEMETRY_APP_NAME),
      encodeURIComponent(deviceQuery['deviceinfo.platform_version']),
      encodeURIComponent(deviceQuery['app.update.channel']),
      encodeURIComponent(deviceQuery['deviceinfo.platform_build_id']),
    ];

    this.url = uriParts.join('/');
    this.url += ('?v=4');
    debug('Telemetry URL is: ' + this.url);
  }

  /*
   * Expose `getData` for unit testing purposes
   */
  AdvancedTelemetryPing.prototype.getData = function(packet) {
    return JSON.stringify(packet);
  };

  // This method converts a raw histogram with the .counts and .ranges arrays
  // in it to one with just .values using the dict format.
  AdvancedTelemetryPing.prototype.packHistogram =
  function packHistogram(hgram) {
    //Constant for Histogram types
    const HISTOGRAM_EXP = 0;

    if (typeof hgram.ranges === 'undefined' ||
      typeof hgram.counts === 'undefined') {
      return hgram;
    }
    let r = hgram.ranges;
    let c = hgram.counts;
    let retgram = {
      range: [r[1], r[r.length - 1]],
      bucket_count: r.length,
      histogram_type: hgram.histogram_type,
      values: {},
      sum: hgram.sum
    };

    if (hgram.histogram_type === HISTOGRAM_EXP) {
      retgram.log_sum = hgram.log_sum;
      retgram.log_sum_squares = hgram.log_sum_squares;
    } else {
      retgram.sum_squares_lo = hgram.sum_squares_lo;
      retgram.sum_squares_hi = hgram.sum_squares_hi;
    }

    let first = true;
    let last = 0;

    for (let i = 0; i < c.length; i++) {
      let value = c[i];
      if (!value) {
        continue;
      }

      // add a lower bound
      if (i && first) {
        retgram.values[r[i - 1]] = 0;
      }
      first = false;
      last = i + 1;
      retgram.values[r[i]] = value;
    }

    // add an upper bound
    if (last && last < c.length) {
      retgram.values[r[last]] = 0;
    }
    return retgram;
  },

  // This method loops through the payload of histograms and converts each
  // histogram payload to the 'dict' format required by the server for parsing.
  AdvancedTelemetryPing.prototype.packHistograms = function() {
    var nameAddonHist = this.packet.payload.addonHistograms;
    var nameKeyHist = this.packet.payload.keyedHistograms;

    // Pack the Addon Histograms.
    for (var addon in nameAddonHist) {
      for (var hist in nameAddonHist[addon]) {
        nameAddonHist[addon][hist] =
          this.packHistogram(nameAddonHist[addon][hist]);
      }
    }

    // Pack the Keyed Histograms.
    for (var key in nameKeyHist) {
      for (var hist2 in nameKeyHist[key]) {
        nameKeyHist[key][hist2] = this.packHistogram(nameKeyHist[key][hist2]);
      }
    }
  };

  AdvancedTelemetryPing.prototype.send = function(xhrAttrs) {
    return new Promise((resolve, reject) => {
      this.data = this.getData(this.packet);
      LazyLoader.load(['/shared/js/gzip/gzip.js']).then(() => {
        Gzip.compress(this.data).then((gzipData) => {
          debug('Length of compressed data:', gzipData.length);
          return this.post(xhrAttrs, gzipData);
        }, (msg) => {
          debug('Error compressing telemetry payload:', msg);
          return this.post(xhrAttrs, this.data);
        }).then(function () {
          resolve();
        });
      });
    });
  };

  AdvancedTelemetryPing.prototype.post = function post(xhrAttrs, gzipData) {
    var xhr = new XMLHttpRequest({ mozSystem: true, mozAnon: true });
    return new Promise((resolve, reject) => {

      xhr.open('POST', this.url);

      if (xhrAttrs) {
        xhr.onload = xhrAttrs.onload;
        xhr.onerror = xhrAttrs.onerror;
        xhr.onabort = xhrAttrs.onabort;
        xhr.ontimeout = xhrAttrs.ontimeout;

        if (xhrAttrs.timeout) {
          xhr.timeout = xhrAttrs.timeout;
        }
      }
      xhr.setRequestHeader('Content-type', 'application/json');
      xhr.setRequestHeader('Content-Encoding', 'gzip');

      xhr.responseType = 'text';

      xhr.send(gzipData);
      resolve();
    });
  };

  /*
   * A helper class that tracks the start time of the current batch.
   */
  function BatchTiming(startup, retry) {
    var self = this;
    this.start = Date.now();
    if (startup) {
      asyncStorage.getItem(AT.BATCH_KEY, function(value) {
        if (value) {
          var oldDate = new Date(value);
          self.interval = self.start - oldDate.getTime();
          if (self.interval > AT.REPORT_INTERVAL) {
            self.interval = AT.REPORT_INTERVAL;
            asyncStorage.setItem(AT.BATCH_KEY, self.start);
          }
        } else {
          // here if it's the first time it's being run and there is nothing
          // saved in async.
          asyncStorage.setItem(AT.BATCH_KEY, self.start);
          self.interval = AT.REPORT_INTERVAL;
        }
      });
    } else {
      asyncStorage.setItem(AT.BATCH_KEY, this.start);
      if (retry) {
        self.interval = AT.RETRY_INTERVAL;
      } else {
        self.interval = AT.REPORT_INTERVAL;
      }
    }
  }

  BatchTiming.prototype.startTime = function() {
    return this.start;
  };

  BatchTiming.prototype.getInterval = function(){
    return this.interval;
  };

  exports.AdvancedTelemetry = AdvancedTelemetry;
  // Exported for unit testing.
  exports.AdvancedTelemetryPing = AdvancedTelemetryPing;
}(window));