b38dea0597
Share Gaia assets and adapters, keep private profiles and development documents local, and update both public READMEs.
1548 lines
56 KiB
JavaScript
Executable file
1548 lines
56 KiB
JavaScript
Executable file
define(
|
|
[
|
|
'logic',
|
|
'../a64',
|
|
'../allback',
|
|
'../date',
|
|
'../syncbase',
|
|
'../util',
|
|
'module',
|
|
'require',
|
|
'exports'
|
|
],
|
|
function(
|
|
logic,
|
|
$a64,
|
|
$allback,
|
|
$date,
|
|
$sync,
|
|
$util,
|
|
$module,
|
|
require,
|
|
exports
|
|
) {
|
|
|
|
/**
|
|
* Lazily evaluated modules
|
|
*/
|
|
var $imaptextparser = null;
|
|
var $imapsnippetparser = null;
|
|
var $imapbodyfetcher = null;
|
|
var $imapchew = null;
|
|
var $imapsync = null;
|
|
|
|
var allbackMaker = $allback.allbackMaker,
|
|
bsearchForInsert = $util.bsearchForInsert,
|
|
bsearchMaybeExists = $util.bsearchMaybeExists,
|
|
cmpHeaderYoungToOld = $util.cmpHeaderYoungToOld,
|
|
DAY_MILLIS = $date.DAY_MILLIS,
|
|
NOW = $date.NOW,
|
|
BEFORE = $date.BEFORE,
|
|
ON_OR_BEFORE = $date.ON_OR_BEFORE,
|
|
SINCE = $date.SINCE,
|
|
TIME_DIR_AT_OR_BEYOND = $date.TIME_DIR_AT_OR_BEYOND,
|
|
TIME_DIR_ADD = $date.TIME_DIR_ADD,
|
|
TIME_DIR_DELTA = $date.TIME_DIR_DELTA,
|
|
makeDaysAgo = $date.makeDaysAgo,
|
|
makeDaysBefore = $date.makeDaysBefore,
|
|
quantizeDate = $date.quantizeDate,
|
|
PASTWARDS = 1, FUTUREWARDS = -1;
|
|
|
|
/**
|
|
* Compact an array in-place with nulls so that the nulls are removed. This
|
|
* is done by a scan with an adjustment delta and a final splice to remove
|
|
* the spares.
|
|
*/
|
|
function compactArray(arr) {
|
|
// this could also be done with a write pointer.
|
|
var delta = 0, len = arr.length;
|
|
for (var i = 0; i < len; i++) {
|
|
var obj = arr[i];
|
|
if (obj === null) {
|
|
delta++;
|
|
continue;
|
|
}
|
|
if (delta)
|
|
arr[i - delta] = obj;
|
|
}
|
|
if (delta)
|
|
arr.splice(len - delta, delta);
|
|
return arr;
|
|
}
|
|
|
|
/**
|
|
* Number of bytes to fetch from the server for snippets.
|
|
*/
|
|
var NUMBER_OF_SNIPPET_BYTES = 256;
|
|
|
|
/**
|
|
* Maximum bytes to request from server in a fetch request (max uint32)
|
|
*/
|
|
var MAX_FETCH_BYTES = (Math.pow(2, 32) - 1);
|
|
|
|
/**
|
|
* Folder connections do the actual synchronization logic. They are associated
|
|
* with one or more `ImapSlice` instances that issue the requests that trigger
|
|
* synchronization. Storage is handled by `FolderStorage` instances. All of
|
|
* the connection life-cycle nitty-gritty is handled by the `ImapAccount`.
|
|
*
|
|
* == Progress
|
|
*
|
|
* Our progress break-down is:
|
|
* - [0.0, 0.1]: Getting the IMAP connection.
|
|
* - (0.1, 0.25]: Getting usable SEARCH results. Bisect back-off does not
|
|
* update progress.
|
|
* - (0.25, 1.0]: Fetching revised flags, headers, and bodies. Since this
|
|
* is primarily a question of network latency, we weight things based
|
|
* on round-trip requests required with reduced cost for number of packets
|
|
* required.
|
|
* - Revised flags: 20 + 1 * number of known headers
|
|
* - New headers: 20 + 5 * number of new headers
|
|
* - Bodies: 30 * number of new headers
|
|
*
|
|
* == IDLE
|
|
*
|
|
* We plan to IDLE in folders that we have active slices in. We are assuming
|
|
* the most basic IDLE implementation where it will tell us when the number
|
|
* of messages increases (EXISTS), or decreases (EXPUNGE and EXISTS), with no
|
|
* notifications when flags change. (This is my current understanding of how
|
|
* gmail operates from internet searches; we're not quite yet to protocol
|
|
* experimentation yet.)
|
|
*
|
|
* The idea is accordingly that we will use IDLE notifications as a hint that
|
|
* we should do a SEARCH for new messages. It is that search that will update
|
|
* our accuracy information and only that.
|
|
*/
|
|
function ImapFolderConn(account, storage) {
|
|
this._account = account;
|
|
this._storage = storage;
|
|
|
|
logic.defineScope(this, 'ImapFolderConn', {
|
|
accountId: account.id,
|
|
folderId: storage.folderId
|
|
});
|
|
|
|
this._conn = null;
|
|
this.box = null;
|
|
|
|
this._deathback = null;
|
|
}
|
|
ImapFolderConn.prototype = {
|
|
/**
|
|
* Acquire a connection and invoke the callback once we have it and we have
|
|
* entered the folder. This method should only be called when running
|
|
* inside `runMutexed`.
|
|
*
|
|
* @args[
|
|
* @param[callback @func[
|
|
* @args[
|
|
* @param[folderConn ImapFolderConn]
|
|
* @param[storage FolderStorage]
|
|
* ]
|
|
* ]]
|
|
* @param[deathback Function]{
|
|
* Invoked if the connection dies.
|
|
* }
|
|
* @param[label String]{
|
|
* A debugging label to name the purpose of the connection.
|
|
* }
|
|
* @param[dieOnConnectFailure #:optional Boolean]{
|
|
* See `ImapAccount.__folderDemandsConnection`.
|
|
* }
|
|
* ]
|
|
*/
|
|
acquireConn: function(callback, deathback, label, dieOnConnectFailure) {
|
|
var self = this;
|
|
this._deathback = deathback;
|
|
this._account.__folderDemandsConnection(
|
|
this._storage.folderId, label,
|
|
function gotconn(conn) {
|
|
self._conn = conn;
|
|
// Now we have a connection, but it's not in the folder.
|
|
// (If we were doing fancier sync like QRESYNC, we would not enter
|
|
// in such a blase fashion.)
|
|
self._conn.selectMailbox(self._storage.folderMeta.path,
|
|
function openedBox(err, box) {
|
|
if (err) {
|
|
console.error('Problem entering folder',
|
|
self._storage.folderMeta.path);
|
|
self._conn = null;
|
|
// hand the connection back, noting a resource problem
|
|
self._account.__folderDoneWithConnection(
|
|
self._conn, false, true);
|
|
if (self._deathback) {
|
|
var deathback = self._deathback;
|
|
self.clearErrorHandler();
|
|
deathback();
|
|
}
|
|
return;
|
|
}
|
|
self.box = box;
|
|
callback(self, self._storage);
|
|
});
|
|
},
|
|
function deadconn() {
|
|
self._conn = null;
|
|
if (self._deathback) {
|
|
var deathback = self._deathback;
|
|
self.clearErrorHandler();
|
|
deathback();
|
|
}
|
|
},
|
|
dieOnConnectFailure);
|
|
},
|
|
|
|
relinquishConn: function() {
|
|
if (!this._conn)
|
|
return;
|
|
|
|
this.clearErrorHandler();
|
|
this._account.__folderDoneWithConnection(this._conn, true, false);
|
|
this._conn = null;
|
|
},
|
|
|
|
/**
|
|
* If no connection, acquires one and also sets up
|
|
* deathback if connection is lost.
|
|
*
|
|
* See `acquireConn` for argument docs.
|
|
*/
|
|
withConnection: function (callback, deathback, label, dieOnConnectFailure) {
|
|
if (!this._conn) {
|
|
this.acquireConn(function () {
|
|
this.withConnection(callback, deathback, label);
|
|
}.bind(this), deathback, label, dieOnConnectFailure);
|
|
return;
|
|
}
|
|
|
|
this._deathback = deathback;
|
|
callback(this);
|
|
},
|
|
|
|
/**
|
|
* Resets error handling that may be triggered during
|
|
* loss of connection.
|
|
*/
|
|
clearErrorHandler: function () {
|
|
this._deathback = null;
|
|
},
|
|
|
|
reselectBox: function(callback) {
|
|
this._conn.selectMailbox(this._storage.folderMeta.path, callback);
|
|
},
|
|
|
|
/**
|
|
* Perform a SEARCH for the purposes of folder synchronization. In the event
|
|
* we are unable to reach the server (we are offline, the server is down,
|
|
* nework troubles), the `abortedCallback` will be invoked. Note that it can
|
|
* take many seconds for us to conclusively fail to reach the server.
|
|
*
|
|
* Track an isRetry flag to ensure we don't fall into an infinite retry loop.
|
|
*/
|
|
_timelySyncSearch: function(searchOptions, searchedCallback,
|
|
abortedCallback, progressCallback, isRetry) {
|
|
var gotSearchResponse = false;
|
|
|
|
// If we don't have a connection, get one, then re-call.
|
|
if (!this._conn) {
|
|
// XXX the abortedCallback should really only be used for the duration
|
|
// of this request, but it will end up being used for the entire duration
|
|
// our folder holds on to the connection. This is not a great idea as
|
|
// long as we are leaving the IMAP connection idling in the folder (which
|
|
// causes us to not release the connection back to the account). We
|
|
// should tie this to the mutex or something else transactional.
|
|
this.acquireConn(
|
|
this._timelySyncSearch.bind(this,
|
|
searchOptions, searchedCallback,
|
|
abortedCallback, progressCallback,
|
|
/* isRetry: */ isRetry),
|
|
abortedCallback, 'sync', true);
|
|
return;
|
|
}
|
|
// We do have a connection. Hopefully the connection is still
|
|
// valid and functional. However, since this connection may have
|
|
// been hanging around a while, sending data now might trigger a
|
|
// connection reset notification. In other words, if the
|
|
// connection has gone stale, we want to grab a new connection and
|
|
// retry before aborting.
|
|
else {
|
|
if (!isRetry) {
|
|
var origAbortedCallback = abortedCallback;
|
|
abortedCallback = (function() {
|
|
// Here, we've acquired an already-connected socket. If we
|
|
// were already connected, but failed to receive a response
|
|
// from the server, this socket is effectively dead. In that
|
|
// case, retry the SEARCH once again with a fresh connection,
|
|
// if we haven't already retried the request.
|
|
if (!gotSearchResponse) {
|
|
console.warn('Broken connection for SEARCH. Retrying.');
|
|
this._timelySyncSearch(searchOptions, searchedCallback,
|
|
origAbortedCallback, progressCallback,
|
|
/* isRetry: */ true);
|
|
}
|
|
// Otherwise, we received an error from this._conn.search
|
|
// below (i.e. there was a legitimate server problem), or we
|
|
// already retried, so we should actually give up.
|
|
else {
|
|
origAbortedCallback();
|
|
}
|
|
}.bind(this));
|
|
}
|
|
this._deathback = abortedCallback;
|
|
}
|
|
|
|
// Having a connection is 10% of the battle
|
|
if (progressCallback)
|
|
progressCallback(0.1);
|
|
|
|
// Gmail IMAP servers cache search results until your connection
|
|
// gets notified of new messages via an unsolicited server
|
|
// response. Sending a command like NOOP is required to flush the
|
|
// cache and force SEARCH to return new messages that have just
|
|
// been received. Other IMAP servers don't need this as far as we know.
|
|
// See <https://bugzilla.mozilla.org/show_bug.cgi?id=933079>.
|
|
if (this._account.isGmail) {
|
|
this._conn.exec('NOOP');
|
|
}
|
|
|
|
this._conn.search(searchOptions, { byUid: true }, function(err, uids) {
|
|
gotSearchResponse = true;
|
|
if (err) {
|
|
console.error('Search error on', searchOptions, 'err:', err);
|
|
abortedCallback();
|
|
return;
|
|
}
|
|
searchedCallback(uids);
|
|
});
|
|
},
|
|
|
|
syncDateRange: function() {
|
|
var args = Array.slice(arguments);
|
|
var self = this;
|
|
|
|
require(['imap/protocol/sync'], function(_sync) {
|
|
$imapsync = _sync;
|
|
(self.syncDateRange = self._lazySyncDateRange).apply(self, args);
|
|
});
|
|
},
|
|
|
|
/**
|
|
* Perform a search to find all the messages in the given date range.
|
|
* Meanwhile, load the set of messages from storage. Infer deletion of the
|
|
* messages we already know about that should exist in the search results but
|
|
* do not. Retrieve information on the messages we don't know anything about
|
|
* and update the metadata on the messages we do know about.
|
|
*
|
|
* An alternate way to accomplish the new/modified/deleted detection for a
|
|
* range might be to do a search over the UID range of new-to-us UIDs and
|
|
* then perform retrieval on what we get back. We would do a flag fetch for
|
|
* all the UIDs we already know about and use that to both get updated
|
|
* flags and infer deletions from UIDs that don't report back. Except that
|
|
* might not work because the standard doesn't seem to say that if we
|
|
* specify gibberish UIDs that it should keep going for the UIDs that are
|
|
* not gibberish. Also, it's not clear what the performance impact of the
|
|
* additional search constraint might be on server performance. (Of course,
|
|
* if the server does not have an index on internaldate, these queries are
|
|
* going to be very expensive and the UID limitation would probably be a
|
|
* mercy to the server.)
|
|
*
|
|
* IMAP servers do not treat the SINCE and BEFORE options to IMAP
|
|
* SEARCH consistently. Because we compare messages in chunks of
|
|
* time-ranges, a message may seem like it has been deleted, when it
|
|
* actually just fell into the adjacent range bucket (Bug 886534).
|
|
* To correct for this, we do the following:
|
|
*
|
|
* 1. When we sync (whether PASTWARDS or FUTUREWARDS), we include
|
|
* messages from a bit before and after the range we asked the
|
|
* server for.
|
|
|
|
* 2. Compare those messages to the list the server returned. For
|
|
* any messages which we have locally, but the server did not
|
|
* return:
|
|
*
|
|
* a) Delete any messages which are unambiguously within our
|
|
* current time range.
|
|
*
|
|
* b) Mark any messages we expected to see (but didn't) with an
|
|
* indicator saying "we asked the server for messages in this
|
|
* time range, but we couldn't find it". If a message was
|
|
* already missing, expand the range to cover the current
|
|
* range also, indicating that the message still wasn't found
|
|
* after a wider search.
|
|
*
|
|
* c) Inspect the "missing range" of each message. If the range
|
|
* covers at least a day before and after the header's date,
|
|
* delete the message. The server didn't return it to us even
|
|
* though we checked a full day before and after the message.
|
|
*
|
|
* d) If the server returns the message in a sync and we haven't
|
|
* deleted it yet, clear the "missing" flag and start over.
|
|
*
|
|
* 3. Because we always sync time ranges farther into the past to
|
|
* show the user new messages, the ambiguity between "deleted or
|
|
* just hidden" disappears as we get information from continued
|
|
* syncs.
|
|
*
|
|
* TLDR: Messages on the ends of SEARCH ranges may fall into
|
|
* adjacent sync ranges. Don't freak out and delete a message
|
|
* just because it didn't show up in this exact range we asked
|
|
* for. Only delete the message if we checked all around where
|
|
* it was supposed to show up, and it never did.
|
|
*
|
|
* @args[
|
|
* @param[startTS @oneof[null DateMS]]{
|
|
* If non-null, inclusive "SINCE" constraint to use, otherwise the
|
|
* constraint is omitted.
|
|
* }
|
|
* @param[endTS @oneof[null DateMS]]{
|
|
* If non-null, exclusive "BEFORE" constraint to use, otherwise the
|
|
* constraint is omitted.
|
|
* }
|
|
* ]
|
|
*/
|
|
_lazySyncDateRange: function(startTS, endTS, accuracyStamp,
|
|
doneCallback, progressCallback) {
|
|
|
|
var scope = logic.subscope(this, { startTS: startTS, endTS: endTS });
|
|
|
|
if (startTS && endTS && SINCE(startTS, endTS)) {
|
|
logic(scope, 'illegalSync');
|
|
doneCallback('invariant');
|
|
return;
|
|
}
|
|
|
|
var self = this;
|
|
var storage = self._storage;
|
|
var completed = false;
|
|
|
|
console.log('syncDateRange:', startTS, endTS);
|
|
logic(scope, 'syncDateRange_begin');
|
|
|
|
// IMAP Search
|
|
|
|
// We don't care about deleted messages, it's best that we're not
|
|
// aware of them. However, it's important to keep in mind that
|
|
// this means that EXISTS provides us with an upper bound on the
|
|
// messages in the folder since we are blinding ourselves to
|
|
// deleted messages.
|
|
var searchOptions = { not: { deleted: true } };
|
|
if (startTS) {
|
|
searchOptions.since = new Date(startTS);
|
|
}
|
|
if (endTS) {
|
|
searchOptions.before = new Date(endTS);
|
|
}
|
|
|
|
var imapSearchPromise = new Promise(function(resolve) {
|
|
this._timelySyncSearch(
|
|
searchOptions,
|
|
resolve,
|
|
function abortedSearch() {
|
|
if (completed)
|
|
return;
|
|
completed = true;
|
|
logic(scope, 'syncDateRange_end', {
|
|
full: 0, flags: 0, deleted: 0
|
|
});
|
|
doneCallback('aborted');
|
|
}.bind(this),
|
|
progressCallback,
|
|
/* isRetry: */ false);
|
|
}.bind(this));
|
|
|
|
// Database Fetch
|
|
|
|
// Fetch messages from the database, extending the search by a day
|
|
// on either side to prevent timezone-related problems (bug 886534).
|
|
|
|
var dbStartTS = (startTS ? startTS - $sync.IMAP_SEARCH_AMBIGUITY_MS : null);
|
|
var dbEndTS = (endTS ? endTS + $sync.IMAP_SEARCH_AMBIGUITY_MS : null);
|
|
logic(scope, 'database-lookup', {
|
|
dbStartTS: dbStartTS,
|
|
dbEndTS: dbEndTS
|
|
});
|
|
var databaseFetchPromise = new Promise(function(resolve) {
|
|
storage.getAllMessagesInImapDateRange(dbStartTS, dbEndTS, resolve);
|
|
});
|
|
|
|
// Combine the results:
|
|
|
|
Promise.all([
|
|
imapSearchPromise,
|
|
databaseFetchPromise
|
|
]).then(function(results) {
|
|
var serverUIDs = results[0];
|
|
var dbHeaders = results[1];
|
|
var effectiveEndTS = endTS || quantizeDate(NOW() + DAY_MILLIS);
|
|
var curDaysDelta = Math.round((effectiveEndTS - startTS) / DAY_MILLIS);
|
|
|
|
// ----------------------------------------------------------------
|
|
// BISECTION SPECIAL CASE: If we have a lot of messages to
|
|
// process and we're searching more than one day, we can shrink
|
|
// our search.
|
|
|
|
var shouldBisect = (serverUIDs.length > $sync.BISECT_DATE_AT_N_MESSAGES &&
|
|
curDaysDelta > 1);
|
|
|
|
console.log(
|
|
'[syncDateRange]',
|
|
'Should bisect?', shouldBisect ? '***YES, BISECT!***' : 'no.',
|
|
'curDaysDelta =', curDaysDelta,
|
|
'serverUIDs.length =', serverUIDs.length);
|
|
|
|
if (shouldBisect) {
|
|
// mark the bisection abort...
|
|
logic(scope, 'syncDateRange_end');
|
|
var bisectInfo = {
|
|
oldStartTS: startTS,
|
|
oldEndTS: endTS,
|
|
numHeaders: serverUIDs.length,
|
|
curDaysDelta: curDaysDelta,
|
|
newStartTS: startTS,
|
|
newEndTS: endTS,
|
|
};
|
|
// If we were being used for a refresh, they may want us to stop
|
|
// and change their sync strategy.
|
|
if (doneCallback('bisect', bisectInfo, null) === 'abort') {
|
|
self.clearErrorHandler();
|
|
doneCallback('bisect-aborted', null);
|
|
} else {
|
|
self.syncDateRange(
|
|
bisectInfo.newStartTS,
|
|
bisectInfo.newEndTS,
|
|
accuracyStamp,
|
|
doneCallback,
|
|
progressCallback);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// end bisection special case
|
|
// ----------------------------------------------------------------
|
|
|
|
if (progressCallback) {
|
|
progressCallback(0.25);
|
|
}
|
|
|
|
// Combine the UIDs from local headers with server UIDs.
|
|
|
|
var uidSet = new Set();
|
|
var serverUidSet = new Set();
|
|
var localHeaderMap = {};
|
|
|
|
dbHeaders.forEach(function(header) {
|
|
// Ignore not-yet-synced local messages (messages without a
|
|
// srvid), such as messages from a partially-completed local
|
|
// move. Because they have no server ID, we can't compare them
|
|
// to anything currently on the server anyway.
|
|
if (header.srvid !== null) {
|
|
uidSet.add(header.srvid);
|
|
localHeaderMap[header.srvid] = header;
|
|
}
|
|
});
|
|
|
|
serverUIDs.forEach(function(uid) {
|
|
uidSet.add(uid);
|
|
serverUidSet.add(uid);
|
|
});
|
|
|
|
var imapSyncOptions = {
|
|
connection: self._conn,
|
|
storage: storage,
|
|
newUIDs: [],
|
|
knownUIDs: [],
|
|
knownHeaders: []
|
|
};
|
|
|
|
var numDeleted = 0;
|
|
var latch = $allback.latch();
|
|
|
|
// Figure out which messages are new, updated, or deleted.
|
|
uidSet.forEach(function(uid) {
|
|
var localHeader = localHeaderMap[uid] || null;
|
|
var hasServer = serverUidSet.has(uid);
|
|
|
|
// New
|
|
if (!localHeader && hasServer) {
|
|
imapSyncOptions.newUIDs.push(uid);
|
|
logic(scope, 'new-uid', { uid: uid });
|
|
}
|
|
// Updated
|
|
else if (localHeader && hasServer) {
|
|
imapSyncOptions.knownUIDs.push(uid);
|
|
imapSyncOptions.knownHeaders.push(localHeader);
|
|
|
|
if (localHeader.imapMissingInSyncRange) {
|
|
localHeader.imapMissingInSyncRange = null;
|
|
logic(scope, 'found-missing-uid', { uid: uid });
|
|
storage.updateMessageHeader(
|
|
localHeader.date, localHeader.id, true, localHeader,
|
|
/* body hint */ null, latch.defer(), { silent: true });
|
|
}
|
|
|
|
logic(scope, 'updated-uid', { uid: uid });
|
|
}
|
|
// Deleted or Ambiguously Deleted
|
|
else if (localHeader && !hasServer) {
|
|
|
|
// So, how long has this message been missing for?
|
|
var fuzz = $sync.IMAP_SEARCH_AMBIGUITY_MS;
|
|
var date = localHeader.date;
|
|
|
|
// There are 3 possible cases for imapMissingInSyncRange:
|
|
// 1) We don't have one, so just use the current search range.
|
|
// 2) It's disjoint from the current search range, so just use the
|
|
// current search range. We do this because we only track one
|
|
// range for the message, and unioning disjoint ranges erroneously
|
|
// assumes that we know something about the gap range *when we do
|
|
// not*. This situation arises because we previously had synced
|
|
// backwards in time so that we were on the "old" ambiguous side
|
|
// of the message. We now must be on the "new" ambiguous side.
|
|
// Since our sync range (currently) only ever moves backwards in
|
|
// time, it is safe for us to discard the information about the
|
|
// "old" side because we'll get that coverage again soon.
|
|
// 3) It overlaps the current range and we can take their union.
|
|
var missingRange;
|
|
if (!localHeader.imapMissingInSyncRange || // (#1)
|
|
((localHeader.imapMissingInSyncRange.endTS < startTS) || // (#2)
|
|
(localHeader.imapMissingInSyncRange.startTS > endTS))) {
|
|
// adopt/clobber!
|
|
// (Note that "Infinity" JSON stringifies to null, so be aware when
|
|
// looking at logs involving this code. But the values are
|
|
// structured cloned for bridge and database purposes and so remain
|
|
// intact.)
|
|
missingRange = localHeader.imapMissingInSyncRange =
|
|
{ startTS: startTS || 0, endTS: endTS || Infinity };
|
|
} else { // (#3, union!)
|
|
missingRange = localHeader.imapMissingInSyncRange;
|
|
// Make sure to treat 'null' startTS and endTS correctly.
|
|
// (This is a union range. We can state that we have not found the
|
|
// message in the time range SINCE missingRange.startTS and BEFORE
|
|
// missingRange.endTS.)
|
|
missingRange.startTS = Math.min(startTS || 0,
|
|
missingRange.startTS || 0);
|
|
missingRange.endTS = Math.max(endTS || Infinity,
|
|
missingRange.endTS || Infinity);
|
|
}
|
|
|
|
// Have we looked all around where the message is supposed
|
|
// to be, and the server never coughed it up? Delete it.
|
|
// (From a range perspective, we want to ensure that the missingRange
|
|
// completely contains the date +/- fuzz range. We use an inclusive
|
|
// comparison in both cases because we are comparing two ranges, not
|
|
// a single date and a range.)
|
|
if (missingRange.startTS <= date - fuzz &&
|
|
missingRange.endTS >= date + fuzz) {
|
|
logic(scope, 'unambiguously-deleted-uid',
|
|
{ uid: uid, date: date, fuzz: fuzz, missingRange: missingRange });
|
|
storage.deleteMessageHeaderAndBodyUsingHeader(localHeader);
|
|
numDeleted++;
|
|
}
|
|
// Or we haven't looked far enough... maybe it will show up
|
|
// later. We've already marked the updated "missing" range above.
|
|
else {
|
|
logic(scope, 'ambiguously-missing-uid',
|
|
{ uid: uid, missingRange: missingRange,
|
|
rangeToDelete: { startTS: date - fuzz, endTS: date + fuzz },
|
|
syncRange: { startTS: startTS, endTS: endTS }});
|
|
storage.updateMessageHeader(
|
|
localHeader.date, localHeader.id, true, localHeader,
|
|
/* body hint */ null, latch.defer(), { silent: true });
|
|
}
|
|
}
|
|
});
|
|
|
|
// Now that we've reconciled the difference between the items
|
|
// listen on the server and the items on the client, we can pass
|
|
// the hard download work into $imapsync.Sync.
|
|
latch.then(function() {
|
|
var uidSync = new $imapsync.Sync(imapSyncOptions);
|
|
uidSync.onprogress = progressCallback;
|
|
uidSync.oncomplete = function(newCount, knownCount) {
|
|
logic(scope, 'syncDateRange_end', {
|
|
full: newCount,
|
|
flags: knownCount,
|
|
deleted: numDeleted
|
|
});
|
|
|
|
// BrowserBox returns an integer modseq, but it's opaque and
|
|
// we already deal with strings, so cast it here.
|
|
var modseq = (self.box.highestModseq || '') + '';
|
|
storage.markSyncRange(startTS, endTS, modseq, accuracyStamp);
|
|
|
|
if (!completed) {
|
|
completed = true;
|
|
self.clearErrorHandler();
|
|
doneCallback(null, null, newCount + knownCount, startTS, endTS);
|
|
}
|
|
};
|
|
});
|
|
}.bind(this));
|
|
},
|
|
|
|
/**
|
|
* Downloads all the body representations for a given message.
|
|
*
|
|
*
|
|
* folder.downloadBodyReps(
|
|
* header,
|
|
* {
|
|
* // maximum number of bytes to fetch total (across all bodyReps)
|
|
* maximumBytesToFetch: N
|
|
* }
|
|
* callback
|
|
* );
|
|
*
|
|
*/
|
|
downloadBodyReps: function() {
|
|
var args = Array.slice(arguments);
|
|
var self = this;
|
|
|
|
require(
|
|
[
|
|
'./imapchew',
|
|
'./protocol/bodyfetcher',
|
|
'./protocol/textparser',
|
|
'./protocol/snippetparser'
|
|
],
|
|
function(
|
|
_imapchew,
|
|
_bodyfetcher,
|
|
_textparser,
|
|
_snippetparser
|
|
) {
|
|
|
|
$imapchew =_imapchew;
|
|
$imapbodyfetcher = _bodyfetcher;
|
|
$imaptextparser = _textparser;
|
|
$imapsnippetparser = _snippetparser;
|
|
|
|
(self.downloadBodyReps = self._lazyDownloadBodyReps).apply(self, args);
|
|
});
|
|
},
|
|
|
|
/**
|
|
* Initiates a request to download all body reps for a single message. If a
|
|
* snippet has not yet been generated this will also generate the snippet...
|
|
*/
|
|
_lazyDownloadBodyReps: function(header, options, callback) {
|
|
if (typeof(options) === 'function') {
|
|
callback = options;
|
|
options = null;
|
|
}
|
|
|
|
options = options || {};
|
|
|
|
var self = this;
|
|
|
|
var gotBody = function gotBody(bodyInfo) {
|
|
// target for snippet generation
|
|
var bodyRepIdx = $imapchew.selectSnippetBodyRep(header, bodyInfo);
|
|
|
|
// assume user always wants entire email unless option is given...
|
|
var overallMaximumBytes = options.maximumBytesToFetch;
|
|
|
|
var bodyParser = $imaptextparser.TextParser;
|
|
|
|
// build the list of requests based on downloading required.
|
|
var requests = [];
|
|
var latch = $allback.latch();
|
|
bodyInfo.bodyReps.forEach(function(rep, idx) {
|
|
// attempt to be idempotent by only requesting the bytes we need if we
|
|
// actually need them...
|
|
if (rep.isDownloaded)
|
|
return;
|
|
|
|
// default to the entire remaining email. We use the estimate * largish
|
|
// multiplier so even if the size estimate is wrong we should fetch more
|
|
// then the requested number of bytes which if truncated indicates the
|
|
// end of the bodies content.
|
|
var bytesToFetch = Math.min(rep.sizeEstimate * 5, MAX_FETCH_BYTES);
|
|
|
|
if (overallMaximumBytes !== undefined) {
|
|
// when we fetch partial results we need to use the snippet parser.
|
|
bodyParser = $imapsnippetparser.SnippetParser;
|
|
|
|
// issued enough downloads
|
|
if (overallMaximumBytes <= 0) {
|
|
return;
|
|
}
|
|
|
|
// if our estimate is greater then expected number of bytes
|
|
// request the maximum allowed.
|
|
if (rep.sizeEstimate > overallMaximumBytes) {
|
|
bytesToFetch = overallMaximumBytes;
|
|
}
|
|
|
|
// subtract the estimated byte size
|
|
overallMaximumBytes -= rep.sizeEstimate;
|
|
}
|
|
|
|
// For a byte-serve request, we need to request at least 1 byte, so
|
|
// request some bytes. This is a logic simplification that should not
|
|
// need to be used because imapchew.js should declare 0-byte files
|
|
// fully downloaded when their parts are created, but better a wasteful
|
|
// network request than breaking here.
|
|
if (bytesToFetch <= 0)
|
|
bytesToFetch = 64;
|
|
|
|
// CONDITIONAL LOGIC BARRIER CONDITIONAL LOGIC BARRIER DITTO DITTO
|
|
// Do not do return/continue after this point because we call
|
|
// latch.defer below, and we break if we call it and then throw away
|
|
// that callback without calling it. (Unsurprisingly.)
|
|
|
|
var request = {
|
|
uid: header.srvid,
|
|
partInfo: rep._partInfo,
|
|
bodyRepIndex: idx,
|
|
createSnippet: idx === bodyRepIdx,
|
|
headerUpdatedCallback: latch.defer(header.srvid + '-' + rep._partInfo)
|
|
};
|
|
|
|
// we may only need a subset of the total number of bytes.
|
|
if (overallMaximumBytes !== undefined || rep.amountDownloaded) {
|
|
// request the remainder
|
|
request.bytes = [
|
|
rep.amountDownloaded,
|
|
bytesToFetch
|
|
];
|
|
}
|
|
|
|
requests.push(request);
|
|
});
|
|
|
|
// we may not have any requests bail early if so.
|
|
if (!requests.length) {
|
|
callback(null, bodyInfo); // no requests === success
|
|
return;
|
|
}
|
|
|
|
var fetch = new $imapbodyfetcher.BodyFetcher(
|
|
self._conn,
|
|
bodyParser,
|
|
requests
|
|
);
|
|
|
|
self._handleBodyFetcher(fetch, header, bodyInfo, latch.defer('body'));
|
|
latch.then(function(results) {
|
|
callback($allback.extractErrFromCallbackArgs(results), bodyInfo);
|
|
});
|
|
};
|
|
|
|
this._storage.getMessageBody(header.suid, header.date, gotBody);
|
|
},
|
|
|
|
/**
|
|
* Wrapper around common bodyRep updates...
|
|
*/
|
|
_handleBodyFetcher: function(fetcher, header, body, bodyUpdatedCallback) {
|
|
var event = {
|
|
changeDetails: {
|
|
bodyReps: []
|
|
}
|
|
};
|
|
|
|
// This will be invoked once per body part that is successfully downloaded
|
|
// or fails to download.
|
|
fetcher.onparsed = function(err, req, resp) {
|
|
if (err) {
|
|
req.headerUpdatedCallback(err);
|
|
return;
|
|
}
|
|
|
|
$imapchew.updateMessageWithFetch(header, body, req, resp);
|
|
|
|
header.bytesToDownloadForBodyDisplay =
|
|
$imapchew.calculateBytesToDownloadForImapBodyDisplay(body);
|
|
|
|
// Always update the header so that we can save
|
|
// bytesToDownloadForBodyDisplay, which will tell the UI whether
|
|
// or not we can show the message body right away.
|
|
this._storage.updateMessageHeader(
|
|
header.date,
|
|
header.id,
|
|
false,
|
|
header,
|
|
body,
|
|
req.headerUpdatedCallback.bind(null, null) // no error
|
|
);
|
|
|
|
event.changeDetails.bodyReps.push(req.bodyRepIndex);
|
|
}.bind(this);
|
|
|
|
// This will be invoked after all of the onparsed events have fired.
|
|
fetcher.onend = function() {
|
|
// Since we no longer have any updates to make to the body, we want to
|
|
// finally update it now.
|
|
this._storage.updateMessageBody(
|
|
header,
|
|
body,
|
|
{},
|
|
event,
|
|
bodyUpdatedCallback.bind(null, null) // we do not/cannot error
|
|
);
|
|
}.bind(this);
|
|
},
|
|
|
|
/**
|
|
* The actual work of downloadBodies, lazily replaces downloadBodies once
|
|
* module deps are loaded.
|
|
*/
|
|
_lazyDownloadBodies: function(headers, options, callback) {
|
|
var downloadsNeeded = 0;
|
|
var latch = $allback.latch();
|
|
for (var i = 0; i < headers.length; i++) {
|
|
// We obviously can't do anything with null header references.
|
|
// To avoid redundant work, we also don't want to do any fetching if we
|
|
// already have a snippet. This could happen because of the extreme
|
|
// potential for a caller to spam multiple requests at us before we
|
|
// service any of them. (Callers should only have one or two outstanding
|
|
// jobs of this and do their own suppression tracking, but bugs happen.)
|
|
var header = headers[i];
|
|
if (!header || header.snippet !== null) {
|
|
continue;
|
|
}
|
|
|
|
// This isn't absolutely guaranteed to be 100% correct, but is good enough
|
|
// for indicating to the caller that we did some work.
|
|
downloadsNeeded++;
|
|
this.downloadBodyReps(headers[i], options, latch.defer(header.suid));
|
|
}
|
|
latch.then(function(results) {
|
|
callback($allback.extractErrFromCallbackArgs(results), downloadsNeeded);
|
|
});
|
|
},
|
|
|
|
/**
|
|
* Download snippets or entire bodies for a set of headers.
|
|
*/
|
|
downloadBodies: function() {
|
|
var args = Array.slice(arguments);
|
|
var self = this;
|
|
|
|
require(
|
|
['./imapchew', './protocol/bodyfetcher', './protocol/snippetparser'],
|
|
function(
|
|
_imapchew,
|
|
_bodyfetcher,
|
|
_snippetparser
|
|
) {
|
|
|
|
$imapchew =_imapchew;
|
|
$imapbodyfetcher = _bodyfetcher;
|
|
$imapsnippetparser = _snippetparser;
|
|
|
|
(self.downloadBodies = self._lazyDownloadBodies).apply(self, args);
|
|
});
|
|
},
|
|
|
|
downloadMessageAttachments: function(uid, partInfos, callback, progress) {
|
|
require(['mimeparser'], function(MimeParser) {
|
|
var conn = this._conn;
|
|
var self = this;
|
|
|
|
var latch = $allback.latch();
|
|
var anyError = null;
|
|
var bodies = [];
|
|
|
|
partInfos.forEach(function(partInfo, index) {
|
|
var partKey = 'body.peek[' + partInfo.part + ']';
|
|
var partDone = latch.defer(partInfo.part);
|
|
conn.listMessages(
|
|
uid,
|
|
[partKey],
|
|
{ byUid: true },
|
|
function(err, messages) {
|
|
if (err) {
|
|
anyError = err;
|
|
console.error('attachments:download-error', {
|
|
error: err,
|
|
part: partInfo.part,
|
|
type: partInfo.type
|
|
});
|
|
partDone();
|
|
return;
|
|
}
|
|
|
|
// We only receive one message per each listMessages call.
|
|
var msg = messages[0];
|
|
|
|
// Find the proper response key of the message. Since this
|
|
// response object is a lightweight wrapper around the
|
|
// response returned from the IRC server and it's possible
|
|
// there are poorly-behaved servers out there, best to err
|
|
// on the side of safety.
|
|
var bodyPart;
|
|
for (var key in msg) {
|
|
if (/body\[/.test(key)) {
|
|
bodyPart = msg[key];
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!bodyPart) {
|
|
console.error('attachments:download-error', {
|
|
error: 'no body part?',
|
|
requestedPart: partKey,
|
|
responseKeys: Object.keys(msg)
|
|
});
|
|
partDone();
|
|
return;
|
|
}
|
|
|
|
// TODO: stream attachments, bug 1047032
|
|
var parser = new MimeParser();
|
|
// TODO: escape partInfo.type/encoding
|
|
parser.write('Content-Type: ' + partInfo.type + '\r\n');
|
|
parser.write('Content-Transfer-Encoding: ' + partInfo.encoding + '\r\n');
|
|
parser.write('\r\n');
|
|
parser.write(bodyPart);
|
|
parser.end(); // Parsing is actually synchronous.
|
|
|
|
var node = parser.node;
|
|
|
|
bodies[index] = new Blob([node.content], {
|
|
type: node.contentType.value
|
|
});
|
|
|
|
partDone();
|
|
});
|
|
});
|
|
|
|
latch.then(function(results) {
|
|
callback(anyError, bodies);
|
|
});
|
|
}.bind(this));
|
|
},
|
|
|
|
shutdown: function() {
|
|
},
|
|
};
|
|
|
|
function ImapFolderSyncer(account, folderStorage) {
|
|
this._account = account;
|
|
this.folderStorage = folderStorage;
|
|
|
|
logic.defineScope(this, 'ImapFolderSyncer', {
|
|
accountId: account.id,
|
|
folderId: folderStorage.folderId
|
|
});
|
|
|
|
this._syncSlice = null;
|
|
/**
|
|
* The timestamp to use for `markSyncRange` for all syncs in this higher
|
|
* level sync. Accuracy time-info does not need high precision, so this
|
|
* results in fewer accuracy structures and simplifies our decision logic
|
|
* in `sliceOpenMostRecent`.
|
|
*/
|
|
this._curSyncAccuracyStamp = null;
|
|
/**
|
|
* @oneof[
|
|
* @case[1]{
|
|
* Growing older/into the past.
|
|
* }
|
|
* @case[-1]{
|
|
* Growing into the present/future.
|
|
* }
|
|
* ]{
|
|
* Sync growth direction. Numeric values chosen to be consistent with
|
|
* slice semantics (which are oriented like they are because the slices
|
|
* display messages from newest to oldest).
|
|
* }
|
|
*/
|
|
this._curSyncDir = 1;
|
|
/**
|
|
* Synchronization is either 'grow' or 'refresh'. Growth is when we just
|
|
* want to learn about some new messages. Refresh is when we know we have
|
|
* already synchronized a time region and want to fully update it and so will
|
|
* keep going until we hit our `syncThroughTS` threshold.
|
|
*/
|
|
this._curSyncIsGrow = null;
|
|
/**
|
|
* The timestamp that will anchor the next synchronization.
|
|
*/
|
|
this._nextSyncAnchorTS = null;
|
|
/**
|
|
* In the event of a bisection, this is the timestamp to fall back to rather
|
|
* than continuing from our
|
|
*/
|
|
this._fallbackOriginTS = null;
|
|
/**
|
|
* The farthest timestamp that we should synchronize through. The value
|
|
* null is potentially meaningful if we are synchronizing FUTUREWARDS.
|
|
*/
|
|
this._syncThroughTS = null;
|
|
/**
|
|
* The number of days we are looking into the past in the current sync step.
|
|
*/
|
|
this._curSyncDayStep = null;
|
|
/**
|
|
* If non-null, then we must synchronize all the way through the provided date
|
|
* before we begin increasing _curSyncDayStep. This helps us avoid
|
|
* oscillation where we make the window too large, shrink it, but then find
|
|
* find nothing. Since we know that there are going to be a lot of messages
|
|
* before we hit this date, it makes sense to keep taking smaller sync steps.
|
|
*/
|
|
this._curSyncDoNotGrowBoundary = null;
|
|
/**
|
|
* The callback to invoke when we complete the sync, regardless of success.
|
|
*/
|
|
this._curSyncDoneCallback = null;
|
|
|
|
this.folderConn = new ImapFolderConn(account, folderStorage);
|
|
}
|
|
exports.ImapFolderSyncer = ImapFolderSyncer;
|
|
ImapFolderSyncer.prototype = {
|
|
/**
|
|
* Although we do have some errbackoff stuff we do, we can always try to
|
|
* synchronize. The errbackoff is just a question of when we will retry.
|
|
*/
|
|
syncable: true,
|
|
|
|
/**
|
|
* Can we grow this sync range? IMAP always lets us do this.
|
|
*/
|
|
get canGrowSync() {
|
|
// Some folders, like localdrafts and outbox, cannot be synced
|
|
// because they are local-only.
|
|
return !this.folderStorage.isLocalOnly;
|
|
},
|
|
|
|
/**
|
|
* Perform an initial synchronization of a folder from now into the past,
|
|
* starting with the specified step size.
|
|
*/
|
|
initialSync: function(slice, initialDays, syncCallback,
|
|
doneCallback, progressCallback) {
|
|
syncCallback('sync', false /* Ignore Headers */);
|
|
// We want to enter the folder and get the box info so we can know if we
|
|
// should trigger our SYNC_WHOLE_FOLDER_AT_N_MESSAGES logic.
|
|
// _timelySyncSearch is what will get called next either way, and it will
|
|
// just reuse the connection and will correctly update the deathback so
|
|
// that our deathback is no longer active.
|
|
this.folderConn.withConnection(
|
|
function(folderConn, storage) {
|
|
// Flag to sync the whole range if we
|
|
var syncWholeTimeRange = false;
|
|
if (folderConn && folderConn.box &&
|
|
folderConn.box.exists <
|
|
$sync.SYNC_WHOLE_FOLDER_AT_N_MESSAGES) {
|
|
syncWholeTimeRange = true;
|
|
}
|
|
|
|
this._startSync(
|
|
slice, PASTWARDS, // sync into the past
|
|
'grow',
|
|
null, // start syncing from the (unconstrained) future
|
|
$sync.OLDEST_SYNC_DATE, // sync no further back than this constant
|
|
null,
|
|
syncWholeTimeRange ? null : initialDays,
|
|
doneCallback, progressCallback);
|
|
}.bind(this),
|
|
function died() {
|
|
doneCallback('aborted');
|
|
},
|
|
'initialSync', true);
|
|
},
|
|
|
|
/**
|
|
* Perform a refresh synchronization covering the requested time range. This
|
|
* may be converted into multiple smaller synchronizations, but the completion
|
|
* notification will only be generated once the entire time span has been
|
|
* synchronized.
|
|
*/
|
|
refreshSync: function(slice, dir, startTS, endTS, origStartTS,
|
|
doneCallback, progressCallback) {
|
|
// timezone compensation happens in the caller
|
|
this._startSync(
|
|
slice, dir,
|
|
'refresh', // this is a refresh, not a grow!
|
|
dir === PASTWARDS ? endTS : startTS,
|
|
dir === PASTWARDS ? startTS : endTS,
|
|
origStartTS,
|
|
/* syncStepDays */ null, doneCallback, progressCallback);
|
|
},
|
|
|
|
/**
|
|
* Synchronize into a time period not currently covered. Growth has an
|
|
* explicit direction and explicit origin timestamp.
|
|
*
|
|
* @args[
|
|
* @param[slice]
|
|
* @param[growthDirection[
|
|
* @param[anchorTS]
|
|
* @param[syncStepDays]
|
|
* @param[doneCallback]
|
|
* @param[progressCallback]
|
|
* ]
|
|
* @return[Boolean]{
|
|
* Returns false if no sync is necessary.
|
|
* }
|
|
*/
|
|
growSync: function(slice, growthDirection, anchorTS, syncStepDays,
|
|
doneCallback, progressCallback) {
|
|
var syncThroughTS;
|
|
if (growthDirection === PASTWARDS) {
|
|
syncThroughTS = $sync.OLDEST_SYNC_DATE;
|
|
}
|
|
else { // FUTUREWARDS
|
|
syncThroughTS = null;
|
|
}
|
|
|
|
this._startSync(slice, growthDirection, 'grow',
|
|
anchorTS, syncThroughTS, null, syncStepDays,
|
|
doneCallback, progressCallback);
|
|
},
|
|
|
|
_startSync: function ifs__startSync(slice, dir, syncTypeStr,
|
|
originTS, syncThroughTS, fallbackOriginTS,
|
|
syncStepDays,
|
|
doneCallback, progressCallback) {
|
|
var startTS, endTS;
|
|
this._syncSlice = slice;
|
|
this._curSyncAccuracyStamp = NOW();
|
|
this._curSyncDir = dir;
|
|
this._curSyncIsGrow = (syncTypeStr === 'grow');
|
|
this._fallbackOriginTS = fallbackOriginTS;
|
|
if (dir === PASTWARDS) {
|
|
endTS = originTS;
|
|
if (syncStepDays) {
|
|
if (endTS)
|
|
this._nextSyncAnchorTS = startTS = endTS - syncStepDays * DAY_MILLIS;
|
|
else
|
|
this._nextSyncAnchorTS = startTS = makeDaysAgo(syncStepDays);
|
|
}
|
|
else {
|
|
startTS = syncThroughTS;
|
|
this._nextSyncAnchorTS = null;
|
|
}
|
|
}
|
|
else { // FUTUREWARDS
|
|
startTS = originTS;
|
|
if (syncStepDays) {
|
|
this._nextSyncAnchorTS = endTS = startTS + syncStepDays * DAY_MILLIS;
|
|
}
|
|
else {
|
|
endTS = syncThroughTS;
|
|
this._nextSyncAnchorTS = null;
|
|
}
|
|
}
|
|
this._syncThroughTS = syncThroughTS;
|
|
this._curSyncDayStep = syncStepDays;
|
|
this._curSyncDoNotGrowBoundary = null;
|
|
this._curSyncDoneCallback = doneCallback;
|
|
|
|
this.folderConn.syncDateRange(startTS, endTS, this._curSyncAccuracyStamp,
|
|
this.onSyncCompleted.bind(this),
|
|
progressCallback);
|
|
},
|
|
|
|
_doneSync: function ifs__doneSync(err) {
|
|
// The desired number of headers is always a rough request value which is
|
|
// intended to be a new thing for each request. So we don't want extra
|
|
// desire building up, so we set it to what we have every time.
|
|
//
|
|
this._syncSlice.desiredHeaders = this._syncSlice.headers.length;
|
|
|
|
// Save our state even if there was an error because we may have accumulated
|
|
// some partial state. Additionally, don't *actually* complete until the
|
|
// save has hit the disk. This is beneficial for both tests and cronsync
|
|
// which has been trying to shut us down in a race with this save
|
|
// triggering.
|
|
this._account.__checkpointSyncCompleted(function () {
|
|
if (this._curSyncDoneCallback)
|
|
this._curSyncDoneCallback(err);
|
|
|
|
this._syncSlice = null;
|
|
this._curSyncAccuracyStamp = null;
|
|
this._curSyncDir = null;
|
|
this._nextSyncAnchorTS = null;
|
|
this._syncThroughTS = null;
|
|
this._curSyncDayStep = null;
|
|
this._curSyncDoNotGrowBoundary = null;
|
|
this._curSyncDoneCallback = null;
|
|
}.bind(this));
|
|
},
|
|
|
|
/**
|
|
* Whatever synchronization we last triggered has now completed; we should
|
|
* either trigger another sync if we still want more data, or close out the
|
|
* current sync.
|
|
*
|
|
* ## Block Flushing
|
|
*
|
|
* We only cause a call to `ImapAccount.__checkpointSyncCompleted` (via a call
|
|
* to `_doneSync`) to happen and cause dirty blocks to be written to disk when
|
|
* we are done with synchronization. This is because this method declares
|
|
* victory once a non-trivial amount of work has been done. In the event that
|
|
* the sync is encountering a lot of deleted messages and so keeps loading
|
|
* blocks, the memory burden is limited because we will be emptying those
|
|
* blocks out so actual memory usage (after GC) is commensurate with the
|
|
* number of (still-)existing messages. And those are what this method uses
|
|
* to determine when it is done.
|
|
*
|
|
* In the cases where we are synchronizing a ton of messages on a single day,
|
|
* we could perform checkpoints during the process, but realistically any
|
|
* device we are operating on should probably have enough memory to deal with
|
|
* these surges, so we're not doing that yet.
|
|
*
|
|
* @args[
|
|
* @param[err]
|
|
* @param[bisectInfo]
|
|
* @param[messagesSeen Number]
|
|
* @param[effStartTS DateMS]{
|
|
* Effective start date in UTC after compensating for server tz offset.
|
|
* }
|
|
* @param[effEndTS @oneof[DateMS null]]{
|
|
* Effective end date in UTC after compensating for server tz offset.
|
|
* If the end date was open-ended, then null is passed instead.
|
|
* }
|
|
* ]
|
|
*/
|
|
onSyncCompleted: function ifs_onSyncCompleted(err, bisectInfo, messagesSeen,
|
|
effStartTS, effEndTS) {
|
|
// In the event the time range had to be bisected, update our info so if
|
|
// we need to take another step we do the right thing.
|
|
if (err === 'bisect') {
|
|
var curDaysDelta = bisectInfo.curDaysDelta,
|
|
numHeaders = bisectInfo.numHeaders;
|
|
|
|
// If we had a fallback TS because we were synced to the dawn of time,
|
|
// use that and start by just cutting the range in thirds rather than
|
|
// doing a weighted bisection since the distribution might include
|
|
// a number of messages earlier than our fallback startTS.
|
|
if (this._curSyncDir === FUTUREWARDS && this._fallbackOriginTS) {
|
|
this.folderStorage.clearSyncedToDawnOfTime(this._fallbackOriginTS);
|
|
bisectInfo.oldStartTS = this._fallbackOriginTS;
|
|
this._fallbackOriginTS = null;
|
|
var effOldEndTS = bisectInfo.oldEndTS ||
|
|
quantizeDate(NOW() + DAY_MILLIS);
|
|
curDaysDelta = Math.round((effOldEndTS - bisectInfo.oldStartTS) /
|
|
DAY_MILLIS);
|
|
numHeaders = $sync.BISECT_DATE_AT_N_MESSAGES * 1.5;
|
|
}
|
|
// Sanity check the time delta; if we grew the bounds to the dawn
|
|
// of time, then our interpolation is useless and it's better for
|
|
// us to crank things way down, even if it's erroneously so.
|
|
else if (curDaysDelta > 1000)
|
|
curDaysDelta = 30;
|
|
|
|
// - Interpolate better time bounds.
|
|
// Assume a linear distribution of messages, but overestimated by
|
|
// a factor of two so we undershoot. Also make sure that we subtract off
|
|
// at least 2 days at a time. This is to ensure that in the case where
|
|
// endTS is null and we end up using makeDaysAgo that we actually shrink
|
|
// by at least 1 day (because of how rounding works for makeDaysAgo).
|
|
var shrinkScale = $sync.BISECT_DATE_AT_N_MESSAGES /
|
|
(numHeaders * 2),
|
|
dayStep = Math.max(1,
|
|
Math.min(curDaysDelta - 2,
|
|
Math.ceil(shrinkScale * curDaysDelta)));
|
|
this._curSyncDayStep = dayStep;
|
|
|
|
if (this._curSyncDir === PASTWARDS) {
|
|
bisectInfo.newEndTS = bisectInfo.oldEndTS;
|
|
this._nextSyncAnchorTS = bisectInfo.newStartTS =
|
|
makeDaysBefore(bisectInfo.newEndTS, dayStep);
|
|
this._curSyncDoNotGrowBoundary = bisectInfo.oldStartTS;
|
|
}
|
|
else { // FUTUREWARDS
|
|
bisectInfo.newStartTS = bisectInfo.oldStartTS;
|
|
this._nextSyncAnchorTS = bisectInfo.newEndTS =
|
|
makeDaysBefore(bisectInfo.newStartTS, -dayStep);
|
|
this._curSyncDoNotGrowBoundary = bisectInfo.oldEndTS;
|
|
}
|
|
|
|
// We return now without calling _doneSync because we are not done; the
|
|
// caller (syncDateRange) will re-trigger itself and keep going.
|
|
return;
|
|
}
|
|
else if (err) {
|
|
this._doneSync(err);
|
|
return;
|
|
}
|
|
|
|
console.log("Sync Completed!", this._curSyncDayStep, "days",
|
|
messagesSeen, "messages synced");
|
|
|
|
// - Slice is dead = we are done
|
|
if (this._syncSlice.isDead) {
|
|
this._doneSync();
|
|
return;
|
|
}
|
|
|
|
// If it now appears we know about all the messages in the folder, then we
|
|
// are done syncing and can mark the entire folder as synchronized. This
|
|
// requires that:
|
|
// - The direction is pastwards. (We check the oldest header, so this
|
|
// is important. We don't really need to do a future-wards variant since
|
|
// we always use pastwards for refreshes and the future-wards variant
|
|
// really does not need a fast-path since the cost of stepping to 'today'
|
|
// is much cheaper thana the cost of walking all the way to 1990.)
|
|
// - The number of messages we know about is the same as the number the
|
|
// server most recently told us are in the folder.
|
|
// - (There are no messages in the folder at all OR)
|
|
// - We have synchronized past the oldest known message header. (This,
|
|
// in combination with the fact that we always open from the most recent
|
|
// set of messages we know about, that we fully synchronize all time
|
|
// intervals (for now!), and our pastwards-direction for refreshes means
|
|
// that we can conclude we have synchronized across all messages and
|
|
// this is a sane conclusion to draw.)
|
|
//
|
|
// NB: If there are any deleted messages, this logic will not save us
|
|
// because we ignored those messages. This is made less horrible by issuing
|
|
// a time-date that expands as we go further back in time.
|
|
//
|
|
// (I have considered asking to see deleted messages too and ignoring them;
|
|
// that might be suitable. We could also just be a jerk and force an
|
|
// expunge.)
|
|
var folderMessageCount = this.folderConn && this.folderConn.box &&
|
|
this.folderConn.box.exists,
|
|
dbCount = this.folderStorage.getKnownMessageCount(),
|
|
syncedThrough =
|
|
((this._curSyncDir === PASTWARDS) ? effStartTS : effEndTS);
|
|
console.log("folder message count", folderMessageCount,
|
|
"dbCount", dbCount,
|
|
"syncedThrough", syncedThrough,
|
|
"oldest known", this.folderStorage.getOldestMessageTimestamp());
|
|
if (this._curSyncDir === PASTWARDS &&
|
|
folderMessageCount === dbCount &&
|
|
(!folderMessageCount ||
|
|
TIME_DIR_AT_OR_BEYOND(this._curSyncDir, syncedThrough,
|
|
this.folderStorage.getOldestMessageTimestamp()))
|
|
) {
|
|
// expand the accuracy range to cover everybody
|
|
this.folderStorage.markSyncedToDawnOfTime();
|
|
this._doneSync();
|
|
return;
|
|
}
|
|
// If we've synchronized to the limits of syncing in the given direction,
|
|
// we're done.
|
|
if (!this._nextSyncAnchorTS ||
|
|
TIME_DIR_AT_OR_BEYOND(this._curSyncDir, this._nextSyncAnchorTS,
|
|
this._syncThroughTS)) {
|
|
this._doneSync();
|
|
return;
|
|
}
|
|
|
|
// - Done if this is a grow and we don't want/need any more headers.
|
|
if (this._curSyncIsGrow &&
|
|
this._syncSlice.headers.length >= this._syncSlice.desiredHeaders) {
|
|
// (limited syncs aren't allowed to expand themselves)
|
|
console.log("SYNCDONE Enough headers retrieved.",
|
|
"have", this._syncSlice.headers.length,
|
|
"want", this._syncSlice.desiredHeaders,
|
|
"conn knows about", this.folderConn.box.exists,
|
|
"sync date", this._curSyncStartTS,
|
|
"[oldest defined as", $sync.OLDEST_SYNC_DATE, "]");
|
|
this._doneSync();
|
|
return;
|
|
}
|
|
|
|
// - Increase our search window size if we aren't finding anything
|
|
// Our goal is that if we are going backwards in time and aren't finding
|
|
// anything, we want to keep expanding our window
|
|
var daysToSearch, lastSyncDaysInPast;
|
|
// If we saw messages, there is no need to increase the window size. We
|
|
// also should not increase the size if we explicitly shrank the window and
|
|
// left a do-not-expand-until marker.
|
|
if (messagesSeen || (this._curSyncDoNotGrowBoundary !== null &&
|
|
!TIME_DIR_AT_OR_BEYOND(this._curSyncDir, this._nextSyncAnchorTS,
|
|
this._curSyncDoNotGrowBoundary))) {
|
|
daysToSearch = this._curSyncDayStep;
|
|
}
|
|
else {
|
|
this._curSyncDoNotGrowBoundary = null;
|
|
// This may be a fractional value because of DST
|
|
lastSyncDaysInPast = (quantizeDate(NOW()) -
|
|
this._nextSyncAnchorTS) / DAY_MILLIS;
|
|
daysToSearch = Math.ceil(this._curSyncDayStep *
|
|
$sync.TIME_SCALE_FACTOR_ON_NO_MESSAGES);
|
|
|
|
// These values used to be more conservative, but the importance of these
|
|
// guards was reduced when we switched to only syncing headers.
|
|
// At current constants (sync=3, scale=2), our doubling in the face of
|
|
// clamping is: 3, 6, 12, 24, 45, ... 90,
|
|
if (lastSyncDaysInPast < 180) {
|
|
if (daysToSearch > 45)
|
|
daysToSearch = 45;
|
|
}
|
|
else if (lastSyncDaysInPast < 365) { // 1 year
|
|
if (daysToSearch > 90)
|
|
daysToSearch = 90;
|
|
}
|
|
else if (lastSyncDaysInPast < 730) { // 2 years
|
|
if (daysToSearch > 120)
|
|
daysToSearch = 120;
|
|
}
|
|
else if (lastSyncDaysInPast < 1825) { // 5 years
|
|
if (daysToSearch > 180)
|
|
daysToSearch = 180;
|
|
}
|
|
else if (lastSyncDaysInPast < 3650) { // 10 years
|
|
if (daysToSearch > 365)
|
|
daysToSearch = 365;
|
|
}
|
|
else if (daysToSearch > 730) {
|
|
daysToSearch = 730;
|
|
}
|
|
this._curSyncDayStep = daysToSearch;
|
|
}
|
|
|
|
// - Move the time range back in time more.
|
|
var startTS, endTS;
|
|
if (this._curSyncDir === PASTWARDS) {
|
|
endTS = this._nextSyncAnchorTS;
|
|
this._nextSyncAnchorTS = startTS = makeDaysBefore(endTS, daysToSearch);
|
|
}
|
|
else { // FUTUREWARDS
|
|
startTS = this._nextSyncAnchorTS;
|
|
this._nextSyncAnchorTS = endTS = makeDaysBefore(startTS, -daysToSearch);
|
|
// If this is a bisection that originally had an endTS of null and this
|
|
// is going to be our last step, convert endTS back to a null.
|
|
// (We use the same step check from above for consistency. We also
|
|
// are trying to do the smallest perturbation to this legacy logic, so
|
|
// we only permute endTS, not _nextSyncAnchorTS although nulling it out
|
|
// should at first glance merely cause us to short-circuit into the
|
|
// _doneSync case before running the check which we know will return
|
|
// true barring the clock jumping backwards.)
|
|
if (this._syncThroughTS === null &&
|
|
TIME_DIR_AT_OR_BEYOND(this._curSyncDir, this._nextSyncAnchorTS,
|
|
this._syncThroughTS)) {
|
|
endTS = null;
|
|
}
|
|
}
|
|
this.folderConn.syncDateRange(startTS, endTS, this._curSyncAccuracyStamp,
|
|
this.onSyncCompleted.bind(this));
|
|
},
|
|
|
|
/**
|
|
* Invoked when there are no longer any live slices on the folder and no more
|
|
* active/enqueued mutex ops.
|
|
*/
|
|
allConsumersDead: function() {
|
|
this.folderConn.relinquishConn();
|
|
},
|
|
|
|
shutdown: function() {
|
|
this.folderConn.shutdown();
|
|
},
|
|
};
|
|
|
|
/**
|
|
* ALL SPECULATIVE RIGHT NOW.
|
|
*
|
|
* Like ImapFolderStorage, but with only one folder and messages named by their
|
|
* X-GM-MSGID value rather than their UID(s).
|
|
*
|
|
* Deletion processing operates slightly differently than for normal IMAP
|
|
* because a message can be removed from one of the folders we synchronize on,
|
|
* but not all of them. We don't want to be overly deletionary in that case,
|
|
* so we maintain a list of folder id's that are keeping each message alive.
|
|
*/
|
|
function GmailMessageStorage() {
|
|
}
|
|
GmailMessageStorage.prototype = {
|
|
};
|
|
|
|
}); // end define
|