b38dea0597
Share Gaia assets and adapters, keep private profiles and development documents local, and update both public READMEs.
1019 lines
41 KiB
JavaScript
1019 lines
41 KiB
JavaScript
/* -*- Mode: js; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- /
|
||
/* vim: set shiftwidth=2 tabstop=2 autoindent cindent expandtab: */
|
||
/* exported Predictions */
|
||
//
|
||
// This is a JavaScript predictive text engine: given a dictionary, a data
|
||
// structure that specifies which keys are near which other keys, and
|
||
// a string of user input, it guesses what the user meant to type or, (if
|
||
// if cannot find a word loosely matching the input) what the user is planning
|
||
// to type.
|
||
//
|
||
// *** Notes on Documentation ***
|
||
//
|
||
// We have moved the overview documentation for prediction mechanism to Mozilla
|
||
// Wiki: https://wiki.mozilla.org/Gaia/System/Keyboard/IME/Latin
|
||
// /Prediction_%26_Auto_Correction
|
||
//
|
||
// The documentation includes definitions of terminology and a few examples on
|
||
// the prediciton process.
|
||
//
|
||
// Please make sure any updates to the codes are reflected in the wiki too.
|
||
//
|
||
//
|
||
// This module defines a single global variable Predictions which is an
|
||
// object with the following methods:
|
||
//
|
||
// setDictionary: specifies the dictionary to use
|
||
//
|
||
// setNearbyKeys: specifies which keys are near which other keys
|
||
//
|
||
// predict: given an input string, asynchronously find corrections or
|
||
// predictions and pass them to a specified callback.
|
||
//
|
||
// The core part of the matching algorithm is in the process() function
|
||
// nested inside the predict() method. To fully understand it, however, you
|
||
// also need to understand the weighted Ternary Search Tree data structure
|
||
// used to represent the dictionary.
|
||
//
|
||
// The illustration of the Ternary Search Tree structure may be found at the
|
||
// Wiki page above.
|
||
//
|
||
// Also see the following online resources which include helpful diagrams:
|
||
//
|
||
// http://en.wikipedia.org/wiki/Ternary_search_tree
|
||
// http://www.strchr.com/ternary_dags
|
||
// http://www.strchr.com/dawg_predictive
|
||
//
|
||
// Note that this implementation does not convert the TST into a DAG to
|
||
// share common suffix nodes as described in the strchr.com blog
|
||
// posts. (That is an optimization that may be possible later: the shared
|
||
// suffix nodes can't hold correct word frequencies, so the search
|
||
// algorithm would have to be modified to carry the correct frequency
|
||
// through while processing a shared suffix.)
|
||
//
|
||
// TODO:
|
||
//
|
||
// Also have to figure out if something is going wrong with Polish.
|
||
// When I type an unaccented character, I'm not confident that I'm
|
||
// getting predictions that include accented versions of that character.
|
||
//
|
||
'use strict';
|
||
|
||
var Predictions = function() {
|
||
var CACHE_SIZE = 255; // how many suggestions to remember
|
||
|
||
// Weights of various permutations we do when matching input
|
||
var VARIANT_FORM_MULTIPLIER = 0.99; // slightly prefer exact match
|
||
var PUNCTUATION_INSERTION_MULTIPLIER = 0.95; // apostrophes are almost free
|
||
var NEARBY_KEY_REPLACEMENT_MULTIPLIER = 1; // adjusted by actual distance
|
||
var TRANSPOSITION_MULTIPLIER = 0.3;
|
||
var INSERTION_MULTIPLIER = 0.3;
|
||
var SUBSTITUTION_MULTIPLIER = 0.2; // for keys that are not nearby
|
||
var DELETION_MULTIPLIER = 0.1;
|
||
var ZERO_CORRECTION_PREFIX_MULTIPLIER = 10;
|
||
// profane words don't have a frequency themselves, so we bump their
|
||
// frequency to a value which will make it pop up (only do if matches input)
|
||
// 15 is a value that still allows very obvious corrections to be made
|
||
// see #803189 for more information.
|
||
var PROFANE_INPUT_MATCH_WEIGHT = 15;
|
||
|
||
// If we can't find enough exact word matches for the user's input
|
||
// we have to expand some of the candidates we found into complete
|
||
// words. But we want words that are close in length to the user's
|
||
// input. The most frequent word beginning with r in the en.us wordlist is
|
||
// received, but we don't want that as a suggestion if the user just
|
||
// type r. We want things like red and run. So for each extra character
|
||
// we have to add, we multiply the weight by this amount.
|
||
var WORD_EXTENSION_MULTIPLIER = 0.4;
|
||
|
||
// How many candidates do we consider before pausing with a setTimeout()?
|
||
// Smaller values make the prediction code more interruptible and
|
||
// possibly result in a more responsive UX. Larger values may reduce the
|
||
// total time required to get predictions
|
||
var CANDIDATES_PER_BATCH = 10;
|
||
|
||
var tree; // A typed array of bytes holding the dictionary tree
|
||
var maxWordLength; // We can reject any input longer than this
|
||
var characterTable = []; // Maps charcodes to frequency in dictionary
|
||
var variants = []; // Maps charcodes to variant forms
|
||
var rootform = []; // Maps charcodes to the root form
|
||
var nearbyKeys; // Maps charcodes to a set of codes of nearby keys
|
||
var cache; // Cache inputs to completions.
|
||
var validChars = null; // An ES6 Set containing valid chars for this dict
|
||
// including possibly valid variant forms and nearby
|
||
// keys.
|
||
|
||
// This function is called to pass our dictionary to us as an ArrayBuffer.
|
||
function setDictionary(buffer) {
|
||
cache = new LRUCache(CACHE_SIZE); // Start with a new cache
|
||
var file = new Uint8Array(buffer);
|
||
|
||
function uint32(offset) {
|
||
return (file[offset] << 24) +
|
||
(file[offset + 1] << 16) +
|
||
(file[offset + 2] << 8) +
|
||
file[offset + 3];
|
||
}
|
||
|
||
function uint16(offset) {
|
||
return (file[offset] << 8) +
|
||
file[offset + 1];
|
||
}
|
||
|
||
if (uint32(0) !== 0x46784F53 || // "FxOS"
|
||
uint32(4) !== 0x44494354) { // "DICT"
|
||
throw new Error('Invalid dictionary file');
|
||
}
|
||
|
||
if (uint32(8) !== 1) {
|
||
throw new Error('Unknown dictionary version');
|
||
}
|
||
|
||
// Read the maximum word length.
|
||
// We add 1 because word predictions can delete characters, so the
|
||
// user could type one extra character and we might still predict it.
|
||
maxWordLength = file[12] + 1;
|
||
|
||
// Read the table of characters and their frequencies
|
||
var numEntries = uint16(13);
|
||
for (var i = 0; i < numEntries; i++) {
|
||
var offset = 15 + i * 6;
|
||
characterTable[uint16(offset)] = uint32(offset + 2);
|
||
}
|
||
|
||
// The dictionary data begins right after the character table
|
||
tree = new Uint8Array(buffer, 15 + numEntries * 6);
|
||
|
||
// The rest of this function processes the character table to create a
|
||
// list of variant forms that we'll accept for each character in the
|
||
// dictionary. Variants cover case differences and unaccented forms of
|
||
// accented letters. Characters with no variants are considered word
|
||
// internal punctuation like apostophes and hyphens.
|
||
|
||
// Map from lowercase ASCII to all known accented forms of the letter
|
||
var rootToAccentedForm = {
|
||
'a': 'ÁáĂăǍǎÂâÄäȦȧẠạȀȁÀàẢảȂȃĀāĄąÅåḀḁȺⱥÃãǼǽǢǣÆæ',
|
||
'b': 'ḂḃḄḅƁɓḆḇɃƀƂƃ',
|
||
'c': 'ĆćČčÇçĈĉĊċƇƈȻȼ',
|
||
'd': 'ĎďḐḑḒḓḊḋḌḍƊɗḎḏĐđƋƌð',
|
||
'e': 'ÉéĔĕĚěȨȩÊêḘḙËëĖėẸẹȄȅÈèẺẻȆȇĒēĘę',
|
||
'f': 'ḞḟƑƒ',
|
||
'g': 'ǴǵĞğǦǧĢģĜĝĠġƓɠḠḡǤǥ',
|
||
'h': 'ḪḫȞȟḨḩĤĥⱧⱨḦḧḢḣḤḥĦħ',
|
||
'i': 'ÍíĬĭǏǐÎîÏïỊịȈȉÌìỈỉȊȋĪīĮįƗɨĨĩḬḭı',
|
||
'j': 'ĴĵɈɉ',
|
||
'k': 'ḰḱǨǩĶķⱩⱪꝂꝃḲḳƘƙḴḵꝀꝁ',
|
||
'l': 'ĹĺȽƚĽľĻļḼḽḶḷⱠⱡꝈꝉḺḻĿŀⱢɫŁł',
|
||
'm': 'ḾḿṀṁṂṃⱮɱ',
|
||
'n': 'ŃńŇňŅņṊṋṄṅṆṇǸǹƝɲṈṉȠƞÑñ',
|
||
'o': 'ÓóŎŏǑǒÔôÖöȮȯỌọŐőȌȍÒòỎỏƠơȎȏꝊꝋꝌꝍŌōǪǫØøÕõŒœ',
|
||
'p': 'ṔṕṖṗꝒꝓƤƥⱣᵽꝐꝑ',
|
||
'q': 'Ꝗꝗ',
|
||
'r': 'ŔŕŘřŖŗṘṙṚṛȐȑȒȓṞṟɌɍⱤɽ',
|
||
's': 'ŚśŠšŞşŜŝȘșṠṡṢṣß',
|
||
't': 'ŤťŢţṰṱȚțȾⱦṪṫṬṭƬƭṮṯƮʈŦŧ',
|
||
'u': 'ÚúŬŭǓǔÛûṶṷÜüṲṳỤụŰűȔȕÙùỦủƯưȖȗŪūŲųŮůŨũṴṵ',
|
||
'v': 'ṾṿƲʋṼṽ',
|
||
'w': 'ẂẃŴŵẄẅẆẇẈẉẀẁⱲⱳ',
|
||
'x': 'ẌẍẊẋ',
|
||
'y': 'ÝýŶŷŸÿẎẏỴỵỲỳƳƴỶỷỾỿȲȳɎɏỸỹ',
|
||
'z': 'ŹźŽžẐẑⱫⱬŻżẒẓȤȥẔẕƵƶ',
|
||
'α': 'άΆ',
|
||
'ε': 'έΈ',
|
||
'η': 'ήΉ',
|
||
'ι': 'ίϊΐΊΪ',
|
||
'ο': 'όΌ',
|
||
'υ': 'ύϋΰΎΫ',
|
||
'ω': 'ώΏ',
|
||
'$': '$'
|
||
};
|
||
|
||
// The reverse mapping from accented forms to the normalized ASCII form
|
||
var accentedFormToRoot = {};
|
||
for (var letter in rootToAccentedForm) {
|
||
var s = rootToAccentedForm[letter];
|
||
for (var j = 0, len = s.length; j < len; j++) {
|
||
accentedFormToRoot[s[j]] = letter;
|
||
}
|
||
}
|
||
|
||
// Now through all the characters that appear in the dictionary
|
||
// and figure out their variant forms.
|
||
for (var charcode in characterTable) {
|
||
// Start off by with an empty set of variants for each character.
|
||
variants[charcode] = '';
|
||
|
||
// Handle upper and lowercase forms
|
||
var ch = String.fromCharCode(charcode);
|
||
var upper = ch.toUpperCase();
|
||
var lower = ch.toLowerCase();
|
||
if (upper !== ch) {
|
||
variants[charcode] += upper;
|
||
// It is not upper case it is probably the root form.
|
||
// If it is an accented character, we'll override this below
|
||
rootform[charcode] = charcode;
|
||
}
|
||
|
||
if (lower !== ch) {
|
||
variants[charcode] += lower;
|
||
rootform[charcode] = lower.charCodeAt(0);
|
||
}
|
||
|
||
// Handle accented forms
|
||
if (accentedFormToRoot[ch]) {
|
||
var root = accentedFormToRoot[ch];
|
||
rootform[charcode] = root.charCodeAt(0);
|
||
|
||
// The root form and its uppercase version are variants we'll
|
||
// accept in user input instead of this accented character.
|
||
variants[charcode] += root + root.toUpperCase();
|
||
}
|
||
|
||
// log("Variants for " + ch + " " + variants[charcode]);
|
||
// log("Root form of " + ch + " " +
|
||
// String.fromCharCode(rootform[charcode]))
|
||
}
|
||
|
||
generateValidChars();
|
||
}
|
||
|
||
// latin.js passes us a data structure that holds the inverse square
|
||
// distance between keys that are near each other on the keyboard.
|
||
// This method just stores it for use later. We use these values as
|
||
// weights for nearby character replacement. With typical FirefoxOS
|
||
// keyboard layouts, adjacent keys on the same row have a value of
|
||
// about .5, keys directly above or below each other have a value of
|
||
// about .25 and keys diagonally adjacent to each other have a value of
|
||
// about .16.
|
||
function setNearbyKeys(data) {
|
||
cache = new LRUCache(CACHE_SIZE); // Discard any cached results
|
||
nearbyKeys = data;
|
||
// log("Nearby Keys: " + JSON.stringify(data));
|
||
}
|
||
|
||
function generateValidChars() {
|
||
// We're called when both |nearbyKeys| and |variants| information is ready.
|
||
// Note this relies on the fact that setNearByKeys is always called before
|
||
// setDictionary, and the setNearByKeys call is optional. Thus, we want to
|
||
// be called at setDictionary.
|
||
//
|
||
// It's not very easy to lessen the constraint, as within this module, we
|
||
// don't have knowledge on whether we're to be reinitialized, with setNBK
|
||
// coming or not, and with setNBK coming before or after setDict.
|
||
//
|
||
// Note at this moment, keys of |variants| object is the character table of
|
||
// this dictionary.
|
||
// We need to consider each variants[ch]'s
|
||
// - the ch itself (the character in the dict)
|
||
// - all the characters of variants[ch] string (different cases and
|
||
// root form (of different cases); the latter is espcially important
|
||
// if ch is accented)
|
||
// furthermore, for that ch, we want to consider as valid all the nearby
|
||
// keys of rootform[ch], and the variant form of such nearby keys.
|
||
|
||
validChars = new Set();
|
||
|
||
variants.forEach(function(variantStr, charCode) {
|
||
validChars.add(String.fromCharCode(charCode));
|
||
|
||
Array.from(variantStr).forEach(function(varCh) {
|
||
validChars.add(varCh);
|
||
});
|
||
|
||
var rootCode = rootform[charCode];
|
||
if (rootCode && nearbyKeys && nearbyKeys[rootCode]) {
|
||
Object.keys(nearbyKeys[rootCode]).forEach(
|
||
function(rootNearByCode) {
|
||
validChars.add(String.fromCharCode(rootNearByCode));
|
||
|
||
// the nearby key isn't necessarily in character table,
|
||
// so test its existence in |variants| first.
|
||
if (variants[rootNearByCode]) {
|
||
Array.from(variants[rootNearByCode]).forEach(
|
||
function(rootNearByVarCode) {
|
||
validChars.add(rootNearByVarCode);
|
||
});
|
||
}
|
||
});
|
||
}
|
||
});
|
||
}
|
||
|
||
//
|
||
// This function asynchronously computes word completions for the specified
|
||
// input string. When called, it immediately returns an object used for
|
||
// communicating with the caller and defers its computations with
|
||
// setTimeout.
|
||
//
|
||
// The returned object has an abort() method that, when called, will
|
||
// cause the predictions to be cancelled the next time it calls setTimeout()
|
||
// to pause.
|
||
//
|
||
// This method communicates with the caller by invoking the specified
|
||
// callback and onerror functions. If an exception occurs while predicting
|
||
// the onerror function is called with an error message.
|
||
//
|
||
// If no error occurs, then the callback function is called with an array
|
||
// argument. Each element of this array is also an array of two elements,
|
||
// a word and a number. The word is a proposed completion or correction,
|
||
// i.e. a 'suggestion', to the input word, and the number is the weight that
|
||
// the prediction algorithm assigns to that suggestion. Higher numbers mean
|
||
// better suggestions. Suggestions may take tens of milliseconds to compute,
|
||
// which is why this method is designed to be asynchronous. Periodically
|
||
// during the search process, the code returns to the event loop with
|
||
// setTimeout(0) which gives other code time to run and potentially call
|
||
// the abort method
|
||
//
|
||
// Before calling this function you must call setDictionary() and
|
||
// setNearbyKeys() to provide the data it needs to make predictions.
|
||
//
|
||
function predict(input, // the user's input
|
||
maxSuggestions, // how many suggestions are requested
|
||
maxCandidates, // how many candidates to consider
|
||
maxCorrections, // how many corrections to allow per
|
||
// suggestion
|
||
callback, // call this on success
|
||
onerror) // and call this on error
|
||
{
|
||
if (!tree || !nearbyKeys) {
|
||
throw Error('not initialized');
|
||
}
|
||
|
||
// The search algorithm compares the user's input to the dictionary tree
|
||
// data structure and generates a set of candidates incrementally,
|
||
// character by character.
|
||
// This variable will store the set of candidates we're evaluating as we
|
||
// do a breadth first search of the dictionary tree and allows us to
|
||
// pull out the best candidates first for further evaluation.
|
||
var candidates = new BoundedPriorityQueue(maxCandidates);
|
||
|
||
// This is where we store the best complete suggestions we've found so far.
|
||
var suggestions = new BoundedPriorityQueue(maxSuggestions);
|
||
|
||
// If the first letter of the input is a capital letter, then we
|
||
// want to capitalize the first letter of all the suggestions.
|
||
// We do this here rather than in latin.js so that we can filter out
|
||
// repeated suggestions that come in both upper and lowercase forms
|
||
var capitalize = (input[0] === input[0].toUpperCase());
|
||
|
||
// This is the object we return. It allows the caller to abort a
|
||
// prediction in progress.
|
||
var status = {
|
||
state: 'predicting',
|
||
abort: function() {
|
||
if (this.state !== 'done' && this.state !== 'aborted') {
|
||
this.state = 'aborting';
|
||
}
|
||
}
|
||
};
|
||
|
||
var cacheKey = input +
|
||
',' + maxSuggestions +
|
||
',' + maxCandidates +
|
||
',' + maxCorrections;
|
||
|
||
// Start searching for suggestions soon...
|
||
setTimeout(getSuggestions);
|
||
|
||
// But first, return the status object to the caller.
|
||
return status;
|
||
|
||
// We use this to check whether the user aborted the search and to
|
||
// set the state property appropriately.
|
||
function aborted() {
|
||
if (status.state === 'aborting') {
|
||
status.state = 'aborted';
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
function getSuggestions() {
|
||
try {
|
||
// Check the cache. If we've seen this input recently we can return
|
||
// suggestions right away.
|
||
var cached_suggestions = cache.get(cacheKey);
|
||
if (cached_suggestions) {
|
||
status.state = 'done';
|
||
status.suggestions = cached_suggestions;
|
||
callback(status.suggestions);
|
||
return;
|
||
}
|
||
|
||
// Check length and check for invalid characters. If the input is
|
||
// bad, we can reject it right away.
|
||
if (input.length > maxWordLength ||
|
||
!Array.from(input).every(function(c) {return validChars.has(c);})) {
|
||
status.state = 'done';
|
||
status.suggestions = [];
|
||
callback(status.suggestions);
|
||
return;
|
||
}
|
||
|
||
// Start off with a single root candidate. The first argument is the
|
||
// address of the root node of the tree
|
||
addCandidate(0, input, '', 1, 1, 0);
|
||
|
||
// And then process it. This will generate more candidates to
|
||
// process. processCandidates() runs until all the suggestions we want
|
||
// have been found or until all possiblities have been tried. It
|
||
// returns to the event loop with setTimeout() so the search can be
|
||
// aborted, but arranges to resume. It calls the callback when done.
|
||
processCandidates();
|
||
}
|
||
catch (e) {
|
||
status.state = 'error';
|
||
status.error = e;
|
||
onerror(e.toString() + '\n' + e.stack);
|
||
}
|
||
}
|
||
|
||
// Add a candidate to the list of promising candidates if frequency *
|
||
// multiplier is high enough. A candidate includes a pointer (byte offset)
|
||
// to a node in the tree, the portion of the user's input that has not yet
|
||
// been considered, the output string that has been generated so far, a
|
||
// number based on the highest frequency word that begins with the
|
||
// output we've generated, a multipler that adjusts that frequency based
|
||
// on how much we've modified the user's input, and a number that
|
||
// indicates how many times we've already corrected the user's input for
|
||
// this candidate.
|
||
function addCandidate(pointer, remaining, output,
|
||
multiplier, frequency, corrections)
|
||
{
|
||
var weight = frequency * multiplier;
|
||
|
||
// If no major corrections have been made to this candidate, then
|
||
// artificially increase its weight so that it appears in the
|
||
// candidates list before any corrected candidates. This should
|
||
// ensure that if the user is typing an infrequent word (but typing it
|
||
// correctly), we don't bump the actual word off the list if there are
|
||
// lots of frequent words that have similar spellings. The artificial
|
||
// weight does not carry through to the list of words, so more frequent
|
||
// words may still be predicted instead of the user's input when the user
|
||
// is typing an infrequent word. But we shouldn't ever not be able to
|
||
// find the user's input as a valid word. Adding letters to the
|
||
// end of partial input does not count as a correction so we also
|
||
// test the multiplier so that we don't boost the weight of every
|
||
// extension. But we do allow one letter to be added on and still
|
||
// get the extra weight.
|
||
if (corrections === 0 &&
|
||
multiplier > WORD_EXTENSION_MULTIPLIER * WORD_EXTENSION_MULTIPLIER) {
|
||
weight += 100;
|
||
}
|
||
// Give it a little boost if corrections are 0 but more than one
|
||
// character has been added to the input, boost depends on freq
|
||
else if (corrections === 0) {
|
||
weight += ((frequency / 32) * ZERO_CORRECTION_PREFIX_MULTIPLIER);
|
||
}
|
||
|
||
// If this candidate could never become a suggestion, don't add it
|
||
if (weight <= suggestions.threshold) {
|
||
return;
|
||
}
|
||
|
||
candidates.add({
|
||
pointer: pointer,
|
||
input: remaining,
|
||
output: output,
|
||
multiplier: multiplier,
|
||
weight: weight,
|
||
corrections: corrections
|
||
}, weight);
|
||
}
|
||
|
||
// Add a suggestion to the priority queue of suggestions
|
||
function addSuggestion(suggestion, weight) {
|
||
// If the input was capitalized, capitalize the suggestion
|
||
if (capitalize) {
|
||
suggestion = suggestion[0].toUpperCase() + suggestion.substring(1);
|
||
}
|
||
|
||
// Make sure we don't already have the suggestion in the queue
|
||
for (var i = 0, n = suggestions.items.length; i < n; i++) {
|
||
if (suggestions.items[i][0] === suggestion) {
|
||
// If the version we already have has higher weight, skip this one
|
||
if (suggestions.priorities[i] >= weight) {
|
||
return;
|
||
} else { // otherwise, remove the existing lower-weight copy
|
||
suggestions.removeItemAt(i);
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
|
||
suggestions.add([suggestion, weight], weight);
|
||
}
|
||
|
||
// Take the highest-ranked candidate from the list of candidates and
|
||
// process it. (This will often add more candidates to the list). After
|
||
// we've processed a batch of candidates this way, use setTimeout() to
|
||
// schedule the processing of the next batch after returning to the
|
||
// event loop. If there are no more candidates or if the highest ranked
|
||
// one is not highly ranked enough, then we're done finding all the
|
||
// possible candidates, and thus, we're doen for making suggestions.
|
||
function processCandidates() {
|
||
try {
|
||
if (aborted()) {
|
||
return;
|
||
}
|
||
|
||
for (var count = 0; count < CANDIDATES_PER_BATCH; count++) {
|
||
var candidate = candidates.remove();
|
||
|
||
// If there are no more candidates, or if the weight isn't
|
||
// high enough, we're done. Call the callback with the current
|
||
// set of suggestions.
|
||
if (!candidate || candidate.weight <= suggestions.threshold) {
|
||
status.state = 'done';
|
||
status.suggestions = suggestions.items;
|
||
cache.add(cacheKey, status.suggestions);
|
||
callback(status.suggestions);
|
||
return;
|
||
}
|
||
|
||
process(candidate);
|
||
|
||
//
|
||
// If the predicted suggestions don't seem right, uncomment these
|
||
// lines to see how the call to process() modifies the set of
|
||
// candiates at each step. The output is verbose, but with careful
|
||
// study it reveals what is going on in the algorithm.
|
||
//
|
||
// var s = "";
|
||
// for(var i = 0; i < candidates.items.length; i++) {
|
||
// s += candidates.priorities[i].toPrecision(2) + " "
|
||
// + candidates.items[i].output + " " +
|
||
// + candidates.items[i].multiplier.toPrecision(2) + ", ";
|
||
// }
|
||
// log(input + " Candidate " + candidate.output +
|
||
// " for " + candidate.input[0] + ": " + s);
|
||
}
|
||
|
||
// After processing one batch of candidates, use setTimeout to
|
||
// schedule another invocation of this function. This returns to
|
||
// the event loop so we can process messages from the main thread
|
||
// and allows us to abort the search if more input arrives.
|
||
setTimeout(processCandidates);
|
||
}
|
||
catch (e) {
|
||
status.state = 'error';
|
||
status.error = e;
|
||
onerror(e.toString() + '\n' + e.stack);
|
||
}
|
||
}
|
||
|
||
//
|
||
// This function is the heart of the dictionary search algorithm. The
|
||
// key to understanding it is that we do not traverse the dictionary
|
||
// tree as we would when looking for an exact match. Instead, at each
|
||
// level, we visit the nodes in frequency order, following the next
|
||
// pointer. If we find nodes that loosely match the first character of
|
||
// the remaining input, we use those nodes to generate new candidates for
|
||
// further evaluation later. This is defined as a GrowOperation in the Wiki
|
||
// article linked at the beginning of this file.
|
||
//
|
||
// Note that by maintaining a list of candidates like this, we're doing a
|
||
// breadth-first search rather than a depth-first search. (But since we
|
||
// weight the candidates and put them in a priority queue, it is not a pure
|
||
// breadth-first search).
|
||
//
|
||
// A candidate specifies a node in the dictionary tree, the remaining part
|
||
// of the user's input, and the output generated so far.
|
||
// This function uses the dictionary to loop through all possible characters
|
||
// that could appear after the current output, and considers those
|
||
// characters in most frequent to least frequent order. It compares each
|
||
// character to the first character (or, in certain GrowOperations, first
|
||
// two characters) of the remainder of user's input and generates new
|
||
// candidates based on that comparison.
|
||
//
|
||
// The candidate generation considers things such as accented characters
|
||
// from the dictionary, nearby keys from the keyboard layout and the
|
||
// possibility of user input errors such as transpositions and
|
||
// omissions.
|
||
//
|
||
function process(candidate) {
|
||
var remaining = candidate.input;
|
||
var output = candidate.output;
|
||
var multiplier = candidate.multiplier;
|
||
var corrections = candidate.corrections;
|
||
var node = {};
|
||
|
||
// The next character of the remainder of user's input
|
||
var char, code;
|
||
if (remaining.length > 0) {
|
||
char = remaining[0];
|
||
code = remaining.charCodeAt(0);
|
||
}
|
||
|
||
for (var curr = candidate.pointer; curr !== -1; curr = node.next) {
|
||
readNode(curr, node);
|
||
|
||
// How common is the most common word under this node?
|
||
var frequency = node.freq;
|
||
var weight = frequency * multiplier;
|
||
|
||
// If this node does not have a high enough weight to make it into
|
||
// the list of candidates, we don't need to continue. None of the
|
||
// nodes that follow in the next pointer linked list will have a
|
||
// higher weight than this one.
|
||
// Note however, that we only use this shortcut if we've already
|
||
// made at least one correction because uncorrected matches are given
|
||
// high weight by addCandidate.
|
||
if (corrections > 0 && weight <= candidates.threshold) {
|
||
break;
|
||
}
|
||
|
||
// If we generate new candidates from this node, this is what
|
||
// their output string will be
|
||
var newoutput = output + String.fromCharCode(node.ch);
|
||
|
||
// The various ways (i.e. GrowOperations) we can generate new candidates
|
||
// from this node follow. Note that each one can have a different
|
||
// associated multiplier. And note that some are considered
|
||
// corrections. To prevent explosive growth in the number of candidates,
|
||
// we limit the number of corrections allowed on any candidate.
|
||
|
||
// If there isn't any more input from the user, then we'll try to
|
||
// extend the output we've already got to find a complete word as a
|
||
// suggestion. But we apply a penalty for each character we add so that
|
||
// shorter completions are favored over longer completions
|
||
if (remaining.length === 0) {
|
||
// If a word ends here, add it to the queue of suggestions
|
||
if (node.ch === 0) {
|
||
// Only suggest profane (freq == 1) words if the input is
|
||
// already the same
|
||
if (node.freq === 1 &&
|
||
input.toUpperCase() === output.toUpperCase()) {
|
||
// Profane words have very low frequency themselves.
|
||
// To make sure they pop up we bump the frequency
|
||
addSuggestion(output, PROFANE_INPUT_MATCH_WEIGHT);
|
||
}
|
||
else if (node.freq !== 1) {
|
||
addSuggestion(output, weight);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
// Insertion-Extend GrowOperation:
|
||
// Otherwise, extend the candidate with the current node. We
|
||
// reduce the multiplier, but do not count this as a correction so
|
||
// that we can extend candidates as far as needed to find suggestions.
|
||
// This is like in the sense of auto completion.
|
||
addCandidate(node.center,
|
||
remaining, // the empty string
|
||
newoutput,
|
||
multiplier * WORD_EXTENSION_MULTIPLIER,
|
||
frequency, corrections);
|
||
|
||
// If there isn't any more input then we don't want to consider
|
||
// any of the other possible corrections below.
|
||
continue;
|
||
}
|
||
|
||
// Handle the case where this node marks the end of a word.
|
||
if (node.ch === 0) {
|
||
// Deletion-AtEnd GrowOperation
|
||
// If there is just one more character of user input remaining,
|
||
// maybe the user accidentally typed an extra character at the
|
||
// end, so try just dropping the last character. Note that this
|
||
// case is unique in that instead of following the center pointer
|
||
// it revisits the same node just without the one remaining
|
||
// character of input.
|
||
if (remaining.length === 1) {
|
||
addCandidate(curr, // stay at this same node
|
||
'', // no more remaining characters
|
||
output, // not newoutput
|
||
multiplier * DELETION_MULTIPLIER,
|
||
frequency, corrections + 1);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
// If we get to here, we know that we're still processing the user's
|
||
// input and that there is a character associated with this node.
|
||
|
||
// These next several GrowOperations are all in disjoint if/else chains,
|
||
// meaning that they're not mutually exclusive.
|
||
// GrowOps within a if/else chain add the same candidate, so it never
|
||
// makes sense for more than one of them to run.
|
||
// But note that it is possible for such GrowOps to happen more than one
|
||
// way, so we have to be sure to do the tests in highest to lowest
|
||
// multiplier order.
|
||
|
||
// Match GrowOperation: an exact match on this character
|
||
if (node.ch === code) {
|
||
addCandidate(node.center,
|
||
remaining.substring(1),
|
||
newoutput,
|
||
multiplier,
|
||
frequency, corrections);
|
||
}
|
||
else if (variants[node.ch].indexOf(char) !== -1) {
|
||
// Substitution-Variant GrowOperation:
|
||
// The user's input is a variant form of the character in this
|
||
// node, so we'll accept that input as a substitute for the node
|
||
// character. This covers case differences and unaccented forms of
|
||
// accented characters. (We don't accept accented forms typed by
|
||
// the user as variants of unaccented characters in the
|
||
// dictionary, however.)
|
||
addCandidate(node.center,
|
||
remaining.substring(1),
|
||
newoutput,
|
||
multiplier * VARIANT_FORM_MULTIPLIER,
|
||
frequency, corrections);
|
||
}
|
||
else if (corrections < maxCorrections) {
|
||
// If we haven't made any corrections on this candidate yet, try
|
||
// substituting the character from this node for the current
|
||
// character in the user's input. If the two keys are near each
|
||
// other on the keyboard, then we do this with higher weight than
|
||
// if they are distant.
|
||
var rootNode = rootform[node.ch];
|
||
var rootInput = rootform[code];
|
||
var nearby =
|
||
nearbyKeys[rootNode] ? nearbyKeys[rootNode][rootInput] : 0;
|
||
if (nearby) {
|
||
var adjust =
|
||
Math.max(nearby * NEARBY_KEY_REPLACEMENT_MULTIPLIER,
|
||
SUBSTITUTION_MULTIPLIER);
|
||
// Substitution-Near GrowOperation:
|
||
// If the node holds a character that is near the one the user
|
||
// typed, try it, assuming that the user has fat fingers and
|
||
// just missed the key. Note that we use a weight based on the
|
||
// distance between the keys. (Keys on the same row are
|
||
// generally closer together than keys above or below each
|
||
// other)
|
||
addCandidate(node.center,
|
||
remaining.substring(1),
|
||
newoutput,
|
||
multiplier * adjust,
|
||
frequency, corrections + 1);
|
||
}
|
||
else if (output.length > 0) {
|
||
// Substitution-Any GrowOperation:
|
||
// If it wasn't a nearby key, try substituting it anyway, but
|
||
// with a much lower weight. This handles the case where the
|
||
// user just doesn't know how to spell the word. We assume that
|
||
// the user knows the correct first letter of the word.
|
||
addCandidate(node.center,
|
||
remaining.substring(1),
|
||
newoutput,
|
||
multiplier * SUBSTITUTION_MULTIPLIER,
|
||
frequency, corrections + 1);
|
||
}
|
||
}
|
||
|
||
// Now we try some other tests that generate different candidates
|
||
// than the above. These involve insertion, deletion or
|
||
// transposition. Note that to avoid exponential blow-up of the
|
||
// search space, we generally only allow maxCorrections (usually 1)
|
||
// correction per candidate.
|
||
|
||
// First, just try inserting this character. Maybe the user forgot
|
||
// to type it or omitted punctuation on purpose. If this character
|
||
// has no variants, then it is a punctuation character and we allow
|
||
// it to be inserted with a high multiplier and no correction
|
||
// penalty. If it is not word punctuation, then the insertion is
|
||
// more costly. Also: assume that the user got the first character
|
||
// correct and don't insert at position 0.
|
||
if (!variants[node.ch]) { // If it is a punctuation character
|
||
// Insertion-Punctuation GrowOperation
|
||
addCandidate(node.center,
|
||
remaining, // insertion, so no substring here
|
||
newoutput,
|
||
multiplier * PUNCTUATION_INSERTION_MULTIPLIER,
|
||
frequency, corrections);
|
||
}
|
||
else if (corrections < maxCorrections && output.length > 0) {
|
||
// Insertion-Any GrowOperation
|
||
addCandidate(node.center,
|
||
remaining,
|
||
newoutput,
|
||
multiplier * INSERTION_MULTIPLIER,
|
||
frequency, corrections + 1);
|
||
}
|
||
|
||
// If there is more input after this character, and if this node of
|
||
// the tree matches the next character of the input, try deleting
|
||
// the current character and try transposing the two. But assume
|
||
// that the user got their first character correct and don't mess
|
||
// with that.
|
||
if (corrections < maxCorrections &&
|
||
remaining.length > 1 && output.length > 0 &&
|
||
(node.ch === remaining.charCodeAt(1) ||
|
||
variants[node.ch].indexOf(remaining[1]) !== -1))
|
||
{
|
||
// Transposition GrowOperation
|
||
addCandidate(node.center,
|
||
remaining[0] + remaining.substring(2),
|
||
newoutput,
|
||
multiplier * TRANSPOSITION_MULTIPLIER,
|
||
frequency, corrections + 1);
|
||
|
||
// Deletion GrowOperation
|
||
addCandidate(node.center,
|
||
remaining.substring(2),
|
||
newoutput,
|
||
multiplier * DELETION_MULTIPLIER,
|
||
frequency, corrections + 1);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
//
|
||
// This function unpacks binary data from the dictionary and returns
|
||
// the nodes of the dictionary tree in expanded form as JS objects.
|
||
// See gaia/dictionaries/xml2dict.py for the corresponding code that
|
||
// serializes the nodes of the tree into this binary format. Full
|
||
// documentation of the binary format is in that file.
|
||
//
|
||
function readNode(offset, node) {
|
||
if (offset === -1) {
|
||
throw Error('Assertion error: followed invalid pointer');
|
||
}
|
||
|
||
var firstbyte = tree[offset++];
|
||
var haschar = firstbyte & 0x80;
|
||
var bigchar = firstbyte & 0x40;
|
||
var hasnext = firstbyte & 0x20;
|
||
node.freq = (firstbyte & 0x1F) + 1; // frequencies range from 1 to 32
|
||
|
||
if (haschar) {
|
||
node.ch = tree[offset++];
|
||
if (bigchar) {
|
||
node.ch = (node.ch << 8) + tree[offset++];
|
||
}
|
||
}
|
||
else {
|
||
node.ch = 0;
|
||
}
|
||
|
||
if (hasnext) {
|
||
node.next =
|
||
(tree[offset++] << 16) +
|
||
(tree[offset++] << 8) +
|
||
tree[offset++];
|
||
}
|
||
else {
|
||
node.next = -1;
|
||
}
|
||
|
||
if (haschar) {
|
||
node.center = offset;
|
||
} else {
|
||
node.center = -1;
|
||
}
|
||
|
||
/*
|
||
log("readNode:" +
|
||
" haschar:" + haschar +
|
||
" bigchar:" + bigchar +
|
||
" hasnext:" + hasnext +
|
||
" freq:" + node.freq +
|
||
" char:" + node.ch +
|
||
" next:" + node.next +
|
||
" center:" + node.center);
|
||
*/
|
||
}
|
||
|
||
//
|
||
// A priority queue with a maximum size.
|
||
//
|
||
// add() inserts an item at a position according to its priority. It
|
||
// returns true if the item was inserted or false if the item's priority
|
||
// was too low for a spot in the fixed-size queue.
|
||
//
|
||
// remove() removes and returns the highest priority item in the queue or
|
||
// null if there are no items
|
||
//
|
||
// threshold is 0 if the queue is not yet full. Otherwise it is the
|
||
// priority of the lowest-priority item in the queue. Items with
|
||
// priorities higher lower than this will never be added to the queue.
|
||
//
|
||
// items is the sorted array of items, with the highest priority item
|
||
// first.
|
||
//
|
||
function BoundedPriorityQueue(maxSize) {
|
||
this.maxSize = maxSize;
|
||
this.threshold = 0;
|
||
this.items = [];
|
||
this.priorities = [];
|
||
}
|
||
|
||
BoundedPriorityQueue.prototype.add = function add(item, priority) {
|
||
// If the array is full we have to reject this item or make room for it
|
||
if (this.items.length === this.maxSize) {
|
||
if (priority <= this.threshold) { // Reject the item.
|
||
return false;
|
||
}
|
||
else { // Make room for it.
|
||
this.items.pop();
|
||
this.priorities.pop();
|
||
}
|
||
}
|
||
|
||
// Search to find the insertion point for this new item
|
||
var index;
|
||
if (this.priorities.length > 60) {
|
||
// Binary search only for relatively long arrays.
|
||
// See http://jsperf.com/linear-or-binary-search for perf data.
|
||
var start = 0, end = this.priorities.length;
|
||
while (start !== end) {
|
||
var mid = Math.floor((start + end) / 2);
|
||
if (priority > this.priorities[mid]) {
|
||
end = mid;
|
||
}
|
||
else {
|
||
start = mid + 1;
|
||
}
|
||
}
|
||
index = start;
|
||
}
|
||
else {
|
||
// Linear search for small arrays
|
||
for (var i = 0, n = this.priorities.length; i < n; i++) {
|
||
if (priority > this.priorities[i]) {
|
||
break;
|
||
}
|
||
}
|
||
index = i;
|
||
}
|
||
|
||
// Insert the new item at that position
|
||
this.items.splice(index, 0, item);
|
||
this.priorities.splice(index, 0, priority);
|
||
|
||
// Update the threshold
|
||
this.threshold = this.priorities[this.maxSize - 1] || 0;
|
||
};
|
||
|
||
BoundedPriorityQueue.prototype.remove = function remove() {
|
||
if (this.items.length === 0) {
|
||
return null;
|
||
}
|
||
this.priorities.shift();
|
||
this.threshold = this.priorities[this.maxSize - 1] || 0;
|
||
return this.items.shift();
|
||
};
|
||
|
||
BoundedPriorityQueue.prototype.removeItemAt = function removeItemAt(index) {
|
||
this.priorities.splice(index, 1);
|
||
this.items.splice(index, 1);
|
||
this.threshold = this.priorities[this.maxSize - 1] || 0;
|
||
};
|
||
|
||
//
|
||
// A very simple Least Recently Used cache. It depends on the fact that the
|
||
// JavaScript for/in loop enumerates properties in order from least recently
|
||
// to most recently added. (Note that this does not work for properties
|
||
// that are numbers, however.)
|
||
//
|
||
function LRUCache(maxsize) {
|
||
this.maxsize = maxsize;
|
||
this.size = 0;
|
||
this.map = Object.create(null); // map keys to values
|
||
}
|
||
|
||
// Cache the key/value pair
|
||
LRUCache.prototype.add = function add(key, value) {
|
||
// If the key is already in the cache, adjust the size since we'll
|
||
// be incrementing below
|
||
if (key in this.map) {
|
||
this.size--;
|
||
}
|
||
|
||
// Now insert the item
|
||
this.map[key] = value;
|
||
this.size++;
|
||
|
||
// If the size is too big delete the first property returned by
|
||
// for/in. This should be the least recently used because the get()
|
||
// method deletes and reinserts
|
||
if (this.size > this.maxsize) {
|
||
for (var p in this.map) {
|
||
delete this.map[p];
|
||
break;
|
||
}
|
||
this.size--;
|
||
}
|
||
};
|
||
|
||
// Look for a cached value matching the specified key
|
||
LRUCache.prototype.get = function(key) {
|
||
if (key in this.map) { // If the key is in the cache
|
||
var value = this.map[key]; // Get the value
|
||
delete this.map[key]; // Delete the key/value mapping
|
||
this.map[key] = value; // And re-insert to make it most recent
|
||
return value; // Return the value
|
||
}
|
||
return null;
|
||
};
|
||
|
||
// This is the Predictions object that is the public API of this module.
|
||
return {
|
||
setDictionary: setDictionary,
|
||
setNearbyKeys: setNearbyKeys,
|
||
predict: predict
|
||
};
|
||
};
|