b38dea0597
Share Gaia assets and adapters, keep private profiles and development documents local, and update both public READMEs.
127 lines
3.8 KiB
JavaScript
Executable file
127 lines
3.8 KiB
JavaScript
Executable file
/* global require, Buffer */
|
|
|
|
// This replaces xml2dict.py, using WordListConverter shared with
|
|
// KeyboardSettingsApp.
|
|
|
|
// Unlike xml2dict.py, we don't care |flags| at all as the attribute is not used
|
|
// anywhere.
|
|
|
|
// Original comments follow.
|
|
|
|
/*
|
|
This script reads a XML-formatted word list and produces a dictionary
|
|
file used by the FirefoxOS virtual keyboard for word suggestions and
|
|
auto corrections.
|
|
|
|
The word lists come from the Android source:
|
|
https://android.googlesource.com/
|
|
platform/packages/inputmethods/LatinIME/+/master/dictionaries/
|
|
|
|
This script currently depends on the XML format of the Android
|
|
wordlists. (Eventually we might want to pre-process the XML files
|
|
to a plain text format and simplify this script so that it will work
|
|
with any plain-text word and frequency list)
|
|
|
|
The sample.xml file from the Android repo looks like this:
|
|
|
|
----------------------------------------------------------------------
|
|
|
|
<!-- This is a sample wordlist that can be converted to a binary
|
|
dictionary for use by the Latin IME. The format of the word
|
|
list is a flat list of word entries. Each entry has a frequency
|
|
between 255 and 0. Highest frequency words get more weight in
|
|
the prediction algorithm. As a special case, a weight of 0 is
|
|
taken to mean profanity - words that should not be considered a
|
|
typo, but that should never be suggested explicitly. You can
|
|
capitalize words that must always be capitalized, such as
|
|
"January". You can have a capitalized and a non-capitalized
|
|
word as separate entries, such as "robin" and "Robin". -->
|
|
|
|
<wordlist>
|
|
<w f="255">this</w>
|
|
<w f="255">is</w>
|
|
<w f="128">sample</w>
|
|
<w f="1">wordlist</w>
|
|
</wordlist>
|
|
----------------------------------------------------------------------
|
|
|
|
This script processes the word list and converts it to a Ternary
|
|
Search Tree (TST), as described in the wiki link below, also in
|
|
|
|
http://en.wikipedia.org/wiki/Ternary_search_tree
|
|
http://www.strchr.com/ternary_dags
|
|
http://www.strchr.com/dawg_predictive
|
|
|
|
Note that the script does not convert the tree into a DAG (by sharing
|
|
common word suffixes) because it cannot maintain separate frequency
|
|
data for each word if the words share nodes.
|
|
|
|
We have moved the documentation (format and example) for the dictionary blob to
|
|
Mozilla Wiki:
|
|
https://wiki.mozilla.org/Gaia/System/Keyboard/IME/Latin/Dictionary_Blob
|
|
|
|
Please make sure any updates to the codes are reflected in the wiki too.
|
|
|
|
*/
|
|
|
|
'use strict';
|
|
|
|
var fs = require('fs');
|
|
|
|
var WordListConverter =
|
|
require('../../../settings/word_list_converter.js').WordListConverter;
|
|
|
|
var argv = require('yargs')
|
|
.demand(1)
|
|
.demand('o')
|
|
.alias('o', 'output')
|
|
.describe('o', 'write output to this file')
|
|
.usage('Usage: $0 -o lang.dict lang_wordlist.xml')
|
|
.argv;
|
|
|
|
var parser = new require('xml2js').Parser();
|
|
|
|
console.log('Reading XML file...');
|
|
|
|
var xmlContent = fs.readFileSync(argv._[0], {encoding: 'utf8'});
|
|
|
|
console.log('Parsing XML file...');
|
|
|
|
parser.parseString(xmlContent, function(error, results){
|
|
if (error) {
|
|
console.error('XML Parser Error:', error);
|
|
return;
|
|
}
|
|
|
|
console.log('Finished parsing.');
|
|
|
|
// from xml results to {w: 'word', f: FREQ_INT}
|
|
var words = results.wordlist.w.map(function(wNode){
|
|
return {
|
|
w: wNode._,
|
|
f: parseInt(wNode.$.f)
|
|
};
|
|
});
|
|
|
|
var highestFreq = words[0].f;
|
|
|
|
// normalize frequency to [0, 1)
|
|
words = words.map(function(word){
|
|
return {
|
|
w: word.w,
|
|
f: word.f / (highestFreq + 1)
|
|
};
|
|
});
|
|
|
|
console.log('Generating TST Tree...');
|
|
|
|
var blob = new WordListConverter(words).toBlob();
|
|
|
|
fs.writeFile(argv.o, new Buffer(new Uint8Array(blob)), 'binary', function(e){
|
|
if (e){
|
|
console.error('Error:', e);
|
|
} else {
|
|
console.log('Finished.');
|
|
}
|
|
});
|
|
});
|