Publish Vulpes Preview 15 sources for desktop, Android and Tundra

Share Gaia assets and adapters, keep private profiles and development documents local, and update both public READMEs.

gaia/apps/keyboard/js/imes/latin/dictionaries/.gitignore

@@ -0,0 +1 @@
+node_modules/

gaia/apps/keyboard/js/imes/latin/dictionaries/Makefile

@@ -0,0 +1,61 @@
+# After building a dictionary here, move the resulting .dict file to
+# apps/keyboard/js/imes/latin/dictionaries/ and check it in.
+#
+# Available:
+# af.dict, bg.dict, bs.dict, ca.dict, cs.dict, da.dict, de.dict, el.dict, en.dict, en_gb.dict, en_us.dict, es.dict, eu.dict, ff.dict,
+# fi.dict, fr.dict, fy.dict, ga.dict, gd.dict, gl.dict, gv.dict, hr.dict, hu.dict, it.dict, lt.dict, lv.dict, nb.dict, nl.dict, pl.dict,
+# pt_br.dict, pt_pt.dict, ro.dict, ru.dict, sk.dict, sl.dict, sq.dict, sr.dict, sr-Cyrl.dict, sr-Latn.dict, sr.dict, sv.dict, tr.dict
+
+WHITELIST = \
+ af.dict \
+ bg.dict \
+ bs.dict \
+ ca.dict \
+ cs.dict \
+ cy.dict \
+ da.dict \
+ de.dict \
+ el.dict \
+ en.dict \
+ en_gb.dict \
+ en_us.dict \
+ es.dict \
+ eu.dict \
+ fi.dict \
+ ff.dict \
+ fr.dict \
+ fy.dict \
+ ga.dict \
+ gd.dict \
+ gl.dict \
+ gv.dict \
+ hr.dict \
+ hu.dict \
+ it.dict \
+ lt.dict \
+ lv.dict \
+ pl.dict \
+ pt_br.dict \
+ nb.dict \
+ nl.dict \
+ pl.dict \
+ pt_br.dict \
+ pt_pt.dict \
+ ro.dict \
+ ru.dict \
+ sk.dict \
+ sl.dict \
+ sq.dict \
+ sr.dict \
+ sr-Cyrl.dict \
+ sr-Latn.dict \
+ sv.dict \
+ tr.dict
+
+%.dict: %_wordlist.xml xml2dict.js dependency
+	node --harmony xml2dict.js -o $@ $<
+
+dependency:
+	npm install
+
+all: $(WHITELIST)

gaia/apps/keyboard/js/imes/latin/dictionaries/package.json

@@ -0,0 +1,6 @@
+{
+  "dependencies": {
+    "xml2js": "^0.4.6",
+    "yargs": "^1.3.2"
+  }
+}

gaia/apps/keyboard/js/imes/latin/dictionaries/xml2dict.js

@@ -0,0 +1,127 @@
+/* global require, Buffer */
+
+// This replaces xml2dict.py, using WordListConverter shared with
+// KeyboardSettingsApp.
+
+// Unlike xml2dict.py, we don't care |flags| at all as the attribute is not used
+// anywhere.
+
+// Original comments follow.
+
+/*
+This script reads a XML-formatted word list and produces a dictionary
+file used by the FirefoxOS virtual keyboard for word suggestions and
+auto corrections.
+
+The word lists come from the Android source:
+https://android.googlesource.com/
+                  platform/packages/inputmethods/LatinIME/+/master/dictionaries/
+
+This script currently depends on the XML format of the Android
+wordlists. (Eventually we might want to pre-process the XML files
+to a plain text format and simplify this script so that it will work
+with any plain-text word and frequency list)
+
+The sample.xml file from the Android repo looks like this:
+
+----------------------------------------------------------------------
+
+    <!-- This is a sample wordlist that can be converted to a binary
+         dictionary for use by the Latin IME. The format of the word
+         list is a flat list of word entries. Each entry has a frequency
+         between 255 and 0. Highest frequency words get more weight in
+         the prediction algorithm. As a special case, a weight of 0 is
+         taken to mean profanity - words that should not be considered a
+         typo, but that should never be suggested explicitly. You can
+         capitalize words that must always be capitalized, such as
+         "January". You can have a capitalized and a non-capitalized
+         word as separate entries, such as "robin" and "Robin". -->
+
+    <wordlist>
+      <w f="255">this</w>
+      <w f="255">is</w>
+      <w f="128">sample</w>
+      <w f="1">wordlist</w>
+    </wordlist>
+----------------------------------------------------------------------
+
+This script processes the word list and converts it to a Ternary
+Search Tree (TST), as described in the wiki link below, also in
+
+  http://en.wikipedia.org/wiki/Ternary_search_tree
+  http://www.strchr.com/ternary_dags
+  http://www.strchr.com/dawg_predictive
+
+Note that the script does not convert the tree into a DAG (by sharing
+common word suffixes) because it cannot maintain separate frequency
+data for each word if the words share nodes.
+
+We have moved the documentation (format and example) for the dictionary blob to
+Mozilla Wiki:
+https://wiki.mozilla.org/Gaia/System/Keyboard/IME/Latin/Dictionary_Blob
+
+Please make sure any updates to the codes are reflected in the wiki too.
+
+*/
+
+'use strict';
+
+var fs = require('fs');
+
+var WordListConverter =
+  require('../../../settings/word_list_converter.js').WordListConverter;
+
+var argv = require('yargs')
+  .demand(1)
+  .demand('o')
+  .alias('o', 'output')
+  .describe('o', 'write output to this file')
+  .usage('Usage: $0 -o lang.dict lang_wordlist.xml')
+  .argv;
+
+var parser = new require('xml2js').Parser();
+
+console.log('Reading XML file...');
+
+var xmlContent = fs.readFileSync(argv._[0], {encoding: 'utf8'});
+
+console.log('Parsing XML file...');
+
+parser.parseString(xmlContent, function(error, results){
+  if (error) {
+    console.error('XML Parser Error:', error);
+    return;
+  }
+
+  console.log('Finished parsing.');
+
+  // from xml results to {w: 'word', f: FREQ_INT}
+  var words = results.wordlist.w.map(function(wNode){
+    return {
+      w: wNode._,
+      f: parseInt(wNode.$.f)
+    };
+  });
+
+  var highestFreq = words[0].f;
+
+  // normalize frequency to [0, 1)
+  words = words.map(function(word){
+    return {
+      w: word.w,
+      f: word.f / (highestFreq + 1)
+    };
+  });
+
+  console.log('Generating TST Tree...');
+
+  var blob = new WordListConverter(words).toBlob();
+
+  fs.writeFile(argv.o, new Buffer(new Uint8Array(blob)), 'binary', function(e){
+    if (e){
+      console.error('Error:', e);
+    } else {
+      console.log('Finished.');
+    }
+  });
+});