Publish Vulpes Preview 15 sources for desktop, Android and Tundra

Share Gaia assets and adapters, keep private profiles and development documents local, and update both public READMEs.

gaia/apps/keyboard/js/imes/jskanji/README

@@ -0,0 +1,17 @@
+# Japanese IME
+
+About Dictionary
+
+The IME dictionary is converted from NAIST Japanese Dictionary project with following steps:
+1. Get a copy of `naist-jdic-0.4.3.tar.gz` at 
+  http://sourceforge.jp/projects/naist-jdic/releases/
+
+2. Extract all .dic files to dict/src/.
+
+3. Run `make` under the `dict` directory to generate dict.json.
+
+The tool of `dict/jcconv.py` is getting from https://github.com/taichino/jcconv.
+
+
+Note:
+  Running `make` on Windows is not recommended. You have to take care of some encoding issues.

gaia/apps/keyboard/js/imes/jskanji/dict/COPYING

@@ -0,0 +1,31 @@
+Copyright (c) 2008, Nara Institute of Science and Technology, Japan.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+Neither the name of the Nara Institute of Science and Technology
+(NAIST) nor the names of its contributors may be used to endorse or
+promote products derived from this software without specific prior
+written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+

gaia/apps/keyboard/js/imes/jskanji/dict/Makefile

@@ -0,0 +1,15 @@
+.PHONY: json clean
+
+json: dict.utf8
+	python conv.py
+
+dict:
+	find src -name "*.dic" -exec cat {} >> $@ \;
+
+dict.utf8: dict
+	iconv -f euc-jp -t utf-8 dict > $@
+
+clean:
+	rm dict
+	rm dict.utf8
+	rm *.pyc

gaia/apps/keyboard/js/imes/jskanji/dict/conv.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import json
+import codecs
+
+from jcconv import kata2hira
+
+file_list = ['dict.utf8',]
+
+temp_obj = dict()
+total_freq = 0
+
+# Converting from EUC-JP to UTF-8 makes unexpected code change for some characters.
+# This table recovers them to its original code.
+utf8_fix_table = {
+    u'‾': u'￣',
+    u'—': u'―',
+    u'〜': u'～',
+    u'‖': u'∥',
+    u'−': u'－',
+    u'¥': u'￥',
+    u'¢': u'￠',
+    u'£': u'￡',
+    u'¬': u'￢',
+    u'¦': u'￤'
+}
+
+naistjdic_fix_table = {
+    u'海・島': u'海鱸島',
+    u'茶・山': u'茶荃山',
+    u'吐・喇列島': u'吐噶喇列島',
+    u'屋那・島': u'屋那覇島',
+    u'流・': u'流硑',
+    u'辰ケ・': u'辰ケ硑',
+    u'穴・湾': u'穴澗湾',
+    u'海・の峰': u'海驢の峰',
+    u'下・島': u'下甑島',
+    u'出来・崎': u'出来澗崎',
+    u'横・岬': u'横澗岬',
+    u'穴・岬': u'穴澗岬',
+    u'京の上・島': u'京の上臈島',
+    u'吐・喇海峡': u'吐噶喇海峡',
+    u'辰ヶ・': u'辰ヶ硑',
+    u'滝ノ・ノ岬': u'滝ノ澗ノ岬',
+    u'中・湾': u'中甑湾',
+    u'丸・ノ鼻': u'丸硑ノ鼻',
+    u'勝・山': u'勝澗山',
+    u'狼・山': u'狼乢山',
+    u'赤・鼻': u'赤硑鼻',
+    u'中ノ・崎': u'中ノ澗崎',
+    u'大松・': u'大松硑',
+    u'中・島': u'中甑島',
+    u'螺・島': u'螺蠑島',
+    u'上・島': u'上甑島'
+}
+
+naistjdic_correction_table = {
+    u'赤硑鼻': u'あかばえはな',
+    u'与那・岳': '',
+    u'村碆・': '',
+    u'笠・岳': ''
+}
+
+def naistjdic_patch(kanji, kana):
+    # Fix encoding convert error by iconv.
+    for c in utf8_fix_table:
+        kanji = kanji.replace(c, utf8_fix_table[c])
+        kana = kana.replace(c, utf8_fix_table[c])
+
+    # Fix some lacked kanji in naist-jdic 0.4.3.
+    if u'・' in kanji:
+        if kanji in naistjdic_fix_table:
+            kanji_key = kanji
+            kanji = naistjdic_fix_table[kanji]
+            del naistjdic_fix_table[kanji_key]
+
+    # Correct kana or remove the item.
+    if kanji in naistjdic_correction_table:
+        kanji_key = kanji
+        kana = naistjdic_correction_table[kanji]
+        del naistjdic_correction_table[kanji_key]
+    return [kanji, kana]
+
+
+# Main loop.
+for file_name in file_list:
+    print file_name
+
+    with codecs.open(file_name, 'r', 'utf-8') as f:
+        for line in f:
+            arr = line.split()
+
+            base = 0
+            for i, e in enumerate(arr):
+                if e[0] <= '9' and e[0] >= '0':
+                    base = i
+            kanji = arr[base-1][1:]
+            freq = arr[base][:-2]
+            kana = kata2hira(arr[base+2][:-1])
+
+            kanji, kana = naistjdic_patch(kanji, kana)
+            if kana == '':
+              # Remove current item from dic.
+              continue
+
+            print kanji.encode('utf-8'), freq, kana
+
+            if len(kana) == 0:
+                continue
+            elif len(kana) > 1 and kana[0] == '{':
+                ks = kana[1:kana.index('}')].split('/')
+                ks = [k + kana[kana.index('}')+1:] for k in ks if len(k) > 0]
+            else:
+                ks = [kana, ]
+
+            freq = int(freq)
+            for k in ks:
+
+                if k not in temp_obj:
+                    temp_obj[k] = [[freq, kanji], ]
+                    total_freq += freq
+                else:
+                    kanji_exist = False
+                    for ele in temp_obj[k]:
+                        if ele[1] == kanji:
+                            kanji_exist = True
+                            if ele[0] > freq:
+                                ele[0] = freq
+
+                    if not kanji_exist:
+                        temp_obj[k].append([freq, kanji])
+                        total_freq += freq
+
+
+print 'Total frequence %d.' % total_freq
+
+json_obj = []
+
+for k, value in temp_obj.iteritems():
+    json_obj.append({'kana' : k, 'terms': [{'kanji': v[1], 'freq': v[0],
+            'kana': k} for v in value]})
+
+print 'Writing to ../dict.json.'
+
+df = codecs.open('../dict.json', 'w', 'utf-8')
+df.write(json.dumps(json_obj, ensure_ascii=False).replace(" {\"terms\":", "\n{\"terms\":"))
+df.close()
+
+print 'Done!'

gaia/apps/keyboard/js/imes/jskanji/dict/jcconv.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+
+__all__ = ['hira2kata', 'kata2hira', 'half2hira', 'hira2half', 'kata2half',
+           'half2kata', 'half2wide', 'wide2half', 'convert',
+           'check_hira', 'check_kata', 'check_half']
+
+import re
+
+# convert hiragana to katakana
+def hira2kata(text, reserved=[]):
+    return convert(text, jcconv.HIRA, jcconv.KATA, reserved)
+
+# convert katakana to hiragana
+def kata2hira(text, reserved=[]):
+    return convert(text, jcconv.KATA, jcconv.HIRA, reserved)
+
+# convert half-width kana to hiragana
+def half2hira(text, reserved=[]):
+    return convert(text, jcconv.HALF, jcconv.HIRA, reserved)
+
+# convert hiragana to half-width kana
+def hira2half(text, reserved=[]):
+    return convert(text, jcconv.HIRA, jcconv.HALF, reserved)
+
+# convert katakana to half-width kana
+def kata2half(text, reserved=[]):
+    return convert(text, jcconv.KATA, jcconv.HALF, reserved)
+
+# convert half-width kana to katakana
+def half2kata(text, reserved=[]):
+    return convert(text, jcconv.HALF, jcconv.KATA, reserved)
+
+# expand half width number and alphabet to wide width
+def half2wide(text, reserved=[]):
+    text = convert(text, jcconv.HNUM, jcconv.WNUM, reserved)
+    text = convert(text, jcconv.HALP, jcconv.WALP, reserved)
+    return convert(text, jcconv.HSYM, jcconv.WSYM, reserved)
+
+# shrink wide width number and alphabet to half width
+def wide2half(text, reserved=[]):
+    text = convert(text, jcconv.WNUM, jcconv.HNUM, reserved)
+    text = convert(text, jcconv.WALP, jcconv.HALP, reserved)
+    return convert(text, jcconv.WSYM, jcconv.HSYM, reserved)
+
+# check if 'text' consists of hiragana
+def check_hira(text):
+    return check(text, jcconv.HIRA)
+
+# check if 'text' consists of katakana
+def check_kata(text):
+    return check(text, jcconv.KATA)
+
+# check if 'text' consists of half-width kana
+def check_half(text):
+    return check(text, jcconv.HALF)
+
+# convert 'frm' charset to 'to' charset
+# input text must be unicode or str(utf-8)
+# 'frm' and 'to' can be specified with (HIRA, KATA, HALF, WNUM, HNUM, WALP, HALP, WSYM, HSYM)
+__regex_storage = {}
+def convert(text, frm, to, reserved=[]):
+    def _multiple_replace(text, dic):
+        key = '|'.join(map(re.escape, dic))
+        if key in __regex_storage:
+            rx = __regex_storage[key]
+        else:
+            rx = re.compile(key)
+            __regex_storage[key] = rx
+        def proc_one(match):
+            return dic[match.group(0)]
+        return rx.sub(proc_one, text)
+
+    uflag = isinstance(text, unicode)
+    f_set = jcconv.char_sets[frm]
+    t_set = jcconv.char_sets[to]
+
+    text = uflag and text or text.decode('utf-8')
+    if len(f_set[0].split(' ')) == len(t_set[0].split(' ')):
+        for i in range(len(f_set)):
+            conv_table = dict(zip(f_set[i].split(' '), t_set[i].split(' ')))
+            for r in reserved:
+                try:
+                    del(conv_table[r])
+                except:
+                    pass
+            text = _multiple_replace(text, conv_table)
+        return uflag and text or text.encode('utf-8')
+    else:
+        raise "Invalid Parameter"
+
+def check(text, char_set_type):
+    uflag = isinstance(text, unicode)
+    text = uflag and text or text.decode('utf-8')
+    char_set = []
+    for set in jcconv.char_sets[char_set_type]:
+        char_set.extend(set.split(' '))
+    return all([text_char in char_set for text_char in text])
+
+# define character sets used in japanese
+class jcconv:
+    (HIRA, KATA, HALF, WNUM, HNUM, WALP, HALP, WSYM, HSYM) = (i for i in range(9))
+    hira = [u'が ぎ ぐ げ ご ざ じ ず ぜ ぞ だ ぢ づ で ど ば び ぶ べ ぼ ぱ ぴ ぷ ぺ ぽ',
+            u'あ い う え お か き く け こ さ し す せ そ た ち つ て と ' + \
+            u'な に ぬ ね の は ひ ふ へ ほ ま み む め も や ゆ よ ら り る れ ろ ' + \
+            u'わ を ん ぁ ぃ ぅ ぇ ぉ ゃ ゅ ょ っ']
+    kata = [u'ガ ギ グ ゲ ゴ ザ ジ ズ ゼ ゾ ダ ヂ ヅ デ ド バ ビ ブ ベ ボ パ ピ プ ペ ポ',
+            u'ア イ ウ エ オ カ キ ク ケ コ サ シ ス セ ソ タ チ ツ テ ト ' + \
+            u'ナ ニ ヌ ネ ノ ハ ヒ フ ヘ ホ マ ミ ム メ モ ヤ ユ ヨ ラ リ ル レ ロ ' + \
+            u'ワ ヲ ン ァ ィ ゥ ェ ォ ャ ュ ョ ッ']
+    half = [u'ｶﾞ ｷﾞ ｸﾞ ｹﾞ ｺﾞ ｻﾞ ｼﾞ ｽﾞ ｾﾞ ｿﾞ ﾀﾞ ﾁﾞ ﾂﾞ ﾃﾞ ﾄﾞ ﾊﾞ ﾋﾞ ﾌﾞ ﾍﾞ ﾎﾞ ﾊﾟ ﾋﾟ ﾌﾟ ﾍﾟ ﾎﾟ',
+            u'ｱ ｲ ｳ ｴ ｵ ｶ ｷ ｸ ｹ ｺ ｻ ｼ ｽ ｾ ｿ ﾀ ﾁ ﾂ ﾃ ﾄ ﾅ ﾆ ﾇ ﾈ ﾉ ﾊ ﾋ ﾌ ﾍ ﾎ ﾏ ﾐ ﾑ ﾒ ﾓ ﾔ ﾕ ﾖ ﾗ ﾘ ﾙ ﾚ ﾛ ' + \
+            u'ﾜ ｦ ﾝ ｧ ｨ ｩ ｪ ｫ ｬ ｭ ｮ ｯ']
+    wnum = [u'０ １ ２ ３ ４ ５ ６ ７ ８ ９']
+    hnum = [u'0 1 2 3 4 5 6 7 8 9']
+    walp = [u'ａ ｂ ｃ ｄ ｅ ｆ ｇ ｈ ｉ ｊ ｋ ｌ ｍ ｎ ｏ ｐ ｑ ｒ ｓ ｔ ｕ ｖ ｗ ｘ ｙ ｚ ' + \
+            u'Ａ Ｂ Ｃ Ｄ Ｅ Ｆ Ｇ Ｈ Ｉ Ｊ Ｋ Ｌ Ｍ Ｎ Ｏ Ｐ Ｑ Ｒ Ｓ Ｔ Ｕ Ｖ Ｗ Ｘ Ｙ Ｚ']
+    halp = [u'a b c d e f g h i j k l m n o p q r s t u v w x y z ' + \
+            u'A B C D E F G H I J K L M N O P Q R S T U V W X Y Z']
+    wsym = [u'！ ” ＃ ＄ ％ ＆ ’ （ ） ＊ ＋ 、 − ． ／ ： ； ＜ ＝ ＞ ？ ＠ 「 ＼ 」 ＾ ＿ ｀ 『 ｜ 』 〜']
+    hsym = [u'! \" # $ % & \' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ ` { | } ~']
+    char_sets = [hira, kata, half, wnum, hnum, walp, halp, wsym, hsym]
+
+
+if __name__ == '__main__':
+    import codecs, sys
+    sys.stdout = codecs.getwriter('utf_8')(sys.stdout)
+    
+    print convert(u'あいうえお', jcconv.HIRA, jcconv.HALF, [u'う'])
+    print convert(u'ばいおりん', jcconv.HIRA, jcconv.HALF)
+    print convert(u'ﾊﾞｲｵﾘﾝ', jcconv.HALF, jcconv.HIRA)
+    print convert(u'12345', jcconv.HNUM, jcconv.WNUM)
+
+    print check_hira(u'ひらがな')
+    print check_hira(u'カタカナ')
+    print check_kata(u'ひらがな')
+    print check_kata(u'カタカナ')