Fix ./autogen.sh, ./configure and build errors.

db

stamp-db

# vim:set noet ts=4:

#

# libpyzy - The Chinese PinYin and Bopomofo conversion library.

#

# Copyright (c) 2008-2010 Peng Huang <shawn.p.huang@gmail.com>

#

# This library is free software; you can redistribute it and/or modify

# it under the terms of the GNU Lesser General Public License as

# published by the Free Software Foundation; either version 2.1 of the

# License, or (at your option) any later version.

#

# This library is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

# Lesser General Public License for more details.

#

# You should have received a copy of the GNU Lesser General Public

# License along with this library; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307

# USA

create_scripts = \

	create_db.py \

	id.py \

	pydict.py \

	valid_hanzi.py \

	$(NULL)

data_files = \

	README \

	rawdict_utf16_65105_freq.txt \

	$(NULL)

if IBUS_BUILD_DB_ANDROID

main_db_DATA = \

	android.db \

	$(NULL)

main_dbdir = $(pkgdatadir)/db

endif

android.db: $(android_raw_data) $(create_scripts)

	$(AM_V_GEN) \

	$(RM) $@; \

	$(srcdir)/create_db.py $(srcdir)/rawdict_utf16_65105_freq.txt | @SQLITE3@ $@ || \

		( $(RM) $@ ; exit 1 )

EXTRA_DIST = \

	$(data_files) \

	$(create_scripts) \

	$(NULL)

CLEANFILES = \

	$(main_db_DATA) \

	$(NULL)

DISTCLEANFILES = \

	$(NULL)

The data is from android open source project.

http://android.git.kernel.org/?p=platform/packages/inputmethods/PinyinIME.git

#!/usr/bin/env python

from pydict import *

from id import *

from valid_hanzi import *

import sys

def get_sheng_yun(pinyin):

    if pinyin == None:

        return None, None

    if pinyin == "ng":

        return "", "en"

    for i in xrange(2, 0, -1):

        t = pinyin[:i]

        if t in SHENGMU_DICT:

            return t, pinyin[len(t):]

    return "", pinyin

def read_phrases(filename):

    buf = file(filename).read()

    buf = unicode(buf, "utf16")

    buf = buf.strip()

    for l in buf.split(u'\n'):

        hanzi, freq, flag, pinyin = l.split(u' ', 3)

        freq = float(freq)

        pinyin = pinyin.split()

        if any(map(lambda c: c not in valid_hanzi, hanzi)):

            continue

        yield hanzi, freq, pinyin

def create_db(filename):

    # import sqlite3

    # con = sqlite3.connect("main.db")

    # con.execute ("PRAGMA synchronous = NORMAL;")

    # con.execute ("PRAGMA temp_store = MEMORY;")

    # con.execute ("PRAGMA default_cache_size = 5000;")

    print "PRAGMA synchronous = NORMAL;"

    print "PRAGMA temp_store = MEMORY;"

    print "PRAGMA default_cache_size = 5000;"

    sql = "CREATE TABLE py_phrase_%d (phrase TEXT, freq INTEGER, %s);"

    for i in range(0, 16):

        column = []

        for j in range(0, i + 1):

            column.append ("s%d INTEGER" % j)

            column.append ("y%d INTEGER" % j)

        print sql % (i, ",".join(column))

        # con.execute(sql % (i, column))

        # con.commit()

    records = list(read_phrases(filename))

    records.sort(lambda a, b: 1 if a[1] > b[1] else -1)

    records_new = []

    i = 0

    max_freq = 0.0

    for hanzi, freq, pinyin in records:

        if max_freq / freq <  1 - 0.001:

            max_freq = freq

            i = i + 1

        records_new.append((hanzi, i, pinyin))

    records_new.reverse()

    print "BEGIN;"

    insert_sql = "INSERT INTO py_phrase_%d VALUES (%s);"

    for hanzi, freq, pinyin in records_new:

        columns = []

        for py in pinyin:

            s, y = get_sheng_yun(py)

            s, y = pinyin_id[s], pinyin_id[y]

            columns.append(s)

            columns.append(y)

        values = "'%s', %d, %s" % (hanzi.encode("utf8"), freq, ",".join(map(str,columns)))

        sql = insert_sql % (len(hanzi) - 1, values)

        print sql

    print "COMMIT;"

    print "VACUUM;"

def main():

    create_db(sys.argv[1])

if __name__ == "__main__":

    main()

def get_validate_hanzi():

    validate_hanzi = file("valid_utf16.txt").read().decode("utf16")

    return set(validate_hanzi)

def main():

    hanzi = get_validate_hanzi()

    hanzi = list(hanzi)

    hanzi.sort()

    print "# -*- coding: utf-8 -*- "

    print "valid_hanzi = set(["

    for c in hanzi:

        print "    u\"%s\"," % c.encode("utf8")

    print "])"

if __name__ == "__main__":

    main()