Commit 8b6cf027 authored by Yuanle Song's avatar Yuanle Song
Browse files

v0.101.0 redesign use of main and user database. 

the old design have flaws and limitations. see :id001: and :id002: in
operational file.

new design:

open :memory: db readwrite, this will be the m_db handler.

try attach main db at known path (readonly if possible) to "maindb".
if failed, try next known path,
if all failed, initDB() should return false and gave up.

try attach user db at known path to "userdb" schema, if failed,
create :memory: db and initialize it as user db. attach it as "userdb"
schema.

update all query against main db to query from maindb.* table.
update all SQL against user db to run again userdb.* table.

benefits:
- access to user db is concurrent safe. multiple libpyzy app won't overwrite
  user's user db file and lose data.
- there is no need to saveUserDB() using timer in the bg.
- there is no need to copy data from user db to :memory:, if user db is
  large, this can lower memory usage.
parent 95b8f0b6

src/Database.cc

+286 −407
Original line number Diff line number Diff line
@@ -44,7 +44,7 @@ namespace PyZy { 
#define DB_BACKUP_TIMEOUT   (60)



#define USER_DICTIONARY_FILE  "user-1.0.db"



#define SQLITE3_MEMORY_DB ":memory:"



std::unique_ptr<Database> Database::m_instance;
@@ -169,6 +169,14 @@ Query::~Query (void) 
{

}



/**

 * query db to find phrases for pinyin.

 *

 * @phrases: put query result in this PhraseArray.

 * @count: query at most this many phrases.

 *

 * Returns: how many phrases fetched from db. 

 */

int

Query::fill (PhraseArray &phrases, int count)

{
@@ -178,7 +186,9 @@ Query::fill (PhraseArray &phrases, int count) 
        if (G_LIKELY (m_stmt.get () == NULL)) {

            m_stmt = Database::instance ().query (

		    m_pinyin, m_pinyin_begin, m_pinyin_len, -1, m_option);

            g_assert (m_stmt.get () != NULL);

	    if (m_stmt == nullptr) {

		    return 0;

	    }

        }



        while (m_stmt->step ()) {
@@ -210,441 +220,277 @@ Query::fill (PhraseArray &phrases, int count) 
    return row;

}



Database::Database (const std::string &user_data_dir)

    : m_db (NULL)

    , m_timeout_id (0)

    , m_timer (g_timer_new ())

    , m_user_data_dir (user_data_dir)

{

	m_user_db_file.clear ();

        m_user_db_file << m_user_data_dir

		       << G_DIR_SEPARATOR_S

		       << USER_DICTIONARY_FILE;

	bool r = open ();

	if (! r) {

		g_error ("open main db failed");

	}

}



Database::~Database (void)

{

	g_timer_destroy (m_timer);

	if (m_timeout_id != 0) {

		bool r = saveUserDB ();

		if (! r) {

			g_warning ("save user db failed");

		}

		gboolean r1 = g_source_remove (m_timeout_id);

		if (! r1) {

			g_warning ("remove timeout source failed, "

				   "source id is %d", m_timeout_id);

		}

		m_timeout_id = 0;

	}

	if (m_db) {

		if (sqlite3_close (m_db) != SQLITE_OK) {

			g_warning ("close sqlite database failed: %d (%s)",

				   sqlite3_errcode (m_db), sqlite3_errmsg (m_db));

		}

		m_db = NULL;

	}

}



bool

Database::executeSQL (const char *sql, sqlite3 *db)

{

    if (! db)

	    db = m_db;

    if (! db) {

	    g_warning ("trying to execute sql %s on db handler NULL", sql);

	    g_assert_not_reached ();

	    return false;

    }

    char *errmsg = NULL;

    if (sqlite3_exec (db, sql, NULL, NULL, &errmsg) != SQLITE_OK) {

        g_warning ("execute sql failed: sql=%s error=%s", sql, errmsg);

        sqlite3_free (errmsg);

        return false;

    }

    return true;

}



/**

 * set sqlite3 pragma on main db to improve performance.

 * attach db to dest handler as schema_name.

 *

 * Returns: true on success, false otherwise.

 */

bool

Database::setPragmaOnMainDB (void)

static bool

attach_db (sqlite3* dest, String db_file_to_attach, const gchar* schema_name)

{

        m_sql.clear ();



	// see https://www.sqlite.org/pragma.html#pragma_synchronous

        m_sql << "PRAGMA synchronous=OFF;\n";



        /* Set the cache size for better performance */

        m_sql << "PRAGMA cache_size=" DB_CACHE_SIZE ";\n";



        /* Using memory for temp store */

        // m_sql << "PRAGMA temp_store=MEMORY;\n";



        /* Set journal mode */

        // m_sql << "PRAGMA journal_mode=PERSIST;\n";



        /* Using EXCLUSIVE locking mode on databases

         * for better performance */

        m_sql << "PRAGMA locking_mode=EXCLUSIVE;\n";



	return executeSQL (m_sql);

	g_message ("attaching db %s as %s",

		   db_file_to_attach.c_str (), schema_name);

	char* sql = sqlite3_mprintf ("ATTACH DATABASE %Q AS %Q;",

				     db_file_to_attach.c_str (),

				     schema_name);

	gboolean result = sqlite3_exec_simple (dest, sql);

	sqlite3_free (sql);

	return result;

}



/**

 * try to open a main database. such as open-phrase.db.

 *

 * Returns: true on success, false otherwise.

 * return TRUE if file exists

 */

bool

Database::open (void)

{

    do {

#if (SQLITE_VERSION_NUMBER >= 3006000)

        sqlite3_initialize ();

#endif

        static const char * maindb [] = {

            PKGDATADIR"/db/local.db",

            PKGDATADIR"/db/open-phrase.db",

            PKGDATADIR"/db/android.db",

            "main.db",

        };



        size_t i;

        for (i = 0; i < G_N_ELEMENTS (maindb); i++) {

	    g_debug ("trying to load main db at %s", maindb[i]);

            if (sqlite3_open_v2 (maindb[i], &m_db, SQLITE_OPEN_READWRITE, NULL) == SQLITE_OK) {

		    g_message ("loading main db at %s", maindb[i]);

		    break;

            }

        }

        if (i == G_N_ELEMENTS (maindb)) {

		g_warning ("Failed to load any known main database");

		break;

static gboolean

file_exists (const char* filename) {

	return g_file_test (filename, G_FILE_TEST_EXISTS);

}



	int r = 0;

	r = setPragmaOnMainDB ();

        if (! r) {

		g_warning ("execute sqlite PRAGMA statements failed");

		break;

/**

 * return the first existing file in given file list.

 * return "" if none of the file exists.

 */

static String

first_existing_file (const std::vector<String> &files)

{

	for (const String &fn : files) {

		if (file_exists (fn.c_str ())) {

			return fn;

		}



        r = loadUserDB ();

	if (! r) {

		g_warning ("load user db failed");

		break;

	}



        /* prefetch some tables */

        // prefetch ();

	g_assert_nonnull (m_db);

        return true;

    } while (0);



    if (m_db) {

        sqlite3_close (m_db);

        m_db = NULL;

    }

    return false;

	return "";

}



/**

 * initialize user db.

 * create tables, index and populate data into desc table.

 * Returns: a main db file name if one exists.

 * an empty string otherwise.

 *

 * Returns: true on success, false otherwise.

 * This function will look for file at these path:

 *   m_user_data_dir + "/main.db"

 *   {PKGDATADIR}/db/local.db

 *   {PKGDATADIR}/db/open-phrase.db

 *   {PKGDATADIR}/db/android.db

 *

 * In ibus-pinyin context those are:

 *   ~/.cache/ibus/pinyin/local.db

 *   /usr/share/pyzy/db/local.db

 *   /usr/share/pyzy/db/open-phrase.db

 *   /usr/share/pyzy/db/android.db

 */

bool

Database::initUserDB (sqlite3 *userdb)

String

Database::getMainDBFile (void)

{

        m_sql = "BEGIN TRANSACTION;\n";

        /* create desc table*/

        m_sql << "CREATE TABLE IF NOT EXISTS desc (name PRIMARY KEY, value TEXT);\n";

        m_sql << "INSERT OR IGNORE INTO desc VALUES ('version', '1.2.0');\n"

              << "INSERT OR IGNORE INTO desc VALUES ('uuid', '" << UUID () << "');\n"

              << "INSERT OR IGNORE INTO desc VALUES ('hostname', '" << Hostname () << "');\n"

              << "INSERT OR IGNORE INTO desc VALUES ('username', '" << Env ("USERNAME") << "');\n"

              << "INSERT OR IGNORE INTO desc VALUES ('create-time', datetime());\n"

              << "INSERT OR IGNORE INTO desc VALUES ('attach-time', datetime());\n";



        /* create phrase tables */

        for (size_t i = 0; i < MAX_PHRASE_LEN; i++) {

		m_sql.appendPrintf ("CREATE TABLE IF NOT EXISTS py_phrase_%d (user_freq, phrase TEXT, freq INTEGER ", i);

		for (size_t j = 0; j <= i; j++)

			m_sql.appendPrintf (",s%d INTEGER, y%d INTEGER", j, j);

		m_sql << ");\n";

        }



        /* create index */

        m_sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << "index_0_0 ON py_phrase_0(s0,y0,phrase);\n";

        m_sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << "index_1_0 ON py_phrase_1(s0,y0,s1,y1,phrase);\n";

        m_sql << "CREATE INDEX IF NOT EXISTS " << "index_1_1 ON py_phrase_1(s0,s1,y1);\n";

        for (size_t i = 2; i < MAX_PHRASE_LEN; i++) {

		m_sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << "index_" << i << "_0 ON py_phrase_" << i

		      << "(s0,y0";

		for (size_t j = 1; j <= i; j++)

			m_sql << ",s" << j << ",y" << j;

		m_sql << ",phrase);\n";

		m_sql << "CREATE INDEX IF NOT EXISTS " << "index_" << i << "_1 ON py_phrase_" << i << "(s0,s1,s2,y2);\n";

        }

        m_sql << "COMMIT;";



	return executeSQL (m_sql, userdb);

	std::vector<String> files;

	files.push_back (m_user_data_dir + "/main.db");

	files.push_back (PKGDATADIR"/db/local.db");

	files.push_back (PKGDATADIR"/db/open-phrase.db");

	files.push_back (PKGDATADIR"/db/android.db");

	return first_existing_file (files);

}



/**

 * copy src_dbname to dest_dbname using sqlite3_backup_step().

 * set pragma on "maindb" and "userdb" on given sqlite3 db handler.

 *

 * dest and src should be opened sqlite3 db handler.

 * dest_dbname and src_dbname are db (schema) names.

 * see document at

 * https://www.sqlite.org/pragma.html

 * 

 * Returns: true on success, false otherwise.

 */

bool

Database::copyDB (sqlite3 *dest, const char* dest_dbname,

		  sqlite3 *src, const char* src_dbname)

static bool

db_set_pragma (sqlite3* db)

{

	bool copy_done = false;

        sqlite3_backup *backup = sqlite3_backup_init (

		dest, dest_dbname, src, src_dbname);

        if (backup) {

            int r = sqlite3_backup_step (backup, -1);

	    if (r == SQLITE_DONE) {

		    copy_done = true;

	    } else {

		    g_warning ("sqlite3_backup_step() failed: %d (%s)",

			       r, sqlite3_errmsg (dest));

	    }

            r = sqlite3_backup_finish (backup);

	    if (r != SQLITE_OK) {

		    g_warning ("sqlite3_backup_finish() failed: %d (%s)",

			       r, sqlite3_errmsg (dest));

	    }

        } else {

		g_warning ("sqlite3_backup_init() failed: %d (%s)",

			   sqlite3_errcode (dest), sqlite3_errmsg (dest));

	}

	return copy_done;

	g_debug ("setting pragma on db");

	const char* sql =

		"PRAGMA maindb.temp_store=MEMORY;"

		"PRAGMA userdb.temp_store=MEMORY;";

	char* errmsg = NULL;

	int r = sqlite3_exec (db, sql, NULL, NULL, &errmsg);

	sqlite3_free (errmsg);

	return r == SQLITE_OK;

}



/**

 * return TRUE if file exists

 */

static gboolean

file_exists (const char* filename) {

	return g_file_test (filename, G_FILE_TEST_EXISTS);

}





/**

 * this will load data from user db to an attached :memory: db on m_db.

 * the attached db is called "userdb".

 *

 * if there is no local user db file (usually

 * ~/.cache/ibus/pinyin/user-1.0.db), create an empty user db in :memory: and

 * use that.

 * Create user db file and init the user db with tables and indices.

 * 

 * Returns: true if the process finished successfully, false otherwise.

 * Returns: TRUE on success, FALSE otherwise.

 */

bool

Database::loadUserDB (void)

gboolean

Database::createUserDBFile ()

{

    sqlite3 *userdb = NULL;

    int r = 0;

    do {

        /* Attach user database */

        m_sql.printf ("ATTACH DATABASE \":memory:\" AS userdb;");

        if (!executeSQL (m_sql))

		break;

	g_return_val_if_fail (! m_user_data_dir.empty (), FALSE);

	g_return_val_if_fail (! m_user_db_file.empty (), FALSE);



        r = g_mkdir_with_parents (m_user_data_dir, 0750);

	g_debug ("ensure dir exists:%s", m_user_data_dir.c_str ());

	gint r = g_mkdir_with_parents (m_user_data_dir, 0750);

	if (r != 0) {

		g_warning ("create dir %s failed: %d (%s)",

			   m_user_data_dir.c_str (), r, g_strerror (r));

		// not critical, libpyzy should still function without a user

		// db file.

		return FALSE;

	}



	g_message ("loading user db at %s", m_user_db_file.c_str ());

	// always open RW because we may need to add additional table or index.

	sqlite3 *userdb = NULL;

	g_message ("creating user db at %s", m_user_db_file.c_str ());

	r = sqlite3_open (m_user_db_file, &userdb);

        if (r != SQLITE_OK) {

		if (file_exists (m_user_db_file.c_str ())) {

			g_warning ("open user db failed: %d (%s)",

				   r, sqlite3_errmsg (userdb));

		}

		// use a :memory: db as userdb, only works for current

		// session.

		r = sqlite3_open (":memory:", &userdb);

		if (r != SQLITE_OK) {

			g_warning ("open :memory: as user db failed: %d (%s)",

				   r, sqlite3_errmsg (userdb));

			break;

		}

	}

	g_assert_nonnull (userdb);



	r = initUserDB (userdb);

	if (! r) {

		break;

	}



	r = copyDB (m_db, "userdb", userdb, "main");

	if (! r) {

		g_warning ("copy user db to (attached :memory: userdb) failed");

		break;

	}



        r = sqlite3_close (userdb);

	if (r != SQLITE_OK) {

		g_warning ("close userdb failed: %d (%s)",

			   r, sqlite3_errmsg (userdb));

		// this is a minor problem.

		// I still want to return true. so no break here.

	}

        return true;

    } while (0);



    r = sqlite3_close (userdb);

    if (r != SQLITE_OK) {

	    g_warning ("close userdb failed: %d (%s)",

		       r, sqlite3_errmsg (userdb));

		sqlite3_close (userdb);

		return FALSE;

	}

    return false;

	bool result = initUserDB (userdb, "main");

	sqlite3_close (userdb);

	return result;

}



/**

 * save :memory: based "userdb" in m_db back to user db file.

 * init m_db handler, make main db accessible at "maindb" schema,

 * make user db accessible at "userdb" schema.

 *

 * This also sets m_main_db_file, m_user_db_file to correct file name.

 *

 * Returns: true on success, false otherwise.

 */

bool

Database::saveUserDB (void)

Database::initDB ()

{

	g_assert_null (m_db);

	if (m_db)

		return TRUE;



	sqlite3* db = NULL;

	int r = 0;

	r = g_mkdir_with_parents (m_user_data_dir, 0750);

	r = sqlite3_open (SQLITE3_MEMORY_DB, &db);

	if (r) {

		g_warning ("create dir %s failed: %d (%s)",

			   m_user_data_dir.c_str (), r, g_strerror (r));

		return false;

		g_warning ("sqlite3 open %s db failed", SQLITE3_MEMORY_DB);

		goto fail;

	}

	String user_db_filename = "";

	user_db_filename << m_user_data_dir << G_DIR_SEPARATOR_S << USER_DICTIONARY_FILE;

	String tmpfile = user_db_filename + "-tmp";

	sqlite3 *userdb = NULL;

	bool save_ok = false;

	do {

		/* remove tmpfile if it exist */

		r = g_unlink (tmpfile);

		if (r) {

			if (file_exists (tmpfile)) {

				g_warning ("delete tmp db %s failed: %d (%s)",

					   tmpfile.c_str (), r, g_strerror (r));

				// do not reuse existing -tmp db, can result

				// in duplicate data when copyDB().

				return false;



	m_main_db_file = getMainDBFile();

	if (m_main_db_file.empty()) {

		g_warning ("failed to find a main db file");

		goto fail;

	}

	g_message ("found main db file at %s", m_main_db_file.c_str ());

	r = attach_db (db, m_main_db_file, "maindb");

	if (!r) {

		g_warning ("attach main db file %s failed",

			   m_main_db_file.c_str ());

		goto fail;

	}

	m_user_db_file = getUserDBFile();

	g_assert (! m_user_db_file.empty());

	if (! file_exists (m_user_db_file.c_str ())) {

		r = createUserDBFile ();

		if (!r) {

			g_warning ("create user db file failed");

			// use a memory db for user db.

			// populate tables after attach.

			m_user_db_file = SQLITE3_MEMORY_DB;

		}

	} else {

			g_debug ("old tmpfile %s removed", tmpfile.c_str ());

		g_message ("found user db file at %s", m_user_db_file.c_str ());

	}

		g_message ("saving in RAM userdb to %s", tmpfile.c_str ());

		unsigned int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;

		g_assert (! file_exists (tmpfile));

		r = sqlite3_open_v2 (tmpfile, &userdb, flags, NULL);

		if (r != SQLITE_OK) {

			g_warning ("open tmp db %s failed: %d (%s)",

				   tmpfile.c_str (),

				   r, sqlite3_errmsg (userdb));

			break;

	r = attach_db (db, m_user_db_file, "userdb");

	if (!r) {

		g_warning ("attach user db file %s failed",

			   m_user_db_file.c_str ());

		goto fail;

	}

		r = copyDB (userdb, "main", m_db, "userdb");

	if (m_user_db_file == SQLITE3_MEMORY_DB) {

		bool r = initUserDB (db, "userdb");

		if (!r) {

			g_warning ("save user db back to file failed");

		} else {

			save_ok = true;

			g_warning ("init in RAM user db failed");

			g_assert_not_reached ();

			goto fail;

		}

		sqlite3_close (userdb);



		if (save_ok) {

			r = g_rename (tmpfile, user_db_filename);

			if (r) {

				g_warning ("rename tmpfile to %s failed: "

					   "%d (%s)", user_db_filename.c_str (),

					   r, g_strerror (r));

				return false;

	}

			g_message ("tmp file renamed. user db %s updated.",

				   user_db_filename.c_str ());

			return true;

	r = db_set_pragma (db);

	if (!r) {

		g_warning ("set pragma on db failed");

		// can continue

	}

		return false;

	} while (0);

	g_assert_nonnull (db);

	m_db = db;

	return true;



	sqlite3_close (userdb);

	g_unlink (tmpfile);

fail:

	sqlite3_close (db);

	g_warning ("initDB() failed, query won't work");

	return false;

}



void

Database::prefetch (void)

String

Database::getUserDBFile (void)

{

	m_sql.clear ();

	for (size_t i = 0; i < DB_PREFETCH_LEN; i++)

		m_sql << "SELECT * FROM py_phrase_" << i << ";\n";

	return m_user_data_dir + G_DIR_SEPARATOR_S + USER_DICTIONARY_FILE;

}



	g_debug ("prefetching ...");

	executeSQL (m_sql);

	g_debug ("done");

Database::Database (const std::string &user_data_dir)

    : m_db (NULL)

    , m_user_data_dir (user_data_dir)

{

	initDB ();		// init m_db

}



Database::~Database (void)

{

	int r = sqlite3_close (m_db);

	if (r != SQLITE_OK) {

		g_warning ("close sqlite database failed: %d (%s)",

			   r, sqlite3_errmsg (m_db));

	}

	m_db = NULL;

}



/**

 * call saveUserDB() if timer has run for DB_BACKUP_TIMEOUT seconds or more.

 * initialize user db.

 * create tables, index and populate data into desc table.

 *

 * used as GSourceFunc() for g_timeout_add_seconds().

 * Returns: true on success, false otherwise.

 */

gboolean

Database::cb_saveUserDB (gpointer user_data)

bool

Database::initUserDB (sqlite3 *userdb, const char* schema)

{

	Database *self = static_cast<Database*> (user_data);

	double elapsed = g_timer_elapsed (self->m_timer, NULL); // in seconds

	if (elapsed + 1 > DB_BACKUP_TIMEOUT) {

		bool r = self->saveUserDB ();

		if (! r) {

			g_warning ("auto save user db failed");

	String sql;

	sql = "BEGIN TRANSACTION;\n";

	// create desc table

	sql << "CREATE TABLE IF NOT EXISTS " << schema << ".desc (name PRIMARY KEY, value TEXT);\n";

	sql << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('version', '1.2.0');\n"

	    << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('uuid', '" << UUID () << "');\n"

	    << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('hostname', '" << Hostname () << "');\n"

	    << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('username', '" << Env ("USERNAME") << "');\n"

	    << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('create-time', datetime());\n"

	    << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('attach-time', datetime());\n";



	/* create phrase tables */

	for (size_t i = 0; i < MAX_PHRASE_LEN; i++) {

		sql.appendPrintf ("CREATE TABLE IF NOT EXISTS %s.py_phrase_%d (user_freq, phrase TEXT, freq INTEGER", schema, i);

		for (size_t j = 0; j <= i; j++)

			sql.appendPrintf (", s%d INTEGER, y%d INTEGER", j, j);

		sql << ");\n";

	}

		self->m_timeout_id = 0;

		return G_SOURCE_REMOVE;



	/* create index */

	sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << schema << ".index_0_0 ON py_phrase_0(s0,y0,phrase);\n";

	sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << schema << ".index_1_0 ON py_phrase_1(s0,y0,s1,y1,phrase);\n";

	sql << "CREATE INDEX IF NOT EXISTS " << schema << ".index_1_1 ON py_phrase_1(s0,s1,y1);\n";

	for (size_t i = 2; i < MAX_PHRASE_LEN; i++) {

		sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << schema << ".index_" << i << "_0 ON py_phrase_" << i

		      << "(s0,y0";

		for (size_t j = 1; j <= i; j++)

			sql << ",s" << j << ",y" << j;

		sql << ",phrase);\n";

		sql << "CREATE INDEX IF NOT EXISTS " << schema << ".index_" << i << "_1 ON py_phrase_" << i << "(s0,s1,s2,y2);\n";

	}

	return G_SOURCE_CONTINUE;

	sql << "COMMIT;";



	return sqlite3_exec_simple (userdb, sql.c_str ());

}



/**

 * this method is called whenever user db is modified.

 * 

 * we will schedule a user db save in DB_BACKUP_TIMEOUT seconds, if no new

 * modification came in between. if there are new modifications, wait for

 * DB_BACKUP_TIMEOUT after last modification.

 */

void

Database::modified (void)

{

	if (m_timeout_id) {

		g_timer_start (m_timer); // reset timer.

	} else {

		static const guint CHECK_INTERVAL = DB_BACKUP_TIMEOUT;

		m_timeout_id = g_timeout_add_seconds (

			CHECK_INTERVAL,

			Database::cb_saveUserDB,

			static_cast<gpointer> (this));

	}

}



/**
@@ -745,6 +591,12 @@ Database::query (const PinyinArray &pinyin, 
                 int                m,

                 guint              option)

{

    if (! m_db) {

        g_warning ("Error: can't query db "

		   "because db init failed");

        return nullptr;

    }



    g_assert (pinyin_begin < pinyin.size ());

    g_assert (pinyin_len <= pinyin.size () - pinyin_begin);

    g_assert (pinyin_len <= MAX_PHRASE_LEN);
@@ -761,8 +613,7 @@ Database::query (const PinyinArray &pinyin, 
        fs2 = pinyin_option_check_sheng (option, p->pinyin_id[0].sheng, p->pinyin_id[2].sheng);



        if (G_LIKELY (i > 0))

            conditions.appendPrintf (0, conditions.size (),

                                       " AND ");

            conditions.appendPrintf (0, conditions.size (), " AND ");



        if (G_UNLIKELY (fs1 || fs2)) {

            if (G_LIKELY (i < DB_INDEX_SIZE)) {
@@ -841,22 +692,21 @@ Database::query (const PinyinArray &pinyin, 
		    sql_condition << " OR (" << conditions[i] << ")\n";

    }



    m_sql.clear ();

    String sql;

    int id = pinyin_len - 1;

    m_sql << "SELECT * FROM ("

                "SELECT 0 AS user_freq, * FROM main.py_phrase_" << id << " WHERE " << sql_condition << " UNION ALL "

    sql << "SELECT * FROM ("

                "SELECT 0 AS user_freq, * FROM maindb.py_phrase_" << id << " WHERE " << sql_condition << " UNION ALL "

                "SELECT * FROM userdb.py_phrase_" << id << " WHERE " << sql_condition << ") "

                    "GROUP BY phrase ORDER BY user_freq DESC, freq DESC ";

    if (m > 0)

        m_sql << " LIMIT " << m;

#if 0

    g_debug ("sql =\n%s", m_sql.c_str ());

#endif

        sql << "LIMIT " << m;



    g_debug ("sql=\n%s", sql.c_str ());



    /* query database */

    SQLStmtPtr stmt (new SQLStmt (m_db));



    if (!stmt->prepare (m_sql)) {

    if (!stmt->prepare (sql)) {

	    stmt.reset ();

    }
@@ -907,22 +757,28 @@ Database::phraseSql (const Phrase &p, String &sql) 
void

Database::commit (const PhraseArray &phrases)

{

	if (! m_db) {

		g_warning ("Error: can't commit new phrase "

			   "because db init failed");

		return;

	}

	Phrase phrase = {""};



    m_sql = "BEGIN TRANSACTION;\n";

	String sql = "BEGIN TRANSACTION;\n";

	for (size_t i = 0; i < phrases.size (); i++) {

		phrase += phrases[i];

        phraseSql (phrases[i], m_sql);

		phraseSql (phrases[i], sql);

	}

	if (phrases.size () > 1)

        phraseSql (phrase, m_sql);

    m_sql << "COMMIT;\n";

		phraseSql (phrase, sql);

	sql << "COMMIT;\n";



    bool r = executeSQL (m_sql);

	gboolean r = sqlite3_exec_simple (m_db, sql);

	if (r) {

		modified ();

	} else {

	    g_warning ("insert phrases to (or update freq for) userdb failed");

		g_warning ("insert phrases to (or update freq for) "

			   "userdb failed");

	}

}
@@ -932,22 +788,45 @@ Database::commit (const PhraseArray &phrases) 
void

Database::remove (const Phrase & phrase)

{

    m_sql = "BEGIN TRANSACTION;\n";

    m_sql << "DELETE FROM userdb.py_phrase_" << phrase.len - 1;

    phraseWhereSql (phrase, m_sql);

    m_sql << ";\n"

	  << "COMMIT;\n";



    executeSQL (m_sql);

	if (! m_db) {

		g_warning ("Error: can't remove phrase "

			   "because db init failed");

		return;

	}

	String sql;

	sql << "BEGIN;\n"

		"DELETE FROM userdb.py_phrase_" << phrase.len - 1;

	phraseWhereSql (phrase, sql);

	sql << ";\n"

		"COMMIT;\n";

	gboolean r = sqlite3_exec_simple (m_db, sql);

	if (r) {

		modified ();

	} else {

		g_warning ("remove phrase %s from userdb failed", phrase.phrase);

	}

}



void

/**

 * create Database singleton instance at Database::m_instance.

 *

 * upstream should check the return value for error. if false is returned,

 * upstream app should not try to access the db.

 *

 * Returns: true on success, false otherwise.

 */

bool

Database::init (const std::string & user_data_dir)

{

	if (m_instance.get () == NULL) {

		m_instance.reset (new Database (user_data_dir));

	}

	if (! m_instance->m_db) {

		// try initDB() again if previous call in Database constructor

		// failed.

		return m_instance->initDB ();

	}

	return true;

}



void

src/Database.h

+19 −26

File changed.

Preview size limit exceeded, changes collapsed.

src/PhraseEditor.cc

+1 −1
Original line number Diff line number Diff line
@@ -152,8 +152,8 @@ PhraseEditor::updateTheFirstCandidate (void) 
        ret = query.fill (m_candidate_0_phrases, 1);

	if (ret != 1) {

		g_warning ("expect query.fill() result be 1, found %d", ret);

		break;

	}

        g_assert (ret == 1);

        begin += m_candidate_0_phrases.back ().len;

    }

}

src/meson.build

+7 −3
Original line number Diff line number Diff line 
# -*- mode: conf -*-

project('pyzy', 'cpp',

project('pyzy', ['cpp', 'c'],

  version: '1.0.1-6',

  license: 'GPL',

  default_options: [
@@ -21,7 +21,10 @@ add_project_arguments( 
  language: 'cpp')



if get_option('buildtype').startswith('release')

  add_project_arguments('-DG_DISABLE_ASSERT', language: 'cpp')

  add_project_arguments(

    '-DG_DISABLE_ASSERT',

    '-DG_DISABLE_CHECKS',

    language: 'cpp')

endif



glib = dependency('glib-2.0')
@@ -44,11 +47,12 @@ lib_src = [ 
  'SpecialPhrase.cc',

  'SpecialPhraseTable.cc',

  'Variant.cc',

  'sqlite3_util.c',

]

shared_library('pyzy-1.0',

  lib_src,

  soversion: '0',

  version: '0.100.1',

  version: '0.101.0',

  dependencies: shared_dep,

  install: true)

src/sqlite3_util.c

0 → 100644
+66 −0

File added.

Preview size limit exceeded, changes collapsed.