From 8b6cf0271f1f06e7664b4edda257d220aefd7699 Mon Sep 17 00:00:00 2001 From: Yuanle Song Date: Fri, 29 Mar 2019 21:31:55 +0800 Subject: [PATCH] v0.101.0 redesign use of main and user database. the old design have flaws and limitations. see :id001: and :id002: in operational file. new design: open :memory: db readwrite, this will be the m_db handler. try attach main db at known path (readonly if possible) to "maindb". if failed, try next known path, if all failed, initDB() should return false and gave up. try attach user db at known path to "userdb" schema, if failed, create :memory: db and initialize it as user db. attach it as "userdb" schema. update all query against main db to query from maindb.* table. update all SQL against user db to run again userdb.* table. benefits: - access to user db is concurrent safe. multiple libpyzy app won't overwrite user's user db file and lose data. - there is no need to saveUserDB() using timer in the bg. - there is no need to copy data from user db to :memory:, if user db is large, this can lower memory usage. --- src/Database.cc | 693 ++++++++++++++++++-------------------------- src/Database.h | 45 ++- src/PhraseEditor.cc | 2 +- src/meson.build | 10 +- src/sqlite3_util.c | 66 +++++ src/sqlite3_util.h | 30 ++ 6 files changed, 409 insertions(+), 437 deletions(-) create mode 100644 src/sqlite3_util.c create mode 100644 src/sqlite3_util.h diff --git a/src/Database.cc b/src/Database.cc index 7e09b9c..272d696 100644 --- a/src/Database.cc +++ b/src/Database.cc @@ -44,7 +44,7 @@ namespace PyZy { #define DB_BACKUP_TIMEOUT (60) #define USER_DICTIONARY_FILE "user-1.0.db" - +#define SQLITE3_MEMORY_DB ":memory:" std::unique_ptr Database::m_instance; @@ -162,13 +162,21 @@ Query::Query (const PinyinArray & pinyin, m_pinyin_len (pinyin_len), m_option (option) { - g_assert (m_pinyin.size () >= pinyin_begin + pinyin_len); + g_assert (m_pinyin.size () >= pinyin_begin + pinyin_len); } Query::~Query (void) { } +/** + * query db to find phrases for pinyin. + * + * @phrases: put query result in this PhraseArray. + * @count: query at most this many phrases. + * + * Returns: how many phrases fetched from db. + */ int Query::fill (PhraseArray &phrases, int count) { @@ -178,7 +186,9 @@ Query::fill (PhraseArray &phrases, int count) if (G_LIKELY (m_stmt.get () == NULL)) { m_stmt = Database::instance ().query ( m_pinyin, m_pinyin_begin, m_pinyin_len, -1, m_option); - g_assert (m_stmt.get () != NULL); + if (m_stmt == nullptr) { + return 0; + } } while (m_stmt->step ()) { @@ -210,441 +220,277 @@ Query::fill (PhraseArray &phrases, int count) return row; } -Database::Database (const std::string &user_data_dir) - : m_db (NULL) - , m_timeout_id (0) - , m_timer (g_timer_new ()) - , m_user_data_dir (user_data_dir) -{ - m_user_db_file.clear (); - m_user_db_file << m_user_data_dir - << G_DIR_SEPARATOR_S - << USER_DICTIONARY_FILE; - bool r = open (); - if (! r) { - g_error ("open main db failed"); - } -} - -Database::~Database (void) -{ - g_timer_destroy (m_timer); - if (m_timeout_id != 0) { - bool r = saveUserDB (); - if (! r) { - g_warning ("save user db failed"); - } - gboolean r1 = g_source_remove (m_timeout_id); - if (! r1) { - g_warning ("remove timeout source failed, " - "source id is %d", m_timeout_id); - } - m_timeout_id = 0; - } - if (m_db) { - if (sqlite3_close (m_db) != SQLITE_OK) { - g_warning ("close sqlite database failed: %d (%s)", - sqlite3_errcode (m_db), sqlite3_errmsg (m_db)); - } - m_db = NULL; - } -} - -bool -Database::executeSQL (const char *sql, sqlite3 *db) -{ - if (! db) - db = m_db; - if (! db) { - g_warning ("trying to execute sql %s on db handler NULL", sql); - g_assert_not_reached (); - return false; - } - char *errmsg = NULL; - if (sqlite3_exec (db, sql, NULL, NULL, &errmsg) != SQLITE_OK) { - g_warning ("execute sql failed: sql=%s error=%s", sql, errmsg); - sqlite3_free (errmsg); - return false; - } - return true; -} - /** - * set sqlite3 pragma on main db to improve performance. + * attach db to dest handler as schema_name. * * Returns: true on success, false otherwise. */ -bool -Database::setPragmaOnMainDB (void) +static bool +attach_db (sqlite3* dest, String db_file_to_attach, const gchar* schema_name) { - m_sql.clear (); - - // see https://www.sqlite.org/pragma.html#pragma_synchronous - m_sql << "PRAGMA synchronous=OFF;\n"; - - /* Set the cache size for better performance */ - m_sql << "PRAGMA cache_size=" DB_CACHE_SIZE ";\n"; - - /* Using memory for temp store */ - // m_sql << "PRAGMA temp_store=MEMORY;\n"; - - /* Set journal mode */ - // m_sql << "PRAGMA journal_mode=PERSIST;\n"; - - /* Using EXCLUSIVE locking mode on databases - * for better performance */ - m_sql << "PRAGMA locking_mode=EXCLUSIVE;\n"; - - return executeSQL (m_sql); + g_message ("attaching db %s as %s", + db_file_to_attach.c_str (), schema_name); + char* sql = sqlite3_mprintf ("ATTACH DATABASE %Q AS %Q;", + db_file_to_attach.c_str (), + schema_name); + gboolean result = sqlite3_exec_simple (dest, sql); + sqlite3_free (sql); + return result; } /** - * try to open a main database. such as open-phrase.db. - * - * Returns: true on success, false otherwise. + * return TRUE if file exists */ -bool -Database::open (void) -{ - do { -#if (SQLITE_VERSION_NUMBER >= 3006000) - sqlite3_initialize (); -#endif - static const char * maindb [] = { - PKGDATADIR"/db/local.db", - PKGDATADIR"/db/open-phrase.db", - PKGDATADIR"/db/android.db", - "main.db", - }; - - size_t i; - for (i = 0; i < G_N_ELEMENTS (maindb); i++) { - g_debug ("trying to load main db at %s", maindb[i]); - if (sqlite3_open_v2 (maindb[i], &m_db, SQLITE_OPEN_READWRITE, NULL) == SQLITE_OK) { - g_message ("loading main db at %s", maindb[i]); - break; - } - } - if (i == G_N_ELEMENTS (maindb)) { - g_warning ("Failed to load any known main database"); - break; - } - - int r = 0; - r = setPragmaOnMainDB (); - if (! r) { - g_warning ("execute sqlite PRAGMA statements failed"); - break; - } - - r = loadUserDB (); - if (! r) { - g_warning ("load user db failed"); - break; - } - - /* prefetch some tables */ - // prefetch (); - g_assert_nonnull (m_db); - return true; - } while (0); - - if (m_db) { - sqlite3_close (m_db); - m_db = NULL; - } - return false; +static gboolean +file_exists (const char* filename) { + return g_file_test (filename, G_FILE_TEST_EXISTS); } /** - * initialize user db. - * create tables, index and populate data into desc table. - * - * Returns: true on success, false otherwise. + * return the first existing file in given file list. + * return "" if none of the file exists. */ -bool -Database::initUserDB (sqlite3 *userdb) +static String +first_existing_file (const std::vector &files) { - m_sql = "BEGIN TRANSACTION;\n"; - /* create desc table*/ - m_sql << "CREATE TABLE IF NOT EXISTS desc (name PRIMARY KEY, value TEXT);\n"; - m_sql << "INSERT OR IGNORE INTO desc VALUES ('version', '1.2.0');\n" - << "INSERT OR IGNORE INTO desc VALUES ('uuid', '" << UUID () << "');\n" - << "INSERT OR IGNORE INTO desc VALUES ('hostname', '" << Hostname () << "');\n" - << "INSERT OR IGNORE INTO desc VALUES ('username', '" << Env ("USERNAME") << "');\n" - << "INSERT OR IGNORE INTO desc VALUES ('create-time', datetime());\n" - << "INSERT OR IGNORE INTO desc VALUES ('attach-time', datetime());\n"; - - /* create phrase tables */ - for (size_t i = 0; i < MAX_PHRASE_LEN; i++) { - m_sql.appendPrintf ("CREATE TABLE IF NOT EXISTS py_phrase_%d (user_freq, phrase TEXT, freq INTEGER ", i); - for (size_t j = 0; j <= i; j++) - m_sql.appendPrintf (",s%d INTEGER, y%d INTEGER", j, j); - m_sql << ");\n"; - } - - /* create index */ - m_sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << "index_0_0 ON py_phrase_0(s0,y0,phrase);\n"; - m_sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << "index_1_0 ON py_phrase_1(s0,y0,s1,y1,phrase);\n"; - m_sql << "CREATE INDEX IF NOT EXISTS " << "index_1_1 ON py_phrase_1(s0,s1,y1);\n"; - for (size_t i = 2; i < MAX_PHRASE_LEN; i++) { - m_sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << "index_" << i << "_0 ON py_phrase_" << i - << "(s0,y0"; - for (size_t j = 1; j <= i; j++) - m_sql << ",s" << j << ",y" << j; - m_sql << ",phrase);\n"; - m_sql << "CREATE INDEX IF NOT EXISTS " << "index_" << i << "_1 ON py_phrase_" << i << "(s0,s1,s2,y2);\n"; - } - m_sql << "COMMIT;"; - - return executeSQL (m_sql, userdb); + for (const String &fn : files) { + if (file_exists (fn.c_str ())) { + return fn; + } + } + return ""; } /** - * copy src_dbname to dest_dbname using sqlite3_backup_step(). + * Returns: a main db file name if one exists. + * an empty string otherwise. * - * dest and src should be opened sqlite3 db handler. - * dest_dbname and src_dbname are db (schema) names. + * This function will look for file at these path: + * m_user_data_dir + "/main.db" + * {PKGDATADIR}/db/local.db + * {PKGDATADIR}/db/open-phrase.db + * {PKGDATADIR}/db/android.db * - * Returns: true on success, false otherwise. + * In ibus-pinyin context those are: + * ~/.cache/ibus/pinyin/local.db + * /usr/share/pyzy/db/local.db + * /usr/share/pyzy/db/open-phrase.db + * /usr/share/pyzy/db/android.db */ -bool -Database::copyDB (sqlite3 *dest, const char* dest_dbname, - sqlite3 *src, const char* src_dbname) +String +Database::getMainDBFile (void) { - bool copy_done = false; - sqlite3_backup *backup = sqlite3_backup_init ( - dest, dest_dbname, src, src_dbname); - if (backup) { - int r = sqlite3_backup_step (backup, -1); - if (r == SQLITE_DONE) { - copy_done = true; - } else { - g_warning ("sqlite3_backup_step() failed: %d (%s)", - r, sqlite3_errmsg (dest)); - } - r = sqlite3_backup_finish (backup); - if (r != SQLITE_OK) { - g_warning ("sqlite3_backup_finish() failed: %d (%s)", - r, sqlite3_errmsg (dest)); - } - } else { - g_warning ("sqlite3_backup_init() failed: %d (%s)", - sqlite3_errcode (dest), sqlite3_errmsg (dest)); - } - return copy_done; + std::vector files; + files.push_back (m_user_data_dir + "/main.db"); + files.push_back (PKGDATADIR"/db/local.db"); + files.push_back (PKGDATADIR"/db/open-phrase.db"); + files.push_back (PKGDATADIR"/db/android.db"); + return first_existing_file (files); } /** - * return TRUE if file exists + * set pragma on "maindb" and "userdb" on given sqlite3 db handler. + * + * see document at + * https://www.sqlite.org/pragma.html + * + * Returns: true on success, false otherwise. */ -static gboolean -file_exists (const char* filename) { - return g_file_test (filename, G_FILE_TEST_EXISTS); +static bool +db_set_pragma (sqlite3* db) +{ + g_debug ("setting pragma on db"); + const char* sql = + "PRAGMA maindb.temp_store=MEMORY;" + "PRAGMA userdb.temp_store=MEMORY;"; + char* errmsg = NULL; + int r = sqlite3_exec (db, sql, NULL, NULL, &errmsg); + sqlite3_free (errmsg); + return r == SQLITE_OK; } - /** - * this will load data from user db to an attached :memory: db on m_db. - * the attached db is called "userdb". - * - * if there is no local user db file (usually - * ~/.cache/ibus/pinyin/user-1.0.db), create an empty user db in :memory: and - * use that. - * - * Returns: true if the process finished successfully, false otherwise. + * Create user db file and init the user db with tables and indices. + * + * Returns: TRUE on success, FALSE otherwise. */ -bool -Database::loadUserDB (void) +gboolean +Database::createUserDBFile () { - sqlite3 *userdb = NULL; - int r = 0; - do { - /* Attach user database */ - m_sql.printf ("ATTACH DATABASE \":memory:\" AS userdb;"); - if (!executeSQL (m_sql)) - break; - - r = g_mkdir_with_parents (m_user_data_dir, 0750); + g_return_val_if_fail (! m_user_data_dir.empty (), FALSE); + g_return_val_if_fail (! m_user_db_file.empty (), FALSE); + + g_debug ("ensure dir exists:%s", m_user_data_dir.c_str ()); + gint r = g_mkdir_with_parents (m_user_data_dir, 0750); if (r != 0) { g_warning ("create dir %s failed: %d (%s)", m_user_data_dir.c_str (), r, g_strerror (r)); - // not critical, libpyzy should still function without a user - // db file. + return FALSE; } - g_message ("loading user db at %s", m_user_db_file.c_str ()); - // always open RW because we may need to add additional table or index. + sqlite3 *userdb = NULL; + g_message ("creating user db at %s", m_user_db_file.c_str ()); r = sqlite3_open (m_user_db_file, &userdb); if (r != SQLITE_OK) { - if (file_exists (m_user_db_file.c_str ())) { - g_warning ("open user db failed: %d (%s)", - r, sqlite3_errmsg (userdb)); - } - // use a :memory: db as userdb, only works for current - // session. - r = sqlite3_open (":memory:", &userdb); - if (r != SQLITE_OK) { - g_warning ("open :memory: as user db failed: %d (%s)", - r, sqlite3_errmsg (userdb)); - break; - } - } - g_assert_nonnull (userdb); - - r = initUserDB (userdb); - if (! r) { - break; - } - - r = copyDB (m_db, "userdb", userdb, "main"); - if (! r) { - g_warning ("copy user db to (attached :memory: userdb) failed"); - break; - } - - r = sqlite3_close (userdb); - if (r != SQLITE_OK) { - g_warning ("close userdb failed: %d (%s)", - r, sqlite3_errmsg (userdb)); - // this is a minor problem. - // I still want to return true. so no break here. + sqlite3_close (userdb); + return FALSE; } - return true; - } while (0); - - r = sqlite3_close (userdb); - if (r != SQLITE_OK) { - g_warning ("close userdb failed: %d (%s)", - r, sqlite3_errmsg (userdb)); - } - return false; + bool result = initUserDB (userdb, "main"); + sqlite3_close (userdb); + return result; } /** - * save :memory: based "userdb" in m_db back to user db file. - * + * init m_db handler, make main db accessible at "maindb" schema, + * make user db accessible at "userdb" schema. + * + * This also sets m_main_db_file, m_user_db_file to correct file name. + * * Returns: true on success, false otherwise. */ bool -Database::saveUserDB (void) +Database::initDB () { + g_assert_null (m_db); + if (m_db) + return TRUE; + + sqlite3* db = NULL; int r = 0; - r = g_mkdir_with_parents (m_user_data_dir, 0750); - if (r) { - g_warning ("create dir %s failed: %d (%s)", - m_user_data_dir.c_str (), r, g_strerror (r)); - return false; + r = sqlite3_open (SQLITE3_MEMORY_DB, &db); + if (r) { + g_warning ("sqlite3 open %s db failed", SQLITE3_MEMORY_DB); + goto fail; } - String user_db_filename = ""; - user_db_filename << m_user_data_dir << G_DIR_SEPARATOR_S << USER_DICTIONARY_FILE; - String tmpfile = user_db_filename + "-tmp"; - sqlite3 *userdb = NULL; - bool save_ok = false; - do { - /* remove tmpfile if it exist */ - r = g_unlink (tmpfile); - if (r) { - if (file_exists (tmpfile)) { - g_warning ("delete tmp db %s failed: %d (%s)", - tmpfile.c_str (), r, g_strerror (r)); - // do not reuse existing -tmp db, can result - // in duplicate data when copyDB(). - return false; - } - } else { - g_debug ("old tmpfile %s removed", tmpfile.c_str ()); - } - g_message ("saving in RAM userdb to %s", tmpfile.c_str ()); - unsigned int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - g_assert (! file_exists (tmpfile)); - r = sqlite3_open_v2 (tmpfile, &userdb, flags, NULL); - if (r != SQLITE_OK) { - g_warning ("open tmp db %s failed: %d (%s)", - tmpfile.c_str (), - r, sqlite3_errmsg (userdb)); - break; - } - r = copyDB (userdb, "main", m_db, "userdb"); - if (! r) { - g_warning ("save user db back to file failed"); - } else { - save_ok = true; - } - sqlite3_close (userdb); - if (save_ok) { - r = g_rename (tmpfile, user_db_filename); - if (r) { - g_warning ("rename tmpfile to %s failed: " - "%d (%s)", user_db_filename.c_str (), - r, g_strerror (r)); - return false; - } - g_message ("tmp file renamed. user db %s updated.", - user_db_filename.c_str ()); - return true; + m_main_db_file = getMainDBFile(); + if (m_main_db_file.empty()) { + g_warning ("failed to find a main db file"); + goto fail; + } + g_message ("found main db file at %s", m_main_db_file.c_str ()); + r = attach_db (db, m_main_db_file, "maindb"); + if (!r) { + g_warning ("attach main db file %s failed", + m_main_db_file.c_str ()); + goto fail; + } + m_user_db_file = getUserDBFile(); + g_assert (! m_user_db_file.empty()); + if (! file_exists (m_user_db_file.c_str ())) { + r = createUserDBFile (); + if (!r) { + g_warning ("create user db file failed"); + // use a memory db for user db. + // populate tables after attach. + m_user_db_file = SQLITE3_MEMORY_DB; + } + } else { + g_message ("found user db file at %s", m_user_db_file.c_str ()); + } + r = attach_db (db, m_user_db_file, "userdb"); + if (!r) { + g_warning ("attach user db file %s failed", + m_user_db_file.c_str ()); + goto fail; + } + if (m_user_db_file == SQLITE3_MEMORY_DB) { + bool r = initUserDB (db, "userdb"); + if (!r) { + g_warning ("init in RAM user db failed"); + g_assert_not_reached (); + goto fail; } - return false; - } while (0); + } + r = db_set_pragma (db); + if (!r) { + g_warning ("set pragma on db failed"); + // can continue + } + g_assert_nonnull (db); + m_db = db; + return true; - sqlite3_close (userdb); - g_unlink (tmpfile); +fail: + sqlite3_close (db); + g_warning ("initDB() failed, query won't work"); return false; } -void -Database::prefetch (void) +String +Database::getUserDBFile (void) { - m_sql.clear (); - for (size_t i = 0; i < DB_PREFETCH_LEN; i++) - m_sql << "SELECT * FROM py_phrase_" << i << ";\n"; + return m_user_data_dir + G_DIR_SEPARATOR_S + USER_DICTIONARY_FILE; +} - g_debug ("prefetching ..."); - executeSQL (m_sql); - g_debug ("done"); +Database::Database (const std::string &user_data_dir) + : m_db (NULL) + , m_user_data_dir (user_data_dir) +{ + initDB (); // init m_db +} + +Database::~Database (void) +{ + int r = sqlite3_close (m_db); + if (r != SQLITE_OK) { + g_warning ("close sqlite database failed: %d (%s)", + r, sqlite3_errmsg (m_db)); + } + m_db = NULL; } /** - * call saveUserDB() if timer has run for DB_BACKUP_TIMEOUT seconds or more. + * initialize user db. + * create tables, index and populate data into desc table. * - * used as GSourceFunc() for g_timeout_add_seconds(). + * Returns: true on success, false otherwise. */ -gboolean -Database::cb_saveUserDB (gpointer user_data) +bool +Database::initUserDB (sqlite3 *userdb, const char* schema) { - Database *self = static_cast (user_data); - double elapsed = g_timer_elapsed (self->m_timer, NULL); // in seconds - if (elapsed + 1 > DB_BACKUP_TIMEOUT) { - bool r = self->saveUserDB (); - if (! r) { - g_warning ("auto save user db failed"); - } - self->m_timeout_id = 0; - return G_SOURCE_REMOVE; + String sql; + sql = "BEGIN TRANSACTION;\n"; + // create desc table + sql << "CREATE TABLE IF NOT EXISTS " << schema << ".desc (name PRIMARY KEY, value TEXT);\n"; + sql << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('version', '1.2.0');\n" + << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('uuid', '" << UUID () << "');\n" + << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('hostname', '" << Hostname () << "');\n" + << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('username', '" << Env ("USERNAME") << "');\n" + << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('create-time', datetime());\n" + << "INSERT OR IGNORE INTO " << schema << ".desc VALUES ('attach-time', datetime());\n"; + + /* create phrase tables */ + for (size_t i = 0; i < MAX_PHRASE_LEN; i++) { + sql.appendPrintf ("CREATE TABLE IF NOT EXISTS %s.py_phrase_%d (user_freq, phrase TEXT, freq INTEGER", schema, i); + for (size_t j = 0; j <= i; j++) + sql.appendPrintf (", s%d INTEGER, y%d INTEGER", j, j); + sql << ");\n"; } - return G_SOURCE_CONTINUE; + + /* create index */ + sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << schema << ".index_0_0 ON py_phrase_0(s0,y0,phrase);\n"; + sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << schema << ".index_1_0 ON py_phrase_1(s0,y0,s1,y1,phrase);\n"; + sql << "CREATE INDEX IF NOT EXISTS " << schema << ".index_1_1 ON py_phrase_1(s0,s1,y1);\n"; + for (size_t i = 2; i < MAX_PHRASE_LEN; i++) { + sql << "CREATE UNIQUE INDEX IF NOT EXISTS " << schema << ".index_" << i << "_0 ON py_phrase_" << i + << "(s0,y0"; + for (size_t j = 1; j <= i; j++) + sql << ",s" << j << ",y" << j; + sql << ",phrase);\n"; + sql << "CREATE INDEX IF NOT EXISTS " << schema << ".index_" << i << "_1 ON py_phrase_" << i << "(s0,s1,s2,y2);\n"; + } + sql << "COMMIT;"; + + return sqlite3_exec_simple (userdb, sql.c_str ()); } /** * this method is called whenever user db is modified. - * - * we will schedule a user db save in DB_BACKUP_TIMEOUT seconds, if no new - * modification came in between. if there are new modifications, wait for - * DB_BACKUP_TIMEOUT after last modification. */ void Database::modified (void) { - if (m_timeout_id) { - g_timer_start (m_timer); // reset timer. - } else { - static const guint CHECK_INTERVAL = DB_BACKUP_TIMEOUT; - m_timeout_id = g_timeout_add_seconds ( - CHECK_INTERVAL, - Database::cb_saveUserDB, - static_cast (this)); - } } /** @@ -745,6 +591,12 @@ Database::query (const PinyinArray &pinyin, int m, guint option) { + if (! m_db) { + g_warning ("Error: can't query db " + "because db init failed"); + return nullptr; + } + g_assert (pinyin_begin < pinyin.size ()); g_assert (pinyin_len <= pinyin.size () - pinyin_begin); g_assert (pinyin_len <= MAX_PHRASE_LEN); @@ -761,8 +613,7 @@ Database::query (const PinyinArray &pinyin, fs2 = pinyin_option_check_sheng (option, p->pinyin_id[0].sheng, p->pinyin_id[2].sheng); if (G_LIKELY (i > 0)) - conditions.appendPrintf (0, conditions.size (), - " AND "); + conditions.appendPrintf (0, conditions.size (), " AND "); if (G_UNLIKELY (fs1 || fs2)) { if (G_LIKELY (i < DB_INDEX_SIZE)) { @@ -841,22 +692,21 @@ Database::query (const PinyinArray &pinyin, sql_condition << " OR (" << conditions[i] << ")\n"; } - m_sql.clear (); + String sql; int id = pinyin_len - 1; - m_sql << "SELECT * FROM (" - "SELECT 0 AS user_freq, * FROM main.py_phrase_" << id << " WHERE " << sql_condition << " UNION ALL " + sql << "SELECT * FROM (" + "SELECT 0 AS user_freq, * FROM maindb.py_phrase_" << id << " WHERE " << sql_condition << " UNION ALL " "SELECT * FROM userdb.py_phrase_" << id << " WHERE " << sql_condition << ") " - "GROUP BY phrase ORDER BY user_freq DESC, freq DESC"; + "GROUP BY phrase ORDER BY user_freq DESC, freq DESC "; if (m > 0) - m_sql << " LIMIT " << m; -#if 0 - g_debug ("sql =\n%s", m_sql.c_str ()); -#endif + sql << "LIMIT " << m; + + g_debug ("sql=\n%s", sql.c_str ()); /* query database */ SQLStmtPtr stmt (new SQLStmt (m_db)); - if (!stmt->prepare (m_sql)) { + if (!stmt->prepare (sql)) { stmt.reset (); } @@ -907,23 +757,29 @@ Database::phraseSql (const Phrase &p, String &sql) void Database::commit (const PhraseArray &phrases) { - Phrase phrase = {""}; + if (! m_db) { + g_warning ("Error: can't commit new phrase " + "because db init failed"); + return; + } + Phrase phrase = {""}; - m_sql = "BEGIN TRANSACTION;\n"; - for (size_t i = 0; i < phrases.size (); i++) { - phrase += phrases[i]; - phraseSql (phrases[i], m_sql); - } - if (phrases.size () > 1) - phraseSql (phrase, m_sql); - m_sql << "COMMIT;\n"; - - bool r = executeSQL (m_sql); - if (r) { - modified (); - } else { - g_warning ("insert phrases to (or update freq for) userdb failed"); - } + String sql = "BEGIN TRANSACTION;\n"; + for (size_t i = 0; i < phrases.size (); i++) { + phrase += phrases[i]; + phraseSql (phrases[i], sql); + } + if (phrases.size () > 1) + phraseSql (phrase, sql); + sql << "COMMIT;\n"; + + gboolean r = sqlite3_exec_simple (m_db, sql); + if (r) { + modified (); + } else { + g_warning ("insert phrases to (or update freq for) " + "userdb failed"); + } } /** @@ -932,22 +788,45 @@ Database::commit (const PhraseArray &phrases) void Database::remove (const Phrase & phrase) { - m_sql = "BEGIN TRANSACTION;\n"; - m_sql << "DELETE FROM userdb.py_phrase_" << phrase.len - 1; - phraseWhereSql (phrase, m_sql); - m_sql << ";\n" - << "COMMIT;\n"; - - executeSQL (m_sql); - modified (); + if (! m_db) { + g_warning ("Error: can't remove phrase " + "because db init failed"); + return; + } + String sql; + sql << "BEGIN;\n" + "DELETE FROM userdb.py_phrase_" << phrase.len - 1; + phraseWhereSql (phrase, sql); + sql << ";\n" + "COMMIT;\n"; + gboolean r = sqlite3_exec_simple (m_db, sql); + if (r) { + modified (); + } else { + g_warning ("remove phrase %s from userdb failed", phrase.phrase); + } } -void +/** + * create Database singleton instance at Database::m_instance. + * + * upstream should check the return value for error. if false is returned, + * upstream app should not try to access the db. + * + * Returns: true on success, false otherwise. + */ +bool Database::init (const std::string & user_data_dir) { - if (m_instance.get () == NULL) { - m_instance.reset (new Database (user_data_dir)); - } + if (m_instance.get () == NULL) { + m_instance.reset (new Database (user_data_dir)); + } + if (! m_instance->m_db) { + // try initDB() again if previous call in Database constructor + // failed. + return m_instance->initDB (); + } + return true; } void diff --git a/src/Database.h b/src/Database.h index ab7c8ca..1b98daa 100644 --- a/src/Database.h +++ b/src/Database.h @@ -27,6 +27,7 @@ #include "String.h" #include "Types.h" #include "Util.h" +#include "sqlite3_util.h" typedef struct sqlite3 sqlite3; @@ -61,10 +62,19 @@ class Database { public: ~Database (); protected: - Database (const std::string & user_data_dir); + Database (const std::string &user_data_dir); public: - static void init (const std::string & data_dir); + static bool init (const std::string &data_dir); + static Database & instance (void) + { + if (m_instance == NULL) { + g_error ("Error: Please call InputContext::init () !"); + g_assert_not_reached (); + } + return *m_instance; + } + static void finalize (void); SQLStmtPtr query (const PinyinArray & pinyin, size_t pinyin_begin, @@ -74,40 +84,23 @@ public: void commit (const PhraseArray & phrases); void remove (const Phrase & phrase); - void conditionsDouble (void); - void conditionsTriple (void); - - static void finalize (void); - static Database & instance (void) - { - if (m_instance == NULL) { - g_error ("Error: Please call InputContext::init () !"); - } - return *m_instance; - } - private: + bool initDB (void); + static bool initUserDB (sqlite3* userdb, const char* schema); + String getMainDBFile (void); + String getUserDBFile (void); + gboolean createUserDBFile (void); bool setPragmaOnMainDB (void); - bool open (void); - bool initUserDB (sqlite3* userdb); - bool copyDB (sqlite3* dest, const char* dest_dbname, - sqlite3* src, const char* src_dbname); - bool loadUserDB (void); - bool saveUserDB (void); void prefetch (void); void phraseSql (const Phrase & p, String & sql); void phraseWhereSql (const Phrase & p, String & sql); - bool executeSQL (const char* sql, sqlite3* db = NULL); static gboolean cb_saveUserDB (gpointer user_data); void modified (void); private: - sqlite3 *m_db; /* sqlite3 database */ - - String m_sql; /* sql stmt */ - unsigned int m_timeout_id; - GTimer *m_timer; + sqlite3 *m_db; /* db handler to access maindb and userdb */ String m_user_data_dir; + String m_main_db_file; /* main db file name with full path */ String m_user_db_file; /* user db file name with full path */ private: diff --git a/src/PhraseEditor.cc b/src/PhraseEditor.cc index a4746c4..c89d3ef 100644 --- a/src/PhraseEditor.cc +++ b/src/PhraseEditor.cc @@ -152,8 +152,8 @@ PhraseEditor::updateTheFirstCandidate (void) ret = query.fill (m_candidate_0_phrases, 1); if (ret != 1) { g_warning ("expect query.fill() result be 1, found %d", ret); + break; } - g_assert (ret == 1); begin += m_candidate_0_phrases.back ().len; } } diff --git a/src/meson.build b/src/meson.build index 2dcacbb..cdf81f3 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,5 +1,5 @@ # -*- mode: conf -*- -project('pyzy', 'cpp', +project('pyzy', ['cpp', 'c'], version: '1.0.1-6', license: 'GPL', default_options: [ @@ -21,7 +21,10 @@ add_project_arguments( language: 'cpp') if get_option('buildtype').startswith('release') - add_project_arguments('-DG_DISABLE_ASSERT', language: 'cpp') + add_project_arguments( + '-DG_DISABLE_ASSERT', + '-DG_DISABLE_CHECKS', + language: 'cpp') endif glib = dependency('glib-2.0') @@ -44,11 +47,12 @@ lib_src = [ 'SpecialPhrase.cc', 'SpecialPhraseTable.cc', 'Variant.cc', + 'sqlite3_util.c', ] shared_library('pyzy-1.0', lib_src, soversion: '0', - version: '0.100.1', + version: '0.101.0', dependencies: shared_dep, install: true) diff --git a/src/sqlite3_util.c b/src/sqlite3_util.c new file mode 100644 index 0000000..2122d43 --- /dev/null +++ b/src/sqlite3_util.c @@ -0,0 +1,66 @@ +#include "sqlite3_util.h" + +/** + * execute sql on sqlite3 db using sqlite3_exec(). + * + * Returns: TRUE on success, FALSE otherwise. if FALSE, errmsg will be printed + * with g_warning(). + */ +gboolean +sqlite3_exec_simple (sqlite3 *db, const char *sql) +{ + if (! db) { + g_warning ("trying to execute sql %s on NULL db handler", sql); + g_assert_not_reached (); + return FALSE; + } + char *errmsg = NULL; + if (sqlite3_exec (db, sql, NULL, NULL, &errmsg) != SQLITE_OK) { + g_warning ("execute sql failed: sql=%s error=%s", sql, errmsg); + sqlite3_free (errmsg); + return FALSE; + } + g_assert_null (errmsg); + return TRUE; +} + +/** + * copy all data from src db to dest db. + * + * it's a wrapper for sqlite3_backup_init(), sqlite3_backup_step(), + * sqlite3_backup_finish(). + * + * TODO probably should add glib based error handling instead of print via + * g_warning and return gboolean for real world use. + * + * @dest: dest db handler + * @dest_dbname: dest db name + * @src: src db handler + * @src_dbname: src db name + */ +gboolean +sqlite3_copy_db (sqlite3 *dest, const char* dest_dbname, + sqlite3 *src, const char* src_dbname) +{ + gboolean copy_done = FALSE; + sqlite3_backup *backup = sqlite3_backup_init ( + dest, dest_dbname, src, src_dbname); + if (backup) { + int r = sqlite3_backup_step (backup, -1); + if (r == SQLITE_DONE) { + copy_done = TRUE; + } else { + g_warning ("sqlite3_backup_step() failed: %d (%s)", + r, sqlite3_errmsg (dest)); + } + r = sqlite3_backup_finish (backup); + if (r != SQLITE_OK) { + g_warning ("sqlite3_backup_finish() failed: %d (%s)", + r, sqlite3_errmsg (dest)); + } + } else { + g_warning ("sqlite3_backup_init() failed: %d (%s)", + sqlite3_errcode (dest), sqlite3_errmsg (dest)); + } + return copy_done; +} diff --git a/src/sqlite3_util.h b/src/sqlite3_util.h new file mode 100644 index 0000000..49fc386 --- /dev/null +++ b/src/sqlite3_util.h @@ -0,0 +1,30 @@ +#ifndef _SQLITE3_UTIL_H_ +#define _SQLITE3_UTIL_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + +gboolean sqlite3_exec_simple (sqlite3 *db, const char *sql); +/** + * copy src_dbname to dest_dbname using sqlite3_backup_step(). + * + * dest and src should be opened sqlite3 db handler. + * dest_dbname and src_dbname are db (schema) names. + * + * Returns: TRUE on success, FALSE otherwise. + */ +gboolean sqlite3_copy_db (sqlite3 *dest, const char* dest_dbname, + sqlite3 *src, const char* src_dbname); + + +#ifdef __cplusplus +} +#endif + +#endif /* _SQLITE3_UTIL_H_ */ -- GitLab