From 323202ab75d658bbd68e5b5c566de922d317b6ae Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 21 Dec 2021 22:48:27 +0800 Subject: [PATCH 01/19] Add relational DB properties Signed-off-by: lianhuix --- .../relational_store_delegate_impl.h | 2 + .../relational/relational_store_instance.h | 10 ++--- .../relational/relational_store_manager.cpp | 18 ++++---- .../storage/include/kvdb_properties.h | 42 ++++++++++++++----- .../include/relational_db_sync_interface.h | 2 + .../storage/src/irelational_store.h | 2 +- .../storage/src/kvdb_properties.cpp | 38 +++++++++++++---- .../storage/src/relational_store_instance.cpp | 18 ++++---- .../relational/sqlite_relational_store.cpp | 7 ++-- .../relational/sqlite_relational_store.h | 6 +-- .../sqlite_relational_store_connection.cpp | 1 + .../sqlite_relational_store_connection.h | 2 +- ...qlite_single_relational_storage_engine.cpp | 10 ++++- ...ingle_ver_relational_storage_executor.cpp} | 10 +++++ 14 files changed, 117 insertions(+), 51 deletions(-) rename services/distributeddataservice/libs/distributeddb/storage/src/sqlite/{sqlite_single_ver_relation_storage_executor.cpp => sqlite_single_ver_relational_storage_executor.cpp} (99%) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 6dc5404c2..25d51d440 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -33,12 +33,14 @@ public: DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) override; + DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, const Query &query, bool wait) override; DBStatus RemoveDeviceData(const std::string &device) override; DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) override; + DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) override; // For connection diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h index 34316ef82..1bed16fe0 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h @@ -28,19 +28,19 @@ public: RelationalStoreInstance(); ~RelationalStoreInstance() = default; - static RelationalStoreConnection *GetDatabaseConnection(const DBProperties &properties, int &errCode); + static RelationalStoreConnection *GetDatabaseConnection(const RelationalDBProperties &properties, int &errCode); static RelationalStoreInstance *GetInstance(); int CheckDatabaseFileStatus(const std::string &id); // public for test mock - static IRelationalStore *GetDataBase(const DBProperties &properties, int &errCode); + static IRelationalStore *GetDataBase(const RelationalDBProperties &properties, int &errCode); private: - IRelationalStore *OpenDatabase(const DBProperties &properties, int &errCode); + IRelationalStore *OpenDatabase(const RelationalDBProperties &properties, int &errCode); - void RemoveKvDBFromCache(const DBProperties &properties); - void SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties); + void RemoveKvDBFromCache(const RelationalDBProperties &properties); + void SaveKvDBToCache(IRelationalStore *store, const RelationalDBProperties &properties); static RelationalStoreInstance *instance_; static std::mutex instanceLock_; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 9a4a60324..47bd5eb62 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -33,22 +33,22 @@ RelationalStoreManager::RelationalStoreManager(const std::string &appId, const s {} static void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, - const std::string &appId, const std::string &userId, DBProperties &properties) + const std::string &appId, const std::string &userId, RelationalDBProperties &properties) { - properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); - properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); - properties.SetStringProp(KvDBProperties::APP_ID, appId); - properties.SetStringProp(KvDBProperties::USER_ID, userId); - properties.SetStringProp(KvDBProperties::STORE_ID, storePath); // same as dir + properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); + properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); + properties.SetStringProp(RelationalDBProperties::APP_ID, appId); + properties.SetStringProp(RelationalDBProperties::USER_ID, userId); + properties.SetStringProp(RelationalDBProperties::STORE_ID, storePath); // same as dir std::string identifier = userId + "-" + appId + "-" + storePath; std::string hashIdentifier = DBCommon::TransferHashString(identifier); - properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); + properties.SetStringProp(RelationalDBProperties::IDENTIFIER_DATA, hashIdentifier); } static const int GET_CONNECT_RETRY = 3; static const int RETRY_GET_CONN_INTER = 30; -static RelationalStoreConnection *GetOneConnectionWithRetry(const DBProperties &properties, int &errCode) +static RelationalStoreConnection *GetOneConnectionWithRetry(const RelationalDBProperties &properties, int &errCode) { for (int i = 0; i < GET_CONNECT_RETRY; i++) { auto conn = RelationalStoreInstance::GetDatabaseConnection(properties, errCode); @@ -77,7 +77,7 @@ void RelationalStoreManager::OpenStore(const std::string &path, const Relational return; } - DBProperties properties; + RelationalDBProperties properties; InitStoreProp(option, path, appId_, userId_, properties); int errCode = E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h index 6337094af..7250af7a5 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h @@ -21,12 +21,13 @@ #include #include "schema_object.h" +#include "relational_schema_object.h" namespace DistributedDB { class DBProperties { public: DBProperties() = default; - ~DBProperties() = default; + virtual ~DBProperties() = default; // Get the string property according the name std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; @@ -45,6 +46,16 @@ public: // Set the integer property for the name void SetIntProp(const std::string &name, int value); + + static const std::string CREATE_IF_NECESSARY; + static const std::string DATABASE_TYPE; + static const std::string DATA_DIR; + static const std::string USER_ID; + static const std::string APP_ID; + static const std::string STORE_ID; + static const std::string IDENTIFIER_DATA; + static const std::string IDENTIFIER_DIR; + protected: std::map stringProperties_; std::map boolProperties_; @@ -54,7 +65,7 @@ protected: class KvDBProperties final : public DBProperties { public: KvDBProperties(); - ~KvDBProperties(); + ~KvDBProperties() override; // Get the sub directory for different type database. static std::string GetStoreSubDirectory(int type); @@ -83,18 +94,10 @@ public: // The upper code will not change the schema if it is already set const SchemaObject &GetSchemaConstRef() const; - static const std::string CREATE_IF_NECESSARY; - static const std::string DATABASE_TYPE; - static const std::string DATA_DIR; - static const std::string USER_ID; - static const std::string APP_ID; - static const std::string STORE_ID; static const std::string FILE_NAME; static const std::string SYNC_MODE; static const std::string MEMORY_MODE; static const std::string ENCRYPTED_MODE; - static const std::string IDENTIFIER_DATA; - static const std::string IDENTIFIER_DIR; static const std::string FIRST_OPEN_IS_READ_ONLY; static const std::string CREATE_DIR_BY_STORE_ID_ONLY; static const std::string SECURITY_LABEL; @@ -114,6 +117,25 @@ private: CipherPassword password_; SchemaObject schema_; }; + +// TODO: move to its own file, or rename this file +class RelationalDBProperties final : public DBProperties { +public: + RelationalDBProperties(); + ~RelationalDBProperties() override; + + // is schema exist + bool IsSchemaExist() const; + + // set schema + void SetSchema(const RelationalSchemaObject &schema); + + // get schema + RelationalSchemaObject GetSchema() const; + +private: + RelationalSchemaObject schema_; +}; } // namespace DistributedDB #endif // KV_DB_PROPERTIES_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h index 94749dba3..eff6dbe54 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h @@ -38,6 +38,8 @@ public: virtual int LocalDataChanged(int notifyEvent, std::vector &queryObj) = 0; virtual int SchemaChanged(int notifyEvent) = 0; + + // TODO: create device table for each distributed table. }; } #endif // RELATIONAL_STORE diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h index 2d8cf45e0..5ff966e9e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h @@ -31,7 +31,7 @@ public: DISABLE_COPY_ASSIGN_MOVE(IRelationalStore); // Open the database. - virtual int Open(const DBProperties &kvDBProp) = 0; + virtual int Open(const RelationalDBProperties &kvDBProp) = 0; virtual void WakeUpSyncer() = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp index ee95611ea..75e98ded8 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp @@ -18,17 +18,18 @@ #include "db_constant.h" namespace DistributedDB { -const std::string KvDBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; -const std::string KvDBProperties::DATABASE_TYPE = "databaseType"; -const std::string KvDBProperties::DATA_DIR = "dataDir"; -const std::string KvDBProperties::USER_ID = "userId"; -const std::string KvDBProperties::APP_ID = "appId"; -const std::string KvDBProperties::STORE_ID = "storeId"; +const std::string DBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; +const std::string DBProperties::DATABASE_TYPE = "databaseType"; +const std::string DBProperties::DATA_DIR = "dataDir"; +const std::string DBProperties::USER_ID = "userId"; +const std::string DBProperties::APP_ID = "appId"; +const std::string DBProperties::STORE_ID = "storeId"; +const std::string DBProperties::IDENTIFIER_DATA = "identifier"; +const std::string DBProperties::IDENTIFIER_DIR = "identifierDir"; + const std::string KvDBProperties::FILE_NAME = "fileName"; const std::string KvDBProperties::MEMORY_MODE = "memoryMode"; const std::string KvDBProperties::ENCRYPTED_MODE = "isEncryptedDb"; -const std::string KvDBProperties::IDENTIFIER_DATA = "identifier"; -const std::string KvDBProperties::IDENTIFIER_DIR = "identifierDir"; const std::string KvDBProperties::FIRST_OPEN_IS_READ_ONLY = "firstOpenIsReadOnly"; const std::string KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY = "createDirByStoreIdOnly"; const std::string KvDBProperties::SECURITY_LABEL = "securityLabel"; @@ -145,4 +146,25 @@ const SchemaObject &KvDBProperties::GetSchemaConstRef() const { return schema_; } + +RelationalDBProperties::RelationalDBProperties() +{} + +RelationalDBProperties::~RelationalDBProperties() +{} + +bool RelationalDBProperties::IsSchemaExist() const +{ + return schema_.IsSchemaValid(); +} + +void RelationalDBProperties::SetSchema(const RelationalSchemaObject &schema) +{ + schema_ = schema; +} + +RelationalSchemaObject RelationalDBProperties::GetSchema() const +{ + return schema_; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp index e48bc8230..9a304762e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -54,10 +54,10 @@ int RelationalStoreInstance::CheckDatabaseFileStatus(const std::string &id) return E_OK; } -static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCode) +static IRelationalStore *GetFromCache(const RelationalDBProperties &properties, int &errCode) { errCode = E_OK; - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); auto iter = dbs_.find(identifier); if (iter == dbs_.end()) { errCode = -E_NOT_FOUND; @@ -76,21 +76,21 @@ static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCo } // Save to IKvDB to the global map -void RelationalStoreInstance::RemoveKvDBFromCache(const DBProperties &properties) +void RelationalStoreInstance::RemoveKvDBFromCache(const RelationalDBProperties &properties) { std::lock_guard lockGuard(storeLock_); - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); dbs_.erase(identifier); } -void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties) +void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const RelationalDBProperties &properties) { if (store == nullptr) { return; } { - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); store->WakeUpSyncer(); if (dbs_.count(identifier) == 0) { dbs_.insert(std::pair(identifier, store)); @@ -98,7 +98,7 @@ void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBP } } -IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &properties, int &errCode) +IRelationalStore *RelationalStoreInstance::OpenDatabase(const RelationalDBProperties &properties, int &errCode) { auto db = new (std::nothrow) SQLiteRelationalStore(); if (db == nullptr) { @@ -122,7 +122,7 @@ IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &prop return db; } -IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &properties, int &errCode) +IRelationalStore *RelationalStoreInstance::GetDataBase(const RelationalDBProperties &properties, int &errCode) { std::lock_guard lockGuard(storeLock_); auto *db = GetFromCache(properties, errCode); @@ -145,7 +145,7 @@ IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &prope return db; } -RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const DBProperties &properties, int &errCode) +RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const RelationalDBProperties &properties, int &errCode) { IRelationalStore *db = GetDataBase(properties, errCode); if (db == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index ed7bdb058..dfaf56757 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -26,7 +26,6 @@ SQLiteRelationalStore::~SQLiteRelationalStore() delete sqliteStorageEngine_; } - // Called when a new connection created. void SQLiteRelationalStore::IncreaseConnectionCounter() { @@ -50,13 +49,13 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) return connection; } -static void InitDataBaseOption(const DBProperties &kvDBProp, OpenDbProperties &option) +static void InitDataBaseOption(const RelationalDBProperties &kvDBProp, OpenDbProperties &option) { option.uri = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); option.createIfNecessary = kvDBProp.GetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, false); } -int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) +int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBProp) { OpenDbProperties option; InitDataBaseOption(kvDBProp, option); @@ -69,7 +68,7 @@ int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) return errCode; } -int SQLiteRelationalStore::Open(const DBProperties &properties) +int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) { sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); if (sqliteStorageEngine_ == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index f41dd0839..0a6f2ab68 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -33,7 +33,7 @@ public: ~SQLiteRelationalStore() override; RelationalStoreConnection *GetDBConnection(int &errCode) override; - int Open(const DBProperties &properties) override; + int Open(const RelationalDBProperties &properties) override; void OnClose(const std::function ¬ifier); SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode) const; @@ -43,7 +43,7 @@ public: void ReleaseDBConnection(RelationalStoreConnection *connection); - void WakeUpSyncer(); + void WakeUpSyncer() override; // for test mock const RelationalSyncAbleStorage *GetStorageEngine() @@ -61,7 +61,7 @@ private: SQLiteSingleRelationalStorageEngine *sqliteStorageEngine_ = nullptr; void IncreaseConnectionCounter(); - int InitStorageEngine(const DBProperties &kvDBProp); + int InitStorageEngine(const RelationalDBProperties &kvDBProp); std::mutex connectMutex_; std::atomic connectionCount_ = 0; std::vector> closeNotifiers_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index d322575d7..a66269b12 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -182,6 +182,7 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) syncParam.relationOnComplete = info.onComplete; syncParam.syncQuery = QuerySyncObject(info.query); syncParam.onFinalize = [this]() { DecObjRef(this); }; + // TODO: check if table permit sync or not int errCode = store->Sync(syncParam); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 42184112f..020560850 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -42,7 +42,7 @@ public: protected: - int Pragma(int cmd, void *parameter); + int Pragma(int cmd, void *parameter) override; private: SQLiteSingleVerRelationalStorageExecutor *GetExecutor(bool isWrite, int &errCode) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index ab12fd5d8..a47129d7c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -57,7 +57,7 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage return errCode; } do { - errCode = Upgrade(db); + errCode = Upgrade(db); // cerate meta_data table. if (errCode != E_OK) { break; } @@ -66,6 +66,14 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage if (errCode != E_OK) { break; } + + // TODO: Get and parse relational schema from meta table + + // TODO: save log table version into meta data + + // TODO: clean the device table + + handle = NewSQLiteStorageExecutor(db, isWrite, false); if (handle == nullptr) { LOGE("[Relational] New SQLiteStorageExecutor[%d] for the pool failed.", isWrite); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp similarity index 99% rename from services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp rename to services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 167adfdb2..be162f3b0 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -36,6 +36,16 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: return errCode; } + // TODO: check if compatible upgrade or not + + // TODO: add not permit sync flag if not compatible upgrade + + // TODO: add analysed table to schema. + + // TODO: reset permit sync flag + + // TODO: upgrade device table + // create log table errCode = SQLiteUtils::CreateRelationalLogTable(dbHandle_, tableName); if (errCode != E_OK) { -- Gitee From 04db30d21358d0bbe63ec1f7cf3ed176b1783572 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 22 Dec 2021 11:45:51 +0800 Subject: [PATCH 02/19] Add relationa build script Signed-off-by: lianhuix --- .../libs/distributeddb/BUILD.gn | 21 +++++++++++++++++++ .../libs/distributeddb/test/BUILD.gn | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index c763dcedc..05ae98eac 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -18,13 +18,16 @@ config("distrdb_config") { "include", "interfaces/include", "interfaces/src", + "interfaces/src/relational", "common/include", + "common/include/relational", "communicator/include", "storage/include", "storage/src", "storage/src/multiver", "storage/src/operation", "storage/src/sqlite", + "storage/src/sqlite/relational", "storage/src/upgrader", "syncer/include", "syncer/src", @@ -41,6 +44,7 @@ config("distrdb_config") { "USING_DB_JSON_EXTRACT_AUTOMATICALLY", "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", + "RELATIONAL_STORE", ] } @@ -48,6 +52,7 @@ config("distrdb_public_config") { visibility = [ "*:*" ] include_dirs = [ "interfaces/include", + "interfaces/include/relational", "include", ] } @@ -82,6 +87,7 @@ ohos_shared_library("distributeddb") { "common/src/query.cpp", "common/src/query_expression.cpp", "common/src/ref_object.cpp", + "common/src/relational/relational_schema_object.cpp", "common/src/runtime_context.cpp", "common/src/runtime_context_impl.cpp", "common/src/schema_object.cpp", @@ -115,6 +121,12 @@ ohos_shared_library("distributeddb") { "interfaces/src/kv_store_nb_delegate_impl.cpp", "interfaces/src/kv_store_result_set_impl.cpp", "interfaces/src/kv_store_snapshot_delegate_impl.cpp", + "interfaces/src/relational/data_value.cpp", + "interfaces/src/relational/relational_store_delegate_impl.cpp", + "interfaces/src/relational/relational_store_manager.cpp", + "interfaces/src/relational/relational_store_sqlite_ext.cpp", + "interfaces/src/relational/runtime_config.cpp", + "storage/src/data_transformer.cpp", "storage/src/default_factory.cpp", "storage/src/generic_kvdb.cpp", "storage/src/generic_kvdb_connection.cpp", @@ -145,10 +157,16 @@ ohos_shared_library("distributeddb") { "storage/src/operation/multi_ver_database_oper.cpp", "storage/src/operation/single_ver_database_oper.cpp", "storage/src/package_file.cpp", + "storage/src/relational_store_connection.cpp", + "storage/src/relational_store_instance.cpp", + "storage/src/relational_sync_able_storage.cpp", "storage/src/result_entries_window.cpp", "storage/src/single_ver_natural_store_commit_notify_data.cpp", "storage/src/sqlite/query_object.cpp", "storage/src/sqlite/query_sync_object.cpp", + "storage/src/sqlite/relational/sqlite_relational_store.cpp", + "storage/src/sqlite/relational/sqlite_relational_store_connection.cpp", + "storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp", "storage/src/sqlite/sqlite_local_kvdb.cpp", "storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -162,6 +180,7 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "storage/src/sqlite/sqlite_single_ver_result_set.cpp", "storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", "storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", @@ -174,6 +193,7 @@ ohos_shared_library("distributeddb") { "storage/src/storage_engine.cpp", "storage/src/storage_engine_manager.cpp", "storage/src/storage_executor.cpp", + "storage/src/sync_able_engine.cpp", "storage/src/sync_able_kvdb.cpp", "storage/src/sync_able_kvdb_connection.cpp", "storage/src/upgrader/single_ver_database_upgrader.cpp", @@ -194,6 +214,7 @@ ohos_shared_library("distributeddb") { "syncer/src/single_ver_data_sync.cpp", "syncer/src/single_ver_data_sync_with_sliding_window.cpp", "syncer/src/single_ver_kv_syncer.cpp", + "syncer/src/single_ver_relational_syncer.cpp", "syncer/src/single_ver_serialize_manager.cpp", "syncer/src/single_ver_sync_engine.cpp", "syncer/src/single_ver_sync_state_machine.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 7b98fee3f..c181d617b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -25,14 +25,18 @@ config("module_private_config") { "./unittest/common/interfaces", "../include", "../interfaces/include", + "../interfaces/include/relational", "../interfaces/src", + "../interfaces/src/relational", "../storage/include", "../storage/src", "../storage/src/multiver", "../storage/src/operation", "../storage/src/sqlite", + "../storage/src/sqlite/relational", "../storage/src/upgrader", "../common/include", + "../common/include/relational", "../common/src", "../communicator/include", "../communicator/src", @@ -53,6 +57,7 @@ config("module_private_config") { "LOW_LEVEL_MEM_DEV", "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", + "RELATIONAL_STORE", ] } @@ -85,6 +90,7 @@ ohos_source_set("src_file") { "../common/src/query.cpp", "../common/src/query_expression.cpp", "../common/src/ref_object.cpp", + "../common/src/relational/relational_schema_object.cpp", "../common/src/runtime_context.cpp", "../common/src/runtime_context_impl.cpp", "../common/src/schema_object.cpp", @@ -118,6 +124,12 @@ ohos_source_set("src_file") { "../interfaces/src/kv_store_nb_delegate_impl.cpp", "../interfaces/src/kv_store_result_set_impl.cpp", "../interfaces/src/kv_store_snapshot_delegate_impl.cpp", + "../interfaces/src/relational/data_value.cpp", + "../interfaces/src/relational/relational_store_delegate_impl.cpp", + "../interfaces/src/relational/relational_store_manager.cpp", + "../interfaces/src/relational/relational_store_sqlite_ext.cpp", + "../interfaces/src/relational/runtime_config.cpp", + "../storage/src/data_transformer.cpp", "../storage/src/default_factory.cpp", "../storage/src/generic_kvdb.cpp", "../storage/src/generic_kvdb_connection.cpp", @@ -148,10 +160,16 @@ ohos_source_set("src_file") { "../storage/src/operation/multi_ver_database_oper.cpp", "../storage/src/operation/single_ver_database_oper.cpp", "../storage/src/package_file.cpp", + "../storage/src/relational_store_connection.cpp", + "../storage/src/relational_store_instance.cpp", + "../storage/src/relational_sync_able_storage.cpp", "../storage/src/result_entries_window.cpp", "../storage/src/single_ver_natural_store_commit_notify_data.cpp", "../storage/src/sqlite/query_object.cpp", "../storage/src/sqlite/query_sync_object.cpp", + "../storage/src/sqlite/relational/sqlite_relational_store.cpp", + "../storage/src/sqlite/relational/sqlite_relational_store_connection.cpp", + "../storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp", "../storage/src/sqlite/sqlite_local_kvdb.cpp", "../storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "../storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -165,6 +183,7 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "../storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "../storage/src/sqlite/sqlite_single_ver_result_set.cpp", "../storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", "../storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", @@ -177,6 +196,7 @@ ohos_source_set("src_file") { "../storage/src/storage_engine.cpp", "../storage/src/storage_engine_manager.cpp", "../storage/src/storage_executor.cpp", + "../storage/src/sync_able_engine.cpp", "../storage/src/sync_able_kvdb.cpp", "../storage/src/sync_able_kvdb_connection.cpp", "../storage/src/upgrader/single_ver_database_upgrader.cpp", @@ -197,6 +217,7 @@ ohos_source_set("src_file") { "../syncer/src/single_ver_data_sync.cpp", "../syncer/src/single_ver_data_sync_with_sliding_window.cpp", "../syncer/src/single_ver_kv_syncer.cpp", + "../syncer/src/single_ver_relational_syncer.cpp", "../syncer/src/single_ver_serialize_manager.cpp", "../syncer/src/single_ver_sync_engine.cpp", "../syncer/src/single_ver_sync_state_machine.cpp", -- Gitee From 557de450c6a1edba2800e201f685a2e73966deaa Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 22 Dec 2021 15:12:19 +0800 Subject: [PATCH 03/19] remove callback in open relational store API Signed-off-by: lianhuix --- .../relational/relational_store_manager.h | 14 +++--- .../relational/relational_store_manager.cpp | 48 ++++++++++--------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 17f669b00..459196bdf 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -36,17 +36,17 @@ public: RelationalStoreManager &operator=(const RelationalStoreManager &) = delete; RelationalStoreManager &operator=(RelationalStoreManager &&) = delete; - // PRAGMA journal_mode=WAL - // PRAGMA synchronous=FULL - // PRAGMA synchronous=NORMAL - DB_API void OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, - const std::function &callback); + DB_API DBStatus OpenStore(const std::string &path, const std::string &storeId, + const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate); - DB_API DBStatus CloseStore(RelationalStoreDelegate *store); + DB_API DBStatus CloseStore(RelationalStoreDelegate *store); // TODO: move interface to delegate - DB_API DBStatus DeleteStore(const std::string &path); + DB_API DBStatus DeleteStore(const std::string &path); // TODO: remove interface private: + void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, + const std::string &storeId, RelationalDBProperties &properties); + std::string appId_; std::string userId_; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 47bd5eb62..cb4fabae3 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -32,15 +32,15 @@ RelationalStoreManager::RelationalStoreManager(const std::string &appId, const s userId_(userId) {} -static void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, - const std::string &appId, const std::string &userId, RelationalDBProperties &properties) +void RelationalStoreManager::InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, + const std::string &storeId, RelationalDBProperties &properties) { properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); - properties.SetStringProp(RelationalDBProperties::APP_ID, appId); - properties.SetStringProp(RelationalDBProperties::USER_ID, userId); - properties.SetStringProp(RelationalDBProperties::STORE_ID, storePath); // same as dir - std::string identifier = userId + "-" + appId + "-" + storePath; + properties.SetStringProp(RelationalDBProperties::APP_ID, appId_); + properties.SetStringProp(RelationalDBProperties::USER_ID, userId_); + properties.SetStringProp(RelationalDBProperties::STORE_ID, storeId); // same as dir + std::string identifier = userId_ + "-" + appId_ + "-" + storeId; std::string hashIdentifier = DBCommon::TransferHashString(identifier); properties.SetStringProp(RelationalDBProperties::IDENTIFIER_DATA, hashIdentifier); } @@ -64,39 +64,41 @@ static RelationalStoreConnection *GetOneConnectionWithRetry(const RelationalDBPr return nullptr; } -void RelationalStoreManager::OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, - const std::function &callback) +DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const std::string &storeId, + const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate) { - if (!callback) { - LOGE("[KvStoreMgr] Invalid callback for kv store!"); - return; + if (delegate != nullptr) { + LOGE("[RelationalStoreMgr] Invalid delegate!"); + return INVALID_ARGS; } - if (!ParamCheckUtils::CheckStoreParameter("Relational_default_id", appId_, userId_) || path.empty()) { - callback(INVALID_ARGS, nullptr); - return; + std::string canonicalDir; + if (!ParamCheckUtils::CheckDataDir(path, canonicalDir)) { + return INVALID_ARGS; + } + + if (!ParamCheckUtils::CheckStoreParameter(storeId, appId_, userId_) || path.empty()) { + return INVALID_ARGS; } RelationalDBProperties properties; - InitStoreProp(option, path, appId_, userId_, properties); + InitStoreProp(option, canonicalDir, storeId, properties); int errCode = E_OK; auto *conn = GetOneConnectionWithRetry(properties, errCode); DBStatus status = TransferDBErrno(errCode); if (conn == nullptr) { - callback(status, nullptr); - return; + return status; } - auto store = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); - if (store == nullptr) { + delegate = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); + if (delegate == nullptr) { conn->Close(); - callback(DB_ERROR, nullptr); - return; + return DB_ERROR; } - (void)conn->TriggerAutoSync(); - callback(OK, store); + (void)conn->TriggerAutoSync(); // TODO: no auto sync + return OK; } DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *store) -- Gitee From a4931b16b5f5adc2efe1362fb9789c8aaf3afd55 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 23 Dec 2021 09:39:26 +0800 Subject: [PATCH 04/19] Add read relational schema from meta Signed-off-by: lianhuix --- .../distributeddb/interfaces/include/types.h | 2 + .../storage/src/relational_store_instance.cpp | 14 ++-- .../relational/sqlite_relational_store.cpp | 73 ++++++++++++++++--- .../relational/sqlite_relational_store.h | 4 + ...qlite_single_relational_storage_engine.cpp | 7 -- 5 files changed, 77 insertions(+), 23 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h index fb12ab170..dca53003d 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h @@ -56,6 +56,8 @@ enum DBStatus { SECURITY_OPTION_CHECK_ERROR, // such as remote device's SecurityOption not equal to local SCHEMA_VIOLATE_VALUE, // Values already exist in dbFile do not match new schema INTERCEPT_DATA_FAIL, // Interceptor push data failed. + RELATIONAL_SCHEMA_NOT_FOUND, // the sync table is not a relational table + RELATIONAL_SCHEMA_CHANGED, // the schema was changed }; struct KvStoreConfig { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp index 9a304762e..366d78863 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -18,8 +18,9 @@ #include #include -#include "sqlite_relational_store.h" +#include "db_common.h" #include "db_errno.h" +#include "sqlite_relational_store.h" #include "log_print.h" namespace DistributedDB { @@ -65,7 +66,6 @@ static IRelationalStore *GetFromCache(const RelationalDBProperties &properties, } auto *db = iter->second; - if (db == nullptr) { LOGE("Store cache is nullptr, there may be a logic error"); errCode = -E_INTERNAL_ERROR; @@ -102,7 +102,7 @@ IRelationalStore *RelationalStoreInstance::OpenDatabase(const RelationalDBProper { auto db = new (std::nothrow) SQLiteRelationalStore(); if (db == nullptr) { - LOGE("Failed to get IKvDB! err:%d", errCode); + LOGE("Failed to get relational store! err:%d", errCode); return nullptr; } @@ -145,8 +145,11 @@ IRelationalStore *RelationalStoreInstance::GetDataBase(const RelationalDBPropert return db; } -RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const RelationalDBProperties &properties, int &errCode) +RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const RelationalDBProperties &properties, + int &errCode) { + std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + LOGD("Begin to get [%s] database connection.", STR_MASK(DBCommon::TransferStringToHex(identifier))); IRelationalStore *db = GetDataBase(properties, errCode); if (db == nullptr) { LOGE("Failed to open the db:%d", errCode); @@ -157,9 +160,8 @@ RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const if (connection == nullptr) { // not kill db, Other operations like import may be used concurrently LOGE("Failed to get the db connect for delegate:%d", errCode); } - RefObject::DecObjRef(db); // restore the reference increased by the cache. - // kvDB = nullptr; + RefObject::DecObjRef(db); // restore the reference increased by the cache. return connection; } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index dfaf56757..051369c0e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -15,12 +15,17 @@ #ifdef RELATIONAL_STORE #include "sqlite_relational_store.h" +#include "db_common.h" #include "db_errno.h" #include "log_print.h" #include "db_types.h" #include "sqlite_relational_store_connection.h" namespace DistributedDB { +namespace { + constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; +} + SQLiteRelationalStore::~SQLiteRelationalStore() { delete sqliteStorageEngine_; @@ -39,7 +44,6 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) { std::lock_guard lock(connectMutex_); RelationalStoreConnection* connection = new (std::nothrow) SQLiteRelationalStoreConnection(this); - if (connection == nullptr) { errCode = -E_OUT_OF_MEMORY; return nullptr; @@ -68,24 +72,73 @@ int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBP return errCode; } -int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) +int SQLiteRelationalStore::GetSchemaFromMeta() { - sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); - if (sqliteStorageEngine_ == nullptr) { - LOGE("[RelationalStore] Create storage engine failed"); - return -E_OUT_OF_MEMORY; + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + int errCode = storageEngine_->GetMetaData(schemaKey, schemaVal); + if (errCode != E_OK && errCode != -E_NOT_FOUND ) { + LOGE("Get relationale schema from meta table failed. %d", errCode); + return errCode; + } else if (errCode == -E_NOT_FOUND) { + LOGW("No relational schema info was found."); + return E_OK; } - int errCode = InitStorageEngine(properties); + std::string schemaStr; + DBCommon::VectorToString(schemaVal, schemaStr); + RelationalSchemaObject schema; + errCode = schema.ParseFromSchemaString(schemaStr); if (errCode != E_OK) { - LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + LOGE("Parse schema string from mata table failed."); return errCode; } - storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); - syncEngine_ = std::make_shared(storageEngine_); + + properties_.SetSchema(schema); return E_OK; } +int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) +{ + sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); + if (sqliteStorageEngine_ == nullptr) { + LOGE("[RelationalStore][Open] Create storage engine failed"); + return -E_OUT_OF_MEMORY; + } + + int errCode = E_OK; + + do { + errCode = InitStorageEngine(properties); + if (errCode != E_OK) { + LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + break; + } + storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); + if (storageEngine_ == nullptr) { + LOGE("[RelationalStore][Open] Create syncable storage failed"); // TODO: + errCode = -E_OUT_OF_MEMORY; + break; + } + + properties_ = properties; + errCode = GetSchemaFromMeta(); + if (errCode != E_OK) { + break; + } + + // TODO: save log table version into meta data + + // TODO: clean the device table + + syncEngine_ = std::make_shared(storageEngine_); + return E_OK; + } while (false); + + // TODO: release resources. + return errCode; +} + void SQLiteRelationalStore::OnClose(const std::function ¬ifier) { AutoLock lockGuard(this); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 0a6f2ab68..9c06e28bf 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -54,6 +54,8 @@ private: // 1 store 1 connection void DecreaseConnectionCounter(); + int GetSchemaFromMeta(); + // use for sync Interactive std::shared_ptr syncEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv @@ -65,6 +67,8 @@ private: std::mutex connectMutex_; std::atomic connectionCount_ = 0; std::vector> closeNotifiers_; + + RelationalDBProperties properties_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index a47129d7c..6c9a5390f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -67,13 +67,6 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage break; } - // TODO: Get and parse relational schema from meta table - - // TODO: save log table version into meta data - - // TODO: clean the device table - - handle = NewSQLiteStorageExecutor(db, isWrite, false); if (handle == nullptr) { LOGE("[Relational] New SQLiteStorageExecutor[%d] for the pool failed.", isWrite); -- Gitee From 7630702b7e8d17d12b4d47af4b62bba4ebf94785 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 23 Dec 2021 10:29:12 +0800 Subject: [PATCH 05/19] Add save table version and clean device table. No Implement, Only Framework. Signed-off-by: lianhuix --- .../relational/sqlite_relational_store.cpp | 49 ++++++++++++++++++- .../relational/sqlite_relational_store.h | 6 +++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 051369c0e..83b3d7979 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -20,10 +20,13 @@ #include "log_print.h" #include "db_types.h" #include "sqlite_relational_store_connection.h" +#include "storage_engine_manager.h" namespace DistributedDB { namespace { constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; + constexpr const char *LOG_TABLE_VERSION_KEY = "log_table_versoin"; + constexpr const char *LOG_TABLE_VERSION_1 = "1.0"; } SQLiteRelationalStore::~SQLiteRelationalStore() @@ -72,6 +75,15 @@ int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBP return errCode; } +void SQLiteRelationalStore::ReleaseResources() +{ + // TODO: Lock + if (sqliteStorageEngine_ != nullptr) { + sqliteStorageEngine_->ClearEnginePasswd(); + (void)StorageEngineManager::ReleaseStorageEngine(sqliteStorageEngine_); + } +} + int SQLiteRelationalStore::GetSchemaFromMeta() { const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); @@ -98,6 +110,32 @@ int SQLiteRelationalStore::GetSchemaFromMeta() return E_OK; } +int SQLiteRelationalStore::SaveLogTableVersionToMeta() +{ + // TODO: save log table version into meta data + LOGD("save log table version to meta table, key: %s, val: %s", LOG_TABLE_VERSION_KEY, LOG_TABLE_VERSION_1); + return E_OK; +} + +int SQLiteRelationalStore::CleanDistributedDeviceTable() +{ + // TODO: clean the device table which is no longer in schema + RelationalSchemaObject schema = properties_.GetSchema(); + for (const auto &table : schema.GetTables()) { + std::string tableName = table.first; + LOGD("Get schema %s.", tableName.c_str()); + } + + int errCode = E_OK; + auto *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + // TODO: Get device table names, and clean + ReleaseHandle(handle); + return E_OK; +} + int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) { sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); @@ -127,15 +165,22 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } - // TODO: save log table version into meta data + errCode = SaveLogTableVersionToMeta(); + if (errCode != E_OK) { + break; + } - // TODO: clean the device table + errCode = CleanDistributedDeviceTable(); + if (errCode != E_OK) { + break; + } syncEngine_ = std::make_shared(storageEngine_); return E_OK; } while (false); // TODO: release resources. + ReleaseResources(); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 9c06e28bf..e927e2aa2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -51,11 +51,17 @@ public: return storageEngine_; } private: + void ReleaseResources(); + // 1 store 1 connection void DecreaseConnectionCounter(); int GetSchemaFromMeta(); + int SaveLogTableVersionToMeta(); + + int CleanDistributedDeviceTable(); + // use for sync Interactive std::shared_ptr syncEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv -- Gitee From e7315a1ad40bddc403c5b966f84710ffc92a9735 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 23 Dec 2021 15:08:10 +0800 Subject: [PATCH 06/19] Add create distributed table implement Signed-off-by: lianhuix --- .../common/include/db_constant.h | 2 ++ .../common/include/param_check_utils.h | 2 ++ .../distributeddb/common/src/db_constant.cpp | 2 ++ .../common/src/param_check_utils.cpp | 5 ++++ .../relational/relational_schema_object.cpp | 6 +++- .../relational/relational_store_delegate.h | 5 +--- .../relational_store_delegate_impl.cpp | 13 ++++++--- .../relational_store_delegate_impl.h | 2 +- .../include/relational_store_connection.h | 3 +- .../relational/sqlite_relational_store.cpp | 28 +++++++++++++++++++ .../relational/sqlite_relational_store.h | 2 ++ .../sqlite_relational_store_connection.cpp | 25 ++++++++--------- .../sqlite_relational_store_connection.h | 3 +- ...single_ver_relational_storage_executor.cpp | 8 +++--- ...e_single_ver_relational_storage_executor.h | 4 ++- 15 files changed, 77 insertions(+), 33 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 99a6944b9..722df4437 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -92,6 +92,8 @@ public: static const std::string UPDATE_META_FUNC; + static const std::string SYSTEM_TABLE_PREFIX; + static constexpr uint32_t AUTO_SYNC_TIMEOUT = 5000; // 5s static constexpr uint32_t MANUAL_SYNC_TIMEOUT = 5000; // 5s diff --git a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h index 0e4c0dbc8..ce07d65e1 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h @@ -49,6 +49,8 @@ public: SchemaObject &schemaObject, std::string &canonicalDir); static uint8_t GetValidCompressionRate(uint8_t compressionRate); + + static bool CheckRelationalTableName(const std::string &tableName); }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp index 410e01137..269b46436 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp @@ -63,4 +63,6 @@ const std::string DBConstant::TRIGGER_REFERENCES_NEW = "NEW."; const std::string DBConstant::TRIGGER_REFERENCES_OLD = "OLD."; const std::string DBConstant::UPDATE_META_FUNC = "update_meta_within_trigger"; + +const std::string DBConstant::SYSTEM_TABLE_PREFIX = "naturalbase_rdb_"; } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp b/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp index e313bd4b4..58ee11566 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp @@ -187,4 +187,9 @@ uint8_t ParamCheckUtils::GetValidCompressionRate(uint8_t compressionRate) } return compressionRate; } + +bool ParamCheckUtils::CheckRelationalTableName(const std::string &tableName) +{ + return tableName.compare(0, DBConstant::SYSTEM_TABLE_PREFIX.size(), DBConstant::SYSTEM_TABLE_PREFIX) != 0; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 2c1ecbf67..ef1578e66 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -425,7 +425,11 @@ const std::map &RelationalSchemaObject::GetTables() cons const TableInfo &RelationalSchemaObject::GetTable(const std::string& tableName) const { - return tables_.at(tableName); + auto it = tables_.find(tableName); + if (it != tables_.end()) { + return it->second; + } + return {}; } int RelationalSchemaObject::CompareAgainstSchemaObject(const std::string &inSchemaString, diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 3e2923b77..6dc424277 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -34,10 +34,7 @@ public: DB_API virtual DBStatus RemoveDeviceData(const std::string &device) = 0; - struct TableOption { - }; - - DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) = 0; + DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index fb4d08542..c9c1b842e 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -16,8 +16,9 @@ #include "relational_store_delegate_impl.h" #include "db_errno.h" -#include "log_print.h" #include "kv_store_errno.h" +#include "log_print.h" +#include "param_check_utils.h" #include "sync_operation.h" namespace DistributedDB { @@ -52,15 +53,19 @@ DBStatus RelationalStoreDelegateImpl::RemoveDeviceData(const std::string &device return NOT_SUPPORT; } -DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName, const TableOption &option) +DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName) { - // check table Name and option + if (!ParamCheckUtils::CheckRelationalTableName(tableName)) { + LOGE("invalid table name."); + return INVALID_ARGS; + } + if (conn_ == nullptr) { LOGE("[RelationalStore Delegate] Invalid connection for operation!"); return DB_ERROR; } - int errCode = conn_->CreateDistributedTable(tableName, option); + int errCode = conn_->CreateDistributedTable(tableName); if (errCode != E_OK) { LOGW("[RelationalStore Delegate] Create Distributed table failed:%d", errCode); return TransferDBErrno(errCode); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 25d51d440..25d7e6c96 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -39,7 +39,7 @@ public: DBStatus RemoveDeviceData(const std::string &device) override; - DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) override; + DBStatus CreateDistributedTable(const std::string &tableName) override; DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) override; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index e502357a7..35f2d91e6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -49,8 +49,7 @@ public: virtual int TriggerAutoSync() = 0; virtual int SyncToDevice(SyncInfo &info) = 0; virtual std::string GetIdentifier() = 0; - virtual int CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) = 0; + virtual int CreateDistributedTable(const std::string &tableName) = 0; protected: // Get the stashed 'KvDB_ pointer' without ref. diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 83b3d7979..51b45a926 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -276,5 +276,33 @@ void SQLiteRelationalStore::WakeUpSyncer() { syncEngine_->WakeUpSyncer(); } + + +int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) +{ + int errCode = E_OK; + auto schema = properties_.GetSchema(); + if (schema.GetTable(tableName).GetTableName() == tableName) { + LOGW("distributed table %s was already created.", tableName.c_str()); + return E_OK; + } + + auto *handle = GetHandle(true, errCode); + if (handle != nullptr) { + return errCode; + } + + TableInfo table; + errCode = handle->CreateDistributedTable(tableName, table); + if (errCode != E_OK) { + LOGE("create distributed table failed. %d", errCode); + } else { + schema.AddRelationalTable(table); + properties_.SetSchema(schema); + } + + ReleaseHandle(handle); + return E_OK; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index e927e2aa2..5358dc5d7 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -50,6 +50,8 @@ public: { return storageEngine_; } + + int CreateDistributedTable(const std::string &tableName); private: void ReleaseResources(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index a66269b12..bc86d438c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -58,7 +58,7 @@ SQLiteSingleVerRelationalStorageExecutor *SQLiteRelationalStoreConnection::GetEx auto *store = GetDB(); if (store == nullptr) { errCode = -E_NOT_INIT; - LOGE("[SingleVerConnection] store is null, get executor failed! errCode = [%d]", errCode); + LOGE("[RelationalConnection] store is null, get executor failed! errCode = [%d]", errCode); return nullptr; } @@ -93,7 +93,7 @@ int SQLiteRelationalStoreConnection::StartTransaction() return errCode; } - LOGD("[SingleVerConnection] Start transaction finish."); + LOGD("[RelationalConnection] Start transaction finish."); writeHandle_ = handle; transactingFlag_.store(true); return E_OK; @@ -135,20 +135,17 @@ int SQLiteRelationalStoreConnection::RollBack() return errCode; } -int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) +int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName) { - int errCode = StartTransaction(); - if (errCode != E_OK && errCode != E_TRANSACT_STATE) { - return errCode; + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null, get DB failed!"); + return -E_INVALID_CONNECTION; } - errCode = writeHandle_->CreateDistributedTable(tableName, option); - if (errCode == E_OK) { - errCode = Commit(); - } else { - int innerCode = RollBack(); - errCode = (innerCode != E_OK) ? innerCode : errCode; + int errCode = store->CreateDistributedTable(tableName); + if (errCode != E_OK) { + LOGE("[RelationalConnection] crete distributed table failed. %d", errCode); } return errCode; } @@ -167,7 +164,7 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) { auto *store = GetDB(); if (store == nullptr) { - LOGE("[SingleVerConnection] store is null, get executor failed!"); + LOGE("[RelationalConnection] store is null, get executor failed!"); return -E_INVALID_CONNECTION; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 020560850..63a533f5f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -37,8 +37,7 @@ public: int TriggerAutoSync() override; int SyncToDevice(SyncInfo &info) override; std::string GetIdentifier() override; - int CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) override; + int CreateDistributedTable(const std::string &tableName) override; protected: diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index be162f3b0..17bd91e9d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -19,17 +19,16 @@ namespace DistributedDB { SQLiteSingleVerRelationalStorageExecutor::SQLiteSingleVerRelationalStorageExecutor(sqlite3 *dbHandle, bool writable) - : SQLiteStorageExecutor(dbHandle, writable, false) {}; + : SQLiteStorageExecutor(dbHandle, writable, false) +{} -int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) +int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, TableInfo &table) { if (dbHandle_ == nullptr) { LOGE("Begin transaction failed, dbHandle is null."); return -E_INVALID_DB; } - TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); @@ -98,6 +97,7 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) { + // TODO: Get table info from schema int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index 301204f61..1ab02ab3b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -32,7 +32,7 @@ public: // Delete the copy and assign constructors DISABLE_COPY_ASSIGN_MOVE(SQLiteSingleVerRelationalStorageExecutor); - int CreateDistributedTable(const std::string &tableName, const RelationalStoreDelegate::TableOption &option); + int CreateDistributedTable(const std::string &tableName, TableInfo &table); int StartTransaction(TransactType type); int Commit(); @@ -57,6 +57,8 @@ public: int SaveSyncItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName, TimeStamp &timeStamp); + int AnalysisRelationalSchema(const std::string &tableName, TableInfo &tableInfo); + private: int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData) const; -- Gitee From a87bf3cc7bc2c7535aef5eed6c8e2deff1f0543b Mon Sep 17 00:00:00 2001 From: lianhuix Date: Fri, 24 Dec 2021 09:25:07 +0800 Subject: [PATCH 07/19] Add relational schema field attribute parse. Signed-off-by: lianhuix --- .../common/include/json_object.h | 2 + .../relational/relational_schema_object.h | 1 + .../distributeddb/common/src/json_object.cpp | 20 +++++ .../relational/relational_schema_object.cpp | 77 ++++++++++++------- .../relational/relational_store_delegate.h | 2 +- .../relational/relational_store_manager.cpp | 4 +- .../relational/sqlite_relational_store.cpp | 15 +++- .../relational/sqlite_relational_store.h | 3 + .../storage/src/sync_able_engine.cpp | 3 +- 9 files changed, 89 insertions(+), 38 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h index c9ecd26e6..0fcaf6478 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h @@ -65,6 +65,8 @@ public: int GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; int GetStringArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; + int GetObjectByFieldPath(const FieldPath &inPath, JsonObject &outArray) const; + // An empty fieldPath indicate the root, the outSubPath should be empty before call, we will not empty it at first. // If inPath is of multiple path, then outSubPath is combination of result of each inPath. int GetSubFieldPath(const FieldPath &inPath, std::set &outSubPath) const; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index 47e4f99aa..c91a1d711 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -153,6 +153,7 @@ private: int ParseCheckTableInfo(const JsonObject &inJsonObject); int ParseCheckTableName(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableDefine(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, FieldInfo &table); int ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableIndex(const JsonObject &inJsonObject, TableInfo &resultTable); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 37a008ddb..80313347a 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -741,6 +741,26 @@ int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const { if (!isValid_) { diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index ef1578e66..5aa861f8c 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -97,15 +97,13 @@ void FieldInfo::SetColumnId(int cid) // return field define string like ("fieldName": "MY INT(21), NOT NULL, DEFAULT 123") std::string FieldInfo::ToAttributeString() const { - std::string attrStr = "\"" + fieldName_ + "\": \""; - attrStr += dataType_; - if (isNotNull_) { - attrStr += ", NOT NULL"; - } + std::string attrStr = "\"" + fieldName_ + "\": {"; + attrStr += "\"TYPE\":\"" + dataType_ + "\","; + attrStr += "\"NOT_NULL\":" + std::string(isNotNull_ ? "true" : "false") + ","; if (hasDefaultValue_) { - attrStr += ", " + defaultValue_; + attrStr += "\"DEFAULT\":\"" + defaultValue_ + "\""; } - attrStr += + "\""; + attrStr += "}"; return attrStr; } @@ -448,8 +446,8 @@ namespace { int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &fieldName, FieldType expectType, bool isNecessary, FieldValue &fieldValue) { - if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName}) && !isNecessary) { - return E_OK; + if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { + return isNecessary ? -E_SCHEMA_PARSE_FAIL : -E_NOT_FOUND; } FieldType fieldType; @@ -600,46 +598,67 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject } for (const auto &field : tableFields) { - if (field.second != FieldType::LEAF_FIELD_STRING) { - LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType STRING but : %s.", + if (field.second != FieldType::LEAF_FIELD_OBJECT) { + LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType OBJECT but : %s.", SchemaUtils::FieldTypeString(field.second).c_str()); return -E_SCHEMA_PARSE_FAIL; } - FieldValue fieldValue; - errCode = inJsonObject.GetFieldValueByFieldPath(field.first, fieldValue); - if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Get schema TABLES DEFINE field value failed: %d.", errCode); - return -E_SCHEMA_PARSE_FAIL; - } - SchemaAttribute outAttr; - errCode = SchemaUtils::ParseAndCheckSchemaAttribute(fieldValue.stringValue, outAttr, false); + JsonObject fieldObj; + errCode = inJsonObject.GetObjectByFieldPath(field.first, fieldObj); if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Parse schema TABLES DEFINE attribute failed: %d.", errCode); + LOGE("[RelationalSchema][Parse] Get table field object failed. %s", errCode); return errCode; } FieldInfo fieldInfo; - fieldInfo.SetFieldName(field.first[1]); - fieldInfo.SetDataType(outAttr.customFieldType); - fieldInfo.SetNotNull(outAttr.hasNotNullConstraint); - if (outAttr.hasDefaultValue) { - fieldInfo.SetDefaultValue(outAttr.defaultValue.stringValue); + fieldInfo.SetFieldName(field.first[0]); // 0 : first element in path + errCode = ParseCheckTableFieldInfo(fieldObj, field.first, fieldInfo); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Parse table field info failed. %d", errCode); + return -E_SCHEMA_PARSE_FAIL; } resultTable.AddField(fieldInfo); } return E_OK; } +int RelationalSchemaObject::ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, + FieldInfo &table) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, "TYPE", FieldType::LEAF_FIELD_STRING, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + table.SetDataType(fieldValue.stringValue); + + errCode = GetMemberFromJsonObject(inJsonObject, "NOT_NULL", FieldType::LEAF_FIELD_BOOL, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + table.SetNotNull(fieldValue.boolValue); + + errCode = GetMemberFromJsonObject(inJsonObject, "DEFAULT", FieldType::LEAF_FIELD_STRING, false, fieldValue); + if (errCode == E_OK) { + table.SetDefaultValue(fieldValue.stringValue); + } else if (errCode != -E_NOT_FOUND) { + return errCode; + } + // TODO: need cid or not? + return E_OK; +} + int RelationalSchemaObject::ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable) { FieldValue fieldValue; - int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, - false, fieldValue); + int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, false, fieldValue); if (errCode == E_OK) { resultTable.SetAutoIncrement(fieldValue.boolValue); + } else if (errCode != -E_NOT_FOUND) { + return errCode; } - return errCode; + return E_OK; } int RelationalSchemaObject::ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable) @@ -685,7 +704,7 @@ int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, LOGE("[RelationalSchema][Parse] Get schema TABLES INDEX field value failed: %d.", errCode); return -E_SCHEMA_PARSE_FAIL; } - resultTable.AddIndexDefine(field.first[1], indexDefine); + resultTable.AddIndexDefine(field.first[1], indexDefine); // 1 : second element in path } return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 6dc424277..1b7f3d3ef 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -27,7 +27,7 @@ public: DB_API virtual ~RelationalStoreDelegate() = default; struct Option { - bool createIfNecessary = true; + // split mode }; DB_API virtual DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index cb4fabae3..a936be02a 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -35,7 +35,7 @@ RelationalStoreManager::RelationalStoreManager(const std::string &appId, const s void RelationalStoreManager::InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, const std::string &storeId, RelationalDBProperties &properties) { - properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); + // properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); properties.SetStringProp(RelationalDBProperties::APP_ID, appId_); properties.SetStringProp(RelationalDBProperties::USER_ID, userId_); @@ -96,8 +96,6 @@ DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const conn->Close(); return DB_ERROR; } - - (void)conn->TriggerAutoSync(); // TODO: no auto sync return OK; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 51b45a926..b689a3737 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -31,7 +31,10 @@ namespace { SQLiteRelationalStore::~SQLiteRelationalStore() { - delete sqliteStorageEngine_; + if (sqliteStorageEngine_ != nullptr) { + delete sqliteStorageEngine_; + sqliteStorageEngine_ = nullptr; + } } // Called when a new connection created. @@ -138,6 +141,12 @@ int SQLiteRelationalStore::CleanDistributedDeviceTable() int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) { + std::lock_guard lock(initalMutex_); + if (isInitialized_) { + LOGD("[RelationalStore][Open] relational db was already inited."); + return E_OK; + } + sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); if (sqliteStorageEngine_ == nullptr) { LOGE("[RelationalStore][Open] Create storage engine failed"); @@ -145,13 +154,13 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) } int errCode = E_OK; - do { errCode = InitStorageEngine(properties); if (errCode != E_OK) { LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); break; } + storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); if (storageEngine_ == nullptr) { LOGE("[RelationalStore][Open] Create syncable storage failed"); // TODO: @@ -176,10 +185,10 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) } syncEngine_ = std::make_shared(storageEngine_); + isInitialized_ = true; return E_OK; } while (false); - // TODO: release resources. ReleaseResources(); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 5358dc5d7..8c7b6127f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -77,6 +77,9 @@ private: std::vector> closeNotifiers_; RelationalDBProperties properties_; + + mutable std::mutex initalMutex_; + bool isInitialized_ = false; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index 5ae456f8f..f2ca51301 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -26,8 +26,7 @@ SyncAbleEngine::SyncAbleEngine(ISyncInterface *store) {} SyncAbleEngine::~SyncAbleEngine() -{ -} +{} // Start a sync action. int SyncAbleEngine::Sync(const ISyncer::SyncParma &parm) -- Gitee From da2c4800627a3d63194e7f9b3476df72701879c9 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Fri, 24 Dec 2021 14:36:36 +0800 Subject: [PATCH 08/19] Fix relational schema issue. Signed-off-by: lianhuix --- .../relational/relational_schema_object.h | 2 +- .../distributeddb/common/src/json_object.cpp | 1 - .../relational/relational_schema_object.cpp | 13 +++++---- ...single_ver_relational_storage_executor.cpp | 28 +++++++++---------- .../storage/src/sqlite/sqlite_utils.cpp | 16 +++++------ 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index c91a1d711..ed59af074 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -138,7 +138,7 @@ public: const std::map &GetTables() const; - const TableInfo &GetTable(const std::string& tableName) const; + TableInfo GetTable(const std::string& tableName) const; private: int CompareAgainstSchemaObject(const std::string &inSchemaString, std::map &cmpRst) const; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 80313347a..cb6c3516f 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -734,7 +734,6 @@ int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &RelationalSchemaObject::GetTables() cons return tables_; } -const TableInfo &RelationalSchemaObject::GetTable(const std::string& tableName) const +TableInfo RelationalSchemaObject::GetTable(const std::string& tableName) const { auto it = tables_.find(tableName); if (it != tables_.end()) { @@ -447,6 +447,7 @@ int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &f bool isNecessary, FieldValue &fieldValue) { if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { + LOGW("[RelationalSchema][Parse] Get schema %s not exist. isNeccessary: %d", fieldName.c_str(), isNecessary); return isNecessary ? -E_SCHEMA_PARSE_FAIL : -E_NOT_FOUND; } @@ -458,11 +459,11 @@ int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &f } if (fieldType != expectType) { - LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but : %d.", fieldName.c_str(), expectType, fieldType); + LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but: %d.", fieldName.c_str(), expectType, fieldType); return -E_SCHEMA_PARSE_FAIL; } - errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {"NAME"}, fieldValue); + errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {fieldName}, fieldValue); if (errCode != E_OK) { LOGE("[RelationalSchema][Parse] Get schema %s value failed: %d.", fieldName.c_str(), errCode); return -E_SCHEMA_PARSE_FAIL; @@ -598,8 +599,8 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject } for (const auto &field : tableFields) { - if (field.second != FieldType::LEAF_FIELD_OBJECT) { - LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType OBJECT but : %s.", + if (field.second != FieldType::INTERNAL_FIELD_OBJECT) { + LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType INTERNAL OBJECT but : %s.", SchemaUtils::FieldTypeString(field.second).c_str()); return -E_SCHEMA_PARSE_FAIL; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 17bd91e9d..489e70eb2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -117,10 +117,10 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSyncDataByTime(TimeStamp } const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = - "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + + "SELECT * FROM naturalbase_rdb_aux_" + table_.GetTableName() + "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; const std::string SELECT_SYNC_ENTRIES_SQL = - "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + + "SELECT * FROM naturalbase_rdb_aux_" + table_.GetTableName() + "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); @@ -395,8 +395,7 @@ int SQLiteSingleVerRelationalStorageExecutor::GetDeletedSyncDataByTimestamp(std: return CheckCorruptedStatus(errCode); } -static const std::string SELECT_META_VALUE_SQL = - "SELECT value FROM distributeddatamgr_aux_metadata WHERE key=?;"; +static const std::string SELECT_META_VALUE_SQL = "SELECT value FROM naturalbase_rdb_aux_metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const { sqlite3_stmt *statement = nullptr; @@ -424,7 +423,7 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v return errCode; } -static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO distributeddatamgr_aux_metadata VALUES(?,?);"; +static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO naturalbase_rdb_aux_metadata VALUES(?,?);"; int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const { sqlite3_stmt *statement = nullptr; @@ -453,7 +452,7 @@ ERROR: return errCode; } -static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM distributeddatamgr_aux_metadata WHERE key=?;"; +static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM naturalbase_rdb_aux_metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const { sqlite3_stmt *statement = nullptr; @@ -480,7 +479,7 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector=? AND key<=?;"; + "DELETE FROM naturalbase_rdb_aux_metadata WHERE key>=? AND key<=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const { sqlite3_stmt *statement = nullptr; @@ -527,8 +526,7 @@ static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) return errCode; } - -static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM distributeddatamgr_aux_metadata;"; +static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM naturalbase_rdb_aux_metadata;"; int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const { sqlite3_stmt *statement = nullptr; @@ -546,7 +544,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj const std::string &deviceName, sqlite3_stmt *&logStmt) const { std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_log"; + const std::string tableName = "naturalbase_rdb_aux_" + object.GetTableName() + "_log"; std::string dataFormat = "?, '" + deviceName + "', ?, ?, ?, ?, ?"; std::string sql = "INSERT OR REPLACE INTO " + tableName + @@ -562,10 +560,10 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const { - // distributeddatamgr_aux_userTableName_deviceHash + // naturalbase_rdb_aux_userTableName_deviceHash // tableName std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_" + + const std::string tableName = "naturalbase_rdb_aux_" + object.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); @@ -603,7 +601,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen Key hashKey; (void)DBCommon::CalcValueHash(dataItem.key, hashKey); std::string hash = std::string(hashKey.begin(), hashKey.end()); - std::string sql = "select * from distributeddatamgr_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; + std::string sql = "select * from naturalbase_rdb_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; sqlite3_stmt *queryStmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, queryStmt); if (errCode != E_OK) { @@ -661,7 +659,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) { std::string devName = DBCommon::TransferHashString(dataItem.dev); - const std::string tableName = "distributeddatamgr_aux_" + table_.GetTableName() + "_" + + const std::string tableName = "naturalbase_rdb_aux_" + table_.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; @@ -784,7 +782,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncItems(const QueryObject &o static int GetLogInfoStatement(sqlite3 *dbHandle, const std::string &tableName, uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement) { - std::string sql = "select * from distributeddatamgr_aux_" + tableName + + std::string sql = "select * from naturalbase_rdb_aux_" + tableName + "_log where flag=0x02 AND timestamp>=? AND timestamp Date: Sat, 25 Dec 2021 11:40:25 +0800 Subject: [PATCH 09/19] Add create distributed table constraint. Signed-off-by: lianhuix --- .../common/include/db_constant.h | 1 + .../relational/sqlite_relational_store.cpp | 8 +++++++ .../relational/sqlite_relational_store.h | 2 ++ ...single_ver_relational_storage_executor.cpp | 16 +++++++++++-- .../storage/src/sqlite/sqlite_utils.cpp | 23 +++++++++++++++++++ .../storage/src/sqlite/sqlite_utils.h | 2 ++ 6 files changed, 50 insertions(+), 2 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 722df4437..9c5d0e65b 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -111,6 +111,7 @@ public: static constexpr size_t MAX_SYNC_BLOCK_SIZE = 31457280; // 30MB static constexpr int DOUBLE_PRECISION = 15; + static constexpr int MAX_DISTRIBUTED_TABLE_COUNT = 32; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index b689a3737..31a26d5f6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -109,6 +109,7 @@ int SQLiteRelationalStore::GetSchemaFromMeta() return errCode; } + std::lock_guard lock(schemaMutex_); properties_.SetSchema(schema); return E_OK; } @@ -123,6 +124,7 @@ int SQLiteRelationalStore::SaveLogTableVersionToMeta() int SQLiteRelationalStore::CleanDistributedDeviceTable() { // TODO: clean the device table which is no longer in schema + std::lock_guard lock(schemaMutex_); RelationalSchemaObject schema = properties_.GetSchema(); for (const auto &table : schema.GetTables()) { std::string tableName = table.first; @@ -290,12 +292,18 @@ void SQLiteRelationalStore::WakeUpSyncer() int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) { int errCode = E_OK; + std::lock_guard lock(schemaMutex_); auto schema = properties_.GetSchema(); if (schema.GetTable(tableName).GetTableName() == tableName) { LOGW("distributed table %s was already created.", tableName.c_str()); return E_OK; } + if (schema.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { + LOGW("The number of distributed tables is exceeds limit."); + return -E_MAX_LIMITS; + } + auto *handle = GetHandle(true, errCode); if (handle != nullptr) { return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 8c7b6127f..1d069659f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -80,6 +80,8 @@ private: mutable std::mutex initalMutex_; bool isInitialized_ = false; + + mutable std::mutex schemaMutex_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 489e70eb2..57e6a7181 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -25,11 +25,23 @@ SQLiteSingleVerRelationalStorageExecutor::SQLiteSingleVerRelationalStorageExecut int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, TableInfo &table) { if (dbHandle_ == nullptr) { - LOGE("Begin transaction failed, dbHandle is null."); + LOGE("[CreateDistributedTable] Begin transaction failed, dbHandle is null."); return -E_INVALID_DB; } - int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); + int historyDataCnt = 0; + int errCode = SQLiteUtils::GetTableCount(dbHandle_, tableName, historyDataCnt); + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] Get the number of table [%s] rows failed. %d", tableName.c_str(), errCode); + return errCode; + } + + if (historyDataCnt > 0) { // 0 : create distributed table should on an empty table + LOGE("[CreateDistributedTable] Create distributed table should on an empty table."); + return -E_NOT_SUPPORT; + } + + errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 1dd2b58c4..32f30f031 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1921,4 +1921,27 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) sqlite3_free(eSql); return E_OK; } + +int GetTableCount(sqlite3 *db, const std::string &tableName, int &count) +{ + if (db == nullptr) { + return -E_INVALID_ARGS; + } + + std::string cntSql = "SELECT COUNT(*) FROM " + tableName + ";"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, cntSql, stmt); + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + count = sqlite3_column_int(stmt, 0); + errCode = E_OK; + } + + SQLiteUtils::ResetStatement(stmt, true, errCode); + return SQLiteUtils::MapSQLiteErrno(errCode); +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 94d714001..d8a9d8d01 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -174,6 +174,8 @@ public: static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); + static int GetTableCount(sqlite3 *db, const std::string &tableName, int &count); + private: static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp); -- Gitee From 18c9db5c22fcdd441f5228ed7c8d3fe57eb67922 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 25 Dec 2021 15:01:25 +0800 Subject: [PATCH 10/19] Fix issues. Signed-off-by: lianhuix --- .../common/src/relational/relational_schema_object.cpp | 2 +- .../src/sqlite/relational/sqlite_relational_store.cpp | 5 +++-- .../libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp | 2 +- .../syncer/distributeddb_relational_ver_p2p_sync_test.cpp | 4 ++-- .../test/unittest/common/syncer/virtual_device.cpp | 4 ++-- .../test/unittest/common/syncer/virtual_device.h | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index e3a3551ee..e5f33dab1 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -447,7 +447,7 @@ int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &f bool isNecessary, FieldValue &fieldValue) { if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { - LOGW("[RelationalSchema][Parse] Get schema %s not exist. isNeccessary: %d", fieldName.c_str(), isNecessary); + LOGW("[RelationalSchema][Parse] Get schema %s not exist. isNecessary: %d", fieldName.c_str(), isNecessary); return isNecessary ? -E_SCHEMA_PARSE_FAIL : -E_NOT_FOUND; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 31a26d5f6..b3f7e305a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -304,8 +304,9 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) return -E_MAX_LIMITS; } + LOGD("Create distributed table for %s.", tableName.c_str()); auto *handle = GetHandle(true, errCode); - if (handle != nullptr) { + if (handle == nullptr) { return errCode; } @@ -319,7 +320,7 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) } ReleaseHandle(handle); - return E_OK; + return errCode; } } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 32f30f031..b900d4779 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1922,7 +1922,7 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) return E_OK; } -int GetTableCount(sqlite3 *db, const std::string &tableName, int &count) +int SQLiteUtils::GetTableCount(sqlite3 *db, const std::string &tableName, int &count) { if (db == nullptr) { return -E_INVALID_ARGS; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index c24f02a1e..1f76b576f 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -185,7 +185,7 @@ namespace { EXPECT_EQ(GetDB(db), SQLITE_OK); EXPECT_EQ(CreateTable(db), SQLITE_OK); - EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName, {}), OK); + EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName), OK); sqlite3_close(db); @@ -254,7 +254,7 @@ void DistributedDBRelationalVerP2PSyncTest::SetUp(void) /** * @tc.setup: create virtual device B, and get a KvStoreNbDelegate as deviceA */ - g_mgr.OpenStore(g_dbDir, {true}, g_kvDelegateCallback); + g_kvDelegateStatus = g_mgr.OpenStore(g_dbDir, "Relational_default_id", {}, g_kvDelegatePtr); ASSERT_TRUE(g_kvDelegateStatus == OK); ASSERT_TRUE(g_kvDelegatePtr != nullptr); g_deviceB = new (std::nothrow) RelationalVirtualDevice(DEVICE_B); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp index eaa1f7764..25749ee91 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp @@ -20,7 +20,7 @@ #include "device_manager.h" #include "log_print.h" #include "multi_ver_sync_state_machine.h" -#include "single_ver_subscribe_manager.h" +#include "subscribe_manager.h" #include "single_ver_sync_state_machine.h" #include "sync_types.h" #include "virtual_communicator.h" @@ -99,7 +99,7 @@ int VirtualDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggrega } if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { context_ = new (std::nothrow) SingleVerSyncTaskContext; - subManager_ = std::make_shared(); + subManager_ = std::make_shared(); static_cast(context_)->SetSubscribeManager(subManager_); } else { context_ = new (std::nothrow) MultiVerSyncTaskContext; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h index f6d93ff99..5132c0dd6 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h @@ -60,7 +60,7 @@ private: std::string remoteDeviceId_; SyncTaskContext *context_; std::function onRemoteDataChanged_; - std::shared_ptr subManager_; + std::shared_ptr subManager_; }; } // namespace DistributedDB -- Gitee From 8ab7f46892055f643636e5b3966afd0acf35fa0c Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 25 Dec 2021 16:27:18 +0800 Subject: [PATCH 11/19] Add transaction for create distributed table. Save schema to meta table Signed-off-by: lianhuix --- .../relational/sqlite_relational_store.cpp | 29 ++++++++++++++++--- .../relational/sqlite_relational_store.h | 1 + 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index b3f7e305a..45d88eb4f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -114,6 +114,18 @@ int SQLiteRelationalStore::GetSchemaFromMeta() return E_OK; } +int SQLiteRelationalStore::SaveSchemaToMeta() +{ + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + DBCommon::StringToVector(properties_.GetSchema().ToSchemaString(), schemaVal); + int errCode = storageEngine_->PutMetaData(schemaKey, schemaVal); + if (errCode != E_OK) { + LOGE("Save relational schema to meta table failed. %d", errCode); + } + return errCode; +} + int SQLiteRelationalStore::SaveLogTableVersionToMeta() { // TODO: save log table version into meta data @@ -310,17 +322,26 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) return errCode; } + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + TableInfo table; errCode = handle->CreateDistributedTable(tableName, table); if (errCode != E_OK) { LOGE("create distributed table failed. %d", errCode); - } else { - schema.AddRelationalTable(table); - properties_.SetSchema(schema); + (void)handle->Rollback(); + ReleaseHandle(handle); + return errCode; } + schema.AddRelationalTable(table); + properties_.SetSchema(schema); + (void)handle->Commit(); ReleaseHandle(handle); - return errCode; + return SaveSchemaToMeta(); } } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 1d069659f..ecb9b1f6d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -59,6 +59,7 @@ private: void DecreaseConnectionCounter(); int GetSchemaFromMeta(); + int SaveSchemaToMeta(); int SaveLogTableVersionToMeta(); -- Gitee From 53dfeab00cc93bd6bb6e53f45dc15b4128672a93 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 25 Dec 2021 17:54:10 +0800 Subject: [PATCH 12/19] Add journal and synchronous mode check Signed-off-by: lianhuix --- .../relational/sqlite_relational_store.cpp | 21 ++++++++ .../relational/sqlite_relational_store.h | 2 +- ...single_ver_relational_storage_executor.cpp | 19 +++++++ ...e_single_ver_relational_storage_executor.h | 2 + .../storage/src/sqlite/sqlite_utils.cpp | 50 +++++++++++++++++++ .../storage/src/sqlite/sqlite_utils.h | 4 ++ 6 files changed, 97 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 45d88eb4f..d882fe91f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -87,6 +87,22 @@ void SQLiteRelationalStore::ReleaseResources() } } +int SQLiteRelationalStore::CheckDBMode() +{ + int errCode = E_OK; + auto *handle = GetHandle(false, errCode); + if (handle == nullptr) { + return errCode; + } + errCode = handle->CheckDBModeForRelational(); + if (errCode != E_OK) { + LOGE("check relational DB mode failed. %d", errCode); + } + + ReleaseHandle(handle); + return errCode; +} + int SQLiteRelationalStore::GetSchemaFromMeta() { const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); @@ -182,6 +198,11 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } + errCode = CheckDBMode(); + if (errCode != E_OK) { + break; + } + properties_ = properties; errCode = GetSchemaFromMeta(); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index ecb9b1f6d..dd0b50360 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -57,7 +57,7 @@ private: // 1 store 1 connection void DecreaseConnectionCounter(); - + int CheckDBMode(); int GetSchemaFromMeta(); int SaveSchemaToMeta(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 57e6a7181..f7ce29b93 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -827,5 +827,24 @@ int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataByQuery(std::vector Date: Sat, 25 Dec 2021 19:10:11 +0800 Subject: [PATCH 13/19] Open relational db without set journal mode Signed-off-by: lianhuix --- .../sqlite_single_relational_storage_engine.cpp | 2 +- .../sqlite_single_ver_relational_storage_executor.cpp | 2 +- .../distributeddb/storage/src/sqlite/sqlite_utils.cpp | 11 +++++++---- .../distributeddb/storage/src/sqlite/sqlite_utils.h | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index 6c9a5390f..6ef4b916e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -52,7 +52,7 @@ int SQLiteSingleRelationalStorageEngine::RegisterFunction(sqlite3 *db) const int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, StorageExecutor *&handle) { sqlite3 *db = nullptr; - int errCode = SQLiteUtils::OpenDatabase(option_, db); + int errCode = SQLiteUtils::OpenDatabase(option_, db, false); if (errCode != E_OK) { return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index f7ce29b93..3c029f027 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -833,7 +833,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CheckDBModeForRelational() std::string journalMode; int errCode = SQLiteUtils::GetJournalMode(dbHandle_, journalMode); if (errCode != E_OK || journalMode != "wal") { - LOGE("Not support journal mode %s for relational db, expect wal mode, %d", journalMode, errCode); + LOGE("Not support journal mode %s for relational db, expect wal mode, %d", journalMode.c_str(), errCode); return -E_NOT_SUPPORT; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index c9cc7d7c8..8d2b3ad1a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -102,7 +102,7 @@ std::string GetTriggerModeString(TriggerModeEnum mode) } } -int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp) +int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal) { uint64_t flag = SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE; if (properties.createIfNecessary) { @@ -114,7 +114,10 @@ int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&db return -E_INVALID_ARGS; } std::string defaultAttachCipher = DEFAULT_ATTACH_CIPHER + cipherName + ";"; - std::vector sqls {WAL_MODE_SQL, defaultAttachCipher, DEFAULT_ATTACH_KDF_ITER}; + std::vector sqls {defaultAttachCipher, DEFAULT_ATTACH_KDF_ITER}; + if (setWal) { + sqls.push_back(WAL_MODE_SQL); + } std::string fileUrl = DBConstant::SQLITE_URL_PRE + properties.uri; int errCode = sqlite3_open_v2(fileUrl.c_str(), &dbTemp, flag, nullptr); @@ -139,7 +142,7 @@ END: return errCode; } -int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db) +int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db, bool setWal) { { // Only for register the sqlite3 log callback @@ -150,7 +153,7 @@ int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db) } } sqlite3 *dbTemp = nullptr; - int errCode = CreateDataBase(properties, dbTemp); + int errCode = CreateDataBase(properties, dbTemp, setWal); if (errCode != E_OK) { goto END; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 864169a63..8452f705a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -69,7 +69,7 @@ struct OpenDbProperties { class SQLiteUtils { public: // Initialize the SQLiteUtils with the given properties. - static int OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db); + static int OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db, bool setWal = true); // Check the statement and prepare the new if statement is null static int GetStatement(sqlite3 *db, const std::string &sql, sqlite3_stmt *&statement); @@ -182,7 +182,7 @@ public: private: - static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp); + static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal); static int SetBusyTimeout(sqlite3 *db, int timeout); -- Gitee From 411622f573d588d83a777308f9ae0de9a779ad74 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 27 Dec 2021 10:05:40 +0800 Subject: [PATCH 14/19] Split DBProperties into KV & Relational Signed-off-by: lianhuix --- .../libs/distributeddb/BUILD.gn | 2 + .../relational/relational_store_manager.h | 2 +- .../relational/relational_store_instance.h | 2 +- .../storage/include/db_properties.h | 61 +++ .../storage/include/kvdb_properties.h | 59 +-- .../storage/include/relationaldb_properties.h | 44 ++ .../storage/src/db_properties.cpp | 72 ++++ .../storage/src/irelational_store.h | 2 +- .../storage/src/kvdb_properties.cpp | 75 ---- .../storage/src/relationaldb_properties.cpp | 39 ++ .../src/single_ver_subscribe_manager.cpp | 394 ------------------ .../syncer/src/single_ver_subscribe_manager.h | 130 ------ .../libs/distributeddb/test/BUILD.gn | 2 + .../unittest/common/syncer/virtual_device.cpp | 305 -------------- .../unittest/common/syncer/virtual_device.h | 67 --- 15 files changed, 224 insertions(+), 1032 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h create mode 100644 services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp delete mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp delete mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h delete mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp delete mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 05ae98eac..924097bb6 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -127,6 +127,7 @@ ohos_shared_library("distributeddb") { "interfaces/src/relational/relational_store_sqlite_ext.cpp", "interfaces/src/relational/runtime_config.cpp", "storage/src/data_transformer.cpp", + "storage/src/db_properties.cpp", "storage/src/default_factory.cpp", "storage/src/generic_kvdb.cpp", "storage/src/generic_kvdb_connection.cpp", @@ -160,6 +161,7 @@ ohos_shared_library("distributeddb") { "storage/src/relational_store_connection.cpp", "storage/src/relational_store_instance.cpp", "storage/src/relational_sync_able_storage.cpp", + "storage/src/relationaldb_properties.cpp", "storage/src/result_entries_window.cpp", "storage/src/single_ver_natural_store_commit_notify_data.cpp", "storage/src/sqlite/query_object.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 459196bdf..71fc2e44c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -22,7 +22,7 @@ #include "auto_launch_export.h" #include "relational_store_delegate.h" #include "irelational_store.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" #include "types.h" namespace DistributedDB { diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h index 1bed16fe0..5c24c1f2c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h @@ -20,7 +20,7 @@ #include #include "irelational_store.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" namespace DistributedDB { class RelationalStoreInstance final { diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h new file mode 100644 index 000000000..1bb6f0baf --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DB_PROPERTIES_H +#define DB_PROPERTIES_H + +#include +#include +#include + +namespace DistributedDB { +class DBProperties { +public: + // Get the string property according the name + std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; + + // Set the string property for the name + void SetStringProp(const std::string &name, const std::string &value); + + // Get the bool property according the name + bool GetBoolProp(const std::string &name, bool defaultValue) const; + + // Set the bool property for the name + void SetBoolProp(const std::string &name, bool value); + + // Get the bool property according the name + int GetIntProp(const std::string &name, int defaultValue) const; + + // Set the integer property for the name + void SetIntProp(const std::string &name, int value); + + static const std::string CREATE_IF_NECESSARY; + static const std::string DATABASE_TYPE; + static const std::string DATA_DIR; + static const std::string USER_ID; + static const std::string APP_ID; + static const std::string STORE_ID; + static const std::string IDENTIFIER_DATA; + static const std::string IDENTIFIER_DIR; + +protected: + DBProperties() = default; + virtual ~DBProperties() = default; + + std::map stringProperties_; + std::map boolProperties_; + std::map intProperties_; +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h index 7250af7a5..c985b0d83 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h @@ -20,48 +20,10 @@ #include #include +#include "db_properties.h" #include "schema_object.h" -#include "relational_schema_object.h" namespace DistributedDB { -class DBProperties { -public: - DBProperties() = default; - virtual ~DBProperties() = default; - - // Get the string property according the name - std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; - - // Set the string property for the name - void SetStringProp(const std::string &name, const std::string &value); - - // Get the bool property according the name - bool GetBoolProp(const std::string &name, bool defaultValue) const; - - // Set the bool property for the name - void SetBoolProp(const std::string &name, bool value); - - // Get the bool property according the name - int GetIntProp(const std::string &name, int defaultValue) const; - - // Set the integer property for the name - void SetIntProp(const std::string &name, int value); - - static const std::string CREATE_IF_NECESSARY; - static const std::string DATABASE_TYPE; - static const std::string DATA_DIR; - static const std::string USER_ID; - static const std::string APP_ID; - static const std::string STORE_ID; - static const std::string IDENTIFIER_DATA; - static const std::string IDENTIFIER_DIR; - -protected: - std::map stringProperties_; - std::map boolProperties_; - std::map intProperties_; -}; - class KvDBProperties final : public DBProperties { public: KvDBProperties(); @@ -117,25 +79,6 @@ private: CipherPassword password_; SchemaObject schema_; }; - -// TODO: move to its own file, or rename this file -class RelationalDBProperties final : public DBProperties { -public: - RelationalDBProperties(); - ~RelationalDBProperties() override; - - // is schema exist - bool IsSchemaExist() const; - - // set schema - void SetSchema(const RelationalSchemaObject &schema); - - // get schema - RelationalSchemaObject GetSchema() const; - -private: - RelationalSchemaObject schema_; -}; } // namespace DistributedDB #endif // KV_DB_PROPERTIES_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h new file mode 100644 index 000000000..4877e7220 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONALDB_PROPERTIES_H +#define RELATIONALDB_PROPERTIES_H + +#include +#include +#include + +#include "db_properties.h" +#include "relational_schema_object.h" + +namespace DistributedDB { +class RelationalDBProperties final : public DBProperties { +public: + RelationalDBProperties(); + ~RelationalDBProperties() override; + + // is schema exist + bool IsSchemaExist() const; + + // set schema + void SetSchema(const RelationalSchemaObject &schema); + + // get schema + RelationalSchemaObject GetSchema() const; + +private: + RelationalSchemaObject schema_; +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp new file mode 100644 index 000000000..92ecca798 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "db_properties.h" + +namespace DistributedDB { +const std::string DBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; +const std::string DBProperties::DATABASE_TYPE = "databaseType"; +const std::string DBProperties::DATA_DIR = "dataDir"; +const std::string DBProperties::USER_ID = "userId"; +const std::string DBProperties::APP_ID = "appId"; +const std::string DBProperties::STORE_ID = "storeId"; +const std::string DBProperties::IDENTIFIER_DATA = "identifier"; +const std::string DBProperties::IDENTIFIER_DIR = "identifierDir"; + +std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const +{ + auto iter = stringProperties_.find(name); + if (iter != stringProperties_.end()) { + return iter->second; + } else { + return defaultValue; + } +} + +void DBProperties::SetStringProp(const std::string &name, const std::string &value) +{ + stringProperties_[name] = value; +} + +bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const +{ + auto iter = boolProperties_.find(name); + if (iter != boolProperties_.end()) { + return iter->second; + } else { + return defaultValue; + } +} + +void DBProperties::SetBoolProp(const std::string &name, bool value) +{ + boolProperties_[name] = value; +} + +int DBProperties::GetIntProp(const std::string &name, int defaultValue) const +{ + auto iter = intProperties_.find(name); + if (iter != intProperties_.end()) { + return iter->second; + } else { + return defaultValue; + } +} + +void DBProperties::SetIntProp(const std::string &name, int value) +{ + intProperties_[name] = value; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h index 5ff966e9e..7695b7877 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h @@ -20,7 +20,7 @@ #include #include "ref_object.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" #include "relational_store_connection.h" namespace DistributedDB { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp index 75e98ded8..c10569d82 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp @@ -18,15 +18,6 @@ #include "db_constant.h" namespace DistributedDB { -const std::string DBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; -const std::string DBProperties::DATABASE_TYPE = "databaseType"; -const std::string DBProperties::DATA_DIR = "dataDir"; -const std::string DBProperties::USER_ID = "userId"; -const std::string DBProperties::APP_ID = "appId"; -const std::string DBProperties::STORE_ID = "storeId"; -const std::string DBProperties::IDENTIFIER_DATA = "identifier"; -const std::string DBProperties::IDENTIFIER_DIR = "identifierDir"; - const std::string KvDBProperties::FILE_NAME = "fileName"; const std::string KvDBProperties::MEMORY_MODE = "memoryMode"; const std::string KvDBProperties::ENCRYPTED_MODE = "isEncryptedDb"; @@ -60,51 +51,6 @@ std::string KvDBProperties::GetStoreSubDirectory(int type) } } -std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const -{ - auto iter = stringProperties_.find(name); - if (iter != stringProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetStringProp(const std::string &name, const std::string &value) -{ - stringProperties_[name] = value; -} - -bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const -{ - auto iter = boolProperties_.find(name); - if (iter != boolProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetBoolProp(const std::string &name, bool value) -{ - boolProperties_[name] = value; -} - -int DBProperties::GetIntProp(const std::string &name, int defaultValue) const -{ - auto iter = intProperties_.find(name); - if (iter != intProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetIntProp(const std::string &name, int value) -{ - intProperties_[name] = value; -} - void KvDBProperties::GetPassword(CipherType &type, CipherPassword &password) const { type = cipherType_; @@ -146,25 +92,4 @@ const SchemaObject &KvDBProperties::GetSchemaConstRef() const { return schema_; } - -RelationalDBProperties::RelationalDBProperties() -{} - -RelationalDBProperties::~RelationalDBProperties() -{} - -bool RelationalDBProperties::IsSchemaExist() const -{ - return schema_.IsSchemaValid(); -} - -void RelationalDBProperties::SetSchema(const RelationalSchemaObject &schema) -{ - schema_ = schema; -} - -RelationalSchemaObject RelationalDBProperties::GetSchema() const -{ - return schema_; -} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp new file mode 100644 index 000000000..3a87aafb5 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "relationaldb_properties.h" + +namespace DistributedDB { +RelationalDBProperties::RelationalDBProperties() +{} + +RelationalDBProperties::~RelationalDBProperties() +{} + +bool RelationalDBProperties::IsSchemaExist() const +{ + return schema_.IsSchemaValid(); +} + +void RelationalDBProperties::SetSchema(const RelationalSchemaObject &schema) +{ + schema_ = schema; +} + +RelationalSchemaObject RelationalDBProperties::GetSchema() const +{ + return schema_; +} +} diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp deleted file mode 100644 index ea78cbe1b..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "single_ver_subscribe_manager.h" -#include "db_common.h" -#include "sync_types.h" - -namespace DistributedDB { -void SingleVerSubscribeManager::ClearRemoteSubscribeQuery(const std::string &device) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - ClearSubscribeQuery(device, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::ClearAllRemoteQuery() -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - remoteSubscribedMap_.clear(); - remoteSubscribedTotalMap_.clear(); -} - -void SingleVerSubscribeManager::ClearLocalSubscribeQuery(const std::string &device) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - unFinishedLocalAutoSubMap_.erase(device); - ClearSubscribeQuery(device, localSubscribeMap_, localSubscribeTotalMap_); -} - -int SingleVerSubscribeManager::ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - int errCode = ReserveSubscribeQuery(device, query, remoteSubscribedMap_, remoteSubscribedTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote reserve err=%d", - STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - int errCode = ActiveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote active err=%d", - STR_MASK(device), STR_MASK(queryId), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - int errCode = ReserveSubscribeQuery(device, query, localSubscribeMap_, localSubscribeTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local reserve err=%d", - STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - int errCode = ActiveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local active err=%d", - STR_MASK(device), STR_MASK(queryId), errCode); - if (errCode != E_OK) { - return errCode; - } - if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && - unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { - unFinishedLocalAutoSubMap_[device].erase(queryId); - } - return errCode; -} - -void SingleVerSubscribeManager::DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - DeleteSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); -} - -void SingleVerSubscribeManager::DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - DeleteSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::PutLocalUnFiniedSubQueries(const std::string &device, - std::vector &subscribeQueries) -{ - LOGI("[SingleVerSubscribeManager] put local unfinished subscribe queries, nums=%d", subscribeQueries.size()); - std::unique_lock lockGuard(localSubscribeMapLock_); - if (subscribeQueries.size() == 0) { - unFinishedLocalAutoSubMap_.erase(device); - return; - } - unFinishedLocalAutoSubMap_[device].clear(); - auto iter = unFinishedLocalAutoSubMap_.find(device); - for (const auto &query : subscribeQueries) { - iter->second.insert(query.GetIdentify()); - } -} - -void SingleVerSubscribeManager::GetAllUnFinishSubQueries( - std::map> &allSyncQueries) const -{ - std::shared_lock lock(localSubscribeMapLock_); - for (auto &item : unFinishedLocalAutoSubMap_) { - if (item.second.size() == 0) { - continue; - } - allSyncQueries[item.first] = {}; - auto iter = allSyncQueries.find(item.first); - for (const auto &queryId : item.second) { - auto iterTmp = localSubscribeTotalMap_.find(queryId); - if (iterTmp == localSubscribeTotalMap_.end()) { - LOGI("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryId)); - continue; - } - iter->second.push_back(iterTmp->second.first); - } - } -} - -void SingleVerSubscribeManager::RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - RemoveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - RemoveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); - if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && - unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { - unFinishedLocalAutoSubMap_[device].erase(queryId); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s delete from UnFinishedMap", - STR_MASK(device), STR_MASK(queryId)); - if (unFinishedLocalAutoSubMap_[device].size() == 0) { - LOGI("[SingleVerSubscribeManager] dev=%s delete from unFinish map", STR_MASK(device)); - unFinishedLocalAutoSubMap_.erase(device); - } - } -} - -void SingleVerSubscribeManager::GetLocalSubscribeQueries(const std::string &device, - std::vector &subscribeQueries) const -{ - std::shared_lock lock(localSubscribeMapLock_); - GetSubscribeQueries(device, localSubscribeMap_, localSubscribeTotalMap_, subscribeQueries); -} - -void SingleVerSubscribeManager::GetRemoteSubscribeQueries(const std::string &device, - std::vector &subscribeQueries) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - GetSubscribeQueries(device, remoteSubscribedMap_, remoteSubscribedTotalMap_, subscribeQueries); -} - -bool SingleVerSubscribeManager::IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - auto iter = remoteSubscribedMap_.find(device); - if (iter == remoteSubscribedMap_.end()) { - LOGD("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); - return false; - } - std::string queryId = query.GetIdentify(); - auto subIter = iter->second.find(queryId); - if (subIter == iter->second.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryId)); - return false; - } - return true; -} - -void SingleVerSubscribeManager::GetRemoteSubscribeQueryIds(const std::string &device, - std::vector &subscribeQueryIds) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - auto iter = remoteSubscribedMap_.find(device); - if (iter == remoteSubscribedMap_.end()) { - LOGI("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : iter->second) { - if (remoteSubscribedTotalMap_.find(queryInfo.first) == remoteSubscribedTotalMap_.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryInfo.first)); - continue; - } - subscribeQueryIds.push_back(queryInfo.first); - } -} - -int SingleVerSubscribeManager::LocalSubscribeLimitCheck(const std::vector &devices, - QuerySyncObject &query) const -{ - std::shared_lock lock(localSubscribeMapLock_); - int devNum = localSubscribeMap_.size(); - for (const auto &device : devices) { - if (localSubscribeMap_.find(device) != localSubscribeMap_.end()) { - continue; - } - devNum++; - if (devNum > MAX_DEVICES_NUM) { - LOGE("[SingleVerSubscribeManager] local subscribe devices is over limit"); - return -E_MAX_LIMITS; - } - } - std::string queryId = query.GetIdentify(); - auto allIter = localSubscribeTotalMap_.find(queryId); - if (allIter == localSubscribeTotalMap_.end() && localSubscribeTotalMap_.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { - LOGE("[SingleVerSubscribeManager] all local subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - return E_OK; -} - -void SingleVerSubscribeManager::ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGI("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : subscribeMap[device]) { - if (subscribedTotalMap.find(queryInfo.first) != subscribedTotalMap.end()) { - if (subscribedTotalMap[queryInfo.first].second > 0) { - subscribedTotalMap[queryInfo.first].second--; - } - if (subscribedTotalMap[queryInfo.first].second == 0) { - LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryInfo.first)); - subscribedTotalMap.erase(queryInfo.first); - } - } - } - subscribeMap.erase(device); - LOGI("[SingleVerSubscribeManager] clear dev=%s` remote subscribe queies finished", STR_MASK(device)); -} - -int SingleVerSubscribeManager::ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - std::string queryId = query.GetIdentify(); - auto iter = subscribeMap.find(device); - auto allIter = subscribedTotalMap.find(queryId); - // limit check - if (allIter == subscribedTotalMap.end() && subscribedTotalMap.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { - LOGE("[SingleVerSubscribeManager] all subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - if (iter == subscribeMap.end() && subscribeMap.size() >= MAX_DEVICES_NUM) { - LOGE("[SingleVerSubscribeManager] subscribe devices is over limit"); - return -E_MAX_LIMITS; - } - if (iter != subscribeMap.end() && iter->second.find(queryId) == iter->second.end() && - iter->second.size() >= MAX_SUBSCRIBE_NUM_PER_DEV) { - LOGE("[SingleVerSubscribeManager] subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - if (iter != subscribeMap.end() && iter->second.find(queryId) != iter->second.end() && - iter->second[queryId] == SubscribeStatus::ACTIVE) { - LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s already active in map", - STR_MASK(device), STR_MASK(queryId)); - return E_OK; - } - - if (iter == subscribeMap.end()) { - subscribeMap[device] = std::map {}; - } - bool isNeedInc = false; - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - subscribeMap[device][queryId] = SubscribeStatus::NOT_ACTIVE; - isNeedInc = true; - } - if (allIter == subscribedTotalMap.end()) { - subscribedTotalMap[queryId] = {query, 1}; - } else if (isNeedInc) { - subscribedTotalMap[queryId].second++; - } - return E_OK; -} - -int SingleVerSubscribeManager::ActiveSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribedTotalMap.find(queryId) == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); - return -E_INTERNAL_ERROR; - } - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s in localSubscribeMap", STR_MASK(device)); - return -E_INTERNAL_ERROR; - } - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); - return -E_INTERNAL_ERROR; - } - subscribeMap[device][queryId] = SubscribeStatus::ACTIVE; - return E_OK; -} - -void SingleVerSubscribeManager::DeleteSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s in map", STR_MASK(device)); - return; - } - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); - return; - } - SubscribeStatus queryStatus = subscribeMap[device][queryId]; - // not permit to delete the query when something wrong this time,because it is subscribed successfully last time - if (queryStatus == SubscribeStatus::ACTIVE) { - LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s is active, no need to del", - STR_MASK(device), STR_MASK(queryId)); - return; - } - subscribeMap[device].erase(queryId); - auto iter = subscribedTotalMap.find(queryId); - if (iter == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); - return; - } - iter->second.second--; - if (iter->second.second <= 0) { - LOGI("[SingleVerSubscribeManager] del queryId=%s from SubscribeTotalMap", STR_MASK(queryId)); - subscribedTotalMap.erase(queryId); - } - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribeMap success", - STR_MASK(device), STR_MASK(queryId)); -} - -void SingleVerSubscribeManager::RemoveSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - auto iter = subscribeMap.find(device); - if (iter == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); - return; - } - if (iter->second.find(queryId) == subscribeMap[device].end()) { - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s not in SubscribedMap", STR_MASK(device), STR_MASK(queryId)); - return; - } - iter->second.erase(queryId); - auto allIter = subscribedTotalMap.find(queryId); - if (allIter == subscribedTotalMap.end()) { - LOGI("[SingleVerSubscribeManager] queryId=%s not in TotalMap", STR_MASK(queryId)); - return; - } - allIter->second.second--; - if (allIter->second.second <= 0) { - subscribedTotalMap.erase(queryId); - LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryId)); - } - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribedMap success", - STR_MASK(device), STR_MASK(queryId)); -} - -void SingleVerSubscribeManager::GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, - const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const -{ - auto iter = subscribeMap.find(device); - if (iter == subscribeMap.end()) { - LOGD("[SingleVerSubscribeManager] dev=%s not in localSubscribeMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : iter->second) { - auto iterTmp = subscribedTotalMap.find(queryInfo.first); - if (iterTmp == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryInfo.first)); - continue; - } - subscribeQueries.push_back(iterTmp->second.first); - } -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h deleted file mode 100644 index c152e0f29..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SINGLE_VER_SUBSCRIBE_MANAGER_H -#define SINGLE_VER_SUBSCRIBE_MANAGER_H - -#include -#include -#include "query_sync_object.h" - -namespace DistributedDB { -enum class SubscribeStatus { - NOT_ACTIVE = 0, - ACTIVE = 1, -}; - -using SubscribeMap = std::map>; -using SubscribedTotalMap = std::map>; - -class SingleVerSubscribeManager { -public: - SingleVerSubscribeManager() = default; - ~SingleVerSubscribeManager() {}; - - DISABLE_COPY_ASSIGN_MOVE(SingleVerSubscribeManager); - - // clear remoteSubscribeMap_[device] list when remote db is closed or dev offline. - void ClearRemoteSubscribeQuery(const std::string &device); - - // clear localSubscribeMap_[device] list when device is offline. - void ClearLocalSubscribeQuery(const std::string &device); - - void ClearAllRemoteQuery(); - - // add query when receive subscribe command - int ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // active query to ACTIVE when send ack ok - int ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // reserve query when user call SubscribeRemoteQuery, status set to NOT_ACTIVE - int ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // active query to ACTIVE when receive ack ok - int ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // delete local subscribe query when recv wrong errCode, only not_active status allowed to del - void DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // delete remote subscribe query when send msg failed, only not_active status allowed to del - void DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // put subscribe queries into unfinished map when remote db online - void PutLocalUnFiniedSubQueries(const std::string &device, std::vector &subscribeQueries); - - // get all device unFinished subscribe queries which triggered by auto subscribe and need retry subscribe - void GetAllUnFinishSubQueries(std::map> &allSyncQueries) const; - - // remove query when receive unsubscribe command - void RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // remove query when user call UnSubscribeRemoteQuery - void RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // get device active subscribeQueries from localSubscribeMap_ - void GetLocalSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; - - // get device remote queryId from remoteSubscribedMap_ while data change - void GetRemoteSubscribeQueryIds(const std::string &device, std::vector &subscribeQueryIds) const; - // get device remote subscribeQueries from remoteSubscribedMap_ while data change - void GetRemoteSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; - - bool IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const; - - int LocalSubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const; -private: - void ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - int ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - int ActiveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void DeleteSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void RemoveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, - const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const; - - mutable std::shared_mutex localSubscribeMapLock_; - // subscribe sponsor, key: device, value: pair map - // status 0: active, 1: not active - SubscribeMap localSubscribeMap_; - - // used retry subscribe in db open scene, key: device value: set - std::map> unFinishedLocalAutoSubMap_; - - // subscribe sponsor total query info, key:queryId, value: - // while use_num is 0, delete item from the map - SubscribedTotalMap localSubscribeTotalMap_; - - mutable std::shared_mutex remoteSubscribedMapLock_; - // subscribed, key: device, value: pair map - // status 0: active, 1: not active - SubscribeMap remoteSubscribedMap_; - - // subscribed total query info, key:queryId, value: - // while use_num is 0, delete item from the map - SubscribedTotalMap remoteSubscribedTotalMap_; -}; -} // namespace DistributedDB - -#endif // SINGLE_VER_SUBSCRIBE_MANAGER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index c181d617b..22581c572 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -130,6 +130,7 @@ ohos_source_set("src_file") { "../interfaces/src/relational/relational_store_sqlite_ext.cpp", "../interfaces/src/relational/runtime_config.cpp", "../storage/src/data_transformer.cpp", + "../storage/src/db_properties.cpp", "../storage/src/default_factory.cpp", "../storage/src/generic_kvdb.cpp", "../storage/src/generic_kvdb_connection.cpp", @@ -163,6 +164,7 @@ ohos_source_set("src_file") { "../storage/src/relational_store_connection.cpp", "../storage/src/relational_store_instance.cpp", "../storage/src/relational_sync_able_storage.cpp", + "../storage/src/relationaldb_properties.cpp", "../storage/src/result_entries_window.cpp", "../storage/src/single_ver_natural_store_commit_notify_data.cpp", "../storage/src/sqlite/query_object.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp deleted file mode 100644 index 25749ee91..000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "virtual_device.h" - -#include -#include - -#include "device_manager.h" -#include "log_print.h" -#include "multi_ver_sync_state_machine.h" -#include "subscribe_manager.h" -#include "single_ver_sync_state_machine.h" -#include "sync_types.h" -#include "virtual_communicator.h" -#include "virtual_communicator_aggregator.h" -#include "virtual_multi_ver_sync_db_interface.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -VirtualDevice::VirtualDevice(const std::string &deviceId) - : communicateHandle_(nullptr), - communicatorAggregator_(nullptr), - storage_(nullptr), - metadata_(nullptr), - deviceId_(deviceId), - remoteDeviceId_("real_device"), - context_(nullptr) -{ -} - -VirtualDevice::~VirtualDevice() -{ - std::mutex cvMutex; - std::condition_variable cv; - bool finished = false; - Offline(); - - if (communicateHandle_ != nullptr) { - communicateHandle_->RegOnMessageCallback(nullptr, nullptr); - communicatorAggregator_->ReleaseCommunicator(communicateHandle_); - communicateHandle_ = nullptr; - } - communicatorAggregator_ = nullptr; - - if (context_ != nullptr) { - IKvDBSyncInterface *storage = storage_; - context_->OnLastRef([storage, &cv, &cvMutex, &finished]() { - if (storage != nullptr) { - delete storage; - } - { - std::lock_guard lock(cvMutex); - finished = true; - } - cv.notify_one(); - }); - RefObject::KillAndDecObjRef(context_); - std::unique_lock lock(cvMutex); - cv.wait(lock, [&finished] {return finished;}); - } else { - delete storage_; - } - context_ = nullptr; - metadata_ = nullptr; - storage_ = nullptr; - subManager_ = nullptr; -} - -int VirtualDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface) -{ - if ((communicatorAggregator == nullptr) || (syncInterface == nullptr)) { - return -E_INVALID_ARGS; - } - - communicatorAggregator_ = communicatorAggregator; - int errCode = E_OK; - communicateHandle_ = communicatorAggregator_->AllocCommunicator(deviceId_, errCode); - if (communicateHandle_ == nullptr) { - return errCode; - } - - storage_ = syncInterface; - metadata_ = std::make_shared(); - if (metadata_->Initialize(storage_) != E_OK) { - LOGE("metadata_ init failed"); - return -E_NOT_SUPPORT; - } - if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { - context_ = new (std::nothrow) SingleVerSyncTaskContext; - subManager_ = std::make_shared(); - static_cast(context_)->SetSubscribeManager(subManager_); - } else { - context_ = new (std::nothrow) MultiVerSyncTaskContext; - } - if (context_ == nullptr) { - return -E_OUT_OF_MEMORY; - } - communicateHandle_->RegOnMessageCallback(std::bind(&VirtualDevice::MessageCallback, this, - std::placeholders::_1, std::placeholders::_2), []() {}); - context_->Initialize(remoteDeviceId_, storage_, metadata_, communicateHandle_); - context_->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); - context_->RegOnSyncTask(std::bind(&VirtualDevice::StartResponseTask, this)); - return E_OK; -} - -void VirtualDevice::SetDeviceId(const std::string &deviceId) -{ - deviceId_ = deviceId; -} - -std::string VirtualDevice::GetDeviceId() const -{ - return deviceId_; -} - -int VirtualDevice::GetData(const Key &key, VirtualDataItem &item) -{ - VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); - return syncAble->GetSyncData(key, item); -} - -int VirtualDevice::GetData(const Key &key, Value &value) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->GetData(key, value); -} - -int VirtualDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) -{ - VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); - LOGI("dev %s put data time %llu", deviceId_.c_str(), time); - return syncAble->PutData(key, value, time, flag); -} - -int VirtualDevice::PutData(const Key &key, const Value &value) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->PutData(key, value); -} - -int VirtualDevice::DeleteData(const Key &key) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->DeleteData(key); -} - -int VirtualDevice::StartTransaction() -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->StartTransaction(); -} - -int VirtualDevice::Commit() -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->Commit(); -} - -int VirtualDevice::MessageCallback(const std::string &deviceId, Message *inMsg) -{ - if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { - if (onRemoteDataChanged_) { - onRemoteDataChanged_(deviceId); - delete inMsg; - inMsg = nullptr; - return E_OK; - } - delete inMsg; - inMsg = nullptr; - return -E_INVALID_ARGS; - } - - LOGD("[VirtualDevice] onMessag, src %s", deviceId.c_str()); - RefObject::IncObjRef(context_); - RefObject::IncObjRef(communicateHandle_); - SyncTaskContext *context = context_; - ICommunicator *communicateHandle = communicateHandle_; - std::thread thread([context, communicateHandle, inMsg]() { - int errCode = context->ReceiveMessageCallback(inMsg); - if (errCode != -E_NOT_NEED_DELETE_MSG) { - delete inMsg; - } - RefObject::DecObjRef(context); - RefObject::DecObjRef(communicateHandle); - }); - thread.detach(); - return E_OK; -} - -void VirtualDevice::OnRemoteDataChanged(const std::function &callback) -{ - onRemoteDataChanged_ = callback; -} - -void VirtualDevice::Online() -{ - static_cast(communicateHandle_)->Enable(); - communicatorAggregator_->OnlineDevice(deviceId_); -} - -void VirtualDevice::Offline() -{ - static_cast(communicateHandle_)->Disable(); - communicatorAggregator_->OfflineDevice(deviceId_); -} - -int VirtualDevice::StartResponseTask() -{ - LOGD("[VirtualDevice] StartResponseTask"); - RefObject::AutoLock lockGuard(context_); - int status = context_->GetTaskExecStatus(); - if ((status == SyncTaskContext::RUNNING) || context_->IsKilled()) { - LOGD("[VirtualDevice] StartResponseTask status:%d", status); - return -E_NOT_SUPPORT; - } - if (context_->IsTargetQueueEmpty()) { - LOGD("[VirtualDevice] StartResponseTask IsTargetQueueEmpty is empty"); - return E_OK; - } - context_->SetTaskExecStatus(ISyncTaskContext::RUNNING); - context_->MoveToNextTarget(); - LOGI("[VirtualDevice] machine StartSync"); - context_->UnlockObj(); - int errCode = context_->StartStateMachine(); - context_->LockObj(); - if (errCode != E_OK) { - LOGE("[VirtualDevice] machine StartSync failed"); - context_->SetOperationStatus(SyncOperation::OP_FAILED); - } - return errCode; -} - -TimeOffset VirtualDevice::GetLocalTimeOffset() const -{ - return metadata_->GetLocalTimeOffset(); -} - -void VirtualDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) -{ - VirtualSingleVerSyncDBInterface *syncInterface = static_cast(storage_); - syncInterface->SetSaveDataDelayTime(milliDelayTime); -} - -int VirtualDevice::Sync(SyncMode mode, bool wait) -{ - auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} - -int VirtualDevice::Subscribe(QuerySyncObject query, bool wait, int id) -{ - auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, SUBSCRIBE_QUERY, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - operation->SetQuery(query); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} - -int VirtualDevice::UnSubscribe(QuerySyncObject query, bool wait, int id) -{ - auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, UNSUBSCRIBE_QUERY, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - operation->SetQuery(query); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h deleted file mode 100644 index 5132c0dd6..000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VIRTUAL_DEVICE_H -#define VIRTUAL_DEVICE_H - -#include "icommunicator.h" - -#include "ikvdb_sync_interface.h" -#include "single_ver_sync_task_context.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -class VirtualCommunicatorAggregator; - -class VirtualDevice { -public: - explicit VirtualDevice(const std::string &deviceId); - ~VirtualDevice(); - - int Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface); - void SetDeviceId(const std::string &deviceId); - std::string GetDeviceId() const; - int GetData(const Key &key, VirtualDataItem &item); - int GetData(const Key &key, Value &value); - int PutData(const Key &key, const Value &value, const TimeStamp &time, int flag); - int PutData(const Key &key, const Value &value); - int DeleteData(const Key &key); - int StartTransaction(); - int Commit(); - int MessageCallback(const std::string &deviceId, Message *inMsg); - void OnRemoteDataChanged(const std::function &callback); - void Online(); - void Offline(); - int StartResponseTask(); - TimeOffset GetLocalTimeOffset() const; - void SetSaveDataDelayTime(uint64_t milliDelayTime); - int Sync(SyncMode mode, bool wait); - int Subscribe(QuerySyncObject query, bool wait, int id); - int UnSubscribe(QuerySyncObject query, bool wait, int id); - -private: - ICommunicator *communicateHandle_; - VirtualCommunicatorAggregator *communicatorAggregator_; - IKvDBSyncInterface *storage_; - std::shared_ptr metadata_; - std::string deviceId_; - std::string remoteDeviceId_; - SyncTaskContext *context_; - std::function onRemoteDataChanged_; - std::shared_ptr subManager_; -}; -} // namespace DistributedDB - -#endif // VIRTUAL_DEVICE_H -- Gitee From 31df7040af8f39bda3b3f0f1b9856b416468edc4 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 27 Dec 2021 10:11:22 +0800 Subject: [PATCH 15/19] Remove TODOs Signed-off-by: lianhuix --- .../relational/relational_schema_object.cpp | 3 +-- .../relational/relational_store_manager.h | 4 ++-- .../include/relational_db_sync_interface.h | 2 -- .../relational/sqlite_relational_store.cpp | 19 +------------------ .../sqlite_relational_store_connection.cpp | 1 - ...single_ver_relational_storage_executor.cpp | 13 +------------ 6 files changed, 5 insertions(+), 37 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index e5f33dab1..cf784be59 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -296,7 +296,6 @@ JsonObject TableInfo::ToJsonObject() const } for (const auto& it : uniqueDefines_) { jsonField.stringValue = VectorJoin(it, ','); - // TODO: add unique to tableJson } return tableJson; } @@ -646,7 +645,7 @@ int RelationalSchemaObject::ParseCheckTableFieldInfo(const JsonObject &inJsonObj } else if (errCode != -E_NOT_FOUND) { return errCode; } - // TODO: need cid or not? + return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 71fc2e44c..74ccc9423 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -39,9 +39,9 @@ public: DB_API DBStatus OpenStore(const std::string &path, const std::string &storeId, const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate); - DB_API DBStatus CloseStore(RelationalStoreDelegate *store); // TODO: move interface to delegate + DB_API DBStatus CloseStore(RelationalStoreDelegate *store); - DB_API DBStatus DeleteStore(const std::string &path); // TODO: remove interface + DB_API DBStatus DeleteStore(const std::string &path); private: void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h index eff6dbe54..94749dba3 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h @@ -38,8 +38,6 @@ public: virtual int LocalDataChanged(int notifyEvent, std::vector &queryObj) = 0; virtual int SchemaChanged(int notifyEvent) = 0; - - // TODO: create device table for each distributed table. }; } #endif // RELATIONAL_STORE diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index d882fe91f..dad1ed2a3 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -80,7 +80,6 @@ int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBP void SQLiteRelationalStore::ReleaseResources() { - // TODO: Lock if (sqliteStorageEngine_ != nullptr) { sqliteStorageEngine_->ClearEnginePasswd(); (void)StorageEngineManager::ReleaseStorageEngine(sqliteStorageEngine_); @@ -144,28 +143,12 @@ int SQLiteRelationalStore::SaveSchemaToMeta() int SQLiteRelationalStore::SaveLogTableVersionToMeta() { - // TODO: save log table version into meta data LOGD("save log table version to meta table, key: %s, val: %s", LOG_TABLE_VERSION_KEY, LOG_TABLE_VERSION_1); return E_OK; } int SQLiteRelationalStore::CleanDistributedDeviceTable() { - // TODO: clean the device table which is no longer in schema - std::lock_guard lock(schemaMutex_); - RelationalSchemaObject schema = properties_.GetSchema(); - for (const auto &table : schema.GetTables()) { - std::string tableName = table.first; - LOGD("Get schema %s.", tableName.c_str()); - } - - int errCode = E_OK; - auto *handle = GetHandle(true, errCode); - if (handle == nullptr) { - return errCode; - } - // TODO: Get device table names, and clean - ReleaseHandle(handle); return E_OK; } @@ -193,7 +176,7 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); if (storageEngine_ == nullptr) { - LOGE("[RelationalStore][Open] Create syncable storage failed"); // TODO: + LOGE("[RelationalStore][Open] Create syncable storage failed"); errCode = -E_OUT_OF_MEMORY; break; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index bc86d438c..160a64d91 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -179,7 +179,6 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) syncParam.relationOnComplete = info.onComplete; syncParam.syncQuery = QuerySyncObject(info.query); syncParam.onFinalize = [this]() { DecObjRef(this); }; - // TODO: check if table permit sync or not int errCode = store->Sync(syncParam); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 3c029f027..892527368 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -46,17 +46,6 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: LOGE("[CreateDistributedTable] analysis table schema failed"); return errCode; } - - // TODO: check if compatible upgrade or not - - // TODO: add not permit sync flag if not compatible upgrade - - // TODO: add analysed table to schema. - - // TODO: reset permit sync flag - - // TODO: upgrade device table - // create log table errCode = SQLiteUtils::CreateRelationalLogTable(dbHandle_, tableName); if (errCode != E_OK) { @@ -109,7 +98,7 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) { - // TODO: Get table info from schema + int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); -- Gitee From ed9d2825fbaa4ea3e28a09fc80285a431f39cc50 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 27 Dec 2021 10:35:54 +0800 Subject: [PATCH 16/19] Change GetTableCount to GetTableRecordCount Signed-off-by: lianhuix --- .../storage/src/sqlite/relational/sqlite_relational_store.cpp | 2 +- .../sqlite/sqlite_single_ver_relational_storage_executor.cpp | 2 +- .../libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp | 2 +- .../libs/distributeddb/storage/src/sqlite/sqlite_utils.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index dad1ed2a3..fe64fc818 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -107,7 +107,7 @@ int SQLiteRelationalStore::GetSchemaFromMeta() const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); Value schemaVal; int errCode = storageEngine_->GetMetaData(schemaKey, schemaVal); - if (errCode != E_OK && errCode != -E_NOT_FOUND ) { + if (errCode != E_OK && errCode != -E_NOT_FOUND) { LOGE("Get relationale schema from meta table failed. %d", errCode); return errCode; } else if (errCode == -E_NOT_FOUND) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 892527368..9fa4e7183 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -30,7 +30,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: } int historyDataCnt = 0; - int errCode = SQLiteUtils::GetTableCount(dbHandle_, tableName, historyDataCnt); + int errCode = SQLiteUtils::GetTableRecordCount(dbHandle_, tableName, historyDataCnt); if (errCode != E_OK) { LOGE("[CreateDistributedTable] Get the number of table [%s] rows failed. %d", tableName.c_str(), errCode); return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 8d2b3ad1a..906d2952c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1975,7 +1975,7 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) return E_OK; } -int SQLiteUtils::GetTableCount(sqlite3 *db, const std::string &tableName, int &count) +int SQLiteUtils::GetTableRecordCount(sqlite3 *db, const std::string &tableName, int &count) { if (db == nullptr) { return -E_INVALID_ARGS; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 8452f705a..623335886 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -178,7 +178,7 @@ public: static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); - static int GetTableCount(sqlite3 *db, const std::string &tableName, int &count); + static int GetTableRecordCount(sqlite3 *db, const std::string &tableName, int &count); private: -- Gitee From 3efd31fbeb92d20efc0a6724b34d2d30187fb80b Mon Sep 17 00:00:00 2001 From: xuhongkang Date: Mon, 27 Dec 2021 15:42:53 +0800 Subject: [PATCH 17/19] add relational testCase Signed-off-by: xuhongkang --- .../include/distributed_rdb_tools.h | 100 +++++ .../src/distributed_rdb_tools.cpp | 256 +++++++++++++ .../src/distributed_rdb_exception_test.cpp | 353 ++++++++++++++++++ 3 files changed, 709 insertions(+) create mode 100644 services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h create mode 100644 services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp create mode 100644 services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h new file mode 100644 index 000000000..15ce29fd9 --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H +#define DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H +#include +#include + +#include "relational_store_delegate.h" +#include "relational_store_manager.h" +#include "types.h" +#include "distributed_test_sysinfo.h" +#include "distributeddb_data_generator.h" +#include "log_print.h" +#ifdef TESTCASES_USING_GTEST +#define HWTEST_F(test_case_name, test_name, level) TEST_F(test_case_name, test_name) +#endif + +struct RdbParameters { + std::string path; + std::string storeId; + std::string appId; + std::string userId; + RdbParameters(str::string pathStr, std::string storeIdStr, std::string appIdStr, std::string userIdStr) + : pathStr(pathStr), storeId(storeIdStr), appId(appIdStr), userId(userIdStr) + { + } +}; + +const static std::string TAG = "DistributedRdbTestTools"; + +const std::string NORMAL_PATH = "/data/test/"; +const std::string NON_EXISTENT_PATH = "/data/test/nonExistent_rdb/"; +const std::string UNREADABLE_PATH = "/data/test/unreadable_rdb/"; +const std::string UNWRITABLE_PATH = "/data/test/unwritable_rdb/"; + +const std::string NULL_STOREID = ""; +const std::string ILLEGAL_STOREID = "rdb_$%#@~%"; +const std::string MODE_STOREID = "rdb_mode"; +const std::string FULL_STOREID = "rdb_full"; +const std::string SUPER_STOREID = "rdb_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +const std::string NON_EXISTENT_TABLE = "non_table"; +const std::string KEYWORD_START_TABLE = "naturalbase_rdb_a"; + +const std::string NORMAL_TABLE = "NORMAL_RDB"; +const std::string CONSTRAINT_TABLE = "CONSTRAINT_RDB"; + +const static RdbParameters g_rdbParameter1(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter2(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter3(NORMAL_PATH, ILLEGAL_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter4(NORMAL_PATH, MODE_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter5(NORMAL_PATH, SUPER_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter6(NON_EXISTENT_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter7(UNREADABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter8(UNWRITABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter9(NORMAL_PATH, FULL_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +class DistributedRdbTestTools final { +public: + DistributedRdbTestTools() {} + ~DistributedRdbTestTools() {} + // Relational Database OpenStore And CreateDistributeTable + static DistributedDB::DBStatus GetOpenStoreStatus(const RelatetionalStoreManager *&manager, RelatetionalStoreDelegate *&delegate, + const RdbParameters ¶m); + static DistributedDB::DBStatus GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + const std::string &tableName); + static bool CloseStore(const DistributedDB::RelatetionalStoreDelegate *&delegate); +private: +} +#endif // DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H \ No newline at end of file diff --git a/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp new file mode 100644 index 000000000..7ef56037d --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "distributed_rdb_tools.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef USE_SQLITE_SYMBOLS +#include "sqlite3.h" +#else +#include "sqlite3sym.h" +#endif +#include "distributeddb_data_generator.h" +#include "distributed_test_sysinfo.h" +#include "platform_specific.h" +#include "securec.h" + +using namespace std; +using namespace DistributedDB; +using namespace DistributedDBDataGenerator; + +namespace { + std::string gtableNameList[33]; + const char *SQL_CREATE_NORMAL_TABLE = "CREATE TABLE IF NOT EXISTS NORMAL_RDB(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "name VARCHAR(100) NOT NULL DEFAULT \"rdb\");"; + + const char *SQL_CREATE_CONSTRAINT_TABLE = "CREATE TABLE IF NOT EXISTS CONSTRAINT_RDB(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "f_id INT," \ + "age INT NOT NULL UNIQUE," \ + "name VARCHAR(100) NOT NULL DEFAULT \"rdb\"," \ + "address CHAR(50) COLLATE NOCASE," \ + "identity INT NOT NULL, CHECK(identity > 10000) PRIMARY KEY(id, identity)," \ + "FOREGIN key(f_id) references NORMAL_RDB(id));"; + + const char *SQL_CREATE_TRIGGER = "CREATE TRIGGER IF NOT EXISTS insertTrigger " \ + "AFTER INSERT ON CONSTRAINT_RDB " \ + "FOR EACH ROW " \ + "BEGIN " \ + "update NORMAL_RDB set name = \"name_001\" " \ + "END"; + + const char *SQL_INSERT_NORMAL_TABLE = "INSERT INTO NORMAL_RDB (id,name)" \ + "VALUES (1, \'rdb_001\'), (2, \'rdb_002\'), (3, \'rdb_003\'), (4, \'rdb_004\'), (5, \'rdb_005\');"; + + + const char *SQL_ADD_FIELD_TABLE1 = "ALTER TABLE RDB_1 ADD COLUMN add_id INI"; + + const char *SQL_ADD_INDEX_TABLE2 = "CREATE INDEX name_index RDB_2 (name)"; + + const char *SQL_DROP_TABLE3 = "DROP TABLE RDB_3"; + + const char *SQL_DROP_CREATE_TABLE3 = "CREATE TABLE IF NOT EXISTS RDB_3(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT);"; + + const char *SQL_DROP_TABLE4 = "DROP TABLE RDB_4"; + + const char *SQL_DROP_CREATE_TABLE4 = "CREATE TABLE IF NOT EXISTS RDB_4(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, name CHAR(100));"; + + const char *SQL_JOURNAL_MODE = "PRAGMA journal_mode = DELETE;"; + + const char *SQL_SYNCHRONOUS_MODE = "PRAGMA synchronous = FULL;"; + + const char *SQL_INSERT_RDB5_TABLE = "INSERT INTO RDB_5 (id,name,age)" \ + "VALUES (1, \'rdb_005\', \'name_rdb5\');"; +} + +DBStatus DistributedRdbTestTools::GetOpenStoreStatus(const RelatetionalStoreManager *&manager, RelatetionalStoreDelegate *&delegate, + const RdbParameters ¶m) +{ + if (manager == nullptr) { + MST_LOG("%s GetRdbStore failed! manager nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + if (delegate != nullptr) { + delegate = nullptr; + } + DBStatus status = manager->OpenStore(param.path, param.storeId, delegate); + if (delegate == nullptr) { + MST_LOG("%s GetRdbStore failed! delegate nullptr.", TAG.c_str()); + } + retrun status; +} + +DBStatus DistributedRdbTestTools::GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + const std::string &tableName) +{ + if (delegate == nullptr) { + MST_LOG("%s CreateDistributedTable failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + DBStatus status = delegate->CreateDistributedTable(tableName); + retrun status; +} + +DBStatus DistributedRdbTestTools::CloseStore(const RelatetionalStoreDelegate *&delegate) +{ + if (delegate == nullptr) { + MST_LOG("%s CloseStore failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + DBStatus status = delegate->CloseStore(); + return status; +} + +bool InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m) +{ + const std::string dbName = param.path + param.storeId + ".db"; + DBStributedDB::OS::RemoveFile(dbName); + int errCode = sqlite3_open(dbName, &db); + if (errCode1 != SQLITE_OK) { + MST_LOG("sqlite3_open Failed!"); + return false; + } + errCode = sqlite3_exec(db, "PRAGMA journal_mode = WAL;", 0, 0, 0); + if (errCode != SQLITE_OK) { + MST_LOG("PRAGMA journal_mode = WAL Failed!"); + return false; + } + return true; +} + +bool InitTableDataAndTRIGGER(const sqlite3 *&db) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + char *errMsg = nullptr; + int errCode1 = sqlite3_exec(db, SQL_CREATE_NORMAL_TABLE, nullptr, nullptr, &errMsg); + if (errCode1 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_CREATE_NORMAL_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + int errCode2 = sqlite3_exec(db, SQL_CREATE_CONSTRAINT_TABLE, nullptr, nullptr, &errMsg); + if (errCode2 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_CREATE_CONSTRAINT_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + int errCode3 = sqlite3_exec(db, SQL_CREATE_TRIGGER, nullptr, nullptr, &errMsg); + if (errCode3 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_CREATE_TRIGGER Failed(%s)", errMsg.c_str()); + return false; + } + + int errCode4 = sqlite3_exec(db, SQL_INSERT_NORMAL_TABLE, nullptr, nullptr, &errMsg); + if (errCode4 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_INSERT_NORMAL_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + for (int i = 1; i <= 33; i++) { + std::string str_0 = "RDB_" + i; + std::string str_1 = "CREATE TABLE IF NOT EXISTS " + std::string str_2 = "( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "name VARCHAR(100) NOT NULL, age VARCHAR(100) NOT NULL);"; + sprintf(str1,"%s%s",str0,str2); + + char *errMsg = nullptr; + int errCode1 = sqlite3_exec(db, str_1.c_str(), nullptr, nullptr, &errMsg); + if (errCode1 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec PresetTables33 Failed(%s)", errMsg.c_str()); + return false; + } + gtableNameList[i-1] = str_0; + } + return true; +} + +bool alterTableAttributes(const sqlite3 *&db) { + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + char *errMsg = nullptr; + int errCode = sqlite3_exec(db, SQL_ADD_FIELD_TABLE1, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_FIELD_TABLE1 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_ADD_INDEX_TABLE2, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_INDEX_TABLE2 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_TABLE3, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_TABLE3 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_CREATE_TABLE3, nullptr, nullptr, &errMsg); + if (errCode1 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_INDEX_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_TABLE4, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_TABLE4 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_CREATE_TABLE4, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_CREATE_TABLE4 Failed(%s)", errMsg.c_str()); + return false; + } + return true; +} + + +bool Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + int errCode = sqlite3_exec(db, sql_name, 0, 0, 0); + if (errCode != SQLITE_OK) { + MST_LOG("%s Failed!", sql_name); + return false; + } + return true; +} + +void CloseSqlite3Store(sqlite3 *&db) +{ + if (db != nullptr) { + sqlite3_close(db); + db = nullptr; + } +} \ No newline at end of file diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp new file mode 100644 index 000000000..3718f5378 --- /dev/null +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "relational_store_delegate.h" +#include "relational_store_manager.h" +#include "distributed_rdb_tools.h" + +using namespace std; +using namespace testing; +#if defined TESTCASES_USING_GTEST_EXT +using namespace testing::ext; +#endif +using namespace DistributedDB; +using namespace DistributedDBDataGenerator; + +// set sqlite3 rdb_A and create 33 dataTables + +namespace DistributedRelatinalDBExceptionOperation { +sqlite3 *gdb = nullptr; + +RelatetionalStoreManager *g_manager = nullptr; + +RelatetionalStoreDelegate *g_delegate = nullptr; + +RelatetionalStoreDelegate *g_delegate1 = nullptr; + +class DistributedRelatinalDBExceptionOperationTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +private: +}; + +void DistributedRelatinalDBExceptionOperationTest::SetUpTestCase(void) +{ + bool init1 = InitSqlite3Store(gdb, g_rdbParameter1); + ASSERT_EQ(init1, true); + bool init2 = InitTableDataAndTRIGGER(gdb); + ASSERT_EQ(init2, true); + g_manager = new (std::nothrow) RelatetionalStoreManager(g_rdbParameter1.appId, g_rdbParameter1.userId); + DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate, g_rdbParameter1); + ASSERT_EQ(status1, DBStatus::OK); +} + +void DistributedRelatinalDBExceptionOperationTest::TearDownTestCase(void) +{ + CloseSqlite3Store(gdb); +} + +void DistributedRelatinalDBExceptionOperationTest::SetUp(void) +{ + UnitTest *test = UnitTest::GetInstance(); + ASSERT_NE(test, nullptr); + const TestInfo *testinfo = test->current_test_info(); + ASSERT_NE(testinfo, nullptr); + string testCaseName = string(testinfo->name()); + MST_LOG("[SetUp] test case %s is start to run", testCaseName.c_str()); +} + +void DistributedRelatinalDBExceptionOperationTest::TearDown(void) +{ + MST_LOG("TearDownTestCase after case."); +} + +/** + * @tc.name: dbPathException001 + * @tc.desc: db path does not exist, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException001, TestSize.Level4) +{ + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter6); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: dbPathException002 + * @tc.desc: db path unreadable, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException002, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = InitSqlite3Store(db, g_rdbParameter7); + ASSERT_EQ(init1, true); + bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IXUSR); + ASSERT_EQ(bpermission, true); + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter7); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); + bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + CloseSqlite3Store(db); +} + +/** + * @tc.name: dbPathException003 + * @tc.desc: db path unwritdable, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException003, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = InitSqlite3Store(db, g_rdbParameter8); + ASSERT_EQ(init1, true); + bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter8); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); + bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IWUSR | S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + CloseSqlite3Store(db); +} + +/** + * @tc.name: storeIdException001 + * @tc.desc: storeId is empty, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException001, TestSize.Level4) +{ + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter2); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: storeIdException002 + * @tc.desc: storeId value larger 128, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException002, TestSize.Level4) +{ + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter5); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: storeIdException003 + * @tc.desc: storeId is illegal, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException003, TestSize.Level4) +{ + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter3); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: dbModeException001 + * @tc.desc: db mode isn't wal or SYNCHRONOUS full, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbModeException001, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = InitSqlite3Store(db, g_rdbParameter4); + ASSERT_EQ(init1, true); + bool res1 = Sqlite3ExecOpration(db, SQL_JOURNAL_MODE); + ASSERT_EQ(res1, true); + DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + ASSERT_EQ(status1, DBStatus::NOT_SUPPORT); + CloseSqlite3Store(db); + + bool init2 = InitSqlite3Store(db, g_rdbParameter9); + ASSERT_EQ(init2, true); + bool res2 = Sqlite3ExecOpration(db, SQL_SYNCHRONOUS_MODE); + ASSERT_EQ(res2, true); + DBStatus status2 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); + CloseSqlite3Store(db); +} + +/** + * @tc.name: tableException001 + * @tc.desc: db hasn't non_table, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, NON_EXISTENT_TABLE); + ASSERT_EQ(status2, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: tableException002 + * @tc.desc: db create natrualbase_rdb_A relatinalTable, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException002, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, KEYWORD_START_TABLE); + ASSERT_EQ(status2, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: tableException003 + * @tc.desc: db NORMAL_RDB table has data, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException003, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, NORMAL_TABLE); + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); +} + +/** + * @tc.name: tableMulCreate001 + * @tc.desc: db create RDB_1 RelatinalTable, multiple createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableMulCreate001, TestSize.Level4) +{ + for (int i = 1; i <= 5; i++) { + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[0]); + ASSERT_EQ(status2, DBStatus::OK); + } +} + +/** + * @tc.name: mulTableCreate001 + * @tc.desc: dbA createDistributedTable 32 tables, successful + * then dbA createDistributedTable the 33'th table failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, mulTableCreate001, TestSize.Level4) +{ + for (auto tableName: gtableNameList) { + DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, tableName); + if (tableName == gtableNameList[32]) { + ASSERT_EQ(status, DBStatus::NOT_SUPPORT); + } else { + ASSERT_EQ(status, DBStatus::OK); + } + } +} + +/** + * @tc.name: mulTableCreate002 + * @tc.desc: dbA createDistributedTable RDB_1...20 tables, And + * dbB createDistributedTable RDB_21...32 tables,successful + * then dbA createDistributedTable the 33'th table failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, mulTableCreate002, TestSize.Level4) +{ + RelatetionalStoreDelegate *delegateA = nullptr; + RelatetionalStoreDelegate *delegateB = nullptr; + DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, delegateA, g_rdbParameter1); + ASSERT_EQ(status1, DBStatus::OK); + DBStatus status2 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, delegateB, g_rdbParameter1); + ASSERT_EQ(status2, DBStatus::OK); + for (int index = 0; index < gtableNameList.size(); index++) { + if (index < 20) { + DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(delegateA, gtableNameList[index]); + ASSERT_EQ(status2, DBStatus::OK); + } else if (index >= 20 && index < 32) + { + DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(delegateB, gtableNameList[index]); + ASSERT_EQ(status2, DBStatus::OK); + } else { + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); + } + } +} + +/** + * @tc.name: relatinalTable001 + * @tc.desc: db has RDB_1 relatinalTable, then alter RDB_1 and createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, relatinalTable001, TestSize.Level4) +{ + bool result = alterTableAttributes(gdb); + ASSERT_EQ(result, true); + for (int i = 0; i < 4; i++) { + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[i]); + ASSERT_EQ(status2, DBStatus::OK); + } +} + +/** + * @tc.name: constraintTable001 + * @tc.desc: db has constraint table, then createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, constraintTable001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, CONSTRAINT_TABLE); + ASSERT_EQ(status2, DBStatus::OK); +} + +/** + * @tc.name: constraintTable001 + * @tc.desc: db has relatinalTable RDB_5, then add data in local RDB_5 and createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, constraintTable001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + ASSERT_EQ(status2, DBStatus::OK); + bool res1 = Sqlite3ExecOpration(db, SQL_INSERT_RDB5_TABLE); + ASSERT_EQ(res1, true); + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + ASSERT_EQ(status2, DBStatus::OK); +} +} -- Gitee From c1c7e4db33b668609d98220048ae4146e988ce28 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Sat, 25 Dec 2021 10:41:58 +0800 Subject: [PATCH 18/19] Some UT. Add new file to BUILD.gn. Change the prefix of table. Signed-off-by: lidwchn --- .../libs/distributeddb/BUILD.gn | 1 + .../common/include/db_constant.h | 4 + .../distributeddb/common/include/db_errno.h | 2 +- .../distributeddb/common/src/db_constant.cpp | 2 + .../relational/relational_sync_able_storage.h | 7 +- .../src/relational_sync_able_storage.cpp | 99 ++--- .../src/sqlite/sqlite_query_helper.cpp | 126 ++++++- .../storage/src/sqlite/sqlite_query_helper.h | 5 +- .../sqlite/sqlite_single_ver_continue_token.h | 5 - ...e_single_ver_relational_continue_token.cpp | 122 +++++++ ...ite_single_ver_relational_continue_token.h | 56 +++ ...single_ver_relational_storage_executor.cpp | 251 ++++--------- ...e_single_ver_relational_storage_executor.h | 16 +- .../sqlite_single_ver_storage_executor.cpp | 6 +- ...lite_single_ver_storage_executor_cache.cpp | 2 +- .../storage/src/sqlite/sqlite_utils.cpp | 19 +- .../libs/distributeddb/test/BUILD.gn | 1 + ...distributeddb_relational_get_data_test.cpp | 344 ++++++++++++++++++ 18 files changed, 777 insertions(+), 291 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h create mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 924097bb6..29f9c8c04 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -182,6 +182,7 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp", "storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "storage/src/sqlite/sqlite_single_ver_result_set.cpp", "storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 9c5d0e65b..fc0a4e6bb 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -112,6 +112,10 @@ public: static constexpr int DOUBLE_PRECISION = 15; static constexpr int MAX_DISTRIBUTED_TABLE_COUNT = 32; + + // For relational + static const std::string RELATIONAL_PREFIX; + static const std::string TIMESTAMP_ALIAS; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h index 5dd9a6e1f..f85331478 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h @@ -115,7 +115,7 @@ constexpr int E_SYSTEM_API_ADAPTER_CALL_FAILED = (E_BASE + 91); // Adapter call constexpr int E_NOT_NEED_DELETE_MSG = (E_BASE + 92); // not need delete msg, will be delete by sliding window receiver constexpr int E_SLIDING_WINDOW_SENDER_ERR = (E_BASE + 93); // sliding window sender err constexpr int E_SLIDING_WINDOW_RECEIVER_INVALID_MSG = (E_BASE + 94); // sliding window receiver invalid msg -constexpr int E_IGNOR_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. +constexpr int E_IGNORE_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. constexpr int E_FORBID_CACHEDB = (E_BASE + 96); // such after rekey can not check passwd due to file control. constexpr int E_INTERCEPT_DATA_FAIL = (E_BASE + 97); // Intercept push data failed. constexpr int E_INVALID_COMPRESS_ALGO = (E_BASE + 98); // The algo is defined, but there's no implement for the algo. diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp index 269b46436..eb90c8a4a 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp @@ -65,4 +65,6 @@ const std::string DBConstant::TRIGGER_REFERENCES_OLD = "OLD."; const std::string DBConstant::UPDATE_META_FUNC = "update_meta_within_trigger"; const std::string DBConstant::SYSTEM_TABLE_PREFIX = "naturalbase_rdb_"; +const std::string DBConstant::RELATIONAL_PREFIX = SYSTEM_TABLE_PREFIX + "aux_"; +const std::string DBConstant::TIMESTAMP_ALIAS = RELATIONAL_PREFIX + "timestamp"; } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 37ef57ada..05bb9b1bf 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -19,7 +19,7 @@ #include "relational_db_sync_interface.h" #include "sqlite_single_relational_storage_engine.h" -#include "sqlite_single_ver_continue_token.h" +#include "sqlite_single_ver_relational_continue_token.h" namespace DistributedDB { class RelationalSyncAbleStorage : public RelationalDBSyncInterface, public virtual RefObject { @@ -67,9 +67,6 @@ public: int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; - // Release the continue token of getting data. - void ReleaseContinueToken(ContinueToken &continueStmtToken) const override; - int PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, const DeviceID &deviceName) override; @@ -112,7 +109,7 @@ private: int SetMaxTimeStamp(TimeStamp timestamp); // get - int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerContinueToken *&continueStmtToken, + int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerRelationalContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const; // put diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index c8082bad6..3f18bdd04 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -196,15 +196,6 @@ const KvDBProperties &RelationalSyncAbleStorage::GetDbProperties() const return properties; } -static void ReleaseKvEntries(std::vector &entries) -{ - for (auto &itemEntry : entries) { - delete itemEntry; - itemEntry = nullptr; - } - entries.clear(); -} - static int GetKvEntriesByDataItems(std::vector &entries, std::vector &dataItems) { int errCode = E_OK; @@ -213,7 +204,7 @@ static int GetKvEntriesByDataItems(std::vector &entries, std if (entry == nullptr) { errCode = -E_OUT_OF_MEMORY; LOGE("GetKvEntries failed, errCode:%d", errCode); - ReleaseKvEntries(entries); + SingleVerKvEntry::Release(entries); break; } entry->SetEntryData(std::move(item)); @@ -237,7 +228,7 @@ static constexpr float QUERY_SYNC_THRESHOLD = 0.50; static bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, size_t appendLen) { - bool reachThreshold = false; + bool reachThreshold = (dataItems.size() >= dataSizeInfo.packetSize); for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { blockSize += GetDataItemSerialSize(dataItems[i], appendLen); reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); @@ -246,7 +237,7 @@ static bool CanHoldDeletedData(const std::vector &dataItems, const Dat } static void ProcessContinueTokenForQuerySync(const std::vector &dataItems, int &errCode, - SQLiteSingleVerContinueToken *&token) + SQLiteSingleVerRelationalContinueToken *&token) { if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. delete token; @@ -261,18 +252,7 @@ static void ProcessContinueTokenForQuerySync(const std::vector &dataIt token = nullptr; return; } - - TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; - if (nextBeginTime > INT64_MAX) { - nextBeginTime = INT64_MAX; - } - bool getDeleteData = ((dataItems.back().flag & DataItem::DELETE_FLAG) != 0); - if (getDeleteData) { - token->FinishGetQueryData(); - token->SetDeletedNextBeginTime("", nextBeginTime); - } else { - token->SetNextBeginTime("", nextBeginTime); - } + token->SetNextBeginTime(dataItems.back()); } /** @@ -280,7 +260,7 @@ static void ProcessContinueTokenForQuerySync(const std::vector &dataIt * If error happened, token will be deleted here. */ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &dataItems, - SQLiteSingleVerContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const + SQLiteSingleVerRelationalContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const { if (storageEngine_ == nullptr) { return -E_INVALID_DB; @@ -298,36 +278,18 @@ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &da goto ERROR; } - // Get query data. - if (!continueStmtToken->IsGetQueryDataFinished()) { - LOGD("[SingleVerNStore] Get query data between %llu and %llu.", continueStmtToken->GetQueryBeginTime(), - continueStmtToken->GetQueryEndTime()); - errCode = handle->GetSyncDataByQuery( - dataItems, Parcel::GetAppendedLen(), continueStmtToken->GetQuery(), dataSizeInfo, - // need modify - std::make_pair(continueStmtToken->GetQueryBeginTime(), continueStmtToken->GetQueryEndTime())); - } - - // Get query data finished. - if (errCode == E_OK || errCode == -E_FINISHED) { - // Clear query timeRange. - continueStmtToken->FinishGetQueryData(); - - // Get delete time next. - if (!continueStmtToken->IsGetDeletedDataFinished() && - CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())) { - LOGD("[SingleVerNStore] Get deleted data between %llu and %llu.", continueStmtToken->GetDeletedBeginTime(), - continueStmtToken->GetDeletedEndTime()); - errCode = handle->GetDeletedSyncDataByTimestamp( - dataItems, Parcel::GetAppendedLen(), - // need modify - continueStmtToken->GetDeletedBeginTime(), continueStmtToken->GetDeletedEndTime(), dataSizeInfo); + do { + errCode = handle->GetSyncDataByQuery(dataItems, + Parcel::GetAppendedLen(), + dataSizeInfo, + std::bind(&SQLiteSingleVerRelationalContinueToken::GetStatement, *continueStmtToken, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + if (errCode == -E_FINISHED) { + continueStmtToken->FinishGetData(); + errCode = continueStmtToken->IsGetAllDataFinished() ? E_OK : -E_UNFINISHED; } - } - - if (errCode == -E_FINISHED) { - errCode = E_OK; - } + } while (errCode == -E_UNFINISHED && + CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())); ERROR: if (errCode != -E_UNFINISHED && errCode != E_OK) { // Error happened. @@ -348,13 +310,24 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return -E_INVALID_ARGS; } - auto token = new (std::nothrow) SQLiteSingleVerContinueToken(timeRange, query); // release in sync module + auto token = new (std::nothrow) SQLiteSingleVerRelationalContinueToken(timeRange, query); if (token == nullptr) { LOGE("[SingleVerNStore] Allocate continue token failed."); return -E_OUT_OF_MEMORY; } - int innerCode; + continueStmtToken = static_cast(token); + return GetSyncDataNext(entries, continueStmtToken, dataSizeInfo); +} + +int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, + ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const +{ + auto token = static_cast(continueStmtToken); + if (!token->CheckValid()) { + return -E_INVALID_ARGS; + } + std::vector dataItems; int errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); if (errCode != E_OK && errCode != -E_UNFINISHED) { // The code need be sent to outside except new error happened. @@ -362,7 +335,7 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return errCode; } - innerCode = GetKvEntriesByDataItems(entries, dataItems); + int innerCode = GetKvEntriesByDataItems(entries, dataItems); if (innerCode != E_OK) { errCode = innerCode; delete token; @@ -372,18 +345,6 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return errCode; } -int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, - ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const -{ - return E_OK; -} - -// Release the continue token of getting data. -void RelationalSyncAbleStorage::ReleaseContinueToken(ContinueToken &continueStmtToken) const -{ - return; -} - int RelationalSyncAbleStorage::PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, const DeviceID &deviceName) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp index 2053fb76b..212b13f30 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp @@ -83,6 +83,38 @@ std::string FieldValue2String(const FieldValue &val, QueryValueType type) return ""; } } + +std::string GetSelectAndFromClauseForRDB(const std::string &tableName) +{ + return "SELECT b.data_key," + "b.device," + "b.ori_device," + "b.timestamp as " + DBConstant::TIMESTAMP_ALIAS + "," + "b.wtimestamp," + "b.flag," + "b.hash_key," + "a.* " + "FROM " + tableName + " AS a INNER JOIN " + DBConstant::RELATIONAL_PREFIX + tableName + "_log AS b " + "ON a.rowid=b.data_key "; +} + +std::string GetTimeRangeClauseForRDB() +{ + return " AND (" + DBConstant::TIMESTAMP_ALIAS + ">=? AND " + DBConstant::TIMESTAMP_ALIAS + "=? AND " + DBConstant::TIMESTAMP_ALIAS + "=? AND naturalbase_rdb_timestamp=? AND naturalbase_rdb_timestamp>> mulDevTableTimeRange_; - std::map>> mulDevTableDeletedTimeRange_; }; } // namespace DistributedDB #endif // SQLITE_SINGLE_VER_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp new file mode 100644 index 000000000..512a9a46e --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_single_ver_relational_continue_token.h" +#include "sqlite_utils.h" + +namespace DistributedDB { +SQLiteSingleVerRelationalContinueToken::SQLiteSingleVerRelationalContinueToken( + const SyncTimeRange &timeRange, const QueryObject &object) + : isGettingDeletedData_(false), queryObj_(object), tableName_(object.GetTableName()), timeRange_(timeRange) +{} + +bool SQLiteSingleVerRelationalContinueToken::CheckValid() const +{ + bool isValid = (magicBegin_ == MAGIC_BEGIN && magicEnd_ == MAGIC_END); + if (!isValid) { + LOGE("Invalid continue token."); + } + return isValid; +} + +int SQLiteSingleVerRelationalContinueToken::GetStatement(sqlite3 *db, sqlite3_stmt *&stmt, bool &isGettingDeletedData) +{ + isGettingDeletedData = isGettingDeletedData_; + if (isGettingDeletedData) { + return GetDeletedDataStmt(db, stmt); + } + return GetQuerySyncStatement(db, stmt); +} + +void SQLiteSingleVerRelationalContinueToken::SetNextBeginTime(const DataItem &theLastItem) +{ + TimeStamp nextBeginTime = theLastItem.timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; + } + if (!isGettingDeletedData_) { + timeRange_.beginTime = nextBeginTime; + return; + } + if ((theLastItem.flag & DataItem::DELETE_FLAG) != 0) { // The last one could be non-deleted. + timeRange_.deleteBeginTime = nextBeginTime; + } +} + +void SQLiteSingleVerRelationalContinueToken::FinishGetData() +{ + if (isGettingDeletedData_) { + timeRange_.deleteEndTime = 0; + return; + } + isGettingDeletedData_ = true; + timeRange_.endTime = 0; + return; +} + +bool SQLiteSingleVerRelationalContinueToken::IsGetAllDataFinished() const +{ + return timeRange_.beginTime >= timeRange_.endTime && timeRange_.deleteBeginTime >= timeRange_.deleteEndTime; +} + +int SQLiteSingleVerRelationalContinueToken::GetQuerySyncStatement(sqlite3 *db, sqlite3_stmt *&stmt) +{ + int errCode = E_OK; + SqliteQueryHelper helper = queryObj_.GetQueryHelper(errCode); + if (errCode != E_OK) { + return errCode; + } + return helper.GetRelationalQueryStatement(db, timeRange_.beginTime, timeRange_.endTime, stmt); +} + +int SQLiteSingleVerRelationalContinueToken::GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const +{ + // get stmt + int errCode = E_OK; + const std::string sql = GetDeletedDataSQL(); + errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + goto ERROR; + } + + // bind stmt + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 1, timeRange_.deleteBeginTime); // 1 means begin time + if (errCode != E_OK) { + goto ERROR; + } + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 2, timeRange_.deleteEndTime); // 2 means end time + if (errCode != E_OK) { + goto ERROR; + } + return errCode; + +ERROR: + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} + +const QueryObject &SQLiteSingleVerRelationalContinueToken::GetQuery() const +{ + return queryObj_; +} + +std::string SQLiteSingleVerRelationalContinueToken::GetDeletedDataSQL() const +{ + std::string tableName = DBConstant::RELATIONAL_PREFIX + tableName_ + "_log"; + return "SELECT * FROM " + tableName + + " WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03 = 0x03) ORDER BY timestamp ASC;"; +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h new file mode 100644 index 000000000..a6542b488 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H +#define SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H +#ifdef RELATIONAL_STORE +#include +#include + +#include "db_types.h" +#include "query_object.h" +#include "single_ver_kvdb_sync_interface.h" + +namespace DistributedDB { +class SQLiteSingleVerRelationalContinueToken { +public: + SQLiteSingleVerRelationalContinueToken(const SyncTimeRange &timeRange, const QueryObject &queryObject); + ~SQLiteSingleVerRelationalContinueToken() = default; + + // Check the magic number at the beginning and end of the SQLiteSingleVerRelationalContinueToken. + bool CheckValid() const; + // The statement should be release by the caller. + int GetStatement(sqlite3 *db, sqlite3_stmt *&stmt, bool &isGettingDeletedData); + void SetNextBeginTime(const DataItem &theLastItem); + void FinishGetData(); + bool IsGetAllDataFinished() const; + const QueryObject &GetQuery() const; + +private: + std::string GetDeletedDataSQL() const; + int GetQuerySyncStatement(sqlite3 *db, sqlite3_stmt *&stmt); + int GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const; + + static const unsigned int MAGIC_BEGIN = 0x600D0AC7; // for token guard + static const unsigned int MAGIC_END = 0x0AC7600D; // for token guard + unsigned int magicBegin_ = MAGIC_BEGIN; + int isGettingDeletedData_ = false; + QueryObject queryObj_; + const std::string &tableName_; + SyncTimeRange timeRange_; + unsigned int magicEnd_ = MAGIC_END; +}; +} // namespace DistributedDB +#endif // RELATIONAL_STORE +#endif // SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 9fa4e7183..56bb8cc70 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -96,7 +96,7 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) +int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(const QueryObject &query) { int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); @@ -106,50 +106,6 @@ int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) return errCode; } -// binding index just for the get sync data sql -static const int BIND_BEGIN_STAMP_INDEX = 1; -static const int BIND_END_STAMP_INDEX = 2; - -int SQLiteSingleVerRelationalStorageExecutor::PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, - sqlite3_stmt *&statement, bool getDeletedData) const -{ - if (dbHandle_ == nullptr) { - return -E_INVALID_DB; - } - - const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = - "SELECT * FROM naturalbase_rdb_aux_" + table_.GetTableName() + - "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; - const std::string SELECT_SYNC_ENTRIES_SQL = - "SELECT * FROM naturalbase_rdb_aux_" + table_.GetTableName() + - "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; - - const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); - int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); - if (errCode != E_OK) { - LOGE("Prepare the sync entries statement error:%d", errCode); - return errCode; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_BEGIN_STAMP_INDEX, begin); - if (errCode != E_OK) { - goto ERROR; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_END_STAMP_INDEX, end); - if (errCode != E_OK) { - goto ERROR; - } - -ERROR: - if (errCode != E_OK) { - LOGE("Bind the timestamp for getting sync data error:%d", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); - } - - return CheckCorruptedStatus(errCode); -} - static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) { switch (type) { @@ -260,7 +216,6 @@ static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) return errCode; } logInfo.originDev = std::string(oriDev.begin(), oriDev.end()); - logInfo.timestamp = static_cast(sqlite3_column_int64(logStatement, 3)); logInfo.wTimeStamp = static_cast(sqlite3_column_int64(logStatement, 4)); logInfo.flag = static_cast(sqlite3_column_int64(logStatement, 5)); @@ -276,68 +231,6 @@ static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) return errCode; } -static sqlite3_stmt *GetDataStmtByPK(sqlite3 *db, const std::string &tableName, int rowId) -{ - std::string sql = "select * from " + tableName + " where rowId=?;"; - sqlite3_stmt *statement = nullptr; - int errCode = SQLiteUtils::GetStatement(db, sql, statement); - if (errCode != E_OK) { - LOGE("[data by rowid statement] Get statement fail!"); - return nullptr; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, 1, rowId); - if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statement, true, errCode); - return nullptr; - } - - return statement; -} - - -int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *logStatement, DataItem &dataItem) const -{ - std::map colInfos = table_.GetFields(); - RowDataWithLog data; - - int errCode = GetLogData(logStatement, data.logInfo); - if (errCode != E_OK) { - LOGE("relational data value transfer to kv fail"); - return errCode; - } - - // dataKey is rowid by pk - // only get local device data - sqlite3_stmt *stmt = GetDataStmtByPK(dbHandle_, table_.GetTableName(), data.logInfo.dataKey); - - errCode = SQLiteUtils::StepWithRetry(stmt, false); - if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW) && errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGE("[SQLiteSingleVerRelationalStorageExecutor] Step fail errCode:%d", errCode); - return errCode; - } - - std::vector fieldInfos; - for (const auto &col: colInfos) { - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - break; - } - DataValue value; - GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId()); - - fieldInfos.push_back(col.second); - data.rowData.push_back(value); - } - - SQLiteUtils::ResetStatement(stmt, true, errCode); - - errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); - if (errCode != E_OK) { - LOGE("relational data value transfer to kv fail"); - } - return errCode; -} - static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) { // timestamp and local flag: 3 * uint64_t, version(uint32_t), key, value, origin dev and the padding size. @@ -349,54 +242,8 @@ static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) return dataSize; } -int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataItems(std::vector &dataItems, - sqlite3_stmt *logStatement, size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const -{ - int errCode; - size_t dataTotalSize = 0; - do { - DataItem item; - errCode = SQLiteUtils::StepWithRetry(logStatement, isMemDb_); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - errCode = GetDataItemForSync(logStatement, item); - } else { - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); - errCode = -E_FINISHED; - } else { - LOGE("Get sync data error:%d", errCode); - } - break; - } - - // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. - dataTotalSize += GetDataItemSerialSize(item, appendLength); - if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || - dataItems.size() >= dataSizeInfo.packetSize) { - errCode = -E_UNFINISHED; - break; - } else { - dataItems.push_back(std::move(item)); - } - } while (true); - return errCode; -} - -int SQLiteSingleVerRelationalStorageExecutor::GetDeletedSyncDataByTimestamp(std::vector &dataItems, - size_t appendLength, TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const -{ - sqlite3_stmt *logStatement = nullptr; - int errCode = PrepareForSyncDataByTime(begin, end, logStatement, true); - if (errCode != E_OK) { - return errCode; - } - - errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); - SQLiteUtils::ResetStatement(logStatement, true, errCode); - return CheckCorruptedStatus(errCode); -} - -static const std::string SELECT_META_VALUE_SQL = "SELECT value FROM naturalbase_rdb_aux_metadata WHERE key=?;"; +static const std::string SELECT_META_VALUE_SQL = + "SELECT value FROM " + DBConstant::RELATIONAL_PREFIX + "metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const { sqlite3_stmt *statement = nullptr; @@ -424,7 +271,8 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v return errCode; } -static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO naturalbase_rdb_aux_metadata VALUES(?,?);"; +static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + + "metadata VALUES(?,?);"; int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const { sqlite3_stmt *statement = nullptr; @@ -453,7 +301,8 @@ ERROR: return errCode; } -static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM naturalbase_rdb_aux_metadata WHERE key=?;"; +static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const { sqlite3_stmt *statement = nullptr; @@ -480,7 +329,7 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector=? AND key<=?;"; + "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + "metadata WHERE key>=? AND key<=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const { sqlite3_stmt *statement = nullptr; @@ -527,7 +376,8 @@ static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) return errCode; } -static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM naturalbase_rdb_aux_metadata;"; + +static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM " + DBConstant::RELATIONAL_PREFIX + "metadata;"; int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const { sqlite3_stmt *statement = nullptr; @@ -545,7 +395,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj const std::string &deviceName, sqlite3_stmt *&logStmt) const { std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "naturalbase_rdb_aux_" + object.GetTableName() + "_log"; + const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_log"; std::string dataFormat = "?, '" + deviceName + "', ?, ?, ?, ?, ?"; std::string sql = "INSERT OR REPLACE INTO " + tableName + @@ -564,7 +414,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryOb // naturalbase_rdb_aux_userTableName_deviceHash // tableName std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "naturalbase_rdb_aux_" + object.GetTableName() + "_" + + const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); @@ -602,7 +452,8 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen Key hashKey; (void)DBCommon::CalcValueHash(dataItem.key, hashKey); std::string hash = std::string(hashKey.begin(), hashKey.end()); - std::string sql = "select * from naturalbase_rdb_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; + std::string sql = "select * from " + DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + + "_log where hash_key = ?;"; sqlite3_stmt *queryStmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, queryStmt); if (errCode != E_OK) { @@ -660,7 +511,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) { std::string devName = DBCommon::TransferHashString(dataItem.dev); - const std::string tableName = "naturalbase_rdb_aux_" + table_.GetTableName() + "_" + + const std::string tableName = DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; @@ -780,40 +631,72 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncItems(const QueryObject &o return errCode; } -static int GetLogInfoStatement(sqlite3 *dbHandle, const std::string &tableName, - uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement) +int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *stmt, DataItem &dataItem, + bool isGettingDeletedData) const { - std::string sql = "select * from naturalbase_rdb_aux_" + tableName + - "_log where flag=0x02 AND timestamp>=? AND timestamp fieldInfos; + if (!isGettingDeletedData) { + for (const auto &col: table_.GetFields()) { + LOGD("[GetDataItemForSync] field:%s type:%d cid:%d", col.second.GetFieldName().c_str(), + col.second.GetStorageType(), col.second.GetColumnId() + 7); + DataValue value; + GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId() + 7); + fieldInfos.push_back(col.second); + data.rowData.push_back(value); + } } - errCode = SQLiteUtils::BindInt64ToStatement(statement, 2, endTime); + errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + LOGE("relational data value transfer to kv fail"); } return errCode; } int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, - QueryObject query, const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const + const DataSizeSpecInfo &dataSizeInfo, std::function getStmt) { - sqlite3_stmt *logStatement = nullptr; - int errCode = GetLogInfoStatement(dbHandle_, query.GetTableName(), timeRange.first, timeRange.second, logStatement); - if (errCode == E_OK) { - errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); + sqlite3_stmt *stmt = nullptr; + bool isGettingDeletedData = false; + int errCode = getStmt(dbHandle_, stmt, isGettingDeletedData); + if (errCode != E_OK) { + return errCode; } - SQLiteUtils::ResetStatement(logStatement, true, errCode); + + size_t dataTotalSize = 0; + do { + DataItem item; + errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = GetDataItemForSync(stmt, item, isGettingDeletedData); + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); + errCode = -E_FINISHED; + break; + } else { + LOGE("Get sync data error:%d", errCode); + break; + } + + // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. + dataTotalSize += GetDataItemSerialSize(item, appendLength); + if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || + dataItems.size() >= dataSizeInfo.packetSize) { + errCode = -E_UNFINISHED; + break; + } else { + dataItems.push_back(std::move(item)); + } + } while (true); + + SQLiteUtils::ResetStatement(stmt, true, errCode); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index 66769f825..ba0057ae1 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -16,8 +16,9 @@ #define SQLITE_SINGLE_VER_RELATIONAL_STORAGE_EXECUTOR_H #ifdef RELATIONAL_STORE -#include "macro_utils.h" +#include "data_transformer.h" #include "db_types.h" +#include "macro_utils.h" #include "sqlite_utils.h" #include "sqlite_storage_executor.h" #include "relational_store_delegate.h" @@ -38,13 +39,11 @@ public: int Commit(); int Rollback(); - int SetTableInfo(QueryObject query); + int SetTableInfo(const QueryObject &query); // For Get sync data - int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, QueryObject query, - const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const; - int GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, - TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; + int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, + const DataSizeSpecInfo &dataSizeInfo, std::function getStmt); // operation of meta data int GetKvData(const Key &key, Value &value) const; @@ -65,10 +64,7 @@ private: int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData) const; - int GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, - size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const; - - int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem) const; + int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem, bool isGettingDeletedData) const; int SaveSyncDataItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName, TimeStamp &timeStamp); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 9d3e9a2ff..84d5b2348 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -1298,9 +1298,9 @@ int SQLiteSingleVerStorageExecutor::PrepareForNotifyConflictAndObserver(DataItem LOGD("[SingleVerExe] Ignore the sync data."); if (isSyncMigrating_) { ResetForMigrateCacheData(); - return -E_IGNOR_DATA; + return -E_IGNORE_DATA; } - return ResetSaveSyncStatements(-E_IGNOR_DATA); + return ResetSaveSyncStatements(-E_IGNORE_DATA); } notify.dataStatus = JudgeSyncSaveType(dataItem, notify.getData, deviceInfo.deviceName, isHashKeyExisted); @@ -1343,7 +1343,7 @@ int SQLiteSingleVerStorageExecutor::SaveSyncDataItem(DataItem &dataItem, const D int errCode = PrepareForNotifyConflictAndObserver(dataItem, deviceInfo, notify); if (errCode != E_OK) { - if (errCode == -E_IGNOR_DATA) { + if (errCode == -E_IGNORE_DATA) { errCode = E_OK; } return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp index 872e87361..9559b8fb4 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp @@ -760,7 +760,7 @@ int SQLiteSingleVerStorageExecutor::PutIntoConflictAndCommitForMigrateCache(Data int errCode = PrepareForNotifyConflictAndObserver(dataItem, deviceInfo, notify); if (errCode != E_OK) { errCode = (errCode == -E_NOT_FOUND ? E_OK : errCode); - if (errCode == -E_IGNOR_DATA) { + if (errCode == -E_IGNORE_DATA) { notify.dataStatus.isDefeated = true; errCode = E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 906d2952c..e69117af7 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1305,7 +1305,7 @@ int SQLiteUtils::RegisterGetSysTime(sqlite3 *db) int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) { std::string sql = - "CREATE TABLE IF NOT EXISTS naturalbase_rdb_aux_metadata(" \ + "CREATE TABLE IF NOT EXISTS " + DBConstant::RELATIONAL_PREFIX + "metadata(" \ "key BLOB PRIMARY KEY NOT NULL," \ "value BLOB);"; @@ -1320,21 +1320,21 @@ int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) int SQLiteUtils::CreateRelationalLogTable(sqlite3 *db, const std::string &oriTableName) { std::string sql = - "CREATE TABLE IF NOT EXISTS naturalbase_rdb_aux_" + oriTableName + "_log(" \ + "CREATE TABLE IF NOT EXISTS " + DBConstant::RELATIONAL_PREFIX + oriTableName + "_log(" \ "data_key INT NOT NULL," \ "device BLOB," \ "ori_device BLOB," \ "timestamp INT NOT NULL," \ "wtimestamp INT NOT NULL," \ "flag INT NOT NULL," \ - "hash_key BLOB PRIMARY KEY NOT NULL);"; + "hash_key BLOB NOT NULL," + "PRIMARY KEY(device,hash_key));"; int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); - return errCode; } - return E_OK; + return errCode; } template @@ -1352,7 +1352,7 @@ int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &tabl "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_INSERT AFTER INSERT \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t INSERT OR REPLACE INTO naturalbase_rdb_aux_%s_log \ + "\t INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + "%s_log \ (data_key, device, ori_device, timestamp, wtimestamp, flag, hash_key)" \ "VALUES (new.rowid, '%s', '%s', get_sys_time(), get_sys_time(), 0x02, calc_hash(new.%s));\n" \ "END;"; @@ -1363,7 +1363,7 @@ int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &tabl "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_UPDATE AFTER UPDATE \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t UPDATE naturalbase_rdb_aux_%s_log SET timestamp=get_sys_time(), device='%s' \ + "\t UPDATE " + DBConstant::RELATIONAL_PREFIX + "%s_log SET timestamp=get_sys_time(), device='%s' \ where hash_key=calc_hash(old.%s) and flag&0x10=0;\n" \ "END;"; updateTrigger = string_format(updateTrigger, table.GetTableName().c_str(), table.GetTableName().c_str(), @@ -1372,7 +1372,7 @@ int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &tabl "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_DELETE BEFORE DELETE \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t UPDATE naturalbase_rdb_aux_%s_log set flag=0x03,timestamp=get_sys_time() \ + "\t UPDATE " + DBConstant::RELATIONAL_PREFIX + "%s_log set flag=0x03,timestamp=get_sys_time() \ WHERE hash_key=calc_hash(old.%s);\n" \ "END;"; deleteTrigger = string_format(deleteTrigger, table.GetTableName().c_str(), @@ -1403,9 +1403,8 @@ int SQLiteUtils::CreateSameStuTable(sqlite3 *db, const std::string &oriTableName int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); - return errCode; } - return E_OK; + return errCode; } int SQLiteUtils::RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func) diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 22581c572..3ca3c7d54 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -185,6 +185,7 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "../storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp", "../storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "../storage/src/sqlite/sqlite_single_ver_result_set.cpp", "../storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp new file mode 100644 index 000000000..db75f5c58 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include + +#include "data_transformer.h" +#include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" +#include "db_types.h" +#include "log_print.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "generic_single_ver_kv_entry.h" +#include "kvdb_properties.h" +#include "relational_store_delegate.h" +#include "relational_store_instance.h" +#include "relational_store_manager.h" +#include "relational_store_sqlite_ext.h" +#include "relational_sync_able_storage.h" +#include "sqlite_relational_store.h" +#include "sqlite_utils.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +string g_testDir; +string g_storePath; +const string g_tableName { "data" }; +DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); +RelationalStoreDelegate *g_delegate = nullptr; + +void CreateDBAndTable() +{ + sqlite3 *db = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + sqlite3_close(db); + return; + } + + const string sql { "CREATE TABLE " + g_tableName + "(key BLOB PRIMARY KEY NOT NULL, value BLOB);" }; + char *zErrMsg = nullptr; + errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK) { + LOGE("sql error:%s",zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_close(db); +} + +int InitRelationalStore() +{ + RelationalStoreDelegate *delegate = nullptr; + DBStatus dbStatus = DBStatus::DB_ERROR; + g_mgr.OpenStore(g_storePath, RelationalStoreDelegate::Option {}, + [&delegate, &dbStatus] (DBStatus status, RelationalStoreDelegate *ptr) { + delegate = ptr; + dbStatus = status; + } + ); + if (dbStatus == DBStatus::OK) { + g_delegate = delegate; + g_delegate->CreateDistributedTable(g_tableName, RelationalStoreDelegate::TableOption()); + } + return dbStatus; +} + +int AddOrUpdateRecord(const Key &key, const Value &value) { + static const string SQL = "INSERT OR REPLACE INTO " + g_tableName + " VALUES(?,?);"; + sqlite3 *db = nullptr; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + goto END; + } + errCode = SQLiteUtils::GetStatement(db, SQL, statement); + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // 1 means key's index + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::BindBlobToStatement(statement, 2, value, true); // 2 means value's index + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::StepWithRetry(statement, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + +END: + SQLiteUtils::ResetStatement(statement, true, errCode); + sqlite3_close(db); + return errCode; +} + +int GetLogData(const Key &key, uint64_t &flag, TimeStamp ×tamp, const DeviceID &device = "") +{ + string tableName = g_tableName; + if (!device.empty()) { + } + const string sql = "SELECT timestamp, flag \ + FROM " + g_tableName + " as a, " + DBConstant::RELATIONAL_PREFIX + g_tableName + "_log as b \ + WHERE a.key=? AND a.rowid=b.data_key;"; + + sqlite3 *db = nullptr; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + goto END; + } + errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // 1 means key's index + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::StepWithRetry(statement, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + timestamp = static_cast(sqlite3_column_int64(statement, 0)); + flag = static_cast(sqlite3_column_int64(statement, 1)); + errCode = E_OK; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = -E_NOT_FOUND; + } + +END: + sqlite3_close(db); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, + const std::string appId, const std::string &userId, DBProperties &properties) +{ + properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); + properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); + properties.SetStringProp(KvDBProperties::APP_ID, appId); + properties.SetStringProp(KvDBProperties::USER_ID, userId); + properties.SetStringProp(KvDBProperties::STORE_ID, storePath); + std::string identifier = userId + "-" + appId + "-" + storePath; + std::string hashIdentifier = DBCommon::TransferHashString(identifier); + properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); +} + +const RelationalSyncAbleStorage *GetRelationalStore() +{ + DBProperties properties; + InitStoreProp(RelationalStoreDelegate::Option {}, g_storePath, APP_ID, USER_ID, properties); + int errCode = E_OK; + auto store = RelationalStoreInstance::GetDataBase(properties, errCode); + if (store == nullptr) { + LOGE("Get db failed:%d", errCode); + return nullptr; + } + return static_cast(store)->GetStorageEngine(); +} +} + +class DistributedDBRelationalGetDataTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBRelationalGetDataTest::SetUpTestCase(void) +{ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_storePath = g_testDir + "/getDataTest.db"; + LOGI("The test db is:%s", g_testDir.c_str()); +} + +void DistributedDBRelationalGetDataTest::TearDownTestCase(void) +{} + +void DistributedDBRelationalGetDataTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + CreateDBAndTable(); +} + +void DistributedDBRelationalGetDataTest::TearDown(void) +{ + if (g_delegate != nullptr) { + EXPECT_EQ(g_mgr.CloseStore(g_delegate), DBStatus::OK); + EXPECT_EQ(g_mgr.DeleteStore(g_storePath), DBStatus::OK); + g_delegate = nullptr; + } + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error."); + } + return; +} + +/** + * @tc.name: LogTbl1 + * @tc.desc: When put sync data to relational store, trigger generate log. + * @tc.type: FUNC + * @tc.require: AR000GK58G + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, LogTbl1, TestSize.Level1) +{ + ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + + /** + * @tc.steps: step1. Put data. + * @tc.expected: Succeed, return OK. + */ + Key insertKey = KEY_1; + Value insertValue = VALUE_1; + EXPECT_EQ(AddOrUpdateRecord(insertKey, insertValue), E_OK); + + /** + * @tc.steps: step2. Check log record. + * @tc.expected: Record exists. + */ + Value value; + uint64_t flag = 0; + TimeStamp timestamp1 = 0; + EXPECT_EQ(GetLogData(insertKey, flag, timestamp1), E_OK); + EXPECT_EQ(flag, DataItem::LOCAL_FLAG); + EXPECT_NE(timestamp1, 0ULL); +} + +/** + * @tc.name: GetSyncData1 + * @tc.desc: GetSyncData interface + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) +{ + ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + + /** + * @tc.steps: step1. Put 500 records. + * @tc.expected: Succeed, return OK. + */ + const int RECORD_COUNT = 500; + for (int i = 0; i < RECORD_COUNT; ++i) { + string key = "key_" + to_string(i); + string value = "value_" + to_string(i); + EXPECT_EQ(AddOrUpdateRecord(Key(key.data(), key.data() + key.size()), + Value(value.data(), value.data() + value.size())), E_OK); + } + + /** + * @tc.steps: step2. Get all data. + * @tc.expected: Succeed and the count is right. + */ + auto store = GetRelationalStore(); + ContinueToken token = nullptr; + QueryObject query(Query::Select(g_tableName)); + std::vector entries; + DataSizeSpecInfo sizeInfo {MTU_SIZE, 50}; + + int errCode = store->GetSyncData(query, SyncTimeRange {}, sizeInfo, token, entries); + int count = entries.size(); + SingleVerKvEntry::Release(entries); + EXPECT_EQ(errCode, -E_UNFINISHED); + while (token != nullptr) { + errCode = store->GetSyncDataNext(entries, token, sizeInfo); + count += entries.size(); + SingleVerKvEntry::Release(entries); + EXPECT_TRUE(errCode == E_OK || errCode == -E_UNFINISHED); + } + EXPECT_EQ(count, RECORD_COUNT); +} + +/** + * @tc.name: GetQuerySyncData1 + * @tc.desc: GetSyncData interface. + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) +{ + ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + + /** + * @tc.steps: step1. Put 100 records. + * @tc.expected: Succeed, return OK. + */ + const int RECORD_COUNT = 100; // 100 records. + for (int i = 0; i < RECORD_COUNT; ++i) { + string key = "k" + to_string(i); + string value = "v" + to_string(i); + EXPECT_EQ(AddOrUpdateRecord(Key(key.data(), key.data() + key.size()), + Value(value.data(), value.data() + value.size())), E_OK); + } + + /** + * @tc.steps: step2. Get data limit 80, offset 30. + * @tc.expected: Get 70 records. + */ + auto store = GetRelationalStore(); + ContinueToken token = nullptr; + const unsigned int LIMIT = 80; // limit as 80. + const unsigned int OFFSET = 30; // offset as 30. + const unsigned int EXPECT_COUNT = RECORD_COUNT - OFFSET; // expect 70 records. + QueryObject query(Query::Select(g_tableName).Limit(LIMIT, OFFSET)); + std::vector entries; + + int errCode = store->GetSyncData(query, SyncTimeRange {}, DataSizeSpecInfo {}, token, entries); + EXPECT_EQ(entries.size(), EXPECT_COUNT); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(token, nullptr); + SingleVerKvEntry::Release(entries); +} +#endif \ No newline at end of file -- Gitee From a6f50ba75b21dd727f7f15fbb7e758f8898998a5 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 30 Dec 2021 10:21:05 +0800 Subject: [PATCH 19/19] Fix compile issues in lower g++ version Signed-off-by: lianhuix --- .../include/relational_store_connection.h | 11 ++++----- .../src/relational_store_connection.cpp | 7 ++++++ ...single_ver_relational_storage_executor.cpp | 23 +++++++++---------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index 35f2d91e6..ac6cb4c13 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -34,11 +34,10 @@ public: const Query &query; bool wait = true; }; - RelationalStoreConnection() = default; - explicit RelationalStoreConnection(IRelationalStore *store) - { - store_ = store; - }; + + RelationalStoreConnection(); + + explicit RelationalStoreConnection(IRelationalStore *store); virtual ~RelationalStoreConnection() = default; @@ -61,7 +60,7 @@ protected: virtual int Pragma(int cmd, void *parameter); IRelationalStore *store_ = nullptr; - std::atomic isExclusive_ = false; + std::atomic isExclusive_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp index 4fc241f37..c04d81193 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp @@ -18,6 +18,13 @@ #include "sqlite_single_ver_relational_storage_executor.h" namespace DistributedDB { +RelationalStoreConnection::RelationalStoreConnection() : isExclusive_(false) +{} + +RelationalStoreConnection::RelationalStoreConnection(IRelationalStore *store) + : store_(store), isExclusive_(false) +{} + int RelationalStoreConnection::Pragma(int cmd, void *parameter) { return E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 56bb8cc70..bd20e5c69 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -242,10 +242,10 @@ static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) return dataSize; } -static const std::string SELECT_META_VALUE_SQL = - "SELECT value FROM " + DBConstant::RELATIONAL_PREFIX + "metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const { + static const std::string SELECT_META_VALUE_SQL = "SELECT value FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_META_VALUE_SQL, statement); if (errCode != E_OK) { @@ -271,10 +271,10 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v return errCode; } -static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + - "metadata VALUES(?,?);"; int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const { + static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + + "metadata VALUES(?,?);"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, INSERT_META_SQL, statement); if (errCode != E_OK) { @@ -301,10 +301,10 @@ ERROR: return errCode; } -static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + - "metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const { + static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_SQL, statement); if (errCode != E_OK) { @@ -328,10 +328,10 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector=? AND key<=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const { + static const std::string REMOVE_META_VALUE_BY_KEY_PREFIX_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key>=? AND key<=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_BY_KEY_PREFIX_SQL, statement); if (errCode != E_OK) { @@ -376,10 +376,9 @@ static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) return errCode; } - -static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM " + DBConstant::RELATIONAL_PREFIX + "metadata;"; int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const { + static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM " + DBConstant::RELATIONAL_PREFIX + "metadata;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_ALL_META_KEYS, statement); if (errCode != E_OK) { @@ -446,8 +445,8 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryOb return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, - const DataItem &dataItem, TimeStamp &maxTimestamp) +int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, const DataItem &dataItem, + TimeStamp &maxTimestamp) { Key hashKey; (void)DBCommon::CalcValueHash(dataItem.key, hashKey); -- Gitee