From e9433c723f27747e7db800e84933b06d4495830c Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 1 Nov 2021 15:25:33 +0800 Subject: [PATCH] Sync the code Signed-off-by: lianhuix --- .../libs/distributeddb/BUILD.gn | 10 +- .../common/include/auto_launch.h | 3 +- .../common/include/data_compression.h | 28 + .../distributeddb/common/include/db_ability.h | 67 + .../distributeddb/common/include/db_common.h | 3 + .../common/include/db_constant.h | 11 + .../common/include/param_check_utils.h | 2 + .../distributeddb/common/include/parcel.h | 76 +- .../common/include/platform_specific.h | 11 + .../common/include/runtime_context.h | 5 +- .../distributeddb/common/include/semaphore.h | 48 + .../distributeddb/common/include/version.h | 5 +- .../distributeddb/common/src/auto_launch.cpp | 19 +- .../common/src/data_compression.cpp | 88 + .../distributeddb/common/src/db_ability.cpp | 160 ++ .../distributeddb/common/src/db_common.cpp | 37 +- .../distributeddb/common/src/db_constant.cpp | 1 + .../common/src/param_check_utils.cpp | 10 + .../libs/distributeddb/common/src/parcel.cpp | 170 +- .../common/src/platform_specific.cpp | 56 +- .../common/src/query_expression.cpp | 13 +- .../common/src/runtime_context_impl.cpp | 18 +- .../common/src/runtime_context_impl.h | 7 +- .../distributeddb/common/src/semaphore.cpp | 61 + .../common/src/time_tick_monitor.h | 2 +- .../include/communicator_aggregator.h | 13 + .../communicator/include/frame_combiner.h | 5 +- .../communicator/include/iadapter.h | 4 + .../communicator/include/icommunicator.h | 6 + .../communicator/include/network_adapter.h | 3 + .../communicator/src/communicator.cpp | 10 + .../communicator/src/communicator.h | 3 + .../src/communicator_aggregator.cpp | 10 + .../communicator/src/communicator_linker.cpp | 10 +- .../communicator/src/network_adapter.cpp | 45 +- .../communicator/src/protocol_proto.cpp | 4 +- .../include/auto_launch_export.h | 6 + .../distributeddb/include/query_expression.h | 29 +- .../libs/distributeddb/include/types_export.h | 21 +- .../include/iprocess_communicator.h | 16 + .../interfaces/include/kv_store_nb_delegate.h | 23 +- .../distributeddb/interfaces/include/types.h | 6 +- .../src/kv_store_delegate_manager.cpp | 10 +- .../src/kv_store_nb_delegate_impl.cpp | 51 + .../src/kv_store_nb_delegate_impl.h | 18 +- .../distributeddb/storage/include/ikvdb.h | 2 + .../storage/include/ikvdb_connection.h | 2 + .../storage/include/ikvdb_sync_interface.h | 6 + .../storage/include/kvdb_manager.h | 9 + .../storage/include/kvdb_pragma.h | 34 +- .../storage/include/kvdb_properties.h | 5 + .../{src => include}/single_ver_kv_entry.h | 0 .../include/single_ver_kvdb_sync_interface.h | 27 + .../storage/src/generic_kvdb.cpp | 5 + .../distributeddb/storage/src/generic_kvdb.h | 2 + .../storage/src/generic_kvdb_connection.cpp | 6 +- .../storage/src/generic_kvdb_connection.h | 2 + .../src/generic_single_ver_kv_entry.cpp | 91 +- .../storage/src/generic_single_ver_kv_entry.h | 15 +- .../storage/src/kvdb_manager.cpp | 218 +- .../storage/src/kvdb_properties.cpp | 4 + .../src/multiver/multi_ver_natural_store.cpp | 10 + .../src/multiver/multi_ver_natural_store.h | 5 + .../storage/src/operation/database_oper.cpp | 2 +- .../src/operation/single_ver_database_oper.h | 2 +- .../storage/src/package_file.cpp | 2 +- ...e_ver_natural_store_commit_notify_data.cpp | 4 +- .../storage/src/sqlite/query_object.cpp | 350 +++- .../storage/src/sqlite/query_object.h | 17 +- .../storage/src/sqlite/query_sync_object.cpp | 273 +++ .../storage/src/sqlite/query_sync_object.h | 56 + .../src/sqlite/sqlite_local_kvdb_snapshot.h | 2 +- .../sqlite_single_ver_continue_token.cpp | 141 ++ .../sqlite/sqlite_single_ver_continue_token.h | 78 + .../sqlite_single_ver_database_upgrader.cpp | 4 +- .../sqlite_single_ver_database_upgrader.h | 2 +- .../sqlite_single_ver_natural_store.cpp | 523 ++++- .../sqlite/sqlite_single_ver_natural_store.h | 62 +- ...te_single_ver_natural_store_connection.cpp | 14 + ...lite_single_ver_natural_store_connection.h | 2 + .../sqlite_single_ver_storage_engine.cpp | 25 +- .../sqlite/sqlite_single_ver_storage_engine.h | 2 +- .../sqlite_single_ver_storage_executor.cpp | 75 +- .../sqlite_single_ver_storage_executor.h | 33 +- ...lite_single_ver_storage_executor_cache.cpp | 50 + .../sqlite_single_ver_storage_executor_sql.h | 15 + .../src/sqlite/sqlite_storage_engine.cpp | 4 +- .../storage/src/sqlite/sqlite_utils.cpp | 57 +- .../storage/src/sqlite/sqlite_utils.h | 4 +- .../storage/src/storage_engine.cpp | 4 +- .../storage/src/storage_engine_manager.cpp | 2 +- .../storage/src/sync_able_kvdb.cpp | 16 +- .../storage/src/sync_able_kvdb.h | 10 +- .../storage/src/sync_able_kvdb_connection.cpp | 134 +- .../storage/src/sync_able_kvdb_connection.h | 9 +- .../syncer/include/isync_engine.h | 7 + .../syncer/include/isync_state_machine.h | 2 - .../syncer/include/isync_target.h | 4 +- .../syncer/include/isync_task_context.h | 12 + .../distributeddb/syncer/include/isyncer.h | 24 +- .../distributeddb/syncer/include/meta_data.h | 60 +- .../include/query_sync_water_mark_helper.h | 173 ++ .../syncer/include/sync_operation.h | 31 +- .../distributeddb/syncer/include/sync_types.h | 48 +- .../syncer/include/syncer_proxy.h | 9 + .../distributeddb/syncer/src/ability_sync.cpp | 422 ++-- .../distributeddb/syncer/src/ability_sync.h | 41 +- .../syncer/src/communicator_proxy.cpp | 254 +++ .../syncer/src/communicator_proxy.h | 60 + .../syncer/src/generic_syncer.cpp | 98 +- .../distributeddb/syncer/src/generic_syncer.h | 11 + .../distributeddb/syncer/src/meta_data.cpp | 300 ++- .../src/multi_ver_sync_state_machine.cpp | 31 +- .../syncer/src/multi_ver_sync_state_machine.h | 2 +- .../syncer/src/multi_ver_syncer.cpp | 4 +- .../src/query_sync_water_mark_helper.cpp | 569 ++++++ .../syncer/src/single_ver_data_packet.cpp | 346 ++++ .../syncer/src/single_ver_data_packet.h | 171 ++ .../syncer/src/single_ver_data_sync.cpp | 1769 ++++++++--------- .../syncer/src/single_ver_data_sync.h | 232 +-- .../src/single_ver_serialize_manager.cpp | 465 +++++ .../syncer/src/single_ver_serialize_manager.h | 60 + .../syncer/src/single_ver_sync_engine.cpp | 3 +- .../src/single_ver_sync_state_machine.cpp | 372 ++-- .../src/single_ver_sync_state_machine.h | 24 +- .../syncer/src/single_ver_sync_target.cpp | 32 +- .../syncer/src/single_ver_sync_target.h | 15 +- .../src/single_ver_sync_task_context.cpp | 87 +- .../syncer/src/single_ver_sync_task_context.h | 38 +- .../syncer/src/single_ver_syncer.cpp | 9 +- .../syncer/src/sliding_window_receiver.cpp | 18 +- .../syncer/src/sliding_window_sender.cpp | 59 +- .../syncer/src/sliding_window_sender.h | 6 +- .../distributeddb/syncer/src/sync_engine.cpp | 220 +- .../distributeddb/syncer/src/sync_engine.h | 29 +- .../syncer/src/sync_operation.cpp | 55 +- .../syncer/src/sync_state_machine.cpp | 29 +- .../syncer/src/sync_state_machine.h | 6 +- .../distributeddb/syncer/src/sync_target.cpp | 5 + .../distributeddb/syncer/src/sync_target.h | 4 +- .../syncer/src/sync_task_context.cpp | 83 +- .../syncer/src/sync_task_context.h | 19 +- .../distributeddb/syncer/src/syncer_proxy.cpp | 26 +- .../distributeddb/syncer/src/time_sync.cpp | 11 +- .../libs/distributeddb/syncer/src/time_sync.h | 2 +- .../libs/distributeddb/test/BUILD.gn | 16 +- .../common/distributeddb_auto_launch_test.cpp | 91 +- .../common/distributeddb_tools_unit_test.cpp | 42 +- .../common/distributeddb_tools_unit_test.h | 27 +- .../common/communicator/adapter_stub.cpp | 10 + .../common/communicator/adapter_stub.h | 2 + ...distributeddb_interfaces_database_test.cpp | 4 +- ...eddb_interfaces_import_and_export_test.cpp | 4 +- .../distributeddb_ability_sync_test.cpp | 46 +- .../distributeddb_anti_dos_sync_test.cpp | 5 +- .../distributeddb_multi_ver_p2p_sync_test.cpp | 21 +- ...buteddb_single_ver_p2p_sync_check_test.cpp | 31 +- ...distributeddb_single_ver_p2p_sync_test.cpp | 15 +- ...stributeddb_syncer_device_manager_test.cpp | 20 +- .../syncer/distributeddb_time_sync_test.cpp | 9 +- ...municator.cpp => virtual_communicator.cpp} | 18 +- ..._communicator.h => virtual_communicator.h} | 9 +- ...pp => virtual_communicator_aggregator.cpp} | 17 +- ...or.h => virtual_communicator_aggregator.h} | 9 +- ...{vitural_device.cpp => virtual_device.cpp} | 69 +- .../{vitural_device.h => virtual_device.h} | 12 +- .../virtual_multi_ver_sync_db_interface.cpp | 10 + .../virtual_multi_ver_sync_db_interface.h | 6 + .../virtual_single_ver_sync_db_Interface.cpp | 145 +- .../virtual_single_ver_sync_db_Interface.h | 17 +- .../syncer/virtual_time_sync_communicator.cpp | 10 + .../syncer/virtual_time_sync_communicator.h | 7 +- .../include/distributeddb_schema_test_tools.h | 7 +- .../src/distributeddb_nb_test_tools.cpp | 14 +- .../src/distributeddb_kv_create_test.cpp | 6 +- .../src/distributeddb_nb_create_test.cpp | 2 +- ...teddb_nb_enable_sync_by_closed_db_test.cpp | 2 +- .../src/distributeddb_nb_schema_test.cpp | 8 + 178 files changed, 8203 insertions(+), 2486 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/common/include/data_compression.h create mode 100644 services/distributeddataservice/libs/distributeddb/common/include/db_ability.h create mode 100644 services/distributeddataservice/libs/distributeddb/common/include/semaphore.h create mode 100644 services/distributeddataservice/libs/distributeddb/common/src/data_compression.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/common/src/semaphore.cpp rename services/distributeddataservice/libs/distributeddb/storage/{src => include}/single_ver_kv_entry.h (100%) create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.h create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/include/query_sync_water_mark_helper.h create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.h create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.h rename services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/{vitural_communicator.cpp => virtual_communicator.cpp} (91%) rename services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/{vitural_communicator.h => virtual_communicator.h} (94%) rename services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/{vitural_communicator_aggregator.cpp => virtual_communicator_aggregator.cpp} (93%) rename services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/{vitural_communicator_aggregator.h => virtual_communicator_aggregator.h} (91%) rename services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/{vitural_device.cpp => virtual_device.cpp} (79%) rename services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/{vitural_device.h => virtual_device.h} (88%) diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 6001241c0..7e9b9fbf6 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -57,6 +57,8 @@ group("build_module") { ohos_shared_library("distributeddb") { sources = [ "common/src/auto_launch.cpp", + "common/src/data_compression.cpp", + "common/src/db_ability.cpp", "common/src/db_common.cpp", "common/src/db_constant.cpp", "common/src/evloop/src/event_impl.cpp", @@ -82,7 +84,7 @@ ohos_shared_library("distributeddb") { "common/src/runtime_context_impl.cpp", "common/src/schema_object.cpp", "common/src/schema_utils.cpp", - "common/src/semaphore_utils.cpp", + "common/src/semaphore.cpp", "common/src/task_pool.cpp", "common/src/task_pool_impl.cpp", "common/src/task_queue.cpp", @@ -142,6 +144,7 @@ ohos_shared_library("distributeddb") { "storage/src/result_entries_window.cpp", "storage/src/single_ver_natural_store_commit_notify_data.cpp", "storage/src/sqlite/query_object.cpp", + "storage/src/sqlite/query_sync_object.cpp", "storage/src/sqlite/sqlite_local_kvdb.cpp", "storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -149,6 +152,7 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_local_storage_executor.cpp", "storage/src/sqlite/sqlite_multi_ver_data_storage.cpp", "storage/src/sqlite/sqlite_multi_ver_transaction.cpp", + "storage/src/sqlite/sqlite_single_ver_continue_token.cpp", "storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp", "storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store.cpp", @@ -170,6 +174,7 @@ ohos_shared_library("distributeddb") { "storage/src/upgrader/single_ver_schema_database_upgrader.cpp", "syncer/src/ability_sync.cpp", "syncer/src/commit_history_sync.cpp", + "syncer/src/communicator_proxy.cpp", "syncer/src/device_manager.cpp", "syncer/src/generic_syncer.cpp", "syncer/src/meta_data.cpp", @@ -178,8 +183,11 @@ ohos_shared_library("distributeddb") { "syncer/src/multi_ver_sync_state_machine.cpp", "syncer/src/multi_ver_sync_task_context.cpp", "syncer/src/multi_ver_syncer.cpp", + "syncer/src/query_sync_water_mark_helper.cpp", + "syncer/src/single_ver_data_packet.cpp", "syncer/src/single_ver_data_sync.cpp", "syncer/src/single_ver_data_sync_with_sliding_window.cpp", + "syncer/src/single_ver_serialize_manager.cpp", "syncer/src/single_ver_sync_engine.cpp", "syncer/src/single_ver_sync_state_machine.cpp", "syncer/src/single_ver_sync_target.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h index fb48035bc..53bed5681 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h @@ -47,6 +47,7 @@ struct AutoLaunchItem { AutoLaunchItemState state = AutoLaunchItemState::UN_INITIAL; bool isDisable = false; bool inObserver = false; + bool isAutoSync = true; }; class AutoLaunch { @@ -60,7 +61,7 @@ public: void SetCommunicatorAggregator(ICommunicatorAggregator *aggregator); int EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier); + const AutoLaunchOption &option); int DisableKvStoreAutoLaunch(const std::string &identifier); diff --git a/services/distributeddataservice/libs/distributeddb/common/include/data_compression.h b/services/distributeddataservice/libs/distributeddb/common/include/data_compression.h new file mode 100644 index 000000000..d8dbb9503 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/include/data_compression.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATA_COMPRESSION_H +#define DATA_COMPRESSION_H + +#include +#include + +namespace DistributedDB { +class DataCompression { +public: + static int Compress(const std::vector &srcData, std::vector &destData); + static int Uncompress(const std::vector &srcData, std::vector &destData, unsigned long destLen); +}; +} // namespace DistributedDB +#endif // DATA_COMPRESSION_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_ability.h b/services/distributeddataservice/libs/distributeddb/common/include/db_ability.h new file mode 100644 index 000000000..b7d3f6450 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_ability.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DB_ABILITY_H +#define DB_ABILITY_H + +#include +#include +#include +#include "macro_utils.h" +#include "parcel.h" +#include "types_export.h" + +namespace DistributedDB { +using AbilityItem = std::pair>; +constexpr uint32_t CURRENT_ABILITY_LEN = 1; // current db ability len +constexpr uint8_t SUPPORT_MARK = 1; // used for set is support one ability +// format: {bit_outer_offset, {bit_inner_offset, used_bits_num}} +constexpr AbilityItem DATABASE_COMPRESSION_ZLIB = {0, {0, 0x0000000000000001}}; // lower 8 bit 0b00000001 +const std::vector ABILITYBITS = {DATABASE_COMPRESSION_ZLIB}; + +class DbAbility { +public: + DbAbility() = default; + DbAbility(const DbAbility &other); + DbAbility& operator=(const DbAbility &other); + ~DbAbility() {}; + + bool operator==(const DbAbility &other) const; + + static int Serialize(Parcel &parcel, const DbAbility &curAbility); + + static int DeSerialize(Parcel &parcel, DbAbility &curAbility); + + static uint32_t CalculateLen(); + + void SetCompressAlgorithm(const std::set &algorithmSet); + + std::set GetCompressAlgorithm() const; + std::string GetCompressAlgorithmStr() const; + + uint64_t GetAbilityItem(const AbilityItem abilityType) const; + + void SetAbilityItem(const AbilityItem &abilityType, uint64_t data); +private: + void SetZlibCompression(uint64_t isCompression); + bool GetZlibCompression() const; + uint64_t zlibCompression_ = 0; // support zlib algorithm or not, 0: not support. 1: support + std::map CompressAlgoMap_ = { + {static_cast(CompressAlgorithm::ZLIB), DATABASE_COMPRESSION_ZLIB} + }; +}; +} // namespace DistributedDB + +#endif // DB_ABILITY_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_common.h b/services/distributeddataservice/libs/distributeddb/common/include/db_common.h index 840a3c8af..3dedbdd0d 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_common.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_common.h @@ -52,10 +52,13 @@ public: static void SetDatabaseIds(KvDBProperties &properties, const std::string &appId, const std::string &userId, const std::string &storeId); + + static std::string StringMasking(const std::string &oriStr, size_t remain = 3); // remain 3 unmask }; // Define short macro substitute for original long expression for convenience of using #define VEC_TO_STR(x) DBCommon::VectorToHexString(x).c_str() +#define STR_MASK(x) DBCommon::StringMasking(x).c_str() } // namespace DistributedDB #endif // DISTRIBUTEDDB_COMMON_H diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index a2f2dbbc1..736a6c837 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -84,12 +84,23 @@ public: static const std::string ID_CONNECTOR; static const std::string DELETE_KVSTORE_REMOVING; + static const std::string DB_LOCK_POSTFIX; static constexpr uint32_t AUTO_SYNC_TIMEOUT = 5000; // 5s static constexpr uint32_t MANUAL_SYNC_TIMEOUT = 5000; // 5s static const size_t MAX_NORMAL_PACK_ITEM_SIZE = 4000; static const size_t MAX_HPMODE_PACK_ITEM_SIZE = 2000; // slide window mode to reduce last ack transfer time + + static constexpr uint32_t MIN_MTU_SIZE = 1024; // 1KB + static constexpr uint32_t MAX_MTU_SIZE = 5242880; // 5MB + + static constexpr uint32_t MIN_TIMEOUT = 5000; // 5s + static constexpr uint32_t MAX_TIMEOUT = 60000; // 60s + + static constexpr uint8_t DEFAULT_COMPTRESS_RATE = 100; + + static constexpr int MAX_SYNC_BLOCK_SIZE = 31457280; // 30MB }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h index 7b7124814..0e4c0dbc8 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h @@ -47,6 +47,8 @@ public: static int CheckAndTransferAutoLaunchParam(const AutoLaunchParam ¶m, SchemaObject &schemaObject, std::string &canonicalDir); + + static uint8_t GetValidCompressionRate(uint8_t compressionRate); }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/parcel.h b/services/distributeddataservice/libs/distributeddb/common/include/parcel.h index 5d7b05ed4..bb5c7dbc7 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/parcel.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/parcel.h @@ -38,6 +38,8 @@ public: bool IsError() const; int WriteInt(int data); uint32_t ReadInt(int &val); + int WriteInt64(int64_t data); + uint32_t ReadInt64(int64_t &val); int WriteUInt32(uint32_t data); uint32_t ReadUInt32(uint32_t &val); int WriteUInt64(uint64_t data); @@ -46,6 +48,7 @@ public: uint32_t ReadVectorChar(std::vector &val); int WriteString(const std::string &inVal); uint32_t ReadString(std::string &outVal); + bool IsContinueRead(); #ifndef OMIT_MULTI_VER int WriteMultiVerCommit(const MultiVerCommitNode &commit); uint32_t ReadMultiVerCommit(MultiVerCommitNode &commit); @@ -58,18 +61,25 @@ public: { static_assert(std::is_pod::value, "type T is not pod"); if (data.size() > INT32_MAX || sizeof(T) > INT32_MAX) { + LOGE("[WriteVector] invalid vector. vec.size:%zu, sizeof(T):%zu", data.size(), sizeof(T)); isError_ = true; return -E_PARSE_FAIL; } + if (IsError()) { + return -E_PARSE_FAIL; + } uint32_t len = data.size(); uint64_t stepLen = static_cast(data.size()) * sizeof(T) + sizeof(uint32_t); len = HostToNet(len); - if (isError_ || bufPtr_ == nullptr || stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + if (bufPtr_ == nullptr || stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + LOGE("[WriteVector] bufPtr:%d, stepLen:%llu, totalLen:%llu, parcelLen:%llu", + bufPtr_ != nullptr, stepLen, totalLen_, parcelLen_); isError_ = true; return -E_PARSE_FAIL; } errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &len, sizeof(uint32_t)); if (errCode != EOK) { + LOGE("[ReadVector] totalLen:%llu, parcelLen:%llu", totalLen_, parcelLen_); isError_ = true; return -E_SECUREC_ERROR; } @@ -87,19 +97,25 @@ public: uint32_t ReadVector(std::vector &val) { static_assert(std::is_pod::value, "type T is not pod"); - uint32_t len; - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_ || sizeof(T) > INT32_MAX) { + if (IsError()) { + return 0; + } + if (bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_ || sizeof(T) > INT32_MAX) { + LOGE("[ReadVector] bufPtr:%d, totalLen:%llu, parcelLen:%llu, sizeof(T):%zu", + bufPtr_ != nullptr, totalLen_, parcelLen_, sizeof(T)); isError_ = true; return 0; } - len = *(reinterpret_cast(bufPtr_)); + uint32_t len = *(reinterpret_cast(bufPtr_)); len = NetToHost(len); if (len > INT32_MAX) { + LOGE("[ReadVector] invalid length:%u", len); isError_ = true; return 0; } uint64_t stepLen = static_cast(len) * sizeof(T) + sizeof(uint32_t); if (stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + LOGE("[ReadVector] stepLen:%llu, totalLen:%llu, parcelLen:%llu", stepLen, totalLen_, parcelLen_); isError_ = true; return 0; } @@ -117,10 +133,11 @@ public: int WriteBlob(const char *buffer, uint32_t bufLen); uint32_t ReadBlob(char *buffer, uint32_t bufLen); - void EightByteAlign(); + void EightByteAlign(); // Avoid reading a single data type across 8 bytes static uint32_t GetIntLen(); static uint32_t GetUInt32Len(); static uint32_t GetUInt64Len(); + static uint32_t GetInt64Len(); static uint32_t GetVectorCharLen(const std::vector &data); template @@ -144,13 +161,62 @@ public: static uint32_t GetMultiVerCommitsLen(const std::vector &commits); #endif static uint32_t GetAppendedLen(); + private: + template + int WriteInteger(T integer); + template + uint32_t ReadInteger(T &integer); + bool isError_ = false; uint8_t *buf_ = nullptr; uint8_t *bufPtr_ = nullptr; uint64_t parcelLen_ = 0; uint64_t totalLen_ = 0; }; + +template +uint32_t Parcel::ReadInteger(T &integer) +{ + if (IsError()) { + return 0; + } + if (bufPtr_ == nullptr || parcelLen_ + sizeof(T) > totalLen_) { + LOGE("[ReadInteger] bufPtr:%d, totalLen:%llu, parcelLen:%llu, sizeof(T):%zu", + bufPtr_ != nullptr, totalLen_, parcelLen_, sizeof(T)); + isError_ = true; + return 0; + } + integer = *(reinterpret_cast(bufPtr_)); + bufPtr_ += sizeof(T); + parcelLen_ += sizeof(T); + integer = NetToHost(integer); + return sizeof(T); +} + +template +int Parcel::WriteInteger(T integer) +{ + if (IsError()) { + return -E_PARSE_FAIL; + } + T inData = HostToNet(integer); + if (parcelLen_ + sizeof(T) > totalLen_) { + LOGE("[WriteInteger] totalLen:%llu, parcelLen:%llu, sizeof(T):%zu", totalLen_, parcelLen_, sizeof(T)); + isError_ = true; + return -E_PARSE_FAIL; + } + errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(T)); + if (errCode != EOK) { + LOGE("[WriteInteger] bufPtr:%d, totalLen:%llu, parcelLen:%llu, sizeof(T):%zu", + bufPtr_ != nullptr, totalLen_, parcelLen_, sizeof(T)); + isError_ = true; + return -E_SECUREC_ERROR; + } + bufPtr_ += sizeof(T); + parcelLen_ += sizeof(T); + return errCode; +} } // namespace DistributedDB #endif // PARCEL_H diff --git a/services/distributeddataservice/libs/distributeddb/common/include/platform_specific.h b/services/distributeddataservice/libs/distributeddb/common/include/platform_specific.h index b95beb866..0c2773f01 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/platform_specific.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/platform_specific.h @@ -34,6 +34,11 @@ struct FileAttr { uint64_t fileLen; }; +// Shield the representation method of file handles on different platforms +struct FileHandle { + int handle = -1; +}; + int CalFileSize(const std::string &fileUrl, uint64_t &size); bool CheckPathExistence(const std::string &filePath); @@ -61,6 +66,12 @@ int GetFilePermissions(const std::string &fileName, uint32_t &permissions); int SetFilePermissions(const std::string &fileName, uint32_t permissions); int RenameFilePath(const std::string &oldFilePath, const std::string &newFilePath); + +int OpenFile(const std::string &fileName, FileHandle &handle); +int CloseFile(FileHandle &handle); + +int FileLock(const FileHandle &handle, bool isBlock); // be careful use block=true, may block process +int FileUnlock(FileHandle &handle); } // namespace OS } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h index 34622e868..ac76d7a6d 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h @@ -44,6 +44,7 @@ public: // Global setting interfaces. virtual void SetProcessLabel(const std::string &label) = 0; virtual std::string GetProcessLabel() const = 0; + // If the pre adapter is not nullptr, set new adapter will release the pre adapter, // must be called after SetCommunicatorAggregator virtual int SetCommunicatorAdapter(IAdapter *adapter) = 0; @@ -78,7 +79,7 @@ public: const std::string &deviceId, uint8_t flag) const = 0; virtual int EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier) = 0; + const AutoLaunchOption &option) = 0; virtual int DisableKvStoreAutoLaunch(const std::string &identifier) = 0; @@ -86,7 +87,7 @@ public: virtual void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) = 0; - virtual NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errorCode) = 0; + virtual NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errCode) = 0; virtual bool IsAccessControlled() const = 0; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/semaphore.h b/services/distributeddataservice/libs/distributeddb/common/include/semaphore.h new file mode 100644 index 000000000..6372363f0 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/include/semaphore.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SEMAPHORE_H +#define SEMAPHORE_H + +#include +#include +#include +#include +#include "macro_utils.h" + +namespace DistributedDB { +class Semaphore { +public: + explicit Semaphore(int count); + ~Semaphore(); + + // Delete the copy and assign constructors + DISABLE_COPY_ASSIGN_MOVE(Semaphore); + + bool WaitSemaphore(int waitSecond); + + void WaitSemaphore(); + + void SendSemaphore(); + +private: + bool CompareCount() const; + std::mutex lockMutex_; + std::condition_variable cv_; + int count_; +}; +} + +#endif diff --git a/services/distributeddataservice/libs/distributeddb/common/include/version.h b/services/distributeddataservice/libs/distributeddb/common/include/version.h index 4ee201df8..a535c85c7 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/version.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/version.h @@ -30,13 +30,14 @@ namespace DistributedDB { // if now version of B is 105, and C is 101, thus version of A is 105; if now release version is 106 and we upgrade // submodule C, if we simply change version of C to 102 then version of A is still 105, but if we change version of C // to 106 then version of A is now 106, so we can know that something had changed for module A. -const std::string SOFTWARE_VERSION_STRING = "1.1.3"; // DistributedDB current version string. +const std::string SOFTWARE_VERSION_STRING = "1.1.4"; // DistributedDB current version string. constexpr uint32_t SOFTWARE_VERSION_BASE = 100; // Software version base value, do not change it constexpr uint32_t SOFTWARE_VERSION_RELEASE_1_0 = SOFTWARE_VERSION_BASE + 1; // 1 for first released version constexpr uint32_t SOFTWARE_VERSION_RELEASE_2_0 = SOFTWARE_VERSION_BASE + 2; // 2 for second released version constexpr uint32_t SOFTWARE_VERSION_RELEASE_3_0 = SOFTWARE_VERSION_BASE + 3; // 3 for second released version +constexpr uint32_t SOFTWARE_VERSION_RELEASE_4_0 = SOFTWARE_VERSION_BASE + 4; // 4 for fourth released version constexpr uint32_t SOFTWARE_VERSION_EARLIEST = SOFTWARE_VERSION_RELEASE_1_0; -constexpr uint32_t SOFTWARE_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_3_0; +constexpr uint32_t SOFTWARE_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_4_0; constexpr int VERSION_INVALID = INT32_MAX; // Storage Related Version diff --git a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp index 1354f8ee4..62aad7ee1 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp @@ -24,7 +24,7 @@ #include "kvdb_manager.h" #include "kv_store_changed_data_impl.h" #include "sync_able_kvdb_connection.h" -#include "semaphore_utils.h" +#include "semaphore.h" #include "kv_store_nb_conflict_data_impl.h" #include "db_common.h" #include "param_check_utils.h" @@ -132,11 +132,12 @@ int AutoLaunch::EnableKvStoreAutoLaunchParmCheck(AutoLaunchItem &autoLaunchItem, } int AutoLaunch::EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier) + const AutoLaunchOption &option) { LOGI("[AutoLaunch] EnableKvStoreAutoLaunch"); std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); - AutoLaunchItem autoLaunchItem{properties, notifier, observer, conflictType, conflictNotifier}; + AutoLaunchItem autoLaunchItem { properties, notifier, option.observer, option.conflictType, option.notifier }; + autoLaunchItem.isAutoSync = option.isAutoSync; int errCode = EnableKvStoreAutoLaunchParmCheck(autoLaunchItem, identifier); if (errCode != E_OK) { LOGE("[AutoLaunch] EnableKvStoreAutoLaunch failed errCode:%d", errCode); @@ -277,14 +278,14 @@ int AutoLaunch::RegisterObserverAndLifeCycleCallback(AutoLaunchItem &autoLaunchI return errCode; } - bool enAutoSync = true; + bool enAutoSync = autoLaunchItem.isAutoSync; errCode = static_cast(autoLaunchItem.conn)->Pragma(PRAGMA_AUTO_SYNC, static_cast(&enAutoSync)); if (errCode != E_OK) { LOGE("[AutoLaunch] PRAGMA_AUTO_SYNC failed, errCode:%d", errCode); return errCode; } - LOGI("[AutoLaunch] PRAGMA_AUTO_SYNC ok"); + LOGI("[AutoLaunch] set PRAGMA_AUTO_SYNC ok, enAutoSync=%d", enAutoSync); return errCode; } @@ -561,7 +562,7 @@ void AutoLaunch::GetConnInDoOpenMap(std::map &doOpe if (doOpenMap.empty()) { return; } - SemaphoreUtils sema(1 - doOpenMap.size()); + Semaphore sema(1 - doOpenMap.size()); for (auto &iter : doOpenMap) { int errCode = RuntimeContext::GetInstance()->ScheduleTask([&sema, &iter, this] { int errCode; @@ -724,6 +725,7 @@ int AutoLaunch::AutoLaunchExt(const std::string &identifier) } AutoLaunchItem autoLaunchItem{properties, param.notifier, param.option.observer, param.option.conflictType, param.option.notifier}; + autoLaunchItem.isAutoSync = param.option.isAutoSync; errCode = RuntimeContext::GetInstance()->ScheduleTask(std::bind(&AutoLaunch::AutoLaunchExtTask, this, identifier, autoLaunchItem)); if (errCode != E_OK) { @@ -904,6 +906,11 @@ int AutoLaunch::GetAutoLaunchProperties(const AutoLaunchParam ¶m, KvDBProper properties.SetIntProp(KvDBProperties::SECURITY_LABEL, param.option.secOption.securityLabel); properties.SetIntProp(KvDBProperties::SECURITY_FLAG, param.option.secOption.securityFlag); } + properties.SetBoolProp(KvDBProperties::COMPRESS_ON_SYNC, param.option.isNeedCompressOnSync); + if (param.option.isNeedCompressOnSync) { + properties.SetIntProp(KvDBProperties::COMPRESSION_RATE, + ParamCheckUtils::GetValidCompressionRate(param.option.compressionRate)); + } DBCommon::SetDatabaseIds(properties, param.appId, param.userId, param.storeId); return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/data_compression.cpp b/services/distributeddataservice/libs/distributeddb/common/src/data_compression.cpp new file mode 100644 index 000000000..3b1df1171 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/src/data_compression.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "data_compression.h" +#ifdef USING_DATA_COMPRESS +#include +#endif + +#include "db_errno.h" +#include "log_print.h" + +namespace DistributedDB { +namespace { +#ifdef USING_DATA_COMPRESS + constexpr int MAX_SYNC_BLOCK_SIZE = 31457280; // 30M +#endif +} + +int DataCompression::Compress(const std::vector &srcData, std::vector &destData) +{ +#ifdef USING_DATA_COMPRESS + auto srcLen = srcData.size(); + auto destLen = compressBound(srcLen); + if (srcLen > MAX_SYNC_BLOCK_SIZE || destLen > MAX_SYNC_BLOCK_SIZE) { + LOGE("Too long to compress, srcLen:%u, destLen:%u.", srcLen, destLen); + return -E_INVALID_ARGS; + } + + // Alloc memory. + destData.resize(destLen, 0); + + // Compress. + LOGI("Src length is %lu.", srcLen); + int errCode = compress(destData.data(), &destLen, srcData.data(), srcLen); + if (errCode != Z_OK) { + LOGE("Compress parcel failed, errCode = %d", errCode); + return -E_SYSTEM_API_FAIL; + } + LOGI("Compressed length is %lu.", destLen); + + destData.resize(destLen); + destData.shrink_to_fit(); +#else + destData = srcData; +#endif + return E_OK; +} + +int DataCompression::Uncompress(const std::vector &srcData, std::vector &destData, + unsigned long destLen) +{ +#ifdef USING_DATA_COMPRESS + auto srcLen = srcData.size(); + if (srcLen > MAX_SYNC_BLOCK_SIZE || destLen > MAX_SYNC_BLOCK_SIZE) { + LOGE("Too long to uncompress, srcLen:%u, destLen:%u.", srcLen, destLen); + return -E_INVALID_ARGS; + } + + // Alloc dest memory. + destData.resize(destLen, 0); + + // Uncompress. + int errCode = uncompress(destData.data(), &destLen, srcData.data(), srcData.size()); + if (errCode != Z_OK) { + LOGE("Uncompress failed, errCode = %d", errCode); + return -E_SYSTEM_API_FAIL; + } + + destData.resize(destLen); + destData.shrink_to_fit(); +#else + destData = srcData; +#endif + return E_OK; +} +} // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp new file mode 100644 index 000000000..1e9c020ab --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "db_ability.h" + +#include +#include +#include +#include "db_errno.h" +#include "types_export.h" + +namespace DistributedDB { +DbAbility::DbAbility(const DbAbility &other) +{ + if (&other != this) { + zlibCompression_ = other.zlibCompression_; + } +} + +DbAbility& DbAbility::operator=(const DbAbility &other) +{ + if (&other != this) { + zlibCompression_ = other.zlibCompression_; + } + return *this; +} + +bool DbAbility::operator==(const DbAbility &other) const +{ + return zlibCompression_ == other.zlibCompression_; +} + +int DbAbility::Serialize(Parcel &parcel, const DbAbility &curAbility) +{ + std::vector dstBuf(CURRENT_ABILITY_LEN, 0); + for (const auto &item : ABILITYBITS) { + dstBuf[item.first] = dstBuf[item.first] | (curAbility.GetAbilityItem(item) << item.second.first); + } + int errCode = parcel.WriteVector(dstBuf); + if (errCode != E_OK) { + return errCode; + } + return E_OK; +} + +int DbAbility::DeSerialize(Parcel &parcel, DbAbility &curAbility) +{ + if (!parcel.IsContinueRead()) { + return E_OK; + } + std::vector dstBuf; + parcel.ReadVector(dstBuf); + if (parcel.IsError()) { + LOGE("[DbAbility][DeSerialize] deserialize failed."); + return -E_LENGTH_ERROR; + } + if (dstBuf.size() == 0) { + LOGE("[DbAbility][DeSerialize] buf length get failed."); + return -E_LENGTH_ERROR; + } + for (const auto &item : ABILITYBITS) { + if (item.first >= dstBuf.size()) { + LOGI("[DbAbility][DeSerialize] recv buf len is less than cur abilitybits[%d]", ABILITYBITS.size()); + break; + } + uint64_t data = (dstBuf[item.first] & item.second.second) >> item.second.first; + curAbility.SetAbilityItem(item, data); + } + return E_OK; +} + +uint32_t DbAbility::CalculateLen() +{ + return Parcel::GetVectorLen(std::vector(CURRENT_ABILITY_LEN, 0)); +} + +void DbAbility::SetCompressAlgorithm(const std::set &algorithmSet) +{ + for (const auto &item : algorithmSet) { + uint8_t compressAlgo = static_cast(item); + if (CompressAlgoMap_.find(compressAlgo) != CompressAlgoMap_.end()) { + SetAbilityItem(CompressAlgoMap_[compressAlgo], SUPPORT_MARK); + } + } +} + +std::set DbAbility::GetCompressAlgorithm() const +{ + std::set remoteCompressAlgoSet; + for (const auto &item : CompressAlgoMap_) { + if (GetAbilityItem(item.second) == SUPPORT_MARK) { + CompressAlgorithm compressAlgo = static_cast(item.first); + remoteCompressAlgoSet.insert(compressAlgo); + } + } + return remoteCompressAlgoSet; +} + +std::string DbAbility::GetCompressAlgorithmStr() const +{ + static std::map algoMap = {{CompressAlgorithm::ZLIB, "zlib"}}; + std::set remoteCompressAlgoSet = GetCompressAlgorithm(); + if (remoteCompressAlgoSet.size() == 0) { + return "none"; + } + std::string currentAlgoStr; + for (const auto &algo : remoteCompressAlgoSet) { + auto iter = algoMap.find(algo); + if (iter != algoMap.end()) { + currentAlgoStr += algoMap[algo] + ","; + } + } + return currentAlgoStr.substr(0, currentAlgoStr.length() - 1); +} + +void DbAbility::SetZlibCompression(uint64_t isCompression) +{ + zlibCompression_ = isCompression; +} + +bool DbAbility::GetZlibCompression() const +{ + return static_cast(zlibCompression_); +} + +uint64_t DbAbility::GetAbilityItem(const AbilityItem abilityType) const +{ + std::map abilityMap = { + {DATABASE_COMPRESSION_ZLIB, zlibCompression_}, + }; + if (abilityMap.find(abilityType) != abilityMap.end()) { + return abilityMap[abilityType]; + } + return 0; +} + +void DbAbility::SetAbilityItem(const AbilityItem &abilityType, uint64_t data) +{ + std::map> itemFuncMap = { + {DATABASE_COMPRESSION_ZLIB, [this](uint64_t data) { SetZlibCompression(data); }} + }; + auto iter = itemFuncMap.find(abilityType); + if (iter != itemFuncMap.end()) { + iter->second(data); + return; + } +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp index d326f97ab..a26f21ec5 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp @@ -31,8 +31,8 @@ namespace { if (item.fileType != type) { continue; } - int errCode = remove(item.fileName.c_str()); - if (errCode != 0) { + int errCode = OS::RemoveFile(item.fileName.c_str()); + if (errCode != E_OK) { LOGE("Remove file failed:%d", errno); } } @@ -40,11 +40,11 @@ namespace { void RemoveDirectories(const std::list &fileList, OS::FileType type) { - for (const auto &item : fileList) { - if (item.fileType != type) { + for (auto item = fileList.rbegin(); item != fileList.rend(); ++item) { + if (item->fileType != type) { continue; } - int errCode = OS::RemoveDBDirectory(item.fileName); + int errCode = OS::RemoveDBDirectory(item->fileName); if (errCode != 0) { LOGE("Remove directory failed:%d", errno); } @@ -175,19 +175,24 @@ int DBCommon::CalcValueHash(const std::vector &value, std::vector remain) { + return oriStr.substr(0, remain); + } +#endif + return oriStr; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp index 827121d4d..b1ecbc6ac 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp @@ -55,4 +55,5 @@ const std::string DBConstant::PATH_BACKUP_POSTFIX = "_bak"; const std::string DBConstant::ID_CONNECTOR = "-"; const std::string DBConstant::DELETE_KVSTORE_REMOVING = "_removing"; +const std::string DBConstant::DB_LOCK_POSTFIX = ".lock"; } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp b/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp index 0086834b9..e313bd4b4 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp @@ -177,4 +177,14 @@ int ParamCheckUtils::CheckAndTransferAutoLaunchParam(const AutoLaunchParam ¶ } return E_OK; } + +uint8_t ParamCheckUtils::GetValidCompressionRate(uint8_t compressionRate) +{ + // Valid when between 1 and 100. When compressionRate is invalid, change it to default rate. + if (compressionRate < 1 || compressionRate > DBConstant::DEFAULT_COMPTRESS_RATE) { + LOGD("Invalid compression rate:%u.", compressionRate); + compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + } + return compressionRate; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp b/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp index 08292ccd1..93a2b23ba 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp @@ -48,92 +48,42 @@ bool Parcel::IsError() const int Parcel::WriteInt(int32_t data) { - int32_t inData = HostToNet(data); - if (isError_ || parcelLen_ + sizeof(int32_t) > totalLen_) { - isError_ = true; - return -E_PARSE_FAIL; - } - errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(int32_t)); - if (errCode != EOK) { - isError_ = true; - return -E_SECUREC_ERROR; - } - bufPtr_ += sizeof(int32_t); - parcelLen_ += sizeof(int32_t); - return errCode; + return WriteInteger(data); } uint32_t Parcel::ReadInt(int32_t &val) { - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(int32_t) > totalLen_) { - isError_ = true; - return 0; - } - val = *(reinterpret_cast(bufPtr_)); - bufPtr_ += sizeof(int32_t); - parcelLen_ += sizeof(int32_t); - val = NetToHost(val); - return sizeof(int32_t); + return ReadInteger(val); +} + +int Parcel::WriteInt64(int64_t data) +{ + return WriteInteger(data); +} + +uint32_t Parcel::ReadInt64(int64_t &val) +{ + return ReadInteger(val); } int Parcel::WriteUInt32(uint32_t data) { - uint32_t inData = HostToNet(data); - if (isError_ || parcelLen_ + sizeof(uint32_t) > totalLen_) { - isError_ = true; - return -E_PARSE_FAIL; - } - errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(uint32_t)); - if (errCode != EOK) { - isError_ = true; - return -E_SECUREC_ERROR; - } - bufPtr_ += sizeof(uint32_t); - parcelLen_ += sizeof(uint32_t); - return errCode; + return WriteInteger(data); } uint32_t Parcel::ReadUInt32(uint32_t &val) { - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_) { - isError_ = true; - return 0; - } - val = *(reinterpret_cast(bufPtr_)); - bufPtr_ += sizeof(uint32_t); - parcelLen_ += sizeof(uint32_t); - val = NetToHost(val); - return sizeof(uint32_t); + return ReadInteger(val); } int Parcel::WriteUInt64(uint64_t data) { - uint64_t inData = HostToNet(data); - if (isError_ || parcelLen_ + sizeof(uint64_t) > totalLen_) { - isError_ = true; - return -E_PARSE_FAIL; - } - errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(uint64_t)); - if (errCode != EOK) { - isError_ = true; - return -E_SECUREC_ERROR; - } - bufPtr_ += sizeof(uint64_t); - parcelLen_ += sizeof(uint64_t); - return errCode; + return WriteInteger(data); } uint32_t Parcel::ReadUInt64(uint64_t &val) { - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(uint64_t) > totalLen_) { - isError_ = true; - return 0; - } - val = *(reinterpret_cast(bufPtr_)); - bufPtr_ += sizeof(uint64_t); - parcelLen_ += sizeof(uint64_t); - val = NetToHost(val); - return sizeof(uint64_t); + return ReadInteger(val); } int Parcel::WriteVectorChar(const std::vector& data) @@ -149,18 +99,24 @@ uint32_t Parcel::ReadVectorChar(std::vector& val) int Parcel::WriteString(const std::string &inVal) { if (inVal.size() > INT32_MAX) { + LOGE("[WriteString] Invalid string, size:%zu.", inVal.size()); isError_ = true; return -E_PARSE_FAIL; } + if (IsError()) { + return -E_PARSE_FAIL; + } uint32_t len = inVal.size(); uint64_t stepLen = sizeof(uint32_t) + static_cast(inVal.size()); len = HostToNet(len); - if (isError_ || stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + if (stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + LOGE("[WriteString] stepLen:%llu, totalLen:%llu, parcelLen:%llu", stepLen, totalLen_, parcelLen_); isError_ = true; return -E_PARSE_FAIL; } errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &len, sizeof(uint32_t)); if (errCode != EOK) { + LOGE("[WriteString] bufPtr:%d, totalLen:%llu, parcelLen:%llu", bufPtr_ != nullptr, totalLen_, parcelLen_); isError_ = true; return -E_SECUREC_ERROR; } @@ -172,18 +128,24 @@ int Parcel::WriteString(const std::string &inVal) } errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_ - sizeof(uint32_t), inVal.c_str(), inVal.size()); if (errCode != EOK) { + LOGE("[WriteString] totalLen:%llu, parcelLen:%llu, inVal.size:%zu.", + totalLen_, parcelLen_, inVal.size()); isError_ = true; return -E_SECUREC_ERROR; } bufPtr_ += inVal.size(); bufPtr_ += BYTE_8_ALIGN(stepLen) - stepLen; parcelLen_ += BYTE_8_ALIGN(stepLen); - return errCode; + return E_OK; } uint32_t Parcel::ReadString(std::string &outVal) { - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_) { + if (IsError()) { + return 0; + } + if (bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_) { + LOGE("[ReadString] bufPtr:%d, totalLen:%llu, parcelLen:%llu", bufPtr_ != nullptr, totalLen_, parcelLen_); isError_ = true; return 0; } @@ -191,6 +153,7 @@ uint32_t Parcel::ReadString(std::string &outVal) len = NetToHost(len); uint64_t stepLen = static_cast(len) + sizeof(uint32_t); if (stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + LOGE("[ReadString] stepLen:%llu, totalLen:%llu, parcelLen:%llu", stepLen, totalLen_, parcelLen_); isError_ = true; return 0; } @@ -202,6 +165,11 @@ uint32_t Parcel::ReadString(std::string &outVal) return static_cast(stepLen); } +bool Parcel::IsContinueRead() +{ + return (parcelLen_ < totalLen_); +} + #ifndef OMIT_MULTI_VER int Parcel::WriteMultiVerCommit(const MultiVerCommitNode &commit) { @@ -271,8 +239,39 @@ int Parcel::WriteMultiVerCommits(const std::vector &commits) return errCode; } for (auto &iter : commits) { - errCode = WriteMultiVerCommit(iter); + errCode = WriteVectorChar(iter.commitId); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write commitId err!"); + return errCode; + } + errCode = WriteVectorChar(iter.leftParent); if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write leftParent err!"); + return errCode; + } + errCode = WriteVectorChar(iter.rightParent); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write rightParent err!"); + return errCode; + } + errCode = WriteUInt64(iter.timestamp); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write timestamp err!"); + return errCode; + } + errCode = WriteUInt64(iter.version); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write version err!"); + return errCode; + } + errCode = WriteUInt64(iter.isLocal); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write isLocal err!"); + return errCode; + } + errCode = WriteString(iter.deviceInfo); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write deviceInfo err!"); return errCode; } EightByteAlign(); @@ -296,7 +295,13 @@ uint32_t Parcel::ReadMultiVerCommits(std::vector &commits) } for (uint64_t i = 0; i < size; i++) { MultiVerCommitNode commit; - len += ReadMultiVerCommit(commit); + len += ReadVectorChar(commit.commitId); + len += ReadVectorChar(commit.leftParent); + len += ReadVectorChar(commit.rightParent); + len += ReadUInt64(commit.timestamp); + len += ReadUInt64(commit.version); + len += ReadUInt64(commit.isLocal); + len += ReadString(commit.deviceInfo); commits.push_back(commit); EightByteAlign(); len = BYTE_8_ALIGN(len); @@ -314,37 +319,49 @@ uint32_t Parcel::ReadMultiVerCommits(std::vector &commits) int Parcel::WriteBlob(const char *buffer, uint32_t bufLen) { if (buffer == nullptr) { + LOGE("[WriteBlob] Invalid buffer."); isError_ = true; return -E_INVALID_ARGS; } - if (isError_ || parcelLen_ + bufLen > totalLen_) { + if (IsError()) { + return -E_PARSE_FAIL; + } + if (parcelLen_ + bufLen > totalLen_) { + LOGE("[WriteBlob] bufLen:%u, totalLen:%llu, parcelLen:%llu", bufLen, totalLen_, parcelLen_); isError_ = true; return -E_PARSE_FAIL; } uint32_t leftLen = static_cast(totalLen_ - parcelLen_); int errCode = memcpy_s(bufPtr_, leftLen, buffer, bufLen); if (errCode != EOK) { + LOGE("[WriteBlob] bufPtr:%d, leftLen:%u, buffer:%p, bufLen:%u", bufPtr_ != nullptr, leftLen, buffer, bufLen); isError_ = true; return -E_SECUREC_ERROR; } uint32_t length = (BYTE_8_ALIGN(bufLen) < leftLen) ? BYTE_8_ALIGN(bufLen) : leftLen; bufPtr_ += length; parcelLen_ += length; - return errCode; + return E_OK; } uint32_t Parcel::ReadBlob(char *buffer, uint32_t bufLen) { if (buffer == nullptr) { + LOGE("[ReadBlob] Invalid buffer."); isError_ = true; return 0; } + if (IsError()) { + return 0; + } uint32_t leftLen = static_cast(totalLen_ - parcelLen_); - if (isError_ || parcelLen_ + bufLen > totalLen_) { + if (parcelLen_ + bufLen > totalLen_) { + LOGE("[ReadBlob] bufLen:%u, totalLen:%llu, parcelLen:%llu", bufLen, totalLen_, parcelLen_); isError_ = true; return 0; } int errCode = memcpy_s(buffer, bufLen, bufPtr_, bufLen); if (errCode != EOK) { + LOGE("[ReadBlob] bufPtr:%d, buffer:%p, bufLen:%u", bufPtr_ != nullptr, buffer, bufLen); isError_ = true; return 0; } @@ -369,6 +386,11 @@ uint32_t Parcel::GetUInt64Len() return sizeof(uint64_t); } +uint32_t Parcel::GetInt64Len() +{ + return sizeof(int64_t); +} + uint32_t Parcel::GetVectorCharLen(const std::vector &data) { return GetVectorLen(data); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/platform_specific.cpp b/services/distributeddataservice/libs/distributeddb/common/src/platform_specific.cpp index c56ba8f53..8869b3860 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/platform_specific.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/platform_specific.cpp @@ -54,6 +54,7 @@ int RenameFilePath(const std::string &oldFilePath, const std::string &newFilePat LOGE("[Rename] Rename file fail. err = %d", errno); return -E_SYSTEM_API_FAIL; } + LOGI("Rename file path successfully!"); return E_OK; } @@ -64,6 +65,7 @@ int RemoveFile(const std::string &filePath) LOGE("[RemoveFile] Remove file fail. err = %d", errno); return -E_SYSTEM_API_FAIL; } + LOGI("Remove file successfully!"); return E_OK; } @@ -71,7 +73,7 @@ int CalFileSize(const std::string &fileUrl, uint64_t &size) { struct stat fileStat; if (fileUrl.empty() || stat(fileUrl.c_str(), &fileStat) < 0 || fileStat.st_size < 0) { - LOGE("Get file[%zu] size failed, errno [%d].", fileUrl.size(), errno); + LOGD("Get file[%zu] size failed, errno [%d].", fileUrl.size(), errno); return -E_INVALID_ARGS; } @@ -258,5 +260,57 @@ int SetFilePermissions(const std::string &fileName, uint32_t permissions) } return E_OK; } + +int OpenFile(const std::string &fileName, FileHandle &handle) +{ + handle.handle = open(fileName.c_str(), (O_WRONLY | O_CREAT), (S_IRUSR | S_IWUSR | S_IRGRP)); + if (handle.handle < 0) { + return -E_SYSTEM_API_FAIL; + } + return E_OK; +} + +int CloseFile(FileHandle &handle) +{ + if (close(handle.handle) != 0) { + return -E_SYSTEM_API_FAIL; + } + handle.handle = -1; + return E_OK; +} + +int FileLock(const FileHandle &handle, bool isBlock) +{ + if (handle.handle < 0) { + return -E_SYSTEM_API_FAIL; + } + struct flock fileLockInfo; + (void)memset_s(&fileLockInfo, sizeof(fileLockInfo), 0, sizeof(fileLockInfo)); + fileLockInfo.l_type = F_WRLCK; + fileLockInfo.l_whence = SEEK_SET; + fileLockInfo.l_start = 0; + fileLockInfo.l_len = 0; + if (fcntl(handle.handle, isBlock ? F_SETLKW : F_SETLK, &fileLockInfo) == -1 && !isBlock) { + return -E_BUSY; + } + return E_OK; +} + +int FileUnlock(FileHandle &handle) +{ + if (handle.handle < 0) { + return E_OK; + } + struct flock fileLockInfo; + (void)memset_s(&fileLockInfo, sizeof(fileLockInfo), 0, sizeof(fileLockInfo)); + fileLockInfo.l_type = F_UNLCK; + fileLockInfo.l_whence = SEEK_SET; + fileLockInfo.l_start = 0; + fileLockInfo.l_len = 0; + if (fcntl(handle.handle, F_SETLK, &fileLockInfo) == -1) { + return -E_SYSTEM_API_FAIL; + } + return CloseFile(handle); +} } // namespace OS } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp index 72aeeab76..7afc9223a 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp @@ -195,12 +195,12 @@ const std::list &QueryExpression::GetQueryExpression() return queryInfo_; } -std::vector QueryExpression::GetPreFixKey() +std::vector QueryExpression::GetPreFixKey() const { return prefixKey_; } -std::string QueryExpression::GetSuggestIndex() +std::string QueryExpression::GetSuggestIndex() const { return suggestIndex_; } @@ -217,6 +217,15 @@ void QueryExpression::EndGroup() QueryValueType::VALUE_TYPE_NULL, std::vector()}); } +void QueryExpression::Reset() +{ + queryInfo_.clear(); + errFlag_ = true; + prefixKey_.clear(); + prefixKey_.shrink_to_fit(); + suggestIndex_.clear(); +} + void QueryExpression::SetErrFlag(bool flag) { errFlag_ = flag; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp index 445de2cc2..faaa671e6 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp @@ -382,9 +382,9 @@ int RuntimeContextImpl::RunPermissionCheck(const std::string &userId, const std: } int RuntimeContextImpl::EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier) + const AutoLaunchOption &option) { - return autoLaunch_.EnableKvStoreAutoLaunch(properties, notifier, observer, conflictType, conflictNotifier); + return autoLaunch_.EnableKvStoreAutoLaunch(properties, notifier, option); } int RuntimeContextImpl::DisableKvStoreAutoLaunch(const std::string &identifier) @@ -408,7 +408,7 @@ NotificationChain::Listener *RuntimeContextImpl::RegisterLockStatusLister(const { std::lock(lockStatusLock_, systemApiAdapterLock_); std::lock_guard lockStatusLock(lockStatusLock_, std::adopt_lock); - std::lock_guard systemApiAdapterLock(systemApiAdapterLock_, std::adopt_lock); + std::lock_guard systemApiAdapterLock(systemApiAdapterLock_, std::adopt_lock); if (lockStatusObserver_ == nullptr) { lockStatusObserver_ = new (std::nothrow) LockStatusObserver(); if (lockStatusObserver_ == nullptr) { @@ -452,7 +452,7 @@ NotificationChain::Listener *RuntimeContextImpl::RegisterLockStatusLister(const bool RuntimeContextImpl::IsAccessControlled() const { - std::lock_guard autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr) { return false; } @@ -461,7 +461,7 @@ bool RuntimeContextImpl::IsAccessControlled() const int RuntimeContextImpl::SetSecurityOption(const std::string &filePath, const SecurityOption &option) const { - std::lock_guard autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr || !OS::CheckPathExistence(filePath)) { LOGI("Adapter is not set, or path not existed, not support set security option!"); return -E_NOT_SUPPORT; @@ -492,7 +492,7 @@ int RuntimeContextImpl::SetSecurityOption(const std::string &filePath, const Sec int RuntimeContextImpl::GetSecurityOption(const std::string &filePath, SecurityOption &option) const { - std::lock_guard autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr) { LOGI("Get Security option, but not set system api adapter!"); return -E_NOT_SUPPORT; @@ -521,7 +521,7 @@ int RuntimeContextImpl::GetSecurityOption(const std::string &filePath, SecurityO bool RuntimeContextImpl::CheckDeviceSecurityAbility(const std::string &devId, const SecurityOption &option) const { - std::lock_guard autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr) { return true; } @@ -532,7 +532,7 @@ int RuntimeContextImpl::SetProcessSystemApiAdapter(const std::shared_ptr lockStatusLock(lockStatusLock_, std::adopt_lock); - std::lock_guard systemApiAdapterLock(systemApiAdapterLock_, std::adopt_lock); + std::lock_guard systemApiAdapterLock(systemApiAdapterLock_, std::adopt_lock); systemApiAdapter_ = adapter; if (systemApiAdapter_ != nullptr && lockStatusObserver_ != nullptr && lockStatusObserver_->IsStarted()) { auto callback = std::bind(&LockStatusObserver::OnStatusChange, @@ -550,7 +550,7 @@ int RuntimeContextImpl::SetProcessSystemApiAdapter(const std::shared_ptr autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr) { return false; } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h index 43f077785..ce5c227ec 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h @@ -70,7 +70,7 @@ public: const std::string &deviceId, uint8_t flag) const override; int EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier) override; + const AutoLaunchOption &option) override; int DisableKvStoreAutoLaunch(const std::string &identifier) override; @@ -78,7 +78,7 @@ public: void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) override; - NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errorCode) override; + NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errCode) override; bool IsAccessControlled() const override; @@ -93,6 +93,7 @@ public: bool IsProcessSystemApiAdapterValid() const override; bool IsCommunicatorAggregatorValid() const override; + // Notify TIME_CHANGE_EVENT. void NotifyTimeStampChanged(TimeOffset offset) const override; @@ -137,7 +138,7 @@ private: AutoLaunch autoLaunch_; // System api - mutable std::mutex systemApiAdapterLock_; + mutable std::recursive_mutex systemApiAdapterLock_; // check security ability may open store std::shared_ptr systemApiAdapter_; mutable std::mutex lockStatusLock_; // Mutex for lockStatusObserver_. LockStatusObserver *lockStatusObserver_; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/semaphore.cpp b/services/distributeddataservice/libs/distributeddb/common/src/semaphore.cpp new file mode 100644 index 000000000..f2ef2096a --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/src/semaphore.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "semaphore.h" + +#include + +namespace DistributedDB { +using std::unique_lock; +using std::mutex; +using std::condition_variable; + +Semaphore::Semaphore(int count) + : count_(count) +{} + +Semaphore::~Semaphore() +{} + +bool Semaphore::WaitSemaphore(int waitSecond) +{ + unique_lock lock(lockMutex_); + bool result = cv_.wait_for(lock, std::chrono::seconds(waitSecond), + std::bind(&Semaphore::CompareCount, this)); + if (result == true) { + --count_; + } + return result; +} + +void Semaphore::WaitSemaphore() +{ + unique_lock lock(lockMutex_); + cv_.wait(lock, std::bind(&Semaphore::CompareCount, this)); + --count_; +} + +void Semaphore::SendSemaphore() +{ + unique_lock lock(lockMutex_); + count_++; + cv_.notify_one(); +} + +bool Semaphore::CompareCount() const +{ + return count_ > 0; +} +} diff --git a/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.h b/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.h index 64d020048..35b409208 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.h @@ -39,7 +39,7 @@ public: NotificationChain::Listener *RegisterTimeChangedLister(const TimeChangedAction &action, int &errCode); // Notify TIME_CHANGE_EVENT. - void NotifyTimeChange(TimeOffset timeChangeOffset) const; + void NotifyTimeChange(TimeOffset offset) const; private: static constexpr uint64_t MONITOR_INTERVAL = 1 * 1000; // 1s static constexpr int64_t MAX_NOISE = 9 * 100 * 1000; // 900ms diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/communicator_aggregator.h b/services/distributeddataservice/libs/distributeddb/communicator/include/communicator_aggregator.h index c91cd42b6..add376027 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/communicator_aggregator.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/communicator_aggregator.h @@ -55,6 +55,7 @@ public: // See ICommunicatorAggregator for detail int Initialize(IAdapter *inAdapter) override; + // Must not call any other functions if Finalize had been called. In fact, Finalize has no chance to be called. void Finalize() override; @@ -69,7 +70,12 @@ public: // return optimal allowed data size(Some header is taken into account and subtract) uint32_t GetCommunicatorAggregatorMtuSize() const; uint32_t GetCommunicatorAggregatorMtuSize(const std::string &target) const; + + // return timeout in range [5s, 60s] + uint32_t GetCommunicatorAggregatorTimeout() const; + uint32_t GetCommunicatorAggregatorTimeout(const std::string &target) const; int GetLocalIdentity(std::string &outTarget) const; + // Get the protocol version of remote target. Return -E_NOT_FOUND if no record. int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const; @@ -104,6 +110,7 @@ private: int OnAppLayerFrameReceive(const std::string &srcTarget, const uint8_t *bytes, uint32_t length, const ParseResult &inResult); int OnAppLayerFrameReceive(const std::string &srcTarget, SerialBuffer *&inFrameBuffer, const ParseResult &inResult); + // Function with suffix NoMutex should be called with mutex in the caller int TryDeliverAppLayerFrameToCommunicatorNoMutex(const std::string &srcTarget, SerialBuffer *&inFrameBuffer, const LabelType &toLabel); @@ -130,6 +137,7 @@ private: std::atomic shutdown_; std::atomic incFrameId_; std::atomic localSourceId_; + // Handle related mutable std::mutex commMapMutex_; std::map> commMap_; // bool true indicate communicator activated @@ -138,21 +146,26 @@ private: SendTaskScheduler scheduler_; IAdapter *adapterHandle_ = nullptr; CommunicatorLinker *commLinker_ = nullptr; + // Thread related std::thread exclusiveThread_; bool wakingSignal_ = false; mutable std::mutex wakingMutex_; std::condition_variable wakingCv_; + // RetryCreateTask related mutable std::mutex retryMutex_; std::condition_variable retryCv_; + // Remote target version related mutable std::mutex versionMapMutex_; std::map versionMap_; + // CommLack Callback related CommunicatorLackCallback onCommLackHandle_; Finalizer onCommLackFinalizer_; mutable std::mutex onCommLackMutex_; + // Connect Callback related OnConnectCallback onConnectHandle_; Finalizer onConnectFinalizer_; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/frame_combiner.h b/services/distributeddataservice/libs/distributeddb/communicator/include/frame_combiner.h index 0fe738698..eed85dc1b 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/frame_combiner.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/frame_combiner.h @@ -19,7 +19,7 @@ #include #include #include -#include "semaphore_utils.h" +#include "semaphore.h" #include "macro_utils.h" #include "parse_result.h" #include "combine_status.h" @@ -42,6 +42,7 @@ public: // Start the timer to supervise the progress void Initialize(); + // Clear the CombineWorkPool and stop the timer void Finalize(); @@ -68,7 +69,7 @@ private: TimerId timerId_ = 0; // 0 is invalid timerId bool isTimerWork_ = false; - SemaphoreUtils timerRemovedIndicator_{0}; + Semaphore timerRemovedIndicator_{0}; uint64_t incProgressId_ = 0; uint64_t totalSizeByByte_ = 0; std::map> combineWorkPool_; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/iadapter.h b/services/distributeddataservice/libs/distributeddb/communicator/include/iadapter.h index 4bdab0b76..c1d5cf821 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/iadapter.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/iadapter.h @@ -42,6 +42,10 @@ public: virtual uint32_t GetMtuSize() = 0; virtual uint32_t GetMtuSize(const std::string &target) = 0; + // Should returns timeout in range [5s, 60s] + virtual uint32_t GetTimeout() = 0; + virtual uint32_t GetTimeout(const std::string &target) = 0; + // Get local target name for identify self virtual int GetLocalIdentity(std::string &outTarget) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/icommunicator.h b/services/distributeddataservice/libs/distributeddb/communicator/include/icommunicator.h index 30060655b..0901be11e 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/icommunicator.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/icommunicator.h @@ -39,8 +39,14 @@ public: // return optimal allowed data size(Some header is taken into account and subtract) virtual uint32_t GetCommunicatorMtuSize() const = 0; virtual uint32_t GetCommunicatorMtuSize(const std::string &target) const = 0; + + // return timeout in range [5s, 60s] + virtual uint32_t GetTimeout() const = 0; + virtual uint32_t GetTimeout(const std::string &target) const = 0; + // Get local target name for identify self virtual int GetLocalIdentity(std::string &outTarget) const = 0; + // Get the protocol version of remote target. Return -E_NOT_FOUND if no record. virtual int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const = 0; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/network_adapter.h b/services/distributeddataservice/libs/distributeddb/communicator/include/network_adapter.h index 1116bf74d..5d610244a 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/network_adapter.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/network_adapter.h @@ -39,6 +39,9 @@ public: uint32_t GetMtuSize() override; uint32_t GetMtuSize(const std::string &target) override; + + uint32_t GetTimeout() override; + uint32_t GetTimeout(const std::string &target) override; int GetLocalIdentity(std::string &outTarget) override; int SendBytes(const std::string &dstTarget, const uint8_t *bytes, uint32_t length) override; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.cpp index 394c6621f..6c6ae2e20 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.cpp @@ -80,6 +80,16 @@ int Communicator::GetLocalIdentity(std::string &outTarget) const return commAggrHandle_->GetLocalIdentity(outTarget); } +uint32_t Communicator::GetTimeout() const +{ + return commAggrHandle_->GetCommunicatorAggregatorTimeout(); +} + +uint32_t Communicator::GetTimeout(const std::string &target) const +{ + return commAggrHandle_->GetCommunicatorAggregatorTimeout(target); +} + int Communicator::SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout) { return SendMessage(dstTarget, inMsg, nonBlock, timeout, nullptr); diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h index e4668978f..cd97d1b9e 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h @@ -43,6 +43,9 @@ public: uint32_t GetCommunicatorMtuSize() const override; uint32_t GetCommunicatorMtuSize(const std::string &target) const override; + + uint32_t GetTimeout() const override; + uint32_t GetTimeout(const std::string &target) const override; int GetLocalIdentity(std::string &outTarget) const override; // Get the protocol version of remote target. Return -E_NOT_FOUND if no record. int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const override; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_aggregator.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_aggregator.cpp index 9bc59ee4e..a8e9e3b79 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_aggregator.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_aggregator.cpp @@ -211,6 +211,16 @@ uint32_t CommunicatorAggregator::GetCommunicatorAggregatorMtuSize(const std::str return adapterHandle_->GetMtuSize(target) - ProtocolProto::GetLengthBeforeSerializedData(); } +uint32_t CommunicatorAggregator::GetCommunicatorAggregatorTimeout() const +{ + return adapterHandle_->GetTimeout(); +} + +uint32_t CommunicatorAggregator::GetCommunicatorAggregatorTimeout(const std::string &target) const +{ + return adapterHandle_->GetTimeout(target); +} + int CommunicatorAggregator::GetLocalIdentity(std::string &outTarget) const { return adapterHandle_->GetLocalIdentity(outTarget); diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.cpp index b3461bb77..87b29ec23 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.cpp @@ -319,11 +319,11 @@ int CommunicatorLinker::TriggerLabelExchangeEvent(const std::string &toTarget) int CommunicatorLinker::TriggerLabelExchangeAckEvent(const std::string &toTarget, uint64_t inSequenceId) { // Build LabelExchangeAck Frame - int error = E_OK; - SerialBuffer *buffer = ProtocolProto::BuildLabelExchangeAck(localDistinctValue_, inSequenceId, errno); - if (error != E_OK) { - LOGE("[Linker][TriggerAck] BuildAck fail, error=%d", error); - return error; + int errCode = E_OK; + SerialBuffer *buffer = ProtocolProto::BuildLabelExchangeAck(localDistinctValue_, inSequenceId, errCode); + if (errCode != E_OK) { + LOGE("[Linker][TriggerAck] BuildAck fail, error=%d", errCode); + return errCode; } // Apply for a latest ackId and update ackTriggerId_ uint64_t ackId; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/network_adapter.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/network_adapter.cpp index b521f9a56..33cf4b474 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/network_adapter.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/network_adapter.cpp @@ -14,6 +14,7 @@ */ #include "network_adapter.h" +#include "db_constant.h" #include "db_errno.h" #include "log_print.h" #include "runtime_context.h" @@ -22,8 +23,6 @@ namespace DistributedDB { namespace { const std::string DEFAULT_PROCESS_LABEL = "Distributeddb_Anonymous_Process"; const std::string SCHEDULE_QUEUE_TAG = "NetworkAdapter"; -constexpr uint32_t MIN_MTU_SIZE = 1024; // 1KB -constexpr uint32_t MAX_MTU_SIZE = 5242880; // 5MB } NetworkAdapter::NetworkAdapter() @@ -121,14 +120,25 @@ void NetworkAdapter::StopAdapter() namespace { uint32_t CheckAndAdjustMtuSize(uint32_t inMtuSize) { - if (inMtuSize < MIN_MTU_SIZE) { - return MIN_MTU_SIZE; - } else if (inMtuSize > MAX_MTU_SIZE) { - return MAX_MTU_SIZE; + if (inMtuSize < DBConstant::MIN_MTU_SIZE) { + return DBConstant::MIN_MTU_SIZE; + } else if (inMtuSize > DBConstant::MAX_MTU_SIZE) { + return DBConstant::MAX_MTU_SIZE; } else { return (inMtuSize - (inMtuSize % sizeof(uint64_t))); // Octet alignment } } + +uint32_t CheckAndAdjustTimeout(uint32_t inTimeout) +{ + if (inTimeout < DBConstant::MIN_TIMEOUT) { + return DBConstant::MIN_TIMEOUT; + } else if (inTimeout > DBConstant::MAX_TIMEOUT) { + return DBConstant::MAX_TIMEOUT; + } else { + return inTimeout; + } +} } uint32_t NetworkAdapter::GetMtuSize() @@ -145,6 +155,12 @@ uint32_t NetworkAdapter::GetMtuSize() uint32_t NetworkAdapter::GetMtuSize(const std::string &target) { +#ifndef OMIT_MTU_CACHE + DeviceInfos devInfo; + devInfo.identifier = target; + uint32_t oriMtuSize = processCommunicator_->GetMtuSize(devInfo); + return CheckAndAdjustMtuSize(oriMtuSize); +#else std::lock_guard mtuSizeLockGuard(mtuSizeMutex_); if (devMapMtuSize_.count(target) == 0) { DeviceInfos devInfo; @@ -154,6 +170,23 @@ uint32_t NetworkAdapter::GetMtuSize(const std::string &target) devMapMtuSize_[target] = CheckAndAdjustMtuSize(oriMtuSize); } return devMapMtuSize_[target]; +#endif +} + +uint32_t NetworkAdapter::GetTimeout() +{ + uint32_t timeout = processCommunicator_->GetTimeout(); + LOGI("[NAdapt][GetTimeout] timeout_=%u ms.", timeout); + return CheckAndAdjustTimeout(timeout); +} + +uint32_t NetworkAdapter::GetTimeout(const std::string &target) +{ + DeviceInfos devInfos; + devInfos.identifier = target; + uint32_t timeout = processCommunicator_->GetTimeout(devInfos); + LOGI("[NAdapt][GetTimeout] timeout=%u ms of target=%s{private}.", timeout, target.c_str()); + return CheckAndAdjustTimeout(timeout); } int NetworkAdapter::GetLocalIdentity(std::string &outTarget) diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.cpp index df5d93fc8..e466f39db 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.cpp @@ -18,6 +18,7 @@ #include #include "hash.h" #include "securec.h" +#include "version.h" #include "db_common.h" #include "log_print.h" #include "macro_utils.h" @@ -28,7 +29,8 @@ namespace DistributedDB { namespace { const uint16_t MAGIC_CODE = 0xAAAA; const uint16_t PROTOCOL_VERSION = 0; -const uint16_t DB_GLOBAL_VERSION = 2; // Compatibility Final Method. 2 Correspond To Version 1.1.3(103) +// Compatibility Final Method. 3 Correspond To Version 1.1.4(104) +const uint16_t DB_GLOBAL_VERSION = SOFTWARE_VERSION_CURRENT - SOFTWARE_VERSION_EARLIEST; const uint8_t PACKET_TYPE_FRAGMENTED = BITX(0); // Use bit 0 const uint8_t PACKET_TYPE_NOT_FRAGMENTED = 0; const uint8_t MAX_PADDING_LEN = 7; diff --git a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h index 2c9e1f1b7..a56b54b6b 100755 --- a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h +++ b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h @@ -33,6 +33,12 @@ struct AutoLaunchOption { int conflictType = 0; KvStoreNbConflictNotifier notifier; SecurityOption secOption; + uint64_t networkId = 0; // Default networkId 0 for P2P_MODE network, [1,10] is reserved and not permitted now + bool isNeedIntegrityCheck = false; + bool isNeedRmCorruptedDb = false; + bool isNeedCompressOnSync = false; + uint8_t compressionRate = 100; // in [1, 100]. + bool isAutoSync = true; }; struct AutoLaunchParam { diff --git a/services/distributeddataservice/libs/distributeddb/include/query_expression.h b/services/distributeddataservice/libs/distributeddb/include/query_expression.h index f1788d0c4..56a979253 100755 --- a/services/distributeddataservice/libs/distributeddb/include/query_expression.h +++ b/services/distributeddataservice/libs/distributeddb/include/query_expression.h @@ -23,7 +23,7 @@ #include "types_export.h" namespace DistributedDB { -enum class QueryValueType { +enum class QueryValueType: int32_t { VALUE_TYPE_INVALID = -1, VALUE_TYPE_NULL, VALUE_TYPE_BOOL, @@ -33,29 +33,30 @@ enum class QueryValueType { VALUE_TYPE_STRING, }; -enum class QueryObjType { - OPER_ILLEGAL = -1, - QUERY_VALUE, - EQUALTO, +// value will const, it will influence query object id +// use high pos bit to distinguish operator type +enum class QueryObjType : uint32_t { + OPER_ILLEGAL = 0x0000, + EQUALTO = 0x0101, NOT_EQUALTO, GREATER_THAN, LESS_THAN, GREATER_THAN_OR_EQUALTO, LESS_THAN_OR_EQUALTO, - LIKE, + LIKE = 0x0201, NOT_LIKE, IS_NULL, IS_NOT_NULL, - IN, + IN = 0x0301, NOT_IN, - QUERY_BY_KEY_PREFIX, - BEGIN_GROUP, + QUERY_BY_KEY_PREFIX = 0x0401, + BEGIN_GROUP = 0x0501, END_GROUP, - AND, + AND = 0x0601, OR, - LIMIT, + LIMIT = 0x0701, ORDERBY, - SUGGEST_INDEX, + SUGGEST_INDEX = 0x0801, }; struct QueryObjNode { @@ -109,9 +110,9 @@ public: void QueryBySuggestIndex(const std::string &indexName); - std::vector GetPreFixKey(); + std::vector GetPreFixKey() const; - std::string GetSuggestIndex(); + std::string GetSuggestIndex() const; const std::list &GetQueryExpression(); diff --git a/services/distributeddataservice/libs/distributeddb/include/types_export.h b/services/distributeddataservice/libs/distributeddb/include/types_export.h index b8063b1f3..c49f147c2 100755 --- a/services/distributeddataservice/libs/distributeddb/include/types_export.h +++ b/services/distributeddataservice/libs/distributeddb/include/types_export.h @@ -109,6 +109,8 @@ struct FieldValue { enum PermissionCheckFlag { CHECK_FLAG_SEND = 1, // send CHECK_FLAG_RECEIVE = 2, // receive + CHECK_FLAG_AUTOSYNC = 4, // autosync flag + CHECK_FLAG_SPONSOR = 8, // sync sponsor }; using PermissionCheckCallback = std::function; using RemotePushFinisheNotifier = RemotePushFinishedNotifier; // To correct spelling errors in the previous version + +enum class CompressAlgorithm : uint8_t { + NONE = 0, + ZLIB = 1 +}; + +enum class NetworkMode : uint8_t { + P2P = 0, + CENTER = 1, +}; + +struct NetworkInfo { + uint32_t size = sizeof(NetworkInfo); // For ABI compatibility + NetworkMode mode = NetworkMode::P2P; + std::vector centers; +}; +} // namespace DistributedDB #endif // DISTRIBUTEDDB_TYPES_EXPORT_H diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/iprocess_communicator.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/iprocess_communicator.h index cdeac53a5..94228dbab 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/iprocess_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/iprocess_communicator.h @@ -97,6 +97,22 @@ public: } return GetMtuSize(); } + + // The valid timeout range from 5s to 60s, value beyond this range will be set to the upper or lower limit. + virtual uint32_t GetTimeout() + { + return 5 * 1000; // 5 * 1000ms + }; + + // The valid timeout range from 5s to 60s, value beyond this range will be set to the upper or lower limit. + virtual uint32_t GetTimeout(const DeviceInfos &devInfo) + { + if (devInfo.identifier.empty()) { + // Error case(would never happen actually) to avoid "unused-parameter" warning. + return 5 * 1000; // 5 * 1000ms + } + return GetTimeout(); + } }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h index 02a0a9005..be51c0690 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h @@ -60,11 +60,16 @@ public: bool createDirByStoreIdOnly = false; SecurityOption secOption; // Add data security level parameter KvStoreObserver *observer = nullptr; - Key key; // The key that needs to be subscribed on obsever, emptye means full subscription + Key key; // The key that needs to be subscribed on obsever, empty means full subscription unsigned int mode = 0; // obsever mode int conflictType = 0; KvStoreNbConflictNotifier notifier = nullptr; int conflictResolvePolicy = LAST_WIN; + uint64_t networkId = 0; // Default networkId 0 for P2P_MODE network, [1,10] is reserved and not permitted now + bool isNeedIntegrityCheck = false; + bool isNeedRmCorruptedDb = false; + bool isNeedCompressOnSync = false; + uint8_t compressionRate = 100; // Valid in [1, 100]. }; DB_API virtual ~KvStoreNbDelegate() {} @@ -195,6 +200,22 @@ public: // If Repeat set, subject to the last time. // If set nullptr, means unregister the notify. DB_API virtual DBStatus SetRemotePushFinishedNotify(const RemotePushFinishedNotifier ¬ifier) = 0; + + // Sync function interface, if wait set true, this function will be blocked until sync finished. + // Param query used to filter the records to be synchronized. + // Now just support push mode and query by prefixKey. + // If Query.limit is used, its query cache will not be recorded, In the same way below, + // the synchronization will still take the full amount. + DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) = 0; + + // Check the integrity of this kvStore. + DB_API virtual DBStatus CheckIntegrity() const = 0; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + DB_API virtual DBStatus SetEqualIdentifier(const std::string &identifier, + const std::vector &targets) = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h index 3a1d93fb6..916c1fc01 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h @@ -43,10 +43,10 @@ enum DBStatus { INVALID_FIELD_TYPE, // invalid put value field type for json schema. CONSTRAIN_VIOLATION, // invalid put value constrain for json schema. INVALID_FORMAT, // invalid put value format for json schema. - STALE, // new record is staler compared to the same key exist in db + STALE, // new record is staler compared to the same key existed in db. LOCAL_DELETED, // local data is deleted by the unpublish. LOCAL_DEFEAT, // local data defeat the sync data while unpublish. - LOCAL_COVERED, // local data is coverd by the sync data while unpublish. + LOCAL_COVERED, // local data is covered by the sync data while unpublish. INVALID_QUERY_FORMAT, INVALID_QUERY_FIELD, PERMISSION_CHECK_FORBID_SYNC, // permission check result , forbid sync. @@ -55,6 +55,7 @@ enum DBStatus { EKEYREVOKED_ERROR, // EKEYREVOKED error when operating db file SECURITY_OPTION_CHECK_ERROR, // such as remote device's SecurityOption not equal to local SCHEMA_VIOLATE_VALUE, // Values already exist in dbFile do not match new schema + NETWORK_MISMATCH, // NetworkMode mismatch or center mismatch }; struct KvStoreConfig { @@ -77,6 +78,7 @@ enum PragmaCmd { SET_WIPE_POLICY, // set the policy of wipe remote stale data RESULT_SET_CACHE_MODE, // Accept ResultSetCacheMode Type As PragmaData RESULT_SET_CACHE_MAX_SIZE, // Allowed Int Type Range [1,16], Unit MB + SET_SYNC_RETRY, }; enum ResolutionPolicyType { diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp index d3493a05d..aab890c2d 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp @@ -89,6 +89,8 @@ namespace { properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); properties.SetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, option.createDirByStoreIdOnly); properties.SetSchema(schema); + properties.SetBoolProp(KvDBProperties::CHECK_INTEGRITY, option.isNeedIntegrityCheck); + properties.SetBoolProp(KvDBProperties::RM_CORRUPTED_DB, option.isNeedRmCorruptedDb); if (RuntimeContext::GetInstance()->IsProcessSystemApiAdapterValid()) { properties.SetIntProp(KvDBProperties::SECURITY_LABEL, option.secOption.securityLabel); properties.SetIntProp(KvDBProperties::SECURITY_FLAG, option.secOption.securityFlag); @@ -98,6 +100,11 @@ namespace { if (option.isEncryptedDb) { properties.SetPassword(option.cipher, option.passwd); } + properties.SetBoolProp(KvDBProperties::COMPRESS_ON_SYNC, option.isNeedCompressOnSync); + if (option.isNeedCompressOnSync) { + properties.SetIntProp(KvDBProperties::COMPRESSION_RATE, + ParamCheckUtils::GetValidCompressionRate(option.compressionRate)); + } } bool CheckObserverConflictParam(const KvStoreNbDelegate::Option &option) @@ -524,8 +531,7 @@ DBStatus KvStoreDelegateManager::EnableKvStoreAutoLaunch(const std::string &user return TransferDBErrno(errCode); } - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(properties, notifier, option.observer, - option.conflictType, option.notifier); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(properties, notifier, option); if (errCode != E_OK) { LOGE("[KvStoreManager] Enable auto launch failed:%d", errCode); return TransferDBErrno(errCode); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp index be459311f..2284bafd6 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp @@ -34,6 +34,7 @@ #include "kv_store_result_set_impl.h" #include "sync_operation.h" #include "performance_analysis.h" +#include "query_object.h" namespace DistributedDB { namespace { @@ -56,6 +57,7 @@ namespace { {SET_WIPE_POLICY, PRAGMA_SET_WIPE_POLICY}, {RESULT_SET_CACHE_MODE, PRAGMA_RESULT_SET_CACHE_MODE}, {RESULT_SET_CACHE_MAX_SIZE, PRAGMA_RESULT_SET_CACHE_MAX_SIZE}, + {SET_SYNC_RETRY, PRAGMA_SET_SYNC_RETRY}, }; const std::string INVALID_CONNECTION = "[KvStoreNbDelegate] Invalid connection for operation"; @@ -490,6 +492,26 @@ DBStatus KvStoreNbDelegateImpl::Sync(const std::vector &devices, Sy return OK; } +DBStatus KvStoreNbDelegateImpl::Sync(const std::vector &devices, SyncMode mode, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); + return DB_ERROR; + } + + QueryObject queryObj(query); + PragmaSync pragmaData(devices, mode, queryObj, std::bind(&KvStoreNbDelegateImpl::OnSyncComplete, + this, std::placeholders::_1, onComplete), wait); + int errCode = conn_->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData); + if (errCode < E_OK) { + LOGE("[KvStoreNbDelegate] QuerySync data failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; +} + DBStatus KvStoreNbDelegateImpl::Pragma(PragmaCmd cmd, PragmaData ¶mData) { if (conn_ == nullptr) { @@ -705,6 +727,16 @@ DBStatus KvStoreNbDelegateImpl::Close() return OK; } +DBStatus KvStoreNbDelegateImpl::CheckIntegrity() const +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); + return DB_ERROR; + } + + return TransferDBErrno(conn_->CheckIntegrity()); +} + DBStatus KvStoreNbDelegateImpl::GetSecurityOption(SecurityOption &option) const { if (conn_ == nullptr) { @@ -815,6 +847,8 @@ void KvStoreNbDelegateImpl::OnSyncComplete(const std::map &sta { static_cast(SyncOperation::EKEYREVOKED_FAILURE), DBStatus::EKEYREVOKED_ERROR }, { static_cast(SyncOperation::SCHEMA_INCOMPATIBLE), DBStatus::SCHEMA_MISMATCH }, { static_cast(SyncOperation::BUSY_FAILURE), DBStatus::BUSY }, + { static_cast(SyncOperation::QUERY_FORMAT_FAILURE), DBStatus::INVALID_QUERY_FORMAT }, + { static_cast(SyncOperation::QUERY_FIELD_FAILURE), DBStatus::INVALID_QUERY_FIELD } }; auto iter = statusMap.find(pair.second); if (iter != statusMap.end()) { @@ -826,4 +860,21 @@ void KvStoreNbDelegateImpl::OnSyncComplete(const std::map &sta onComplete(result); } } + +DBStatus KvStoreNbDelegateImpl::SetEqualIdentifier(const std::string &identifier, + const std::vector &targets) +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); + return DB_ERROR; + } + + PragmaSetEqualIdentifier pragma(identifier, targets); + int errCode = conn_->Pragma(PRAGMA_ADD_EQUAL_IDENTIFIER, reinterpret_cast(&pragma)); + if (errCode != E_OK) { + LOGE("[KvStoreNbDelegate] Set store equal identifier failed : %d", errCode); + } + + return TransferDBErrno(errCode); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h index 241ca620f..b277d23ae 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h @@ -83,15 +83,15 @@ public: std::string GetStoreId() const override; // Sync function interface, if wait set true, this function will be blocked until sync finished - DB_API DBStatus Sync(const std::vector &devices, SyncMode mode, + DBStatus Sync(const std::vector &devices, SyncMode mode, const std::function &devicesMap)> &onComplete, bool wait) override; // Special pragma interface, see PragmaCmd and PragmaData, - DB_API DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) override; + DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) override; // Set the conflict notifier for getting the specified type conflict data. - DB_API DBStatus SetConflictNotifier(int conflictType, const KvStoreNbConflictNotifier ¬ifier) override; + DBStatus SetConflictNotifier(int conflictType, const KvStoreNbConflictNotifier ¬ifier) override; // Rekey the database. DBStatus Rekey(const CipherPassword &password) override; @@ -125,6 +125,18 @@ public: DBStatus Close(); + // Sync function interface, if wait set true, this function will be blocked until sync finished. + // Param query used to filter the records to be synchronized. + // Now just support push mode and query by prefixKey. + DBStatus Sync(const std::vector &devices, SyncMode mode, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) override; + + DBStatus CheckIntegrity() const override; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + DBStatus SetEqualIdentifier(const std::string &identifier, const std::vector &targets) override; + private: DBStatus GetInner(const IOption &option, const Key &key, Value &value) const; DBStatus PutInner(const IOption &option, const Key &key, const Value &value); diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb.h b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb.h index e21c0455e..1aa687b98 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb.h @@ -55,6 +55,8 @@ public: virtual int GetKvDBSize(const KvDBProperties &properties, uint64_t &size) const = 0; virtual void EnableAutonomicUpgrade() = 0; + + virtual int CheckIntegrity() const = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h index 37ca17240..754062729 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h @@ -124,6 +124,8 @@ public: // Get the securityLabel and securityFlag virtual int GetSecurityOption(int &securityLabel, int &securityFlag) const = 0; + + virtual int CheckIntegrity() const = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_sync_interface.h index bd698dbe0..23c6480fd 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_sync_interface.h @@ -54,6 +54,12 @@ public: // Put meta data as a key-value entry. virtual int PutMetaData(const Key &key, const Value &value) = 0; + // Delete multiple meta data records in a transaction. + virtual int DeleteMetaData(const std::vector &keys) = 0; + + // Delete multiple meta data records with key prefix in a transaction. + virtual int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const = 0; + // Get all meta data keys. virtual int GetAllMetaKeys(std::vector &keys) const = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_manager.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_manager.h index af8832202..57324bb0a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_manager.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_manager.h @@ -25,6 +25,7 @@ #include "db_errno.h" #include "ikvdb.h" #include "ikvdb_factory.h" +#include "platform_specific.h" namespace DistributedDB { class KvDBManager final { @@ -89,6 +90,8 @@ private: void SetAllDatabaseCorruptionHander(const KvStoreCorruptionHandler &handler); + IKvDB *CreateDataBase(const KvDBProperties &property, int &errCode); + IKvDB *GetDataBase(const KvDBProperties &property, int &errCode, bool isNeedIfOpened); void DataBaseCorruptNotify(const std::string &appId, const std::string &userId, const std::string &storeId); @@ -117,9 +120,15 @@ private: IKvDB *GetKvDBFromCacheByIdentify(const std::string &identifier, const std::map &cache) const; + static int CheckRemoveStateAndRetry(const KvDBProperties &property); + + static int TryLockDB(const KvDBProperties &kvDBProp, int retryTimes); + static int UnlockDB(const KvDBProperties &kvDBProp); + static KvDBManager *instance_; static std::mutex kvDBLock_; static std::mutex instanceLock_; + static std::map locks_; std::map localKvDBs_; std::map multiVerNaturalStores_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h index c6ecc0464..58e4bfb9f 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h @@ -21,6 +21,7 @@ #include #include "types.h" +#include "query_object.h" namespace DistributedDB { enum : int { @@ -44,18 +45,40 @@ enum : int { PRAGMA_RESULT_SET_CACHE_MAX_SIZE, PRAGMA_TRIGGER_TO_MIGRATE_DATA, PRAGMA_REMOTE_PUSH_FINISHED_NOTIFY, + PRAGMA_SET_SYNC_RETRY, + PRAGMA_ADD_EQUAL_IDENTIFIER, }; struct PragmaSync { + PragmaSync(const std::vector &devices, int mode, const QueryObject &query, + const std::function &devicesMap)> &onComplete, + bool wait = false) + : devices_(devices), + mode_(mode), + onComplete_(onComplete), + wait_(wait), + isQuerySync_(true), + query_(query) + { + } + PragmaSync(const std::vector &devices, int mode, const std::function &devicesMap)> &onComplete, bool wait = false) - : devices_(devices), mode_(mode), onComplete_(onComplete), wait_(wait) {} + : devices_(devices), + mode_(mode), + onComplete_(onComplete), + wait_(wait), + isQuerySync_(false) + { + } std::vector devices_; int mode_; std::function &devicesMap)> onComplete_; bool wait_; + bool isQuerySync_; + QueryObject query_; }; struct PragmaRemotePushNotify { @@ -63,6 +86,15 @@ struct PragmaRemotePushNotify { RemotePushFinishedNotifier notifier_; }; + +struct PragmaSetEqualIdentifier { + PragmaSetEqualIdentifier(const std::string &identifier, const std::vector &targets) + : identifier_(identifier), + targets_(targets) {} + + std::string identifier_; + std::vector targets_; +}; } // namespace DistributedDB #endif // KV_DB_PRAGMA_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h index 9c8a66383..c90894c21 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h @@ -68,6 +68,7 @@ public: int GetSecLabel() const; int GetSecFlag() const; + // Get schema const reference if you can guarantee the lifecycle of this KvDBProperties // The upper code will not change the schema if it is already set const SchemaObject &GetSchemaConstRef() const; @@ -89,6 +90,10 @@ public: static const std::string SECURITY_LABEL; static const std::string SECURITY_FLAG; static const std::string CONFLICT_RESOLVE_POLICY; + static const std::string CHECK_INTEGRITY; + static const std::string RM_CORRUPTED_DB; + static const std::string COMPRESS_ON_SYNC; + static const std::string COMPRESSION_RATE; static const int LOCAL_TYPE = 1; static const int MULTI_VER_TYPE = 2; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_kv_entry.h b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kv_entry.h similarity index 100% rename from services/distributeddataservice/libs/distributeddb/storage/src/single_ver_kv_entry.h rename to services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kv_entry.h diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h index 828c4de22..1652cfc46 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h @@ -19,8 +19,24 @@ #include "ikvdb_sync_interface.h" #include "single_ver_kv_entry.h" #include "iprocess_system_api_adapter.h" +#include "query_object.h" namespace DistributedDB { +using MulDevTimeRanges = std::map>; +using MulDevSinVerKvEntry = std::map>; +using MulDevDataItems = std::map>; + +struct SyncTimeRange { + TimeStamp beginTime = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp endTime = static_cast(INT64_MAX); + TimeStamp deleteEndTime = static_cast(INT64_MAX); + bool IsValid() const + { + return (beginTime <= endTime && deleteBeginTime <= deleteEndTime); + } +}; + class SingleVerKvDBSyncInterface : public IKvDBSyncInterface { public: ~SingleVerKvDBSyncInterface() override {}; @@ -34,6 +50,11 @@ public: virtual int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const = 0; + // Get the data which would be synced with query condition + virtual int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const = 0; + // Get the data using the token allocated by GetSyncData, the token would be release automatically when finished. virtual int GetSyncDataNext(std::vector &dataItems, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const = 0; @@ -62,6 +83,12 @@ public: virtual bool IsReadable() const = 0; virtual void NotifyRemotePushFinished(const std::string &targetId) const = 0; + + // Get the timestamp when database created or imported + virtual int GetDatabaseCreateTimeStamp(TimeStamp &outTime) const = 0; + + virtual int GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const = 0; + virtual int GetCompressionAlgo(std::set &algorithmSet) const = 0; }; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.cpp index 158a435d7..9802edf55 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.cpp @@ -374,6 +374,11 @@ void GenericKvDB::SetConnectionFlag(bool isExisted) const return; } +int GenericKvDB::CheckIntegrity() const +{ + return E_OK; +} + void GenericKvDB::GetStoreDirectory(const KvDBProperties &properties, int dbType, std::string &storeDir, std::string &storeOnlyDir) const { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.h b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.h index 17b9585b3..5cacf0189 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.h @@ -126,6 +126,8 @@ public: virtual void SetConnectionFlag(bool isExisted) const; + int CheckIntegrity() const override; + protected: // Create a connection object, no DB ref increased. virtual GenericKvDBConnection *NewConnection(int &errCode) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.cpp index 19935e922..8bc0c9026 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.cpp @@ -321,5 +321,9 @@ int GenericKvDBConnection::GetEventType(unsigned mode, std::list &eventType return TranslateObserverModeToEventTypes(mode, eventTypes); } -} +int GenericKvDBConnection::CheckIntegrity() const +{ + return E_OK; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.h index 91bac5720..e5e2378e1 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.h @@ -72,6 +72,8 @@ public: int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) override; int GetSecurityOption(int &securityLabel, int &securityFlag) const override; + + int CheckIntegrity() const override; protected: // Get the stashed 'KvDB_ pointer' without ref. template diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.cpp index da289231d..10ff94ded 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.cpp @@ -16,6 +16,7 @@ #include "generic_single_ver_kv_entry.h" #include +#include "data_compression.h" #include "db_errno.h" #include "parcel.h" #include "version.h" @@ -105,10 +106,10 @@ int GenericSingleVerKvEntry::SerializeData(Parcel &parcel, uint32_t targetVersio int GenericSingleVerKvEntry::SerializeDatas(const std::vector &kvEntries, Parcel &parcel, uint32_t targetVersion) { - LOGD("GenericSingleVerKvEntry::SerialDatas targetVersion:%d", targetVersion); uint32_t size = kvEntries.size(); int errCode = parcel.WriteUInt32(size); if (errCode != E_OK) { + LOGE("[SerializeDatas] write entries size failed, errCode=%d.", errCode); return errCode; } parcel.EightByteAlign(); @@ -118,6 +119,7 @@ int GenericSingleVerKvEntry::SerializeDatas(const std::vectorSerializeData(parcel, targetVersion); if (errCode != E_OK) { + LOGE("[SerializeDatas] write kvEntry failed, errCode=%d.", errCode); return errCode; } } @@ -138,7 +140,6 @@ uint32_t GenericSingleVerKvEntry::CalculateLen(uint32_t targetVersion) uint32_t GenericSingleVerKvEntry::CalculateLens(const std::vector &kvEntries, uint32_t targetVersion) { - LOGD("GenericSingleVerKvEntry::CalculateLen targetVersion:%d", targetVersion); uint64_t len = 0; len += Parcel::GetUInt32Len(); len = BYTE_8_ALIGN(len); @@ -324,4 +325,90 @@ int GenericSingleVerKvEntry::DeSerializeByVersion(uint32_t targetVersion, Parcel DeSerializeByLaterVersion(len, parcel); return E_OK; } + +uint32_t GenericSingleVerKvEntry::CalculateCompressedLens(const std::vector &compressedData) +{ + // No compressed data in sync. + if (compressedData.empty()) { + return 0; + } + + // Calculate compressed data length. + uint64_t len = 0; + len += Parcel::GetUInt32Len(); // srcLen. + len += Parcel::GetUInt32Len(); // compression algorithm type. + len += Parcel::GetVectorLen(compressedData); // compressed data. + return (len > INT32_MAX) ? 0 : len; +} + +int GenericSingleVerKvEntry::Compress(const std::vector &kvEntries, std::vector &destData, + const CompressInfo &compressInfo) +{ + // Calculate length. + auto srcLen = CalculateLens(kvEntries, compressInfo.targetVersion); + if (srcLen == 0) { + LOGE("Over limit size, cannot compress."); + return -E_INVALID_ARGS; + } + + // Serialize data. + std::vector srcData(srcLen, 0); + Parcel parcel(srcData.data(), srcData.size()); + int errCode = SerializeDatas(kvEntries, parcel, compressInfo.targetVersion); + if (errCode != E_OK) { + return errCode; + } + + // Compress data. + return DataCompression::Compress(srcData, destData); +} + +int GenericSingleVerKvEntry::Uncompress(const std::vector &srcData, std::vector &kvEntries, + unsigned long destLen) +{ + // Uncompress data. + std::vector destData(destLen, 0); + int errCode = DataCompression::Uncompress(srcData, destData, destLen); + if (errCode != E_OK) { + return errCode; + } + + // Deserialize data. + Parcel parcel(destData.data(), destData.size()); + if (DeSerializeDatas(kvEntries, parcel) == 0) { + return -E_PARSE_FAIL; + } + return E_OK; +} + +int GenericSingleVerKvEntry::SerializeCompressedDatas(const std::vector &kvEntries, + const std::vector &compressedEntries, Parcel &parcel, uint32_t targetVersion, CompressAlgorithm algo) +{ + uint32_t srcLen = CalculateLens(kvEntries, targetVersion); + (void)parcel.WriteUInt32(static_cast(CompressAlgorithm::ZLIB)); + (void)parcel.WriteUInt32(srcLen); + (void)parcel.WriteVector(compressedEntries); + return parcel.IsError() ? -E_PARSE_FAIL : E_OK; +} + +int GenericSingleVerKvEntry::DeSerializeCompressedDatas(std::vector &kvEntries, Parcel &parcel) +{ + // Get compression algo type. + uint32_t algoType = 0; + (void)parcel.ReadUInt32(algoType); + + // Get buffer length. + uint32_t destLen = 0; + (void)parcel.ReadUInt32(destLen); + + // Get compressed data. + std::vector srcData; + (void)parcel.ReadVector(srcData); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + + // Uncompress data. + return Uncompress(srcData, kvEntries, destLen); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.h b/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.h index 0dce405e3..e7e85fc62 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.h @@ -24,6 +24,11 @@ #include "single_ver_kv_entry.h" namespace DistributedDB { +struct CompressInfo { + CompressAlgorithm compressAlgo; + uint32_t targetVersion; +}; + class GenericSingleVerKvEntry : public SingleVerKvEntry { public: GenericSingleVerKvEntry(); @@ -48,7 +53,7 @@ public: uint64_t GetFlag() const override; - void SetEntryData(DataItem &&dateItem); + void SetEntryData(DataItem &&dataItem); int SerializeData(Parcel &parcel, uint32_t targetVersion) override; @@ -61,6 +66,14 @@ public: static int DeSerializeDatas(std::vector &kvEntries, Parcel &parcel); static uint32_t CalculateLens(const std::vector &kvEntries, uint32_t targetVersion); + static uint32_t CalculateCompressedLens(const std::vector &compressedData); + static int Compress(const std::vector &kvEntries, std::vector &destData, + const CompressInfo &compressInfo); + static int Uncompress(const std::vector &srcData, std::vector &kvEntries, + unsigned long destLen); + static int SerializeCompressedDatas(const std::vector &kvEntries, + const std::vector &compressedEntries, Parcel &parcel, uint32_t targetVersion, CompressAlgorithm algo); + static int DeSerializeCompressedDatas(std::vector &kvEntries, Parcel &parcel); private: int AdaptToVersion(int operType, uint32_t targetVersion, Parcel &parcel, uint64_t &datalen); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_manager.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_manager.cpp index 47fa951eb..39b53da06 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_manager.cpp @@ -14,7 +14,6 @@ */ #include "kvdb_manager.h" -#include "platform_specific.h" #include "log_print.h" #include "db_common.h" #include "runtime_context.h" @@ -22,17 +21,19 @@ #include "default_factory.h" #include "generic_kvdb.h" #include "db_constant.h" +#include "res_finalizer.h" namespace DistributedDB { const std::string KvDBManager::PROCESS_LABEL_CONNECTOR = "-"; KvDBManager *KvDBManager::instance_ = nullptr; std::mutex KvDBManager::kvDBLock_; std::mutex KvDBManager::instanceLock_; +std::map KvDBManager::locks_; namespace { DefaultFactory g_defaultFactory; - int CreateDataBase(const KvDBProperties &property, IKvDB *&kvDB) + int CreateDataBaseInstance(const KvDBProperties &property, IKvDB *&kvDB) { IKvDBFactory *factory = IKvDBFactory::GetCurrent(); if (factory == nullptr) { @@ -70,34 +71,38 @@ namespace { } return errCode; } +} - int CheckRemoveStateAndRetry(const KvDBProperties &property) - { - std::string dataDir = property.GetStringProp(KvDBProperties::DATA_DIR, ""); - std::string identifier = property.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); - std::string identifierName = DBCommon::TransferStringToHex(identifier); - std::string storeDir = dataDir + "/" + identifierName + DBConstant::DELETE_KVSTORE_REMOVING; +int KvDBManager::CheckRemoveStateAndRetry(const KvDBProperties &property) +{ + std::string dataDir = property.GetStringProp(KvDBProperties::DATA_DIR, ""); + std::string identifier = property.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifierName = DBCommon::TransferStringToHex(identifier); + std::string storeDir = dataDir + "/" + identifierName + DBConstant::DELETE_KVSTORE_REMOVING; - if (OS::CheckPathExistence(storeDir)) { - KvDBManager::RemoveDatabase(property); - } - // Re-detection deleted had been finish - if (OS::CheckPathExistence(storeDir)) { - LOGD("Deletekvstore unfinished, can not create new same identifier kvstore!"); - return -E_REMOVE_FILE; - } - return E_OK; + if (OS::CheckPathExistence(storeDir)) { + KvDBManager::ExecuteRemoveDatabase(property); + } + // Re-detection deleted had been finish + if (OS::CheckPathExistence(storeDir)) { + LOGD("Deletekvstore unfinished, can not create new same identifier kvstore!"); + return -E_REMOVE_FILE; } + return E_OK; } int KvDBManager::ExecuteRemoveDatabase(const KvDBProperties &properties) { + int errCode = CheckDatabaseFileStatus(properties); + if (errCode != E_OK) { + return errCode; + } IKvDBFactory *factory = IKvDBFactory::GetCurrent(); if (factory == nullptr) { return -E_INVALID_DB; } - int errCode = CreateRemoveStateFlagFile(properties); + errCode = CreateRemoveStateFlagFile(properties); if (errCode != E_OK) { LOGE("create ctrl file failed:%d.", errCode); return errCode; @@ -173,6 +178,75 @@ void KvDBManager::ExitDBOpenCloseProcess(const std::string &identifier) kvDBOpenCondition_.notify_all(); } +// one time 100ms +// In order to prevent long-term blocking of the process, a retry method is used +// The dimensions of the lock by appid-userid-storeid +int KvDBManager::TryLockDB(const KvDBProperties &kvDBProp, int retryTimes) +{ + std::string dataDir = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); + bool isMemoryDb = kvDBProp.GetBoolProp(KvDBProperties::MEMORY_MODE, false); + std::string id = KvDBManager::GenerateKvDBIdentifier(kvDBProp); + if (dataDir.back() != '/') { + dataDir += "/"; + } + + if (isMemoryDb) { + LOGI("MemoryDb not need lock!"); + return E_OK; + } + + if (locks_.count(id) != 0) { + LOGI("db has been locked!"); + return E_OK; + } + + std::string hexHashId = DBCommon::TransferStringToHex((id)); + OS::FileHandle handle; + int errCode = OS::OpenFile(dataDir + hexHashId + DBConstant::DB_LOCK_POSTFIX, handle); + if (errCode != E_OK) { + LOGE("Open lock file fail errCode = [%d], errno:%d", errCode, errno); + return errCode; + } + + while (retryTimes-- > 0) { + errCode = OS::FileLock(handle, false); // not block process + if (errCode == E_OK) { + LOGI("[%s]locked!", STR_MASK(DBCommon::TransferStringToHex(KvDBManager::GenerateKvDBIdentifier(kvDBProp)))); + locks_[id] = handle; + return errCode; + } else if (errCode == -E_BUSY) { + LOGD("DB already held by process lock!"); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait for 100ms + continue; + } else { + LOGE("Try lock db failed, errCode = [%d] errno:%d", errCode, errno); + OS::CloseFile(handle); + return errCode; + } + } + OS::CloseFile(handle); + return -E_BUSY; +} + +int KvDBManager::UnlockDB(const KvDBProperties &kvDBProp) +{ + bool isMemoryDb = kvDBProp.GetBoolProp(KvDBProperties::MEMORY_MODE, false); + if (isMemoryDb) { + return E_OK; + } + std::string identifierDir = KvDBManager::GenerateKvDBIdentifier(kvDBProp); + if (locks_.count(identifierDir) == 0) { + return E_OK; + } + int errCode = OS::FileUnlock(locks_[identifierDir]); + LOGI("DB unlocked! errCode = [%d]", errCode); + if (errCode != E_OK) { + return errCode; + } + locks_.erase(identifierDir); + return E_OK; +} + // Used to open a kvdb with the given property IKvDBConnection *KvDBManager::GetDatabaseConnection(const KvDBProperties &properties, int &errCode, bool isNeedIfOpened) @@ -184,7 +258,9 @@ IKvDBConnection *KvDBManager::GetDatabaseConnection(const KvDBProperties &proper } IKvDBConnection *connection = nullptr; std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + LOGD("Begin to get [%s] database connection.", STR_MASK(DBCommon::TransferStringToHex(identifier))); manager->EnterDBOpenCloseProcess(identifier); + IKvDB *kvDB = manager->GetDataBase(properties, errCode, isNeedIfOpened); if (kvDB == nullptr) { if (isNeedIfOpened) { @@ -192,12 +268,13 @@ IKvDBConnection *KvDBManager::GetDatabaseConnection(const KvDBProperties &proper } } else { connection = kvDB->GetDBConnection(errCode); - if (connection == nullptr) { + if (connection == nullptr) { // not kill kvdb, Other operations like import may be used concurrently LOGE("Failed to get the db connect for delegate:%d", errCode); } RefObject::DecObjRef(kvDB); // restore the reference increased by the cache. kvDB = nullptr; } + manager->ExitDBOpenCloseProcess(identifier); if (errCode == -E_INVALID_PASSWD_OR_CORRUPTED_DB) { std::string appId = properties.GetStringProp(KvDBProperties::APP_ID, ""); @@ -231,6 +308,37 @@ int KvDBManager::ReleaseDatabaseConnection(IKvDBConnection *connection) return errCode; } +IKvDB *KvDBManager::CreateDataBase(const KvDBProperties &property, int &errCode) +{ + IKvDB *kvDB = OpenNewDatabase(property, errCode); + if (kvDB == nullptr) { + LOGE("Failed to open the new database."); + if (errCode == -E_INVALID_PASSWD_OR_CORRUPTED_DB && + property.GetBoolProp(KvDBProperties::RM_CORRUPTED_DB, false)) { + LOGI("Remove the corrupted database while open"); + ExecuteRemoveDatabase(property); + kvDB = OpenNewDatabase(property, errCode); + } + return kvDB; + } + + if (property.GetBoolProp(KvDBProperties::CHECK_INTEGRITY, false)) { + int integrityStatus = kvDB->CheckIntegrity(); + if (integrityStatus == -E_INVALID_PASSWD_OR_CORRUPTED_DB) { + RemoveKvDBFromCache(kvDB); + RefObject::KillAndDecObjRef(kvDB); + kvDB = nullptr; + errCode = -E_INVALID_PASSWD_OR_CORRUPTED_DB; + if (property.GetBoolProp(KvDBProperties::RM_CORRUPTED_DB, false)) { + LOGI("Remove the corrupted database for the integrity check"); + ExecuteRemoveDatabase(property); + kvDB = OpenNewDatabase(property, errCode); + } + } + } + return kvDB; +} + IKvDB *KvDBManager::GetDataBase(const KvDBProperties &property, int &errCode, bool isNeedIfOpened) { bool isMemoryDb = property.GetBoolProp(KvDBProperties::MEMORY_MODE, false); @@ -243,16 +351,41 @@ IKvDB *KvDBManager::GetDataBase(const KvDBProperties &property, int &errCode, bo errCode = -E_ALREADY_OPENED; kvDB = nullptr; } - } else { - if (isMemoryDb && !isCreateNecessary) { - LOGI("IsCreateNecessary is false, Not need create database"); - } else if (errCode == -E_NOT_FOUND) { - kvDB = OpenNewDatabase(property, errCode); - if (kvDB == nullptr) { - LOGE("Failed to open the new database."); - } + return kvDB; + } + if (isMemoryDb && !isCreateNecessary) { + LOGI("IsCreateNecessary is false, Not need create database"); + return nullptr; + } + if (errCode != -E_NOT_FOUND) { + return nullptr; + } + + // Taking into account the compatibility of version delivery, + // temporarily use isNeedIntegrityCheck this field to avoid multi-process concurrency + bool isNeedIntegrityCheck = property.GetBoolProp(KvDBProperties::CHECK_INTEGRITY, false); + if (isNeedIntegrityCheck) { + LOGI("db need lock, need check integrity is [%d]", isNeedIntegrityCheck); + errCode = KvDBManager::TryLockDB(property, 10); // default 10 times retry + if (errCode != E_OK) { + return nullptr; } } + + ResFinalizer unlock([&errCode, &property, &kvDB]() { + int err = KvDBManager::UnlockDB(property); + if (err != E_OK) { + LOGE("GetDataBase unlock failed! err [%d] errCode [%d]", err, errCode); + errCode = err; + RefObject::KillAndDecObjRef(kvDB); + kvDB = nullptr; + } + }); + + kvDB = CreateDataBase(property, errCode); + if (errCode != E_OK) { + LOGE("Create data base failed, errCode = [%d]", errCode); + } return kvDB; } @@ -367,7 +500,7 @@ IKvDB *KvDBManager::OpenNewDatabase(const KvDBProperties &property, int &errCode } IKvDB *kvDB = nullptr; - errCode = CreateDataBase(property, kvDB); + errCode = CreateDataBaseInstance(property, kvDB); if (errCode != E_OK) { LOGE("Failed to get IKvDB! err:%d", errCode); return nullptr; @@ -385,6 +518,7 @@ IKvDB *KvDBManager::OpenNewDatabase(const KvDBProperties &property, int &errCode LOGI("Database identifier:%.6s, dir:%.6s", identifier.c_str(), dbDir.c_str()); // Register the callback function when the database is closed, triggered when kvdb free kvDB->OnClose([kvDB, this]() { + LOGI("Remove from the cache"); this->RemoveKvDBFromCache(kvDB); }); @@ -401,15 +535,37 @@ IKvDB *KvDBManager::OpenNewDatabase(const KvDBProperties &property, int &errCode // return BUSY if in use int KvDBManager::RemoveDatabase(const KvDBProperties &properties) { - int errCode = CheckDatabaseFileStatus(properties); - if (errCode != E_OK) { - return errCode; + KvDBManager *manager = GetInstance(); + if (manager == nullptr) { + LOGE("Failed to get kvdb manager while removing the db!"); + return -E_OUT_OF_MEMORY; + } + std::string identifier = GenerateKvDBIdentifier(properties); + manager->EnterDBOpenCloseProcess(identifier); + + LOGI("KvDBManager::RemoveDatabase begin try lock the database!"); + std::string lockFile = properties.GetStringProp(KvDBProperties::DATA_DIR, "") + "/" + + DBCommon::TransferStringToHex(identifier) + DBConstant::DB_LOCK_POSTFIX; + int errCode = E_OK; + if (OS::CheckPathExistence(lockFile)) { + errCode = KvDBManager::TryLockDB(properties, 10); // default 10 times retry + if (errCode != E_OK) { + manager->ExitDBOpenCloseProcess(identifier); + return errCode; + } } errCode = ExecuteRemoveDatabase(properties); if (errCode != E_OK) { LOGE("[KvDBManager] Remove database failed:%d", errCode); } + int err = KvDBManager::UnlockDB(properties); // unlock and delete lock file befor delete dir + if (err != E_OK) { + LOGE("[KvDBManager][RemoveDatabase] UnlockDB failed:%d, errno:%d", err, errno); + errCode = err; + } + + manager->ExitDBOpenCloseProcess(identifier); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp index d13027d83..8b832de5d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp @@ -34,6 +34,10 @@ const std::string KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY = "createDirByStor const std::string KvDBProperties::SECURITY_LABEL = "securityLabel"; const std::string KvDBProperties::SECURITY_FLAG = "securityFlag"; const std::string KvDBProperties::CONFLICT_RESOLVE_POLICY = "conflictResolvePolicy"; +const std::string KvDBProperties::CHECK_INTEGRITY = "checkIntegrity"; +const std::string KvDBProperties::RM_CORRUPTED_DB = "rmCorruptedDb"; +const std::string KvDBProperties::COMPRESS_ON_SYNC = "needCompressOnSync"; +const std::string KvDBProperties::COMPRESSION_RATE = "compressionRate"; KvDBProperties::KvDBProperties() : cipherType_(CipherType::AES_256_GCM) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.cpp index fc2b15138..4a79f6074 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.cpp @@ -545,6 +545,11 @@ int MultiVerNaturalStore::PutMetaData(const Key &key, const Value &value) return errCode; } +int MultiVerNaturalStore::DeleteMetaData(const std::vector &keys) +{ + return -E_NOT_SUPPORT; +} + // Get all meta data keys. int MultiVerNaturalStore::GetAllMetaKeys(std::vector &keys) const { @@ -1177,6 +1182,11 @@ int MultiVerNaturalStore::GetVersionFilePath(const KvDBProperties &kvDBProp, std return E_OK; } +int MultiVerNaturalStore::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + return -E_NOT_SUPPORT; +} + DEFINE_OBJECT_TAG_FACILITIES(MultiVerNaturalStore) } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.h index 72594171d..3b6ae0445 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.h @@ -74,6 +74,11 @@ public: // Put meta data as a key-value entry. int PutMetaData(const Key &key, const Value &value) override; + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys) override; + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + // Get all meta data keys. int GetAllMetaKeys(std::vector &keys) const override; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/operation/database_oper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/operation/database_oper.cpp index ffd876fbc..065dadf45 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/operation/database_oper.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/operation/database_oper.cpp @@ -366,7 +366,7 @@ int DatabaseOper::RemoveFile(const std::string &fileName) return E_OK; } - if (remove(fileName.c_str()) != 0) { + if (OS::RemoveFile(fileName.c_str()) != E_OK) { LOGE("Remove file failed:%d", errno); return -E_REMOVE_FILE; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/operation/single_ver_database_oper.h b/services/distributeddataservice/libs/distributeddb/storage/src/operation/single_ver_database_oper.h index 946409d7a..b0ea78704 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/operation/single_ver_database_oper.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/operation/single_ver_database_oper.h @@ -68,7 +68,7 @@ private: int ImportUnpackedMetaDatabase(const ImportFileInfo &info) const; - int SetSecOpt(const std::string &dir, bool isDir = true) const; + int SetSecOpt(const std::string &path, bool isDir = true) const; int BackupDatabase(const ImportFileInfo &info) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/package_file.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/package_file.cpp index cc1625c6b..85eee2744 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/package_file.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/package_file.cpp @@ -60,7 +60,7 @@ static void Clear(ofstream &target, string targetFile) if (target.is_open()) { target.close(); } - if (remove(targetFile.c_str()) != EOK) { + if (OS::RemoveFile(targetFile.c_str()) != E_OK) { LOGE("Remove file failed."); } return; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_natural_store_commit_notify_data.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_natural_store_commit_notify_data.cpp index 2aec796fe..fd3db2a6c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_natural_store_commit_notify_data.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_natural_store_commit_notify_data.cpp @@ -197,7 +197,7 @@ int SingleVerNaturalStoreCommitNotifyData::GetConflictedNotifiedFlag() const } bool SingleVerNaturalStoreCommitNotifyData::IsConflictedNotifyMatched(const DataItem &itemPut, - const DataItem &itemOri) const + const DataItem &itemGet) const { int dataConflictedType = 0; // Local put @@ -205,7 +205,7 @@ bool SingleVerNaturalStoreCommitNotifyData::IsConflictedNotifyMatched(const Data dataConflictedType = SINGLE_VER_CONFLICT_NATIVE_ALL; } else { // Compare the origin device of the get and put item. - if (itemPut.origDev != itemOri.origDev) { + if (itemPut.origDev != itemGet.origDev) { dataConflictedType = SINGLE_VER_CONFLICT_FOREIGN_KEY_ORIG; } else { dataConflictedType = SINGLE_VER_CONFLICT_FOREIGN_KEY_ONLY; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp index bb7aeb6c4..7f2ceae99 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp @@ -14,6 +14,7 @@ */ #include "query_object.h" +#include #include "db_errno.h" #include "schema_utils.h" #include "get_query_info.h" @@ -24,100 +25,76 @@ namespace DistributedDB { namespace { - const std::string PRE_QUERY_KV_SQL = "SELECT key, value FROM sync_data "; - const std::string PRE_QUERY_ROWID_SQL = "SELECT rowid FROM sync_data "; - const std::string PRE_GET_COUNT_SQL = "SELECT count(*) FROM sync_data "; - const std::string FILTER_NATIVE_DATA_SQL = "WHERE (flag&0x01=0) "; - const std::string USING_INDEX = "INDEXED BY "; - const int MAX_SQL_LEN = 1024 * 1024; // 1M bytes - const int LIMIT_FIELD_VALUE_SIZE = 2; - const int SINGLE_FIELD_VALUE_SIZE = 1; - const int INVALID_LIMIT = INT_MAX; - const int MAX_CONDITIONS_SIZE = 128; - const int MAX_SQLITE_BIND_SIZE = 50000; - - enum SymbolType { - INVALID_SYMBOL = -1, - COMPARE_SYMBOL, // relation symbol use to compare - RELATIONAL_SYMBOL, - RANGE_SYMBOL, - LOGIC_SYMBOL, - LINK_SYMBOL, // use to link relatonal symbol - SPECIAL_SYMBOL, // need special precess and need at the last - PREFIXKEY_SYMBOL, - SUGGEST_INDEX_SYMBOL, - }; - - bool IsNeedCheckEqualFormat(SymbolType type) - { - return type == COMPARE_SYMBOL || type == RELATIONAL_SYMBOL || type == RANGE_SYMBOL; - } - - // Considering our company's style, give up hexadecimal expressions state machines to avoid unreadable - const std::map SYMBOL_TYPE_DIC { - {QueryObjType::EQUALTO, COMPARE_SYMBOL}, - {QueryObjType::NOT_EQUALTO, COMPARE_SYMBOL}, - {QueryObjType::GREATER_THAN, COMPARE_SYMBOL}, - {QueryObjType::LESS_THAN, COMPARE_SYMBOL}, - {QueryObjType::GREATER_THAN_OR_EQUALTO, COMPARE_SYMBOL}, - {QueryObjType::LESS_THAN_OR_EQUALTO, COMPARE_SYMBOL}, - {QueryObjType::LIKE, RELATIONAL_SYMBOL}, - {QueryObjType::NOT_LIKE, RELATIONAL_SYMBOL}, - {QueryObjType::IS_NULL, RELATIONAL_SYMBOL}, - {QueryObjType::IS_NOT_NULL, RELATIONAL_SYMBOL}, - {QueryObjType::IN, RANGE_SYMBOL}, - {QueryObjType::NOT_IN, RANGE_SYMBOL}, - {QueryObjType::BEGIN_GROUP, LOGIC_SYMBOL}, - {QueryObjType::END_GROUP, LOGIC_SYMBOL}, - {QueryObjType::AND, LINK_SYMBOL}, - {QueryObjType::OR, LINK_SYMBOL}, - {QueryObjType::LIMIT, SPECIAL_SYMBOL}, - {QueryObjType::ORDERBY, SPECIAL_SYMBOL}, - {QueryObjType::QUERY_BY_KEY_PREFIX, PREFIXKEY_SYMBOL}, - {QueryObjType::SUGGEST_INDEX, SUGGEST_INDEX_SYMBOL} - }; - - SymbolType GetSymbolType(const QueryObjType &queryObjType) - { - if (SYMBOL_TYPE_DIC.find(queryObjType) == SYMBOL_TYPE_DIC.end()) { - return INVALID_SYMBOL; - } - return SYMBOL_TYPE_DIC.at(queryObjType); - } - - const std::map RELATIONAL_SYMBOL_TO_SQL { - {QueryObjType::EQUALTO, "= "}, - {QueryObjType::NOT_EQUALTO, "!= "}, - {QueryObjType::GREATER_THAN, "> "}, - {QueryObjType::LESS_THAN, "< "}, - {QueryObjType::GREATER_THAN_OR_EQUALTO, ">= "}, - {QueryObjType::LESS_THAN_OR_EQUALTO, "<= "}, - {QueryObjType::LIKE, " LIKE "}, - {QueryObjType::NOT_LIKE, " NOT LIKE "}, - {QueryObjType::IS_NULL, " IS NULL "}, - {QueryObjType::IS_NOT_NULL, " IS NOT NULL "}, - {QueryObjType::IN, " IN ("}, - {QueryObjType::NOT_IN, " NOT IN ("}, - }; - - const std::map LOGIC_SYMBOL_TO_SQL { - {QueryObjType::AND, " AND "}, - {QueryObjType::OR, " OR "}, - {QueryObjType::BEGIN_GROUP, "("}, - {QueryObjType::END_GROUP, ")"}, - }; +const std::string PRE_QUERY_KV_SQL = "SELECT key, value FROM sync_data "; +const std::string PRE_QUERY_ITEM_SQL = "SELECT * FROM sync_data "; +const std::string PRE_QUERY_ROWID_SQL = "SELECT rowid FROM sync_data "; +const std::string PRE_GET_COUNT_SQL = "SELECT count(*) FROM sync_data "; +const std::string FILTER_NATIVE_DATA_SQL = "WHERE (flag&0x01=0) "; +const std::string FILTER_REMOTE_QUERY = "WHERE (flag&0x03=0x02)"; +const std::string USING_INDEX = "INDEXED BY "; +const int MAX_SQL_LEN = 1024 * 1024; // 1M bytes +const int LIMIT_FIELD_VALUE_SIZE = 2; +const int SINGLE_FIELD_VALUE_SIZE = 1; +const int INVALID_LIMIT = INT_MAX; +const int MAX_CONDITIONS_SIZE = 128; +const int MAX_SQLITE_BIND_SIZE = 50000; +const uint32_t SYMBOL_TYPE_MASK = 0xff00; + +enum SymbolType : uint32_t { + INVALID_SYMBOL = 0x0000, + COMPARE_SYMBOL = 0x0100, // relation symbol use to compare + RELATIONAL_SYMBOL = 0x0200, + RANGE_SYMBOL = 0x0300, + PREFIXKEY_SYMBOL = 0x0400, + LOGIC_SYMBOL = 0x0500, + LINK_SYMBOL = 0x0600, // use to link relatonal symbol + SPECIAL_SYMBOL = 0x0700, // need special precess and need at the last + SUGGEST_INDEX_SYMBOL = 0x0800, +}; + +bool IsNeedCheckEqualFormat(SymbolType type) +{ + return type == COMPARE_SYMBOL || type == RELATIONAL_SYMBOL || type == RANGE_SYMBOL; +} - int CheckLinkerBefor(std::list::iterator &iter) - { - auto preIter = std::prev(iter, 1); - SymbolType symbolType = GetSymbolType(preIter->operFlag); - if (symbolType != COMPARE_SYMBOL && symbolType != RELATIONAL_SYMBOL && symbolType != LOGIC_SYMBOL && - symbolType != RANGE_SYMBOL && symbolType != PREFIXKEY_SYMBOL) { - LOGE("Must be a comparison operation before the connective! operFage = %s", VNAME(preIter->operFlag)); - return -E_INVALID_QUERY_FORMAT; - } - return E_OK; +SymbolType GetSymbolType(const QueryObjType &queryObjType) +{ + return static_cast(static_cast(queryObjType) & SYMBOL_TYPE_MASK); +} + +const std::map RELATIONAL_SYMBOL_TO_SQL { + {QueryObjType::EQUALTO, "= "}, + {QueryObjType::NOT_EQUALTO, "!= "}, + {QueryObjType::GREATER_THAN, "> "}, + {QueryObjType::LESS_THAN, "< "}, + {QueryObjType::GREATER_THAN_OR_EQUALTO, ">= "}, + {QueryObjType::LESS_THAN_OR_EQUALTO, "<= "}, + {QueryObjType::LIKE, " LIKE "}, + {QueryObjType::NOT_LIKE, " NOT LIKE "}, + {QueryObjType::IS_NULL, " IS NULL "}, + {QueryObjType::IS_NOT_NULL, " IS NOT NULL "}, + {QueryObjType::IN, " IN ("}, + {QueryObjType::NOT_IN, " NOT IN ("}, +}; + +const std::map LOGIC_SYMBOL_TO_SQL { + {QueryObjType::AND, " AND "}, + {QueryObjType::OR, " OR "}, + {QueryObjType::BEGIN_GROUP, "("}, + {QueryObjType::END_GROUP, ")"}, +}; + +int CheckLinkerBefor(std::list::iterator &iter) +{ + auto preIter = std::prev(iter, 1); + SymbolType symbolType = GetSymbolType(preIter->operFlag); + if (symbolType != COMPARE_SYMBOL && symbolType != RELATIONAL_SYMBOL && symbolType != LOGIC_SYMBOL && + symbolType != RANGE_SYMBOL && symbolType != PREFIXKEY_SYMBOL) { + LOGE("Must be a comparison operation before the connective! operFlag = %s", VNAME(preIter->operFlag)); + return -E_INVALID_QUERY_FORMAT; } + return E_OK; +} } QueryObject::QueryObject() @@ -134,8 +111,15 @@ QueryObject::QueryObject() {} QueryObject::QueryObject(const Query &query) - : limit_(INVALID_LIMIT), offset_(0), orderByCounts_(0), transformed_(false), hasOrderBy_(false), - hasLimit_(false), isOrderByAppeared_(false), hasPrefixKey_(false), isNeedOrderbyKey_(true) + : limit_(INVALID_LIMIT), + offset_(0), + orderByCounts_(0), + transformed_(false), + hasOrderBy_(false), + hasLimit_(false), + isOrderByAppeared_(false), + hasPrefixKey_(false), + isNeedOrderbyKey_(true) { QueryExpression queryExpressions = GetQueryInfo::GetQueryExpression(query); queryObjNodes_ = queryExpressions.GetQueryExpression(); @@ -144,6 +128,21 @@ QueryObject::QueryObject(const Query &query) suggestIndex_ = queryExpressions.GetSuggestIndex(); } +QueryObject::QueryObject(const std::list &queryObjNodes, const std::vector prefixKey) + : queryObjNodes_(queryObjNodes), + prefixKey_(prefixKey), + limit_(INVALID_LIMIT), + offset_(0), + orderByCounts_(0), + isValid_(true), + transformed_(false), + hasOrderBy_(false), + hasLimit_(false), + isOrderByAppeared_(false), + hasPrefixKey_(false), + isNeedOrderbyKey_(true) +{} + int QueryObject::GetQuerySql(std::string &sql, bool onlyRowid) { int errCode = E_OK; @@ -152,7 +151,7 @@ int QueryObject::GetQuerySql(std::string &sql, bool onlyRowid) } const std::string &querySqlForUse = (onlyRowid ? PRE_QUERY_ROWID_SQL : PRE_QUERY_KV_SQL); - sql = AssembleSqlForSuggestIndex(querySqlForUse); + sql = AssembleSqlForSuggestIndex(querySqlForUse, FILTER_NATIVE_DATA_SQL); sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); if (transformed_) { LOGD("This query object has been parsed."); @@ -180,7 +179,7 @@ int QueryObject::GetCountQuerySql(std::string &sql) if (errCode != E_OK) { return errCode; } - sql = AssembleSqlForSuggestIndex(PRE_GET_COUNT_SQL); + sql = AssembleSqlForSuggestIndex(PRE_GET_COUNT_SQL, FILTER_NATIVE_DATA_SQL); sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); sql += countSql_; return E_OK; @@ -242,7 +241,7 @@ void QueryObject::FilterSymbolToAddBracketLink(bool &isNeedEndBracket, std::stri } } -int QueryObject::ToQuerySql() +int QueryObject::ParseQueryObjNodeToSQL() { if (queryObjNodes_.empty()) { querySql_ += ";"; @@ -270,6 +269,16 @@ int QueryObject::ToQuerySql() querySql_ += ") "; } + return errCode; +} + +int QueryObject::ToQuerySql() +{ + int errCode = ParseQueryObjNodeToSQL(); + if (errCode != E_OK) { + return errCode; + } + // Limit needs to be placed after orderby and processed separately in the limit branch if (hasPrefixKey_ && !hasOrderBy_ && !hasLimit_ && isNeedOrderbyKey_) { LOGD("Need add order by key at last when has prefixkey no need order by value and limit!"); @@ -280,6 +289,24 @@ int QueryObject::ToQuerySql() return errCode; } +int QueryObject::ToQuerySyncSql(bool hasSubQuery) +{ + int errCode = ParseQueryObjNodeToSQL(); + if (errCode != E_OK) { + return errCode; + } + + // Order by time when no order by and no limit and no need order by key. + if (!hasOrderBy_ && !hasLimit_ && !isNeedOrderbyKey_) { + querySql_ += "ORDER BY timestamp ASC"; + } + + if (!hasSubQuery) { + querySql_ += ";"; + } + return errCode; +} + int QueryObject::ToGetCountSql() { countSql_.clear(); @@ -342,6 +369,7 @@ int QueryObject::GetQuerySqlStatement(sqlite3 *dbHandle, const std::string &sql, // bind the prefix key for the first and second args. errCode = SQLiteUtils::BindPrefixKey(statement, 1, prefixKey_); if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); LOGE("[Query] Get statement when bind prefix key, errCode = %d", errCode); return errCode; } @@ -351,6 +379,7 @@ int QueryObject::GetQuerySqlStatement(sqlite3 *dbHandle, const std::string &sql, for (const QueryObjNode &objNode : queryObjNodes_) { errCode = BindFieldValue(statement, objNode, index); if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); LOGE("[Query] Get statement fail when bind field value, errCode = %d", errCode); return errCode; } @@ -371,6 +400,7 @@ int QueryObject::GetCountSqlStatement(sqlite3 *dbHandle, const std::string &coun // bind the prefix key for the first and second args. errCode = SQLiteUtils::BindPrefixKey(countStmt, 1, prefixKey_); if (errCode != E_OK) { + SQLiteUtils::ResetStatement(countStmt, true, errCode); LOGE("[Query] Get count statement fail when bind prefix key, errCode = %d", errCode); return errCode; } @@ -383,6 +413,7 @@ int QueryObject::GetCountSqlStatement(sqlite3 *dbHandle, const std::string &coun } errCode = BindFieldValue(countStmt, objNode, index); if (errCode != E_OK) { + SQLiteUtils::ResetStatement(countStmt, true, errCode); LOGE("[Query] Get count statement fail when bind field value, errCode = %d", errCode); return errCode; } @@ -704,14 +735,14 @@ int QueryObject::ParseQueryExpression(const QueryObjNode &queryNode, std::string return E_OK; } -std::string QueryObject::AssembleSqlForSuggestIndex(const std::string &baseSql) const +std::string QueryObject::AssembleSqlForSuggestIndex(const std::string &baseSql, const std::string &filter) const { std::string formatIndex = CheckAndFormatSuggestIndex(); if (formatIndex.empty()) { - return baseSql + FILTER_NATIVE_DATA_SQL; + return baseSql + filter; } - return baseSql + USING_INDEX + "'" + formatIndex + "' " + FILTER_NATIVE_DATA_SQL; + return baseSql + USING_INDEX + "'" + formatIndex + "' " + filter; } std::string QueryObject::CheckAndFormatSuggestIndex() const @@ -733,4 +764,123 @@ std::string QueryObject::CheckAndFormatSuggestIndex() const return SchemaUtils::FieldPathString(indexName); } + +int QueryObject::GetSyncDataQuerySql(std::string &sql) +{ + int errCode = E_OK; + if (!IsValid(errCode)) { + return errCode; + } + + bool hasSubQuery = false; + if (hasLimit_) { + hasSubQuery = true; // Need sub query. + } else { + isNeedOrderbyKey_ = false; // Need order by timestamp. + } + + sql = AssembleSqlForSuggestIndex(PRE_QUERY_ITEM_SQL, FILTER_REMOTE_QUERY); + sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); + sql = hasSubQuery ? sql : (sql + " AND (timestamp>=? AND timestamp= ? AND timestamp < ?) ORDER BY timestamp;"; + } + + return errCode; +} + +int QueryObject::GetQuerySyncStatement(sqlite3 *dbHandle, uint64_t beginTime, uint64_t endTime, + sqlite3_stmt *&statement) +{ + std::string sql; + int errCode = GetSyncDataQuerySql(sql); + if (errCode != E_OK) { + LOGE("[Query] Get SQL fail!"); + return -E_INVALID_QUERY_FORMAT; + } + + errCode = SQLiteUtils::GetStatement(dbHandle, sql, statement); + if (errCode != E_OK) { + LOGE("[Query] Get statement fail!"); + return -E_INVALID_QUERY_FORMAT; + } + + int index = 1; // begin with 1. + if (hasPrefixKey_) { + // bind the prefix key for the first and second args. + errCode = SQLiteUtils::BindPrefixKey(statement, index, prefixKey_); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + LOGE("[Query] Get statement when bind prefix key, errCode = %d", errCode); + return errCode; + } + index = 3; // begin with 3 next if prefix key exists. + } + + errCode = SQLiteUtils::BindInt64ToStatement(statement, index++, beginTime); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; + } + + errCode = SQLiteUtils::BindInt64ToStatement(statement, index++, endTime); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; + } + + for (const QueryObjNode &objNode : queryObjNodes_) { + errCode = BindFieldValue(statement, objNode, index); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + LOGE("[Query] Get statement fail when bind field value, errCode = %d", errCode); + return errCode; + } + } + return errCode; +} + +// For query sync serialized +const std::list &QueryObject::GetQueryObjNodes() const +{ + return queryObjNodes_; +} + +const std::vector &QueryObject::GetPrefixKey() const +{ + return prefixKey_; +} + +const std::string &QueryObject::GetSuggestIndex() const +{ + return suggestIndex_; +} + +bool QueryObject::IsValid() const +{ + return isValid_; +} + +bool QueryObject::IsQueryOnlyByKey() const +{ + for (const auto &iter : queryObjNodes_) { + if (iter.operFlag != QueryObjType::LIMIT && iter.operFlag != QueryObjType::QUERY_BY_KEY_PREFIX) { + return false; + } + } + return true; +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h index 4884867e9..43e4ebbed 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h @@ -25,6 +25,7 @@ class QueryObject { public: QueryObject(); explicit QueryObject(const Query &query); + QueryObject(const std::list &queryObjNodes, const std::vector prefixKey); ~QueryObject() = default; int GetQuerySql(std::string &sql, bool onlyRowid = false); int GetCountQuerySql(std::string &sql); @@ -35,8 +36,21 @@ public: void SetSchema(const SchemaObject &schema); int GetQuerySqlStatement(sqlite3 *dbHandle, const std::string &sql, sqlite3_stmt *&statement); int GetCountSqlStatement(sqlite3 *dbHandle, const std::string &countSql, sqlite3_stmt *&countStmt); + + // For query Sync + int GetQuerySyncStatement(sqlite3 *dbHandle, uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement); + + // For query sync serialized + const std::list &GetQueryObjNodes() const; + const std::vector &GetPrefixKey() const; + const std::string &GetSuggestIndex() const; + bool IsValid() const; + bool IsQueryOnlyByKey() const; + private: + int ParseQueryObjNodeToSQL(); int ToQuerySql(); + int ToQuerySyncSql(bool hasSubQuery = false); int ToGetCountSql(); int CheckEqualFormat(std::list::iterator &iter) const; int CheckLinkerFormat(std::list::iterator &iter) const; @@ -54,8 +68,9 @@ private: static std::string MapCastTypeSql(const FieldType &type); int BindFieldValue(sqlite3_stmt *statement, const QueryObjNode &queryNode, int &index) const; void FilterSymbolToAddBracketLink(bool &isNeedEndBracket, std::string &querySql) const; - std::string AssembleSqlForSuggestIndex(const std::string &baseSql) const; + std::string AssembleSqlForSuggestIndex(const std::string &baseSql, const std::string &filter) const; std::string CheckAndFormatSuggestIndex() const; + int GetSyncDataQuerySql(std::string &sql); SchemaObject schema_; std::list queryObjNodes_; std::vector prefixKey_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp new file mode 100644 index 000000000..d09982680 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "query_sync_object.h" + +#include "db_errno.h" +#include "log_print.h" +#include "db_common.h" +#include "version.h" + +namespace DistributedDB { +namespace { +const std::string MAGIC = "remote query"; + +int SerializeDataObjNode(Parcel &parcel, const QueryObjNode &objNode) +{ + if (objNode.operFlag == QueryObjType::OPER_ILLEGAL) { + return -E_INVALID_QUERY_FORMAT; + } + (void)parcel.WriteUInt32(static_cast(objNode.operFlag)); + parcel.EightByteAlign(); + (void)parcel.WriteString(objNode.fieldName); + (void)parcel.WriteInt(static_cast(objNode.type)); + (void)parcel.WriteUInt32(objNode.fieldValue.size()); + + for (const FieldValue &value : objNode.fieldValue) { + (void)parcel.WriteString(value.stringValue); + + // string may not closely arranged continuously + // longValue is maximum length in union + (void)parcel.WriteInt64(value.longValue); + } + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + return E_OK; +} + +int DeSerializeDataObjNode(Parcel &parcel, QueryObjNode &objNode) +{ + uint32_t readOperFlag = 0; + (void)parcel.ReadUInt32(readOperFlag); + objNode.operFlag = static_cast(readOperFlag); + parcel.EightByteAlign(); + + (void)parcel.ReadString(objNode.fieldName); + + int readInt = -1; + (void)parcel.ReadInt(readInt); + objNode.type = static_cast(readInt); + + uint32_t valueSize = 0; + (void)parcel.ReadUInt32(valueSize); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + + for (size_t i = 0; i < valueSize; i++) { + FieldValue value; + (void)parcel.ReadString(value.stringValue); + + (void)parcel.ReadInt64(value.longValue); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + objNode.fieldValue.push_back(value); + } + return E_OK; +} +} + +QuerySyncObject::QuerySyncObject() +{} + +QuerySyncObject::QuerySyncObject(const QueryObject &queryObj) + : queryObj_(queryObj) +{} + +int QuerySyncObject::GetObjContext(ObjContext &objContext) const +{ + if (!queryObj_.IsValid()) { + return -E_INVALID_QUERY_FORMAT; + } + objContext.prefixKey.assign(queryObj_.GetPrefixKey().begin(), queryObj_.GetPrefixKey().end()); + objContext.suggestIndex = queryObj_.GetSuggestIndex(); + objContext.queryObjNodes = queryObj_.GetQueryObjNodes(); + return E_OK; +} + +std::string QuerySyncObject::GetIdentify() +{ + if (!queryObj_.IsValid()) { + return std::string(); + } + uint64_t len = Parcel::GetVectorCharLen(queryObj_.GetPrefixKey()); + // suggestionIndex is local attribute, do not need to be propagated to remote + for (const QueryObjNode &node : queryObj_.GetQueryObjNodes()) { + if (node.operFlag == QueryObjType::LIMIT || node.operFlag == QueryObjType::ORDERBY) { + continue; + } + // operFlag and valueType is int + len += Parcel::GetUInt32Len() + Parcel::GetIntLen() + Parcel::GetStringLen(node.fieldName); + for (const FieldValue &value : node.fieldValue) { + len += Parcel::GetStringLen(value.stringValue) + Parcel::GetInt64Len(); + } + } + + std::vector buff(len, 0); // It will affect the hash result, the default value cannot be modified + Parcel parcel(buff.data(), len); + + // The order needs to be consistent, otherwise it will affect the hash result + (void)parcel.WriteVectorChar(queryObj_.GetPrefixKey()); + for (const QueryObjNode &node : queryObj_.GetQueryObjNodes()) { + if (node.operFlag == QueryObjType::LIMIT || node.operFlag == QueryObjType::ORDERBY) { + continue; + } + (void)parcel.WriteUInt32(static_cast(node.operFlag)); + (void)parcel.WriteInt(static_cast(node.type)); + (void)parcel.WriteString(node.fieldName); + for (const FieldValue &value : node.fieldValue) { + (void)parcel.WriteInt64(value.longValue); + (void)parcel.WriteString(value.stringValue); + } + } + + if (parcel.IsError()) { + return std::string(); + } + + std::vector hashBuff; + int errCode = DBCommon::CalcValueHash(buff, hashBuff); + if (errCode != E_OK) { + return std::string(); + } + return DBCommon::VectorToHexString(hashBuff); +} + +const QueryObject &QuerySyncObject::GetQueryObj() const +{ + return queryObj_; +} + +uint32_t QuerySyncObject::CalculateParcelLen(uint32_t softWareVersion) const +{ + if (softWareVersion == SOFTWARE_VERSION_CURRENT) { + return CalculateLen(); + } + LOGE("current not support!"); + return 0; +} + +bool QuerySyncObject::HasLimit() const +{ + for (const QueryObjNode &node : queryObj_.GetQueryObjNodes()) { + if (node.operFlag == QueryObjType::LIMIT) { + return true; + } + } + return false; +} + +int QuerySyncObject::SerializeData(Parcel &parcel, uint32_t softWareVersion) +{ + ObjContext context; + int errCode = GetObjContext(context); + if (errCode != E_OK) { + return errCode; + } + + (void)parcel.WriteString(MAGIC); + (void)parcel.WriteUInt32(context.version); + (void)parcel.WriteVectorChar(context.prefixKey); + (void)parcel.WriteString(context.suggestIndex); + (void)parcel.WriteUInt32(context.queryObjNodes.size()); + + parcel.EightByteAlign(); + + for (const QueryObjNode &node : context.queryObjNodes) { + errCode = SerializeDataObjNode(parcel, node); + if (errCode != E_OK) { + return errCode; + } + } + if (parcel.IsError()) { // parcel almost success + return -E_INVALID_ARGS; + } + + parcel.EightByteAlign(); + return E_OK; +} + +int QuerySyncObject::DeSerializeData(Parcel &parcel, QueryObject &queryObj) +{ + std::string magic; + int errCode = parcel.ReadString(magic); + if (magic != MAGIC) { + return -E_INVALID_ARGS; + } + + ObjContext context; + (void)parcel.ReadUInt32(context.version); + if (context.version != 0) { + LOGE("Parcel version and deserialize version not matched!"); + return -E_VERSION_NOT_SUPPORT; + } + + (void)parcel.ReadVectorChar(context.prefixKey); + (void)parcel.ReadString(context.suggestIndex); + + uint32_t nodesSize = 0; + (void)parcel.ReadUInt32(nodesSize); + if (parcel.IsError()) { // almost success + return -E_INVALID_ARGS; + } + parcel.EightByteAlign(); + for (size_t i = 0; i < nodesSize; i++) { + QueryObjNode node; + errCode = DeSerializeDataObjNode(parcel, node); + if (errCode != E_OK) { + return errCode; + } + context.queryObjNodes.emplace_back(node); + } + + if (parcel.IsError()) { // almost success + return -E_INVALID_ARGS; + } + + queryObj = QueryObject(context.queryObjNodes, context.prefixKey); + return E_OK; +} + +uint32_t QuerySyncObject::CalculateLen() const +{ + uint64_t len = Parcel::GetStringLen(MAGIC); + len += Parcel::GetUInt32Len(); // version + len += Parcel::GetVectorCharLen(queryObj_.GetPrefixKey()); + len += Parcel::GetStringLen(queryObj_.GetSuggestIndex()); + len += Parcel::GetUInt32Len(); // nodes size + len = Parcel::GetEightByteAlign(len); + for (const QueryObjNode &node : queryObj_.GetQueryObjNodes()) { + if (node.operFlag == QueryObjType::OPER_ILLEGAL) { + LOGE("contain illegal operator for query sync!"); + return 0; + } + // operflag, fieldName, query value type, value size, union max size, string value + len += Parcel::GetUInt32Len(); + len = Parcel::GetEightByteAlign(len); + len += Parcel::GetStringLen(node.fieldName) + + Parcel::GetIntLen() + Parcel::GetUInt32Len(); + for (size_t i = 0; i < node.fieldValue.size(); i++) { + len += Parcel::GetInt64Len() + Parcel::GetStringLen(node.fieldValue[i].stringValue); + } + } + len = Parcel::GetEightByteAlign(len); + if (len > INT32_MAX) { + return 0; + } + return len; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h new file mode 100644 index 000000000..b52a4fc4f --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef QUERY_SYNC_OBJECT_H +#define QUERY_SYNC_OBJECT_H + +#include + +#include "query_object.h" +#include "parcel.h" + +namespace DistributedDB { +struct ObjContext { + uint32_t version = 0; // serialized struct version + std::vector prefixKey{}; + std::string suggestIndex{}; + std::list queryObjNodes{}; +}; + +class QuerySyncObject { +public: + QuerySyncObject(); + explicit QuerySyncObject(const QueryObject &queryObj); + ~QuerySyncObject() = default; + + std::string GetIdentify(); + bool HasLimit() const; + + int SerializeData(Parcel &parcel, uint32_t softWareVersion); + + // Before use QueryObject, should call Parcel.IsError() to Get result. + static int DeSerializeData(Parcel &parcel, QueryObject &queryObj); + uint32_t CalculateParcelLen(uint32_t softWareVersion) const; + + const QueryObject &GetQueryObj() const; + +private: + uint32_t CalculateLen() const; + int GetObjContext(ObjContext &objContext) const; + + QueryObject queryObj_; +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_local_kvdb_snapshot.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_local_kvdb_snapshot.h index c94962f75..cc8b7a1ce 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_local_kvdb_snapshot.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_local_kvdb_snapshot.h @@ -37,7 +37,7 @@ public: int Get(const Key &key, Value &value) const override; // Get the entries of the key set - int GetEntries(const Key &keyPrefix, std::vector &entry) const override; + int GetEntries(const Key &keyPrefix, std::vector &entries) const override; void Close(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.cpp new file mode 100644 index 000000000..6f7a84826 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sqlite_single_ver_continue_token.h" + +namespace DistributedDB { +SQLiteSingleVerContinueToken::SQLiteSingleVerContinueToken(TimeStamp begin, TimeStamp end) + : timeRanges_(MulDevTimeRanges{{"", {begin, end}}}) +{} + +SQLiteSingleVerContinueToken::SQLiteSingleVerContinueToken( + const SyncTimeRange &timeRange, const QueryObject &queryObject) + : queryObject_(std::map{{"", queryObject}}), + timeRanges_(MulDevTimeRanges{{"", {timeRange.beginTime, timeRange.endTime}}}), + deleteTimeRanges_(MulDevTimeRanges{{"", {timeRange.deleteBeginTime, timeRange.deleteEndTime}}}) +{} + +SQLiteSingleVerContinueToken::SQLiteSingleVerContinueToken(MulDevTimeRanges timeRanges) + : timeRanges_(timeRanges) +{} + +SQLiteSingleVerContinueToken::~SQLiteSingleVerContinueToken() +{} + +bool SQLiteSingleVerContinueToken::CheckValid() const +{ + return ((magicBegin_ == MAGIC_BEGIN) && (magicEnd_ == MAGIC_END)); +} + +TimeStamp SQLiteSingleVerContinueToken::GetQueryBeginTime() const +{ + return GetBeginTimeStamp(timeRanges_); +} + +TimeStamp SQLiteSingleVerContinueToken::GetQueryEndTime() const +{ + return GetEndTimeStamp(timeRanges_); +} + +TimeStamp SQLiteSingleVerContinueToken::GetDeletedBeginTime() const +{ + return GetBeginTimeStamp(deleteTimeRanges_); +} + +TimeStamp SQLiteSingleVerContinueToken::GetDeletedEndTime() const +{ + return GetEndTimeStamp(deleteTimeRanges_); +} + +void SQLiteSingleVerContinueToken::SetNextBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime) +{ + RemovePrevDevAndSetBeginTime(deviceID, nextBeginTime, timeRanges_); +} + +const MulDevTimeRanges& SQLiteSingleVerContinueToken::GetTimeRanges() +{ + return timeRanges_; +} + +void SQLiteSingleVerContinueToken::SetDeletedNextBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime) +{ + RemovePrevDevAndSetBeginTime(deviceID, nextBeginTime, deleteTimeRanges_); +} + +const MulDevTimeRanges& SQLiteSingleVerContinueToken::GetDeletedTimeRanges() const +{ + return deleteTimeRanges_; +} + +void SQLiteSingleVerContinueToken::FinishGetQueryData() +{ + timeRanges_.clear(); +} + +void SQLiteSingleVerContinueToken::FinishGetDeletedData() +{ + deleteTimeRanges_.clear(); +} + +bool SQLiteSingleVerContinueToken::IsGetQueryDataFinished() const +{ + return timeRanges_.empty(); +} + +bool SQLiteSingleVerContinueToken::IsGetDeletedDataFinished() const +{ + return deleteTimeRanges_.empty(); +} + +bool SQLiteSingleVerContinueToken::IsQuerySync() const +{ + return !queryObject_.empty(); +} + +QueryObject SQLiteSingleVerContinueToken::GetQuery() const +{ + if (!queryObject_.empty()) { + return queryObject_.begin()->second; + } + return QueryObject{}; +} + +void SQLiteSingleVerContinueToken::RemovePrevDevAndSetBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime, + MulDevTimeRanges &timeRanges) +{ + auto iter = timeRanges.find(deviceID); + if (iter == timeRanges.end()) { + return; + } + iter = timeRanges.erase(timeRanges.begin(), iter); + iter->second.first = nextBeginTime; +} + +TimeStamp SQLiteSingleVerContinueToken::GetBeginTimeStamp(const MulDevTimeRanges &timeRanges) const +{ + if (timeRanges.empty()) { + return 0; + } + return timeRanges.begin()->second.first; +} + +TimeStamp SQLiteSingleVerContinueToken::GetEndTimeStamp(const MulDevTimeRanges &timeRanges) const +{ + if (timeRanges.empty()) { + return static_cast(INT64_MAX); + } + return timeRanges.begin()->second.second; +} +} // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.h new file mode 100644 index 000000000..c8a8d7163 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_SINGLE_VER_CONTINUE_TOKEN_H +#define SQLITE_SINGLE_VER_CONTINUE_TOKEN_H + +#include + +#include "db_types.h" +#include "query_object.h" +#include "single_ver_kvdb_sync_interface.h" + +namespace DistributedDB { +class SQLiteSingleVerContinueToken { +public: + // For one device. + SQLiteSingleVerContinueToken(TimeStamp begin, TimeStamp end); + + // For one device in query sync. + SQLiteSingleVerContinueToken(const SyncTimeRange &timeRange, const QueryObject &queryObject); + + // For multiple device. + explicit SQLiteSingleVerContinueToken(MulDevTimeRanges timeRanges); + + ~SQLiteSingleVerContinueToken(); + /* + * function: Check the magic number at the beginning and end of the SingleVerContinueToken. + * returnValue: Return true if the begin and end magic number is OK. + * Return false if the begin or end magic number is error. + */ + bool CheckValid() const; + + TimeStamp GetQueryBeginTime() const; + TimeStamp GetQueryEndTime() const; + TimeStamp GetDeletedBeginTime() const; + TimeStamp GetDeletedEndTime() const; + + void SetNextBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime); + const MulDevTimeRanges &GetTimeRanges(); + void SetDeletedNextBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime); + const MulDevTimeRanges &GetDeletedTimeRanges() const; + + void FinishGetQueryData(); + void FinishGetDeletedData(); + + bool IsGetQueryDataFinished() const; + bool IsGetDeletedDataFinished() const; + + bool IsQuerySync() const; + QueryObject GetQuery() const; + +private: + void RemovePrevDevAndSetBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime, MulDevTimeRanges &timeRanges); + + TimeStamp GetBeginTimeStamp(const MulDevTimeRanges &timeRanges) const; + TimeStamp GetEndTimeStamp(const MulDevTimeRanges &timeRanges) const; + + static const unsigned int MAGIC_BEGIN = 0x600D0AC7; // for token guard + static const unsigned int MAGIC_END = 0x0AC7600D; // for token guard + unsigned int magicBegin_ = MAGIC_BEGIN; + std::map queryObject_; + MulDevTimeRanges timeRanges_; + MulDevTimeRanges deleteTimeRanges_; + unsigned int magicEnd_ = MAGIC_END; +}; +} // namespace DistributedDB +#endif // SQLITE_SINGLE_VER_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp index 3ad498707..70d7cec2c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp @@ -238,9 +238,9 @@ void SQLiteSingleVerDatabaseUpgrader::SetMetaUpgrade(const SecurityOption &curre } } -void SQLiteSingleVerDatabaseUpgrader::SetSubdir(const std::string &parentDir) +void SQLiteSingleVerDatabaseUpgrader::SetSubdir(const std::string &subDir) { - subDir_ = parentDir; + subDir_ = subDir; } int SQLiteSingleVerDatabaseUpgrader::SetPathSecOptWithCheck(const std::string &path, const SecurityOption &secOption, diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.h index 28ce76587..3d5190e44 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.h @@ -32,7 +32,7 @@ public: static int CreateDbDir(); void SetMetaUpgrade(const SecurityOption ¤tOpt, const SecurityOption &expectOpt, const std::string &subDir); - void SetSubdir(const std::string &subdir); + void SetSubdir(const std::string &subDir); static int SetPathSecOptWithCheck(const std::string &path, const SecurityOption &secOption, const std::string &dbStore, bool isWithChecked = false); static int SetSecOption(const std::string &path, const SecurityOption &secOption, bool isWithChecked); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index 8f38ccbb0..1817294ab 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -45,9 +45,14 @@ namespace DistributedDB { } while (0) namespace { - constexpr int MAX_SYNC_BLOCK_SIZE = 31457280; // 30MB constexpr int DEF_LIFE_CYCLE_TIME = 60000; // 60s constexpr int WAIT_DELEGATE_CALLBACK_TIME = 100; + // In querySync, when getting query data finished, + // if the block size reach the half of max block size, will get deleted data next; + // if the block size not reach the half of max block size, will not get deleted data. + constexpr float QUERY_SYNC_THRESHOLD = 0.50; + + const std::string CREATE_DB_TIME = "createDBTime"; // Currently this function only suitable to be call from sync in insert_record_from_sync procedure // Take attention if future coder attempt to call it in other situation procedure @@ -92,25 +97,74 @@ namespace { return errCode; } - void ProcessContinueToken(int &errCode, TimeStamp end, const std::vector &dataItems, - ContinueTokenStruct *&token) + // Called when get multiple dev data. + // deviceID is the device which currently being getting. When getting one dev data, deviceID is "". + // dataItems is the DataItems which already be get from DB sorted by timestamp. + // token must not be null. + void ProcessContinueToken(const DeviceID &deviceID, const std::vector &dataItems, int &errCode, + SQLiteSingleVerContinueToken *&token) { - if (token == nullptr) { + if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. + delete token; + token = nullptr; return; } - if (!dataItems.empty()) { - TimeStamp timestamp = dataItems.back().timeStamp; - if (timestamp > INT64_MAX - 1) { - token->SetBeginTimeStamp(INT64_MAX); - } else { - token->SetBeginTimeStamp(timestamp + 1); // Add 1 for the timestamp. - } - token->SetEndTimeStamp(end); - } else { + + if (dataItems.empty()) { + errCode = -E_INTERNAL_ERROR; + LOGE("Get data unfinished but dataitems is empty."); + delete token; + token = nullptr; + return; + } + + TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; + } + token->SetNextBeginTime(deviceID, nextBeginTime); + return; + } + + // Called when get one dev data. + void ProcessContinueToken(const std::vector &dataItems, int &errCode, + SQLiteSingleVerContinueToken *&token) + { + ProcessContinueToken("", dataItems, errCode, token); + } + + // Called when get query sync data. + // dataItems is the DataItems which already be get from DB sorted by timestamp. + // token must not be null. + void ProcessContinueTokenForQuerySync(const std::vector &dataItems, int &errCode, + SQLiteSingleVerContinueToken *&token) + { + if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. delete token; token = nullptr; + return; + } + + if (dataItems.empty()) { errCode = -E_INTERNAL_ERROR; + LOGE("Get data unfinished but dataitems is empty."); + delete token; + token = nullptr; + return; + } + + TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; + } + bool getDeleteData = ((dataItems.back().flag & DataItem::DELETE_FLAG) != 0); + if (getDeleteData) { + token->FinishGetQueryData(); + token->SetDeletedNextBeginTime("", nextBeginTime); + } else { + token->SetNextBeginTime("", nextBeginTime); } + return; } void UpdateSecProperties(KvDBProperties &properties, bool isReadOnly, const SchemaObject &savedSchemaObj, @@ -129,6 +183,43 @@ namespace { properties.SetIntProp(KvDBProperties::SECURITY_LABEL, engine->GetSecurityOption().securityLabel); properties.SetIntProp(KvDBProperties::SECURITY_FLAG, engine->GetSecurityOption().securityFlag); } + + void ReleaseKvEntries(std::vector &entries) + { + for (auto &itemEntry : entries) { + delete itemEntry; + itemEntry = nullptr; + } + entries.clear(); + } + + int GetKvEntriesByDataItems(std::vector &entries, std::vector &dataItems) + { + int errCode = E_OK; + for (auto &item : dataItems) { + auto entry = new (std::nothrow) GenericSingleVerKvEntry(); + if (entry == nullptr) { + errCode = -E_OUT_OF_MEMORY; + LOGE("GetKvEntries failed, errCode:%d", errCode); + ReleaseKvEntries(entries); + break; + } + entry->SetEntryData(std::move(item)); + entries.push_back(entry); + } + return errCode; + } + + bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, + size_t appendLen) + { + bool reachThreshold = false; + for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { + blockSize += SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(dataItems[i], appendLen); + reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); + } + return !reachThreshold; + } } SQLiteSingleVerNaturalStore::SQLiteSingleVerNaturalStore() @@ -140,7 +231,8 @@ SQLiteSingleVerNaturalStore::SQLiteSingleVerNaturalStore() isReadOnly_(false), lifeCycleNotifier_(nullptr), lifeTimerId_(0), - autoLifeTime_(DEF_LIFE_CYCLE_TIME) + autoLifeTime_(DEF_LIFE_CYCLE_TIME), + createDBTime_(0) {} SQLiteSingleVerNaturalStore::~SQLiteSingleVerNaturalStore() @@ -270,14 +362,11 @@ inline bool AmendValueShouldBeUse(int errCode) return (errCode == -E_VALUE_MATCH_AMENDED); } -inline bool ValueIsSomehowWrong(int errCode) +inline bool IsValueMismatched(int errCode) { - if (errCode == -E_VALUE_MISMATCH_FEILD_COUNT || - errCode == -E_VALUE_MISMATCH_FEILD_TYPE || - errCode == -E_VALUE_MISMATCH_CONSTRAINT) { - return true; - } - return false; + return (errCode == -E_VALUE_MISMATCH_FEILD_COUNT || + errCode == -E_VALUE_MISMATCH_FEILD_TYPE || + errCode == -E_VALUE_MISMATCH_CONSTRAINT); } } @@ -315,7 +404,7 @@ int SQLiteSingleVerNaturalStore::CheckValueAndAmendIfNeed(ValueSource sourceType useAmendValue = true; return E_OK; } - if (ValueIsSomehowWrong(errCode)) { + if (IsValueMismatched(errCode)) { return errCode; } } else { @@ -378,6 +467,22 @@ int SQLiteSingleVerNaturalStore::CheckDatabaseRecovery(const KvDBProperties &kvD return errCode; } +int SQLiteSingleVerNaturalStore::GetStorageEngine(const KvDBProperties &kvDBProp) +{ + int errCode = E_OK; + storageEngine_ = + static_cast(StorageEngineManager::GetStorageEngine(kvDBProp, errCode)); + if (storageEngine_ == nullptr) { + return errCode; + } + + if (storageEngine_->IsEngineCorrupted()) { + LOGE("[SqlSinStore][GetStorageEngine] database engine is corrupted, not need continue to open!"); + errCode = -E_INVALID_PASSWD_OR_CORRUPTED_DB; + } + return errCode; +} + int SQLiteSingleVerNaturalStore::Open(const KvDBProperties &kvDBProp) { std::lock_guard lock(initialMutex_); @@ -392,15 +497,9 @@ int SQLiteSingleVerNaturalStore::Open(const KvDBProperties &kvDBProp) bool isReadOnly = false; SchemaObject savedSchemaObj; - storageEngine_ = - static_cast(StorageEngineManager::GetStorageEngine(kvDBProp, errCode)); - if (errCode != E_OK) { - goto ERROR; - } - if (storageEngine_->IsEngineCorrupted()) { - errCode = -E_INVALID_PASSWD_OR_CORRUPTED_DB; - LOGE("[SqlSinStore][Open] database engine is corrupted, not need continue to open! errCode = [%d]", errCode); + errCode = GetStorageEngine(kvDBProp); + if (errCode != E_OK) { goto ERROR; } @@ -429,6 +528,13 @@ int SQLiteSingleVerNaturalStore::Open(const KvDBProperties &kvDBProp) UpdateSecProperties(MyProp(), isReadOnly, savedSchemaObj, storageEngine_); StartSyncer(); + OnKill([this](){ ReleaseResources(); }); + + errCode = SaveCreateDBTimeIfNotExisted(); + if (errCode != E_OK) { + goto ERROR; + } + InitialLocalDataTimestamp(); isInitialized_ = true; isReadOnly_ = isReadOnly; @@ -440,7 +546,6 @@ ERROR: void SQLiteSingleVerNaturalStore::Close() { - SyncAbleKvDB::Close(); ReleaseResources(); } @@ -529,6 +634,34 @@ int SQLiteSingleVerNaturalStore::PutMetaData(const Key &key, const Value &value) return errCode; } +// Delete multiple meta data records in a transaction. +int SQLiteSingleVerNaturalStore::DeleteMetaData(const std::vector &keys) +{ + for (const auto &key : keys) { + if (key.empty() || key.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; + } + } + int errCode = E_OK; + auto handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + handle->StartTransaction(TransactType::IMMEDIATE); + errCode = handle->DeleteMetaData(keys); + if (errCode != E_OK) { + handle->Rollback(); + LOGE("[SinStore] DeleteMetaData failed, errCode = %d", errCode); + } else { + handle->Commit(); + } + + ReleaseHandle(handle); + HeartBeatForLifeCycle(); + return errCode; +} + int SQLiteSingleVerNaturalStore::GetAllMetaKeys(std::vector &keys) const { CHECK_STORAGE_ENGINE; @@ -592,11 +725,7 @@ int SQLiteSingleVerNaturalStore::GetSyncData(TimeStamp begin, TimeStamp end, std ERROR: if (errCode != E_OK && errCode != -E_UNFINISHED) { - for (auto &itemEntry : entries) { - delete itemEntry; - itemEntry = nullptr; - } - entries.clear(); + ReleaseKvEntries(entries); } HeartBeatForLifeCycle(); return errCode; @@ -605,11 +734,11 @@ ERROR: int SQLiteSingleVerNaturalStore::GetSyncData(TimeStamp begin, TimeStamp end, std::vector &dataItems, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const { - if (begin >= end || dataSizeInfo.blockSize > MAX_SYNC_BLOCK_SIZE) { + if (begin >= end || dataSizeInfo.blockSize > DBConstant::MAX_SYNC_BLOCK_SIZE) { return -E_INVALID_ARGS; } - auto token = new (std::nothrow) ContinueTokenStruct; + auto token = new (std::nothrow) SQLiteSingleVerContinueToken(begin, end); if (token == nullptr) { LOGE("[SQLiteSingleVerNaturalStore][NewToken] Bad alloc."); return -E_OUT_OF_MEMORY; @@ -627,23 +756,103 @@ int SQLiteSingleVerNaturalStore::GetSyncData(TimeStamp begin, TimeStamp end, std } ERROR: - if (errCode != -E_UNFINISHED) { - if (errCode != E_OK) { - dataItems.clear(); - } + if (errCode != -E_UNFINISHED && errCode != E_OK) { + dataItems.clear(); + } + ProcessContinueToken(dataItems, errCode, token); + continueStmtToken = static_cast(token); + + ReleaseHandle(handle); + return errCode; +} + +int SQLiteSingleVerNaturalStore::GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const +{ + if (!timeRange.IsValid()) { + return -E_INVALID_ARGS; + } + int errCode = CheckReadDataControlled(); + if (errCode != E_OK) { + LOGE("[GetEntries] Existed cache prevents the reading from query sync[%d]!", errCode); + return errCode; + } + + auto token = new (std::nothrow) SQLiteSingleVerContinueToken(timeRange, query); + if (token == nullptr) { + LOGE("[SingleVerNStore] Allocate continue token failed."); + return -E_OUT_OF_MEMORY; + } + + int innerCode; + std::vector dataItems; + errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); + if (errCode != E_OK && errCode != -E_UNFINISHED) { // The code need be sent to outside except new error happened. + goto ERROR; + } + + innerCode = GetKvEntriesByDataItems(entries, dataItems); + if (innerCode != E_OK) { + errCode = innerCode; delete token; token = nullptr; - } else { - ProcessContinueToken(errCode, end, dataItems, token); } - if (handle != nullptr) { - ReleaseHandle(handle); - } +ERROR: continueStmtToken = static_cast(token); return errCode; } +/** + * Caller must ensure that parameter continueStmtToken is valid. + * If error happened, token will be deleted here. + */ +int SQLiteSingleVerNaturalStore::GetSyncDataForQuerySync(std::vector &dataItems, + SQLiteSingleVerContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const +{ + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetHandle(false, errCode); + if (handle == nullptr) { + goto ERROR; + } + + // Get query data. + if (!continueStmtToken->IsGetQueryDataFinished()) { + LOGD("[SingleVerNStore] Get query data between %llu and %llu.", continueStmtToken->GetQueryBeginTime(), + continueStmtToken->GetQueryEndTime()); + errCode = handle->GetSyncDataByQuery(dataItems, GetAppendedLen(), continueStmtToken->GetQuery(), dataSizeInfo, + std::make_pair(continueStmtToken->GetQueryBeginTime(), continueStmtToken->GetQueryEndTime())); + } + + // Get query data finished. + if (errCode == E_OK || errCode == -E_FINISHED) { + // Clear query timeRange. + continueStmtToken->FinishGetQueryData(); + + // Get delete time next. + if (!continueStmtToken->IsGetDeletedDataFinished() && + CanHoldDeletedData(dataItems, dataSizeInfo, GetAppendedLen())) { + LOGD("[SingleVerNStore] Get deleted data between %llu and %llu.", continueStmtToken->GetDeletedBeginTime(), + continueStmtToken->GetDeletedEndTime()); + errCode = handle->GetDeletedSyncDataByTimestamp(dataItems, GetAppendedLen(), + continueStmtToken->GetDeletedBeginTime(), continueStmtToken->GetDeletedEndTime(), dataSizeInfo); + } + } + + if (errCode == -E_FINISHED) { + errCode = E_OK; + } + +ERROR: + if (errCode != -E_UNFINISHED && errCode != E_OK) { // Error happened. + dataItems.clear(); + } + ProcessContinueTokenForQuerySync(dataItems, errCode, continueStmtToken); + ReleaseHandle(handle); + return errCode; +} + int SQLiteSingleVerNaturalStore::GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const { @@ -654,30 +863,23 @@ int SQLiteSingleVerNaturalStore::GetSyncDataNext(std::vector } std::vector dataItems; - errCode = GetSyncDataNext(dataItems, continueStmtToken, dataSizeInfo); - if (errCode != E_OK && errCode != -E_UNFINISHED) { - LOGE("GetSyncDataNext errCode:%d", errCode); - goto ERROR; + auto token = static_cast(continueStmtToken); + if (token->IsQuerySync()) { + errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); + continueStmtToken = static_cast(token); + } else { + errCode = GetSyncDataNext(dataItems, continueStmtToken, dataSizeInfo); } - for (auto &item : dataItems) { - GenericSingleVerKvEntry *entry = new (std::nothrow) GenericSingleVerKvEntry(); - if (entry == nullptr) { - errCode = -E_OUT_OF_MEMORY; - LOGE("GetSyncDataNext errCode:%d", errCode); - goto ERROR; - } - entry->SetEntryData(std::move(item)); - entries.push_back(entry); + if (errCode != E_OK && errCode != -E_UNFINISHED) { + LOGE("GetSyncDataNext errCode:%d", errCode); + return errCode; } -ERROR: - if (errCode != E_OK && errCode != -E_UNFINISHED) { - for (auto &itemEntry : entries) { - delete itemEntry; - itemEntry = nullptr; - } - entries.clear(); + int innerErrCode = GetKvEntriesByDataItems(entries, dataItems); + if (innerErrCode != E_OK) { + errCode = innerErrCode; + ReleaseContinueToken(continueStmtToken); } return errCode; } @@ -685,11 +887,11 @@ ERROR: int SQLiteSingleVerNaturalStore::GetSyncDataNext(std::vector &dataItems, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const { - if (dataSizeInfo.blockSize > MAX_SYNC_BLOCK_SIZE) { + if (dataSizeInfo.blockSize > DBConstant::MAX_SYNC_BLOCK_SIZE) { return -E_INVALID_ARGS; } - auto token = static_cast(continueStmtToken); + auto token = static_cast(continueStmtToken); if (token == nullptr || !(token->CheckValid())) { LOGE("[SingleVerNaturalStore][GetSyncDataNext] invalid continue token."); return -E_INVALID_ARGS; @@ -702,35 +904,22 @@ int SQLiteSingleVerNaturalStore::GetSyncDataNext(std::vector &dataItem return errCode; } - errCode = handle->GetSyncDataByTimestamp(dataItems, GetAppendedLen(), token->GetBeginTimeStamp(), - token->GetEndTimeStamp(), dataSizeInfo); - if (errCode == -E_UNFINISHED) { - // update the begin timestamp of the next fetch. - if (!dataItems.empty()) { - TimeStamp timestamp = dataItems.back().timeStamp; - if (timestamp >= INT64_MAX) { - token->SetBeginTimeStamp(INT64_MAX); - } else { - token->SetBeginTimeStamp(timestamp + 1); // Add 1 for the next fetch. - } - continueStmtToken = static_cast(token); - } else { - ReleaseContinueToken(continueStmtToken); - errCode = -E_INTERNAL_ERROR; - } - } else { - ReleaseContinueToken(continueStmtToken); - if (errCode == -E_FINISHED) { - errCode = E_OK; - } + errCode = handle->GetSyncDataByTimestamp(dataItems, GetAppendedLen(), token->GetQueryBeginTime(), + token->GetQueryEndTime(), dataSizeInfo); + if (errCode == -E_FINISHED) { + errCode = E_OK; } + + ProcessContinueToken(dataItems, errCode, token); + continueStmtToken = static_cast(token); + ReleaseHandle(handle); return errCode; } void SQLiteSingleVerNaturalStore::ReleaseContinueToken(ContinueToken &continueStmtToken) const { - auto token = static_cast(continueStmtToken); + auto token = static_cast(continueStmtToken); if (token == nullptr || !(token->CheckValid())) { LOGE("[SQLiteSingleVerNaturalStore][ReleaseContinueToken] Input is not a continue token."); return; @@ -820,7 +1009,7 @@ int SQLiteSingleVerNaturalStore::RemoveDeviceData(const std::string &deviceName, return -E_NOT_PERMIT; } // Call the syncer module to erase the water mark. - int errCode = EraseDeviceWaterMark(deviceName); + int errCode = EraseDeviceWaterMark(deviceName, true); if (errCode != E_OK) { LOGE("[SingleVerNStore] erase water mark failed:%d", errCode); return errCode; @@ -993,6 +1182,7 @@ int SQLiteSingleVerNaturalStore::RegisterNotification() void SQLiteSingleVerNaturalStore::ReleaseResources() { + SyncAbleKvDB::Close(); if (notificationEventsRegistered_) { UnRegisterNotificationEventType(static_cast(SQLITE_GENERAL_NS_SYNC_EVENT)); UnRegisterNotificationEventType(static_cast(SQLITE_GENERAL_NS_PUT_EVENT)); @@ -1282,6 +1472,14 @@ int SQLiteSingleVerNaturalStore::Import(const std::string &filePath, const Ciphe operation = std::make_unique(this, storageEngine_); operation->SetLocalDevId(localDev); errCode = operation->Import(filePath, passwd); + if (errCode != E_OK) { + goto END; + } + + // Save create db time. + storageEngine_->Enable(OperatePerm::IMPORT_MONOPOLIZE_PERM); + errCode = SaveCreateDBTime(); + END: // restore the storage engine and the syncer. storageEngine_->Enable(OperatePerm::IMPORT_MONOPOLIZE_PERM); @@ -1365,8 +1563,7 @@ void SQLiteSingleVerNaturalStore::InitDataBaseOption(const KvDBProperties &kvDBP securityOpt.securityFlag = kvDBProp.GetSecFlag(); } - option = {uri, isCreateNecessary, isMemoryDb, createTableSqls, cipherType, passwd, schemaStr, - subDir, securityOpt}; + option = {uri, isCreateNecessary, isMemoryDb, createTableSqls, cipherType, passwd, schemaStr, subDir, securityOpt}; option.conflictReslovePolicy = kvDBProp.GetIntProp(KvDBProperties::CONFLICT_RESOLVE_POLICY, DEFAULT_LAST_WIN); option.createDirByStoreIdOnly = kvDBProp.GetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, false); } @@ -1531,7 +1728,8 @@ int SQLiteSingleVerNaturalStore::GetKvDBSize(const KvDBProperties &properties, u std::string storeOnlyIdentDir; std::string storeIdentDir; GenericKvDB::GetStoreDirectory(properties, KvDBProperties::SINGLE_VER_TYPE, storeIdentDir, storeOnlyIdentDir); - std::vector> dbDir {{DBConstant::MAINDB_DIR, DBConstant::SINGLE_VER_DATA_STORE}, + std::vector> dbDir { + {DBConstant::MAINDB_DIR, DBConstant::SINGLE_VER_DATA_STORE}, {DBConstant::METADB_DIR, DBConstant::SINGLE_VER_META_STORE}, {DBConstant::CACHEDB_DIR, DBConstant::SINGLE_VER_CACHE_STORE}}; int errCode = -E_NOT_FOUND; @@ -1631,13 +1829,11 @@ int SQLiteSingleVerNaturalStore::StopLifeCycleTimer() const bool SQLiteSingleVerNaturalStore::IsDataMigrating() const { - if (storageEngine_ != nullptr) { - EngineState state = storageEngine_->GetEngineState(); - if (state == EngineState::MIGRATING || state == EngineState::ENGINE_BUSY) { - return true; - } + if (storageEngine_ == nullptr) { + return false; } - return false; + EngineState state = storageEngine_->GetEngineState(); + return (state == EngineState::MIGRATING || state == EngineState::ENGINE_BUSY); } void SQLiteSingleVerNaturalStore::SetConnectionFlag(bool isExisted) const @@ -1807,5 +2003,130 @@ void SQLiteSingleVerNaturalStore::NotifyRemotePushFinished(const std::string &ta NotifyRemotePushFinishedInner(targetId); } +int SQLiteSingleVerNaturalStore::GetDatabaseCreateTimeStamp(TimeStamp &outTime) const +{ + // Found in memory. + { + std::lock_guard autoLock(createDBTimeMutex_); + if (createDBTime_ != 0) { + outTime = createDBTime_; + return E_OK; + } + } + + const Key key(CREATE_DB_TIME.begin(), CREATE_DB_TIME.end()); + Value value; + int errCode = GetMetaData(key, value); + if (errCode != E_OK) { + LOGD("GetDatabaseCreateTimeStamp failed, errCode = %d.", errCode); + return errCode; + } + + TimeStamp createDBTime = 0; + Parcel parcel(value.data(), value.size()); + (void)parcel.ReadUInt64(createDBTime); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + outTime = createDBTime; + std::lock_guard autoLock(createDBTimeMutex_); + createDBTime_ = createDBTime; + return E_OK; +} + +int SQLiteSingleVerNaturalStore::CheckIntegrity() const +{ + int errCode = E_OK; + auto handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->CheckIntegrity(); + ReleaseHandle(handle); + return errCode; +} + +int SQLiteSingleVerNaturalStore::SaveCreateDBTime() +{ + TimeStamp createDBTime = GetCurrentTimeStamp(); + const Key key(CREATE_DB_TIME.begin(), CREATE_DB_TIME.end()); + Value value(Parcel::GetUInt64Len()); + Parcel parcel(value.data(), Parcel::GetUInt64Len()); + (void)parcel.WriteUInt64(createDBTime); + if (parcel.IsError()) { + LOGE("SaveCreateDBTime failed, something wrong in parcel."); + return -E_PARSE_FAIL; + } + + int errCode = PutMetaData(key, value); + if (errCode != E_OK) { + LOGE("SaveCreateDBTime failed, errCode = %d", errCode); + return errCode; + } + + // save in memory. + std::lock_guard autoLock(createDBTimeMutex_); + createDBTime_ = createDBTime; + return errCode; +} + +int SQLiteSingleVerNaturalStore::SaveCreateDBTimeIfNotExisted() +{ + TimeStamp createDBTime = 0; + int errCode = GetDatabaseCreateTimeStamp(createDBTime); + if (errCode == -E_NOT_FOUND) { + errCode = SaveCreateDBTime(); + } + if (errCode != E_OK) { + LOGE("SaveCreateDBTimeIfNotExisted failed, errCode=%d.", errCode); + } + return errCode; +} + +int SQLiteSingleVerNaturalStore::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + if (keyPrefix.empty() || keyPrefix.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; + } + + int errCode = E_OK; + auto handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->DeleteMetaDataByPrefixKey(keyPrefix); + if (errCode != E_OK) { + LOGE("[SinStore] DeleteMetaData by prefix key failed, errCode = %d", errCode); + } + + ReleaseHandle(handle); + HeartBeatForLifeCycle(); + return errCode; +} + +int SQLiteSingleVerNaturalStore::GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const +{ +#ifdef USING_DATA_COMPRESS + needCompressOnSync = GetDbProperties().GetBoolProp(KvDBProperties::COMPRESS_ON_SYNC, false); + compressionRate = GetDbProperties().GetIntProp(KvDBProperties::COMPRESSION_RATE, + DBConstant::DEFAULT_COMPTRESS_RATE); +#else + needCompressOnSync = false; + compressionRate = 100; // no need compress means the ratio is 100%. +#endif + return E_OK; +} + +int SQLiteSingleVerNaturalStore::GetCompressionAlgo(std::set &algorithmSet) const +{ + algorithmSet.clear(); +#ifdef USING_DATA_COMPRESS + algorithmSet.insert(CompressAlgorithm::ZLIB); +#endif + return E_OK; +} + DEFINE_OBJECT_TAG_FACILITIES(SQLiteSingleVerNaturalStore) } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h index 71b488d5b..0cc778eba 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h @@ -24,44 +24,9 @@ #include "single_ver_kvdb_sync_interface.h" #include "kv_store_nb_conflict_data_impl.h" #include "runtime_context.h" +#include "sqlite_single_ver_continue_token.h" namespace DistributedDB { -struct ContinueTokenStruct { - /* - * function: Check the magic number at the beginning and end of the ContinueTokenStruct. - * returnValue: Return true if the begin and end magic number is OK. - * Return false if the begin or end magic number is error. - */ - bool CheckValid() const - { - return ((magicBegin_ == MAGIC_BEGIN) && (magicEnd_ == MAGIC_END)); - } - TimeStamp GetBeginTimeStamp() const - { - return begin_; - } - void SetBeginTimeStamp(TimeStamp begin) - { - begin_ = begin; - } - TimeStamp GetEndTimeStamp() const - { - return end_; - } - void SetEndTimeStamp(TimeStamp end) - { - end_ = end; - } - -private: - static const unsigned int MAGIC_BEGIN = 0x600D0AC7; // for token guard - static const unsigned int MAGIC_END = 0x0AC7600D; // for token guard - unsigned int magicBegin_ = MAGIC_BEGIN; - TimeStamp begin_ = 0; - TimeStamp end_ = 0; - unsigned int magicEnd_ = MAGIC_END; -}; - class SQLiteSingleVerNaturalStore : public SyncAbleKvDB, public SingleVerKvDBSyncInterface { public: SQLiteSingleVerNaturalStore(); @@ -98,6 +63,11 @@ public: int PutMetaData(const Key &key, const Value &value) override; + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys) override; + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + int GetAllMetaKeys(std::vector &keys) const override; int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &dataItems, ContinueToken &continueStmtToken, @@ -106,6 +76,9 @@ public: int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; + int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, const DataSizeSpecInfo &dataSizeInfo, + ContinueToken &continueStmtToken, std::vector &entries) const override; + int GetSyncDataNext(std::vector &dataItems, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; @@ -193,6 +166,13 @@ public: void NotifyRemotePushFinished(const std::string &targetId) const override; + int GetDatabaseCreateTimeStamp(TimeStamp &outTime) const override; + + int CheckIntegrity() const override; + + int GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const override; + int GetCompressionAlgo(std::set &algorithmSet) const override; + private: int CheckDatabaseRecovery(const KvDBProperties &kvDBProp); @@ -237,6 +217,7 @@ private: void InitConflictNotifiedFlag(SingleVerNaturalStoreCommitNotifyData *committedData); void AsyncDataMigration() const; + // Change value that should be amended, and neglect value that is incompatible void CheckAmendValueContentForSyncProcedure(std::vector &dataItems) const; @@ -253,6 +234,13 @@ private: int ClearIncompleteDatabase(const KvDBProperties &kvDBPro) const; + int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerContinueToken *&continueStmtToken, + const DataSizeSpecInfo &dataSizeInfo) const; + + int SaveCreateDBTime(); + int SaveCreateDBTimeIfNotExisted(); + + int GetStorageEngine(const KvDBProperties &kvDBProp); DECLARE_OBJECT_TAG(SQLiteSingleVerNaturalStore); TimeStamp currentMaxTimeStamp_ = 0; @@ -268,6 +256,8 @@ private: mutable DatabaseLifeCycleNotifier lifeCycleNotifier_; mutable TimerId lifeTimerId_; uint32_t autoLifeTime_; + mutable TimeStamp createDBTime_; + mutable std::mutex createDBTimeMutex_; }; } #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp index cf37553e4..6f4ec0966 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp @@ -708,6 +708,20 @@ int SQLiteSingleVerNaturalStoreConnection::PreClose() return E_OK; } +int SQLiteSingleVerNaturalStoreConnection::CheckIntegrity() const +{ + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetExecutor(true, errCode); + if (handle == nullptr) { + LOGW("Failed to get the executor for the integrity check."); + return errCode; + } + + errCode = handle->CheckIntegrity(); + ReleaseExecutor(handle); + return errCode; +} + int SQLiteSingleVerNaturalStoreConnection::CheckMonoStatus(OperatePerm perm) { // 1. Get the connection number diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h index 8a619ff69..01acfd6a4 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h @@ -50,6 +50,7 @@ public: int GetEntries(const IOption &option, const Query &query, std::vector &entries) const override; int GetCount(const IOption &option, const Query &query, int &count) const override; + // Put the batch values to the database. int PutBatch(const IOption &option, const std::vector &entries) override; @@ -102,6 +103,7 @@ public: // Called when Close and delete the connection. int PreClose() override; + int CheckIntegrity() const override; private: int CheckMonoStatus(OperatePerm perm); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp index 61bfe157b..d75223044 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp @@ -311,7 +311,7 @@ int SQLiteSingleVerStorageEngine::ReleaseExecutor(SQLiteSingleVerStorageExecutor int SQLiteSingleVerStorageEngine::FinishMigrateData(SQLiteSingleVerStorageExecutor *&handle, EngineState stateBeforeMigrate) { - LOGD("Begin to finish migrate and reinit db state!"); + LOGI("Begin to finish migrate and reinit db state!"); int errCode; if (handle == nullptr) { return -E_INVALID_ARGS; @@ -483,7 +483,7 @@ bool SQLiteSingleVerStorageEngine::IsEngineCorrupted() const StorageExecutor *SQLiteSingleVerStorageEngine::NewSQLiteStorageExecutor(sqlite3 *dbHandle, bool isWrite, bool isMemDb) { - auto executor = new (std::nothrow) SQLiteSingleVerStorageExecutor(dbHandle, isWrite, isMemDb); + auto executor = new (std::nothrow) SQLiteSingleVerStorageExecutor(dbHandle, isWrite, isMemDb, executorState_); if (executor == nullptr) { return executor; } @@ -491,7 +491,7 @@ StorageExecutor *SQLiteSingleVerStorageEngine::NewSQLiteStorageExecutor(sqlite3 return executor; } -int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(sqlite3 *&db) +int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(bool isWrite, sqlite3 *&db) { // Only could get the main database handle in the uninitialized and the main status. if (GetEngineState() != EngineState::INVALID && GetEngineState() != EngineState::MAINDB) { @@ -504,7 +504,12 @@ int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(sqlite3 *&db) DBConstant::SQLITE_DB_EXTENSION; } - int errCode = SQLiteUtils::OpenDatabase(option_, db); + OpenDbProperties optionTemp = option_; + if (!isWrite) { + optionTemp.createIfNecessary = false; + } + + int errCode = SQLiteUtils::OpenDatabase(optionTemp, db); if (errCode != E_OK) { if (errno == EKEYREVOKED) { LOGI("Failed to open the main database for key revoked[%d]", errCode); @@ -535,9 +540,9 @@ int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(sqlite3 *&db) int SQLiteSingleVerStorageEngine::GetDbHandle(bool isWrite, const SecurityOption &secOpt, sqlite3 *&dbHandle) { - int errCode = TryToOpenMainDatabase(dbHandle); - LOGD("Finish to open the main database, write[%d], label[%d], flag[%d], errCode[%d]", - isWrite, secOpt.securityLabel, secOpt.securityFlag, errCode); + int errCode = TryToOpenMainDatabase(isWrite, dbHandle); + LOGD("Finish to open the main database, write[%d], label[%d], flag[%d], id[%.6s], errCode[%d]", isWrite, + secOpt.securityLabel, secOpt.securityFlag, DBCommon::TransferStringToHex(identifier_).c_str(), errCode); if (!(ParamCheckUtils::IsS3SECEOpt(secOpt) && errCode == -E_EKEYREVOKED)) { return errCode; } @@ -546,7 +551,7 @@ int SQLiteSingleVerStorageEngine::GetDbHandle(bool isWrite, const SecurityOption DBConstant::SQLITE_DB_EXTENSION; if (!isWrite || GetEngineState() != EngineState::INVALID || OS::CheckPathExistence(cacheDbPath)) { - LOGI("[SQLiteSingleStorageEng][GetDbHandle]Only use for first create cache db! [%d] [%d]", + LOGI("[SQLiteSingleStorageEng][GetDbHandle] Only use for first create cache db! [%d] [%d]", isWrite, GetEngineState()); return -E_EKEYREVOKED; } @@ -1020,8 +1025,8 @@ void SQLiteSingleVerStorageEngine::SetMaxTimeStamp(TimeStamp maxTimeStamp) const void SQLiteSingleVerStorageEngine::CommitNotifyForMigrateCache(NotifyMigrateSyncData &syncData) const { - auto &isRemote = syncData.isRemote; - auto &isRemoveDeviceData = syncData.isRemoveDeviceData; + const auto &isRemote = syncData.isRemote; + const auto &isRemoveDeviceData = syncData.isRemoveDeviceData; auto &committedData = syncData.committedData; auto &entries = syncData.entries; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.h index 87a7bbdaa..af8e85fe3 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.h @@ -87,7 +87,7 @@ private: int EraseDeviceWaterMark(SQLiteSingleVerStorageExecutor *&handle, const std::vector &dataItems); // For db. - int TryToOpenMainDatabase(sqlite3 *&db); + int TryToOpenMainDatabase(bool isWrite, sqlite3 *&db); int GetCacheDbHandle(sqlite3 *&db); int GetDbHandle(bool isWrite, const SecurityOption &secOpt, sqlite3 *&dbHandle); int AttachMetaDatabase(sqlite3 *dbHandle, const OpenDbProperties &option) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 892bd7edf..b22c91817 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -283,7 +283,11 @@ int SQLiteSingleVerStorageExecutor::PutKvData(SingleVerDataType type, const Key if (isLocal && committedData != nullptr) { Entry entry = {key, value}; - committedData->InsertCommittedData(std::move(entry), (isExisted ? DataType::UPDATE : DataType::INSERT), true); + if (isExisted) { + committedData->InsertCommittedData(std::move(entry), DataType::UPDATE, true); + } else { + committedData->InsertCommittedData(std::move(entry), DataType::INSERT, true); + } } return E_OK; } @@ -403,12 +407,14 @@ ERROR: } int SQLiteSingleVerStorageExecutor::PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, - sqlite3_stmt *&statement) const + sqlite3_stmt *&statement, bool getDeletedData) const { if (dbHandle_ == nullptr) { return -E_INVALID_DB; } - int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_SYNC_ENTRIES_SQL, statement); + + const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); if (errCode != E_OK) { LOGE("Prepare the sync entries statement error:%d", errCode); return errCode; @@ -444,17 +450,12 @@ void SQLiteSingleVerStorageExecutor::ReleaseContinueStatement() } } -int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector &dataItems, size_t appenedLength, - TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const +int SQLiteSingleVerStorageExecutor::GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, + size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const { + int errCode; size_t dataTotalSize = 0; - sqlite3_stmt *statement = nullptr; - int errCode = PrepareForSyncDataByTime(begin, end, statement); - if (errCode != E_OK) { - return errCode; - } - dataItems.clear(); do { DataItem dataItem; errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); @@ -462,7 +463,7 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector errCode = GetDataItemForSync(statement, dataItem); } else { if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGD("Get sync data finished, number: %zu, size: %zu", dataTotalSize, dataItems.size()); + LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); errCode = -E_FINISHED; } else { LOGE("Get sync data error:%d", errCode); @@ -471,7 +472,7 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector } // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. - dataTotalSize += GetDataItemSerialSize(dataItem, appenedLength); + dataTotalSize += GetDataItemSerialSize(dataItem, appendLength); if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || dataItems.size() >= dataSizeInfo.packetSize) { errCode = -E_UNFINISHED; @@ -480,6 +481,45 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector dataItems.push_back(std::move(dataItem)); } } while (true); + return errCode; +} + +int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, + TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = PrepareForSyncDataByTime(begin, end, statement); + if (errCode != E_OK) { + return errCode; + } + + errCode = GetSyncDataItems(dataItems, statement, appendLength, dataSizeInfo); + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerStorageExecutor::GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, + TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = PrepareForSyncDataByTime(begin, end, statement, true); + if (errCode != E_OK) { + return errCode; + } + + errCode = GetSyncDataItems(dataItems, statement, appendLength, dataSizeInfo); + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerStorageExecutor::GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, + QueryObject query, const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = query.GetQuerySyncStatement(dbHandle_, timeRange.first, timeRange.second, statement); + if (errCode == E_OK) { + errCode = GetSyncDataItems(dataItems, statement, appendLength, dataSizeInfo); + } SQLiteUtils::ResetStatement(statement, true, errCode); return CheckCorruptedStatus(errCode); @@ -1903,4 +1943,13 @@ void SQLiteSingleVerStorageExecutor::SetConflictResolvePolicy(int policy) conflictResolvePolicy_ = policy; } } + +int SQLiteSingleVerStorageExecutor::CheckIntegrity() const +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + + return SQLiteUtils::CheckIntegrity(dbHandle_, CHECK_DB_INTEGRITY_SQL); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h index a1ea09068..cc2d91030 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h @@ -137,8 +137,13 @@ public: void ReleaseContinueStatement(); - int GetSyncDataByTimestamp(std::vector &dataItems, size_t appenedLength, TimeStamp begin, + int GetSyncDataByTimestamp(std::vector &dataItems, size_t appendedLength, TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; + int GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendedLength, TimeStamp begin, + TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; + + int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, QueryObject query, + const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const; int GetDeviceIdentifier(PragmaEntryDeviceIdentifier *identifier); @@ -212,6 +217,15 @@ public: void SetConflictResolvePolicy(int policy); + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys); + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix); + + int CheckIntegrity() const; + + static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen); + private: struct SaveRecordStatements { sqlite3_stmt *queryStatement = nullptr; @@ -229,8 +243,6 @@ private: static int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem); - static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen); - static void PutConflictData(const DataItem &itemPut, const DataItem &itemGet, const DeviceInfo &deviceInfo, const DataOperStatus &dataStatus, SingleVerNaturalStoreCommitNotifyData *commitData); @@ -245,7 +257,8 @@ private: int GetSyncDataPreByHashKey(const Key &hashKey, DataItem &itemGet) const; - int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement) const; + int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData = false) + const; int StepForResultEntries(sqlite3_stmt *statement, std::vector &entries) const; @@ -308,43 +321,55 @@ private: // Process timestamp for syncdata in cacheDB when migrating. int ProcessTimeStampForSyncDataInCacheDB(std::vector &dataItems); + // Get migrateTimeOffset_. int InitMigrateTimeStampOffset(); + // Get min timestamp of local data in sync_data, cacheDB. int GetMinTimestampInCacheDB(TimeStamp &minStamp) const; // Prepare conflict notify and commit notify data. int PrepareForNotifyConflictAndObserver(DataItem &dataItem, const DeviceInfo &deviceInfo, NotifyConflictAndObserverData ¬ify); + // Put observer and conflict data into commit notify when migrating cacheDB. int PutIntoConflictAndCommitForMigrateCache(DataItem &dataItem, const DeviceInfo &deviceInfo, NotifyConflictAndObserverData ¬ify); + // Notify when migrating cacheDB. int NotifyForMigrateCacheDB(std::vector &dataItems, NotifyMigrateSyncData &syncData); int GetEntriesForNotifyRemoveDevData(const DataItem &item, std::vector &entries) const; // Reset migrateSyncStatements_. int ResetForMigrateCacheData(); + // Init migrating data. int InitMigrateData(); int MigrateRmDevData(const DataItem &dataItem) const; int VacuumLocalData() const; + int GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, + size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const; + sqlite3_stmt *getSyncStatement_; sqlite3_stmt *getResultRowIdStatement_; sqlite3_stmt *getResultEntryStatement_; SaveRecordStatements saveSyncStatements_; SaveRecordStatements saveLocalStatements_; + // Used for migrating sync_data. SaveRecordStatements migrateSyncStatements_; bool isTransactionOpen_; bool attachMetaMode_; // true for attach meta mode ExecutorState executorState_; + // Max timestamp in mainDB. Used for migrating. TimeStamp maxTimeStampInMainDB_; + // The offset between min timestamp in cacheDB and max timestamp in mainDB. Used for migrating. TimeOffset migrateTimeOffset_; + // Migrating sync flag. When the flag is true, mainDB and cacheDB are attached, migrateSyncStatements_ is set, // maxTimeStampInMainDB_ and migrateTimeOffset_ is meaningful. bool isSyncMigrating_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp index ea9a1a1d3..b3354a3cf 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp @@ -871,4 +871,54 @@ int SQLiteSingleVerStorageExecutor::GetMaxTimeStampDuringMigrating(TimeStamp &ma maxTimeStamp = maxTimeStampInMainDB_; return E_OK; } + +int SQLiteSingleVerStorageExecutor::DeleteMetaData(const std::vector &keys) +{ + sqlite3_stmt *statement = nullptr; + const std::string sql = attachMetaMode_ ? REMOVE_ATTACH_META_VALUE_SQL : REMOVE_META_VALUE_SQL; + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); + if (errCode != E_OK) { + return errCode; + } + + for (const auto &key : keys) { + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // first arg. + if (errCode != E_OK) { + break; + } + + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + break; + } + errCode = E_OK; + SQLiteUtils::ResetStatement(statement, false, errCode); + } + + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) +{ + sqlite3_stmt *statement = nullptr; + const std::string sql = attachMetaMode_ ? + REMOVE_ATTACH_META_VALUE_BY_KEY_PREFIX_SQL : REMOVE_META_VALUE_BY_KEY_PREFIX_SQL; + + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::BindPrefixKey(statement, 1, keyPrefix); // 1 is first arg. + if (errCode == E_OK) { + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + } + + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_sql.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_sql.h index ff7b927a0..b88779641 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_sql.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_sql.h @@ -103,6 +103,9 @@ namespace DistributedDB { const std::string SELECT_SYNC_ENTRIES_SQL = "SELECT * FROM sync_data WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; + const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = + "SELECT * FROM sync_data WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; + const std::string SELECT_SYNC_PREFIX_SQL = "SELECT key, value FROM sync_data WHERE key>=? AND key<=? AND (flag&0x01=0) ORDER BY key ASC;"; @@ -160,6 +163,18 @@ namespace DistributedDB { const std::string SELECT_MAIN_SYNC_HASH_SQL_FROM_CACHEHANDLE = "SELECT * FROM maindb.sync_data WHERE hash_key=?;"; + const std::string REMOVE_META_VALUE_SQL = + "DELETE FROM meta_data WHERE key=?;"; + const std::string REMOVE_ATTACH_META_VALUE_SQL = + "DELETE FROM meta.meta_data WHERE key=?;"; + + const std::string CHECK_DB_INTEGRITY_SQL = "PRAGMA integrity_check;"; + + const std::string REMOVE_META_VALUE_BY_KEY_PREFIX_SQL = + "DELETE FROM meta_data WHERE key>=? AND key<=?;"; + const std::string REMOVE_ATTACH_META_VALUE_BY_KEY_PREFIX_SQL = + "DELETE FROM meta.meta_data WHERE key>=? AND key<=?;"; + const int BIND_KV_KEY_INDEX = 1; const int BIND_KV_VAL_INDEX = 2; const int BIND_LOCAL_TIMESTAMP_INDEX = 3; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp index b3ce9d9ff..009400728 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp @@ -102,7 +102,7 @@ int SQLiteStorageEngine::ReInit() bool SQLiteStorageEngine::IsNeedTobeReleased() const { EngineState engineState = GetEngineState(); - return ((engineState == MAINDB) || (engineState == INVALID)); + return ((engineState == EngineState::MAINDB) || (engineState == EngineState::INVALID)); } const std::string &SQLiteStorageEngine::GetIdentifier() const @@ -168,7 +168,7 @@ int SQLiteStorageEngine::CheckEngineOption(const KvDBProperties &kvDBProp) const securityOpt.securityLabel = kvDBProp.GetSecLabel(); securityOpt.securityFlag = kvDBProp.GetSecFlag(); } - std::string schemaStr = kvDBProp.GetSchema().ToSchemaString(); + int conflictReslovePolicy = kvDBProp.GetIntProp(KvDBProperties::CONFLICT_RESOLVE_POLICY, DEFAULT_LAST_WIN); bool createDirByStoreIdOnly = kvDBProp.GetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, false); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index e6a5aaf49..73c7f596d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -385,12 +385,17 @@ int SQLiteUtils::ExecuteRawSQL(sqlite3 *db, const std::string &sql) return SQLiteUtils::MapSQLiteErrno(errCode); } -int SQLiteUtils::SetKey(sqlite3 *db, CipherType type, const CipherPassword &passwd) +int SQLiteUtils::SetKey(sqlite3 *db, CipherType type, const CipherPassword &passwd, const bool &isMemDb) { if (db == nullptr) { return -E_INVALID_DB; } + // in memory mode no need cipher + if (isMemDb) { + return E_OK; + } + if (passwd.GetSize() != 0) { #ifndef OMIT_ENCRYPT int errCode = sqlite3_key(db, static_cast(passwd.GetData()), static_cast(passwd.GetSize())); @@ -416,6 +421,9 @@ int SQLiteUtils::SetKey(sqlite3 *db, CipherType type, const CipherPassword &pass if (errno == EKEYREVOKED) { return -E_EKEYREVOKED; } + if (errCode == -E_BUSY) { + return errCode; + } return -E_INVALID_PASSWD_OR_CORRUPTED_DB; } return E_OK; @@ -500,6 +508,45 @@ int SQLiteUtils::CreateMetaDatabase(const std::string &metaDbPath) return errCode; } +int SQLiteUtils::CheckIntegrity(sqlite3 *db, const std::string &sql) +{ + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + LOGE("Prepare the integrity check statement error:%d", errCode); + return errCode; + } + int resultCnt = 0; + bool checkResultOK = false; + do { + errCode = SQLiteUtils::StepWithRetry(statement); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + break; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + auto result = reinterpret_cast(sqlite3_column_text(statement, 0)); + if (result == nullptr) { + continue; + } + resultCnt = (resultCnt > 1) ? resultCnt : (resultCnt + 1); + if (strcmp(result, "ok") == 0) { + checkResultOK = true; + } + } else { + checkResultOK = false; + LOGW("Step for the integrity check failed:%d", errCode); + break; + } + } while (true); + if (resultCnt == 1 && checkResultOK) { + errCode = E_OK; + } else { + errCode = -E_INVALID_PASSWD_OR_CORRUPTED_DB; + } + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + + #ifndef OMIT_ENCRYPT int SQLiteUtils::ExportDatabase(sqlite3 *db, CipherType type, const CipherPassword &passwd, const std::string &newDbName) @@ -570,7 +617,8 @@ int SQLiteUtils::GetVersion(const OpenDbProperties &properties, int &version) LOGE("Open database failed: %d, sys:%d", errCode, errno); goto END; } - errCode = SQLiteUtils::SetKey(dbTemp, properties.cipherType, properties.passwd); + + errCode = SQLiteUtils::SetKey(dbTemp, properties.cipherType, properties.passwd, properties.isMemDb); if (errCode != E_OK) { LOGE("Set key failed: %d", errCode); goto END; @@ -1254,7 +1302,7 @@ int SQLiteUtils::GetDbSize(const std::string &dir, const std::string &dbName, ui uint64_t localDbSize = 0; int errCode = OS::CalFileSize(dataDir, localDbSize); if (errCode < 0) { - LOGE("Failed to get the db file size, errno:%d", errno); + LOGD("Failed to get the db file size, errno:%d", errno); if (errno == ENOENT) { return -E_NOT_FOUND; } @@ -1341,7 +1389,8 @@ int SQLiteUtils::SetDataBaseProperty(sqlite3 *db, const OpenDbProperties &proper if (errCode != E_OK) { return errCode; } - errCode = SQLiteUtils::SetKey(db, properties.cipherType, properties.passwd); + + errCode = SQLiteUtils::SetKey(db, properties.cipherType, properties.passwd, properties.isMemDb); if (errCode != E_OK) { LOGD("SQLiteUtils::SetKey fail!!![%d]", errCode); return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 0ec724843..7f4f9cb2d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -79,7 +79,7 @@ public: static int ExecuteRawSQL(sqlite3 *db, const std::string &sql); - static int SetKey(sqlite3 *db, CipherType type, const CipherPassword &passwd); + static int SetKey(sqlite3 *db, CipherType type, const CipherPassword &passwd, const bool &isMemDb); static int GetColumnBlobValue(sqlite3_stmt *statement, int index, std::vector &value); @@ -128,6 +128,8 @@ public: const std::string &attachDbAbsPath, const std::string &attachAsName = "backup"); static int CreateMetaDatabase(const std::string &metaDbPath); + + static int CheckIntegrity(sqlite3 *db, const std::string &sql); private: static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.cpp index 3939e4ced..992069d49 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.cpp @@ -360,6 +360,7 @@ void StorageEngine::AddStorageExecutor(StorageExecutor *handle) } else { readIdleList_.push_back(handle); } + LOGD("add [%d] storage executor to handle pool.", handle->GetWritable()); } void StorageEngine::CloseExecutor() @@ -409,9 +410,8 @@ StorageExecutor *StorageEngine::FetchStorageExecutor(bool isWrite, std::listCheckEngineOption(property); if (errCode != E_OK) { LOGE("kvdb property mismatch engine option! errCode = [%d]", errCode); - return nullptr; + storageEngine = nullptr; } } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp index 3ae8676f7..8659cd04e 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp @@ -67,9 +67,7 @@ void SyncAbleKvDB::Close() } // Start a sync action. -int SyncAbleKvDB::Sync(const std::vector &devices, int mode, - const std::function &)> &onComplete, - const std::function &onFinalize, bool wait = false) +int SyncAbleKvDB::Sync(const ISyncer::SyncParma &parma) { if (!started_) { StartSyncer(); @@ -77,7 +75,7 @@ int SyncAbleKvDB::Sync(const std::vector &devices, int mode, return -E_NOT_INIT; } } - return syncer_.Sync(devices, mode, onComplete, onFinalize, wait); + return syncer_.Sync(parma); } void SyncAbleKvDB::EnableAutoSync(bool enable) @@ -232,4 +230,14 @@ void SyncAbleKvDB::NotifyRemotePushFinishedInner(const std::string &targetId) co remotePushNotifyChain_->NotifyEvent(REMOTE_PUSH_FINISHED, static_cast(&info)); } } + +int SyncAbleKvDB::SetSyncRetry(bool isRetry) +{ + return syncer_.SetSyncRetry(isRetry); +} + +int SyncAbleKvDB::SetEqualIdentifier(const std::string &identifier, const std::vector &targets) +{ + return syncer_.SetEqualIdentifier(identifier, targets); +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h index 1b8116245..81aa8a4a4 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h @@ -38,9 +38,7 @@ public: void Close() override; // Start a sync action. - int Sync(const std::vector &devices, int mode, - const std::function &)> &onComplete, - const std::function &onFinalize, bool wait); + int Sync(const ISyncer::SyncParma &parma); // Enable auto sync void EnableAutoSync(bool enable); @@ -70,12 +68,16 @@ public: int SetStaleDataWipePolicy(WipePolicy policy); - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash = true); + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash); NotificationChain::Listener *AddRemotePushFinishedNotify(const RemotePushFinishedNotifier ¬ifier, int errCode); void NotifyRemotePushFinishedInner(const std::string &targetId) const; + int SetSyncRetry(bool isRetry); + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const std::string &identifier, const std::vector &targets); + protected: virtual IKvDBSyncInterface *GetSyncInterface() = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp index 94ac9f826..a3392bbca 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp @@ -51,52 +51,68 @@ SyncAbleKvDBConnection::~SyncAbleKvDBConnection() remotePushFinishedListener_ = nullptr; } +void SyncAbleKvDBConnection::InitPragmaFunc() +{ + if (!pragmaFunc_.empty()) { + return; + } + pragmaFunc_ = { + {PRAGMA_SYNC_DEVICES, + [this]( + void *parameter, int &errCode) { errCode = PragmaSyncAction(static_cast(parameter)); }}, + {PRAGMA_AUTO_SYNC, + [this](void *parameter, int &errCode) { errCode = EnableAutoSync(*(static_cast(parameter))); }}, + {PRAGMA_PERFORMANCE_ANALYSIS_GET_REPORT, + [](void *parameter, int &errCode) { + *(static_cast(parameter)) = PerformanceAnalysis::GetInstance()->GetStatistics(); + }}, + {PRAGMA_PERFORMANCE_ANALYSIS_OPEN, + [](void *parameter, int &errCode) { PerformanceAnalysis::GetInstance()->OpenPerformanceAnalysis(); }}, + {PRAGMA_PERFORMANCE_ANALYSIS_CLOSE, + [](void *parameter, int &errCode) { PerformanceAnalysis::GetInstance()->ClosePerformanceAnalysis(); }}, + {PRAGMA_PERFORMANCE_ANALYSIS_SET_REPORTFILENAME, + [](void *parameter, int &errCode) { + PerformanceAnalysis::GetInstance()->SetFileNumber(*(static_cast(parameter))); + }}, + {PRAGMA_GET_QUEUED_SYNC_SIZE, + [this](void *parameter, int &errCode) { errCode = GetQueuedSyncSize(static_cast(parameter)); }}, + {PRAGMA_SET_QUEUED_SYNC_LIMIT, + [this](void *parameter, int &errCode) { errCode = SetQueuedSyncLimit(static_cast(parameter)); }}, + {PRAGMA_GET_QUEUED_SYNC_LIMIT, + [this](void *parameter, int &errCode) { errCode = GetQueuedSyncLimit(static_cast(parameter)); }}, + {PRAGMA_SET_WIPE_POLICY, + [this](void *parameter, int &errCode) { + errCode = SetStaleDataWipePolicy(static_cast(parameter)); + }}, + {PRAGMA_REMOTE_PUSH_FINISHED_NOTIFY, + [this](void *parameter, int &errCode) { + errCode = SetRemotePushFinishedNotify(static_cast(parameter)); + }}, + {PRAGMA_SET_SYNC_RETRY, + [this](void *parameter, int &errCode) { errCode = SetSyncRetry(*(static_cast(parameter))); }}, + {PRAGMA_ADD_EQUAL_IDENTIFIER, + [this](void *parameter, int &errCode) { + errCode = SetEqualIdentifier(static_cast(parameter)); + }}, + }; +} + int SyncAbleKvDBConnection::Pragma(int cmd, void *parameter) { int errCode = PragmaParamCheck(cmd, parameter); if (errCode != E_OK) { return -E_INVALID_ARGS; } - switch (cmd) { - case PRAGMA_SYNC_DEVICES: - errCode = PragmaSyncAction(static_cast(parameter)); - break; - case PRAGMA_AUTO_SYNC: - errCode = EnableAutoSync(*(static_cast(parameter))); - break; - case PRAGMA_PERFORMANCE_ANALYSIS_GET_REPORT: - *(static_cast(parameter)) = PerformanceAnalysis::GetInstance()->GetStatistics(); - break; - case PRAGMA_PERFORMANCE_ANALYSIS_OPEN: - PerformanceAnalysis::GetInstance()->OpenPerformanceAnalysis(); - break; - case PRAGMA_PERFORMANCE_ANALYSIS_CLOSE: - PerformanceAnalysis::GetInstance()->ClosePerformanceAnalysis(); - break; - case PRAGMA_PERFORMANCE_ANALYSIS_SET_REPORTFILENAME: - PerformanceAnalysis::GetInstance()->SetFileNumber(*(static_cast(parameter))); - break; - case PRAGMA_GET_QUEUED_SYNC_SIZE: - errCode = GetQueuedSyncSize(static_cast(parameter)); - break; - case PRAGMA_SET_QUEUED_SYNC_LIMIT: - errCode = SetQueuedSyncLimit(static_cast(parameter)); - break; - case PRAGMA_GET_QUEUED_SYNC_LIMIT: - errCode = GetQueuedSyncLimit(static_cast(parameter)); - break; - case PRAGMA_SET_WIPE_POLICY: - errCode = SetStaleDataWipePolicy(static_cast(parameter)); - break; - case PRAGMA_REMOTE_PUSH_FINISHED_NOTIFY: - errCode = SetRemotePushFinishedNotify(static_cast(parameter)); - break; - default: - // Call Pragma() of super class. - errCode = GenericKvDBConnection::Pragma(cmd, parameter); - break; + + InitPragmaFunc(); + auto iter = pragmaFunc_.find(cmd); + if (iter != pragmaFunc_.end()) { + iter->second(parameter, errCode); + return errCode; } - return errCode; + + // Call Pragma() of super class. + return GenericKvDBConnection::Pragma(cmd, parameter); } int SyncAbleKvDBConnection::PragmaParamCheck(int cmd, const void *parameter) @@ -105,6 +121,7 @@ int SyncAbleKvDBConnection::PragmaParamCheck(int cmd, const void *parameter) case PRAGMA_AUTO_SYNC: case PRAGMA_PERFORMANCE_ANALYSIS_GET_REPORT: case PRAGMA_PERFORMANCE_ANALYSIS_SET_REPORTFILENAME: + case PRAGMA_SET_SYNC_RETRY: if (parameter == nullptr) { return -E_INVALID_ARGS; } @@ -136,11 +153,17 @@ int SyncAbleKvDBConnection::PragmaSyncAction(const PragmaSync *syncParameter) } IncObjRef(this); } - int errCode = kvDB->Sync(syncParameter->devices_, syncParameter->mode_, - std::bind(&SyncAbleKvDBConnection::OnSyncComplete, this, std::placeholders::_1, syncParameter->onComplete_, - syncParameter->wait_), [this]() { - DecObjRef(this); - }, syncParameter->wait_); + + ISyncer::SyncParma syncParam; + syncParam.devices = syncParameter->devices_; + syncParam.mode = syncParameter->mode_; + syncParam.wait = syncParameter->wait_; + syncParam.isQuerySync = syncParameter->isQuerySync_; + syncParam.query = syncParameter->query_; + syncParam.onFinalize = [this]() { DecObjRef(this); }; + syncParam.onComplete = std::bind(&SyncAbleKvDBConnection::OnSyncComplete, this, std::placeholders::_1, + syncParameter->onComplete_, syncParameter->wait_); + int errCode = kvDB->Sync(syncParam); if (errCode <= 0) { DecObjRef(this); } else if (!syncParameter->wait_) { @@ -289,4 +312,27 @@ int SyncAbleKvDBConnection::SetRemotePushFinishedNotify(PragmaRemotePushNotify * remotePushFinishedListener_ = tmpListener; return errCode; } + +int SyncAbleKvDBConnection::SetSyncRetry(bool isRetry) +{ + SyncAbleKvDB *kvDB = GetDB(); + if (kvDB == nullptr) { + return -E_INVALID_CONNECTION; + } + return kvDB->SetSyncRetry(isRetry); +} + +int SyncAbleKvDBConnection::SetEqualIdentifier(const PragmaSetEqualIdentifier *param) +{ + if (param == nullptr) { + return -E_INVALID_ARGS; + } + + SyncAbleKvDB *kvDB = GetDB(); + if (kvDB == nullptr) { + return -E_INVALID_CONNECTION; + } + + return kvDB->SetEqualIdentifier(param->identifier_, param->targets_); +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h index c6c77128f..454be782a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h @@ -16,13 +16,14 @@ #ifndef SYNC_ABLE_KVDB_CONNECTION_H #define SYNC_ABLE_KVDB_CONNECTION_H -#include "ref_object.h" #include "generic_kvdb_connection.h" +#include "ref_object.h" namespace DistributedDB { class SyncAbleKvDB; struct PragmaSync; struct PragmaRemotePushNotify; +struct PragmaSetEqualIdentifier; class SyncAbleKvDBConnection : public GenericKvDBConnection, public virtual RefObject { public: @@ -42,6 +43,7 @@ private: // Do pragma-sync action. int PragmaParamCheck(int cmd, const void *parameter); int PragmaSyncAction(const PragmaSync *syncParameter); + void InitPragmaFunc(); // If enable is true, it will enable auto sync int EnableAutoSync(bool enable); @@ -59,11 +61,16 @@ private: int SetRemotePushFinishedNotify(PragmaRemotePushNotify *notifyParma); + int SetSyncRetry(bool isRetry); + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const PragmaSetEqualIdentifier *param); + // For sync in progress. std::list syncIdList_; std::mutex remotePushFinishedListenerLock_; NotificationChain::Listener *remotePushFinishedListener_; + std::map> pragmaFunc_{}; }; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isync_engine.h index fd66bb154..7fb5ee821 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isync_engine.h @@ -53,8 +53,15 @@ public: // Get local deviceId, is hashed virtual int GetLocalIdentity(std::string &outTarget) const = 0; + // Get the database identifier virtual std::string GetLabel() const = 0; + // Set Manual Sync retry config + virtual void SetSyncRetry(bool isRetry) = 0; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + virtual int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) = 0; + protected: virtual ~ISyncEngine() {}; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isync_state_machine.h index 26994f19c..4ffc923e7 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isync_state_machine.h @@ -26,8 +26,6 @@ namespace DistributedDB { class ISyncStateMachine { public: - const static int RETRY_TIME = 3; - virtual ~ISyncStateMachine() {}; // Init the SyncStateMachine, this function must be called before any other call. diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_target.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isync_target.h index fc05ebcc0..36ca06578 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_target.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isync_target.h @@ -43,7 +43,7 @@ public: // Get the mode of this task request or response virtual int GetMode() const = 0; - // Set a Sync Status, it will increase the ref of operation + // Set a SyncOperation virtual void SetSyncOperation(SyncOperation *operation) = 0; // Get a SyncOperation @@ -51,6 +51,8 @@ public: // Is this target is a auto sync virtual bool IsAutoSync() const = 0; + + virtual uint32_t GetResponseSessionId() const = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isync_task_context.h index 95b5337a3..fd92349c3 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isync_task_context.h @@ -79,6 +79,14 @@ public: virtual bool IsAutoSync() const = 0; + virtual bool IsSyncTaskNeedRetry() const = 0; + + virtual void SetSyncRetry(bool isRetry) = 0; + + virtual int GetSyncRetryTimes() const = 0; + + virtual int GetSyncRetryTimeout(int retryTime) const = 0; + // Set a Timer used for timeout virtual int StartTimer() = 0; @@ -149,6 +157,10 @@ public: virtual void SetTaskErrCode(int errCode) = 0; + virtual bool IsAutoLiftWaterMark() const = 0; + + virtual void IncNegotiationCount() = 0; + protected: virtual ~ISyncTaskContext() {}; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h index 6fdf05bd2..1b662cf4d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h @@ -21,11 +21,22 @@ #include #include "ikvdb_sync_interface.h" -#include "types_export.h" +#include "types_export.h" +#include "query_object.h" namespace DistributedDB { class ISyncer { public: + struct SyncParma { + std::vector devices; + std::function &devicesMap)> onComplete; + std::function onFinalize; + int mode = 0; + bool wait = false; + bool isQuerySync = false; + QueryObject query; + }; + virtual ~ISyncer() {}; // Init the Syncer modules @@ -44,10 +55,13 @@ public: const std::function &)> &onComplete, const std::function &onFinalize, bool wait) = 0; + // Sync function. use SyncParma to reduce paramter. + virtual int Sync(const SyncParma ¶m) = 0; + // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. virtual int RemoveSyncOperation(int syncId) = 0; - // Get The current vitural timestamp + // Get The current virtual timestamp virtual uint64_t GetTimeStamp() = 0; // Enable auto sync function @@ -79,6 +93,12 @@ public: // Set stale data wipe policy virtual int SetStaleDataWipePolicy(WipePolicy policy) = 0; + + // Set Manual Sync retry config + virtual int SetSyncRetry(bool isRetry) = 0; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + virtual int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/meta_data.h b/services/distributeddataservice/libs/distributeddb/syncer/include/meta_data.h index 1cb6356fe..2533f8949 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/meta_data.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/meta_data.h @@ -23,6 +23,7 @@ #include "db_types.h" #include "ikvdb_sync_interface.h" #include "ref_object.h" +#include "query_sync_water_mark_helper.h" namespace DistributedDB { struct MetaDataValue { @@ -30,6 +31,8 @@ struct MetaDataValue { uint64_t lastUpdateTime = 0; uint64_t localWaterMark = 0; uint64_t peerWaterMark = 0; + TimeStamp dbCreateTime = 0; + uint64_t clearDeviceDataMark = 0; // Default 0 for not remove device data. }; class Metadata { @@ -49,23 +52,60 @@ public: void GetPeerWaterMark(const DeviceID &deviceId, uint64_t &outValue); - int SavePeerWaterMark(const DeviceID &deviceId, uint64_t inValue, bool isNeedHash = true); + int SavePeerWaterMark(const DeviceID &deviceId, uint64_t inValue, bool isNeedHash); int SaveLocalTimeOffset(TimeOffset timeOffset); TimeOffset GetLocalTimeOffset() const; - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash = true); + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash); void SetLastLocalTime(TimeStamp lastLocalTime); TimeStamp GetLastLocalTime() const; + int SetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark); + + // the querySync's sendWatermark will increase by the device watermark + // if the sendWatermark less than device watermark + int GetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, WaterMark &waterMark, bool isAutoLift = true); + + int SetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark); + + // the querySync's recvWatermark will increase by the device watermark + // if the watermark less than device watermark + int GetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, WaterMark &waterMark); + + int SetSendDeleteSyncWaterMark(const std::string &deviceId, const WaterMark &waterMark); + + // the deleteSync's sendWatermark will increase by the device watermark + // if the sendWatermark less than device watermark + int GetSendDeleteSyncWaterMark(const std::string &deviceId, WaterMark &waterMark, bool isAutoLift = true); + + int SetRecvDeleteSyncWaterMark(const std::string &deviceId, const WaterMark &waterMark); + + // the deleteSync's recvWatermark will increase by the device watermark + // if the recvWatermark less than device watermark + int GetRecvDeleteSyncWaterMark(const std::string &deviceId, WaterMark &waterMark); + + void GetDbCreateTime(const DeviceID &deviceId, uint64_t &outValue); + + int SetDbCreateTime(const DeviceID &deviceId, uint64_t inValue, bool isNeedHash); + + int ResetMetaDataAfterRemoveData(const DeviceID &deviceId); + + void GetRemoveDataMark(const DeviceID &deviceId, uint64_t &outValue); + private: int SaveMetaDataValue(const DeviceID &deviceId, const MetaDataValue &inValue); - void GetMetaDataValue(const DeviceID &deviceId, MetaDataValue &outValue, bool isNeedHash = true); + // sync module need hash devices id + void GetMetaDataValue(const DeviceID &deviceId, MetaDataValue &outValue, bool isNeedHash); int SerializeMetaData(const MetaDataValue &inValue, std::vector &outValue); @@ -75,6 +115,9 @@ private: int SetMetadataToDb(const std::vector &key, const std::vector &inValue); + int DeleteMetaDataFromDB(const std::vector &keys) const; + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const; + void PutMetadataToMap(const DeviceID &deviceId, const MetaDataValue &value); void GetMetadataFromMap(const DeviceID &deviceId, MetaDataValue &outValue); @@ -87,7 +130,14 @@ private: uint64_t GetRandTimeOffset() const; - void GetHashDeviceId(const DeviceID &deviceId, DeviceID &hashDeviceId, bool isNeedHash = true); + void GetHashDeviceId(const DeviceID &deviceId, DeviceID &hashDeviceId, bool isNeedHash); + + // this function will read data from db by metaData's key + // and then serialize it and put to map + int LoadDeviceIdDataToMap(const Key &key); + + // reset the waterMark to zero + int ResetRecvQueryWaterMark(const DeviceID &deviceId); // store localTimeOffset in ram; if change, should add a lock first, change here and metadata, // then release lock @@ -104,6 +154,8 @@ private: // store localTimeOffset in ram, used to make timestamp increase mutable std::mutex lastLocalTimeLock_; TimeStamp lastLocalTime_; + + QuerySyncWaterMarkHelper querySyncWaterMarkHelper_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/query_sync_water_mark_helper.h b/services/distributeddataservice/libs/distributeddb/syncer/include/query_sync_water_mark_helper.h new file mode 100644 index 000000000..1000c52ad --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/query_sync_water_mark_helper.h @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef QUERY_SYNC_WATER_MARK_HELPER_H +#define QUERY_SYNC_WATER_MARK_HELPER_H + +#include "db_types.h" + +#include +#include +#include +#include +#include +#include "ikvdb_sync_interface.h" + +namespace DistributedDB { +struct QueryWaterMark { + uint32_t version = 0; + WaterMark sendWaterMark = 0; + WaterMark recvWaterMark = 0; + TimeStamp lastUsedTime = 0; // use for delete data + std::string sql; // for analyze sql from logs +}; + +struct DeleteWaterMark { + uint32_t version = 0; + WaterMark sendWaterMark = 0; + WaterMark recvWaterMark = 0; +}; + +// LRU map +class LruMap final { +public: + LruMap() = default; + ~LruMap() = default; + + DISABLE_COPY_ASSIGN_MOVE(LruMap); + + int Get(const std::string &key, QueryWaterMark &outValue); + int Put(const std::string &key, const QueryWaterMark &inValue); + void RemoveWithPrefixKey(const std::string &prefixKey); +private: + int Elimination(const std::string &key, const QueryWaterMark &inQueryWaterMark); + + std::mutex lruLock_; + std::map cache_; + std::deque> eliminationChain_; +}; + +class QuerySyncWaterMarkHelper { +public: + QuerySyncWaterMarkHelper(); + ~QuerySyncWaterMarkHelper(); + + DISABLE_COPY_ASSIGN_MOVE(QuerySyncWaterMarkHelper); + + int Initialize(IKvDBSyncInterface *storage); + + int GetQueryWaterMark(const std::string &queryIdentify, const std::string &deviceId, + QueryWaterMark &queryWaterMark); + + int SetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark); + + int SetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark); + + int GetDeleteSyncWaterMark(const std::string &deviceId, DeleteWaterMark &deleteWaterMark); + + int SetSendDeleteSyncWaterMark(const std::string &deviceId, const WaterMark &waterMark); + + int SetRecvDeleteSyncWaterMark(const std::string &deviceId, const WaterMark &waterMark); + + // this function will read deleteWaterMark from db by it's deleteWaterMarkKey + // and then serialize it and put to cache + int LoadDeleteSyncDataToCache(const Key &deleteWaterMarkKey); + + // this function will remove data in db + int RemoveLeastUsedQuerySyncItems(const std::vector &querySyncIds); + + // reset the waterMark to zero + int ResetRecvQueryWaterMark(const DeviceID &deviceId); + + static std::string GetQuerySyncPrefixKey(); + + static std::string GetDeleteSyncPrefixKey(); + +private: + + int GetMetadataFromDb(const std::vector &key, std::vector &outValue); + + int SetMetadataToDb(const std::vector &key, const std::vector &inValue); + + int DeleteMetaDataFromDB(const std::vector &keys) const; + + int SaveQueryWaterMarkToDB(const DeviceID &dbKeyString, const QueryWaterMark &queryWaterMark); + + int GetQueryWaterMarkFromDB(const DeviceID &dbKeyString, QueryWaterMark &queryWaterMark); + + int SetRecvQueryWaterMarkWithoutLock(const std::string &deviceId, const std::string &cacheKey, + const WaterMark &waterMark); + + // search the queryWaterMark from db or cache_ + // and ensure it exit in cache_ + int GetQueryWaterMarkInCacheAndDb(const std::string &cacheKey, QueryWaterMark &queryWaterMark); + + // only first create queryWaterMark will call this function + // it will create a queryWaterMark and save to db + int PutQueryWaterMarkToDB(const DeviceID &dbKeyString, QueryWaterMark &queryWaterMark); + + // get the querySync hashId in cache_ or generate one and then put it in to cache_ + // the hashId is made up of "QUERY_SYNC_PREFIX_KEY" + hash(deviceId) + queryId + void GetHashQuerySyncDeviceId(const DeviceID &deviceId, + const DeviceID &queryId, DeviceID &hashQuerySyncId); + + // put queryWaterMark to lru cache_ and then save to db + int UpdateCacheAndSave(const std::string &cacheKey, const std::string &deviceId, QueryWaterMark &queryWaterMark); + + // search the deleteWaterMark from db or cache_ + // and ensure it exit in cache_ + int GetDeleteWaterMarkFromCache(const DeviceID &hashDeviceId, DeleteWaterMark &deleteWaterMark); + + // get the deleteSync hashId in cache_ or generate one and then put it in to cache_ + // the hashId is made up of "DELETE_SYNC_PREFIX_KEY" + hash(deviceId) + void GetHashDeleteSyncDeviceId(const DeviceID &deviceId, DeviceID &hashDeleteSyncId); + + int SaveDeleteWaterMarkToDB(const DeviceID &hashDeviceId, const DeleteWaterMark &deleteWaterMark); + + int GetDeleteWaterMarkFromDB(const DeviceID &hashDeviceId, DeleteWaterMark &deleteWaterMark); + + // put queryWaterMark to lru cache_ and then save to db + int UpdateDeleteSyncCacheAndSave(const std::string &dbKey, const DeleteWaterMark &deleteWaterMark); + + static int SerializeQueryWaterMark(const QueryWaterMark &queryWaterMark, std::vector &outValue); + + static int DeSerializeQueryWaterMark(const std::vector &dbQueryWaterMark, QueryWaterMark &queryWaterMark); + + static uint64_t CalculateQueryWaterMarkSize(const QueryWaterMark &queryWaterMark); + + static int SerializeDeleteWaterMark(const DeleteWaterMark &deleteWaterMark, std::vector &outValue); + + static int DeSerializeDeleteWaterMark(const std::vector &inValue, DeleteWaterMark &deleteWaterMark); + + static uint64_t CalculateDeleteWaterMarkSize(const DeleteWaterMark &deleteWaterMark); + + // store or visit queryWaterMark should add a lock + // because it will change the eliminationChain + // and the queryWaterMark use a LRU Map to store in ram + std::mutex queryWaterMarkLock_; + LruMap querySyncCache_; + std::map> deviceIdToHashQuerySyncIdMap_; + + // also store deleteKeyWaterMark should add a lock + std::mutex deleteSyncLock_; + std::map deleteSyncCache_; + std::map deviceIdToHashDeleteSyncIdMap_; + + IKvDBSyncInterface *storage_; +}; +} // namespace DistributedDB +#endif diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_operation.h b/services/distributeddataservice/libs/distributeddb/syncer/include/sync_operation.h index 913252f4f..0fc735789 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_operation.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/sync_operation.h @@ -24,9 +24,11 @@ #include "ikvdb_sync_interface.h" #include "ref_object.h" -#include "semaphore_utils.h" +#include "semaphore.h" +#include "sync_types.h" #include "notification_chain.h" #include "runtime_context.h" +#include "query_sync_object.h" namespace DistributedDB { class SyncOperation : public RefObject { @@ -44,19 +46,11 @@ public: EKEYREVOKED_FAILURE, // EKEYREVOKED error BUSY_FAILURE, SCHEMA_INCOMPATIBLE, + QUERY_FORMAT_FAILURE, + QUERY_FIELD_FAILURE, FAILED }; - enum SyncMode { - PUSH, - PULL, - PUSH_AND_PULL, - AUTO_PUSH, - AUTO_PULL, - RESPONSE_PULL, - INVALID - }; - using UserCallback = std::function)>; using OnSyncFinished = std::function; using OnSyncFinalize = std::function; @@ -108,6 +102,13 @@ public: // Check if All devices sync finished. bool CheckIsAllFinished() const; + // For query sync + void SetQuery(const QueryObject &query); + QuerySyncObject GetQuery() const; + bool IsQuerySync() const; + static int GetSyncType(int mode); + static int TransferSyncMode(int mode); + protected: virtual ~SyncOperation(); @@ -117,6 +118,9 @@ private: // called by destruction void Finalize(); + // Transfer sync mode from interface to inner + void TransferQuerySyncMode(); + // The device list const std::vector devices_; @@ -148,7 +152,10 @@ private: bool isFinished_; // Used for block sync - std::unique_ptr semaphore_; + std::unique_ptr semaphore_; + + QuerySyncObject query_; + bool isQuerySync; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_types.h b/services/distributeddataservice/libs/distributeddb/syncer/include/sync_types.h index 94ada2dea..10214e8e5 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_types.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/sync_types.h @@ -16,6 +16,8 @@ #ifndef SYNC_TYPES_H #define SYNC_TYPES_H +#include + namespace DistributedDB { enum MessageId { TIME_SYNC_MESSAGE = 1, @@ -25,10 +27,52 @@ enum MessageId { VALUE_SLICE_SYNC_MESSAGE, LOCAL_DATA_CHANGED, ABILITY_SYNC_MESSAGE, + QUERY_SYNC_MESSAGE, + UNKNOW_MESSAGE, +}; + +enum SyncModeType { + PUSH, + PULL, + PUSH_AND_PULL, + AUTO_PUSH, + AUTO_PULL, + RESPONSE_PULL, + QUERY_PUSH, + QUERY_PULL, + QUERY_PUSH_PULL, + INVALID_MODE +}; + +enum SyncType { + MANUAL_FULL_SYNC_TYPE = 1, + AUTO_SYNC_TYPE, + QUERY_SYNC_TYPE, + INVALID_SYNC_TYPE, +}; + +struct UpdateWaterMark { + bool normalUpdateMark = false; + bool deleteUpdateMark = false; }; -const static uint32_t SEND_TIME_OUT = 3000; // 3s -const int NOT_SURPPORT_SEC_CLASSIFICATION = 0xff; +constexpr uint32_t SEND_TIME_OUT = 3000; // 3s +constexpr int NOT_SURPPORT_SEC_CLASSIFICATION = 0xff; +constexpr uint8_t QUERY_SYNC_MODE_BASE = SyncModeType::QUERY_PUSH; +constexpr int AUTO_RETRY_TIMES = 3; +constexpr int MANUAL_RETRY_TIMES = 1; +constexpr int TIME_SYNC_WAIT_TIME = 5000; // 5s +constexpr int FEEDDOG_MAX_MTU_SIZE = 30 * 1024; // 30k + +// index 0 for packetId in data request +// if ack reserve size is 1, reserve is {localWaterMark} +// if ack reserve size is above 2, reserve is {localWaterMark, packetId, deletedWaterMark...} +constexpr uint32_t REQUEST_PACKET_RESERVED_INDEX_PACKETID = 0; +constexpr uint32_t ACK_PACKET_RESERVED_INDEX_PACKETID = 1; // index 0 for packetId +constexpr uint32_t ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK = 0; // index 0 for localWaterMark +constexpr uint32_t ACK_PACKET_RESERVED_INDEX_DELETE_WATER_MARK = 2; // index 2 for deleteDataWaterMark +constexpr uint64_t MAX_TIMESTAMP = INT64_MAX; +constexpr uint8_t REMOVE_DEVICE_DATA_MARK = 1; } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h index 9e2ba913d..5f3a60809 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h @@ -46,6 +46,9 @@ public: const std::function &)> &onComplete, const std::function &onFinalize, bool wait) override; + // Sync function. use SyncParma to reduce paramter. + int Sync(const SyncParma ¶m) override; + // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; @@ -82,6 +85,12 @@ public: // Set stale data wipe policy int SetStaleDataWipePolicy(WipePolicy policy) override; + // Set Manual Sync retry config + int SetSyncRetry(bool isRetry) override; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) override; + private: std::mutex syncerLock_; std::shared_ptr syncer_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp index 9d1cc7e07..087de658b 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp @@ -20,6 +20,7 @@ #include "db_errno.h" #include "log_print.h" #include "sync_types.h" +#include "db_common.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_task_context.h" @@ -30,7 +31,8 @@ AbilitySyncRequestPacket::AbilitySyncRequestPacket() softwareVersion_(SOFTWARE_VERSION_CURRENT), secLabel_(0), secFlag_(0), - schemaType_(0) + schemaType_(0), + dbCreateTime_(0) { } @@ -73,9 +75,9 @@ void AbilitySyncRequestPacket::SetSchema(const std::string &schema) schema_ = schema; } -void AbilitySyncRequestPacket::GetSchema(std::string &schema) const +std::string AbilitySyncRequestPacket::GetSchema() const { - schema = schema_; + return schema_; } void AbilitySyncRequestPacket::SetSchemaType(uint32_t schemaType) @@ -108,6 +110,16 @@ int32_t AbilitySyncRequestPacket::GetSecFlag() const return secFlag_; } +void AbilitySyncRequestPacket::SetDbCreateTime(uint64_t dbCreateTime) +{ + dbCreateTime_ = dbCreateTime; +} + +uint64_t AbilitySyncRequestPacket::GetDbCreateTime() const +{ + return dbCreateTime_; +} + uint32_t AbilitySyncRequestPacket::CalculateLen() const { uint64_t len = 0; @@ -123,6 +135,8 @@ uint32_t AbilitySyncRequestPacket::CalculateLen() const len += Parcel::GetIntLen(); // secLabel_ len += Parcel::GetIntLen(); // secFlag_ len += Parcel::GetUInt32Len(); // schemaType_ + len += Parcel::GetUInt64Len(); // dbCreateTime_ + len += DbAbility::CalculateLen(); // dbAbility_ // the reason why not 8-byte align is that old version is not 8-byte align // so it is not possible to set 8-byte align for high version. if (len > INT32_MAX) { @@ -132,6 +146,16 @@ uint32_t AbilitySyncRequestPacket::CalculateLen() const return len; } +DbAbility AbilitySyncRequestPacket::GetDbAbility() const +{ + return dbAbility_; +} + +void AbilitySyncRequestPacket::SetDbAbility(const DbAbility &dbAbility) +{ + dbAbility_ = dbAbility; +} + AbilitySyncAckPacket::AbilitySyncAckPacket() : protocolVersion_(ABILITY_SYNC_VERSION_V1), softwareVersion_(SOFTWARE_VERSION_CURRENT), @@ -140,7 +164,8 @@ AbilitySyncAckPacket::AbilitySyncAckPacket() secFlag_(0), schemaType_(0), permitSync_(0), - requirePeerConvert_(0) + requirePeerConvert_(0), + dbCreateTime_(0) { } @@ -183,9 +208,9 @@ void AbilitySyncAckPacket::SetSchema(const std::string &schema) schema_ = schema; } -void AbilitySyncAckPacket::GetSchema(std::string &schema) const +std::string AbilitySyncAckPacket::GetSchema() const { - schema = schema_; + return schema_; } void AbilitySyncAckPacket::SetSchemaType(uint32_t schemaType) @@ -238,6 +263,16 @@ uint32_t AbilitySyncAckPacket::GetRequirePeerConvert() const return requirePeerConvert_; } +void AbilitySyncAckPacket::SetDbCreateTime(uint64_t dbCreateTime) +{ + dbCreateTime_ = dbCreateTime; +} + +uint64_t AbilitySyncAckPacket::GetDbCreateTime() const +{ + return dbCreateTime_; +} + uint32_t AbilitySyncAckPacket::CalculateLen() const { uint64_t len = 0; @@ -255,6 +290,8 @@ uint32_t AbilitySyncAckPacket::CalculateLen() const len += Parcel::GetUInt32Len(); // schemaType_ len += Parcel::GetUInt32Len(); // permitSync_ len += Parcel::GetUInt32Len(); // requirePeerConvert_ + len += Parcel::GetUInt64Len(); // dbCreateTime_ + len += DbAbility::CalculateLen(); // dbAbility_ if (len > INT32_MAX) { LOGE("[AbilitySyncAckPacket][CalculateLen] err len:%llu", len); return 0; @@ -262,9 +299,20 @@ uint32_t AbilitySyncAckPacket::CalculateLen() const return len; } +DbAbility AbilitySyncAckPacket::GetDbAbility() const +{ + return dbAbility_; +} + +void AbilitySyncAckPacket::SetDbAbility(const DbAbility &dbAbility) +{ + dbAbility_ = dbAbility; +} + AbilitySync::AbilitySync() : communicator_(nullptr), storageInterface_(nullptr), + metadata_(nullptr), syncFinished_(false) { } @@ -273,15 +321,18 @@ AbilitySync::~AbilitySync() { communicator_ = nullptr; storageInterface_ = nullptr; + metadata_ = nullptr; } -int AbilitySync::Initialize(ICommunicator *inCommunicator, IKvDBSyncInterface *inStorage, const std::string &deviceId) +int AbilitySync::Initialize(ICommunicator *inCommunicator, IKvDBSyncInterface *inStorage, + std::shared_ptr &inMetadata, const std::string &deviceId) { - if (inCommunicator == nullptr || inStorage == nullptr || deviceId.empty()) { + if (inCommunicator == nullptr || inStorage == nullptr || deviceId.empty() || inMetadata == nullptr) { return -E_INVALID_ARGS; } communicator_ = inCommunicator; storageInterface_ = inStorage; + metadata_ = inMetadata; deviceId_ = deviceId; return E_OK; } @@ -289,30 +340,17 @@ int AbilitySync::Initialize(ICommunicator *inCommunicator, IKvDBSyncInterface *i int AbilitySync::SyncStart(uint32_t sessionId, uint32_t sequenceId, uint16_t remoteCommunicatorVersion, const CommErrHandler &handler) { + AbilitySyncRequestPacket packet; + int errCode = SetAbilityRequestBodyInfo(packet, remoteCommunicatorVersion); + if (errCode != E_OK) { + return errCode; + } Message *message = new (std::nothrow) Message(ABILITY_SYNC_MESSAGE); if (message == nullptr) { return -E_OUT_OF_MEMORY; } - AbilitySyncRequestPacket packet; - packet.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); - packet.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); - SchemaObject schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); - // 102 version is forbidden to sync with 103 json-schema or flatbuffer-schema - // so schema should put null string while remote is 102 version to avoid this bug. - if (remoteCommunicatorVersion == 1) { - packet.SetSchema(""); - packet.SetSchemaType(0); - } else { - packet.SetSchema(schemaObj.ToSchemaString()); - packet.SetSchemaType(static_cast(schemaObj.GetSchemaType())); - } - SecurityOption option; - GetPacketSecOption(option); - packet.SetSecLabel(option.securityLabel); - packet.SetSecFlag(option.securityFlag); - message->SetMessageType(TYPE_REQUEST); - int errCode = message->SetCopiedObject<>(packet); + errCode = message->SetCopiedObject<>(packet); if (errCode != E_OK) { LOGE("[AbilitySync][SyncStart] SetCopiedObject failed, err %d", errCode); delete message; @@ -322,8 +360,6 @@ int AbilitySync::SyncStart(uint32_t sessionId, uint32_t sequenceId, uint16_t rem message->SetVersion(MSG_VERSION_EXT); message->SetSessionId(sessionId); message->SetSequenceId(sequenceId); - LOGI("[AbilitySync][SyncStart] software version = %u, Label = %d, Flag = %d", SOFTWARE_VERSION_CURRENT, - option.securityLabel, option.securityFlag); errCode = communicator_->SendMessage(deviceId_, message, false, SEND_TIME_OUT, handler); if (errCode != E_OK) { LOGE("[AbilitySync][SyncStart] SendPacket failed, err %d", errCode); @@ -348,8 +384,7 @@ int AbilitySync::AckRecv(const Message *message, ISyncTaskContext *context) } uint32_t remoteSoftwareVersion = packet->GetSoftwareVersion(); context->SetRemoteSoftwareVersion(remoteSoftwareVersion); - std::string schema; - packet->GetSchema(schema); + std::string schema = packet->GetSchema(); SingleVerKvDBSyncInterface *storage = static_cast(storageInterface_); if (remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_2_0) { HandleVersionV3AckSecOptionParam(packet, context); @@ -360,6 +395,15 @@ int AbilitySync::AckRecv(const Message *message, ISyncTaskContext *context) LOGE("[AbilitySync][AckRecv] scheme check failed"); return -E_SCHEMA_MISMATCH; } + if (remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_3_0) { + errCode = metadata_->SetDbCreateTime(deviceId_, packet->GetDbCreateTime(), true); + if (errCode != E_OK) { + LOGE("[AbilitySync][AckRecv] set db create time failed,errCode=%d", errCode); + return errCode; + } + } + DbAbility remoteDbAbility = packet->GetDbAbility(); + (static_cast(context))->SetDbAbility(remoteDbAbility); (void)SendAck(message, storage->GetSchemaInfo(), AbilitySync::CHECK_SUCCESS, localSyncOpinion, true); (static_cast(context))->SetIsSchemaSync(true); } else { @@ -392,8 +436,7 @@ int AbilitySync::RequestRecv(const Message *message, ISyncTaskContext *context) return -E_VERSION_NOT_SUPPORT; } - std::string schema; - packet->GetSchema(schema); + std::string schema = packet->GetSchema(); bool isCompatible = storage->CheckCompatible(schema); if (!isCompatible) { (static_cast(context))->SetTaskErrCode(-E_SCHEMA_MISMATCH); @@ -413,8 +456,11 @@ int AbilitySync::RequestRecv(const Message *message, ISyncTaskContext *context) remoteSoftwareVersion, isCompatible); return SendAck(message, SchemaObject(), E_OK, localSyncOpinion, false); } - LOGI("[AbilitySync][RequestRecv] remote dev = %s{private}, version = %u, CheckSchemaCompatible = %d", - deviceId_.c_str(), remoteSoftwareVersion, isCompatible); + if (ackCode == E_OK && remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_3_0) { + ackCode = metadata_->SetDbCreateTime(deviceId_, packet->GetDbCreateTime(), true); + } + LOGI("[AbilitySync][RequestRecv] remote dev=%s,ver=%u,schemaCompatible=%d", STR_MASK(deviceId_), + remoteSoftwareVersion, isCompatible); return SendAck(message, storage->GetSchemaInfo(), ackCode, localSyncOpinion, false); } @@ -438,13 +484,12 @@ int AbilitySync::AckNotifyRecv(const Message *message, ISyncTaskContext *context LOGE("[AbilitySync][AckNotifyRecv] received a errCode %d", errCode); return errCode; } - std::string schema; - packet->GetSchema(schema); + std::string schema = packet->GetSchema(); uint32_t remoteSoftwareVersion = packet->GetSoftwareVersion(); context->SetRemoteSoftwareVersion(remoteSoftwareVersion); SyncOpinion localSyncOpinion = HandleVersionV3AckSchemaParam(packet, schema, context); - LOGI("[AckNotifyRecv] receive dev = %s{private} ack notify, remoteSoftwareVersion = %u, ackCode = %d", - deviceId_.c_str(), remoteSoftwareVersion, errCode); + LOGI("[AckNotifyRecv] receive dev = %s ack notify, remoteSoftwareVersion = %u, ackCode = %d", + STR_MASK(deviceId_), remoteSoftwareVersion, errCode); (static_cast(context))->SetIsSchemaSync(true); (void)SendAck(message, SchemaObject(), AbilitySync::LAST_NOTIFY, localSyncOpinion, true); return E_OK; @@ -492,6 +537,8 @@ SyncOpinion AbilitySync::HandleVersionV3RequestParam(const AbilitySyncRequestPac int32_t remoteSecLabel = packet->GetSecLabel(); int32_t remoteSecFlag = packet->GetSecFlag(); SecurityOption secOption = {remoteSecLabel, remoteSecFlag}; + DbAbility remoteDbAbility = packet->GetDbAbility(); + (static_cast(context))->SetDbAbility(remoteDbAbility); (static_cast(context))->SetRemoteSeccurityOption(secOption); (static_cast(context))->SetReceivcPermitCheck(false); uint8_t remoteSchemaType = packet->GetSchemaType(); @@ -527,7 +574,7 @@ SyncOpinion AbilitySync::HandleVersionV3AckSchemaParam(const AbilitySyncAckPacke return localOpinion; } -void AbilitySync::GetPacketSecOption(SecurityOption &option) +void AbilitySync::GetPacketSecOption(SecurityOption &option) const { int errCode = (static_cast(storageInterface_))->GetSecurityOption(option); if (errCode == -E_NOT_SUPPORT) { @@ -622,27 +669,17 @@ int AbilitySync::DeSerialization(const uint8_t *buffer, uint32_t length, Message int AbilitySync::SendAck(const Message *inMsg, const SchemaObject &schemaObj, int ackCode, SyncOpinion localOpinion, bool isAckNotify) { + AbilitySyncAckPacket ackPacket; + int errCode = SetAbilityAckBodyInfo(ackPacket, schemaObj, ackCode, localOpinion, isAckNotify); + if (errCode != E_OK) { + return errCode; + } Message *ackMessage = new (std::nothrow) Message(ABILITY_SYNC_MESSAGE); if (ackMessage == nullptr) { LOGE("[AbilitySync][SendAck] message create failed, may be memleak!"); return -E_OUT_OF_MEMORY; } - - AbilitySyncAckPacket ackPacket; - ackPacket.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); - ackPacket.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); - ackPacket.SetSchema(schemaObj.ToSchemaString()); - ackPacket.SetSchemaType(static_cast(schemaObj.GetSchemaType())); - ackPacket.SetAckCode(ackCode); - if (!isAckNotify) { - SecurityOption option; - GetPacketSecOption(option); - ackPacket.SetSecLabel(option.securityLabel); - ackPacket.SetSecFlag(option.securityFlag); - } - ackPacket.SetPermitSync(localOpinion.permitSync); - ackPacket.SetRequirePeerConvert(localOpinion.requirePeerConvert); - int errCode = ackMessage->SetCopiedObject<>(ackPacket); + errCode = ackMessage->SetCopiedObject<>(ackPacket); if (errCode != E_OK) { LOGE("[AbilitySync][SendAck] SetCopiedObject failed, err %d", errCode); delete ackMessage; @@ -653,7 +690,6 @@ int AbilitySync::SendAck(const Message *inMsg, const SchemaObject &schemaObj, in ackMessage->SetTarget(deviceId_); ackMessage->SetSessionId(inMsg->GetSessionId()); ackMessage->SetSequenceId(inMsg->GetSequenceId()); - errCode = communicator_->SendMessage(deviceId_, ackMessage, false, SEND_TIME_OUT); if (errCode != E_OK) { LOGE("[AbilitySync][SendAck] SendPacket failed, err %d", errCode); @@ -693,35 +729,17 @@ int AbilitySync::RequestPacketSerialization(uint8_t *buffer, uint32_t length, co } Parcel parcel(buffer, length); - int errCode = parcel.WriteUInt32(packet->GetProtocolVersion()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSendCode()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetSoftwareVersion()); - if (errCode != E_OK) { - return errCode; - } - std::string schema; - packet->GetSchema(schema); - errCode = parcel.WriteString(schema); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSecLabel()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSecFlag()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetSchemaType()); - if (errCode != E_OK) { - return errCode; + parcel.WriteUInt32(packet->GetProtocolVersion()); + parcel.WriteInt(packet->GetSendCode()); + parcel.WriteUInt32(packet->GetSoftwareVersion()); + parcel.WriteString(packet->GetSchema()); + parcel.WriteInt(packet->GetSecLabel()); + parcel.WriteInt(packet->GetSecFlag()); + parcel.WriteUInt32(packet->GetSchemaType()); + parcel.WriteUInt64(packet->GetDbCreateTime()); + int errCode = DbAbility::Serialize(parcel, packet->GetDbAbility()); + if (parcel.IsError() || errCode != E_OK) { + return -E_PARSE_FAIL; } return E_OK; } @@ -734,43 +752,19 @@ int AbilitySync::AckPacketSerialization(uint8_t *buffer, uint32_t length, const } Parcel parcel(buffer, length); - int errCode = parcel.WriteUInt32(ABILITY_SYNC_VERSION_V1); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(SOFTWARE_VERSION_CURRENT); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetAckCode()); - if (errCode != E_OK) { - return errCode; - } - std::string schema; - packet->GetSchema(schema); - errCode = parcel.WriteString(schema); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSecLabel()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSecFlag()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetSchemaType()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetPermitSync()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetRequirePeerConvert()); - if (errCode != E_OK) { - return errCode; + parcel.WriteUInt32(ABILITY_SYNC_VERSION_V1); + parcel.WriteUInt32(SOFTWARE_VERSION_CURRENT); + parcel.WriteInt(packet->GetAckCode()); + parcel.WriteString(packet->GetSchema()); + parcel.WriteInt(packet->GetSecLabel()); + parcel.WriteInt(packet->GetSecFlag()); + parcel.WriteUInt32(packet->GetSchemaType()); + parcel.WriteUInt32(packet->GetPermitSync()); + parcel.WriteUInt32(packet->GetRequirePeerConvert()); + parcel.WriteUInt64(packet->GetDbCreateTime()); + int errCode = DbAbility::Serialize(parcel, packet->GetDbAbility()); + if (parcel.IsError() || errCode != E_OK) { + return -E_PARSE_FAIL; } return E_OK; } @@ -805,10 +799,8 @@ int AbilitySync::RequestPacketDeSerialization(const uint8_t *buffer, uint32_t le parcel.ReadInt(sendCode); parcel.ReadUInt32(softwareVersion); parcel.ReadString(schema); - if (!parcel.IsError() && softwareVersion > SOFTWARE_VERSION_RELEASE_2_0) { - RequestPacketDeSerializationTailPart(parcel, packet); - } - if (parcel.IsError()) { + errCode = RequestPacketDeSerializationTailPart(parcel, packet, softwareVersion); + if (parcel.IsError() || errCode != E_OK) { goto ERROR_OUT; } packet->SetSendCode(sendCode); @@ -826,36 +818,67 @@ ERROR_OUT: return errCode; } -void AbilitySync::RequestPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncRequestPacket *packet) +int AbilitySync::RequestPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncRequestPacket *packet, + uint32_t version) { - int32_t secLabel = 0; - int32_t secFlag = 0; - uint32_t schemaType = 0; - parcel.ReadInt(secLabel); - parcel.ReadInt(secFlag); - parcel.ReadUInt32(schemaType); - packet->SetSecLabel(secLabel); - packet->SetSecFlag(secFlag); - packet->SetSchemaType(schemaType); + if (!parcel.IsError() && version > SOFTWARE_VERSION_RELEASE_2_0) { + int32_t secLabel = 0; + int32_t secFlag = 0; + uint32_t schemaType = 0; + parcel.ReadInt(secLabel); + parcel.ReadInt(secFlag); + parcel.ReadUInt32(schemaType); + packet->SetSecLabel(secLabel); + packet->SetSecFlag(secFlag); + packet->SetSchemaType(schemaType); + } + if (!parcel.IsError() && version > SOFTWARE_VERSION_RELEASE_3_0) { + uint64_t dbCreateTime = 0; + parcel.ReadUInt64(dbCreateTime); + packet->SetDbCreateTime(dbCreateTime); + } + DbAbility remoteDbAbility; + int errCode = DbAbility::DeSerialize(parcel, remoteDbAbility); + if (errCode != E_OK) { + LOGE("[AbilitySync] request packet DeSerializ failed."); + return errCode; + } + packet->SetDbAbility(remoteDbAbility); + return E_OK; } -void AbilitySync::AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAckPacket *packet) -{ - int32_t secLabel = 0; - int32_t secFlag = 0; - uint32_t schemaType = 0; - uint32_t permitSync = 0; - uint32_t requirePeerConvert = 0; - parcel.ReadInt(secLabel); - parcel.ReadInt(secFlag); - parcel.ReadUInt32(schemaType); - parcel.ReadUInt32(permitSync); - parcel.ReadUInt32(requirePeerConvert); - packet->SetSecLabel(secLabel); - packet->SetSecFlag(secFlag); - packet->SetSchemaType(schemaType); - packet->SetPermitSync(permitSync); - packet->SetRequirePeerConvert(requirePeerConvert); +int AbilitySync::AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAckPacket *packet, uint32_t version) +{ + if (!parcel.IsError() && version > SOFTWARE_VERSION_RELEASE_2_0) { + int32_t secLabel = 0; + int32_t secFlag = 0; + uint32_t schemaType = 0; + uint32_t permitSync = 0; + uint32_t requirePeerConvert = 0; + parcel.ReadInt(secLabel); + parcel.ReadInt(secFlag); + parcel.ReadUInt32(schemaType); + parcel.ReadUInt32(permitSync); + parcel.ReadUInt32(requirePeerConvert); + packet->SetSecLabel(secLabel); + packet->SetSecFlag(secFlag); + packet->SetSchemaType(schemaType); + packet->SetPermitSync(permitSync); + packet->SetRequirePeerConvert(requirePeerConvert); + } + if (!parcel.IsError() && version > SOFTWARE_VERSION_RELEASE_3_0) { + uint64_t dbCreateTime = 0; + parcel.ReadUInt64(dbCreateTime); + packet->SetDbCreateTime(dbCreateTime); + } + DbAbility remoteDbAbility; + int errCode = DbAbility::DeSerialize(parcel, remoteDbAbility); + if (errCode != E_OK) { + LOGE("[AbilitySync] ack packet DeSerializ failed."); + return errCode; + } + packet->SetDbAbility(remoteDbAbility); + return E_OK; } int AbilitySync::AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) @@ -889,10 +912,8 @@ int AbilitySync::AckPacketDeSerialization(const uint8_t *buffer, uint32_t length parcel.ReadUInt32(softwareVersion); parcel.ReadInt(ackCode); parcel.ReadString(schema); - if (!parcel.IsError() && softwareVersion > SOFTWARE_VERSION_RELEASE_2_0) { - AckPacketDeSerializationTailPart(parcel, packet); - } - if (parcel.IsError()) { + errCode = AckPacketDeSerializationTailPart(parcel, packet, softwareVersion); + if (parcel.IsError() || errCode != E_OK) { LOGE("[AbilitySync][RequestDeSerialization] DeSerialization failed!"); errCode = -E_PARSE_FAIL; goto ERROR_OUT; @@ -928,4 +949,89 @@ int AbilitySync::CheckAckCode(const Message *message, ISyncTaskContext *context, } return E_OK; } + +int AbilitySync::SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uint16_t remoteCommunicatorVersion) const +{ + uint64_t dbCreateTime; + int errCode = + (static_cast(storageInterface_))->GetDatabaseCreateTimeStamp(dbCreateTime); + if (errCode != E_OK) { + LOGE("[AbilitySync][FillAbilityRequest] GetDatabaseCreateTimeStamp failed, err %d", errCode); + return errCode; + } + SecurityOption option; + GetPacketSecOption(option); + SchemaObject schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); + DbAbility dbAbility; + errCode = GetDbAbilityInfo(dbAbility); + if (errCode != E_OK) { + LOGE("[AbilitySync][FillAbilityRequest] GetDbAbility failed, err %d", errCode); + return errCode; + } + // 102 version is forbidden to sync with 103 json-schema or flatbuffer-schema + // so schema should put null string while remote is 102 version to avoid this bug. + if (remoteCommunicatorVersion == 1) { + packet.SetSchema(""); + packet.SetSchemaType(0); + } else { + packet.SetSchema(schemaObj.ToSchemaString()); + packet.SetSchemaType(static_cast(schemaObj.GetSchemaType())); + } + packet.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); + packet.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); + packet.SetSecLabel(option.securityLabel); + packet.SetSecFlag(option.securityFlag); + packet.SetDbCreateTime(dbCreateTime); + packet.SetDbAbility(dbAbility); + LOGI("[AbilitySync][FillRequest] ver=%u,Lab=%d,Flag=%d,dbCreateTime=%llu", SOFTWARE_VERSION_CURRENT, + option.securityLabel, option.securityFlag, dbCreateTime); + return E_OK; +} + +int AbilitySync::SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const SchemaObject &schemaObj, int ackCode, + SyncOpinion localOpinion, bool isAckNotify) +{ + int errCode = E_OK; + ackPacket.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); + ackPacket.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); + ackPacket.SetSchema(schemaObj.ToSchemaString()); + ackPacket.SetSchemaType(static_cast(schemaObj.GetSchemaType())); + if (!isAckNotify) { + SecurityOption option; + GetPacketSecOption(option); + ackPacket.SetSecLabel(option.securityLabel); + ackPacket.SetSecFlag(option.securityFlag); + uint64_t dbCreateTime = 0; + errCode = + (static_cast(storageInterface_))->GetDatabaseCreateTimeStamp(dbCreateTime); + if (errCode != E_OK) { + LOGE("[AbilitySync][SyncStart] GetDatabaseCreateTimeStamp failed, err %d", errCode); + ackCode = errCode; + } + DbAbility dbAbility; + errCode = GetDbAbilityInfo(dbAbility); + if (errCode != E_OK) { + LOGE("[AbilitySync][FillAbilityRequest] GetDbAbility failed, err %d", errCode); + return errCode; + } + ackPacket.SetDbCreateTime(dbCreateTime); + ackPacket.SetDbAbility(dbAbility); + } + ackPacket.SetAckCode(ackCode); + ackPacket.SetPermitSync(localOpinion.permitSync); + ackPacket.SetRequirePeerConvert(localOpinion.requirePeerConvert); + return E_OK; +} + +int AbilitySync::GetDbAbilityInfo(DbAbility &dbAbility) const +{ + std::set localAlgorithmSet; + int errCode = + (static_cast(storageInterface_))->GetCompressionAlgo(localAlgorithmSet); + if (errCode != E_OK) { + return errCode; + } + dbAbility.SetCompressAlgorithm(localAlgorithmSet); + return E_OK; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h index 4304e1856..1fa61c7dc 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h @@ -19,6 +19,7 @@ #include #include +#include "db_ability.h" #include "icommunicator.h" #include "ikvdb_sync_interface.h" #include "isync_task_context.h" @@ -40,7 +41,7 @@ public: uint32_t GetSoftwareVersion() const; void SetSchema(const std::string &schema); - void GetSchema(std::string &schema) const; + std::string GetSchema() const; void SetSchemaType(uint32_t schemaType); uint32_t GetSchemaType() const; @@ -51,8 +52,15 @@ public: void SetSecFlag(int32_t secFlag); int32_t GetSecFlag() const; + void SetDbCreateTime(uint64_t dbCreateTime); + uint64_t GetDbCreateTime() const; + uint32_t CalculateLen() const; + DbAbility GetDbAbility() const; + + void SetDbAbility(const DbAbility &dbAbility); + private: uint32_t protocolVersion_; int32_t sendCode_; @@ -61,6 +69,8 @@ private: int32_t secLabel_; int32_t secFlag_; uint32_t schemaType_; + uint64_t dbCreateTime_; + DbAbility dbAbility_; }; class AbilitySyncAckPacket { @@ -78,7 +88,7 @@ public: int32_t GetAckCode() const; void SetSchema(const std::string &schema); - void GetSchema(std::string &schema) const; + std::string GetSchema() const; void SetSchemaType(uint32_t schemaType); uint32_t GetSchemaType() const; @@ -95,8 +105,15 @@ public: void SetRequirePeerConvert(uint32_t requirePeerConvert); uint32_t GetRequirePeerConvert() const; + void SetDbCreateTime(uint64_t dbCreateTime); + uint64_t GetDbCreateTime() const; + uint32_t CalculateLen() const; + DbAbility GetDbAbility() const; + + void SetDbAbility(const DbAbility &dbAbility); + private: uint32_t protocolVersion_; uint32_t softwareVersion_; @@ -107,6 +124,8 @@ private: uint32_t schemaType_; uint32_t permitSync_; uint32_t requirePeerConvert_; + uint64_t dbCreateTime_; + DbAbility dbAbility_; }; class AbilitySync { @@ -120,7 +139,8 @@ public: int SyncStart(uint32_t sessionId, uint32_t sequenceId, uint16_t remoteCommunicatorVersion, const CommErrHandler &handler = nullptr); - int Initialize(ICommunicator *inCommunicator, IKvDBSyncInterface *inStorage, const std::string &deviceId); + int Initialize(ICommunicator *inCommunicator, IKvDBSyncInterface *inStorage, std::shared_ptr &inMetadata, + const std::string &deviceId); int AckRecv(const Message *message, ISyncTaskContext *context); @@ -156,9 +176,10 @@ private: static int AckPacketCalculateLen(const Message *inMsg, uint32_t &len); - static void RequestPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncRequestPacket *packet); + static int RequestPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncRequestPacket *packet, + uint32_t version); - static void AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAckPacket *packet); + static int AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAckPacket *packet, uint32_t version); bool SecLabelCheck(const AbilitySyncRequestPacket *packet) const; @@ -171,12 +192,20 @@ private: SyncOpinion HandleVersionV3AckSchemaParam(const AbilitySyncAckPacket *packet, const std::string &remoteSchema, ISyncTaskContext *context) const; - void GetPacketSecOption(SecurityOption &option); + void GetPacketSecOption(SecurityOption &option) const; int CheckAckCode(const Message *message, ISyncTaskContext *context, int errCode); + int SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uint16_t remoteCommunicatorVersion) const; + + int SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const SchemaObject &schemaObj, int ackCode, + SyncOpinion localOpinion, bool isAckNotify); + + int GetDbAbilityInfo(DbAbility &dbAbility) const; + ICommunicator *communicator_; IKvDBSyncInterface *storageInterface_; + std::shared_ptr metadata_; std::string deviceId_; bool syncFinished_; static const int FAILED_GET_SEC_CLASSIFICATION = 0x55; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp new file mode 100644 index 000000000..c6b6a5019 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "communicator_proxy.h" +#include "db_constant.h" +#include "log_print.h" + +namespace DistributedDB { +CommunicatorProxy::CommunicatorProxy() : mainComm_(nullptr) +{ +} + +CommunicatorProxy::~CommunicatorProxy() +{ + if (mainComm_ != nullptr) { + RefObject::DecObjRef(mainComm_); + } + mainComm_ = nullptr; + + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + RefObject::DecObjRef(devCommMap_[iter.first]); + } + devCommMap_.clear(); +} + +int CommunicatorProxy::RegOnMessageCallback(const OnMessageCallback &onMessage, const Finalizer &inOper) +{ + if (mainComm_ != nullptr) { + (void) mainComm_->RegOnMessageCallback(onMessage, inOper); + } + + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + (void) devCommMap_[iter.first]->RegOnMessageCallback(onMessage, inOper); + } + return E_OK; +} + +int CommunicatorProxy::RegOnConnectCallback(const OnConnectCallback &onConnect, const Finalizer &inOper) +{ + if (mainComm_ != nullptr) { + (void) mainComm_->RegOnConnectCallback(onConnect, inOper); + } + + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + (void) devCommMap_[iter.first]->RegOnConnectCallback(onConnect, inOper); + } + + return E_OK; +} + +int CommunicatorProxy::RegOnSendableCallback(const std::function &onSendable, const Finalizer &inOper) +{ + if (mainComm_ != nullptr) { + (void) mainComm_->RegOnSendableCallback(onSendable, inOper); + } + + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + (void) devCommMap_[iter.first]->RegOnSendableCallback(onSendable, inOper); + } + + return E_OK; +} + +void CommunicatorProxy::Activate() +{ + if (mainComm_ != nullptr) { + mainComm_->Activate(); + } + + // use temp map to avoid active in lock + std::map tempMap; + { + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + tempMap[iter.first] = devCommMap_[iter.first]; + RefObject::IncObjRef(devCommMap_[iter.first]); + } + } + + for (const auto &iter : tempMap) { + tempMap[iter.first]->Activate(); + RefObject::DecObjRef(tempMap[iter.first]); + } +} + +uint32_t CommunicatorProxy::GetCommunicatorMtuSize() const +{ + if (mainComm_ == nullptr) { + return DBConstant::MIN_MTU_SIZE; + } + return mainComm_->GetCommunicatorMtuSize(); +} + +uint32_t CommunicatorProxy::GetCommunicatorMtuSize(const std::string &target) const +{ + ICommunicator *targetCommunicator = nullptr; + { + std::lock_guard lock(devCommMapLock_); + if (devCommMap_.count(target) != 0) { + targetCommunicator = devCommMap_.at(target); + RefObject::IncObjRef(targetCommunicator); + } + } + if (targetCommunicator != nullptr) { + uint32_t mtuSize = targetCommunicator->GetCommunicatorMtuSize(target); + RefObject::DecObjRef(targetCommunicator); + return mtuSize; + } + + if (mainComm_ != nullptr) { + return mainComm_->GetCommunicatorMtuSize(target); + } + + return DBConstant::MIN_MTU_SIZE; +} + +uint32_t CommunicatorProxy::GetTimeout() const +{ + if (mainComm_ == nullptr) { + return DBConstant::MIN_TIMEOUT; + } + return mainComm_->GetTimeout(); +} + +uint32_t CommunicatorProxy::GetTimeout(const std::string &target) const +{ + ICommunicator *targetCommunicator = nullptr; + { + std::lock_guard lock(devCommMapLock_); + if (devCommMap_.count(target) != 0) { + targetCommunicator = devCommMap_.at(target); + RefObject::IncObjRef(targetCommunicator); + } + } + if (targetCommunicator != nullptr) { + uint32_t timeout = targetCommunicator->GetTimeout(target); + RefObject::DecObjRef(targetCommunicator); + return timeout; + } + + if (mainComm_ != nullptr) { + return mainComm_->GetTimeout(target); + } + + return DBConstant::MIN_TIMEOUT; +} + +int CommunicatorProxy::GetLocalIdentity(std::string &outTarget) const +{ + return mainComm_->GetLocalIdentity(outTarget); +} + +int CommunicatorProxy::GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const +{ + ICommunicator *targetCommunicator = nullptr; + { + std::lock_guard lock(devCommMapLock_); + if (devCommMap_.count(target) != 0) { + targetCommunicator = devCommMap_.at(target); + RefObject::IncObjRef(targetCommunicator); + } + } + if (targetCommunicator != nullptr) { + int errCode = targetCommunicator->GetRemoteCommunicatorVersion(target, outVersion); + RefObject::DecObjRef(targetCommunicator); + return errCode; + } + + if (mainComm_ != nullptr) { + return mainComm_->GetRemoteCommunicatorVersion(target, outVersion); + } + + return -E_NOT_INIT; +} + +int CommunicatorProxy::SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout) +{ + return SendMessage(dstTarget, inMsg, nonBlock, timeout, nullptr); +} + +int CommunicatorProxy::SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout, + const OnSendEnd &onEnd) +{ + ICommunicator *targetCommunicator = nullptr; + { + std::lock_guard lock(devCommMapLock_); + if (devCommMap_.count(dstTarget) != 0) { + targetCommunicator = devCommMap_[dstTarget]; + RefObject::IncObjRef(targetCommunicator); + } + } + if (targetCommunicator != nullptr) { + LOGI("[CommProxy] use equal label to send data"); + int errCode = targetCommunicator->SendMessage(dstTarget, inMsg, nonBlock, timeout, onEnd); + RefObject::DecObjRef(targetCommunicator); + return errCode; + } + + if (mainComm_ != nullptr) { + return mainComm_->SendMessage(dstTarget, inMsg, nonBlock, timeout, onEnd); + } + + return -E_NOT_INIT; +} + +void CommunicatorProxy::SetMainCommunicator(ICommunicator *communicator) +{ + mainComm_ = communicator; + RefObject::IncObjRef(mainComm_); +} + +void CommunicatorProxy::SetEqualCommunicator(ICommunicator *communicator, const std::vector &targets) +{ + std::lock_guard lock(devCommMapLock_); + // Clear offline target + for (auto dev = devCommMap_.begin(); dev != devCommMap_.end();) { + auto iter = std::find_if(targets.begin(), targets.end(), + [&dev](const std::string &target) { + return target == dev->first; + }); + if (iter == targets.end()) { + RefObject::DecObjRef(devCommMap_[dev->first]); + dev = devCommMap_.erase(dev); + continue; + } + dev++; + } + + // Add new online target + for (const auto &target : targets) { + if (devCommMap_.count(target) != 0) { + // change the identifier and dev relation + RefObject::DecObjRef(devCommMap_[target]); + } + RefObject::IncObjRef(communicator); + devCommMap_[target] = communicator; + } +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h new file mode 100644 index 000000000..3a0432027 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COMMUNICATOR_PROXY_H +#define COMMUNICATOR_PROXY_H + +#include +#include +#include +#include +#include +#include +#include "icommunicator.h" +#include "message.h" + +namespace DistributedDB { +class CommunicatorProxy : public ICommunicator { +public: + CommunicatorProxy(); + ~CommunicatorProxy(); + + int RegOnMessageCallback(const OnMessageCallback &onMessage, const Finalizer &inOper) override; + int RegOnConnectCallback(const OnConnectCallback &onConnect, const Finalizer &inOper) override; + int RegOnSendableCallback(const std::function &onSendable, const Finalizer &inOper) override; + void Activate() override; + uint32_t GetCommunicatorMtuSize() const override; + uint32_t GetCommunicatorMtuSize(const std::string &target) const override; + uint32_t GetTimeout() const override; + uint32_t GetTimeout(const std::string &target) const override; + int GetLocalIdentity(std::string &outTarget) const override; + int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const override; + int SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout) override; + int SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout, + const OnSendEnd &onEnd) override; + + // Set an Main communicator for this database, used userid & appId & storeId + void SetMainCommunicator(ICommunicator *communicator); + + // Set an equal communicator for this database, After this called, send msg to the target will use this communicator + void SetEqualCommunicator(ICommunicator *communicator, const std::vector &targets); + +private: + ICommunicator *mainComm_; + mutable std::mutex devCommMapLock_; + std::map devCommMap_; +}; +} // namespace DistributedDB +#endif // COMMUNICATOR_PROXY_H diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index 1c933f24d..7faa11442 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -15,6 +15,7 @@ #include "generic_syncer.h" +#include "db_common.h" #include "db_errno.h" #include "log_print.h" #include "ref_object.h" @@ -29,6 +30,7 @@ #include "device_manager.h" #include "db_constant.h" #include "ability_sync.h" +#include "single_ver_serialize_manager.h" namespace DistributedDB { const int GenericSyncer::MIN_VALID_SYNC_ID = 1; @@ -45,7 +47,6 @@ GenericSyncer::GenericSyncer() queuedManualSyncLimit_(DBConstant::QUEUED_SYNC_LIMIT_DEFAULT), manualSyncEnable_(true), closing_(false) - { } @@ -145,6 +146,17 @@ int GenericSyncer::Close() int GenericSyncer::Sync(const std::vector &devices, int mode, const std::function &)> &onComplete, const std::function &onFinalize, bool wait = false) +{ + SyncParma param; + param.devices = devices; + param.mode = mode; + param.onComplete = onComplete; + param.onFinalize = onFinalize; + param.wait = wait; + return Sync(param); +} + +int GenericSyncer::Sync(const SyncParma ¶m) { std::lock_guard lock(syncerLock_); if (!initialized_) { @@ -155,41 +167,45 @@ int GenericSyncer::Sync(const std::vector &devices, int mode, LOGE("[Syncer] Syncer is closing, return!"); return -E_BUSY; } - if (!IsValidDevices(devices) || !IsValidMode(mode)) { + if (!IsValidDevices(param.devices) || !IsValidMode(param.mode)) { return -E_INVALID_ARGS; } - if (IsQueuedManualSyncFull(mode, wait)) { + if (IsQueuedManualSyncFull(param.mode, param.wait)) { LOGE("[Syncer] -E_BUSY"); return -E_BUSY; } - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); + if (param.isQuerySync && !param.query.IsQueryOnlyByKey()) { + LOGE("[Syncer] query sync only support prefix key"); + return -E_NOT_SUPPORT; } + + PerformanceAnalysis::GetInstance()->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); uint32_t syncId = GenerateSyncId(); - LOGI("[Syncer] GenerateSyncId %d, mode = %d, wait = %d , label = %s, devicesNum = %d", syncId, mode, wait, - label_.c_str(), devices.size()); - SyncOperation *operation = new (std::nothrow) SyncOperation(syncId, devices, mode, onComplete, wait); + LOGI("[Syncer] GenerateSyncId %d, mode = %d, wait = %d , label = %s, devices = %s", syncId, param.mode, + param.wait, label_.c_str(), GetSyncDevicesStr(param.devices).c_str()); + SyncOperation *operation = + new (std::nothrow) SyncOperation(syncId, param.devices, param.mode, param.onComplete, param.wait); if (operation == nullptr) { - LOGE("[Syncer] SyncOperation alloc failed when sync called, may be out of memory"); return -E_OUT_OF_MEMORY; } + operation->Initialize(); operation->OnKill(std::bind(&GenericSyncer::SyncOperationKillCallback, this, operation->GetSyncId())); std::function onFinished = std::bind(&GenericSyncer::OnSyncFinished, this, std::placeholders::_1); operation->SetOnSyncFinished(onFinished); - operation->SetOnSyncFinalize(onFinalize); + operation->SetOnSyncFinalize(param.onFinalize); + if (param.isQuerySync) { + operation->SetQuery(param.query); + } int errCode = AddSyncOperation(operation); if (errCode != E_OK) { LOGE("[Syncer] AddSyncOperation failed when sync called, err %d", errCode); RefObject::KillAndDecObjRef(operation); return errCode; } - AddQueuedManualSyncSize(mode, wait); - LOGD("[Syncer] AddSyncOperation end"); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); - } + + AddQueuedManualSyncSize(param.mode, param.wait); + PerformanceAnalysis::GetInstance()->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); return syncId; } @@ -356,7 +372,7 @@ uint32_t GenericSyncer::GenerateSyncId() bool GenericSyncer::IsValidMode(int mode) const { - if ((mode > SyncOperation::AUTO_PULL) || (mode < SyncOperation::PUSH)) { + if ((mode >= SyncModeType::INVALID_MODE) || (mode < SyncModeType::PUSH)) { LOGE("[Syncer] Sync mode is not valid!"); return false; } @@ -409,7 +425,7 @@ int GenericSyncer::SyncResourceInit() LOGE("Register timesync message transform func ERR!"); return errCode; } - errCode = SingleVerDataSync::RegisterTransformFunc(); + errCode = SingleVerSerializeManager::RegisterTransformFunc(); if (errCode != E_OK) { LOGE("Register SingleVerDataSync message transform func ERR!"); return errCode; @@ -479,8 +495,9 @@ int GenericSyncer::GetQueuedSyncLimit(int *queuedSyncLimit) const void GenericSyncer::AddQueuedManualSyncSize(int mode, bool wait) { - if (((mode == SyncOperation::PULL) || (mode == SyncOperation::PUSH) || (mode == SyncOperation::PUSH_AND_PULL)) && - (wait == false)) { + int tmpMode = SyncOperation::TransferSyncMode(mode); + if (((tmpMode == SyncModeType::PULL) || (tmpMode == SyncModeType::PUSH) || (tmpMode == SyncModeType::PUSH_AND_PULL)) + && (wait == false)) { std::lock_guard lock(queuedManualSyncLock_); queuedManualSyncSize_++; } @@ -488,20 +505,22 @@ void GenericSyncer::AddQueuedManualSyncSize(int mode, bool wait) bool GenericSyncer::IsQueuedManualSyncFull(int mode, bool wait) const { + int tmpMode = SyncOperation::TransferSyncMode(mode); std::lock_guard lock(queuedManualSyncLock_); - if (((mode == SyncOperation::PULL) || (mode == SyncOperation::PUSH) || (mode == SyncOperation::PUSH_AND_PULL)) && - (manualSyncEnable_ == false)) { + if (((tmpMode == SyncModeType::PULL) || (tmpMode == SyncModeType::PUSH) || (tmpMode == SyncModeType::PUSH_AND_PULL)) + && (manualSyncEnable_ == false)) { LOGI("[GenericSyncer] manualSyncEnable_:false"); return true; } - if (((mode == SyncOperation::PULL) || (mode == SyncOperation::PUSH) || (mode == SyncOperation::PUSH_AND_PULL)) && - (wait == false)) { + if (((tmpMode == SyncModeType::PULL) || (tmpMode == SyncModeType::PUSH) || (tmpMode == SyncModeType::PUSH_AND_PULL)) + && (wait == false)) { if (queuedManualSyncSize_ < queuedManualSyncLimit_) { return false; + } else { + LOGD("[GenericSyncer] queuedManualSyncSize_:%d < queuedManualSyncLimit_:%d", queuedManualSyncSize_, + queuedManualSyncLimit_); + return true; } - LOGD("[GenericSyncer] queuedManualSyncSize_:%d < queuedManualSyncLimit_:%d", queuedManualSyncSize_, - queuedManualSyncLimit_); - return true; } return false; } @@ -578,4 +597,29 @@ void GenericSyncer::GetOnlineDevices(std::vector &devices) const syncEngine_->GetOnlineDevices(devices); } } + +int GenericSyncer::SetSyncRetry(bool isRetry) +{ + syncEngine_->SetSyncRetry(isRetry); + return E_OK; +} + +int GenericSyncer::SetEqualIdentifier(const std::string &identifier, const std::vector &targets) +{ + std::lock_guard lock(syncerLock_); + if (syncEngine_ == nullptr) { + return -E_NOT_INIT; + } + return syncEngine_->SetEqualIdentifier(identifier, targets); +} + +std::string GenericSyncer::GetSyncDevicesStr(const std::vector &devices) const +{ + std::string syncDevices; + for (const auto &dev:devices) { + syncDevices += STR_MASK(dev); + syncDevices += ","; + } + return syncDevices.substr(0, syncDevices.size() - 1); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h index c02195a5f..4036c3ccf 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h @@ -50,6 +50,9 @@ public: const std::function &)> &onComplete, const std::function &onFinalize, bool wait) override; + // Sync function. use SyncParma to reduce paramter. + int Sync(const SyncParma ¶m) override; + // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; @@ -74,6 +77,12 @@ public: // Get local deviceId, is hashed int GetLocalIdentity(std::string &outTarget) const override; + // Set Manual Sync retry config + int SetSyncRetry(bool isRetry) override; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) override; + protected: // Remote data changed callback virtual void RemoteDataChanged(const std::string &device) = 0; @@ -123,6 +132,8 @@ protected: void GetOnlineDevices(std::vector &devices) const; + std::string GetSyncDevicesStr(const std::vector &devices) const; + static int SyncModuleInit(); static int SyncResourceInit(); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp index 7d1da4efd..f1945186a 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp @@ -19,13 +19,14 @@ #include "securec.h" #include "db_errno.h" #include "db_common.h" +#include "hash.h" #include "log_print.h" +#include "sync_types.h" #include "time_helper.h" -#include "hash.h" namespace DistributedDB { namespace { - const int STR_TO_LL_BY_DEC = 10; + const int STR_TO_LL_BY_DEX = 10; // store local timeoffset;this is a special key; const std::string LOCALTIMEOFFSET_KEY = "localTimeOffset"; const std::string DEVICEID_PREFIX_KEY = "deviceId"; @@ -61,6 +62,7 @@ int Metadata::Initialize(IKvDBSyncInterface* storage) std::lock_guard lockGuard(metadataLock_); metadataMap_.clear(); } + (void)querySyncWaterMarkHelper_.Initialize(storage); return LoadAllMetadata(); } @@ -68,10 +70,10 @@ int Metadata::SaveTimeOffset(const DeviceID &deviceId, TimeOffset inValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); metadata.timeOffset = inValue; metadata.lastUpdateTime = TimeHelper::GetSysCurrentTime(); - LOGD("Metadata::SaveTimeOffset = %lld dev %s{private}", inValue, deviceId.c_str()); + LOGD("Metadata::SaveTimeOffset = %lld dev %s", inValue, STR_MASK(deviceId)); return SaveMetaDataValue(deviceId, metadata); } @@ -79,7 +81,7 @@ void Metadata::GetTimeOffset(const DeviceID &deviceId, TimeOffset &outValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); outValue = metadata.timeOffset; } @@ -87,7 +89,7 @@ void Metadata::GetLocalWaterMark(const DeviceID &deviceId, uint64_t &outValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); outValue = metadata.localWaterMark; } @@ -95,9 +97,9 @@ int Metadata::SaveLocalWaterMark(const DeviceID &deviceId, uint64_t inValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); metadata.localWaterMark = inValue; - LOGD("Metadata::SaveLocalWaterMark = %llu\n", inValue); + LOGD("Metadata::SaveLocalWaterMark = %llu", inValue); return SaveMetaDataValue(deviceId, metadata); } @@ -105,7 +107,7 @@ void Metadata::GetPeerWaterMark(const DeviceID &deviceId, uint64_t &outValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); outValue = metadata.peerWaterMark; } @@ -128,7 +130,7 @@ int Metadata::SaveLocalTimeOffset(TimeOffset timeOffset) std::lock_guard lockGuard(localTimeOffsetLock_); localTimeOffset_ = timeOffset; - LOGD("Metadata::SaveLocalTimeOffset offset = %lld\n", timeOffset); + LOGD("Metadata::SaveLocalTimeOffset offset = %lld", timeOffset); int errCode = SetMetadataToDb(localTimeOffsetValue, timeOffsetValue); if (errCode != E_OK) { LOGE("Metadata::SaveLocalTimeOffset SetMetadataToDb failed errCode:%d", errCode); @@ -138,12 +140,33 @@ int Metadata::SaveLocalTimeOffset(TimeOffset timeOffset) TimeOffset Metadata::GetLocalTimeOffset() const { - return localTimeOffset_.load(std::memory_order_seq_cst); + TimeOffset localTimeOffset = localTimeOffset_.load(std::memory_order_seq_cst); + return localTimeOffset; } int Metadata::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) { - return SavePeerWaterMark(deviceId, 0, isNeedHash); + // try to erase all the waterMark + // erase deleteSync recv waterMark + WaterMark waterMark = 0; + int errCodeDeleteSync = SetRecvDeleteSyncWaterMark(deviceId, waterMark); + // erase querySync recv waterMark + int errCodeQuerySync = ResetRecvQueryWaterMark(deviceId); + // peerWaterMark must be erased at last + int errCode = SavePeerWaterMark(deviceId, 0, isNeedHash); + if (errCode != E_OK) { + LOGE("[Metadata] erase peerWaterMark failed errCode:%d", errCode); + return errCode; + } + if (errCodeQuerySync != E_OK) { + LOGE("[Metadata] erase queryWaterMark failed errCode:%d", errCodeQuerySync); + return errCodeQuerySync; + } + if (errCodeDeleteSync != E_OK) { + LOGE("[Metadata] erase deleteWaterMark failed errCode:%d", errCodeDeleteSync); + return errCodeDeleteSync; + } + return E_OK; } void Metadata::SetLastLocalTime(TimeStamp lastLocalTime) @@ -169,12 +192,12 @@ int Metadata::SaveMetaDataValue(const DeviceID &deviceId, const MetaDataValue &i } DeviceID hashDeviceId; - GetHashDeviceId(deviceId, hashDeviceId); + GetHashDeviceId(deviceId, hashDeviceId, true); std::vector key; DBCommon::StringToVector(hashDeviceId, key); errCode = SetMetadataToDb(key, value); if (errCode != E_OK) { - LOGE("Metadata::SetMetadataToDb failed errCode:%d\n", errCode); + LOGE("Metadata::SetMetadataToDb failed errCode:%d", errCode); return errCode; } PutMetadataToMap(hashDeviceId, inValue); @@ -227,6 +250,14 @@ int Metadata::SetMetadataToDb(const std::vector &key, const std::vector return naturalStoragePtr_->PutMetaData(key, inValue); } +int Metadata::DeleteMetaDataFromDB(const std::vector &keys) const +{ + if (naturalStoragePtr_ == nullptr) { + return -E_INVALID_DB; + } + return naturalStoragePtr_->DeleteMetaData(keys); +} + void Metadata::PutMetadataToMap(const DeviceID &deviceId, const MetaDataValue &value) { metadataMap_[deviceId] = value; @@ -240,8 +271,8 @@ void Metadata::GetMetadataFromMap(const DeviceID &deviceId, MetaDataValue &outVa int64_t Metadata::StringToLong(const std::vector &value) { std::string valueString(value.begin(), value.end()); - int64_t longData = std::strtoll(valueString.c_str(), nullptr, STR_TO_LL_BY_DEC); - LOGD("Metadata::StringToLong longData = %lld\n", longData); + int64_t longData = std::strtoll(valueString.c_str(), nullptr, STR_TO_LL_BY_DEX); + LOGD("Metadata::StringToLong longData = %lld", longData); return longData; } @@ -253,33 +284,59 @@ int Metadata::GetAllMetadataKey(std::vector> &keys) return naturalStoragePtr_->GetAllMetaKeys(keys); } -int Metadata::LoadAllMetadata() +namespace { +bool IsMetaDataKey(const Key &inKey, const std::string &expectPrefix) { - std::vector> deviceIds; - std::vector value; - int errCode = E_OK; - GetAllMetadataKey(deviceIds); + if (inKey.size() < expectPrefix.size()) { + return false; + } + std::string prefixInKey(inKey.begin(), inKey.begin() + expectPrefix.size()); + if (prefixInKey != expectPrefix) { + return false; + } + return true; +} +} - for (auto it = deviceIds.begin(); it != deviceIds.end(); ++it) { - if (it->size() < DEVICEID_PREFIX_KEY.size()) { - continue; - } - std::string prefixKey(it->begin(), it->begin() + DEVICEID_PREFIX_KEY.size()); - if (prefixKey != DEVICEID_PREFIX_KEY) { - continue; - } - errCode = GetMetadataFromDb(*it, value); - if (errCode != E_OK) { - return errCode; - } - MetaDataValue metadata; - std::string deviceId(it->begin(), it->end()); - errCode = DeSerializeMetaData(value, metadata); - { - std::lock_guard lockGuard(metadataLock_); - PutMetadataToMap(deviceId, metadata); +int Metadata::LoadAllMetadata() +{ + std::vector> metaDataKeys; + GetAllMetadataKey(metaDataKeys); + + std::vector> querySyncIds; + for (const auto &deviceId : metaDataKeys) { + if (IsMetaDataKey(deviceId, DEVICEID_PREFIX_KEY)) { + int errCode = LoadDeviceIdDataToMap(deviceId); + if (errCode != E_OK) { + return errCode; + } + } else if (IsMetaDataKey(deviceId, QuerySyncWaterMarkHelper::GetQuerySyncPrefixKey())) { + querySyncIds.push_back(deviceId); + } else if (IsMetaDataKey(deviceId, QuerySyncWaterMarkHelper::GetDeleteSyncPrefixKey())) { + int errCode = querySyncWaterMarkHelper_.LoadDeleteSyncDataToCache(deviceId); + if (errCode != E_OK) { + return errCode; + } } } + return querySyncWaterMarkHelper_.RemoveLeastUsedQuerySyncItems(querySyncIds); +} + +int Metadata::LoadDeviceIdDataToMap(const Key &key) +{ + std::vector value; + int errCode = GetMetadataFromDb(key, value); + if (errCode != E_OK) { + return errCode; + } + MetaDataValue metaValue; + std::string metaKey(key.begin(), key.end()); + errCode = DeSerializeMetaData(value, metaValue); + if (errCode != E_OK) { + return errCode; + } + std::lock_guard lockGuard(metadataLock_); + PutMetadataToMap(metaKey, metaValue); return errCode; } @@ -309,4 +366,167 @@ void Metadata::GetHashDeviceId(const DeviceID &deviceId, DeviceID &hashDeviceId, hashDeviceId = deviceIdToHashDeviceIdMap_[deviceId]; } } -} // namespace DistributedDB + +int Metadata::GetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, WaterMark &waterMark) +{ + QueryWaterMark queryWaterMark; + int errCode = querySyncWaterMarkHelper_.GetQueryWaterMark(queryIdentify, deviceId, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + WaterMark peerWaterMark; + GetPeerWaterMark(deviceId, peerWaterMark); + waterMark = std::max(queryWaterMark.recvWaterMark, peerWaterMark); + return E_OK; +} + +int Metadata::SetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark) +{ + return querySyncWaterMarkHelper_.SetRecvQueryWaterMark(queryIdentify, deviceId, waterMark); +} + +int Metadata::GetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, WaterMark &waterMark, bool isAutoLift) +{ + QueryWaterMark queryWaterMark; + int errCode = querySyncWaterMarkHelper_.GetQueryWaterMark(queryIdentify, deviceId, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + if (isAutoLift) { + WaterMark localWaterMark; + GetLocalWaterMark(deviceId, localWaterMark); + waterMark = std::max(queryWaterMark.sendWaterMark, localWaterMark); + } else { + waterMark = queryWaterMark.sendWaterMark; + } + return E_OK; +} + +int Metadata::SetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark) +{ + return querySyncWaterMarkHelper_.SetSendQueryWaterMark(queryIdentify, deviceId, waterMark); +} + +int Metadata::GetSendDeleteSyncWaterMark(const DeviceID &deviceId, WaterMark &waterMark, bool isAutoLift) +{ + DeleteWaterMark deleteWaterMark; + int errCode = querySyncWaterMarkHelper_.GetDeleteSyncWaterMark(deviceId, deleteWaterMark); + if (errCode != E_OK) { + return errCode; + } + if (isAutoLift) { + WaterMark localWaterMark; + GetLocalWaterMark(deviceId, localWaterMark); + waterMark = std::max(deleteWaterMark.sendWaterMark, localWaterMark); + } else { + waterMark = deleteWaterMark.sendWaterMark; + } + return E_OK; +} + +int Metadata::SetSendDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMark &waterMark) +{ + return querySyncWaterMarkHelper_.SetSendDeleteSyncWaterMark(deviceId, waterMark); +} + +int Metadata::GetRecvDeleteSyncWaterMark(const DeviceID &deviceId, WaterMark &waterMark) +{ + DeleteWaterMark deleteWaterMark; + int errCode = querySyncWaterMarkHelper_.GetDeleteSyncWaterMark(deviceId, deleteWaterMark); + if (errCode != E_OK) { + return errCode; + } + WaterMark peerWaterMark; + GetPeerWaterMark(deviceId, peerWaterMark); + waterMark = std::max(deleteWaterMark.recvWaterMark, peerWaterMark); + return E_OK; +} + +int Metadata::SetRecvDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMark &waterMark) +{ + return querySyncWaterMarkHelper_.SetRecvDeleteSyncWaterMark(deviceId, waterMark); +} + +int Metadata::ResetRecvQueryWaterMark(const DeviceID &deviceId) +{ + return querySyncWaterMarkHelper_.ResetRecvQueryWaterMark(deviceId); +} + +void Metadata::GetDbCreateTime(const DeviceID &deviceId, uint64_t &outValue) +{ + MetaDataValue metadata; + std::lock_guard lockGuard(metadataLock_); + DeviceID hashDeviceId; + GetHashDeviceId(deviceId, hashDeviceId, true); + if (metadataMap_.find(hashDeviceId) != metadataMap_.end()) { + metadata = metadataMap_[hashDeviceId]; + outValue = metadata.dbCreateTime; + return; + } + outValue = 0; + LOGI("Metadata::GetDbCreateTime, not found dev = %s dbCreateTime", STR_MASK(deviceId)); +} + +int Metadata::SetDbCreateTime(const DeviceID &deviceId, uint64_t inValue, bool isNeedHash) +{ + MetaDataValue metadata; + std::lock_guard lockGuard(metadataLock_); + DeviceID hashDeviceId; + GetHashDeviceId(deviceId, hashDeviceId, true); + if (metadataMap_.find(hashDeviceId) != metadataMap_.end()) { + metadata = metadataMap_[hashDeviceId]; + if (metadata.dbCreateTime != 0 && metadata.dbCreateTime != inValue) { + metadata.clearDeviceDataMark = REMOVE_DEVICE_DATA_MARK; + LOGI("Metadata::SetDbCreateTime,set cleardata mark,dev=%s,dbCreateTime=%llu", STR_MASK(deviceId), inValue); + } + if (metadata.dbCreateTime == 0) { + LOGI("Metadata::SetDbCreateTime,update dev=%s,dbCreateTime=%llu", STR_MASK(deviceId), inValue); + } + } + metadata.dbCreateTime = inValue; + return SaveMetaDataValue(deviceId, metadata); +} + +int Metadata::ResetMetaDataAfterRemoveData(const DeviceID &deviceId) +{ + MetaDataValue metadata; + std::lock_guard lockGuard(metadataLock_); + DeviceID hashDeviceId; + GetHashDeviceId(deviceId, hashDeviceId, true); + if (metadataMap_.find(hashDeviceId) != metadataMap_.end()) { + metadata = metadataMap_[hashDeviceId]; + metadata.clearDeviceDataMark = 0; + // when finished remove data, should reset send watermark between high version sync + // recv watermark has already reset in removedata func. + metadata.localWaterMark = 0; + return SaveMetaDataValue(deviceId, metadata); + } + return -E_NOT_FOUND; +} + +void Metadata::GetRemoveDataMark(const DeviceID &deviceId, uint64_t &outValue) +{ + MetaDataValue metadata; + std::lock_guard lockGuard(metadataLock_); + DeviceID hashDeviceId; + GetHashDeviceId(deviceId, hashDeviceId, true); + if (metadataMap_.find(hashDeviceId) != metadataMap_.end()) { + metadata = metadataMap_[hashDeviceId]; + outValue = metadata.clearDeviceDataMark; + return; + } + outValue = 0; +} + +int Metadata::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + if (naturalStoragePtr_ == nullptr) { + return -E_INVALID_DB; + } + return naturalStoragePtr_->DeleteMetaDataByPrefixKey(keyPrefix); +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.cpp index ba3231deb..1fb8d4e07 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.cpp @@ -102,7 +102,7 @@ void MultiVerSyncStateMachine::StepToIdle() StopWatchDog(); context_->Clear(); PerformanceAnalysis::GetInstance()->TimeRecordEnd(); - LOGD("[MultiVerSyncStateMachine][%s{private}] step to idle", context_->GetDeviceId().c_str()); + LOGD("[MultiVerSyncStateMachine][%s] step to idle", STR_MASK(context_->GetDeviceId())); } int MultiVerSyncStateMachine::MessageCallbackCheck(const Message *inMsg) @@ -195,8 +195,7 @@ void MultiVerSyncStateMachine::SyncStepInnerLocked() goto SYNC_STEP_OUT; } - LOGD("[MultiVerSyncStateMachine] SyncStep dst=%s{private}, state = %d", - context_->GetDeviceId().c_str(), currentState_); + LOGD("[MultiVerSyncStateMachine] SyncStep dst=%s, state = %d", STR_MASK(context_->GetDeviceId()), currentState_); int errCode; { std::lock_guard lock(stateMachineLock_); @@ -279,7 +278,7 @@ int MultiVerSyncStateMachine::PrepareNextSyncTask() return StartSyncInner(); } -void MultiVerSyncStateMachine::SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) +void MultiVerSyncStateMachine::SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) { } @@ -402,14 +401,14 @@ void MultiVerSyncStateMachine::Finish() if (commitsSize > 0) { context_->GetCommit(commitsSize - 1, commit); context_->GetCommits(commits); - LOGD("MultiVerSyncStateMachine::Finish merge src=%s{private}", context_->GetDeviceId().c_str()); + LOGD("MultiVerSyncStateMachine::Finish merge src=%s", STR_MASK(context_->GetDeviceId())); PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { performance->StepTimeRecordStart(MV_TEST_RECORDS::RECORD_MERGE); } int errCode = multiVerDataSync_->MergeSyncCommit(commit, commits); - LOGD("MultiVerSyncStateMachine::Finish merge src=%s{private}, MergeSyncCommit errCode:%d", - context_->GetDeviceId().c_str(), errCode); + LOGD("MultiVerSyncStateMachine::Finish merge src=%s, MergeSyncCommit errCode:%d", + STR_MASK(context_->GetDeviceId()), errCode); if (performance != nullptr) { performance->StepTimeRecordEnd(MV_TEST_RECORDS::RECORD_MERGE); } @@ -429,15 +428,15 @@ int MultiVerSyncStateMachine::OneCommitSyncFinish() int errCode = E_OK; int commitIndex = context_->GetCommitIndex(); - LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s{private}, commitIndex = %d,", - context_->GetDeviceId().c_str(), commitIndex); + LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s, commitIndex = %d,", STR_MASK(context_->GetDeviceId()), + commitIndex); if (commitIndex > 0) { context_->GetCommit(commitIndex - 1, commit); deviceName = context_->GetDeviceId(); context_->GetEntries(entries); - LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s{private}, entries size = %lu", - context_->GetDeviceId().c_str(), entries.size()); - errCode = timeSync_->GetTimeOffset(outOffset); + LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s, entries size = %lu", + STR_MASK(context_->GetDeviceId()), entries.size()); + errCode = timeSync_->GetTimeOffset(outOffset, TIME_SYNC_WAIT_TIME); if (errCode != E_OK) { LOGI("MultiVerSyncStateMachine::OneCommitSyncFinish GetTimeOffset fail errCode:%d", errCode); return errCode; @@ -447,8 +446,8 @@ int MultiVerSyncStateMachine::OneCommitSyncFinish() // Due to time sync error, commit timestamp may bigger than currentLocalTime, we need to fix the timestamp TimeOffset timefixOffset = (commit.timestamp < currentLocalTime) ? 0 : (commit.timestamp - currentLocalTime); - LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s{private}, timefixOffset = %lld", - context_->GetDeviceId().c_str(), timefixOffset); + LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s, timefixOffset = %lld", + STR_MASK(context_->GetDeviceId()), timefixOffset); commit.timestamp -= timefixOffset; for (MultiVerKvEntry *entry : entries) { TimeStamp timeStamp; @@ -461,8 +460,8 @@ int MultiVerSyncStateMachine::OneCommitSyncFinish() performance->StepTimeRecordStart(MV_TEST_RECORDS::RECORD_PUT_COMMIT_DATA); } errCode = multiVerDataSync_->PutCommitData(commit, entries, deviceName); - LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish PutCommitData src=%s{private}, errCode = %d", - context_->GetDeviceId().c_str(), errCode); + LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish PutCommitData src=%s, errCode = %d", + STR_MASK(context_->GetDeviceId()), errCode); if (performance != nullptr) { performance->StepTimeRecordEnd(MV_TEST_RECORDS::RECORD_PUT_COMMIT_DATA); } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.h index 312e34730..8f8a3ce8c 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.h @@ -72,7 +72,7 @@ protected: int PrepareNextSyncTask() override; // Called by StartSaveDataNotifyTimer, used to send a save data notify packet - void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) override; + void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) override; private: enum State { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp index 94b1a8d26..5ac04dce0 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp @@ -52,7 +52,7 @@ void MultiVerSyncer::EnableAutoSync(bool enable) return; } - int syncId = Sync(devices, SyncOperation::AUTO_PULL, nullptr, nullptr, false); + int syncId = Sync(devices, SyncModeType::AUTO_PULL, nullptr, nullptr, false); if (syncId < MIN_VALID_SYNC_ID) { LOGE("[Syncer] sync start by EnableAutoSync failed err %d", syncId); } @@ -83,7 +83,7 @@ void MultiVerSyncer::RemoteDataChanged(const std::string &device) if (autoSyncEnable_) { std::vector devices; devices.push_back(device); - int syncId = Sync(devices, SyncOperation::AUTO_PULL, nullptr, nullptr, false); + int syncId = Sync(devices, SyncModeType::AUTO_PULL, nullptr, nullptr, false); if (syncId < MIN_VALID_SYNC_ID) { LOGE("[MultiVerSyncer] sync start by RemoteDataChanged failed err %d", syncId); } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp new file mode 100644 index 000000000..1568bc9ce --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp @@ -0,0 +1,569 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "query_sync_water_mark_helper.h" + +#include +#include +#include "platform_specific.h" +#include "parcel.h" +#include "db_errno.h" +#include "db_common.h" +#include "log_print.h" + +namespace DistributedDB { +namespace { + const int MAX_CACHE_ITEMS = 200; + const uint32_t MAX_STORE_ITEMS = 100000; + // WaterMark Version + constexpr uint32_t QUERY_WATERMARK_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_3_0; + constexpr uint32_t DELETE_WATERMARK_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_3_0; + // Prefix Key in db + const std::string QUERY_SYNC_PREFIX_KEY = "querySync"; + const std::string DELETE_SYNC_PREFIX_KEY = "deleteSync"; +} + +QuerySyncWaterMarkHelper::QuerySyncWaterMarkHelper() + : storage_(nullptr) +{} + +QuerySyncWaterMarkHelper::~QuerySyncWaterMarkHelper() +{ + storage_ = nullptr; + deviceIdToHashQuerySyncIdMap_.clear(); + deleteSyncCache_.clear(); + deviceIdToHashDeleteSyncIdMap_.clear(); +} + +int LruMap::Put(const std::string &key, const QueryWaterMark &inValue) +{ + std::lock_guard autoLock(lruLock_); + cache_[key] = inValue; + return Elimination(key, inValue); +} + +int LruMap::Get(const std::string &key, QueryWaterMark &outValue) +{ + std::lock_guard autoLock(lruLock_); + if (cache_.count(key) == 0) { + return -E_NOT_FOUND; + } + outValue = cache_[key]; + return Elimination(key, outValue); +} + +void LruMap::RemoveWithPrefixKey(const std::string &prefixKey) +{ + std::lock_guard autoLock(lruLock_); + auto iterator = eliminationChain_.begin(); + while (iterator != eliminationChain_.end()) { + const std::string &key = (*iterator).first; + if (key.find(prefixKey) == 0) { + (void)cache_.erase(key); + iterator = eliminationChain_.erase(iterator); + } else { + iterator++; + } + } +} + +// move the node to last and remove the first node until the size less than limit +int LruMap::Elimination(const std::string &key, const QueryWaterMark &inQueryWaterMark) +{ + auto iterator = eliminationChain_.begin(); + while (iterator != eliminationChain_.end()) { + if ((*iterator).first == key) { + eliminationChain_.erase(iterator); + break; + } + iterator++; + } + std::pair entry = {key, inQueryWaterMark}; + eliminationChain_.push_back(entry); + while (eliminationChain_.size() > MAX_CACHE_ITEMS) { + std::pair &pair = eliminationChain_.front(); + cache_.erase(pair.first); + eliminationChain_.pop_front(); + } + return E_OK; +} + +int QuerySyncWaterMarkHelper::GetMetadataFromDb(const std::vector &key, std::vector &outValue) +{ + if (storage_ == nullptr) { + return -E_INVALID_DB; + } + return storage_->GetMetaData(key, outValue); +} + +int QuerySyncWaterMarkHelper::SetMetadataToDb(const std::vector &key, const std::vector &inValue) +{ + if (storage_ == nullptr) { + return -E_INVALID_DB; + } + return storage_->PutMetaData(key, inValue); +} + +int QuerySyncWaterMarkHelper::DeleteMetaDataFromDB(const std::vector &keys) const +{ + if (storage_ == nullptr) { + return -E_INVALID_DB; + } + return storage_->DeleteMetaData(keys); +} + +int QuerySyncWaterMarkHelper::Initialize(IKvDBSyncInterface *storage) +{ + storage_ = storage; + return E_OK; +} + +int QuerySyncWaterMarkHelper::LoadDeleteSyncDataToCache(const Key &deleteWaterMarkKey) +{ + std::vector value; + int errCode = GetMetadataFromDb(deleteWaterMarkKey, value); + if (errCode != E_OK) { + return errCode; + } + DeleteWaterMark deleteWaterMark; + std::string dbKey(deleteWaterMarkKey.begin(), deleteWaterMarkKey.end()); + errCode = DeSerializeDeleteWaterMark(value, deleteWaterMark); + if (errCode != E_OK) { + return errCode; + } + std::lock_guard autoLock(deleteSyncLock_); + deleteSyncCache_[dbKey] = deleteWaterMark; + return errCode; +} + +int QuerySyncWaterMarkHelper::GetQueryWaterMarkInCacheAndDb(const std::string &cacheKey, + QueryWaterMark &queryWaterMark) +{ + // first get from cache_ + int errCode = querySyncCache_.Get(cacheKey, queryWaterMark); + bool addToCache = false; + if (errCode == -E_NOT_FOUND) { + // second get from db + errCode = GetQueryWaterMarkFromDB(cacheKey, queryWaterMark); + addToCache = true; + } + if (errCode == -E_NOT_FOUND) { + // third generate one and save to db + errCode = PutQueryWaterMarkToDB(cacheKey, queryWaterMark); + } + // something error return + if (errCode != E_OK) { + LOGE("[Meta]GetQueryWaterMark Fail code = %d", errCode); + return errCode; + } + // remember add to cache_ + if (addToCache) { + querySyncCache_.Put(cacheKey, queryWaterMark); + } + return errCode; +} + +int QuerySyncWaterMarkHelper::GetQueryWaterMark(const std::string &queryIdentify, const std::string &deviceId, + QueryWaterMark &queryWaterMark) +{ + std::string cacheKey; + GetHashQuerySyncDeviceId(deviceId, queryIdentify, cacheKey); + std::lock_guard autoLock(queryWaterMarkLock_); + return GetQueryWaterMarkInCacheAndDb(cacheKey, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::SetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark) +{ + std::string cacheKey; + GetHashQuerySyncDeviceId(deviceId, queryIdentify, cacheKey); + std::lock_guard autoLock(queryWaterMarkLock_); + return SetRecvQueryWaterMarkWithoutLock(deviceId, cacheKey, waterMark); +} + +int QuerySyncWaterMarkHelper::SetRecvQueryWaterMarkWithoutLock(const std::string &deviceId, + const std::string &cacheKey, const WaterMark &waterMark) +{ + QueryWaterMark queryWaterMark; + int errCode = GetQueryWaterMarkInCacheAndDb(cacheKey, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + queryWaterMark.recvWaterMark = waterMark; + return UpdateCacheAndSave(cacheKey, deviceId, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::SetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark) +{ + std::string cacheKey; + GetHashQuerySyncDeviceId(deviceId, queryIdentify, cacheKey); + QueryWaterMark queryWaterMark; + std::lock_guard autoLock(queryWaterMarkLock_); + int errCode = GetQueryWaterMarkInCacheAndDb(cacheKey, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + queryWaterMark.sendWaterMark = waterMark; + return UpdateCacheAndSave(cacheKey, deviceId, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::UpdateCacheAndSave(const std::string &cacheKey, + const std::string &deviceId, QueryWaterMark &queryWaterMark) +{ + // update lastUsedTime + int errCode = OS::GetCurrentSysTimeInMicrosecond(queryWaterMark.lastUsedTime); + if (errCode != E_OK) { + return errCode; + } + // save db first + errCode = SaveQueryWaterMarkToDB(cacheKey, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + querySyncCache_.Put(cacheKey, queryWaterMark); + return errCode; +} + +int QuerySyncWaterMarkHelper::PutQueryWaterMarkToDB(const DeviceID &dbKeyString, QueryWaterMark &queryWaterMark) +{ + int errCode = OS::GetCurrentSysTimeInMicrosecond(queryWaterMark.lastUsedTime); + if (errCode != E_OK) { + return errCode; + } + queryWaterMark.version = QUERY_WATERMARK_VERSION_CURRENT; + return SaveQueryWaterMarkToDB(dbKeyString, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::SaveQueryWaterMarkToDB(const DeviceID &dbKeyString, const QueryWaterMark &queryWaterMark) +{ + // serialize value + Value dbValue; + int errCode = SerializeQueryWaterMark(queryWaterMark, dbValue); + if (errCode != E_OK) { + return errCode; + } + // serialize key + Key dbKey; + DBCommon::StringToVector(dbKeyString, dbKey); + // save + errCode = SetMetadataToDb(dbKey, dbValue); + if (errCode != E_OK) { + LOGE("QuerySyncWaterMarkHelper::SaveQueryWaterMarkToDB failed errCode:%d", errCode); + } + return errCode; +} + +int QuerySyncWaterMarkHelper::GetQueryWaterMarkFromDB(const DeviceID &dbKeyString, QueryWaterMark &queryWaterMark) +{ + // serialize key + Key dbKey; + DBCommon::StringToVector(dbKeyString, dbKey); + // search in db + Value dbValue; + int errCode = GetMetadataFromDb(dbKey, dbValue); + if (errCode != E_OK) { + return errCode; + } + return DeSerializeQueryWaterMark(dbValue, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::SerializeQueryWaterMark(const QueryWaterMark &queryWaterMark, Value &outValue) +{ + uint64_t length = CalculateQueryWaterMarkSize(queryWaterMark); + outValue.resize(length); + Parcel parcel(outValue.data(), outValue.size()); + parcel.WriteUInt32(queryWaterMark.version); + parcel.EightByteAlign(); + parcel.WriteUInt64(queryWaterMark.sendWaterMark); + parcel.WriteUInt64(queryWaterMark.recvWaterMark); + parcel.WriteUInt64(queryWaterMark.lastUsedTime); + parcel.WriteString(queryWaterMark.sql); + if (parcel.IsError()) { + LOGE("[Meta] Parcel error when serialize queryWaterMark"); + return -E_PARSE_FAIL; + } + return E_OK; +} + +int QuerySyncWaterMarkHelper::DeSerializeQueryWaterMark(const Value &dbQueryWaterMark, QueryWaterMark &queryWaterMark) +{ + Parcel parcel(const_cast(dbQueryWaterMark.data()), dbQueryWaterMark.size()); + parcel.ReadUInt32(queryWaterMark.version); + parcel.EightByteAlign(); + parcel.ReadUInt64(queryWaterMark.sendWaterMark); + parcel.ReadUInt64(queryWaterMark.recvWaterMark); + parcel.ReadUInt64(queryWaterMark.lastUsedTime); + parcel.ReadString(queryWaterMark.sql); + if (parcel.IsError()) { + LOGE("[Meta] Parcel error when deserialize queryWaterMark"); + return -E_PARSE_FAIL; + } + return E_OK; +} + +uint64_t QuerySyncWaterMarkHelper::CalculateQueryWaterMarkSize(const QueryWaterMark &queryWaterMark) +{ + uint64_t length = Parcel::GetUInt32Len(); // version + length = Parcel::GetEightByteAlign(length); + length += Parcel::GetUInt64Len(); // sendWaterMark + length += Parcel::GetUInt64Len(); // recvWaterMark + length += Parcel::GetUInt64Len(); // lastUsedTime + length += Parcel::GetStringLen(queryWaterMark.sql); + return length; +} + +void QuerySyncWaterMarkHelper::GetHashQuerySyncDeviceId(const DeviceID &deviceId, + const DeviceID &queryId, DeviceID &hashQuerySyncId) +{ + std::lock_guard autoLock(queryWaterMarkLock_); + if (deviceIdToHashQuerySyncIdMap_[deviceId].count(queryId) == 0) { + // do not modify this + hashQuerySyncId = QUERY_SYNC_PREFIX_KEY + DBCommon::TransferHashString(deviceId) + queryId; + deviceIdToHashQuerySyncIdMap_[deviceId][queryId] = hashQuerySyncId; + } else { + hashQuerySyncId = deviceIdToHashQuerySyncIdMap_[deviceId][queryId]; + } +} + +int QuerySyncWaterMarkHelper::GetDeleteSyncWaterMark(const std::string &deviceId, DeleteWaterMark &deleteWaterMark) +{ + std::string hashId; + GetHashDeleteSyncDeviceId(deviceId, hashId); + return GetDeleteWaterMarkFromCache(hashId, deleteWaterMark); +} + +int QuerySyncWaterMarkHelper::SetSendDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMark &waterMark) +{ + std::string hashId; + GetHashDeleteSyncDeviceId(deviceId, hashId); + DeleteWaterMark deleteWaterMark; + GetDeleteWaterMarkFromCache(hashId, deleteWaterMark); + deleteWaterMark.sendWaterMark = waterMark; + std::lock_guard autoLock(deleteSyncLock_); + return UpdateDeleteSyncCacheAndSave(hashId, deleteWaterMark); +} + +int QuerySyncWaterMarkHelper::SetRecvDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMark &waterMark) +{ + std::string hashId; + GetHashDeleteSyncDeviceId(deviceId, hashId); + DeleteWaterMark deleteWaterMark; + GetDeleteWaterMarkFromCache(hashId, deleteWaterMark); + deleteWaterMark.recvWaterMark = waterMark; + std::lock_guard autoLock(deleteSyncLock_); + return UpdateDeleteSyncCacheAndSave(hashId, deleteWaterMark); +} + +int QuerySyncWaterMarkHelper::UpdateDeleteSyncCacheAndSave(const std::string &dbKey, + const DeleteWaterMark &deleteWaterMark) +{ + // save db first + int errCode = SaveDeleteWaterMarkToDB(dbKey, deleteWaterMark); + if (errCode != E_OK) { + return errCode; + } + // modify cache + deleteSyncCache_[dbKey] = deleteWaterMark; + return errCode; +} + +int QuerySyncWaterMarkHelper::GetDeleteWaterMarkFromCache(const DeviceID &hashDeviceId, + DeleteWaterMark &deleteWaterMark) +{ + // lock prevent different thread visit deleteSyncCache_ + std::lock_guard autoLock(deleteSyncLock_); + // if not found + if (deleteSyncCache_.find(hashDeviceId) == deleteSyncCache_.end()) { + DeleteWaterMark waterMark; + waterMark.version = DELETE_WATERMARK_VERSION_CURRENT; + int errCode = GetDeleteWaterMarkFromDB(hashDeviceId, waterMark); + if (errCode == -E_NOT_FOUND) { + deleteWaterMark.sendWaterMark = 0; + deleteWaterMark.recvWaterMark = 0; + errCode = E_OK; + } + if (errCode != E_OK) { + LOGE("[Meta]GetDeleteWaterMark Fail code = %d", errCode); + return errCode; + } + deleteSyncCache_.insert(std::pair(hashDeviceId, waterMark)); + } + deleteWaterMark = deleteSyncCache_[hashDeviceId]; + return E_OK; +} + +int QuerySyncWaterMarkHelper::GetDeleteWaterMarkFromDB(const DeviceID &hashDeviceId, + DeleteWaterMark &deleteWaterMark) +{ + Key dbKey; + DBCommon::StringToVector(hashDeviceId, dbKey); + // search in db + Value dbValue; + int errCode = GetMetadataFromDb(dbKey, dbValue); + if (errCode != E_OK) { + return errCode; + } + // serialize value + return DeSerializeDeleteWaterMark(dbValue, deleteWaterMark); +} + +int QuerySyncWaterMarkHelper::SaveDeleteWaterMarkToDB(const DeviceID &hashDeviceId, + const DeleteWaterMark &deleteWaterMark) +{ + // serialize value + Value dbValue; + int errCode = SerializeDeleteWaterMark(deleteWaterMark, dbValue); + if (errCode != E_OK) { + return errCode; + } + Key dbKey; + DBCommon::StringToVector(hashDeviceId, dbKey); + // save + errCode = SetMetadataToDb(dbKey, dbValue); + if (errCode != E_OK) { + LOGE("QuerySyncWaterMarkHelper::SaveDeleteWaterMarkToDB failed errCode:%d", errCode); + } + return errCode; +} + +void QuerySyncWaterMarkHelper::GetHashDeleteSyncDeviceId(const DeviceID &deviceId, DeviceID &hashDeleteSyncId) +{ + std::lock_guard autoLock(deleteSyncLock_); + if (deviceIdToHashDeleteSyncIdMap_.count(deviceId) == 0) { + hashDeleteSyncId = DELETE_SYNC_PREFIX_KEY + DBCommon::TransferHashString(deviceId); + deviceIdToHashDeleteSyncIdMap_.insert(std::pair(deviceId, hashDeleteSyncId)); + } else { + hashDeleteSyncId = deviceIdToHashDeleteSyncIdMap_[deviceId]; + } +} + +int QuerySyncWaterMarkHelper::SerializeDeleteWaterMark(const DeleteWaterMark &deleteWaterMark, + std::vector &outValue) +{ + uint64_t length = CalculateDeleteWaterMarkSize(deleteWaterMark); + outValue.resize(length); + Parcel parcel(outValue.data(), outValue.size()); + parcel.WriteUInt32(deleteWaterMark.version); + parcel.EightByteAlign(); + parcel.WriteUInt64(deleteWaterMark.sendWaterMark); + parcel.WriteUInt64(deleteWaterMark.recvWaterMark); + if (parcel.IsError()) { + LOGE("[Meta] Parcel error when serialize deleteWaterMark."); + return -E_PARSE_FAIL; + } + return E_OK; +} + +int QuerySyncWaterMarkHelper::DeSerializeDeleteWaterMark(const std::vector &inValue, + DeleteWaterMark &deleteWaterMark) +{ + Parcel parcel(const_cast(inValue.data()), inValue.size()); + parcel.ReadUInt32(deleteWaterMark.version); + parcel.EightByteAlign(); + parcel.ReadUInt64(deleteWaterMark.sendWaterMark); + parcel.ReadUInt64(deleteWaterMark.recvWaterMark); + if (parcel.IsError()) { + LOGE("[Meta] Parcel error when deserialize deleteWaterMark."); + return -E_PARSE_FAIL; + } + return E_OK; +} + +uint64_t QuerySyncWaterMarkHelper::CalculateDeleteWaterMarkSize(const DeleteWaterMark &deleteWaterMark) +{ + uint64_t length = Parcel::GetUInt32Len(); // version + length = Parcel::GetEightByteAlign(length); + length += Parcel::GetUInt64Len(); // sendWaterMark + length += Parcel::GetUInt64Len(); // recvWaterMark + return length; +} + +std::string QuerySyncWaterMarkHelper::GetQuerySyncPrefixKey() +{ + return QUERY_SYNC_PREFIX_KEY; +} + +std::string QuerySyncWaterMarkHelper::GetDeleteSyncPrefixKey() +{ + return DELETE_SYNC_PREFIX_KEY; +} + +int QuerySyncWaterMarkHelper::RemoveLeastUsedQuerySyncItems(const std::vector &querySyncIds) +{ + if (querySyncIds.size() < MAX_STORE_ITEMS) { + return E_OK; + } + std::vector> allItems; + std::map> idMap; + std::vector> waitToRemove; + for (const auto &id : querySyncIds) { + Value value; + int errCode = GetMetadataFromDb(id, value); + if (errCode != E_OK) { + waitToRemove.push_back(id); + continue; // may be this failure cause by wrong data + } + QueryWaterMark queryWaterMark; + std::string queryKey(id.begin(), id.end()); + errCode = DeSerializeQueryWaterMark(value, queryWaterMark); + if (errCode != E_OK) { + waitToRemove.push_back(id); + continue; // may be this failure cause by wrong data + } + idMap.insert({queryKey, id}); + allItems.emplace_back(queryKey, queryWaterMark.lastUsedTime); + } + // we only remove broken data below + // 1. common data size less then 10w + // 2. allItems.size() - MAX_STORE_ITEMS - waitToRemove.size() < 0 + // so we only let allItems.size() < MAX_STORE_ITEMS + waitToRemove.size() + if (allItems.size() < MAX_STORE_ITEMS + waitToRemove.size()) { + // remove in db + return DeleteMetaDataFromDB(waitToRemove); + } + uint32_t removeCount = allItems.size() - MAX_STORE_ITEMS - waitToRemove.size(); + // quick select the k_th least used + std::nth_element(allItems.begin(), allItems.begin() + removeCount, allItems.end(), + [](std::pair &w1, std::pair &w2) { + return w1.second < w2.second; + }); + for (uint32_t i = 0; i < removeCount; ++i) { + waitToRemove.push_back(idMap[allItems[i].first]); + } + // remove in db + return DeleteMetaDataFromDB(waitToRemove); +} + +int QuerySyncWaterMarkHelper::ResetRecvQueryWaterMark(const DeviceID &deviceId) +{ + // lock prevent other thread modify queryWaterMark at this moment + { + std::lock_guard autoLock(queryWaterMarkLock_); + std::string prefixKeyStr = QUERY_SYNC_PREFIX_KEY + DBCommon::TransferHashString(deviceId); + // remove in db + Key prefixKey; + DBCommon::StringToVector(prefixKeyStr, prefixKey); + int errCode = storage_->DeleteMetaDataByPrefixKey(prefixKey); + if (errCode != E_OK) { + LOGE("[META]ResetRecvQueryWaterMark fail errCode:%d", errCode); + return errCode; + } + // clean cache + querySyncCache_.RemoveWithPrefixKey(prefixKeyStr); + } + return E_OK; +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.cpp new file mode 100644 index 000000000..7417e766c --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.cpp @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "single_ver_data_packet.h" +#include "icommunicator.h" +#include "single_ver_kvdb_sync_interface.h" +#include "query_sync_object.h" +#include "generic_single_ver_kv_entry.h" +#include "sync_types.h" +#include "version.h" +#include "parcel.h" + + +namespace DistributedDB { +DataRequestPacket::~DataRequestPacket() +{ + for (auto &entry : data_) { + delete entry; + entry = nullptr; + } +} + +void DataRequestPacket::SetData(std::vector &data) +{ + data_ = std::move(data); +} + +const std::vector &DataRequestPacket::GetData() const +{ + return data_; +} + +void DataRequestPacket::SetCompressData(std::vector &compressData) +{ + compressData_ = std::move(compressData); +} + +const std::vector &DataRequestPacket::GetCompressData() const +{ + return compressData_; +} + +void DataRequestPacket::SetEndWaterMark(WaterMark waterMark) +{ + endWaterMark_ = waterMark; +} + +WaterMark DataRequestPacket::GetEndWaterMark() const +{ + return endWaterMark_; +} + +void DataRequestPacket::SetLocalWaterMark(WaterMark waterMark) +{ + localWaterMark_ = waterMark; +} + +WaterMark DataRequestPacket::GetLocalWaterMark() const +{ + return localWaterMark_; +} + +void DataRequestPacket::SetPeerWaterMark(WaterMark waterMark) +{ + peerWaterMark_ = waterMark; +} + +WaterMark DataRequestPacket::GetPeerWaterMark() const +{ + return peerWaterMark_; +} + +void DataRequestPacket::SetSendCode(int32_t errCode) +{ + sendCode_ = errCode; +} + +int32_t DataRequestPacket::GetSendCode() const +{ + return sendCode_; +} + +void DataRequestPacket::SetMode(int32_t mode) +{ + mode_ = mode; +} + +int32_t DataRequestPacket::GetMode() const +{ + return mode_; +} + +void DataRequestPacket::SetSessionId(uint32_t sessionId) +{ + sessionId_ = sessionId; +} + +uint32_t DataRequestPacket::GetSessionId() const +{ + return sessionId_; +} + +void DataRequestPacket::SetVersion(uint32_t version) +{ + version_ = version; +} + +uint32_t DataRequestPacket::GetVersion() const +{ + return version_; +} + +void DataRequestPacket::SetReserved(std::vector &reserved) +{ + reserved_ = std::move(reserved); +} + +std::vector DataRequestPacket::GetReserved() const +{ + return reserved_; +} + +uint64_t DataRequestPacket::GetPacketId() const +{ + uint64_t packetId = 0; + std::vector DataRequestReserve = GetReserved(); + if (DataRequestReserve.size() > REQUEST_PACKET_RESERVED_INDEX_PACKETID) { + return DataRequestReserve[REQUEST_PACKET_RESERVED_INDEX_PACKETID]; + } else { + return packetId; + } +} + +uint32_t DataRequestPacket::CalculateLen(uint32_t messageId) const +{ + uint64_t totalLen = GenericSingleVerKvEntry::CalculateLens( + IsCompressData() ? std::vector {} : data_, version_); // for data + totalLen += Parcel::GetUInt64Len(); // endWaterMark + totalLen += Parcel::GetUInt64Len(); // localWaterMark + totalLen += Parcel::GetUInt64Len(); // peerWaterMark + totalLen += Parcel::GetIntLen(); // sendCode + totalLen += Parcel::GetIntLen(); // mode + totalLen += Parcel::GetUInt32Len(); // sessionId + totalLen += Parcel::GetUInt32Len(); // version + totalLen += Parcel::GetVectorLen(reserved_); // reserved + + if (version_ > SOFTWARE_VERSION_RELEASE_2_0) { + totalLen += Parcel::GetUInt32Len(); // flag bit0 used for isLastSequence + } + totalLen = Parcel::GetEightByteAlign(totalLen); // 8-byte align + if (totalLen > INT32_MAX) { + return 0; + } + if (messageId == QUERY_SYNC_MESSAGE) { + // deleted watermark + totalLen += Parcel::GetUInt64Len(); + // query id + totalLen += Parcel::GetStringLen(queryId_); + // add for queryObject + totalLen += query_.CalculateParcelLen(SOFTWARE_VERSION_CURRENT); + } + if (IsCompressData()) { + totalLen += GenericSingleVerKvEntry::CalculateCompressedLens(compressData_); // add forcompressData_ + } + if (totalLen > INT32_MAX) { + return 0; + } + return totalLen; +} + +void DataRequestPacket::SetFlag(uint32_t flag) +{ + flag_ = flag; +} + +uint32_t DataRequestPacket::GetFlag() const +{ + return flag_; +} + +bool DataRequestPacket::IsLastSequence() const +{ + return ((flag_ & IS_LAST_SEQUENCE) == IS_LAST_SEQUENCE); +} + +void DataRequestPacket::SetLastSequence() +{ + flag_ = flag_ | IS_LAST_SEQUENCE; +} + +bool DataRequestPacket::IsNeedUpdateWaterMark() const +{ + return !((flag_ & IS_UPDATE_WATER) == IS_UPDATE_WATER); +} + +void DataRequestPacket::SetUpdateWaterMark() +{ + flag_ = flag_ | IS_UPDATE_WATER; +} + +void DataRequestPacket::SetCompressDataMark() +{ + flag_ = flag_ | IS_COMPRESS_DATA; +} + +bool DataRequestPacket::IsCompressData() const +{ + return ((flag_ & IS_COMPRESS_DATA) == IS_COMPRESS_DATA); +} + +void DataRequestPacket::SetCompressAlgo(CompressAlgorithm algo) +{ + algo_ = algo; +} + +CompressAlgorithm DataRequestPacket::GetCompressAlgo() const +{ + return algo_; +} + +void DataRequestPacket::SetBasicInfo(int sendCode, uint32_t version, int32_t mode) +{ + SetSendCode(sendCode); + SetVersion(version); + SetMode(mode); +} + +void DataRequestPacket::SetWaterMark(WaterMark localMark, WaterMark peerMark, WaterMark deletedWatermark) +{ + localWaterMark_ = localMark; + peerWaterMark_ = peerMark; + deletedWatermark_ = deletedWatermark; +} + +void DataRequestPacket::SetQuery(const QuerySyncObject &query) +{ + query_ = query; +} + +QuerySyncObject DataRequestPacket::GetQuery() const +{ + return query_; +} + +void DataRequestPacket::SetQueryId(const std::string &queryId) +{ + queryId_ = queryId; +} + +std::string DataRequestPacket::GetQueryId() const +{ + return queryId_; +} + +void DataRequestPacket::SetDeletedWaterMark(WaterMark watermark) +{ + deletedWatermark_ = watermark; +} + +WaterMark DataRequestPacket::GetDeletedWaterMark() const +{ + return deletedWatermark_; +} + +void DataAckPacket::SetData(uint64_t data) +{ + data_ = data; +} + +uint64_t DataAckPacket::GetData() const +{ + return data_; +} + +void DataAckPacket::SetRecvCode(int32_t errorCode) +{ + recvCode_ = errorCode; +} + +int32_t DataAckPacket::GetRecvCode() const +{ + return recvCode_; +} + +void DataAckPacket::SetVersion(uint32_t version) +{ + version_ = version; +} + +uint32_t DataAckPacket::GetVersion() const +{ + return version_; +} + +void DataAckPacket::SetReserved(std::vector &reserved) +{ + reserved_ = std::move(reserved); +} + +std::vector DataAckPacket::GetReserved() const +{ + return reserved_; +} + +uint64_t DataAckPacket::GetPacketId() const +{ + uint64_t packetId = 0; + std::vector DataAckReserve = GetReserved(); + if (DataAckReserve.size() > ACK_PACKET_RESERVED_INDEX_PACKETID) { + return DataAckReserve[ACK_PACKET_RESERVED_INDEX_PACKETID]; + } else { + return packetId; + } +} + +bool DataAckPacket::IsPacketIdValid(uint64_t packetId) +{ + return (packetId > 0); +} + +uint32_t DataAckPacket::CalculateLen() const +{ + uint64_t len = Parcel::GetUInt64Len(); // ackWaterMark + len += Parcel::GetIntLen(); // recvCode + len += Parcel::GetUInt32Len(); // version + len += Parcel::GetVectorLen(reserved_); // reserved + + len = Parcel::GetEightByteAlign(len); + if (len > INT32_MAX) { + return 0; + } + return len; +} +} // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.h new file mode 100644 index 000000000..b113ec2a4 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.h @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SINGLE_VER_DATA_PACKET_NEW_H +#define SINGLE_VER_DATA_PACKET_NEW_H + +#include "icommunicator.h" +#include "single_ver_kvdb_sync_interface.h" +#include "query_sync_object.h" +#include "sync_types.h" +#include "version.h" +#include "parcel.h" + +namespace DistributedDB { +using SendDataItem = SingleVerKvEntry *; + +class DataRequestPacket { +public: + DataRequestPacket() {}; + virtual ~DataRequestPacket(); + + void SetData(std::vector &data); + + const std::vector &GetData() const; + const std::vector &GetCompressedData() const; + + void SetCompressData(std::vector &compressData); + + const std::vector &GetCompressData() const; + + void SetEndWaterMark(WaterMark waterMark); + + WaterMark GetEndWaterMark() const; + + void SetLocalWaterMark(WaterMark waterMark); + + WaterMark GetLocalWaterMark() const; + + void SetPeerWaterMark(WaterMark waterMark); + + WaterMark GetPeerWaterMark() const; + + void SetSendCode(int32_t errCode); + + int32_t GetSendCode() const; + + void SetMode(int32_t mode); + + int32_t GetMode() const; + + void SetSessionId(uint32_t sessionId); + + uint32_t GetSessionId() const; + + void SetVersion(uint32_t version); + + uint32_t GetVersion() const; + + uint32_t CalculateLen(uint32_t messageId) const; + + void SetReserved(std::vector &reserved); + + std::vector GetReserved() const; + + uint64_t GetPacketId() const; + + void SetFlag(uint32_t flag); + + uint32_t GetFlag() const; + + bool IsLastSequence() const; + + void SetLastSequence(); + + bool IsNeedUpdateWaterMark() const; + + void SetUpdateWaterMark(); + + void SetBasicInfo(int sendCode, uint32_t version, int32_t mode); + + void SetWaterMark(WaterMark localMark, WaterMark peerMark, WaterMark deletedWatermark); + + void SetQuery(const QuerySyncObject &query); + QuerySyncObject GetQuery() const; + + void SetQueryId(const std::string &queryId); + std::string GetQueryId() const; + + void SetDeletedWaterMark(WaterMark watermark); + WaterMark GetDeletedWaterMark() const; + + void SetCompressDataMark(); + bool IsCompressData() const; + + void SetCompressAlgo(CompressAlgorithm algo); + CompressAlgorithm GetCompressAlgo() const; + +protected: + std::vector data_; + WaterMark endWaterMark_ = 0; + WaterMark localWaterMark_ = 0; + WaterMark peerWaterMark_ = 0; + int32_t sendCode_ = 0; + int32_t mode_ = SyncModeType::INVALID_MODE; + uint32_t sessionId_ = 0; + uint32_t version_ = SOFTWARE_VERSION_CURRENT; + std::vector reserved_; + uint32_t flag_ = 0; // bit 0 used for isLastSequence + // add for query sync mode + QuerySyncObject query_; + std::string queryId_; + WaterMark deletedWatermark_ = 0; + std::vector compressData_; // if compressData size is above 0, means use compressData and ignore data_ + CompressAlgorithm algo_ = CompressAlgorithm::NONE; // used for param while serialize compress data + static const uint32_t IS_LAST_SEQUENCE = 0x1; // bit 0 used for isLastSequence, 1: is last, 0: not last + static const uint32_t IS_UPDATE_WATER = 0x2; // bit 1 used for update watermark, 0: update, 1: not update + static const uint32_t IS_COMPRESS_DATA = 0x4; // bit 3 used for compress data, 0: raw data, 1: compress data +}; + +class DataAckPacket { +public: + DataAckPacket() {}; + virtual ~DataAckPacket() {}; + + void SetData(uint64_t data); + + uint64_t GetData() const; + + void SetRecvCode(int32_t errorCode); + + int32_t GetRecvCode() const; + + void SetVersion(uint32_t version); + + uint32_t GetVersion() const; + + void SetReserved(std::vector &reserved); + + std::vector GetReserved() const; + + uint64_t GetPacketId() const; + + static bool IsPacketIdValid(uint64_t packetId); + + uint32_t CalculateLen() const; + +private: + /* + * data_ is waterMark when revCode_ == LOCAL_WATER_MARK_NOT_INIT || revCode_ == E_OK; + * data_ is timer in milliSeconds when revCode_ == -E_SAVE_DATA_NOTIFY && data_ != 0. + */ + uint64_t data_ = 0; + int32_t recvCode_ = 0; + uint32_t version_ = SOFTWARE_VERSION_CURRENT; + std::vector reserved_; +}; +} // namespace DistributedDB + +#endif // SINGLE_VER_DATA_SYNC_NEW_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index 9bce54fc6..bdd2684f6 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -16,23 +16,16 @@ #include "single_ver_data_sync.h" #include "db_common.h" +#include "db_types.h" +#include "generic_single_ver_kv_entry.h" #include "log_print.h" -#include "single_ver_sync_state_machine.h" -#include "performance_analysis.h" #include "message_transform.h" -#include "generic_single_ver_kv_entry.h" +#include "performance_analysis.h" +#include "single_ver_sync_state_machine.h" #include "single_ver_sync_target.h" -#include "db_constant.h" namespace DistributedDB { namespace { - // index 0 for packetId in data request - // if ack reserve size is 1, reserve is {localWaterMark} - // if ack reserve size is above 2, reserve is {localWaterMark, packetId ...} - constexpr uint32_t REQUEST_PACKET_RESERVED_INDEX_PACKETID = 0; - constexpr uint32_t ACK_PACKET_RESERVED_INDEX_PACKETID = 1; - constexpr uint32_t ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK = 0; // index 0 for localWaterMark - void SetMessageHeadInfo(Message &message, uint16_t inMsgType, const std::string &inTarget, uint32_t inSequenceId, uint32_t inSessionId) { @@ -43,237 +36,6 @@ namespace { } } -DataRequestPacket::~DataRequestPacket() -{ - for (auto &entry : data_) { - delete entry; - entry = nullptr; - } -} - -void DataRequestPacket::SetData(std::vector &data) -{ - data_ = std::move(data); -} - -const std::vector &DataRequestPacket::GetData() const -{ - return data_; -} - -void DataRequestPacket::SetEndWaterMark(WaterMark waterMark) -{ - endWaterMark_ = waterMark; -} - -WaterMark DataRequestPacket::GetEndWaterMark() const -{ - return endWaterMark_; -} - -void DataRequestPacket::SetLocalWaterMark(WaterMark waterMark) -{ - localWaterMark_ = waterMark; -} - -WaterMark DataRequestPacket::GetLocalWaterMark() const -{ - return localWaterMark_; -} - -void DataRequestPacket::SetPeerWaterMark(WaterMark waterMark) -{ - peerWaterMark_ = waterMark; -} - -WaterMark DataRequestPacket::GetPeerWaterMark() const -{ - return peerWaterMark_; -} - -void DataRequestPacket::SetSendCode(int32_t errCode) -{ - sendCode_ = errCode; -} - -int32_t DataRequestPacket::GetSendCode() const -{ - return sendCode_; -} - -void DataRequestPacket::SetMode(int32_t mode) -{ - mode_ = mode; -} - -int32_t DataRequestPacket::GetMode() const -{ - return mode_; -} - -void DataRequestPacket::SetSessionId(uint32_t sessionId) -{ - sessionId_ = sessionId; -} - -uint32_t DataRequestPacket::GetSessionId() const -{ - return sessionId_; -} - -void DataRequestPacket::SetVersion(uint32_t version) -{ - version_ = version; -} - -uint32_t DataRequestPacket::GetVersion() const -{ - return version_; -} - -void DataRequestPacket::SetReserved(std::vector &reserved) -{ - reserved_ = std::move(reserved); -} - -std::vector DataRequestPacket::GetReserved() const -{ - return reserved_; -} - -uint64_t DataRequestPacket::GetPacketId() const -{ - uint64_t packetId = 0; - std::vector DataRequestReserve = GetReserved(); - if (DataRequestReserve.size() > REQUEST_PACKET_RESERVED_INDEX_PACKETID) { - return DataRequestReserve[REQUEST_PACKET_RESERVED_INDEX_PACKETID]; - } else { - return packetId; - } -} - -uint32_t DataRequestPacket::CalculateLen() const -{ - uint64_t totalLen = GenericSingleVerKvEntry::CalculateLens(data_, version_); // for data - totalLen += Parcel::GetUInt64Len(); // endWaterMark - totalLen += Parcel::GetUInt64Len(); // localWaterMark - totalLen += Parcel::GetUInt64Len(); // peerWaterMark - totalLen += Parcel::GetIntLen(); // sendCode - totalLen += Parcel::GetIntLen(); // mode - totalLen += Parcel::GetUInt32Len(); // sessionId - totalLen += Parcel::GetUInt32Len(); // version - totalLen += Parcel::GetVectorLen(reserved_); // reserved - - if (version_ > SOFTWARE_VERSION_RELEASE_2_0) { - totalLen += Parcel::GetUInt32Len(); // flag bit0 used for isLastSequence - } - totalLen = Parcel::GetEightByteAlign(totalLen); // 8-byte align - if (totalLen > INT32_MAX) { - return 0; - } - return totalLen; -} - -void DataRequestPacket::SetFlag(uint32_t flag) -{ - flag_ = flag; -} - -uint32_t DataRequestPacket::GetFlag() const -{ - return flag_; -} - -bool DataRequestPacket::IsLastSequence() const -{ - return flag_ & IS_LAST_SEQUENCE; -} - -void DataRequestPacket::SetLastSequence() -{ - flag_ = flag_ | IS_LAST_SEQUENCE; -} - -void DataRequestPacket::SetBasicInfo(int sendCode, uint32_t version, WaterMark localMark, WaterMark peerMark, - int32_t mode) -{ - SetSendCode(sendCode); - SetVersion(version); - SetLocalWaterMark(localMark); - SetPeerWaterMark(peerMark); - SetMode(mode); -} - -void DataAckPacket::SetData(uint64_t data) -{ - data_ = data; -} - -uint64_t DataAckPacket::GetData() const -{ - return data_; -} - -void DataAckPacket::SetRecvCode(int32_t errorCode) -{ - recvCode_ = errorCode; -} - -int32_t DataAckPacket::GetRecvCode() const -{ - return recvCode_; -} - -void DataAckPacket::SetVersion(uint32_t version) -{ - version_ = version; -} - -uint32_t DataAckPacket::GetVersion() const -{ - return version_; -} - -void DataAckPacket::SetReserved(std::vector &reserved) -{ - reserved_ = std::move(reserved); -} - -std::vector DataAckPacket::GetReserved() const -{ - return reserved_; -} - -uint64_t DataAckPacket::GetPacketId() const -{ - uint64_t packetId = 0; - std::vector DataAckReserve = GetReserved(); - if (DataAckReserve.size() > ACK_PACKET_RESERVED_INDEX_PACKETID) { - return DataAckReserve[ACK_PACKET_RESERVED_INDEX_PACKETID]; - } else { - return packetId; - } -} - -bool DataAckPacket::IsPacketIdValid(uint64_t packetId) -{ - return (packetId > 0); -} - -uint32_t DataAckPacket::CalculateLen() const -{ - uint64_t len = Parcel::GetUInt64Len(); // ackWaterMark - len += Parcel::GetIntLen(); // recvCode - len += Parcel::GetUInt32Len(); // version - len += Parcel::GetVectorLen(reserved_); // reserved - - len = Parcel::GetEightByteAlign(len); - if (len > INT32_MAX) { - return 0; - } - return len; -} - SingleVerDataSync::SingleVerDataSync() : mtuSize_(0), storage_(nullptr), @@ -289,17 +51,6 @@ SingleVerDataSync::~SingleVerDataSync() metadata_ = nullptr; } -int SingleVerDataSync::RegisterTransformFunc() -{ - TransformFunc func; - func.computeFunc = std::bind(&SingleVerDataSync::CalculateLen, std::placeholders::_1); - func.serializeFunc = std::bind(&SingleVerDataSync::Serialization, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3); - func.deserializeFunc = std::bind(&SingleVerDataSync::DeSerialization, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3); - return MessageTransform::RegTransformFunction(DATA_SYNC_MESSAGE, func); -} - int SingleVerDataSync::Initialize(IKvDBSyncInterface *inStorage, ICommunicator *inCommunicateHandle, std::shared_ptr &inMetadata, const std::string &deviceId) { @@ -317,107 +68,6 @@ int SingleVerDataSync::Initialize(IKvDBSyncInterface *inStorage, ICommunicator * return E_OK; } -bool SingleVerDataSync::IsPacketValid(const Message *inMsg) -{ - if (inMsg == nullptr) { - return false; - } - if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE) { - LOGE("[DataSync][IsPacketValid] Message Id ERROR! messageId=%d", inMsg->GetMessageId()); - return false; - } - int msgType = inMsg->GetMessageType(); - if (msgType != TYPE_REQUEST && msgType != TYPE_RESPONSE && msgType != TYPE_NOTIFY) { - LOGE("[DataSync][IsPacketValid] Message type ERROR! message type=%d", msgType); - return false; - } - return true; -} - -int SingleVerDataSync::Serialization(uint8_t *buffer, uint32_t length, const Message *inMsg) -{ - if ((buffer == nullptr) || !(IsPacketValid(inMsg))) { - return -E_MESSAGE_ID_ERROR; - } - - switch (inMsg->GetMessageType()) { - case TYPE_REQUEST: - return DataPacketSerialization(buffer, length, inMsg); - case TYPE_RESPONSE: - case TYPE_NOTIFY: - return AckPacketSerialization(buffer, length, inMsg); - default: - return -E_MESSAGE_TYPE_ERROR; - } -} - -int SingleVerDataSync::DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) -{ - if ((buffer == nullptr) || !(IsPacketValid(inMsg))) { - return -E_MESSAGE_ID_ERROR; - } - - switch (inMsg->GetMessageType()) { - case TYPE_REQUEST: - return DataPacketDeSerialization(buffer, length, inMsg); - case TYPE_RESPONSE: - case TYPE_NOTIFY: - return AckPacketDeSerialization(buffer, length, inMsg); - default: - return -E_MESSAGE_TYPE_ERROR; - } -} - -uint32_t SingleVerDataSync::CalculateLen(const Message *inMsg) -{ - if (!(IsPacketValid(inMsg))) { - return 0; - } - uint32_t len = 0; - int errCode; - switch (inMsg->GetMessageType()) { - case TYPE_REQUEST: - errCode = DataPacketCalculateLen(inMsg, len); - if (errCode != E_OK) { - LOGE("[DataSync][CalculateLen] calculate data request packet len failed, errCode=%d", errCode); - return 0; - } - return len; - case TYPE_RESPONSE: - case TYPE_NOTIFY: - errCode = AckPacketCalculateLen(inMsg, len); - if (errCode != E_OK) { - LOGE("[DataSync][CalculateLen] calculate data notify packet len failed errCode=%d", errCode); - return 0; - } - return len; - default: - return 0; - } -} - -int SingleVerDataSync::DataPacketCalculateLen(const Message *inMsg, uint32_t &len) -{ - const DataRequestPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - - len = packet->CalculateLen(); - return E_OK; -} - -int SingleVerDataSync::AckPacketCalculateLen(const Message *inMsg, uint32_t &len) -{ - const DataAckPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - - len = packet->CalculateLen(); - return E_OK; -} - std::string SingleVerDataSync::GetLocalDeviceName() { std::string deviceInfo; @@ -446,237 +96,13 @@ std::string SingleVerDataSync::TransferLocalOrigDevName(const std::string &origN return origName; } -int SingleVerDataSync::DataPacketSyncerPartSerialization(Parcel &parcel, const DataRequestPacket *packet) -{ - // endWaterMark - int errCode = parcel.WriteUInt64(packet->GetEndWaterMark()); - if (errCode != E_OK) { - return errCode; - } - // localWaterMark - errCode = parcel.WriteUInt64(packet->GetLocalWaterMark()); - if (errCode != E_OK) { - return errCode; - } - // peerWaterMark - errCode = parcel.WriteUInt64(packet->GetPeerWaterMark()); - if (errCode != E_OK) { - return errCode; - } - // sendCode - errCode = parcel.WriteInt(packet->GetSendCode()); - if (errCode != E_OK) { - return errCode; - } - // mode - errCode = parcel.WriteInt(packet->GetMode()); - if (errCode != E_OK) { - return errCode; - } - // sessionId - errCode = parcel.WriteUInt32(packet->GetSessionId()); - if (errCode != E_OK) { - return errCode; - } - // reserved - errCode = parcel.WriteVector(packet->GetReserved()); - if (errCode != E_OK) { - return errCode; - } - if (packet->GetVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - errCode = parcel.WriteUInt32(packet->GetFlag()); - if (errCode != E_OK) { - return errCode; - } - } - parcel.EightByteAlign(); - return errCode; -} - -int SingleVerDataSync::DataPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) -{ - const DataRequestPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - Parcel parcel(buffer, length); - - // version - int errCode = parcel.WriteUInt32(packet->GetVersion()); - if (errCode != E_OK) { - return errCode; - } - // sendDataItems - const std::vector &data = packet->GetData(); - errCode = GenericSingleVerKvEntry::SerializeDatas(data, parcel, packet->GetVersion()); - if (errCode != E_OK) { - return errCode; - } - return SingleVerDataSync::DataPacketSyncerPartSerialization(parcel, packet); -} - -int SingleVerDataSync::DataPacketSyncerPartDeSerialization(Parcel &parcel, DataRequestPacket *packet, - uint32_t packLen, uint32_t length, uint32_t version) -{ - WaterMark waterMark; - WaterMark localWaterMark; - WaterMark peerWaterMark; - int32_t sendCode; - int32_t mode; - uint32_t sessionId; - uint32_t flag = 0; - std::vector reserved; - - packLen += parcel.ReadUInt64(waterMark); - packLen += parcel.ReadUInt64(localWaterMark); - packLen += parcel.ReadUInt64(peerWaterMark); - packLen += parcel.ReadInt(sendCode); - packLen += parcel.ReadInt(mode); - packLen += parcel.ReadUInt32(sessionId); - packLen += parcel.ReadVector(reserved); - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - packLen += parcel.ReadUInt32(flag); - packet->SetFlag(flag); - } - packLen = Parcel::GetEightByteAlign(packLen); - if (packLen != length || parcel.IsError()) { - LOGE("[DataSync][DataPacketDeSerialization] deserialize failed! input len=%lu,packLen=%lu", length, packLen); - return -E_LENGTH_ERROR; - } - packet->SetEndWaterMark(waterMark); - packet->SetLocalWaterMark(localWaterMark); - packet->SetPeerWaterMark(peerWaterMark); - packet->SetSendCode(sendCode); - packet->SetMode(mode); - packet->SetSessionId(sessionId); - packet->SetReserved(reserved); - return E_OK; -} - -int SingleVerDataSync::DataPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) -{ - std::vector dataItems; - uint32_t version; - Parcel parcel(const_cast(buffer), length); - uint32_t packLen = parcel.ReadUInt32(version); - if (parcel.IsError()) { - return -E_INVALID_ARGS; - } - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket(); - if (packet == nullptr) { - return -E_OUT_OF_MEMORY; - } - if (version > SOFTWARE_VERSION_CURRENT) { - packet->SetVersion(version); - packet->SetSendCode(-E_VERSION_NOT_SUPPORT); - int errCode = inMsg->SetExternalObject<>(packet); - if (errCode != E_OK) { - delete packet; - packet = nullptr; - } - return errCode; - } - packet->SetVersion(version); - packLen += GenericSingleVerKvEntry::DeSerializeDatas(dataItems, parcel); - packet->SetData(dataItems); - int errCode = SingleVerDataSync::DataPacketSyncerPartDeSerialization(parcel, packet, packLen, length, version); - if (errCode != E_OK) { - delete packet; - packet = nullptr; - return errCode; - } - errCode = inMsg->SetExternalObject<>(packet); - if (errCode != E_OK) { - delete packet; - packet = nullptr; - } - return errCode; -} - -int SingleVerDataSync::AckPacketSyncerPartSerializationV1(Parcel &parcel, const DataAckPacket *packet) -{ - int errCode = parcel.WriteUInt64(packet->GetData()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetRecvCode()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteVector(packet->GetReserved()); - if (errCode != E_OK) { - return errCode; - } - parcel.EightByteAlign(); - return errCode; -} - -int SingleVerDataSync::AckPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) -{ - const DataAckPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - - Parcel parcel(buffer, length); - int errCode = parcel.WriteUInt32(packet->GetVersion()); - if (errCode != E_OK) { - return errCode; - } - // now V1 compatible for softWareVersion :{101, 102} - return SingleVerDataSync::AckPacketSyncerPartSerializationV1(parcel, packet); -} - -int SingleVerDataSync::AckPacketSyncerPartDeSerializationV1(Parcel &parcel, DataAckPacket &packet) -{ - WaterMark mark; - int32_t errCode; - std::vector reserved; - - parcel.ReadUInt64(mark); - parcel.ReadInt(errCode); - parcel.ReadVector(reserved); - if (parcel.IsError()) { - return -E_INVALID_ARGS; - } - packet.SetData(mark); - packet.SetRecvCode(errCode); - packet.SetReserved(reserved); - return E_OK; -} - -int SingleVerDataSync::AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) -{ - DataAckPacket packet; - Parcel parcel(const_cast(buffer), length); - uint32_t version; - - parcel.ReadUInt32(version); - if (parcel.IsError()) { - return -E_INVALID_ARGS; - } - if (version > SOFTWARE_VERSION_CURRENT) { - packet.SetVersion(version); - packet.SetRecvCode(-E_VERSION_NOT_SUPPORT); - return inMsg->SetCopiedObject<>(packet); - } - packet.SetVersion(version); - // now V1 compatible for softWareVersion :{101, 102} - int errCode = SingleVerDataSync::AckPacketSyncerPartDeSerializationV1(parcel, packet); - if (errCode != E_OK) { - return errCode; - } - - return inMsg->SetCopiedObject<>(packet); -} - int SingleVerDataSync::Send(SingleVerSyncTaskContext *context, const Message *message, const CommErrHandler &handler, uint32_t packetLen) { bool startFeedDogRet = false; - if (packetLen > mtuSize_ && mtuSize_ > 0) { + if (packetLen > mtuSize_ && mtuSize_ > FEEDDOG_MAX_MTU_SIZE) { uint32_t time = static_cast(static_cast(packetLen) * - static_cast(DBConstant::AUTO_SYNC_TIMEOUT) / mtuSize_); // no overflow + static_cast(context->GetTimeoutTime()) / mtuSize_); // no overflow startFeedDogRet = context->StartFeedDogForSync(time, SyncDirectionFlag::SEND); } int errCode = communicateHandle_->SendMessage(context->GetDeviceId(), message, false, SEND_TIME_OUT, handler); @@ -692,6 +118,7 @@ int SingleVerDataSync::Send(SingleVerSyncTaskContext *context, const Message *me int SingleVerDataSync::GetData(SingleVerSyncTaskContext *context, std::vector &outData, size_t packetSize) { int errCode; + UpdateMtuSize(); if (context->GetRetryStatus() == SyncTaskContext::NEED_RETRY) { context->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); LOGI("[DataSync][GetData] resend data"); @@ -707,44 +134,55 @@ int SingleVerDataSync::GetData(SingleVerSyncTaskContext *context, std::vectorSetTaskErrCode(-E_EKEYREVOKED); - } - if (errCode == -E_BUSY) { - context->SetTaskErrCode(-E_BUSY); + if (errCode == E_OK || errCode == -E_UNFINISHED) { + TransDbDataItemToSendDataItem(context, outData); + return errCode; } + context->SetTaskErrCode(errCode); return errCode; } -int SingleVerDataSync::GetDataWithRerformanceRecord(SingleVerSyncTaskContext *context, - std::vector &outData) +int SingleVerDataSync::GetDataWithPerformanceRecord(SingleVerSyncTaskContext *context, SyncEntry &syncOutData) { uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); size_t packetSize = (version > SOFTWARE_VERSION_RELEASE_2_0) ? DBConstant::MAX_HPMODE_PACK_ITEM_SIZE : DBConstant::MAX_NORMAL_PACK_ITEM_SIZE; + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_READ_DATA); } - int errCode = GetData(context, outData, packetSize); + int errCode = GetData(context, syncOutData.entries, packetSize); if (performance != nullptr) { performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_READ_DATA); } + if (errCode != E_OK && errCode != -E_UNFINISHED) { + return errCode; + } + CompressAlgorithm remoteAlgo = context->ChooseCompressAlgo(); + if (needCompressOnSync && remoteAlgo != CompressAlgorithm::NONE) { + int compressCode = GenericSingleVerKvEntry::Compress(syncOutData.entries, syncOutData.compressedEntries, + { remoteAlgo, version }); + if (compressCode != E_OK) { + return compressCode; + } + } return errCode; } int SingleVerDataSync::GetUnsyncData(SingleVerSyncTaskContext *context, std::vector &outData, size_t packetSize) { - WaterMark startMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), startMark); - WaterMark endMark = context->GetEndMark(); + int errCode; + WaterMark startMark = 0; + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + GetLocalWaterMark(curType, context->GetQuery().GetIdentify(), context, startMark); + WaterMark endMark = MAX_TIMESTAMP; if ((endMark == 0) || (startMark > endMark)) { return E_OK; } @@ -753,8 +191,16 @@ int SingleVerDataSync::GetUnsyncData(SingleVerSyncTaskContext *context, std::vec if (token != nullptr) { storage_->ReleaseContinueToken(token); } - DataSizeSpecInfo syncDataSizeInfo = {mtuSize_, packetSize}; - int errCode = storage_->GetSyncData(startMark, endMark, outData, token, syncDataSizeInfo); + DataSizeSpecInfo syncDataSizeInfo = GetDataSizeSpecInfo(packetSize); + if (curType != SyncType::QUERY_SYNC_TYPE) { + errCode = storage_->GetSyncData(startMark, endMark, outData, token, syncDataSizeInfo); + } else { + WaterMark deletedStartMark = 0; + GetLocalDeleteSyncWaterMark(context, deletedStartMark); + QueryObject queryObj = context->GetQueryObj(); + errCode = storage_->GetSyncData(queryObj, SyncTimeRange{ startMark, deletedStartMark, endMark, endMark}, + syncDataSizeInfo, token, outData); + } context->SetContinueToken(token); if (errCode != E_OK && errCode != -E_UNFINISHED) { LOGE("[DataSync][GetUnsyncData] get unsync data failed,errCode=%d", errCode); @@ -767,7 +213,7 @@ int SingleVerDataSync::GetNextUnsyncData(SingleVerSyncTaskContext *context, std: { ContinueToken token; context->GetContinueToken(token); - DataSizeSpecInfo syncDataSizeInfo = {mtuSize_, packetSize}; + DataSizeSpecInfo syncDataSizeInfo = GetDataSizeSpecInfo(packetSize); int errCode = storage_->GetSyncDataNext(outData, token, syncDataSizeInfo); context->SetContinueToken(token); if (errCode != E_OK && errCode != -E_UNFINISHED) { @@ -797,13 +243,9 @@ int SingleVerDataSync::SaveData(const SingleVerSyncTaskContext *context, const s return errCode; } -TimeStamp SingleVerDataSync::GetMaxSendDataTime(const std::vector &inData, bool isNeedInit, - WaterMark localMark) +TimeStamp SingleVerDataSync::GetMaxSendDataTime(const std::vector &inData) { TimeStamp stamp = 0; - if (isNeedInit) { - stamp = localMark; - } for (size_t i = 0; i < inData.size(); i++) { if (inData[i] == nullptr) { continue; @@ -816,6 +258,80 @@ TimeStamp SingleVerDataSync::GetMaxSendDataTime(const std::vector return stamp; } +SyncTimeRange SingleVerDataSync::GetSyncDataTimeRange(int syncType, SingleVerSyncTaskContext *context, + const std::vector &inData, UpdateWaterMark &isUpdate) +{ + WaterMark localMark = 0; + WaterMark deleteMark = 0; + GetLocalWaterMark(syncType, context->GetQuery().GetIdentify(), context, localMark); + GetLocalDeleteSyncWaterMark(context, deleteMark); + if (syncType != SyncType::QUERY_SYNC_TYPE) { + return GetFullSyncDataTimeRange(inData, localMark, isUpdate); + } + return GetQuerySyncDataTimeRange(inData, localMark, deleteMark, isUpdate); +} + +SyncTimeRange SingleVerDataSync::GetRecvDataTimeRange(int syncType, SingleVerSyncTaskContext *context, + const std::vector &data, UpdateWaterMark &isUpdate) +{ + if (syncType != SyncType::QUERY_SYNC_TYPE) { + return GetFullSyncDataTimeRange(data, 0, isUpdate); + } + return GetQuerySyncDataTimeRange(data, 0, 0, isUpdate); +} + +SyncTimeRange SingleVerDataSync::GetFullSyncDataTimeRange(const std::vector &inData, + WaterMark localMark, UpdateWaterMark &isUpdate) +{ + TimeStamp maxTimeStamp = localMark; + TimeStamp minTimeStamp = localMark; + for (size_t i = 0; i < inData.size(); i++) { + if (inData[i] == nullptr) { + continue; + } + TimeStamp tempStamp = inData[i]->GetTimestamp(); + if (maxTimeStamp < tempStamp) { + maxTimeStamp = tempStamp; + } + if (minTimeStamp > tempStamp) { + minTimeStamp = tempStamp; + } + isUpdate.normalUpdateMark = true; + } + return {minTimeStamp, 0, maxTimeStamp, 0}; +} + +SyncTimeRange SingleVerDataSync::GetQuerySyncDataTimeRange(const std::vector &inData, + WaterMark localMark, WaterMark deletelocalMark, UpdateWaterMark &isUpdate) +{ + SyncTimeRange dataTimeRange = {localMark, deletelocalMark, localMark, deletelocalMark}; + for (size_t i = 0; i < inData.size(); i++) { + if (inData[i] == nullptr) { + continue; + } + TimeStamp tempStamp = inData[i]->GetTimestamp(); + if ((inData[i]->GetFlag() & DataItem::DELETE_FLAG) == 0) { // query data + if (dataTimeRange.endTime < tempStamp) { + dataTimeRange.endTime = tempStamp; + } + if (dataTimeRange.beginTime > tempStamp) { + dataTimeRange.beginTime = tempStamp; + } + isUpdate.normalUpdateMark = true; + } + if ((inData[i]->GetFlag() & DataItem::DELETE_FLAG) != 0) { // delete data + if (dataTimeRange.deleteEndTime < tempStamp) { + dataTimeRange.deleteEndTime = tempStamp; + } + if (dataTimeRange.deleteBeginTime > tempStamp) { + dataTimeRange.deleteBeginTime = tempStamp; + } + isUpdate.deleteUpdateMark = true; + } + } + return dataTimeRange; +} + TimeStamp SingleVerDataSync::GetMinSendDataTime(const std::vector &inData, WaterMark localMark) { TimeStamp stamp = localMark; @@ -831,15 +347,61 @@ TimeStamp SingleVerDataSync::GetMinSendDataTime(const std::vector return stamp; } -int SingleVerDataSync::SaveLocalWaterMark(const DeviceID &deviceId, WaterMark waterMark) +SyncTimeRange SingleVerDataSync::ReviseLocalMark(int syncType, SyncTimeRange &dataTimeRange, + UpdateWaterMark updateMark) { - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SAVE_LOCAL_WATERMARK); + SyncTimeRange tmpDataTime = dataTimeRange; + if (updateMark.deleteUpdateMark && syncType == SyncType::QUERY_SYNC_TYPE) { + tmpDataTime.deleteEndTime += 1; } - int errCode = metadata_->SaveLocalWaterMark(deviceId, waterMark); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SAVE_LOCAL_WATERMARK); + if (updateMark.normalUpdateMark) { + tmpDataTime.endTime += 1; + } + return tmpDataTime; +} + +int SingleVerDataSync::SaveLocalWaterMark(int syncType, const SingleVerSyncTaskContext *context, + SyncTimeRange dataTimeRange, bool isCheckBeforUpdate) const +{ + WaterMark localMark = 0; + int errCode = E_OK; + const std::string &deviceId = context->GetDeviceId(); + const std::string &queryId = context->GetQuery().GetIdentify(); + if (syncType != SyncType::QUERY_SYNC_TYPE) { + if (isCheckBeforUpdate) { + GetLocalWaterMark(syncType, queryId, context, localMark); + if (localMark >= dataTimeRange.endTime) { + return E_OK; + } + } + errCode = metadata_->SaveLocalWaterMark(deviceId, dataTimeRange.endTime); + } else { + bool isNeedUpdateMark = true; + bool isNeedUpdateDeleteMark = true; + if (isCheckBeforUpdate) { + WaterMark deleteDataWaterMark = 0; + GetLocalWaterMark(syncType, queryId, context, localMark); + GetLocalDeleteSyncWaterMark(context, deleteDataWaterMark); + if (localMark >= dataTimeRange.endTime) { + isNeedUpdateMark = false; + } + if (deleteDataWaterMark >= dataTimeRange.deleteEndTime) { + isNeedUpdateDeleteMark = false; + } + } + if (isNeedUpdateMark) { + LOGD("label=%s,dev=%s,endTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), dataTimeRange.endTime); + errCode = metadata_->SetSendQueryWaterMark(queryId, deviceId, dataTimeRange.endTime); + if (errCode != E_OK) { + LOGE("[DataSync][SaveLocalWaterMark] save query metadata watermark failed,errCode=%d", errCode); + return errCode; + } + } + if (isNeedUpdateDeleteMark) { + LOGD("label=%s,dev=%s,deleteEndTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), + dataTimeRange.deleteEndTime); + errCode = metadata_->SetSendDeleteSyncWaterMark(deviceId, dataTimeRange.deleteEndTime); + } } if (errCode != E_OK) { LOGE("[DataSync][SaveLocalWaterMark] save metadata local watermark failed,errCode=%d", errCode); @@ -847,23 +409,116 @@ int SingleVerDataSync::SaveLocalWaterMark(const DeviceID &deviceId, WaterMark wa return errCode; } -int SingleVerDataSync::SavePeerWaterMark(const DeviceID &deviceId, WaterMark waterMark) +void SingleVerDataSync::GetPeerWaterMark(int syncType, const std::string &queryIdentify, + const DeviceID &deviceId, WaterMark &waterMark) const +{ + if (syncType != SyncType::QUERY_SYNC_TYPE) { + metadata_->GetPeerWaterMark(deviceId, waterMark); + return; + } + metadata_->GetRecvQueryWaterMark(queryIdentify, deviceId, waterMark); +} + +void SingleVerDataSync::GetPeerDeleteSyncWaterMark(const DeviceID &deviceId, WaterMark &waterMark) +{ + metadata_->GetRecvDeleteSyncWaterMark(deviceId, waterMark); +} + +void SingleVerDataSync::GetLocalDeleteSyncWaterMark(const SingleVerSyncTaskContext *context, + WaterMark &waterMark) const +{ + metadata_->GetSendDeleteSyncWaterMark(context->GetDeviceId(), waterMark, context->IsAutoLiftWaterMark()); +} + +void SingleVerDataSync::GetLocalWaterMark(int syncType, const std::string &queryIdentify, + const SingleVerSyncTaskContext *context, WaterMark &waterMark) const { - return metadata_->SavePeerWaterMark(deviceId, waterMark); + if (syncType != SyncType::QUERY_SYNC_TYPE) { + metadata_->GetLocalWaterMark(context->GetDeviceId(), waterMark); + return; + } + metadata_->GetSendQueryWaterMark(queryIdentify, context->GetDeviceId(), + waterMark, context->IsAutoLiftWaterMark()); } -int SingleVerDataSync::RemoveDeviceData(SingleVerSyncTaskContext *context, const Message *message, +int SingleVerDataSync::RemoveDeviceDataHandle(SingleVerSyncTaskContext *context, const Message *message, WaterMark maxSendDataTime) { + bool isNeedClearRemoteData = false; + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_3_0) { + uint64_t clearDeviceDataMark = 0; + metadata_->GetRemoveDataMark(context->GetDeviceId(), clearDeviceDataMark); + isNeedClearRemoteData = (clearDeviceDataMark == REMOVE_DEVICE_DATA_MARK); + } else { + const DataRequestPacket *packet = message->GetObject(); + if (packet == nullptr) { + LOGE("[RemoveDeviceDataHandle] get packet object failed"); + return -E_INVALID_ARGS; + } + int curType = SyncOperation::GetSyncType(packet->GetMode()); + WaterMark packetLocalMark = packet->GetLocalWaterMark(); + WaterMark peerMark = 0; + GetPeerWaterMark(curType, packet->GetQueryId(), context->GetDeviceId(), peerMark); + isNeedClearRemoteData = ((packetLocalMark == 0) && (peerMark != 0)); + } + if (!isNeedClearRemoteData) { + return E_OK; + } + int errCode = E_OK; + if (context->IsNeedClearRemoteStaleData()) { + // need to clear remote device history data + errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); + if (errCode != E_OK) { + (void)SendAck(context, message, errCode, maxSendDataTime); + return errCode; + } + if (context->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_EARLIEST) { + // avoid repeat clear in ack + metadata_->SaveLocalWaterMark(context->GetDeviceId(), 0); + } + } + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_3_0) { + errCode = metadata_->ResetMetaDataAfterRemoveData(context->GetDeviceId()); + if (errCode != E_OK) { + (void)SendAck(context, message, errCode, maxSendDataTime); + return errCode; + } + } + return E_OK; +} + +int SingleVerDataSync::DealRemoveDeviceDataByAck(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, + const std::vector &reserved) +{ + bool isNeedClearRemoteData = false; + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_3_0) { + uint64_t clearDeviceDataMark = 0; + metadata_->GetRemoveDataMark(context->GetDeviceId(), clearDeviceDataMark); + isNeedClearRemoteData = (clearDeviceDataMark != 0); + } else if (reserved.empty()) { + WaterMark localMark = 0; + GetLocalWaterMark(curType, context->GetQuery().GetIdentify(), context, localMark); + isNeedClearRemoteData = ((localMark != 0) && (ackWaterMark == 0)); + } else { + WaterMark peerMark = 0; + GetPeerWaterMark(curType, context->GetQuery().GetIdentify(), context->GetDeviceId(), peerMark); + isNeedClearRemoteData = ((reserved[ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK] == 0) && (peerMark != 0)); + } + if (!isNeedClearRemoteData) { + return E_OK; + } + // need to clear remote historydata + LOGI("[DataSync][WaterMarkException] AckRecv reserved not empty,rebuilted,clear historydata,label=%s,dev=%s", + label_.c_str(), STR_MASK(GetDeviceId())); int errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); if (errCode != E_OK) { - (void)SendAck(context, message, errCode, maxSendDataTime); return errCode; } - if (context->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_EARLIEST) { - (void)SaveLocalWaterMark(context->GetDeviceId(), 0); // avoid repeat clear in ack + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_3_0) { + errCode = metadata_->ResetMetaDataAfterRemoveData(context->GetDeviceId()); } - return E_OK; + return errCode; } void SingleVerDataSync::TransDbDataItemToSendDataItem(const SingleVerSyncTaskContext *context, @@ -907,99 +562,114 @@ void SingleVerDataSync::TransSendDataItemToLocal(const SingleVerSyncTaskContext } } -int SingleVerDataSync::PushStart(SingleVerSyncTaskContext *context) +void SingleVerDataSync::FillRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, + SyncEntry &syncData, int sendCode, int mode) { - if (context == nullptr) { - return -E_INVALID_ARGS; - } - std::vector outData; - WaterMark localMark; - WaterMark peerMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - LOGD("[DataSync][PushStart] localMark=%llu,endmark=%llu,peerMark=%llu,label=%s,dev=%s{private}", localMark, - context->GetEndMark(), peerMark, label_.c_str(), GetDeviceId().c_str()); + WaterMark localMark = 0; + WaterMark peerMark = 0; + WaterMark deleteMark = 0; + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + GetLocalWaterMark(curType, context->GetQuery().GetIdentify(), context, localMark); + GetPeerWaterMark(curType, context->GetQuery().GetIdentify(), context->GetDeviceId(), peerMark); + GetLocalDeleteSyncWaterMark(context, deleteMark); + if (((mode != SyncModeType::RESPONSE_PULL && sendCode == E_OK)) || + (mode == SyncModeType::RESPONSE_PULL && sendCode == SEND_FINISHED)) { + packet->SetLastSequence(); + } + int tmpMode = mode; + if (mode == SyncModeType::RESPONSE_PULL) { + tmpMode = (curType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH : SyncModeType::PUSH; + } + packet->SetData(syncData.entries); + packet->SetCompressData(syncData.compressedEntries); + packet->SetBasicInfo(sendCode, version, tmpMode); + packet->SetWaterMark(localMark, peerMark, deleteMark); + if (SyncOperation::TransferSyncMode(mode) == SyncModeType::PUSH_AND_PULL) { + packet->SetEndWaterMark(context->GetEndMark()); + packet->SetQuery(context->GetQuery()); + packet->SetSessionId(context->GetRequestSessionId()); + } + packet->SetQueryId(context->GetQuery().GetIdentify()); + CompressAlgorithm curAlgo = context->ChooseCompressAlgo(); + if (needCompressOnSync && curAlgo != CompressAlgorithm::NONE) { + packet->SetCompressDataMark(); + packet->SetCompressAlgo(curAlgo); + } + SetPacketId(packet, context, version); + if (curType == SyncType::QUERY_SYNC_TYPE && context->GetQuery().HasLimit()) { + packet->SetUpdateWaterMark(); + } + LOGD("[DataSync] curType=%d,local=%llu,del=%llu,end=%llu,label=%s,dev=%s,isCompress=%d", curType, localMark, + deleteMark, context->GetEndMark(), label_.c_str(), STR_MASK(GetDeviceId()), packet->IsCompressData()); +} + +int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) +{ + int errCode = RemoveDeviceDataIfNeed(context); + if (errCode != E_OK) { + return errCode; + } + SyncEntry syncData; // get data - int errCode = GetDataWithRerformanceRecord(context, outData); + errCode = GetDataWithPerformanceRecord(context, syncData); + // once get data occur E_EKEYREVOKED error, should also send request to remote dev to pull data. + if (SyncOperation::TransferSyncMode(mode) == SyncModeType::PUSH_AND_PULL && + context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && errCode == -E_EKEYREVOKED) { + errCode = E_OK; + } if (errCode != E_OK && errCode != -E_UNFINISHED) { LOGE("[DataSync][PushStart] get data failed, errCode=%d", errCode); return errCode; } - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; if (packet == nullptr) { LOGE("[DataSync][PushStart] new DataRequestPacket error"); return -E_OUT_OF_MEMORY; } - bool isUpdateWaterMark = (outData.size() > 0); - TimeStamp maxSendDateTime = GetMaxSendDataTime(outData, true, localMark); - TimeStamp minSendDateTime = GetMinSendDataTime(outData, localMark); - context->SetSequenceStartAndEndTimeStamp(minSendDateTime, maxSendDateTime); + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + UpdateWaterMark isUpdateWaterMark; + SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); + context->SetSequenceStartAndEndTimeStamp(dataTime); if (errCode == E_OK) { - context->SetSessionEndTimeStamp(maxSendDateTime); - packet->SetLastSequence(); + context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } - packet->SetData(outData); - packet->SetBasicInfo(errCode, version, localMark, peerMark, SyncOperation::PUSH); - SetPacketId(packet, context, version); - errCode = SendDataPacket(packet, context); + FillRequestPacket(packet, context, syncData, errCode, mode); + errCode = SendDataPacket(curType, packet, context); PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_MACHINE_START_TO_PUSH_SEND); } - if (errCode == E_OK && isUpdateWaterMark) { - SaveLocalWaterMark(context->GetDeviceId(), maxSendDateTime + 1); + if (errCode == E_OK) { + if (curType == SyncType::QUERY_SYNC_TYPE && context->GetQuery().HasLimit()) { + LOGI("[DataSync][RequestStart] query contain limit, no need to update watermark."); + return E_OK; + } + SyncTimeRange tmpDataTime = ReviseLocalMark(curType, dataTime, isUpdateWaterMark); + SaveLocalWaterMark(curType, context, tmpDataTime); } return errCode; } -int SingleVerDataSync::PushPullStart(SingleVerSyncTaskContext *context) +int SingleVerDataSync::PushStart(SingleVerSyncTaskContext *context) { if (context == nullptr) { return -E_INVALID_ARGS; } - WaterMark localMark; - WaterMark peerMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - LOGD("[DataSync][PushPull] localMark=%llu,endmark=%llu,peerMark=%llu,label=%s,dev=%s{private}", localMark, - context->GetEndMark(), peerMark, label_.c_str(), GetDeviceId().c_str()); - // get data - std::vector outData; - int errCode = GetDataWithRerformanceRecord(context, outData); - // once get data occur E_EKEYREVOKED error, should also send request to remote dev to pull data. - if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && errCode == -E_EKEYREVOKED) { - errCode = E_OK; - } - if (errCode != E_OK && errCode != -E_UNFINISHED) { - return errCode; - } + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + return RequestStart(context, + (curType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH : SyncModeType::PUSH); +} - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; - if (packet == nullptr) { - LOGE("[DataSync][PushPullStart] new DataRequestPacket error"); - return -E_OUT_OF_MEMORY; - } - bool isUpdateWaterMark = (outData.size() > 0); - TimeStamp maxSendDateTime = GetMaxSendDataTime(outData, true, localMark); - TimeStamp minSendDateTime = GetMinSendDataTime(outData, localMark); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - context->SetSequenceStartAndEndTimeStamp(minSendDateTime, maxSendDateTime); - if (errCode == E_OK) { - context->SetSessionEndTimeStamp(maxSendDateTime); - packet->SetLastSequence(); - } - packet->SetData(outData); - packet->SetBasicInfo(errCode, version, localMark, peerMark, SyncOperation::PUSH_AND_PULL); - packet->SetEndWaterMark(context->GetEndMark()); - packet->SetSessionId(context->GetRequestSessionId()); - SetPacketId(packet, context, version); - int sendErrCode = SendDataPacket(packet, context); - if (sendErrCode == E_OK && isUpdateWaterMark) { - SaveLocalWaterMark(context->GetDeviceId(), maxSendDateTime + 1); +int SingleVerDataSync::PushPullStart(SingleVerSyncTaskContext *context) +{ + if (context == nullptr) { + return -E_INVALID_ARGS; } - return sendErrCode; + return RequestStart(context, context->GetMode()); } int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) @@ -1007,28 +677,39 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) if (context == nullptr) { return -E_INVALID_ARGS; } + int errCode = RemoveDeviceDataIfNeed(context); + if (errCode != E_OK) { + return errCode; + } DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; if (packet == nullptr) { LOGE("[DataSync][PullRequest]new DataRequestPacket error"); return -E_OUT_OF_MEMORY; } - WaterMark peerMark; - WaterMark localMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - packet->SetLocalWaterMark(localMark); - packet->SetPeerWaterMark(peerMark); - packet->SetMode(SyncOperation::PULL); + int syncType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + WaterMark peerMark = 0; + WaterMark localMark = 0; + WaterMark deleteMark = 0; + GetPeerWaterMark(syncType, context->GetQuery().GetIdentify(), context->GetDeviceId(), peerMark); + GetLocalWaterMark(syncType, context->GetQuery().GetIdentify(), context, localMark); + GetLocalDeleteSyncWaterMark(context, deleteMark); + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); WaterMark endMark = context->GetEndMark(); + SyncTimeRange dataTime = {localMark, deleteMark, localMark, deleteMark}; + context->SetSequenceStartAndEndTimeStamp(dataTime); + + packet->SetBasicInfo(E_OK, version, context->GetMode()); + packet->SetWaterMark(localMark, peerMark, deleteMark); packet->SetEndWaterMark(endMark); packet->SetSessionId(context->GetRequestSessionId()); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - packet->SetVersion(version); + packet->SetQuery(context->GetQuery()); + packet->SetQueryId(context->GetQuery().GetIdentify()); packet->SetLastSequence(); SetPacketId(packet, context, version); - LOGD("[DataSync][PullRequest]peerMark=%llu,localMark=%llu,endMark=%llu,IsLastSeq=%d,label=%s,dev=%s{private}", - peerMark, localMark, endMark, packet->IsLastSequence(), label_.c_str(), GetDeviceId().c_str()); - return SendDataPacket(packet, context); + + LOGD("[DataSync][Pull] curType=%d,local=%llu,del=%llu,end=%llu,peer=%llu,label=%s,dev=%s", syncType, localMark, + deleteMark, peerMark, endMark, label_.c_str(), STR_MASK(GetDeviceId())); + return SendDataPacket(syncType, packet, context); } int SingleVerDataSync::PullResponseStart(SingleVerSyncTaskContext *context) @@ -1036,22 +717,15 @@ int SingleVerDataSync::PullResponseStart(SingleVerSyncTaskContext *context) if (context == nullptr) { return -E_INVALID_ARGS; } - WaterMark localMark; - WaterMark peerMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - LOGD("[DataSync][PullResponse] localMark=%llu,pullendmark=%llu,peerMark=%llu,label=%s,dev=%s{private}", - localMark, context->GetEndMark(), peerMark, label_.c_str(), GetDeviceId().c_str()); // get data - std::vector outData; - int errCode = GetDataWithRerformanceRecord(context, outData); + SyncEntry syncData; + int errCode = GetDataWithPerformanceRecord(context, syncData); if (errCode != E_OK && errCode != -E_UNFINISHED) { if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - SendPullResponseDataPkt(errCode, outData, context); + SendPullResponseDataPkt(errCode, syncData, context); } return errCode; } - // if send finished int ackCode = E_OK; ContinueToken token = nullptr; @@ -1060,33 +734,60 @@ int SingleVerDataSync::PullResponseStart(SingleVerSyncTaskContext *context) LOGD("[DataSync][PullResponse] send last frame end"); ackCode = SEND_FINISHED; } - bool isUpdateWaterMark = (outData.size() > 0); - TimeStamp maxSendDateTime = GetMaxSendDataTime(outData, true, localMark); - TimeStamp minSendDateTime = GetMinSendDataTime(outData, localMark); - context->SetSequenceStartAndEndTimeStamp(minSendDateTime, maxSendDateTime); + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + UpdateWaterMark isUpdateWaterMark; + SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); + context->SetSequenceStartAndEndTimeStamp(dataTime); if (errCode == E_OK) { - context->SetSessionEndTimeStamp(maxSendDateTime); + context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } - errCode = SendPullResponseDataPkt(ackCode, outData, context); - if (errCode == E_OK && isUpdateWaterMark) { - SaveLocalWaterMark(context->GetDeviceId(), maxSendDateTime + 1); + errCode = SendPullResponseDataPkt(ackCode, syncData, context); + if (errCode == E_OK) { + if (curType == SyncType::QUERY_SYNC_TYPE && context->GetQuery().HasLimit()) { + LOGI("[DataSync][PullResponseStart] query contain limit, no need to update watermark."); + return E_OK; + } + SyncTimeRange tmpDataTime = ReviseLocalMark(curType, dataTime, isUpdateWaterMark); + SaveLocalWaterMark(curType, context, tmpDataTime); } return errCode; } -void SingleVerDataSync::UpdatePeerWaterMark(const SingleVerSyncTaskContext *context, WaterMark peerWatermark) +void SingleVerDataSync::UpdateQueryPeerWaterMark(int syncType, const std::string &queryId, SyncTimeRange &dataTime, + const SingleVerSyncTaskContext *context, UpdateWaterMark isUpdateWaterMark) { - if (peerWatermark == 0) { - return; + WaterMark tmpPeerWatermark = dataTime.endTime; + WaterMark tmpPeerDeletedWatermark = dataTime.deleteEndTime; + if (isUpdateWaterMark.normalUpdateMark) { + tmpPeerWatermark++; } - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SAVE_PEER_WATERMARK); + if (isUpdateWaterMark.deleteUpdateMark) { + tmpPeerDeletedWatermark++; } + UpdatePeerWaterMark(syncType, queryId, context, tmpPeerWatermark, tmpPeerDeletedWatermark); +} - int errCode = SavePeerWaterMark(context->GetDeviceId(), peerWatermark + 1); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SAVE_PEER_WATERMARK); +void SingleVerDataSync::UpdatePeerWaterMark(int syncType, const std::string &queryId, + const SingleVerSyncTaskContext *context, WaterMark peerWatermark, WaterMark peerDeletedWatermark) +{ + if (peerWatermark == 0 && peerDeletedWatermark == 0) { + return; + } + int errCode = E_OK; + if (syncType != SyncType::QUERY_SYNC_TYPE) { + errCode = metadata_->SavePeerWaterMark(context->GetDeviceId(), peerWatermark, true); + } else { + if (peerWatermark != 0) { + LOGD("label=%s,dev=%s,endTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), peerWatermark); + errCode = metadata_->SetRecvQueryWaterMark(queryId, context->GetDeviceId(), peerWatermark); + if (errCode != E_OK) { + LOGE("[DataSync][UpdatePeerWaterMark] save query peer water mark failed,errCode=%d", errCode); + } + } + if (peerDeletedWatermark != 0) { + LOGD("label=%s,dev=%s,peerDeletedTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), peerDeletedWatermark); + errCode = metadata_->SetRecvDeleteSyncWaterMark(context->GetDeviceId(), peerDeletedWatermark); + } } if (errCode != E_OK) { LOGE("[DataSync][UpdatePeerWaterMark] save peer water mark failed,errCode=%d", errCode); @@ -1140,6 +841,10 @@ int SingleVerDataSync::RequestRecvPre(SingleVerSyncTaskContext *context, const M if (version > SOFTWARE_VERSION_RELEASE_2_0) { errCode = CheckSchemaStrategy(context, message); } + if (IsQueryByKeyCheck(packet->GetMode(), packet->GetQuery(), context, message) != E_OK) { + return -E_WAIT_NEXT_MESSAGE; + } + return errCode; } @@ -1152,60 +857,60 @@ int SingleVerDataSync::RequestRecv(SingleVerSyncTaskContext *context, const Mess } const DataRequestPacket *packet = message->GetObject(); const std::vector &data = packet->GetData(); - LOGI("[DataSync][RequestRecv] remote ver=%u,size=%d,errCode=%d,Label=%s,dev=%s{private}", packet->GetVersion(), - data.size(), packet->GetSendCode(), label_.c_str(), GetDeviceId().c_str()); - WaterMark packetLocalMark = packet->GetLocalWaterMark(); - WaterMark peerMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); + int curType = SyncOperation::GetSyncType(packet->GetMode()); + LOGI("[DataSync][RequestRecv] curType=%d,remote ver=%u,size=%d,errCode=%d,Label=%s,dev=%s", curType, + packet->GetVersion(), data.size(), packet->GetSendCode(), label_.c_str(), STR_MASK(GetDeviceId())); context->SetReceiveWaterMarkErr(false); - WaterMark maxSendDataTime = GetMaxSendDataTime(data, false); - if (packetLocalMark > peerMark) { - LOGI("[DataSync][RequestRecv] packetLocalMark=%llu,current=%llu", packetLocalMark, peerMark); - return SendLocalWaterMarkAck(context, message); - } else if ((packetLocalMark == 0) && (peerMark != 0) && context->IsNeedClearRemoteStaleData()) { - // need to clear remote device history data - errCode = RemoveDeviceData(context, message, maxSendDataTime); - if (errCode != E_OK) { - return errCode; - } + UpdateWaterMark isUpdateWaterMark; + SyncTimeRange dataTime = GetRecvDataTimeRange(curType, context, data, isUpdateWaterMark); + errCode = RemoveDeviceDataHandle(context, message, dataTime.endTime); + if (errCode != E_OK) { + return errCode; + } + bool isWaterErr = WaterMarkErrHandle(curType, context, message); + if (isWaterErr) { + return E_OK; } GetPullEndWatermark(context, packet, pullEndWatermark); // save data first errCode = SaveData(context, data); if (errCode != E_OK) { - (void)SendAck(context, message, errCode, maxSendDataTime); + (void)SendAck(context, message, errCode, dataTime.endTime); return errCode; } if (pullEndWatermark > 0 && !storage_->IsReadable()) { // pull mode pullEndWatermark = 0; - errCode = SendAck(context, message, -E_EKEYREVOKED, maxSendDataTime); + errCode = SendAck(context, message, -E_EKEYREVOKED, dataTime.endTime); } else { // if data is empty, we don't know the max timestap of this packet. - errCode = SendAck(context, message, !data.empty() ? E_OK : WATER_MARK_INVALID, maxSendDataTime); + errCode = SendAck(context, message, !data.empty() ? E_OK : WATER_MARK_INVALID, dataTime.endTime); } RemotePushFinished(packet->GetSendCode(), packet->GetMode(), message->GetSessionId(), context->GetRequestSessionId()); - UpdatePeerWaterMark(context, maxSendDataTime); + if (curType != SyncType::QUERY_SYNC_TYPE && isUpdateWaterMark.normalUpdateMark) { + UpdatePeerWaterMark(curType, "", context, dataTime.endTime + 1, 0); + } else if (curType == SyncType::QUERY_SYNC_TYPE && packet->IsNeedUpdateWaterMark()) { + UpdateQueryPeerWaterMark(curType, packet->GetQueryId(), dataTime, context, isUpdateWaterMark); + } if (errCode != E_OK) { return errCode; } if (packet->GetSendCode() == SEND_FINISHED) { return -E_RECV_FINISHED; } - return errCode; } -int SingleVerDataSync::SendDataPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context) +int SingleVerDataSync::SendDataPacket(int syncType, const DataRequestPacket *packet, SingleVerSyncTaskContext *context) { - Message *message = new (std::nothrow) Message(DATA_SYNC_MESSAGE); + Message *message = new (std::nothrow) Message(GetMessageId(syncType)); if (message == nullptr) { LOGE("[DataSync][SendDataPacket] new message error"); delete packet; packet = nullptr; return -E_OUT_OF_MEMORY; } - uint32_t packetLen = packet->CalculateLen(); + uint32_t packetLen = packet->CalculateLen(GetMessageId(syncType)); int errCode = message->SetExternalObject(packet); if (errCode != E_OK) { delete packet; @@ -1221,7 +926,6 @@ int SingleVerDataSync::SendDataPacket(const DataRequestPacket *packet, SingleVer if (performance != nullptr) { performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_SEND_REQUEST_TO_ACK_RECV); } - CommErrHandler handler = std::bind(&SyncTaskContext::CommErrHandlerFunc, std::placeholders::_1, context, message->GetSessionId()); errCode = Send(context, message, handler, packetLen); @@ -1233,7 +937,7 @@ int SingleVerDataSync::SendDataPacket(const DataRequestPacket *packet, SingleVer return errCode; } -int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, std::vector &inData, +int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, SyncEntry &syncOutData, SingleVerSyncTaskContext *context) { DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; @@ -1241,28 +945,16 @@ int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, std::vectorGetLocalWaterMark(context->GetDeviceId(), localMark); - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - packet->SetBasicInfo(ackCode, version, localMark, peerMark, SyncOperation::PUSH); - if (ackCode == SEND_FINISHED) { - packet->SetLastSequence(); - } - SetPacketId(packet, context, version); - packet->SetData(inData); - uint32_t packetLen = packet->CalculateLen(); - Message *message = new (std::nothrow) Message(DATA_SYNC_MESSAGE); + int syncType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + FillRequestPacket(packet, context, syncOutData, ackCode, SyncModeType::RESPONSE_PULL); + uint32_t packetLen = packet->CalculateLen(GetMessageId(syncType)); + Message *message = new (std::nothrow) Message(GetMessageId(syncType)); if (message == nullptr) { LOGE("[DataSync][SendPullResponseDataPkt] new message error"); delete packet; packet = nullptr; return -E_OUT_OF_MEMORY; } - LOGI("[DataSync][SendPullResponseDataPkt] size=%d,code=%d,LastSequence=%d,label=%s,dev=%s{private}", - size, ackCode, packet->IsLastSequence(), label_.c_str(), GetDeviceId().c_str()); int errCode = message->SetExternalObject(packet); if (errCode != E_OK) { delete packet; @@ -1275,7 +967,6 @@ int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, std::vectorGetDeviceId(), context->GetSequenceId(), context->GetResponseSessionId()); SendResetWatchDogPacket(context, packetLen); - errCode = Send(context, message, nullptr, packetLen); if (errCode != E_OK) { delete message; @@ -1284,19 +975,6 @@ int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, std::vectorGetPeerWaterMark(context->GetDeviceId(), mark); - ackPacket.SetData(mark); - } -} - int SingleVerDataSync::SendAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, WaterMark maxSendDataTime) { @@ -1304,27 +982,13 @@ int SingleVerDataSync::SendAck(SingleVerSyncTaskContext *context, const Message if (packet == nullptr) { return -E_INVALID_ARGS; } - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); + Message *ackMessage = new (std::nothrow) Message(message->GetMessageId()); if (ackMessage == nullptr) { LOGE("[DataSync][SendAck] new message error"); return -E_OUT_OF_MEMORY; } DataAckPacket ack; - ack.SetRecvCode(recvCode); - SetAckData(ack, context, recvCode, maxSendDataTime); - WaterMark localMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - std::vector reserved {localMark}; - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - uint64_t packetId = 0; - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value - } - if (version > SOFTWARE_VERSION_RELEASE_2_0 && packetId > 0) { - reserved.push_back(packetId); - } - ack.SetReserved(reserved); - ack.SetVersion(version); + SetAckPacket(ack, context, packet, recvCode, maxSendDataTime); int errCode = ackMessage->SetCopiedObject(ack); if (errCode != E_OK) { delete ackMessage; @@ -1340,59 +1004,6 @@ int SingleVerDataSync::SendAck(SingleVerSyncTaskContext *context, const Message delete ackMessage; ackMessage = nullptr; } - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); - } - return errCode; -} - -int SingleVerDataSync::SendLocalWaterMarkAck(SingleVerSyncTaskContext *context, const Message *message) -{ - context->SetReceiveWaterMarkErr(true); - const DataRequestPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); - if (ackMessage == nullptr) { - LOGE("[DataSync][LocalWaterMarkAck] new message error"); - return -E_OUT_OF_MEMORY; - } - - DataAckPacket ack; - WaterMark peerMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - ack.SetData(peerMark); - WaterMark localMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - uint64_t packetId = 0; - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value - } - std::vector reserved {localMark}; - if (version > SOFTWARE_VERSION_RELEASE_2_0 && packetId > 0) { - reserved.push_back(packetId); - } - ack.SetReserved(reserved); - ack.SetRecvCode(LOCAL_WATER_MARK_NOT_INIT); - ack.SetVersion(version); - int errCode = ackMessage->SetCopiedObject(ack); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - LOGE("[DataSync][LocalWaterMarkAck] set copied object failed, errcode=%d", errCode); - return errCode; - } - SetMessageHeadInfo(*ackMessage, TYPE_RESPONSE, context->GetDeviceId(), message->GetSequenceId(), - message->GetSessionId()); - errCode = Send(context, ackMessage, nullptr, 0); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - } - LOGI("[DataSync][LocalWaterMarkAck] LOCAL_WATER_MARK_NOT_INIT peerMark=%llu,localMark=%llu", peerMark, localMark); return errCode; } @@ -1407,8 +1018,8 @@ int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message return -E_INVALID_ARGS; } int32_t recvCode = packet->GetRecvCode(); - LOGD("[DataSync][AckRecv] ver=%u,recvCode=%d,myversion=%u,label=%s,dev=%s{private}", packet->GetVersion(), - recvCode, SOFTWARE_VERSION_CURRENT, label_.c_str(), GetDeviceId().c_str()); + LOGD("[DataSync][AckRecv] ver=%u,recvCode=%d,myversion=%u,label=%s,dev=%s", packet->GetVersion(), recvCode, + SOFTWARE_VERSION_CURRENT, label_.c_str(), STR_MASK(GetDeviceId())); if (recvCode == -E_VERSION_NOT_SUPPORT) { LOGE("[DataSync][AckRecv] Version mismatch"); return -E_VERSION_NOT_SUPPORT; @@ -1417,8 +1028,8 @@ int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message if (recvCode == -E_NEED_ABILITY_SYNC || recvCode == -E_NOT_PERMIT) { // after set sliding window sender err, we can ReleaseContinueToken, avoid crash context->SetSlidingWindowSenderErr(true); - LOGI("[DataSync][AckRecv] Data sync abort,recvCode = %d,label = %s,dev = %s{private}", recvCode, - label_.c_str(), GetDeviceId().c_str()); + LOGI("[DataSync][AckRecv] Data sync abort,recvCode =%d,label =%s,dev=%s", recvCode, label_.c_str(), + STR_MASK(GetDeviceId())); context->ReleaseContinueToken(); return recvCode; } @@ -1430,13 +1041,13 @@ int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message if (recvCode == -E_SAVE_DATA_NOTIFY && data != 0) { // data only use low 32bit context->StartFeedDogForSync(static_cast(data), SyncDirectionFlag::RECEIVE); - LOGI("[DataSync][AckRecv] notify ResetWatchDog=%llu,label=%s,dev=%s{private}", data, label_.c_str(), - GetDeviceId().c_str()); + LOGI("[DataSync][AckRecv] notify ResetWatchDog=%llu,label=%s,dev=%s", data, label_.c_str(), + STR_MASK(GetDeviceId())); } if (recvCode != E_OK && recvCode != WATER_MARK_INVALID) { - LOGW("[DataSync][AckRecv] Received a uncatched recvCode=%d,label=%s,dev=%s{private}", recvCode, - label_.c_str(), GetDeviceId().c_str()); + LOGW("[DataSync][AckRecv] Received a uncatched recvCode=%d,label=%s,dev=%s", recvCode, + label_.c_str(), STR_MASK(GetDeviceId())); return recvCode; } @@ -1453,9 +1064,13 @@ int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message } void SingleVerDataSync::SendSaveDataNotifyPacket(SingleVerSyncTaskContext *context, uint32_t pktVersion, - uint32_t sessionId, uint32_t sequenceId) + uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) { - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); + if (inMsgId != DATA_SYNC_MESSAGE && inMsgId != QUERY_SYNC_MESSAGE) { + LOGE("[SingleVerDataSync] messageId not avaiable."); + return; + } + Message *ackMessage = new (std::nothrow) Message(inMsgId); if (ackMessage == nullptr) { LOGE("[DataSync][SaveDataNotify] new message failed"); return; @@ -1478,8 +1093,8 @@ void SingleVerDataSync::SendSaveDataNotifyPacket(SingleVerSyncTaskContext *conte delete ackMessage; ackMessage = nullptr; } - LOGD("[DataSync][SaveDataNotify] Send SaveDataNotify packet Finished,errcode=%d,label=%s,dev=%s{private}", - errCode, label_.c_str(), GetDeviceId().c_str()); + LOGD("[DataSync][SaveDataNotify] Send SaveDataNotify packet Finished,errcode=%d,label=%s,dev=%s", + errCode, label_.c_str(), STR_MASK(GetDeviceId())); } void SingleVerDataSync::GetPullEndWatermark(const SingleVerSyncTaskContext *context, const DataRequestPacket *packet, @@ -1488,13 +1103,14 @@ void SingleVerDataSync::GetPullEndWatermark(const SingleVerSyncTaskContext *cont if (packet == nullptr) { return; } - if ((packet->GetMode() == SyncOperation::PULL) || (packet->GetMode() == SyncOperation::PUSH_AND_PULL)) { + int mode = SyncOperation::TransferSyncMode(packet->GetMode()); + if ((mode == SyncModeType::PULL) || (mode == SyncModeType::PUSH_AND_PULL)) { WaterMark endMark = packet->GetEndWaterMark(); TimeOffset offset; metadata_->GetTimeOffset(context->GetDeviceId(), offset); pullEndWatermark = endMark - offset; - LOGD("[DataSync][PullEndWatermark] packetEndMark=%llu,offset=%llu,endWaterMark=%llu,label=%s,dev=%s{private}", - endMark, offset, pullEndWatermark, label_.c_str(), GetDeviceId().c_str()); + LOGD("[DataSync][PullEndWatermark] packetEndMark=%llu,offset=%llu,endWaterMark=%llu,label=%s,dev=%s", + endMark, offset, pullEndWatermark, label_.c_str(), STR_MASK(GetDeviceId())); } } @@ -1504,45 +1120,30 @@ int SingleVerDataSync::DealWaterMarkException(SingleVerSyncTaskContext *context, // after set sliding window sender err, we can SaveLocalWaterMark, avoid sliding window sender re save a wrong // waterMark again. context->SetSlidingWindowSenderErr(true); - WaterMark localMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - LOGI("[DataSync][WaterMarkException] AckRecv LOCAL_WATER_MARK_NOT_INIT mark=%llu,label=%s,dev=%s{private}", - ackWaterMark, label_.c_str(), GetDeviceId().c_str()); - int errCode = SaveLocalWaterMark(context->GetDeviceId(), ackWaterMark); + WaterMark deletedWaterMark = 0; + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + if (curType == SyncType::QUERY_SYNC_TYPE) { + if (reserved.size() <= ACK_PACKET_RESERVED_INDEX_DELETE_WATER_MARK) { + LOGE("[DataSync] get packet reserve size failed"); + return -E_INVALID_ARGS; + } + deletedWaterMark = reserved[ACK_PACKET_RESERVED_INDEX_DELETE_WATER_MARK]; + } + LOGI("[DataSync][WaterMarkException] AckRecv water error, mark=%llu,deleteMark=%llu,label=%s,dev=%s", ackWaterMark, + deletedWaterMark, label_.c_str(), STR_MASK(GetDeviceId())); + int errCode = SaveLocalWaterMark(curType, context, {0, 0, ackWaterMark, deletedWaterMark}); if (errCode != E_OK) { return errCode; } context->SetRetryStatus(SyncTaskContext::NEED_RETRY); - // for push_and_pull mode it may be EKEYREVOKED error before receive watermarkexception - // should clear errCode and restart pushpull request. - if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && - context->GetMode() == SyncOperation::PUSH_AND_PULL && context->GetTaskErrCode() == -E_EKEYREVOKED) { - context->SetTaskErrCode(E_OK); + context->IncNegotiationCount(); + PushAndPUllKeyRevokHandle(context); + if (!context->IsNeedClearRemoteStaleData()) { + return -E_RE_SEND_DATA; } - - if (reserved.empty()) { - if (localMark != 0 && ackWaterMark == 0 && context->IsNeedClearRemoteStaleData()) { - // need to clear remote historydata - LOGI("[DataSync][WaterMarkException] AckRecv rebuilted,clear historydata,label=%s,dev=%s{private}", - label_.c_str(), GetDeviceId().c_str()); - errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); - if (errCode != E_OK) { - return errCode; - } - } - } else { - WaterMark peerMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - if (reserved[ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK] == 0 && peerMark != 0 && - context->IsNeedClearRemoteStaleData()) { - // need to clear remote historydata - LOGI("[DataSync][WaterMarkException] AckRecv reserved not empty,rebuilted,clear historydata,label=%s," - "dev = %s{private}", label_.c_str(), GetDeviceId().c_str()); - errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); - if (errCode != E_OK) { - return errCode; - } - } + errCode = DealRemoveDeviceDataByAck(context, ackWaterMark, reserved); + if (errCode != E_OK) { + return errCode; } return -E_RE_SEND_DATA; } @@ -1554,12 +1155,12 @@ int SingleVerDataSync::RunPermissionCheck(SingleVerSyncTaskContext *context, con std::string userId = storage_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); std::string storeId = storage_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); uint8_t flag; - int32_t mode = packet->GetMode(); - if (mode == SyncOperation::PUSH) { + int mode = SyncOperation::TransferSyncMode(packet->GetMode()); + if (mode == SyncModeType::PUSH) { flag = CHECK_FLAG_RECEIVE; - } else if (mode == SyncOperation::PULL) { + } else if (mode == SyncModeType::PULL) { flag = CHECK_FLAG_SEND; - } else if (mode == SyncOperation::PUSH_AND_PULL) { + } else if (mode == SyncModeType::PUSH_AND_PULL) { flag = CHECK_FLAG_SEND | CHECK_FLAG_RECEIVE; } else { // before add permissionCheck, PushStart packet and pullResponse packet do not setMode. @@ -1568,17 +1169,18 @@ int SingleVerDataSync::RunPermissionCheck(SingleVerSyncTaskContext *context, con int errCode = RuntimeContext::GetInstance()->RunPermissionCheck(userId, appId, storeId, context->GetDeviceId(), flag); if (errCode != E_OK) { - LOGE("[DataSync][RunPermissionCheck] check failed flag=%d,Label=%s,dev=%s{private}", flag, label_.c_str(), - GetDeviceId().c_str()); + LOGE("[DataSync][RunPermissionCheck] check failed flag=%d,Label=%s,dev=%s", flag, label_.c_str(), + STR_MASK(GetDeviceId())); if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_EARLIEST) { // ver 101 can't handle this errCode (void)SendAck(context, message, -E_NOT_PERMIT, 0); } return -E_NOT_PERMIT; } const std::vector &data = packet->GetData(); - WaterMark maxSendDataTime = GetMaxSendDataTime(data, false); + WaterMark maxSendDataTime = GetMaxSendDataTime(data); uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - if (version > SOFTWARE_VERSION_RELEASE_2_0 && (mode != SyncOperation::PULL) && !context->GetReceivcPermitCheck()) { + if (version > SOFTWARE_VERSION_RELEASE_2_0 && (mode != SyncModeType::PULL) && + !context->GetReceivcPermitCheck()) { bool permitReceive = CheckPermitReceiveData(context); if (permitReceive) { context->SetReceivcPermitCheck(true); @@ -1593,10 +1195,10 @@ int SingleVerDataSync::RunPermissionCheck(SingleVerSyncTaskContext *context, con // used in pull response void SingleVerDataSync::SendResetWatchDogPacket(SingleVerSyncTaskContext *context, uint32_t packetLen) { - if (mtuSize_ >= packetLen || mtuSize_ == 0) { + if (mtuSize_ >= packetLen || mtuSize_ < FEEDDOG_MAX_MTU_SIZE) { return; } - uint64_t data = static_cast(packetLen) * static_cast(DBConstant::AUTO_SYNC_TIMEOUT) / mtuSize_; + uint64_t data = static_cast(packetLen) * static_cast(context->GetTimeoutTime()) / mtuSize_; Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); if (ackMessage == nullptr) { @@ -1622,45 +1224,10 @@ void SingleVerDataSync::SendResetWatchDogPacket(SingleVerSyncTaskContext *contex if (errCode != E_OK) { delete ackMessage; ackMessage = nullptr; - LOGE("[DataSync][ResetWatchDog] Send packet failed,errcode=%d,label=%s,dev = %s{private}", errCode, - label_.c_str(), GetDeviceId().c_str()); + LOGE("[DataSync][ResetWatchDog] Send packet failed,errcode=%d,label=%s,dev=%s", errCode, label_.c_str(), + STR_MASK(GetDeviceId())); } else { - LOGI("[DataSync][ResetWatchDog] data = %llu,label=%s,dev=%s{private}", data, label_.c_str(), - GetDeviceId().c_str()); - } -} - -void SingleVerDataSync::SendAck(SingleVerSyncTaskContext *context, uint32_t sessionId, uint32_t sequenceId, - uint64_t packetId) -{ - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); - if (ackMessage == nullptr) { - LOGE("[DataSync][SendAck] new message error"); - return; - } - DataAckPacket ack; - ack.SetRecvCode(E_OK); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - WaterMark localMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - std::vector reserved {localMark}; - if (version > SOFTWARE_VERSION_RELEASE_2_0 && packetId > 0) { - reserved.push_back(packetId); - } - ack.SetReserved(reserved); - ack.SetVersion(version); - int errCode = ackMessage->SetCopiedObject(ack); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - LOGE("[DataSync][SendAck] set copied object failed, errcode=%d", errCode); - return; - } - SetMessageHeadInfo(*ackMessage, TYPE_RESPONSE, context->GetDeviceId(), sequenceId, sessionId); - errCode = Send(context, ackMessage, nullptr, 0); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; + LOGI("[DataSync][ResetWatchDog] data = %llu,label=%s,dev=%s", data, label_.c_str(), STR_MASK(GetDeviceId())); } } @@ -1669,48 +1236,29 @@ int32_t SingleVerDataSync::ReSend(SingleVerSyncTaskContext *context, DataSyncReS if (context == nullptr) { return -E_INVALID_ARGS; } - LOGI("[DataSync][ReSend] start=%llu,end=%llu,label=%s,dev=%s{private}", reSendInfo.start, reSendInfo.end, - label_.c_str(), GetDeviceId().c_str()); - std::vector outData; - ContinueToken token = nullptr; - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - size_t packetSize = (version > SOFTWARE_VERSION_RELEASE_2_0) ? - DBConstant::MAX_HPMODE_PACK_ITEM_SIZE : DBConstant::MAX_NORMAL_PACK_ITEM_SIZE; - DataSizeSpecInfo reSendDataSizeInfo = {mtuSize_, packetSize}; - int errCode = storage_->GetSyncData(reSendInfo.start, reSendInfo.end + 1, outData, token, reSendDataSizeInfo); - if (token != nullptr) { - storage_->ReleaseContinueToken(token); - } - if (errCode == -E_BUSY || errCode == -E_EKEYREVOKED) { - context->SetTaskErrCode(errCode); - return errCode; - } + SyncEntry syncData; + int errCode = GetReSendData(syncData, context, reSendInfo); if (errCode != E_OK && errCode != -E_UNFINISHED) { return errCode; } - WaterMark localMark; - WaterMark peerMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; if (packet == nullptr) { LOGE("[DataSync][ReSend] new DataRequestPacket error"); return -E_OUT_OF_MEMORY; } - if (context->GetSessionEndTimeStamp() == reSendInfo.end) { - LOGI("[DataSync][ReSend] lastid,label=%s,dev=%s{private}", label_.c_str(), GetDeviceId().c_str()); - packet->SetLastSequence(); - } - packet->SetData(outData); - packet->SetBasicInfo(errCode, version, reSendInfo.start, peerMark, SyncOperation::PUSH); - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - std::vector reserved {reSendInfo.packetId}; - packet->SetReserved(reserved); - } + FillRequestReSendPacket(context, packet, reSendInfo, syncData, errCode); errCode = SendReSendPacket(packet, context, reSendInfo.sessionId, reSendInfo.sequenceId); - if (errCode == E_OK && localMark < reSendInfo.end) { + if (errCode == E_OK && SyncOperation::TransferSyncMode(context->GetMode()) != SyncModeType::PULL) { // resend.end may not update in localwatermark while E_TIMEOUT occurred in send message last time. - SaveLocalWaterMark(context->GetDeviceId(), reSendInfo.end + 1); + SyncTimeRange dataTime {reSendInfo.start, reSendInfo.deleteDataStart, reSendInfo.end, reSendInfo.deleteDataEnd}; + if (reSendInfo.deleteDataEnd > reSendInfo.deleteDataStart && curType == SyncType::QUERY_SYNC_TYPE) { + dataTime.deleteEndTime += 1; + } + if (reSendInfo.end > reSendInfo.start) { + dataTime.endTime += 1; + } + SaveLocalWaterMark(curType, context, dataTime, true); } return errCode; } @@ -1718,14 +1266,15 @@ int32_t SingleVerDataSync::ReSend(SingleVerSyncTaskContext *context, DataSyncReS int SingleVerDataSync::SendReSendPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context, uint32_t sessionId, uint32_t sequenceId) { - Message *message = new (std::nothrow) Message(DATA_SYNC_MESSAGE); + int syncType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + Message *message = new (std::nothrow) Message(GetMessageId(syncType)); if (message == nullptr) { LOGE("[DataSync][SendDataPacket] new message error"); delete packet; packet = nullptr; return -E_OUT_OF_MEMORY; } - uint32_t packetLen = packet->CalculateLen(); + uint32_t packetLen = packet->CalculateLen(GetMessageId(syncType)); int errCode = message->SetExternalObject(packet); if (errCode != E_OK) { delete packet; @@ -1766,11 +1315,12 @@ bool SingleVerDataSync::IsPermitLocalDeviceRecvData(const std::string &deviceId, return RuntimeContext::GetInstance()->CheckDeviceSecurityAbility(deviceId, remoteSecOption); } -int SingleVerDataSync::CheckPermitSendData(int mode, SingleVerSyncTaskContext *context) +int SingleVerDataSync::CheckPermitSendData(int inMode, SingleVerSyncTaskContext *context) { uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + int mode = SyncOperation::TransferSyncMode(inMode); // for pull mode it just need to get data, no need to send data. - if (version <= SOFTWARE_VERSION_RELEASE_2_0 || mode == SyncOperation::PULL) { + if (version <= SOFTWARE_VERSION_RELEASE_2_0 || mode == SyncModeType::PULL) { return E_OK; } if (context->GetSendPermitCheck()) { @@ -1779,22 +1329,22 @@ int SingleVerDataSync::CheckPermitSendData(int mode, SingleVerSyncTaskContext *c bool isPermitSync = true; std::string deviceId = context->GetDeviceId(); SecurityOption remoteSecOption = context->GetRemoteSeccurityOption(); - if (mode == SyncOperation::PUSH || mode == SyncOperation::PUSH_AND_PULL || mode == SyncOperation::RESPONSE_PULL) { + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::RESPONSE_PULL) { isPermitSync = IsPermitRemoteDeviceRecvData(deviceId, remoteSecOption); } - LOGI("[DataSync][PermitSendData] mode=%d,dev=%s{private},label=%d,flag=%d,PermitSync=%d", mode, deviceId.c_str(), + LOGI("[DataSync][PermitSendData] mode=%d,dev=%s,label=%d,flag=%d,PermitSync=%d", mode, STR_MASK(deviceId_), remoteSecOption.securityLabel, remoteSecOption.securityFlag, isPermitSync); if (isPermitSync) { context->SetSendPermitCheck(true); return E_OK; } - if (mode == SyncOperation::PUSH || mode == SyncOperation::PUSH_AND_PULL) { + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { context->SetTaskErrCode(-E_SECURITY_OPTION_CHECK_ERROR); return -E_SECURITY_OPTION_CHECK_ERROR; } - if (mode == SyncOperation::RESPONSE_PULL) { - std::vector outData; - SendPullResponseDataPkt(-E_SECURITY_OPTION_CHECK_ERROR, outData, context); + if (mode == SyncModeType::RESPONSE_PULL) { + SyncEntry syncData; + SendPullResponseDataPkt(-E_SECURITY_OPTION_CHECK_ERROR, syncData, context); return -E_SECURITY_OPTION_CHECK_ERROR; } return E_OK; @@ -1810,6 +1360,37 @@ std::string SingleVerDataSync::GetDeviceId() const return deviceId_; } +bool SingleVerDataSync::WaterMarkErrHandle(int syncType, SingleVerSyncTaskContext *context, const Message *message) +{ + const DataRequestPacket *packet = message->GetObject(); + if (packet == nullptr) { + LOGE("[WaterMarkErrHandle] get packet object failed"); + return -E_INVALID_ARGS; + } + WaterMark packetLocalMark = packet->GetLocalWaterMark(); + WaterMark packetDeletedMark = packet->GetDeletedWaterMark(); + WaterMark peerMark = 0; + WaterMark deletedMark = 0; + GetPeerWaterMark(syncType, packet->GetQueryId(), context->GetDeviceId(), peerMark); + if (syncType == SyncType::QUERY_SYNC_TYPE) { + GetPeerDeleteSyncWaterMark(context->GetDeviceId(), deletedMark); + } + if (syncType != SyncType::QUERY_SYNC_TYPE && packetLocalMark > peerMark) { + LOGI("[DataSync][RequestRecv] packetLocalMark=%llu,current=%llu", packetLocalMark, peerMark); + context->SetReceiveWaterMarkErr(true); + SendAck(context, message, LOCAL_WATER_MARK_NOT_INIT, 0); + return true; + } + if (syncType == SyncType::QUERY_SYNC_TYPE && (packetLocalMark > peerMark || packetDeletedMark > deletedMark)) { + LOGI("[DataSync][RequestRecv] packetDeletedMark=%llu,deletedMark=%llu,packetLocalMark=%llu,peerMark=%llu", + packetDeletedMark, deletedMark, packetLocalMark, peerMark); + context->SetReceiveWaterMarkErr(true); + SendAck(context, message, LOCAL_WATER_MARK_NOT_INIT, 0); + return true; + } + return false; +} + bool SingleVerDataSync::CheckPermitReceiveData(const SingleVerSyncTaskContext *context) { SecurityOption remoteSecOption = context->GetRemoteSeccurityOption(); @@ -1822,8 +1403,8 @@ bool SingleVerDataSync::CheckPermitReceiveData(const SingleVerSyncTaskContext *c if (isPermitSync) { return isPermitSync; } - LOGE("[DataSync][PermitReceiveData] check failed: permitReceive=%d, localDev=%s{private}, seclabel=%d, secflag=%d", - isPermitSync, localDeviceId.c_str(), remoteSecOption.securityLabel, remoteSecOption.securityFlag); + LOGE("[DataSync][PermitReceiveData] check failed: permitReceive=%d, localDev=%s, seclabel=%d, secflag=%d", + isPermitSync, STR_MASK(localDeviceId), remoteSecOption.securityLabel, remoteSecOption.securityFlag); return isPermitSync; } @@ -1843,9 +1424,11 @@ int SingleVerDataSync::CheckSchemaStrategy(SingleVerSyncTaskContext *context, co return E_OK; } -void SingleVerDataSync::RemotePushFinished(int sendCode, int mode, uint32_t msgSessionId, uint32_t contextSessionId) +void SingleVerDataSync::RemotePushFinished(int sendCode, int inMode, uint32_t msgSessionId, uint32_t contextSessionId) { - if ((mode != SyncOperation::PUSH) && (mode != SyncOperation::PUSH_AND_PULL)) { + int mode = SyncOperation::TransferSyncMode(inMode); + if ((mode != SyncModeType::PUSH) && (mode != SyncModeType::PUSH_AND_PULL) && (mode != SyncModeType::QUERY_PUSH) && + (mode != SyncModeType::QUERY_PUSH_PULL)) { return; } @@ -1862,4 +1445,216 @@ void SingleVerDataSync::SetPacketId(DataRequestPacket *packet, SingleVerSyncTask packet->SetReserved(reserved); } } + +int SingleVerDataSync::GetMessageId(int syncType) const +{ + if (syncType == SyncType::QUERY_SYNC_TYPE) { + return QUERY_SYNC_MESSAGE; + } + return DATA_SYNC_MESSAGE; +} + +void SingleVerDataSync::PushAndPUllKeyRevokHandle(SingleVerSyncTaskContext *context) +{ + // for push_and_pull mode it may be EKEYREVOKED error before receive watermarkexception + // should clear errCode and restart pushpull request. + int mode = SyncOperation::TransferSyncMode(context->GetMode()); + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && mode == SyncModeType::PUSH_AND_PULL && + context->GetTaskErrCode() == -E_EKEYREVOKED) { + context->SetTaskErrCode(E_OK); + } +} + +void SingleVerDataSync::SetAckPacket(DataAckPacket &ackPacket, SingleVerSyncTaskContext *context, + const DataRequestPacket *packet, int32_t recvCode, WaterMark maxSendDataTime) +{ + int curType = SyncOperation::GetSyncType(packet->GetMode()); + WaterMark localMark = 0; + GetLocalWaterMark(curType, packet->GetQueryId(), context, localMark); + ackPacket.SetRecvCode(recvCode); + // send ack packet + if ((recvCode == E_OK) && (maxSendDataTime != 0)) { + ackPacket.SetData(maxSendDataTime + 1); // + 1 to next start + } else if (recvCode != WATER_MARK_INVALID) { + WaterMark mark = 0; + GetPeerWaterMark(curType, packet->GetQueryId(), context->GetDeviceId(), mark); + ackPacket.SetData(mark); + } + std::vector reserved {localMark}; + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + uint64_t packetId = 0; + if (version > SOFTWARE_VERSION_RELEASE_2_0) { + packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value + } + if (version > SOFTWARE_VERSION_RELEASE_2_0 && packetId > 0) { + reserved.push_back(packetId); + } + // while recv is not E_OK, data is peerMark, reserve[2] is deletedPeerMark value + if (curType == SyncType::QUERY_SYNC_TYPE && recvCode != WATER_MARK_INVALID) { + WaterMark deletedPeerMark; + GetPeerDeleteSyncWaterMark(context->GetDeviceId(), deletedPeerMark); + reserved.push_back(deletedPeerMark); // query sync mode, reserve[2] store deletedPeerMark value + } + ackPacket.SetReserved(reserved); + ackPacket.SetVersion(version); +} + +int SingleVerDataSync::GetReSendData(SyncEntry &syncData, SingleVerSyncTaskContext *context, + DataSyncReSendInfo reSendInfo) +{ + int mode = SyncOperation::TransferSyncMode(context->GetMode()); + if (mode == SyncModeType::PULL) { + return E_OK; + } + ContinueToken token = nullptr; + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + size_t packetSize = (version > SOFTWARE_VERSION_RELEASE_2_0) ? + DBConstant::MAX_HPMODE_PACK_ITEM_SIZE : DBConstant::MAX_NORMAL_PACK_ITEM_SIZE; + DataSizeSpecInfo reSendDataSizeInfo = GetDataSizeSpecInfo(packetSize); + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + int errCode; + if (curType != SyncType::QUERY_SYNC_TYPE) { + errCode = storage_->GetSyncData(reSendInfo.start, reSendInfo.end + 1, syncData.entries, token, + reSendDataSizeInfo); + } else { + QueryObject queryObj = context->GetQuery().GetQueryObj(); + errCode = storage_->GetSyncData(queryObj, SyncTimeRange { reSendInfo.start, reSendInfo.deleteDataStart, + reSendInfo.end + 1, reSendInfo.deleteDataEnd + 1 }, reSendDataSizeInfo, token, syncData.entries); + } + if (token != nullptr) { + storage_->ReleaseContinueToken(token); + } + if (errCode == -E_BUSY || errCode == -E_EKEYREVOKED) { + context->SetTaskErrCode(errCode); + return errCode; + } + if (errCode != E_OK && errCode != -E_UNFINISHED) { + return errCode; + } + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + CompressAlgorithm remoteAlgo = context->ChooseCompressAlgo(); + if (needCompressOnSync && remoteAlgo != CompressAlgorithm::NONE) { + int compressCode = GenericSingleVerKvEntry::Compress(syncData.entries, syncData.compressedEntries, + { remoteAlgo, version }); + if (compressCode != E_OK) { + return compressCode; + } + } + return errCode; +} + +int SingleVerDataSync::GetReSendMode(int mode, uint32_t sequenceId, int syncType) +{ + int curMode = SyncOperation::TransferSyncMode(mode); + if (curMode == SyncModeType::PUSH || curMode == SyncModeType::PULL) { + return mode; + } + if (curMode == SyncModeType::RESPONSE_PULL) { + return (syncType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH : SyncModeType::PUSH; + } + // set push_and_pull mode when resend first sequenceId to inform remote to run RESPONSE_PULL task + // for sequenceId which is larger than first, only need to send data, means to set push or query_push mode + if (sequenceId == 1) { + return (syncType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH_PULL : SyncModeType::PUSH_AND_PULL; + } + return (syncType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH : SyncModeType::PUSH; +} + +int SingleVerDataSync::RemoveDeviceDataIfNeed(SingleVerSyncTaskContext *context) +{ + if (context->GetRemoteSoftwareVersion() <= SOFTWARE_VERSION_RELEASE_3_0) { + return E_OK; + } + uint64_t clearRemoteDataMark = 0; + metadata_->GetRemoveDataMark(context->GetDeviceId(), clearRemoteDataMark); + if (clearRemoteDataMark == 0) { + return E_OK; + } + int errCode = E_OK; + if (context->IsNeedClearRemoteStaleData() && clearRemoteDataMark == REMOVE_DEVICE_DATA_MARK) { + errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); + if (errCode != E_OK) { + LOGE("clear remote %s data failed,errCode=%d", STR_MASK(GetDeviceId()), errCode); + return errCode; + } + } + if (clearRemoteDataMark == REMOVE_DEVICE_DATA_MARK) { + errCode = metadata_->ResetMetaDataAfterRemoveData(context->GetDeviceId()); + if (errCode != E_OK) { + LOGE("set %s removeDataWaterMark to false failed,errCode=%d", STR_MASK(GetDeviceId()), errCode); + return errCode; + } + } + return E_OK; +} + +void SingleVerDataSync::UpdateMtuSize() +{ + mtuSize_ = communicateHandle_->GetCommunicatorMtuSize(deviceId_) * 9 / 10; // get the 9/10 of the size +} + +void SingleVerDataSync::FillRequestReSendPacket(const SingleVerSyncTaskContext *context, DataRequestPacket *packet, + DataSyncReSendInfo reSendInfo, SyncEntry &syncData, int sendCode) +{ + int curType = (context->IsQuerySync()) ? QUERY_SYNC_TYPE : MANUAL_FULL_SYNC_TYPE; + WaterMark peerMark = 0; + GetPeerWaterMark(curType, context->GetQuery().GetIdentify(), context->GetDeviceId(), peerMark); + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + // transer reSend mode, RESPONSE_PULL transfer to push or query push + // PUSH_AND_PULL mode which sequenceId lager than first transfer to push or query push + int reSendMode = GetReSendMode(context->GetMode(), reSendInfo.sequenceId, curType); + if (context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) || + SyncOperation::TransferSyncMode(context->GetMode()) == SyncModeType::PULL) { + LOGI("[DataSync][ReSend] set lastid,label=%s,dev=%s", label_.c_str(), STR_MASK(GetDeviceId())); + packet->SetLastSequence(); + } + if (sendCode == E_OK && context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) && + context->GetMode() == SyncModeType::RESPONSE_PULL) { + sendCode = SEND_FINISHED; + } + packet->SetData(syncData.entries); + packet->SetCompressData(syncData.compressedEntries); + packet->SetBasicInfo(sendCode, version, reSendMode); + packet->SetWaterMark(reSendInfo.start, peerMark, reSendInfo.deleteDataStart); + if (SyncOperation::TransferSyncMode(reSendMode) != SyncModeType::PUSH) { + packet->SetEndWaterMark(context->GetEndMark()); + packet->SetQuery(context->GetQuery()); + } + packet->SetQueryId(context->GetQuery().GetIdentify()); + if (version > SOFTWARE_VERSION_RELEASE_2_0) { + std::vector reserved {reSendInfo.packetId}; + packet->SetReserved(reserved); + } + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + CompressAlgorithm curAlgo = context->ChooseCompressAlgo(); + if (needCompressOnSync && curAlgo != CompressAlgorithm::NONE) { + packet->SetCompressDataMark(); + packet->SetCompressAlgo(curAlgo); + } +} + +DataSizeSpecInfo SingleVerDataSync::GetDataSizeSpecInfo(size_t packetSize) +{ + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + uint32_t blockSize = std::min(static_cast(DBConstant::MAX_SYNC_BLOCK_SIZE), + mtuSize_ * 100 / compressionRate); // compressionRate max is 100 + return {blockSize, packetSize}; +} + +int SingleVerDataSync::IsQueryByKeyCheck(int inMode, const QuerySyncObject &query, SingleVerSyncTaskContext *context, + const Message *msg) +{ + if (SyncOperation::GetSyncType(inMode) == SyncType::QUERY_SYNC_TYPE && !query.GetQueryObj().IsQueryOnlyByKey()) { + LOGE("[DataSync][RequestRecvPre] only support prefix key query sync"); + SendAck(context, msg, -E_NOT_SUPPORT, 0); + return -E_NOT_SUPPORT; + } + return E_OK; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index 1aee6bfb3..4d432d356 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -20,9 +20,11 @@ #include "meta_data.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_task_context.h" +#include "single_ver_data_packet.h" #include "sync_types.h" #include "version.h" #include "parcel.h" +#include "query_sync_object.h" namespace DistributedDB { using SendDataItem = SingleVerKvEntry *; @@ -30,133 +32,25 @@ using SendDataItem = SingleVerKvEntry *; struct DataSyncReSendInfo { uint32_t sessionId = 0; uint32_t sequenceId = 0; - TimeStamp start = 0; // means localwatermark + TimeStamp start = 0; // means normal or sync data localwatermark TimeStamp end = 0; + TimeStamp deleteDataStart = 0; // means delete data localwatermark + TimeStamp deleteDataEnd = 0; uint64_t packetId = 0; }; -class DataRequestPacket { -public: - DataRequestPacket() {}; - ~DataRequestPacket(); - - void SetData(std::vector &data); - - const std::vector &GetData() const; - - void SetEndWaterMark(WaterMark waterMark); - - WaterMark GetEndWaterMark() const; - - void SetLocalWaterMark(WaterMark waterMark); - - WaterMark GetLocalWaterMark() const; - - void SetPeerWaterMark(WaterMark waterMark); - - WaterMark GetPeerWaterMark() const; - - void SetSendCode(int32_t errCode); - - int32_t GetSendCode() const; - - void SetMode(int32_t mode); - - int32_t GetMode() const; - - void SetSessionId(uint32_t sessionId); - - uint32_t GetSessionId() const; - - void SetVersion(uint32_t version); - - uint32_t GetVersion() const; - - uint32_t CalculateLen() const; - - void SetReserved(std::vector &reserved); - - std::vector GetReserved() const; - - uint64_t GetPacketId() const; - - void SetFlag(uint32_t flag); - - uint32_t GetFlag() const; - - bool IsLastSequence() const; - - void SetLastSequence(); - - void SetBasicInfo(int sendCode, uint32_t version, WaterMark localMark, WaterMark peerMark, int32_t mode); - -private: - std::vector data_; - WaterMark endWaterMark_ = 0; - WaterMark localWaterMark_ = 0; - WaterMark peerWaterMark_ = 0; - int32_t sendCode_ = 0; - int32_t mode_ = SyncOperation::INVALID; - uint32_t sessionId_ = 0; - uint32_t version_ = SOFTWARE_VERSION_CURRENT; - std::vector reserved_; - uint32_t flag_ = 0; // bit 0 used for isLastSequence - static const uint32_t IS_LAST_SEQUENCE = 0x1; // bit 0 used for isLastSequence, 1: is last, 0: not last -}; - -class DataAckPacket { -public: - DataAckPacket() {}; - ~DataAckPacket() {}; - - void SetData(uint64_t data); - - uint64_t GetData() const; - - void SetRecvCode(int32_t errorCode); - - int32_t GetRecvCode() const; - - void SetVersion(uint32_t version); - - uint32_t GetVersion() const; - - void SetReserved(std::vector &reserved); - - std::vector GetReserved() const; - - uint64_t GetPacketId() const; - - static bool IsPacketIdValid(uint64_t packetId); - - uint32_t CalculateLen() const; - -private: - /* - * data_ is waterMark when revCode_ == LOCAL_WATER_MARK_NOT_INIT || revCode_ == E_OK; - * data_ is timer in milliSeconds when revCode_ == -E_SAVE_DATA_NOTIFY && data_ != 0. - */ - uint64_t data_ = 0; - int32_t recvCode_ = 0; - uint32_t version_ = SOFTWARE_VERSION_CURRENT; - std::vector reserved_; +struct SyncEntry { + std::vector entries; + std::vector compressedEntries; }; class SingleVerDataSync { public: SingleVerDataSync(); - ~SingleVerDataSync(); + virtual ~SingleVerDataSync(); DISABLE_COPY_ASSIGN_MOVE(SingleVerDataSync); - static int Serialization(uint8_t *buffer, uint32_t length, const Message *inMsg); - - static int DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); - - static uint32_t CalculateLen(const Message *inMsg); - - static int RegisterTransformFunc(); - int Initialize(IKvDBSyncInterface *inStorage, ICommunicator *inCommunicateHandle, std::shared_ptr &inMetadata, const std::string &deviceId); @@ -173,47 +67,51 @@ public: int AckRecv(SingleVerSyncTaskContext *context, const Message *message); void SendSaveDataNotifyPacket(SingleVerSyncTaskContext *context, uint32_t pktVersion, uint32_t sessionId, - uint32_t sequenceId); + uint32_t sequenceId, uint32_t inMsgId); - void SendAck(SingleVerSyncTaskContext *context, uint32_t sessionId, uint32_t sequenceId, uint64_t packetId); + int SendAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, WaterMark maxSendDataTime); int32_t ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo); - int CheckPermitSendData(int mode, SingleVerSyncTaskContext *context); + int CheckPermitSendData(int inMode, SingleVerSyncTaskContext *context); std::string GetLabel() const; std::string GetDeviceId() const; -private: + bool WaterMarkErrHandle(int syncType, SingleVerSyncTaskContext *context, const Message *message); + +protected: static const int SEND_FINISHED = 0xff; static const int LOCAL_WATER_MARK_NOT_INIT = 0xaa; static const int PEER_WATER_MARK_NOT_INIT = 0x55; static const int WATER_MARK_INVALID = 0xbb; static const int MTU_SIZE = 28311552; // 27MB - static int AckPacketCalculateLen(const Message *inMsg, uint32_t &len); + TimeStamp GetMaxSendDataTime(const std::vector &inData); - static int DataPacketCalculateLen(const Message *inMsg, uint32_t &len); + TimeStamp GetMinSendDataTime(const std::vector &inData, WaterMark localMark); - static int DataPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + void FillRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, + SyncEntry &syncData, int sendCode, int mode); - static int DataPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + int RequestStart(SingleVerSyncTaskContext *context, int mode); - static int AckPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + SyncTimeRange GetSyncDataTimeRange(int syncType, SingleVerSyncTaskContext *context, + const std::vector &inData, UpdateWaterMark &isUpdate); - static int AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + SyncTimeRange GetRecvDataTimeRange(int syncType, SingleVerSyncTaskContext *context, + const std::vector &data, UpdateWaterMark &isUpdate); - static bool IsPacketValid(const Message *inMsg); + SyncTimeRange GetFullSyncDataTimeRange(const std::vector &inData, WaterMark localMark, + UpdateWaterMark &isUpdate); - static TimeStamp GetMaxSendDataTime(const std::vector &inData, bool isNeedInit, - WaterMark localMark = 0); - - static TimeStamp GetMinSendDataTime(const std::vector &inData, WaterMark localMark); + SyncTimeRange GetQuerySyncDataTimeRange(const std::vector &inData, WaterMark localMark, + WaterMark deletelocalMark, UpdateWaterMark &isUpdate); int GetData(SingleVerSyncTaskContext *context, std::vector &outData, size_t packetSize); - int GetDataWithRerformanceRecord(SingleVerSyncTaskContext *context, std::vector &outData); + int GetDataWithPerformanceRecord(SingleVerSyncTaskContext *context, SyncEntry &syncOutData); int Send(SingleVerSyncTaskContext *context, const Message *message, const CommErrHandler &handler, uint32_t packetLen); @@ -222,13 +120,29 @@ private: int GetNextUnsyncData(SingleVerSyncTaskContext *context, std::vector &outData, size_t packetSize); + int GetMessageId(int syncType) const; + int SaveData(const SingleVerSyncTaskContext *context, const std::vector &inData); - int SaveLocalWaterMark(const DeviceID &deviceId, WaterMark waterMark); + SyncTimeRange ReviseLocalMark(int syncType, SyncTimeRange &dataTimeRange, UpdateWaterMark updateMark); + + int SaveLocalWaterMark(int syncType, const SingleVerSyncTaskContext *context, + SyncTimeRange dataTimeRange, bool isCheckBeforUpdate = false) const; + + void GetLocalWaterMark(int syncType, const std::string &queryIdentify, const SingleVerSyncTaskContext *context, + WaterMark &watermark) const; + + void GetPeerWaterMark(int syncType, const std::string &queryIdentify, const DeviceID &deviceId, + WaterMark &watermark) const; + + void GetPeerDeleteSyncWaterMark(const DeviceID &deviceId, WaterMark &waterMark); - int SavePeerWaterMark(const DeviceID &deviceId, WaterMark waterMark); + void GetLocalDeleteSyncWaterMark(const SingleVerSyncTaskContext *context, WaterMark &waterMark) const; - int RemoveDeviceData(SingleVerSyncTaskContext *context, const Message *message, WaterMark maxSendDataTime); + int RemoveDeviceDataHandle(SingleVerSyncTaskContext *context, const Message *message, WaterMark maxSendDataTime); + + int DealRemoveDeviceDataByAck(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, + const std::vector &reserved); void TransSendDataItemToLocal(const SingleVerSyncTaskContext *context, const std::vector &data); @@ -236,17 +150,13 @@ private: void TransDbDataItemToSendDataItem(const SingleVerSyncTaskContext *context, std::vector &outData); - int SendDataPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context); - - void SetAckData(DataAckPacket &ackPacket, SingleVerSyncTaskContext *context, int32_t recvCode, - WaterMark maxSendDataTime) const; + int SendDataPacket(int syncType, const DataRequestPacket *packet, SingleVerSyncTaskContext *context); - int SendAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, - WaterMark maxSendDataTime); + void UpdateQueryPeerWaterMark(int syncType, const std::string &queryId, SyncTimeRange &dataTime, + const SingleVerSyncTaskContext *context, UpdateWaterMark isUpdateWaterMark); - int SendLocalWaterMarkAck(SingleVerSyncTaskContext *context, const Message *message); - - void UpdatePeerWaterMark(const SingleVerSyncTaskContext *context, WaterMark peerWatermark); + void UpdatePeerWaterMark(int syncType, const std::string &queryId, const SingleVerSyncTaskContext *context, + WaterMark peerWatermark, WaterMark peerDeletedWatermark); std::string GetLocalDeviceName(); @@ -262,15 +172,6 @@ private: int DealWaterMarkException(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, const std::vector &reserved); - static int DataPacketSyncerPartSerialization(Parcel &parcel, const DataRequestPacket *packet); - - static int DataPacketSyncerPartDeSerialization(Parcel &parcel, DataRequestPacket *packet, uint32_t packLen, - uint32_t length, uint32_t version); - - static int AckPacketSyncerPartSerializationV1(Parcel &parcel, const DataAckPacket *packet); - - static int AckPacketSyncerPartDeSerializationV1(Parcel &parcel, DataAckPacket &packet); - int RunPermissionCheck(SingleVerSyncTaskContext *context, const Message *message, const DataRequestPacket *packet); @@ -279,8 +180,7 @@ private: int SendReSendPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context, uint32_t sessionId, uint32_t sequenceId); - int SendPullResponseDataPkt(int ackCode, std::vector &inData, - SingleVerSyncTaskContext *context); + int SendPullResponseDataPkt(int ackCode, SyncEntry &syncOutData, SingleVerSyncTaskContext *context); void SetPacketId(DataRequestPacket *packet, SingleVerSyncTaskContext *context, uint32_t version); @@ -292,7 +192,29 @@ private: int CheckSchemaStrategy(SingleVerSyncTaskContext *context, const Message *message); - void RemotePushFinished(int sendCode, int mode, uint32_t msgSessionId, uint32_t contetSessionId); + void RemotePushFinished(int sendCode, int inMode, uint32_t msgSessionId, uint32_t contextSessionId); + + void PushAndPUllKeyRevokHandle(SingleVerSyncTaskContext *context); + + void SetAckPacket(DataAckPacket &ackPacket, SingleVerSyncTaskContext *context, const DataRequestPacket *packet, + int32_t recvCode, WaterMark maxSendDataTime); + + int GetReSendData(SyncEntry &syncData, SingleVerSyncTaskContext *context, + DataSyncReSendInfo reSendInfo); + + int RemoveDeviceDataIfNeed(SingleVerSyncTaskContext *context); + + int GetReSendMode(int mode, uint32_t sequenceId, int syncType); + + void FillRequestReSendPacket(const SingleVerSyncTaskContext *context, DataRequestPacket *packet, + DataSyncReSendInfo reSendInfo, SyncEntry &syncData, int sendCode); + + int IsQueryByKeyCheck(int inMode, const QuerySyncObject &query, SingleVerSyncTaskContext *context, + const Message *msg); + + void UpdateMtuSize(); + + DataSizeSpecInfo GetDataSizeSpecInfo(size_t packetSize); uint32_t mtuSize_; SingleVerKvDBSyncInterface* storage_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.cpp new file mode 100644 index 000000000..289eeda85 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.cpp @@ -0,0 +1,465 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "single_ver_serialize_manager.h" +#include "icommunicator.h" +#include "log_print.h" +#include "message_transform.h" +#include "parcel.h" +#include "generic_single_ver_kv_entry.h" +#include "sync_types.h" +#include "version.h" +#include "db_common.h" + +namespace DistributedDB { +int SingleVerSerializeManager::Serialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + if ((buffer == nullptr) || !(IsPacketValid(inMsg))) { + return -E_MESSAGE_ID_ERROR; + } + + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + return DataPacketSerialization(buffer, length, inMsg); + case TYPE_RESPONSE: + case TYPE_NOTIFY: + return AckPacketSerialization(buffer, length, inMsg); + default: + return -E_MESSAGE_TYPE_ERROR; + } +} + +int SingleVerSerializeManager::DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + if ((buffer == nullptr) || !(IsPacketValid(inMsg))) { + return -E_MESSAGE_ID_ERROR; + } + + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + return DataPacketDeSerialization(buffer, length, inMsg); + case TYPE_RESPONSE: + case TYPE_NOTIFY: + return AckPacketDeSerialization(buffer, length, inMsg); + default: + return -E_MESSAGE_TYPE_ERROR; + } +} + +uint32_t SingleVerSerializeManager::CalculateLen(const Message *inMsg) +{ + if (!(IsPacketValid(inMsg))) { + return 0; + } + uint32_t len = 0; + int errCode; + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + errCode = DataPacketCalculateLen(inMsg, len); + if (errCode != E_OK) { + LOGE("[DataSync][CalculateLen] calculate data request packet len failed, errCode=%d", errCode); + return 0; + } + return len; + case TYPE_RESPONSE: + case TYPE_NOTIFY: + errCode = AckPacketCalculateLen(inMsg, len); + if (errCode != E_OK) { + LOGE("[DataSync][CalculateLen] calculate data notify packet len failed errCode=%d", errCode); + return 0; + } + return len; + default: + return 0; + } +} +int SingleVerSerializeManager::RegisterTransformFunc() +{ + TransformFunc func; + func.computeFunc = std::bind(&SingleVerSerializeManager::CalculateLen, std::placeholders::_1); + func.serializeFunc = std::bind(&SingleVerSerializeManager::Serialization, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3); + func.deserializeFunc = std::bind(&SingleVerSerializeManager::DeSerialization, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3); + + int errCode = MessageTransform::RegTransformFunction(QUERY_SYNC_MESSAGE, func); + if (errCode != E_OK) { + return errCode; + } + return MessageTransform::RegTransformFunction(DATA_SYNC_MESSAGE, func); +} + +int SingleVerSerializeManager::DataPacketSyncerPartSerialization(Parcel &parcel, const DataRequestPacket *packet) +{ + // endWaterMark + int errCode = parcel.WriteUInt64(packet->GetEndWaterMark()); + if (errCode != E_OK) { + return errCode; + } + // localWaterMark + errCode = parcel.WriteUInt64(packet->GetLocalWaterMark()); + if (errCode != E_OK) { + return errCode; + } + // peerWaterMark + errCode = parcel.WriteUInt64(packet->GetPeerWaterMark()); + if (errCode != E_OK) { + return errCode; + } + // sendCode + errCode = parcel.WriteInt(packet->GetSendCode()); + if (errCode != E_OK) { + return errCode; + } + // mode + errCode = parcel.WriteInt(packet->GetMode()); + if (errCode != E_OK) { + return errCode; + } + // sessionId + errCode = parcel.WriteUInt32(packet->GetSessionId()); + if (errCode != E_OK) { + return errCode; + } + // reserved + errCode = parcel.WriteVector(packet->GetReserved()); + if (errCode != E_OK) { + return errCode; + } + if (packet->GetVersion() > SOFTWARE_VERSION_RELEASE_2_0) { + errCode = parcel.WriteUInt32(packet->GetFlag()); + if (errCode != E_OK) { + return errCode; + } + } + parcel.EightByteAlign(); + return errCode; +} + +int SingleVerSerializeManager::DataPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + Parcel parcel(buffer, length); + + // version + int errCode = parcel.WriteUInt32(packet->GetVersion()); + if (errCode != E_OK) { + LOGE("[DataPacketSerialization] Serialize version failed"); + return errCode; + } + // sendDataItems + errCode = GenericSingleVerKvEntry::SerializeDatas( + (packet->IsCompressData() ? std::vector {} : packet->GetData()), parcel, packet->GetVersion()); + if (errCode != E_OK) { + LOGE("[DataPacketSerialization] Serialize Data failed"); + return errCode; + } + + // data sync + errCode = DataPacketSyncerPartSerialization(parcel, packet); + if (errCode != E_OK) { + LOGE("[DataPacketSerialization] Serialize Data failed"); + return errCode; + } + if (inMsg->GetMessageId() == QUERY_SYNC_MESSAGE) { + errCode = DataPacketQuerySyncSerialization(parcel, packet); // for query sync + if (errCode != E_OK) { + return errCode; + } + } + if (packet->IsCompressData()) { + // serialize compress data + errCode = GenericSingleVerKvEntry::SerializeCompressedDatas(packet->GetData(), packet->GetCompressData(), + parcel, packet->GetVersion(), packet->GetCompressAlgo()); + if (errCode != E_OK) { + LOGE("[DataPacketSerialization] Serialize compress Data failed"); + return errCode; + } + } + return E_OK; +} + +int SingleVerSerializeManager::DataPacketQuerySyncSerialization(Parcel &parcel, const DataRequestPacket *packet) +{ + // deleted record send watermark + int errCode = parcel.WriteUInt64(packet->GetDeletedWaterMark()); + if (errCode != E_OK) { + LOGE("[QuerySerialization] Serialize deleted record send watermark failed!"); + return errCode; + } + + // query identify + QuerySyncObject queryObj = packet->GetQuery(); + errCode = parcel.WriteString(packet->GetQueryId()); + if (errCode != E_OK) { + LOGE("[QuerySerialization] Serialize query id failed!"); + return errCode; + } + if (packet->GetMode() != QUERY_PUSH) { + // need to check. + errCode = queryObj.SerializeData(parcel, SOFTWARE_VERSION_CURRENT); + } + return errCode; +} + +int SingleVerSerializeManager::DataPacketCalculateLen(const Message *inMsg, uint32_t &len) +{ + const DataRequestPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + + len = packet->CalculateLen(inMsg->GetMessageId()); + return E_OK; +} + +int SingleVerSerializeManager::AckPacketCalculateLen(const Message *inMsg, uint32_t &len) +{ + const DataAckPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + + len = packet->CalculateLen(); + return E_OK; +} + +bool SingleVerSerializeManager::IsPacketValid(const Message *inMsg) +{ + if (inMsg == nullptr) { + return false; + } + + int msgType = inMsg->GetMessageType(); + if (msgType != TYPE_REQUEST && msgType != TYPE_RESPONSE && msgType != TYPE_NOTIFY) { + LOGE("[DataSync][IsPacketValid] Message type ERROR! message type=%d", msgType); + return false; + } + return true; +} + +int SingleVerSerializeManager::AckPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + const DataAckPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + + Parcel parcel(buffer, length); + int errCode = parcel.WriteUInt32(packet->GetVersion()); + if (errCode != E_OK) { + return errCode; + } + // now V1 compatible for softWareVersion :{101, 102} + return AckPacketSyncerPartSerializationV1(parcel, packet); +} + +int SingleVerSerializeManager::AckPacketSyncerPartSerializationV1(Parcel &parcel, const DataAckPacket *packet) +{ + int errCode = parcel.WriteUInt64(packet->GetData()); + if (errCode != E_OK) { + return errCode; + } + errCode = parcel.WriteInt(packet->GetRecvCode()); + if (errCode != E_OK) { + return errCode; + } + errCode = parcel.WriteVector(packet->GetReserved()); + if (errCode != E_OK) { + return errCode; + } + parcel.EightByteAlign(); + return errCode; +} + +int SingleVerSerializeManager::DataPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + std::vector dataItems; + uint32_t version; + Parcel parcel(const_cast(buffer), length); + uint32_t packLen = parcel.ReadUInt32(version); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + + if (version > SOFTWARE_VERSION_CURRENT) { + return -E_VERSION_NOT_SUPPORT; + } + + auto packet = new (std::nothrow) DataRequestPacket(); + if (packet == nullptr) { + return -E_OUT_OF_MEMORY; + } + + packet->SetVersion(version); + packLen += GenericSingleVerKvEntry::DeSerializeDatas(dataItems, parcel); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + + packet->SetData(dataItems); + int errCode = DataPacketSyncerPartDeSerialization(parcel, packet, packLen, length, version); + if (errCode != E_OK) { + goto ERROR; + } + if (inMsg->GetMessageId() == QUERY_SYNC_MESSAGE) { + errCode = DataPacketQuerySyncDeSerialization(parcel, packet); + if (errCode != E_OK) { + goto ERROR; + } + } + if (packet->IsCompressData()) { + errCode = DataPacketCompressDataDeSerialization(parcel, packet); + if (errCode != E_OK) { + goto ERROR; + } + } + + errCode = inMsg->SetExternalObject<>(packet); + if (errCode != E_OK) { + goto ERROR; + } + return errCode; + +ERROR: + delete packet; + packet = nullptr; + return errCode; +} + +int SingleVerSerializeManager::DataPacketQuerySyncDeSerialization(Parcel &parcel, DataRequestPacket *packet) +{ + WaterMark deletedWatermark = 0; + parcel.ReadUInt64(deletedWatermark); + std::string queryId; + parcel.ReadString(queryId); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + // query identify + QueryObject queryObj; + int errCode = E_OK; + if (packet->GetMode() != QUERY_PUSH) { + // need to check. + errCode = QuerySyncObject::DeSerializeData(parcel, queryObj); + } + if (errCode != E_OK) { + LOGI("[SingleVerSerializeManager] DeSerializeData object failed."); + } + packet->SetDeletedWaterMark(deletedWatermark); + packet->SetQueryId(queryId); + if (packet->GetMode() != QUERY_PUSH) { + packet->SetQuery(QuerySyncObject(queryObj)); + } + return E_OK; +} + +int SingleVerSerializeManager::DataPacketCompressDataDeSerialization(Parcel &parcel, DataRequestPacket *packet) +{ + std::vector originalData; + int errCode = GenericSingleVerKvEntry::DeSerializeCompressedDatas(originalData, parcel); + if (errCode != E_OK) { + LOGE("[SingleVerSerializeManager] DeSerializeComptressData failed, errCode=%d", errCode); + return errCode; + } + packet->SetData(originalData); + return E_OK; +} + +int SingleVerSerializeManager::DataPacketSyncerPartDeSerialization(Parcel &parcel, DataRequestPacket *packet, + uint32_t packLen, uint32_t length, uint32_t version) +{ + WaterMark waterMark; + WaterMark localWaterMark; + WaterMark peerWaterMark; + int32_t sendCode; + int32_t mode; + uint32_t sessionId; + uint32_t flag = 0; + std::vector reserved; + + packLen += parcel.ReadUInt64(waterMark); + packLen += parcel.ReadUInt64(localWaterMark); + packLen += parcel.ReadUInt64(peerWaterMark); + packLen += parcel.ReadInt(sendCode); + packLen += parcel.ReadInt(mode); + packLen += parcel.ReadUInt32(sessionId); + packLen += parcel.ReadVector(reserved); + if (version > SOFTWARE_VERSION_RELEASE_2_0) { + packLen += parcel.ReadUInt32(flag); + packet->SetFlag(flag); + } + packLen = Parcel::GetEightByteAlign(packLen); + if (parcel.IsError()) { + LOGE("[DataSync][DataPacketDeSerialization] deserialize failed! input len=%lu,packLen=%lu", length, packLen); + return -E_LENGTH_ERROR; + } + parcel.EightByteAlign(); + packet->SetEndWaterMark(waterMark); + packet->SetLocalWaterMark(localWaterMark); + packet->SetPeerWaterMark(peerWaterMark); + packet->SetSendCode(sendCode); + packet->SetMode(mode); + packet->SetSessionId(sessionId); + packet->SetReserved(reserved); + return E_OK; +} + +int SingleVerSerializeManager::AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + DataAckPacket packet; + Parcel parcel(const_cast(buffer), length); + uint32_t version; + + parcel.ReadUInt32(version); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + if (version > SOFTWARE_VERSION_CURRENT) { + packet.SetVersion(version); + packet.SetRecvCode(-E_VERSION_NOT_SUPPORT); + return inMsg->SetCopiedObject<>(packet); + } + packet.SetVersion(version); + // now V1 compatible for softWareVersion :{101, 102} + int errCode = AckPacketSyncerPartDeSerializationV1(parcel, packet); + if (errCode != E_OK) { + return errCode; + } + + return inMsg->SetCopiedObject<>(packet); +} + +int SingleVerSerializeManager::AckPacketSyncerPartDeSerializationV1(Parcel &parcel, DataAckPacket &packet) +{ + WaterMark mark; + int32_t errCode; + std::vector reserved; + + parcel.ReadUInt64(mark); + parcel.ReadInt(errCode); + parcel.ReadVector(reserved); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + packet.SetData(mark); + packet.SetRecvCode(errCode); + packet.SetReserved(reserved); + return E_OK; +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.h new file mode 100644 index 000000000..b0e681dd8 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SINGLE_VER_SERIALIZE_MANAGER_NEW_H +#define SINGLE_VER_SERIALIZE_MANAGER_NEW_H + +#include "icommunicator.h" +#include "parcel.h" +#include "single_ver_data_packet.h" + +namespace DistributedDB { +class SingleVerSerializeManager { +public: + SingleVerSerializeManager(); + virtual ~SingleVerSerializeManager(); + + DISABLE_COPY_ASSIGN_MOVE(SingleVerSerializeManager); + + static int Serialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + + static int DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + + static uint32_t CalculateLen(const Message *inMsg); + + static int RegisterTransformFunc(); +private: + static bool IsPacketValid(const Message *inMsg); + static int DataPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + static int DataPacketSyncerPartSerialization(Parcel &parcel, const DataRequestPacket *packet); + static int DataPacketQuerySyncSerialization(Parcel &parcel, const DataRequestPacket *packet); + static int DataPacketCalculateLen(const Message *inMsg, uint32_t &len); + + static int DataPacketQuerySyncDeSerialization(Parcel &parcel, DataRequestPacket *packet); + static int DataPacketCompressDataDeSerialization(Parcel &parcel, DataRequestPacket *packet); + static int DataPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + static int DataPacketSyncerPartDeSerialization(Parcel &parcel, DataRequestPacket *packet, uint32_t packLen, + uint32_t length, uint32_t version); + + static int AckPacketCalculateLen(const Message *inMsg, uint32_t &len); + static int AckPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + static int AckPacketSyncerPartSerializationV1(Parcel &parcel, const DataAckPacket *packet); + + static int AckPacketSyncerPartDeSerializationV1(Parcel &parcel, DataAckPacket &packet); + static int AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); +}; +} // namespace DistributedDB + +#endif // SINGLE_VER_SERIALIZE_MANAGER_NEW_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp index 1c332d388..c298af391 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp @@ -20,11 +20,12 @@ namespace DistributedDB { ISyncTaskContext *SingleVerSyncEngine::CreateSyncTaskContext() { - auto context = new (std::nothrow) SingleVerSyncTaskContext; + auto context = new (std::nothrow) SingleVerSyncTaskContext(); if (context == nullptr) { LOGE("[SingleVerSyncEngine][CreateSyncTaskContext] create failed, may be out of memory"); return nullptr; } + context->SetSyncRetry(GetSyncRetry()); context->EnableClearRemoteStaleData(needClearRemoteStaleData_); return context; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp index adc30e2d2..ff402f797 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp @@ -24,9 +24,11 @@ #include "sync_operation.h" #include "message_transform.h" #include "sync_types.h" +#include "db_common.h" #include "runtime_context.h" #include "performance_analysis.h" #include "single_ver_sync_target.h" +#include "single_ver_data_sync.h" namespace DistributedDB { namespace { @@ -45,13 +47,13 @@ namespace { {TIME_SYNC, INNER_ERR_EVENT, INNER_ERR}, // In ABILITY_SYNC state, compare version num and schema - {ABILITY_SYNC, VERSION_NOT_SUPPORT_EVENT, INNER_ERR}, + {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, {ABILITY_SYNC, SWITCH_TO_PROCTOL_V1_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, - // In START_INITIACTIVE_DATA_SYNC state, send a sync request, and send first packet of data sync + // In START_INITIACTIVE_DATA_SYNC state, send a sync request, and send first packt of data sync {START_INITIACTIVE_DATA_SYNC, SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, {START_INITIACTIVE_DATA_SYNC, RESPONSE_PUSH_REMAINDER_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_INITIACTIVE_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, @@ -68,7 +70,7 @@ namespace { {INACTIVE_PUSH_REMAINDER_DATA, RE_SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, {INACTIVE_PUSH_REMAINDER_DATA, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - // In START_PASSIVE_DATA_SYNC state, do response pull request, and send first packet of data sync + // In START_PASSIVE_DATA_SYNC state, do response pull request, and send first packt of data sync {START_PASSIVE_DATA_SYNC, SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_PASSIVE_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, {START_PASSIVE_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, @@ -111,19 +113,19 @@ namespace { {TIME_SYNC, INNER_ERR_EVENT, INNER_ERR}, // In ABILITY_SYNC state, compare version num and schema - {ABILITY_SYNC, VERSION_NOT_SUPPORT_EVENT, INNER_ERR}, + {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, - // In START_INITIACTIVE_SLIDING_DATA_SYNC state, send a sync request, and send first packet of data sync + // In START_INITIACTIVE_SLIDING_DATA_SYNC state, send a sync request, and send first packt of data sync {START_INITIACTIVE_SLIDING_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, {START_INITIACTIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {START_INITIACTIVE_SLIDING_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, {START_INITIACTIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, {START_INITIACTIVE_SLIDING_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, - // In START_PASSIVE_SLIDING_DATA_SYNC state, do response pull request, and send first packet of data sync + // In START_PASSIVE_SLIDING_DATA_SYNC state, do response pull request, and send first packt of data sync {START_PASSIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, {START_PASSIVE_SLIDING_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, {START_PASSIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, @@ -193,7 +195,7 @@ int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, IKvDBSyncIn if (errCode != E_OK) { goto ERROR_OUT; } - errCode = abilitySync_->Initialize(communicator, syncInterface, context->GetDeviceId()); + errCode = abilitySync_->Initialize(communicator, syncInterface, metaData, context->GetDeviceId()); if (errCode != E_OK) { goto ERROR_OUT; } @@ -246,6 +248,7 @@ int SingleVerSyncStateMachine::ReceiveMessageCallback(Message *inMsg) errCode = AbilitySyncRecv(inMsg); break; case DATA_SYNC_MESSAGE: + case QUERY_SYNC_MESSAGE: errCode = DataPktRecv(inMsg); break; default: @@ -278,8 +281,8 @@ void SingleVerSyncStateMachine::SyncStepInner() if (iter != stateMapping_.end()) { event = static_cast(iter->second()); } else { - LOGE("[StateMachine][SyncStepInner] can not find state=%d,label=%s,dev=%s{private}", currentState_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGE("[StateMachine][SyncStepInner] can not find state=%d,label=%s,dev=%s", currentState_, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); break; } } while (event != Event::WAIT_ACK_EVENT && SwitchMachineState(event) == E_OK && currentState_ != IDLE); @@ -305,10 +308,10 @@ int SingleVerSyncStateMachine::StartSyncInner() void SingleVerSyncStateMachine::AbortInner() { - LOGE("[StateMachine][AbortInner] error occurred,abort,label=%s,dev=%s{private}", dataSync_->GetLabel().c_str(), - context_->GetDeviceId().c_str()); - if (context_->GetMode() == SyncOperation::PUSH_AND_PULL || context_->GetMode() == SyncOperation::PULL || - context_->IsKilled()) { + LOGE("[StateMachine][AbortInner] error occurred,abort,label=%s,dev=%s", dataSync_->GetLabel().c_str(), + STR_MASK(context_->GetDeviceId())); + int mode = SyncOperation::TransferSyncMode(context_->GetMode()); + if (mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL || context_->IsKilled()) { dataSyncWithSlidingWindow_->ReceiverClear(); } dataSyncWithSlidingWindow_->SenderClear(); @@ -335,17 +338,17 @@ int SingleVerSyncStateMachine::PrepareNextSyncTask() } if (currentState_ != State::IDLE && currentState_ != State::SYNC_TASK_FINISHED) { - LOGW("[StateMachine][PrepareNextSyncTask] PreSync may get an err, state=%d,dev=%s{private}", - currentState_, context_->GetDeviceId().c_str()); + LOGW("[StateMachine][PrepareNextSyncTask] PreSync may get an err, state=%d,dev=%s", + currentState_, STR_MASK(context_->GetDeviceId())); currentState_ = State::IDLE; } return E_OK; } -void SingleVerSyncStateMachine::SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) +void SingleVerSyncStateMachine::SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) { dataSync_->SendSaveDataNotifyPacket(context_, - std::min(context_->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT), sessionId, sequenceId); + std::min(context_->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT), sessionId, sequenceId, inMsgId); } void SingleVerSyncStateMachine::CommErrAbort() @@ -422,19 +425,20 @@ void SingleVerSyncStateMachine::InitStateMapping() Event SingleVerSyncStateMachine::DoInitiactiveDataSync() { int errCode = E_OK; - switch (context_->GetMode()) { - case SyncOperation::PUSH: + int mode = SyncOperation::TransferSyncMode(context_->GetMode()); + switch (mode) { + case SyncModeType::PUSH: context_->SetOperationStatus(SyncOperation::RECV_FINISHED); errCode = dataSync_->PushStart(context_); break; - case SyncOperation::PULL: + case SyncModeType::PULL: context_->SetOperationStatus(SyncOperation::SEND_FINISHED); errCode = dataSync_->PullRequestStart(context_); break; - case SyncOperation::PUSH_AND_PULL: + case SyncModeType::PUSH_AND_PULL: errCode = dataSync_->PushPullStart(context_); break; - case SyncOperation::RESPONSE_PULL: + case SyncModeType::RESPONSE_PULL: // In response pull mode, reminader data should send in // PASSIVE_PUSH_REMAINDER_DATA return Event::RESPONSE_PUSH_REMAINDER_EVENT; @@ -450,23 +454,26 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSync() Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() { - LOGD("[StateMachine][activeDataSync] mode=%d,label=%s,dev=%s{private}", context_->GetMode(), - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGD("[StateMachine][activeDataSync] mode=%d,label=%s,dev=%s", context_->GetMode(), + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); int errCode = E_OK; switch (context_->GetMode()) { - case SyncOperation::PUSH: + case SyncModeType::PUSH: + case SyncModeType::QUERY_PUSH: context_->SetOperationStatus(SyncOperation::RECV_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::PUSH, context_, dataSync_); + errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); break; - case SyncOperation::PULL: + case SyncModeType::PULL: + case SyncModeType::QUERY_PULL: context_->SetOperationStatus(SyncOperation::SEND_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::PULL, context_, dataSync_); + errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); break; - case SyncOperation::PUSH_AND_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::PUSH_AND_PULL, context_, dataSync_); + case SyncModeType::PUSH_AND_PULL: + case SyncModeType::QUERY_PUSH_PULL: + errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); break; - case SyncOperation::RESPONSE_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::RESPONSE_PULL, context_, dataSync_); + case SyncModeType::RESPONSE_PULL: + errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); break; default: errCode = -E_NOT_SUPPORT; @@ -476,7 +483,8 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() return Event::WAIT_ACK_EVENT; } // once E_EKEYREVOKED error occurred, PUSH_AND_PULL mode should wait for ack to pull remote data. - if (context_->GetMode() == SyncOperation::PUSH_AND_PULL && errCode == -E_EKEYREVOKED) { + if (SyncOperation::TransferSyncMode(context_->GetMode()) == SyncModeType::PUSH_AND_PULL && + errCode == -E_EKEYREVOKED) { return Event::WAIT_ACK_EVENT; } return TransformErrCodeToEvent(errCode); @@ -509,7 +517,7 @@ Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() return RESPONSE_TASK_FINISHED_EVENT; } } - int errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::RESPONSE_PULL, context_, dataSync_); + int errCode = dataSyncWithSlidingWindow_->SenderStart(SyncModeType::RESPONSE_PULL, context_, dataSync_); if (errCode == E_OK) { return Event::WAIT_ACK_EVENT; } @@ -519,13 +527,14 @@ Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() Event SingleVerSyncStateMachine::DoInitiactivePushRemainderData() { int errCode; - switch (context_->GetMode()) { - case SyncOperation::PULL: - case SyncOperation::RESPONSE_PULL: + int mode = SyncOperation::TransferSyncMode(context_->GetMode()); + switch (mode) { + case SyncModeType::PULL: + case SyncModeType::RESPONSE_PULL: // In pull or response pul mode, don't need to do INACTIVE_PUSH return Event::SEND_FINISHED_EVENT; - case SyncOperation::PUSH: - case SyncOperation::PUSH_AND_PULL: + case SyncModeType::PUSH: + case SyncModeType::PUSH_AND_PULL: errCode = dataSync_->PushStart(context_); break; default: @@ -556,7 +565,7 @@ Event SingleVerSyncStateMachine::DoWaitForDataRecv() const if (context_->GetOperationStatus() == SyncOperation::FINISHED_ALL) { return RECV_FINISHED_EVENT; } - if (context_->GetMode() == SyncOperation::PUSH_AND_PULL && + if (SyncOperation::TransferSyncMode(context_->GetMode()) == SyncModeType::PUSH_AND_PULL && context_->GetOperationStatus() == SyncOperation::EKEYREVOKED_FAILURE && context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { return RECV_FINISHED_EVENT; @@ -604,8 +613,8 @@ Event SingleVerSyncStateMachine::DoAbilitySync() CommErrHandler handler = std::bind(&SyncTaskContext::CommErrHandlerFunc, std::placeholders::_1, context_, context_->GetRequestSessionId()); - LOGI("[StateMachine][AbilitySync] start abilitySync,label=%s,dev=%s{private}", dataSync_->GetLabel().c_str(), - context_->GetDeviceId().c_str()); + LOGI("[StateMachine][AbilitySync] start abilitySync,label=%s,dev=%s", dataSync_->GetLabel().c_str(), + STR_MASK(context_->GetDeviceId())); errCode = abilitySync_->SyncStart(context_->GetRequestSessionId(), context_->GetSequenceId(), remoteCommunicatorVersion, handler); if (errCode != E_OK) { @@ -641,16 +650,9 @@ Event SingleVerSyncStateMachine::DoInnerErr() RefObject::AutoLock lock(context_); if (!context_->IsCommNormal()) { context_->Abort(SyncOperation::COMM_ABNORMAL); - } else if (context_->GetTaskErrCode() == -E_SCHEMA_MISMATCH) { - context_->Abort(SyncOperation::SCHEMA_INCOMPATIBLE); - } else if (context_->GetTaskErrCode() == -E_EKEYREVOKED) { - context_->Abort(SyncOperation::EKEYREVOKED_FAILURE); - } else if (context_->GetTaskErrCode() == -E_SECURITY_OPTION_CHECK_ERROR) { - context_->Abort(SyncOperation::SECURITY_OPTION_CHECK_FAILURE); - } else if (context_->GetTaskErrCode() == -E_BUSY) { - context_->Abort(SyncOperation::BUSY_FAILURE); } else { - context_->Abort(SyncOperation::FAILED); + int status = GetSyncOperationStatus(context_->GetTaskErrCode()); + context_->Abort(status); } context_->Clear(); AbortInner(); @@ -663,7 +665,7 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) return abilitySync_->RequestRecv(inMsg, context_); } - if (inMsg->GetMessageType() == TYPE_RESPONSE) { + if (inMsg->GetMessageType() == TYPE_RESPONSE && AbilityMsgSessionIdCheck(inMsg)) { int errCode = abilitySync_->AckRecv(inMsg, context_); std::lock_guard lock(stateMachineLock_); (void)ResetWatchDog(); @@ -672,8 +674,8 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) SwitchStateAndStep(TransformErrCodeToEvent(errCode)); } else if (context_->GetRemoteSoftwareVersion() <= SOFTWARE_VERSION_RELEASE_2_0) { abilitySync_->SetAbilitySyncFinishedStatus(true); - LOGI("[StateMachine][AbilitySyncRecv] ability Sync Finished,label=%s,dev=%s{private}", - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[StateMachine][AbilitySyncRecv] ability Sync Finished,label=%s,dev=%s", + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); currentRemoteVersionId_ = context_->GetRemoteSoftwareVersionId(); SwitchStateAndStep(ABILITY_SYNC_FINISHED_EVENT); } @@ -691,15 +693,17 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) SwitchStateAndStep(Event::INNER_ERR_EVENT); return E_OK; } - if (ackCode == AbilitySync::LAST_NOTIFY) { + if (ackCode == AbilitySync::LAST_NOTIFY && AbilityMsgSessionIdCheck(inMsg)) { abilitySync_->SetAbilitySyncFinishedStatus(true); - LOGI("[StateMachine][AbilitySyncRecv] ability sync finished,label=%s,dev=%s{private}", - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + // while recv last notify means ability sync finished,it is better to reset watchDog to avoid timeout. + (void)ResetWatchDog(); + LOGI("[StateMachine][AbilitySyncRecv] ability sync finished,label=%s,dev=%s", + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); currentRemoteVersionId_ = context_->GetRemoteSoftwareVersionId(); (static_cast(context_))->SetIsSchemaSync(true); std::lock_guard lock(stateMachineLock_); SwitchStateAndStep(ABILITY_SYNC_FINISHED_EVENT); - } else { + } else if (ackCode != AbilitySync::LAST_NOTIFY) { abilitySync_->AckNotifyRecv(inMsg, context_); } return E_OK; @@ -711,6 +715,10 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) { + PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); + if (performance != nullptr) { + performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); + } StopFeedDogForSync(SyncDirectionFlag::RECEIVE); { std::lock_guard lockWatchDog(stateMachineLock_); @@ -721,9 +729,12 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) // RequestRecv will save data, it may cost a long time. // So we need to send save data notify to keep remote alive. - bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId()); + bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); WaterMark pullEndWaterkark = 0; int errCode = dataSync_->RequestRecv(context_, inMsg, pullEndWaterkark); + if (performance != nullptr) { + performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); + } if (isNeedStop) { StopSaveDataNotify(); } @@ -734,34 +745,9 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) return errCode; } std::lock_guard lock(stateMachineLock_); - if (IsNeedErrCodeHandle(inMsg->GetSessionId())) { - switch (errCode) { - case E_OK: - break; - case -E_RECV_FINISHED: - context_->SetOperationStatus(SyncOperation::RECV_FINISHED); - SwitchStateAndStep(Event::RECV_FINISHED_EVENT); - break; - case -E_EKEYREVOKED: - PushPullDataRequestEvokeErrHandle(context_); - break; - case -E_BUSY: - context_->SetTaskErrCode(-E_BUSY); - SwitchStateAndStep(Event::INNER_ERR_EVENT); - break; - case -E_SECURITY_OPTION_CHECK_ERROR: - context_->SetTaskErrCode(-E_SECURITY_OPTION_CHECK_ERROR); - SwitchStateAndStep(Event::INNER_ERR_EVENT); - break; - case -E_NEED_ABILITY_SYNC: - return errCode; - default: - SwitchStateAndStep(Event::INNER_ERR_EVENT); - break; - } - } + DataRecvErrCodeHandle(inMsg->GetSessionId(), errCode); if (pullEndWaterkark > 0) { - AddPullResponseTarget(pullEndWaterkark, inMsg->GetSessionId()); + AddPullResponseTarget(inMsg, pullEndWaterkark); } return E_OK; } @@ -810,6 +796,13 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) { StopFeedDogForSync(SyncDirectionFlag::SEND); std::lock_guard lock(stateMachineLock_); + // Unfortunately we use stateMachineLock_ in many sync process + // So a bad ack will check before the lock and wait + // And then another process is running, it will get the lock.After this process, the ack became invalid. + // If we don't check ack again, it will be delivered to dataSyncer. + if (!IsPacketValid(inMsg)) { + return -E_INVALID_ARGS; + } if (IsNeedResetWatchdog(inMsg)) { (void)ResetWatchDog(); } @@ -827,35 +820,15 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) // AckRecv will save meta data, it may cost a long time. if another thread is saving data // So we need to send save data notify to keep remote alive. // eg. remote do pull sync - bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId()); + bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); errCode = dataSync_->AckRecv(context_, inMsg); if (isNeedStop) { StopSaveDataNotify(); } - - switch (errCode) { - case -E_NEED_ABILITY_SYNC: - NeedAbilitySyncHandle(); - break; - case -E_NO_DATA_SEND: - // when version is higher than 102, this step will be handle in slide windows function. - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - context_->SetOperationStatus(SyncOperation::SEND_FINISHED); - } - break; - case -E_NOT_PERMIT: - context_->SetOperationStatus(SyncOperation::PERMISSION_CHECK_FAILED); - break; - case -E_EKEYREVOKED: - case -E_SECURITY_OPTION_CHECK_ERROR: - case -E_BUSY: - context_->SetTaskErrCode(errCode); - break; - case -E_SAVE_DATA_NOTIFY: - return errCode; - default: - break; + if (errCode == -E_SAVE_DATA_NOTIFY) { + return errCode; } + AckRecvErrCodeHandle(errCode); if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { SwitchStateAndStep(TransformErrCodeToEvent(errCode)); return errCode; @@ -869,19 +842,18 @@ int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); int errCode; TimeOffset offset = 0; + uint32_t timeout = TIME_SYNC_WAIT_TIME; switch (inMsg->GetMessageType()) { case TYPE_REQUEST: + timeout = communicator_->GetTimeout(context_->GetDeviceId()); // If message is data sync request, we should check timeoffset. - errCode = timeSync_->GetTimeOffset(offset); + errCode = timeSync_->GetTimeOffset(offset, timeout); if (errCode != E_OK) { LOGE("[StateMachine][DataPktRecv] GetTimeOffset err! errCode=%d", errCode); return errCode; } context_->SetTimeOffset(offset); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); - } if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { // higher than 102 version may go to here, but it is ok not use slwr,after abilitysync would go slwr. errCode = HandleDataRequestRecv(inMsg); @@ -910,10 +882,29 @@ void SingleVerSyncStateMachine::StepToTimeout() SwitchStateAndStep(Event::TIME_OUT_EVENT); } +int SingleVerSyncStateMachine::GetSyncOperationStatus(int errCode) const +{ + static const std::map statusMap = { + { -E_SCHEMA_MISMATCH, SyncOperation::SCHEMA_INCOMPATIBLE }, + { -E_EKEYREVOKED, SyncOperation::EKEYREVOKED_FAILURE }, + { -E_SECURITY_OPTION_CHECK_ERROR, SyncOperation::SECURITY_OPTION_CHECK_FAILURE }, + { -E_BUSY, SyncOperation::BUSY_FAILURE }, + { -E_NOT_PERMIT, SyncOperation::PERMISSION_CHECK_FAILED }, + { -E_TIMEOUT, SyncOperation::TIMEOUT }, + { -E_INVALID_QUERY_FORMAT, SyncOperation::QUERY_FORMAT_FAILURE }, + { -E_INVALID_QUERY_FIELD, SyncOperation::QUERY_FIELD_FAILURE } + }; + auto iter = statusMap.find(errCode); + if (iter != statusMap.end()) { + return iter->second; + } + return SyncOperation::FAILED; +} + int SingleVerSyncStateMachine::TimeMarkSyncRecv(const Message *inMsg) { - LOGD("[StateMachine][TimeMarkSyncRecv] type=%d,label=%s,dev=%s{private}", inMsg->GetMessageType(), - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGD("[StateMachine][TimeMarkSyncRecv] type=%d,label=%s,dev=%s", inMsg->GetMessageType(), + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); { std::lock_guard lock(stateMachineLock_); (void)ResetWatchDog(); @@ -950,26 +941,34 @@ bool SingleVerSyncStateMachine::IsPacketValid(const Message *inMsg) const return false; } - if ((inMsg->GetMessageId() < TIME_SYNC_MESSAGE) || (inMsg->GetMessageId() > ABILITY_SYNC_MESSAGE)) { + if ((inMsg->GetMessageId() < TIME_SYNC_MESSAGE) || (inMsg->GetMessageId() >= UNKNOW_MESSAGE)) { LOGE("[StateMachine][IsPacketValid] Message is invalid, id=%d", inMsg->GetMessageId()); return false; } + // filter invalid ack at first + bool isResponseType = (inMsg->GetMessageType() == TYPE_RESPONSE); + if (isResponseType && (inMsg->GetMessageId() != TIME_SYNC_MESSAGE) && + (inMsg->GetSessionId() != context_->GetRequestSessionId()) && + (inMsg->GetSessionId() != context_->GetResponseSessionId())) { + LOGE("[StateMachine][IsPacketValid] Message is invalid, label=%s, dev=%s", + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); + return false; + } + // timeSync and abilitySync don't need to check sequenceId if (inMsg->GetMessageId() == TIME_SYNC_MESSAGE || inMsg->GetMessageId() == ABILITY_SYNC_MESSAGE) { return true; } + + // sliding window don't need to check sequenceId here, just return if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { return true; } - if (inMsg->GetMessageType() == TYPE_RESPONSE) { - if ((inMsg->GetSequenceId() != context_->GetSequenceId()) || - ((inMsg->GetSessionId() != context_->GetRequestSessionId()) && - (inMsg->GetSessionId() != context_->GetResponseSessionId()))) { - LOGE("[StateMachine][IsPacketValid] Message is invalid,inMsg SequenceId=%d,seqId=%d,syncId=%d", - inMsg->GetSequenceId(), context_->GetSequenceId(), context_->GetSyncId()); - return false; - } + if (isResponseType && (inMsg->GetSequenceId() != context_->GetSequenceId())) { + LOGE("[StateMachine][IsPacketValid] Message is invalid,inMsg SequenceId=%d,seqId=%d,syncId=%d", + inMsg->GetSequenceId(), context_->GetSequenceId(), context_->GetSyncId()); + return false; } return true; @@ -981,13 +980,15 @@ void SingleVerSyncStateMachine::PreStartPullResponse() context_->PopResponseTarget(target); context_->SetEndMark(target.GetEndWaterMark()); context_->SetResponseSessionId(target.GetResponseSessionId()); - context_->SetMode(SyncOperation::RESPONSE_PULL); + context_->SetMode(SyncModeType::RESPONSE_PULL); context_->ReSetSequenceId(); + context_->SetQuerySync(target.IsQuerySync()); + context_->SetQuery(target.GetQuery()); } bool SingleVerSyncStateMachine::IsRightDataResponsePkt(const Message *inMsg) const { - if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE) { + if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE || inMsg->GetMessageId() != QUERY_SYNC_MESSAGE) { return false; } @@ -995,7 +996,7 @@ bool SingleVerSyncStateMachine::IsRightDataResponsePkt(const Message *inMsg) con return false; } - if ((context_->GetMode() != SyncOperation::INVALID) && (inMsg->GetMessageType() == TYPE_RESPONSE)) { + if ((context_->GetMode() != SyncModeType::INVALID_MODE) && (inMsg->GetMessageType() == TYPE_RESPONSE)) { return true; } @@ -1028,19 +1029,34 @@ int SingleVerSyncStateMachine::MessageCallbackPre(const Message *inMsg) return E_OK; } -void SingleVerSyncStateMachine::AddPullResponseTarget(WaterMark pullEndWatermark, uint32_t sessionId) +void SingleVerSyncStateMachine::AddPullResponseTarget(const Message *inMsg, WaterMark pullEndWatermark) { + int messageType = inMsg->GetMessageId(); + uint32_t sessionId = inMsg->GetSessionId(); if (pullEndWatermark == 0) { LOGE("[StateMachine][AddPullResponseTarget] pullEndWatermark is 0!"); return; } + if (context_->GetResponseSessionId() == sessionId || context_->FindResponseSyncTarget(sessionId)) { + LOGI("[StateMachine][AddPullResponseTarget] task is already running"); + return; + } + const DataRequestPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + LOGE("[AddPullResponseTarget] get packet object failed"); + return; + } SingleVerSyncTarget *targetTmp = new (std::nothrow) SingleVerSyncTarget; if (targetTmp == nullptr) { LOGE("[StateMachine][AddPullResponseTarget] add failed, may oom"); return; } targetTmp->SetTaskType(ISyncTarget::RESPONSE); - targetTmp->SetMode(SyncOperation::RESPONSE_PULL); + if (messageType == QUERY_SYNC_MESSAGE) { + targetTmp->SetQuery(packet->GetQuery()); + targetTmp->SetQuerySync(true); + } + targetTmp->SetMode(SyncModeType::RESPONSE_PULL); targetTmp->SetEndWaterMark(pullEndWatermark); targetTmp->SetResponseSessionId(sessionId); if (context_->AddSyncTarget(targetTmp) != E_OK) { @@ -1057,8 +1073,8 @@ Event SingleVerSyncStateMachine::TransformErrCodeToEvent(int errCode) switch (errCode) { case -E_TIMEOUT: return TransforTimeOutErrCodeToEvent(); - case -VERSION_NOT_SUPPORT_EVENT: - return Event::VERSION_NOT_SUPPORT_EVENT; + case -VERSION_NOT_SUPPOR_EVENT: + return Event::VERSION_NOT_SUPPOR_EVENT; case -E_SEND_DATA: return Event::SEND_DATA_EVENT; case -E_NO_DATA_SEND: @@ -1101,10 +1117,11 @@ bool SingleVerSyncStateMachine::IsNeedResetWatchdog(const Message *inMsg) const Event SingleVerSyncStateMachine::TransforTimeOutErrCodeToEvent() { - if (syncContext_->IsAutoSync() && (syncContext_->GetRetryTime() < RETRY_TIME)) { + if (syncContext_->IsSyncTaskNeedRetry() && (syncContext_->GetRetryTime() < syncContext_->GetSyncRetryTimes())) { return Event::WAIT_TIME_OUT_EVENT; + } else { + return Event::TIME_OUT_EVENT; } - return Event::TIME_OUT_EVENT; } void SingleVerSyncStateMachine::SetSlidingWindowSenderErr(bool isErr) @@ -1114,8 +1131,8 @@ void SingleVerSyncStateMachine::SetSlidingWindowSenderErr(bool isErr) bool SingleVerSyncStateMachine::IsNeedErrCodeHandle(uint32_t sessionId) const { - // version_102 omit to set sessionId so version_3 should skip to compare sessionid. - if (sessionId == context_->GetRequestSessionId() || context_->GetRequestSessionId() == 0 || + // Release2.0 omit to set sessionId so version_3 should skip to compare sessionid. + if (sessionId == context_->GetRequestSessionId() || context_->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_RELEASE_2_0) { return true; } else { @@ -1123,18 +1140,89 @@ bool SingleVerSyncStateMachine::IsNeedErrCodeHandle(uint32_t sessionId) const } } -void SingleVerSyncStateMachine::PushPullDataRequestEvokeErrHandle(SingleVerSyncTaskContext *context) +void SingleVerSyncStateMachine::PushPullDataRequestEvokeErrHandle() { // the pushpull sync task should wait for send finished after remote dev get data occur E_EKEYREVOKED error. - if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && - context->GetMode() == SyncOperation::PUSH_AND_PULL) { + if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && + SyncOperation::TransferSyncMode(context_->GetMode()) == SyncModeType::PUSH_AND_PULL) { LOGI("data request errCode = %d, wait for send finished", -E_EKEYREVOKED); - context->SetTaskErrCode(-E_EKEYREVOKED); - context->SetOperationStatus(SyncOperation::RECV_FINISHED); + context_->SetTaskErrCode(-E_EKEYREVOKED); + context_->SetOperationStatus(SyncOperation::RECV_FINISHED); SwitchStateAndStep(Event::RECV_FINISHED_EVENT); } else { - context->SetTaskErrCode(-E_EKEYREVOKED); + context_->SetTaskErrCode(-E_EKEYREVOKED); SwitchStateAndStep(Event::INNER_ERR_EVENT); } } + +void SingleVerSyncStateMachine::DataRecvErrCodeHandle(uint32_t sessionId, int errCode) +{ + if (IsNeedErrCodeHandle(sessionId)) { + switch (errCode) { + case E_OK: + break; + case -E_RECV_FINISHED: + context_->SetOperationStatus(SyncOperation::RECV_FINISHED); + SwitchStateAndStep(Event::RECV_FINISHED_EVENT); + break; + case -E_EKEYREVOKED: + PushPullDataRequestEvokeErrHandle(); + break; + case -E_BUSY: + case -E_SECURITY_OPTION_CHECK_ERROR: + case -E_INVALID_QUERY_FORMAT: + case -E_INVALID_QUERY_FIELD: + context_->SetTaskErrCode(errCode); + SwitchStateAndStep(Event::INNER_ERR_EVENT); + break; + default: + SwitchStateAndStep(Event::INNER_ERR_EVENT); + break; + } + } +} + +bool SingleVerSyncStateMachine::AbilityMsgSessionIdCheck(const Message *inMsg) +{ + if (inMsg != nullptr && inMsg->GetSessionId() == context_->GetRequestSessionId()) { + return true; + } + LOGE("[AbilitySync] session check failed,dev=%s", STR_MASK(context_->GetDeviceId())); + return false; +} + +int SingleVerSyncStateMachine::GetSyncType(uint32_t messageId) const +{ + if (messageId == QUERY_SYNC_MESSAGE) { + return SyncType::QUERY_SYNC_TYPE; + } + return SyncType::MANUAL_FULL_SYNC_TYPE; +} + +void SingleVerSyncStateMachine::AckRecvErrCodeHandle(int errCode) +{ + switch (errCode) { + case -E_NEED_ABILITY_SYNC: + NeedAbilitySyncHandle(); + break; + case -E_NO_DATA_SEND: + // when version is higher than 102, this step will be handle in slide windows function. + if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { + context_->SetOperationStatus(SyncOperation::SEND_FINISHED); + } + break; + case -E_NOT_PERMIT: + context_->SetOperationStatus(SyncOperation::PERMISSION_CHECK_FAILED); + break; + case -E_EKEYREVOKED: + case -E_SECURITY_OPTION_CHECK_ERROR: + case -E_BUSY: + case -E_INVALID_QUERY_FORMAT: + case -E_INVALID_QUERY_FIELD: + context_->SetTaskErrCode(errCode); + break; + default: + break; + } +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h index 44d5733d7..6f5a5af40 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h @@ -21,7 +21,7 @@ #include "sync_state_machine.h" #include "sync_target.h" -#include "semaphore_utils.h" +#include "semaphore.h" #include "message.h" #include "single_ver_sync_task_context.h" #include "time_sync.h" @@ -42,7 +42,7 @@ namespace { INACTIVE_PUSH_REMAINDER_DATA, // push remainded data if initactive sync data more than on frame PASSIVE_PUSH_REMAINDER_DATA, // push remainded data if passive sync data more than on frame WAIT_FOR_RECEIVE_DATA_FINISH, // all data send finished, wait for data revice if has pull request - SYNC_TASK_FINISHED, // current sync task finished, try to schedule next sync task + SYNC_TASK_FINISHED, // current sync task finihsed, try to schedule next sync task SYNC_TIME_OUT, INNER_ERR, START_INITIACTIVE_SLIDING_DATA_SYNC, // used for do sync started by local device, use sliding window @@ -53,7 +53,7 @@ namespace { START_SYNC_EVENT = 1, TIME_SYNC_FINISHED_EVENT, ABILITY_SYNC_FINISHED_EVENT, - VERSION_NOT_SUPPORT_EVENT, + VERSION_NOT_SUPPOR_EVENT, SWITCH_TO_PROCTOL_V1_EVENT, SEND_DATA_EVENT, SEND_FINISHED_EVENT, @@ -95,7 +95,9 @@ public: bool IsNeedErrCodeHandle(uint32_t sessionId) const; - void PushPullDataRequestEvokeErrHandle(SingleVerSyncTaskContext *context); + void PushPullDataRequestEvokeErrHandle(); + + void DataRecvErrCodeHandle(uint32_t sessionId, int errCode); protected: // Step the SingleVerSyncStateMachine @@ -122,7 +124,7 @@ protected: int PrepareNextSyncTask() override; // Called by StartSaveDataNotifyTimer, used to send a save data notify packet - void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) override; + void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) override; private: // Used to init sync state machine switchbables @@ -154,7 +156,7 @@ private: // Waiting for pull data revice finish, if coming a pull request, should goto START_PASSIVE_DATA_SYNC state Event DoWaitForDataRecv() const; - // Sync task finished, should do some data clear and exec next task. + // Sync task finihsed, should do some data clear and exec next task. Event DoSyncTaskFinished(); // Do something when sync timeout. @@ -167,6 +169,8 @@ private: Event DoPassiveDataSyncWithSlidingWindow(); + int GetSyncOperationStatus(int errCode) const; + int TimeMarkSyncRecv(const Message *inMsg); int AbilitySyncRecv(const Message *inMsg); @@ -193,7 +197,7 @@ private: int MessageCallbackPre(const Message *inMsg); - void AddPullResponseTarget(WaterMark pullEndWatermark, uint32_t sessionId); + void AddPullResponseTarget(const Message *inMsg, WaterMark pullEndWatermark); Event TransformErrCodeToEvent(int errCode); @@ -201,6 +205,12 @@ private: Event TransforTimeOutErrCodeToEvent(); + bool AbilityMsgSessionIdCheck(const Message *inMsg); + + int GetSyncType(uint32_t messageId) const; + + void AckRecvErrCodeHandle(int errCode); + DISABLE_COPY_ASSIGN_MOVE(SingleVerSyncStateMachine); static std::mutex stateSwitchTableLock_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.cpp index 40c328d4a..e4e053db3 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.cpp @@ -23,7 +23,8 @@ namespace DistributedDB { SingleVerSyncTarget::SingleVerSyncTarget() - : endWaterMark_(0) + : endWaterMark_(0), + query_(QueryObject()) { } @@ -31,6 +32,15 @@ SingleVerSyncTarget::~SingleVerSyncTarget() { } +void SingleVerSyncTarget::SetSyncOperation(SyncOperation *operation) +{ + SyncTarget::SetSyncOperation(operation); + if ((operation != nullptr) && !operation->IsKilled()) { + query_ = operation->GetQuery(); + isQuerySync_ = operation->IsQuerySync(); + } +} + void SingleVerSyncTarget::SetEndWaterMark(WaterMark waterMark) { endWaterMark_ = waterMark; @@ -50,4 +60,24 @@ uint32_t SingleVerSyncTarget::GetResponseSessionId() const { return responseSessionId_; } + +void SingleVerSyncTarget::SetQuery(const QuerySyncObject &query) +{ + query_ = query; +} + +QuerySyncObject SingleVerSyncTarget::GetQuery() const +{ + return query_; +} + +void SingleVerSyncTarget::SetQuerySync(bool isQuerySync) +{ + isQuerySync_ = isQuerySync; +} + +bool SingleVerSyncTarget::IsQuerySync() const +{ + return isQuerySync_; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.h index 6d1fbde39..d7d5c1ee3 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.h @@ -18,6 +18,7 @@ #include "db_types.h" #include "sync_target.h" +#include "query_sync_object.h" namespace DistributedDB { class SingleVerSyncTarget final : public SyncTarget { @@ -25,19 +26,29 @@ public: SingleVerSyncTarget(); ~SingleVerSyncTarget() override; + // Set a syncOperation + void SetSyncOperation(SyncOperation *operation) override; + // Set the end water mark of this task void SetEndWaterMark(WaterMark waterMark); // Get the end water mark of this task WaterMark GetEndWaterMark() const; + // For pull response sync void SetResponseSessionId(uint32_t responseSessionId); + uint32_t GetResponseSessionId() const override; - uint32_t GetResponseSessionId() const; - + // For query sync + void SetQuery(const QuerySyncObject &query); + QuerySyncObject GetQuery() const; + void SetQuerySync(bool isQuerySync); + bool IsQuerySync() const; private: WaterMark endWaterMark_; uint32_t responseSessionId_ = 0; + QuerySyncObject query_; + bool isQuerySync_ = false; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp index 97c855411..2459511dc 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp @@ -16,7 +16,7 @@ #include "single_ver_sync_task_context.h" #include - +#include "db_common.h" #include "db_errno.h" #include "log_print.h" #include "isyncer.h" @@ -182,9 +182,11 @@ void SingleVerSyncTaskContext::CopyTargetData(const ISyncTarget *target) SyncTaskContext::CopyTargetData(target); mode_ = targetTmp->GetMode(); endMark_ = targetTmp->GetEndWaterMark(); - if (mode_ == SyncOperation::RESPONSE_PULL) { + if (mode_ == SyncModeType::RESPONSE_PULL) { responseSessionId_ = targetTmp->GetResponseSessionId(); } + query_ = targetTmp->GetQuery(); + isQuerySync_ = targetTmp->IsQuerySync(); } void SingleVerSyncTaskContext::Clear() @@ -192,12 +194,14 @@ void SingleVerSyncTaskContext::Clear() retryTime_ = 0; ClearSyncOperation(); SyncTaskContext::Clear(); - SetMode(SyncOperation::INVALID); + SetMode(SyncModeType::INVALID_MODE); syncId_ = 0; isAutoSync_ = false; SetOperationStatus(SyncOperation::WAITING); SetEndMark(0); SetResponseSessionId(0); + query_ = QuerySyncObject(QueryObject()); + isQuerySync_ = false; } void SingleVerSyncTaskContext::Abort(int status) @@ -240,21 +244,16 @@ void SingleVerSyncTaskContext::StopFeedDogForSync(SyncDirectionFlag flag) stateMachine_->StopFeedDogForSync(flag); } -void SingleVerSyncTaskContext::SetSequenceStartAndEndTimeStamp(TimeStamp start, TimeStamp end) +void SingleVerSyncTaskContext::SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange) { - sequenceStartTimeStamp_ = start; - sequenceEndTimeStamp_ = end; + dataTimeRange_ = dataTimeRange; } -TimeStamp SingleVerSyncTaskContext::GetSequenceStartTimeStamp() const +SyncTimeRange SingleVerSyncTaskContext::GetDataTimeRange() const { - return sequenceStartTimeStamp_; + return dataTimeRange_; } -TimeStamp SingleVerSyncTaskContext::GetSequenceEndTimeStamp() const -{ - return sequenceEndTimeStamp_; -} void SingleVerSyncTaskContext::SetSessionEndTimeStamp(TimeStamp end) { @@ -341,12 +340,74 @@ bool SingleVerSyncTaskContext::GetIsSchemaSync() const bool SingleVerSyncTaskContext::IsSkipTimeoutError(int errCode) const { - if (errCode == -E_TIMEOUT && IsAutoSync() && (GetRetryTime() < ISyncStateMachine::RETRY_TIME)) { + if (errCode == -E_TIMEOUT && IsSyncTaskNeedRetry() && (GetRetryTime() < GetSyncRetryTimes())) { LOGE("[SingleVerSyncTaskContext] send message timeout error occurred"); return true; } else { return false; } } + +bool SingleVerSyncTaskContext::FindResponseSyncTarget(uint32_t responseSessionId) const +{ + std::lock_guard lock(targetQueueLock_); + auto iter = std::find_if(responseTargetQueue_.begin(), responseTargetQueue_.end(), + [responseSessionId](const ISyncTarget *target) { + return target->GetResponseSessionId() == responseSessionId; + }); + if (iter == responseTargetQueue_.end()) { + return false; + } + return true; +} + +void SingleVerSyncTaskContext::SetQuery(const QuerySyncObject &query) +{ + query_ = query; +} + +QueryObject SingleVerSyncTaskContext::GetQueryObj() const +{ + return query_.GetQueryObj(); +} + +QuerySyncObject SingleVerSyncTaskContext::GetQuery() const +{ + return query_; +} + +void SingleVerSyncTaskContext::SetQuerySync(bool isQuerySync) +{ + isQuerySync_ = isQuerySync; +} + +bool SingleVerSyncTaskContext::IsQuerySync() const +{ + return isQuerySync_; +} + +void SingleVerSyncTaskContext::SetDbAbility(DbAbility &remoteDbAbility) +{ + LOGI("[SingleVerSyncTaskContext] set dev=%s compressAlgo=%s", STR_MASK(GetDeviceId()), + remoteDbAbility.GetCompressAlgorithmStr().c_str()); + remoteDbAbility_ = remoteDbAbility; +} + +CompressAlgorithm SingleVerSyncTaskContext::ChooseCompressAlgo() const +{ + std::set remoteAlgo = remoteDbAbility_.GetCompressAlgorithm(); + if (remoteAlgo.size() == 0) { + return CompressAlgorithm::NONE; + } + std::set localAlgorithmSet; + (void)(static_cast(syncInterface_))->GetCompressionAlgo(localAlgorithmSet); + std::set algoIntersection; + set_intersection(remoteAlgo.begin(), remoteAlgo.end(), localAlgorithmSet.begin(), localAlgorithmSet.end(), + inserter(algoIntersection, algoIntersection.begin())); + if (algoIntersection.size() == 0) { + return CompressAlgorithm::NONE; + } + return *(algoIntersection.begin()); +} DEFINE_OBJECT_TAG_FACILITIES(SingleVerSyncTaskContext) } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h index 5f2f60137..49de65875 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h @@ -20,16 +20,19 @@ #include #include +#include "db_ability.h" #include "sync_target.h" #include "sync_task_context.h" #include "time_helper.h" #include "single_ver_sync_target.h" +#include "single_ver_kvdb_sync_interface.h" +#include "query_sync_object.h" namespace DistributedDB { class SingleVerSyncTaskContext final : public SyncTaskContext { public: - SingleVerSyncTaskContext(); + explicit SingleVerSyncTaskContext(); DISABLE_COPY_ASSIGN_MOVE(SingleVerSyncTaskContext); @@ -78,16 +81,12 @@ public: // stop timer to ResetWatchDog when sync data one (key,value) size bigger than mtu void StopFeedDogForSync(SyncDirectionFlag flag); - // if sended by sliding window, get the start timeStamp of data in a sequence - TimeStamp GetSequenceStartTimeStamp() const; - - // if sended by sliding window, get the end timeStamp of data in a sequence - TimeStamp GetSequenceEndTimeStamp() const; + SyncTimeRange GetDataTimeRange() const; int HandleDataRequestRecv(const Message *msg); // if sended by sliding window, set the start and and timeStamp of data in a sequence - void SetSequenceStartAndEndTimeStamp(TimeStamp start, TimeStamp end); + void SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange); // if sended by sliding window, set the last data timeStamp in a sync session void SetSessionEndTimeStamp(TimeStamp end); @@ -125,6 +124,17 @@ public: bool GetIsSchemaSync() const; bool IsSkipTimeoutError(int errCode) const; + + bool FindResponseSyncTarget(uint32_t responseSessionId) const; + + // For query sync + void SetQuery(const QuerySyncObject &query); + QueryObject GetQueryObj() const; + QuerySyncObject GetQuery() const; + void SetQuerySync(bool isQuerySync); + bool IsQuerySync() const; + void SetDbAbility(DbAbility &remoteDbAbility); + CompressAlgorithm ChooseCompressAlgo() const; protected: ~SingleVerSyncTaskContext() override; void CopyTargetData(const ISyncTarget *target) override; @@ -145,14 +155,20 @@ private: SyncStrategy syncStrategy_; bool isSchemaSync_ = false; - // in a sync session, if sended by sliding window, the min timeStamp of data in a sequence - TimeStamp sequenceStartTimeStamp_ = 0; - // in a sync session, if sended by sliding window, the max timeStamp of data in a sequence - TimeStamp sequenceEndTimeStamp_ = 0; + // normal data or delete data start timestamp and end timestamp,recorded for slws resend. + SyncTimeRange dataTimeRange_; // in a sync session, the last data timeStamp TimeStamp sessionEndTimeStamp_ = 0; + // is receive waterMark err, peerWaterMark bigger than remote localWaterMark bool isReceiveWaterMarkErr_ = false; + + // For querySync + QuerySyncObject query_; + bool isQuerySync_ = false; + + // For db ability + DbAbility remoteDbAbility_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp index 4b87eb81f..9f0e0ad4a 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp @@ -22,6 +22,7 @@ #include "ikvdb_sync_interface.h" #include "meta_data.h" #include "log_print.h" +#include "db_common.h" #include "sqlite_single_ver_natural_store.h" #include "single_ver_sync_engine.h" @@ -58,7 +59,7 @@ void SingleVerSyncer::EnableAutoSync(bool enable) LOGI("[Syncer] EnableAutoSync no online devices"); return; } - int syncId = Sync(devices, SyncOperation::AUTO_PUSH, nullptr, nullptr, false); + int syncId = Sync(devices, SyncModeType::AUTO_PUSH, nullptr, nullptr, false); if (syncId < MIN_VALID_SYNC_ID) { LOGE("[Syncer] sync start by EnableAutoSync failed err %d", syncId); } @@ -98,7 +99,7 @@ void SingleVerSyncer::LocalDataChanged(int notifyEvent) return; } - int syncId = Sync(devices, SyncOperation::AUTO_PUSH, nullptr, nullptr, false); + int syncId = Sync(devices, SyncModeType::AUTO_PUSH, nullptr, nullptr, false); if (syncId < MIN_VALID_SYNC_ID) { LOGE("[Syncer] sync start by RemoteDataChanged failed err %d", syncId); } @@ -108,13 +109,13 @@ void SingleVerSyncer::LocalDataChanged(int notifyEvent) // Remote data changed callback void SingleVerSyncer::RemoteDataChanged(const std::string &device) { - LOGI("[SingleVerSyncer] device online dev %s{private}", device.c_str()); + LOGI("[SingleVerSyncer] device online dev %s", STR_MASK(device)); if (autoSyncEnable_) { RefObject::IncObjRef(syncEngine_); int retCode = RuntimeContext::GetInstance()->ScheduleTask([this, device] { std::vector devices; devices.push_back(device); - int syncId = Sync(devices, SyncOperation::AUTO_PUSH, nullptr, nullptr, false); + int syncId = Sync(devices, SyncModeType::AUTO_PUSH, nullptr, nullptr, false); if (syncId < MIN_VALID_SYNC_ID) { LOGE("[SingleVerSyncer] sync start by RemoteDataChanged failed err %d", syncId); } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp index 35185ff11..326cb9710 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp @@ -15,6 +15,7 @@ #include "sliding_window_receiver.h" #include "sync_task_context.h" +#include "db_common.h" namespace DistributedDB { SlidingWindowReceiver::~SlidingWindowReceiver() @@ -69,11 +70,10 @@ int SlidingWindowReceiver::PutMsg(Message *inMsg) uint32_t sessionId = inMsg->GetSessionId(); uint32_t sequenceId = inMsg->GetSequenceId(); bool isLastSequence = packet->IsLastSequence(); - uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value std::unique_lock lock(lock_); if (workingId_ != 0 && sessionId_ != 0) { LOGI("[PutMsg] task is running, wait for workdingId=%u end,seId=%u", workingId_, sequenceId); - workingTaskcv_.wait(lock); + workingTaskcv_.wait(lock, [this] { return workingId_ == 0; }); } if (sessionId_ != sessionId) { ResetInfo(); @@ -86,7 +86,7 @@ int SlidingWindowReceiver::PutMsg(Message *inMsg) if (sequenceId <= hasFinishedMaxId_) { LOGI("[slwr] seId=%u,FinishedMId_=%u,label=%s", sequenceId, hasFinishedMaxId_, dataSync_->GetLabel().c_str()); lock.unlock(); - dataSync_->SendAck(context_, sessionId_, sequenceId, packetId); + dataSync_->SendAck(context_, inMsg, E_OK, 0); return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; } int errCode = ErrHandle(sequenceId); @@ -110,15 +110,15 @@ void SlidingWindowReceiver::DealMsg() { std::lock_guard lock(lock_); if (workingId_ != 0 || messageMap_.count(hasFinishedMaxId_ + 1) == 0) { - LOGI("[slwr] DealMsg do nothing workingId_=%u,hasFinishedMaxId_=%u,label=%s,deviceId=%s{private}", - workingId_, hasFinishedMaxId_, dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[slwr] DealMsg do nothing workingId_=%u,hasFinishedMaxId_=%u,label=%s,deviceId=%s", + workingId_, hasFinishedMaxId_, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); return; } workingId_ = hasFinishedMaxId_ + 1; msg = messageMap_[workingId_]; messageMap_.erase(workingId_); - LOGI("[slwr] DealMsg workingId_=%u,label=%s,deviceId=%s{private}", workingId_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[slwr] DealMsg workingId_=%u,label=%s,deviceId=%s", workingId_, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); } int errCode = context_->HandleDataRequestRecv(msg); delete msg; @@ -134,8 +134,8 @@ void SlidingWindowReceiver::DealMsg() isWaterMarkErrHappened_ = true; } else { hasFinishedMaxId_++; - LOGI("[slwr] DealMsg ok hasFinishedMaxId_=%u,label=%s,deviceId=%s{private}", hasFinishedMaxId_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[slwr] DealMsg ok hasFinishedMaxId_=%u,label=%s,deviceId=%s", hasFinishedMaxId_, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); } context_->SetReceiveWaterMarkErr(false); workingTaskcv_.notify_all(); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp index 5bbb5e383..2d22afe8c 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp @@ -14,7 +14,7 @@ */ #include "sliding_window_sender.h" - +#include "db_common.h" namespace DistributedDB { SlidingWindowSender::~SlidingWindowSender() { @@ -37,16 +37,17 @@ int SlidingWindowSender::ParamCheck(int32_t mode, const SingleVerSyncTaskContext return E_OK; } -void SlidingWindowSender::Init(int32_t mode, SingleVerSyncTaskContext *context, +void SlidingWindowSender::Init(int32_t inMode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync) { isErr_ = false; - mode_ = mode; + mode_ = inMode; context_->SetSessionEndTimeStamp(0); context_->ReSetSequenceId(); maxSequenceIdhasSent_ = 0; windowSize_ = MAX_WINDOW_SIZE; - if (mode == SyncOperation::PUSH || mode == SyncOperation::PUSH_AND_PULL || mode == SyncOperation::PULL) { + int mode = SyncOperation::TransferSyncMode(inMode); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL) { sessionId_ = context->GetRequestSessionId(); } else { sessionId_ = context->GetResponseSessionId(); @@ -71,8 +72,6 @@ int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *conte if (errCode != E_OK) { return errCode; } - LOGI("[slws] SendStart,mode=%d,label=%s,device=%s{private}", mode_, dataSync->GetLabel().c_str(), - context->GetDeviceId().c_str()); errCode = dataSync->CheckPermitSendData(mode, context); if (errCode != E_OK) { return errCode; @@ -82,11 +81,14 @@ int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *conte return ReSend(); } Init(mode, context, dataSync); - if (mode == SyncOperation::PUSH) { + LOGI("[slws] SendStart,mode=%d,label=%s,device=%s", mode_, dataSync->GetLabel().c_str(), + STR_MASK(context->GetDeviceId())); + int tmpMode = SyncOperation::TransferSyncMode(mode); + if (tmpMode == SyncModeType::PUSH) { errCode = dataSync->PushStart(context); - } else if (mode == SyncOperation::PUSH_AND_PULL) { + } else if (tmpMode == SyncModeType::PUSH_AND_PULL) { errCode = dataSync->PushPullStart(context); - } else if (mode == SyncOperation::PULL) { + } else if (tmpMode == SyncModeType::PULL) { errCode = dataSync->PullRequestStart(context); } else { errCode = dataSync->PullResponseStart(context); @@ -102,7 +104,7 @@ int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *conte return errCode; } errCode = UpdateInfo(); - if (mode == SyncOperation::PUSH_AND_PULL && context_->GetTaskErrCode() == -E_EKEYREVOKED) { + if (tmpMode == SyncModeType::PUSH_AND_PULL && context_->GetTaskErrCode() == -E_EKEYREVOKED) { LOGE("wait for recv finished for push and pull mode"); return -E_EKEYREVOKED; } @@ -115,15 +117,19 @@ int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *conte int SlidingWindowSender::UpdateInfo() { ReSendInfo reSendInfo; - reSendInfo.start = context_->GetSequenceStartTimeStamp(); - reSendInfo.end = context_->GetSequenceEndTimeStamp(); + SyncTimeRange reSendDataTimeRange = context_->GetDataTimeRange(); + reSendInfo.start = reSendDataTimeRange.beginTime; + reSendInfo.end = reSendDataTimeRange.endTime; + reSendInfo.deleteBeginTime = reSendDataTimeRange.deleteBeginTime; + reSendInfo.deleteEndTime = reSendDataTimeRange.deleteEndTime; reSendInfo.packetId = context_->GetPacketId(); maxSequenceIdhasSent_++; reSendMap_[maxSequenceIdhasSent_] = reSendInfo; windowSize_--; - LOGI("[slws] mode=%d,start=%llu,end=%llu,seqId=%d,packetId=%llu,window_size=%d,label=%s,device=%s{private}", mode_, - reSendInfo.start, reSendInfo.end, maxSequenceIdhasSent_, reSendInfo.packetId, windowSize_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[slws] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," + "label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, maxSequenceIdhasSent_, reSendInfo.packetId, windowSize_, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); ContinueToken token; context_->GetContinueToken(token); if (token == nullptr) { @@ -147,12 +153,13 @@ int SlidingWindowSender::InnerSend() } int errCode; context_->IncSequenceId(); - if (mode_ == SyncOperation::PUSH || mode_ == SyncOperation::PUSH_AND_PULL) { + int mode = SyncOperation::TransferSyncMode(mode_); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { errCode = dataSync_->PushStart(context_); } else { errCode = dataSync_->PullResponseStart(context_); } - if (mode_ == SyncOperation::PUSH_AND_PULL && errCode == -E_EKEYREVOKED) { + if ((mode == SyncModeType::PUSH_AND_PULL) && errCode == -E_EKEYREVOKED) { LOGE("errCode = %d, wait for recv finished for push and pull mode", errCode); isAllDataHasSent_ = true; return -E_EKEYREVOKED; @@ -214,8 +221,8 @@ int SlidingWindowSender::AckRecv(const Message *message) uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value uint32_t sessionId = message->GetSessionId(); uint32_t sequenceId = message->GetSequenceId(); - LOGI("[slws] AckRecv sequecneId=%d,packetId=%llu,label=%s,dev=%s{private}", sequenceId, packetId, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[slws] AckRecv sequecneId=%d,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); std::lock_guard lock(lock_); if (sessionId != sessionId_) { @@ -234,8 +241,8 @@ int SlidingWindowSender::AckRecv(const Message *message) return InnerSend(); } else if (reSendMap_.size() == 0) { context_->SetOperationStatus(SyncOperation::SEND_FINISHED); - LOGI("[slws] AckRecv all finished,label=%s,dev=%s{private}", dataSync_->GetLabel().c_str(), - context_->GetDeviceId().c_str()); + LOGI("[slws] AckRecv all finished,label=%s,dev=%s", dataSync_->GetLabel().c_str(), + STR_MASK(context_->GetDeviceId())); InnerClear(); return -E_FINISHED; } @@ -254,10 +261,12 @@ int SlidingWindowSender::ReSend() const } uint32_t sequenceId = reSendMap_.begin()->first; ReSendInfo reSendInfo = reSendMap_.begin()->second; - LOGI("[slws] ReSend mode=%d,start=%llu,end=%llu,seqId=%d,packetId=%llu,windowsize=%d,label=%s,deviceId=%s{private}", - mode_, reSendInfo.start, reSendInfo.end, sequenceId, reSendInfo.packetId, windowSize_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); - DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, reSendInfo.packetId}; + LOGI("[slws] ReSend mode=%d,start=%llu,end=%llu,delStart=%llu,delEnd=%llu,seqId=%d,packetId=%llu,windowsize=%d," + "label=%s,deviceId=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, sequenceId, reSendInfo.packetId, windowSize_, dataSync_->GetLabel().c_str(), + STR_MASK(context_->GetDeviceId())); + DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, + reSendInfo.deleteBeginTime, reSendInfo.deleteEndTime, reSendInfo.packetId}; return dataSync_->ReSend(context_, dataReSendInfo); } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h index 74727ac4e..15136228b 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h @@ -25,6 +25,8 @@ namespace DistributedDB { struct ReSendInfo { TimeStamp start = 0; TimeStamp end = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp deleteEndTime = 0; // packetId is used for matched ackpacket packetId which saved in ackPacket.reserve // if equaled, means need to handle the ack, or drop. it is always increased uint64_t packetId = 0; @@ -46,14 +48,14 @@ public: private: int ParamCheck(int32_t mode, const SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - void Init(int32_t mode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); + void Init(int32_t inMode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); int UpdateInfo(); int InnerSend(); void InnerClear(); int ReSend() const; // when timeout, used for reSend data // initial max window size - static const int MAX_WINDOW_SIZE = 5; + static const int MAX_WINDOW_SIZE = 3; std::mutex lock_; // 0 is default invalid sessionId. uint32_t sessionId_ = 0; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp index 658fa94ba..d8bda1d2d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp @@ -30,7 +30,7 @@ #include "multi_ver_data_sync.h" #include "value_slice_sync.h" #endif -#include "single_ver_data_sync.h" +#include "single_ver_serialize_manager.h" #include "ability_sync.h" #include "device_manager.h" #include "db_common.h" @@ -47,7 +47,9 @@ SyncEngine::SyncEngine() deviceManager_(nullptr), metadata_(nullptr), timeChangedListener_(nullptr), - execTaskCount_(0) + execTaskCount_(0), + isSyncRetry_(false), + communicatorProxy_(nullptr) { } @@ -115,6 +117,7 @@ int SyncEngine::Initialize(IKvDBSyncInterface *syncInterface, std::shared_ptrDrop(true); timeChangedListener_ = nullptr; } - if (communicator_ != nullptr) { - communicator_->RegOnMessageCallback(nullptr, nullptr); - communicator_->RegOnConnectCallback(nullptr, nullptr); - communicator_->RegOnSendableCallback(nullptr, nullptr); - } + UnRegCommunicatorsCallback(); + // Clear SyncContexts { std::unique_lock lock(contextMapLock_); @@ -143,17 +143,8 @@ int SyncEngine::Close() lock.lock(); } } - if (communicator_ != nullptr) { - ICommunicatorAggregator *communicatorAggregator = nullptr; - int errCode = RuntimeContext::GetInstance()->GetCommunicatorAggregator(communicatorAggregator); - if (communicatorAggregator == nullptr) { - LOGF("[SyncEngine] ICommunicatorAggregator get failed when fialize SyncEngine err %d", errCode); - return errCode; - } - communicatorAggregator->ReleaseCommunicator(communicator_); - communicator_ = nullptr; - } + ReleaseCommunicators(); std::lock_guard msgLock(queueLock_); while (!msgQueue_.empty()) { Message *inMsg = msgQueue_.front(); @@ -176,7 +167,7 @@ int SyncEngine::AddSyncOperation(SyncOperation *operation) std::vector devices = operation->GetDevices(); for (const auto &deviceId : devices) { - int checkErrCode = RunPermissionCheck(deviceId, operation->GetMode()); + int checkErrCode = RunPermissionCheck(deviceId, operation->GetMode(), operation->IsAutoSync()); if (checkErrCode == E_OK) { operation->SetStatus(deviceId, SyncOperation::WAITING); int errCode = AddSyncOperForContext(deviceId, operation); @@ -241,7 +232,7 @@ int SyncEngine::InitDeviceManager(const std::function &onRemo return -E_OUT_OF_MEMORY; } - int errCode = deviceManager_->Initialize(communicator_, onRemoteDataChanged); + int errCode = deviceManager_->Initialize(communicatorProxy_, onRemoteDataChanged); if (errCode != E_OK) { LOGE("[SyncEngine] deviceManager init failed! err %d", errCode); delete deviceManager_; @@ -259,6 +250,7 @@ int SyncEngine::InitComunicator(const IKvDBSyncInterface *syncInterface) LOGE("[SyncEngine] Get ICommunicatorAggregator error when init the sync engine err = %d", errCode); return errCode; } + std::vector label = syncInterface->GetIdentifier(); communicator_ = communicatorAggregator->AllocCommunicator(label, errCode); if (communicator_ == nullptr) { @@ -267,7 +259,7 @@ int SyncEngine::InitComunicator(const IKvDBSyncInterface *syncInterface) } errCode = communicator_->RegOnMessageCallback( - std::bind(&SyncEngine::MessageReceiveCallback, this, std::placeholders::_1, std::placeholders::_2), + std::bind(&SyncEngine::MessageReciveCallback, this, std::placeholders::_1, std::placeholders::_2), []() {}); if (errCode != E_OK) { LOGE("[SyncEngine] SyncRequestCallback register failed! err = %d", errCode); @@ -275,6 +267,15 @@ int SyncEngine::InitComunicator(const IKvDBSyncInterface *syncInterface) communicator_ = nullptr; return errCode; } + + communicatorProxy_ = new (std::nothrow) CommunicatorProxy(); + if (communicatorProxy_ == nullptr) { + communicatorAggregator->ReleaseCommunicator(communicator_); + communicator_ = nullptr; + return -E_OUT_OF_MEMORY; + } + + communicatorProxy_->SetMainCommunicator(communicator_); label.resize(3); // only show 3 Bytes enough label_ = DBCommon::VectorToHexString(label); LOGD("[SyncEngine] RegOnConnectCallback"); @@ -308,12 +309,12 @@ int SyncEngine::AddSyncOperForContext(const std::string &deviceId, SyncOperation return errCode; } -void SyncEngine::MessageReceiveCallbackTask(ISyncTaskContext *context, const ICommunicator *communicator, +void SyncEngine::MessageReciveCallbackTask(ISyncTaskContext *context, const ICommunicator *communicator, Message *inMsg) { std::string deviceId = context->GetDeviceId(); if (inMsg == nullptr) { - LOGE("[SyncEngine] MessageReceiveCallback inMsg is null!"); + LOGE("[SyncEngine] MessageReciveCallback inMsg is null!"); goto MSG_CALLBACK_OUT; } @@ -406,16 +407,16 @@ ISyncTaskContext *SyncEngine::GetConextForMsg(const std::string &targetDev, int int SyncEngine::ScheduleDealMsg(ISyncTaskContext *context, Message *inMsg) { - RefObject::IncObjRef(communicator_); + RefObject::IncObjRef(communicatorProxy_); { std::lock_guard incLock(queueLock_); execTaskCount_++; } - int errCode = RuntimeContext::GetInstance()->ScheduleTask(std::bind(&SyncEngine::MessageReceiveCallbackTask, - this, context, communicator_, inMsg)); + int errCode = RuntimeContext::GetInstance()->ScheduleTask(std::bind(&SyncEngine::MessageReciveCallbackTask, + this, context, communicatorProxy_, inMsg)); if (errCode != E_OK) { - LOGE("[SyncEngine] MessageReceiveCallbackTask Schedule failed err %d", errCode); - RefObject::DecObjRef(communicator_); + LOGE("[SyncEngine] MessageReciveCallbackTask Schedule failed err %d", errCode); + RefObject::DecObjRef(communicatorProxy_); { std::lock_guard decLock(queueLock_); execTaskCount_--; @@ -424,20 +425,20 @@ int SyncEngine::ScheduleDealMsg(ISyncTaskContext *context, Message *inMsg) return errCode; } -void SyncEngine::MessageReceiveCallback(const std::string &targetDev, Message *inMsg) +void SyncEngine::MessageReciveCallback(const std::string &targetDev, Message *inMsg) { - int errCode = MessageReceiveCallbackInner(targetDev, inMsg); + int errCode = MessageReciveCallbackInner(targetDev, inMsg); if (errCode != E_OK) { delete inMsg; inMsg = nullptr; - LOGE("[SyncEngine] MessageReceiveCallback failed!"); + LOGE("[SyncEngine] MessageReciveCallback failed!"); } } -int SyncEngine::MessageReceiveCallbackInner(const std::string &targetDev, Message *inMsg) +int SyncEngine::MessageReciveCallbackInner(const std::string &targetDev, Message *inMsg) { if (targetDev.empty() || inMsg == nullptr) { - LOGE("[SyncEngine][MessageReceiveCallback] from a invalid device or inMsg is null "); + LOGE("[SyncEngine][MessageReciveCallback] from a invalid device or inMsg is null "); return -E_INVALID_ARGS; } @@ -467,7 +468,7 @@ int SyncEngine::MessageReceiveCallbackInner(const std::string &targetDev, Messag return errCode; } - LOGD("[SyncEngine] MessageReceiveCallback MSG ID = %d", inMsg->GetMessageId()); + LOGD("[SyncEngine] MessageReciveCallback MSG ID = %d", inMsg->GetMessageId()); return ScheduleDealMsg(nextContext, inMsg); } @@ -497,7 +498,8 @@ int SyncEngine::GetMsgSize(const Message *inMsg) const case ABILITY_SYNC_MESSAGE: return AbilitySync::CalculateLen(inMsg); case DATA_SYNC_MESSAGE: - return SingleVerDataSync::CalculateLen(inMsg); + case QUERY_SYNC_MESSAGE: + return SingleVerSerializeManager::CalculateLen(inMsg); #ifndef OMIT_MULTI_VER case COMMIT_HISTORY_SYNC_MESSAGE: return CommitHistorySync::CalculateLen(inMsg); @@ -532,9 +534,9 @@ ISyncTaskContext *SyncEngine::GetSyncTaskContext(const std::string &deviceId, in LOGE("[SyncEngine] SyncTaskContext alloc failed, may be no memory avliad!"); return nullptr; } - errCode = context->Initialize(deviceId, syncInterface_, metadata_, communicator_); + errCode = context->Initialize(deviceId, syncInterface_, metadata_, communicatorProxy_); if (errCode != E_OK) { - LOGE("[SyncEngine] context init failed err %d, dev %s{private}", errCode, deviceId.c_str()); + LOGE("[SyncEngine] context init failed err %d, dev %s", errCode, STR_MASK(deviceId)); RefObject::DecObjRef(context); context = nullptr; return nullptr; @@ -543,7 +545,7 @@ ISyncTaskContext *SyncEngine::GetSyncTaskContext(const std::string &deviceId, in // IncRef for SyncEngine to make sure SyncEngine is valid when context access RefObject::IncObjRef(this); context->OnLastRef([this, deviceId]() { - LOGD("[SyncEngine] SyncTaskContext for id %s{private} finalized", deviceId.c_str()); + LOGD("[SyncEngine] SyncTaskContext for id %s finalized", STR_MASK(deviceId)); RefObject::DecObjRef(this); }); context->RegOnSyncTask(std::bind(&SyncEngine::ExecSyncTask, this, context)); @@ -601,38 +603,42 @@ void SyncEngine::SetMaxQueueCacheSize(int value) int SyncEngine::GetLocalIdentity(std::string &outTarget) const { - if (communicator_ == nullptr) { - LOGE("[SyncEngine] communicator_ is nullptr, return!"); + if (communicatorProxy_ == nullptr) { + LOGE("[SyncEngine] communicatorProxy_ is nullptr, return!"); return -E_NOT_INIT; } std::string deviceId; - int errCode = communicator_->GetLocalIdentity(deviceId); + int errCode = communicatorProxy_->GetLocalIdentity(deviceId); if (errCode != E_OK) { - LOGE("[SyncEngine] communicator_ GetLocalIdentity fail errCode:%d", errCode); + LOGE("[SyncEngine] communicatorProxy_ GetLocalIdentity fail errCode:%d", errCode); return errCode; } outTarget = DBCommon::TransferHashString(deviceId); return E_OK; } -int SyncEngine::RunPermissionCheck(const std::string &deviceId, int mode) const +int SyncEngine::RunPermissionCheck(const std::string &deviceId, int inMode, bool isAutoSync) const { uint8_t flag = 0; - if (mode == SyncOperation::PUSH) { + int mode = SyncOperation::TransferSyncMode(inMode); + if (mode == SyncModeType::PUSH) { flag = CHECK_FLAG_SEND; - } else if (mode == SyncOperation::PULL) { + } else if (mode == SyncModeType::PULL) { flag = CHECK_FLAG_RECEIVE; - } else if (mode == SyncOperation::PUSH_AND_PULL) { + } else if (mode == SyncModeType::PUSH_AND_PULL) { flag = CHECK_FLAG_SEND | CHECK_FLAG_RECEIVE; } + if (isAutoSync) { + flag = flag | CHECK_FLAG_AUTOSYNC; + } std::string appId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); std::string userId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); std::string storeId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); int errCode = RuntimeContext::GetInstance()->RunPermissionCheck(userId, appId, storeId, deviceId, flag); if (errCode != E_OK) { - LOGE("[SyncEngine] RunPermissionCheck not pass errCode:%d, flag:%d, %s{private} Label=%s", - errCode, flag, deviceId.c_str(), label_.c_str()); + LOGE("[SyncEngine] RunPermissionCheck not pass errCode:%d, flag:%d, %s Label=%s", + errCode, flag, STR_MASK(deviceId), label_.c_str()); } return errCode; } @@ -641,4 +647,124 @@ std::string SyncEngine::GetLabel() const { return label_; } + +bool SyncEngine::GetSyncRetry() const +{ + return isSyncRetry_; +} + +void SyncEngine::SetSyncRetry(bool isRetry) +{ + if (isSyncRetry_ == isRetry) { + LOGI("sync retry is equal, syncTry=%d, no need to set.", isRetry); + return; + } + isSyncRetry_ = isRetry; + std::lock_guard lock(contextMapLock_); + for (auto &iter : syncTaskContextMap_) { + ISyncTaskContext *context = iter.second; + if (context != nullptr) { + context->SetSyncRetry(isRetry); + } + } +} + +int SyncEngine::SetEqualIdentifier(const std::string &identifier, const std::vector &targets) +{ + ICommunicator *communicator = nullptr; + { + std::lock_guard lock(equalCommunicatorsLock_); + if (equalCommunicators_.count(identifier) != 0) { + communicator = equalCommunicators_[identifier]; + } else { + int errCode = E_OK; + communicator = AllocCommunicator(identifier, errCode); + if (communicator == nullptr) { + return errCode; + } + equalCommunicators_[identifier] = communicator; + } + } + LOGI("[SyncEngine] set equal identifier %s, original %s", + DBCommon::TransferStringToHex(identifier).c_str(), label_.c_str()); + communicatorProxy_->SetEqualCommunicator(communicator, targets); + communicator->Activate(); + return E_OK; +} + +ICommunicator *SyncEngine::AllocCommunicator(const std::string &identifier, int &errCode) +{ + ICommunicatorAggregator *communicatorAggregator = nullptr; + errCode = RuntimeContext::GetInstance()->GetCommunicatorAggregator(communicatorAggregator); + if (communicatorAggregator == nullptr) { + LOGE("[SyncEngine] Get ICommunicatorAggregator error when SetEqualIdentifier err = %d", errCode); + return nullptr; + } + std::vector identifierVect(identifier.begin(), identifier.end()); + auto communicator = communicatorAggregator->AllocCommunicator(identifierVect, errCode); + if (communicator == nullptr) { + LOGE("[SyncEngine] AllocCommunicator error when SetEqualIdentifier! err = %d", errCode); + return communicator; + } + + errCode = communicator->RegOnMessageCallback( + std::bind(&SyncEngine::MessageReciveCallback, this, std::placeholders::_1, std::placeholders::_2), + []() {}); + if (errCode != E_OK) { + LOGE("[SyncEngine] SyncRequestCallback register failed in SetEqualIdentifier! err = %d", errCode); + communicatorAggregator->ReleaseCommunicator(communicator); + return nullptr; + } + + errCode = communicator->RegOnConnectCallback( + std::bind(&DeviceManager::OnDeviceConnectCallback, deviceManager_, + std::placeholders::_1, std::placeholders::_2), nullptr); + if (errCode != E_OK) { + LOGE("[SyncEngine][RegConnCB] register failed in SetEqualIdentifier! err %d", errCode); + communicator->RegOnMessageCallback(nullptr, nullptr); + communicatorAggregator->ReleaseCommunicator(communicator); + return nullptr; + } + + return communicator; +} + +void SyncEngine::UnRegCommunicatorsCallback() +{ + if (communicator_ != nullptr) { + communicator_->RegOnMessageCallback(nullptr, nullptr); + communicator_->RegOnConnectCallback(nullptr, nullptr); + communicator_->RegOnSendableCallback(nullptr, nullptr); + } + + std::lock_guard lock(equalCommunicatorsLock_); + for (const auto &iter : equalCommunicators_) { + iter.second->RegOnMessageCallback(nullptr, nullptr); + iter.second->RegOnConnectCallback(nullptr, nullptr); + iter.second->RegOnSendableCallback(nullptr, nullptr); + } +} + +void SyncEngine::ReleaseCommunicators() +{ + RefObject::KillAndDecObjRef(communicatorProxy_); + communicatorProxy_ = nullptr; + ICommunicatorAggregator *communicatorAggregator = nullptr; + int errCode = RuntimeContext::GetInstance()->GetCommunicatorAggregator(communicatorAggregator); + if (communicatorAggregator == nullptr) { + LOGF("[SyncEngine] ICommunicatorAggregator get failed when fialize SyncEngine err %d", errCode); + return; + } + + if (communicator_ != nullptr) { + communicatorAggregator->ReleaseCommunicator(communicator_); + communicator_ = nullptr; + } + + std::lock_guard lock(equalCommunicatorsLock_); + for (auto &iter : equalCommunicators_) { + communicatorAggregator->ReleaseCommunicator(iter.second); + } + equalCommunicators_.clear(); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h index 859607f49..32dd2988b 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h @@ -20,10 +20,11 @@ #include #include +#include "communicator_proxy.h" +#include "device_manager.h" #include "isync_engine.h" #include "isync_task_context.h" #include "task_pool.h" -#include "device_manager.h" namespace DistributedDB { constexpr uint16_t NEW_SEND_TASK = 1; @@ -73,6 +74,12 @@ public: std::string GetLabel() const override; + bool GetSyncRetry() const; + void SetSyncRetry(bool isRetry) override; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) override; + protected: // Create a context virtual ISyncTaskContext *CreateSyncTaskContext() = 0; @@ -98,13 +105,13 @@ private: int AddSyncOperForContext(const std::string &deviceId, SyncOperation *operation); // Sync Request CallbackTask run at a sub thread. - void MessageReceiveCallbackTask(ISyncTaskContext *context, const ICommunicator *communicator, Message *inMsg); + void MessageReciveCallbackTask(ISyncTaskContext *context, const ICommunicator *communicator, Message *inMsg); - // wrapper of MessageReceiveCallbackTask - void MessageReceiveCallback(const std::string &targetDev, Message *inMsg); + // wrapper of MessageReciveCallbackTask + void MessageReciveCallback(const std::string &targetDev, Message *inMsg); // Sync Request Callback - int MessageReceiveCallbackInner(const std::string &targetDev, Message *inMsg); + int MessageReciveCallbackInner(const std::string &targetDev, Message *inMsg); // Exec the given SyncTarget. and callback onComplete. int ExecSyncTask(ISyncTaskContext *context); @@ -126,7 +133,13 @@ private: ISyncTaskContext *GetConextForMsg(const std::string &targetDev, int &errCode); - int RunPermissionCheck(const std::string &deviceId, int mode) const; + int RunPermissionCheck(const std::string &deviceId, int inMode, bool isAutoSync) const; + + ICommunicator *AllocCommunicator(const std::string &identifier, int &errCode); + + void UnRegCommunicatorsCallback(); + + void ReleaseCommunicators(); IKvDBSyncInterface *syncInterface_; ICommunicator *communicator_; @@ -137,6 +150,10 @@ private: NotificationChain::Listener *timeChangedListener_; unsigned int execTaskCount_; std::string label_; + bool isSyncRetry_; + CommunicatorProxy *communicatorProxy_; + std::mutex equalCommunicatorsLock_; + std::map equalCommunicators_; static int queueCacheSize_; static int maxQueueCacheSize_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp index e5c6996fd..9509d9c81 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp @@ -14,7 +14,6 @@ */ #include "sync_operation.h" - #include "db_errno.h" #include "log_print.h" #include "performance_analysis.h" @@ -29,7 +28,9 @@ SyncOperation::SyncOperation(uint32_t syncId, const std::vector &de isBlockSync_(isBlockSync), isAutoSync_(false), isFinished_(false), - semaphore_(nullptr) + semaphore_(nullptr), + query_(QueryObject()), + isQuerySync(false) { } @@ -46,6 +47,7 @@ int SyncOperation::Initialize() for (const std::string &deviceId : devices_) { statuses_.insert(std::pair(deviceId, WAITING)); } + if (mode_ == AUTO_PUSH) { mode_ = PUSH; isAutoSync_ = true; @@ -53,9 +55,8 @@ int SyncOperation::Initialize() mode_ = PULL; isAutoSync_ = true; } - if (isBlockSync_) { - semaphore_ = std::make_unique(0); + semaphore_ = std::make_unique(0); } return E_OK; @@ -187,5 +188,51 @@ void SyncOperation::Finalize() } } +void SyncOperation::SetQuery(const QueryObject &query) +{ + query_ = QuerySyncObject(query); + isQuerySync = true; + mode_ += QUERY_SYNC_MODE_BASE; +} + +QuerySyncObject SyncOperation::GetQuery() const +{ + return query_; +} + +bool SyncOperation::IsQuerySync() const +{ + return isQuerySync; +} + +int SyncOperation::GetSyncType(int mode) +{ + static const std::map syncTypeMap = { + {SyncModeType::PUSH, SyncType::MANUAL_FULL_SYNC_TYPE}, + {SyncModeType::PULL, SyncType::MANUAL_FULL_SYNC_TYPE}, + {SyncModeType::PUSH_AND_PULL, SyncType::MANUAL_FULL_SYNC_TYPE}, + {SyncModeType::RESPONSE_PULL, SyncType::MANUAL_FULL_SYNC_TYPE}, + {SyncModeType::AUTO_PULL, SyncType::AUTO_SYNC_TYPE}, + {SyncModeType::AUTO_PUSH, SyncType::AUTO_SYNC_TYPE}, + {SyncModeType::QUERY_PUSH, SyncType::QUERY_SYNC_TYPE}, + {SyncModeType::QUERY_PULL, SyncType::QUERY_SYNC_TYPE}, + {SyncModeType::QUERY_PUSH_PULL, SyncType::QUERY_SYNC_TYPE}, + }; + auto iter = syncTypeMap.find(mode); + if (iter != syncTypeMap.end()) { + return iter->second; + } + return INVALID_SYNC_TYPE; +} + +int SyncOperation::TransferSyncMode(int mode) +{ + // AUTO_PUSH and AUTO_PULL mode is used before sync, RESPONSE_PULL is regarded as push or query push mode. + // so for the three mode, it is no need to transfered. + if (mode >= SyncModeType::QUERY_PUSH && mode <= SyncModeType::QUERY_PUSH_PULL) { + return (mode - QUERY_SYNC_MODE_BASE); + } + return mode; +} DEFINE_OBJECT_TAG_FACILITIES(SyncOperation) } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp index 911f698d2..e5f3023be 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp @@ -15,7 +15,6 @@ #include "sync_state_machine.h" -#include #include #include "log_print.h" @@ -85,7 +84,7 @@ int SyncStateMachine::TimeoutCallback(TimerId timerId) } int retryTime = syncContext_->GetRetryTime(); - if (retryTime >= RETRY_TIME || !syncContext_->IsAutoSync()) { + if (retryTime >= syncContext_->GetSyncRetryTimes() || !syncContext_->IsSyncTaskNeedRetry()) { LOGI("[SyncStateMachine][Timeout] TimeoutCallback retryTime:%d", retryTime); syncContext_->UnlockObj(); StepToTimeout(); @@ -99,9 +98,7 @@ int SyncStateMachine::TimeoutCallback(TimerId timerId) syncContext_->IncSequenceId(); } syncContext_->SetRetryStatus(SyncTaskContext::NEED_RETRY); - int timeoutTime = syncContext_->GetTimeoutTime(); - // set the new timeout value with 2 raised to the power of retryTime. - timeoutTime = timeoutTime * static_cast(pow(2, retryTime)); + int timeoutTime = syncContext_->GetSyncRetryTimeout(retryTime); syncContext_->ModifyTimer(timeoutTime); LOGI("[SyncStateMachine][Timeout] Schedule task, timeoutTime = %d, retryTime = %d", timeoutTime, retryTime); SyncStep(); @@ -148,7 +145,7 @@ int SyncStateMachine::SwitchMachineState(uint8_t event) const std::map &table = (*tableIter).switchTable; auto eventToStateIter = table.find(currentState_); if (eventToStateIter == table.end()) { - LOGE("[SyncStateMachine][SwitchState] ver:%d, Can't find EventToState with currentSate %u", + LOGE("[SyncStateMachine][SwitchState] tableVer:%d, Can't find EventToState with currentSate %u", (*tableIter).version, currentState_); SetCurStateErrStatus(); return E_OK; @@ -157,13 +154,13 @@ int SyncStateMachine::SwitchMachineState(uint8_t event) const EventToState &eventToState = eventToStateIter->second; auto stateIter = eventToState.find(event); if (stateIter == eventToState.end()) { - LOGD("[SyncStateMachine][SwitchState] ver:%d, Can't find event %u int currentSate %u ignore", + LOGD("[SyncStateMachine][SwitchState] tableVer:%d, Can't find event %u int currentSate %u ignore", (*tableIter).version, event, currentState_); return -E_NOT_FOUND; } currentState_ = stateIter->second; - LOGD("[SyncStateMachine][SwitchState] ver:%d, from state %u move to state %u with event %u dev %s{private}", + LOGD("[SyncStateMachine][SwitchState] tableVer:%d, from state %u move to state %u with event %u dev %s{private}", (*tableIter).version, eventToStateIter->first, currentState_, event, syncContext_->GetDeviceId().c_str()); return E_OK; } @@ -219,7 +216,7 @@ void SyncStateMachine::StopWatchDog() syncContext_->StopTimer(); } -bool SyncStateMachine::StartSaveDataNotify(uint32_t sessionId, uint32_t sequenceId) +bool SyncStateMachine::StartSaveDataNotify(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) { std::lock_guard lockGuard(saveDataNotifyLock_); if (saveDataNotifyTimerId_ > 0) { @@ -232,7 +229,7 @@ bool SyncStateMachine::StartSaveDataNotify(uint32_t sessionId, uint32_t sequence RefObject::IncObjRef(syncContext_); int errCode = RuntimeContext::GetInstance()->SetTimer( SAVE_DATA_NOTIFY_INTERVAL, - [this, sessionId, sequenceId](TimerId timerId) { + [this, sessionId, sequenceId, inMsgId](TimerId timerId) { { std::lock_guard lock(stateMachineLock_); (void)ResetWatchDog(); @@ -242,7 +239,7 @@ bool SyncStateMachine::StartSaveDataNotify(uint32_t sessionId, uint32_t sequence StopSaveDataNotifyNoLock(); return E_OK; } - SendSaveDataNotifyPacket(sessionId, sequenceId); + SendSaveDataNotifyPacket(sessionId, sequenceId, inMsgId); saveDataNotifyCount_++; return E_OK; }, @@ -289,7 +286,7 @@ bool SyncStateMachine::StartFeedDogForSync(uint32_t time, SyncDirectionFlag flag LOGW("[SyncStateMachine][feedDog] timer has been started!, flag:%d", flag); return false; } - int cnt = time / SAVE_DATA_NOTIFY_INTERVAL; + int cnt = GetFeedDogTimeout(time / SAVE_DATA_NOTIFY_INTERVAL); LOGI("[SyncStateMachine][feedDog] start cnt:%d, flag:%d", cnt, flag); // Incref to make sure context still alive before timer stopped. @@ -323,6 +320,14 @@ bool SyncStateMachine::StartFeedDogForSync(uint32_t time, SyncDirectionFlag flag return true; } +uint8_t SyncStateMachine::GetFeedDogTimeout(int timeoutCount) const +{ + if (timeoutCount > UINT8_MAX) { + return UINT8_MAX; + } + return timeoutCount; +} + void SyncStateMachine::StopFeedDogForSync(SyncDirectionFlag flag) { if (flag != SyncDirectionFlag::SEND && flag != SyncDirectionFlag::RECEIVE) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.h index 8d6c2066a..c922f6e45 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.h @@ -51,6 +51,8 @@ public: // start a timer to ResetWatchDog when sync data one (key,value) size bigger than mtu bool StartFeedDogForSync(uint32_t time, SyncDirectionFlag flag) override; + uint8_t GetFeedDogTimeout(int timeoutCount) const; + // stop timer to ResetWatchDog when sync data one (key,value) size bigger than mtu void StopFeedDogForSync(SyncDirectionFlag flag) override; protected: @@ -83,7 +85,7 @@ protected: virtual int PrepareNextSyncTask() = 0; // Called by StartSaveDataNotifyTimer, Sub class should realize this function to send a heartbeet packet - virtual void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) = 0; + virtual void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) = 0; // Used to parse state table to switch machine state, this function must be called in stateMachineLock int SwitchMachineState(uint8_t event); @@ -104,7 +106,7 @@ protected: void StopWatchDog(); // Start a timer to send data notify packet to keep remote device not timeout - bool StartSaveDataNotify(uint32_t sessionId, uint32_t sequenceId); + bool StartSaveDataNotify(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId); // Stop send save data notify void StopSaveDataNotify(); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.cpp index d6d350add..68343ba16 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.cpp @@ -78,5 +78,10 @@ bool SyncTarget::IsAutoSync() const } return operation_->IsAutoSync(); } + +uint32_t SyncTarget::GetResponseSessionId() const +{ + return 0; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.h index 93c01f117..3f4a04b1d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.h @@ -39,7 +39,7 @@ public: // Get the mode of this task request or response int GetMode() const override; - // Set a Sync Status, it will increase the ref of operation + // Set a SyncOperation void SetSyncOperation(SyncOperation *operation) override; // Get a SyncOperation @@ -48,6 +48,8 @@ public: // Is this target is a auto sync bool IsAutoSync() const override; + virtual uint32_t GetResponseSessionId() const override; + protected: SyncOperation *operation_; int taskType_; // sync task or response task; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp index e2a7b940d..a93dcda9f 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp @@ -16,18 +16,23 @@ #include "sync_task_context.h" #include +#include +#include "db_constant.h" #include "db_errno.h" -#include "log_print.h" -#include "isync_state_machine.h" #include "hash.h" +#include "isync_state_machine.h" +#include "log_print.h" #include "time_helper.h" -#include "db_constant.h" namespace DistributedDB { std::mutex SyncTaskContext::synTaskContextSetLock_; std::set SyncTaskContext::synTaskContextSet_; +namespace { + const int NEGOTIATION_LIMIT = 2; +} + SyncTaskContext::SyncTaskContext() : syncOperation_(nullptr), syncId_(0), @@ -43,7 +48,10 @@ SyncTaskContext::SyncTaskContext() remoteSoftwareVersion_(0), remoteSoftwareVersionId_(0), isCommNormal_(true), - taskErrCode_(E_OK) + taskErrCode_(E_OK), + syncTaskRetryStatus_(false), + isSyncRetry_(false), + negotiationCount_(0) { } @@ -130,7 +138,7 @@ void SyncTaskContext::Clear() isAutoSync_ = false; requestSessionId_ = 0; isNeedRetry_ = NO_NEED_RETRY; - mode_ = SyncOperation::INVALID; + mode_ = SyncModeType::INVALID_MODE; status_ = SyncOperation::WAITING; taskErrCode_ = E_OK; packetId_ = 0; @@ -191,6 +199,7 @@ int SyncTaskContext::GetOperationStatus() const } return syncOperation_->GetStatus(deviceId_); } + void SyncTaskContext::SetMode(int mode) { mode_ = mode; @@ -200,6 +209,7 @@ int SyncTaskContext::GetMode() const { return mode_; } + void SyncTaskContext::MoveToNextTarget() { ClearSyncOperation(); @@ -260,11 +270,6 @@ bool SyncTaskContext::IsAutoSync() const int SyncTaskContext::StartTimer() { - if (!isAutoSync_) { - timeout_ = DBConstant::MANUAL_SYNC_TIMEOUT; - } else { - timeout_ = DBConstant::AUTO_SYNC_TIMEOUT; - } std::lock_guard lockGuard(timerLock_); if (timerId_ > 0) { return -E_UNEXPECTED_DATA; @@ -518,6 +523,9 @@ void SyncTaskContext::CopyTargetData(const ISyncTarget *target) isNeedRetry_ = SyncTaskContext::NO_NEED_RETRY; taskErrCode_ = E_OK; packetId_ = 0; + syncTaskRetryStatus_ = isSyncRetry_; + timeout_ = communicator_->GetTimeout(deviceId_); + negotiationCount_ = 0; target->GetSyncOperation(syncOperation_); ReSetSequenceId(); @@ -526,13 +534,17 @@ void SyncTaskContext::CopyTargetData(const ISyncTarget *target) RefObject::IncObjRef(syncOperation_); syncId_ = syncOperation_->GetSyncId(); isAutoSync_ = syncOperation_->IsAutoSync(); + if (isAutoSync_) { + syncTaskRetryStatus_ = true; + } requestSessionId_ = Hash::Hash32Func(deviceId_ + std::to_string(syncId_) + std::to_string(TimeHelper::GetSysCurrentTime())); - LOGI("[SyncTaskContext][copyTarget] mode_ = %d, syncId_ = %d, isAutoSync_ = %d, dev = %s{private}", - mode_, syncId_, isAutoSync_, deviceId_.c_str()); + LOGI("[SyncTaskContext][copyTarget] mode=%d,syncId=%d,isAutoSync=%d,isRetry=%d,dev=%s{private}", + mode_, syncId_, isAutoSync_, syncTaskRetryStatus_, deviceId_.c_str()); } else { isAutoSync_ = false; - LOGI("[SyncTaskContext][copyTarget] for response data dev %s{private} ", deviceId_.c_str()); + LOGI("[SyncTaskContext][copyTarget] for response data dev %s{private},isRetry=%d", deviceId_.c_str(), + syncTaskRetryStatus_); } } @@ -574,9 +586,10 @@ void SyncTaskContext::CancelCurrentSyncRetryIfNeed(int newTargetMode) if (!isAutoSync_) { return; } - if (newTargetMode == mode_ || newTargetMode == SyncOperation::PUSH_AND_PULL) { - SetRetryTime(ISyncStateMachine::RETRY_TIME); - ModifyTimer(DBConstant::AUTO_SYNC_TIMEOUT); + int mode = SyncOperation::TransferSyncMode(newTargetMode); + if (newTargetMode == mode_ || mode == SyncModeType::PUSH_AND_PULL) { + SetRetryTime(AUTO_RETRY_TIMES); + ModifyTimer(timeout_); } } @@ -589,4 +602,42 @@ void SyncTaskContext::SetTaskErrCode(int errCode) { taskErrCode_ = errCode; } + +bool SyncTaskContext::IsSyncTaskNeedRetry() const +{ + return syncTaskRetryStatus_; +} + +void SyncTaskContext::SetSyncRetry(bool isRetry) +{ + isSyncRetry_ = isRetry; +} + +int SyncTaskContext::GetSyncRetryTimes() const +{ + if (IsAutoSync()) { + return AUTO_RETRY_TIMES; + } + return MANUAL_RETRY_TIMES; +} + +int SyncTaskContext::GetSyncRetryTimeout(int retryTime) const +{ + int timeoutTime = GetTimeoutTime(); + if (IsAutoSync()) { + // set the new timeout value with 2 raised to the power of retryTime. + return timeoutTime * static_cast(pow(2, retryTime)); + } + return timeoutTime; +} + +bool SyncTaskContext::IsAutoLiftWaterMark() const +{ + return negotiationCount_ < NEGOTIATION_LIMIT; +} + +void SyncTaskContext::IncNegotiationCount() +{ + negotiationCount_++; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.h index 3a542cafe..0ef0db4b6 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.h @@ -21,7 +21,7 @@ #include "isync_task_context.h" #include "sync_target.h" -#include "semaphore_utils.h" +#include "semaphore.h" #include "sync_operation.h" #include "icommunicator.h" #include "ikvdb_sync_interface.h" @@ -174,6 +174,18 @@ public: void SetTaskErrCode(int errCode) override; + bool IsSyncTaskNeedRetry() const override; + + void SetSyncRetry(bool isRetry) override; + + int GetSyncRetryTimes() const override; + + int GetSyncRetryTimeout(int retryTime) const override; + + bool IsAutoLiftWaterMark() const override; + + void IncNegotiationCount() override; + protected: const static int KILL_WAIT_SECONDS = INT32_MAX; @@ -233,8 +245,11 @@ protected: bool isCommNormal_; int taskErrCode_; uint64_t packetId_ = 0; // used for assignment to reSendMap_.ReSendInfo.packetId in 103 version or above + bool syncTaskRetryStatus_; + bool isSyncRetry_; + uint32_t negotiationCount_; - // For gloable ISyncTaskContext Set, used by CommErrCallback. + // For global ISyncTaskContext Set, used by CommErrCallback. static std::mutex synTaskContextSetLock_; static std::set synTaskContextSet_; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp index 365ae3a8f..a1b31aaae 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp @@ -56,7 +56,7 @@ int SyncerProxy::Close() int SyncerProxy::Sync(const std::vector &devices, int mode, const std::function &)> &onComplete, - const std::function &onFinalize, bool wait = false) + const std::function &onFinalize, bool wait) { if (syncer_ == nullptr) { return -E_NOT_INIT; @@ -64,6 +64,14 @@ int SyncerProxy::Sync(const std::vector &devices, int mode, return syncer_->Sync(devices, mode, onComplete, onFinalize, wait); } +int SyncerProxy::Sync(const SyncParma &parma) +{ + if (syncer_ == nullptr) { + return -E_NOT_INIT; + } + return syncer_->Sync(parma); +} + int SyncerProxy::RemoveSyncOperation(int syncId) { if (syncer_ == nullptr) { @@ -159,4 +167,20 @@ int SyncerProxy::SetStaleDataWipePolicy(WipePolicy policy) } return syncer_->SetStaleDataWipePolicy(policy); } + +int SyncerProxy::SetSyncRetry(bool isRetry) +{ + if (syncer_ == nullptr) { + return -E_NOT_INIT; + } + return syncer_->SetSyncRetry(isRetry); +} + +int SyncerProxy::SetEqualIdentifier(const std::string &identifier, const std::vector &targets) +{ + if (syncer_ == nullptr) { + return -E_NOT_INIT; + } + return syncer_->SetEqualIdentifier(identifier, targets); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.cpp index adf1348e8..8b5d55082 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.cpp @@ -29,7 +29,6 @@ namespace { constexpr uint64_t TIME_SYNC_INTERVAL = 24 * 60 * 60 * 1000; // 24h constexpr int TRIP_DIV_HALF = 2; constexpr int64_t MAX_TIME_OFFSET_NOISE = 1 * 1000 * 10000; // 1s for 100ns - constexpr int TIME_SYNC_WAIT_TIME = 5; // 5s } // Class TimeSyncPacket @@ -483,7 +482,7 @@ int TimeSync::TimeSyncDriver(TimerId timerId) return E_OK; } -int TimeSync::GetTimeOffset(TimeOffset &outOffset) +int TimeSync::GetTimeOffset(TimeOffset &outOffset, uint32_t timeout) { if (!isSynced_) { { @@ -492,15 +491,17 @@ int TimeSync::GetTimeOffset(TimeOffset &outOffset) } CommErrHandler handler = std::bind(&TimeSync::CommErrHandlerFunc, std::placeholders::_1, this); int errCode = SyncStart(handler); - LOGD("TimeSync::GetTimeOffset start, current = %llu, errCode:%d", TimeHelper::GetSysCurrentTime(), errCode); + LOGD("TimeSync::GetTimeOffset start, current time = %llu, errCode = %d，timeout = %u ms", + TimeHelper::GetSysCurrentTime(), errCode, timeout); std::unique_lock lock(cvLock_); - if (errCode != E_OK || !conditionVar_.wait_for(lock, std::chrono::seconds(TIME_SYNC_WAIT_TIME), + if (errCode != E_OK || !conditionVar_.wait_for(lock, std::chrono::milliseconds(timeout), [this](){ return this->isAckReceived_ == true; })) { LOGD("TimeSync::GetTimeOffset, retryTime_ = %d", retryTime_); retryTime_++; if (retryTime_ < MAX_RETRY_TIME) { lock.unlock(); - return GetTimeOffset(outOffset); + LOGI("TimeSync::GetTimeOffset timeout, try again"); + return GetTimeOffset(outOffset, timeout); } retryTime_ = 0; return -E_TIMEOUT; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.h index 7730e5c34..12afeba59 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.h @@ -82,7 +82,7 @@ public: int RequestRecv(const Message *message); // Get timeoffset from metadata - int GetTimeOffset(TimeOffset &outOffset); + int GetTimeOffset(TimeOffset &outOffset, uint32_t timeout); bool IsNeedSync() const; diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 10dffa5ac..a4e01532d 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -60,6 +60,8 @@ ohos_source_set("src_file") { sources = [ "../common/src/auto_launch.cpp", + "../common/src/data_compression.cpp", + "../common/src/db_ability.cpp", "../common/src/db_common.cpp", "../common/src/db_constant.cpp", "../common/src/evloop/src/event_impl.cpp", @@ -85,7 +87,7 @@ ohos_source_set("src_file") { "../common/src/runtime_context_impl.cpp", "../common/src/schema_object.cpp", "../common/src/schema_utils.cpp", - "../common/src/semaphore_utils.cpp", + "../common/src/semaphore.cpp", "../common/src/task_pool.cpp", "../common/src/task_pool_impl.cpp", "../common/src/task_queue.cpp", @@ -145,6 +147,7 @@ ohos_source_set("src_file") { "../storage/src/result_entries_window.cpp", "../storage/src/single_ver_natural_store_commit_notify_data.cpp", "../storage/src/sqlite/query_object.cpp", + "../storage/src/sqlite/query_sync_object.cpp", "../storage/src/sqlite/sqlite_local_kvdb.cpp", "../storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "../storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -152,6 +155,7 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_local_storage_executor.cpp", "../storage/src/sqlite/sqlite_multi_ver_data_storage.cpp", "../storage/src/sqlite/sqlite_multi_ver_transaction.cpp", + "../storage/src/sqlite/sqlite_single_ver_continue_token.cpp", "../storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp", "../storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store.cpp", @@ -173,6 +177,7 @@ ohos_source_set("src_file") { "../storage/src/upgrader/single_ver_schema_database_upgrader.cpp", "../syncer/src/ability_sync.cpp", "../syncer/src/commit_history_sync.cpp", + "../syncer/src/communicator_proxy.cpp", "../syncer/src/device_manager.cpp", "../syncer/src/generic_syncer.cpp", "../syncer/src/meta_data.cpp", @@ -181,8 +186,11 @@ ohos_source_set("src_file") { "../syncer/src/multi_ver_sync_state_machine.cpp", "../syncer/src/multi_ver_sync_task_context.cpp", "../syncer/src/multi_ver_syncer.cpp", + "../syncer/src/query_sync_water_mark_helper.cpp", + "../syncer/src/single_ver_data_packet.cpp", "../syncer/src/single_ver_data_sync.cpp", "../syncer/src/single_ver_data_sync_with_sliding_window.cpp", + "../syncer/src/single_ver_serialize_manager.cpp", "../syncer/src/single_ver_sync_engine.cpp", "../syncer/src/single_ver_sync_state_machine.cpp", "../syncer/src/single_ver_sync_target.cpp", @@ -203,11 +211,11 @@ ohos_source_set("src_file") { "unittest/common/common/distributeddb_data_generate_unit_test.cpp", "unittest/common/common/distributeddb_tools_unit_test.cpp", "unittest/common/interfaces/process_system_api_adapter_impl.cpp", + "unittest/common/syncer/virtual_communicator.cpp", + "unittest/common/syncer/virtual_communicator_aggregator.cpp", + "unittest/common/syncer/virtual_device.cpp", "unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp", "unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp", - "unittest/common/syncer/vitural_communicator.cpp", - "unittest/common/syncer/vitural_communicator_aggregator.cpp", - "unittest/common/syncer/vitural_device.cpp", ] configs = [ ":module_private_config" ] diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp index 989c2a3e5..0313df605 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp @@ -20,7 +20,7 @@ #include "db_common.h" #include "kvdb_manager.h" #include "distributeddb_tools_unit_test.h" -#include "vitural_communicator_aggregator.h" +#include "virtual_communicator_aggregator.h" #include "platform_specific.h" #include "kv_store_nb_conflict_data.h" #include "kvdb_pragma.h" @@ -198,7 +198,9 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch001, TestSize.Level3) * @tc.steps: step1. right param A enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, nullptr, nullptr, 0, nullptr); + AutoLaunchOption option; + option.notifier = nullptr; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, nullptr, option); EXPECT_TRUE(errCode == E_OK); /** @@ -206,14 +208,14 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch001, TestSize.Level3) * @tc.expected: step2. failed. */ g_propB.SetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, option); EXPECT_TRUE(errCode != E_OK); /** * @tc.steps: step3. right param C enable * @tc.expected: step3. success. */ - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, nullptr, option); EXPECT_TRUE(errCode == E_OK); /** @@ -270,10 +272,11 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch002, TestSize.Level3) * @tc.steps: step1. right param A B enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, observer, 0, nullptr); - EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, observer, 0, nullptr); - EXPECT_TRUE(errCode == E_OK); + AutoLaunchOption option; + option.notifier = nullptr; + option.observer = observer; + EXPECT_TRUE(RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, option) == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, option) == E_OK); /** * @tc.steps: step2. RunOnConnectCallback @@ -302,10 +305,8 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch002, TestSize.Level3) * @tc.steps: step4. param A B disable * @tc.expected: step4. notifier WRITE_CLOSED */ - errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierA); - EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierB); - EXPECT_TRUE(errCode == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierA) == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierB) == E_OK); std::unique_lock lock(cvMutex); cv.wait(lock, [&finished] {return finished;}); @@ -345,9 +346,12 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch003, TestSize.Level3) * @tc.steps: step1. right param A B enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, observer, 0, nullptr); + AutoLaunchOption option; + option.notifier = nullptr; + option.observer = observer; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, observer, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, option); EXPECT_TRUE(errCode == E_OK); /** @@ -401,28 +405,30 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch004, TestSize.Level3) * @tc.steps: step1. right param A~H enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, nullptr, nullptr, 0, nullptr); + AutoLaunchOption option; + option.notifier = nullptr; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propD, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propD, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propE, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propE, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propF, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propF, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propG, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propG, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propH, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propH, nullptr, option); EXPECT_TRUE(errCode == E_OK); /** * @tc.steps: step2. right param I enable * @tc.expected: step2. -E_MAX_LIMITS. */ - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propI, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propI, nullptr, option); EXPECT_TRUE(errCode == -E_MAX_LIMITS); /** @@ -436,7 +442,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch004, TestSize.Level3) * @tc.steps: step4. right param I enable * @tc.expected: step4. E_OK. */ - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propI, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propI, nullptr, option); EXPECT_TRUE(errCode == E_OK); /** @@ -498,7 +504,10 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch005, TestSize.Level3) * @tc.steps: step2. right param A enable * @tc.expected: step2. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, observer, 0, nullptr); + AutoLaunchOption option; + option.notifier = nullptr; + option.observer = observer; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, option); EXPECT_TRUE(errCode == E_OK); /** @@ -562,11 +571,13 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch006, TestSize.Level3) cv.notify_one(); }); thread.detach(); - + AutoLaunchOption option; + option.notifier = nullptr; + option.observer = observer; for (int i = 0; i < TEST_ENABLE_CNT; i++) { - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, observer, 0, nullptr); + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, observer, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, option); EXPECT_TRUE(errCode == E_OK); errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierA); @@ -847,10 +858,14 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch012, TestSize.Level3) * @tc.steps: step1. right param A B enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, TestAutoLaunchNotifier, nullptr, - CONFLICT_FOREIGN_KEY_ONLY, ConflictNotifierCallback); + AutoLaunchOption option; + option.notifier = ConflictNotifierCallback; + option.conflictType = CONFLICT_FOREIGN_KEY_ONLY; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, TestAutoLaunchNotifier, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, nullptr, 0, nullptr); + AutoLaunchOption option1; + option1.notifier = nullptr; + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, option1); EXPECT_TRUE(errCode == E_OK); /** @@ -917,10 +932,10 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch013, TestSize.Level3) * @tc.steps: step1. right param b c enable, a SetAutoLaunchRequestCallback * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, nullptr, 0, nullptr); - EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, notifier, nullptr, 0, nullptr); - EXPECT_TRUE(errCode == E_OK); + AutoLaunchOption option; + option.notifier = nullptr; + EXPECT_TRUE(RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, option) == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, notifier, option) == E_OK); KvStoreObserverUnitTest *observer = new (std::nothrow) KvStoreObserverUnitTest; ASSERT_TRUE(observer != nullptr); @@ -978,9 +993,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch013, TestSize.Level3) * @tc.steps: step4. param A B disable * @tc.expected: step4. notifier WRITE_CLOSED */ - errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierB); - EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierC); - EXPECT_TRUE(errCode == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierB) == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierC) == E_OK); g_communicatorAggregator->RunOnConnectCallback(REMOTE_DEVICE_ID, false); } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp index 308d2964f..ef67ff9f1 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp @@ -17,10 +17,12 @@ #include "platform_specific.h" #include +#include #include #include #include +#include #include #include @@ -513,7 +515,8 @@ bool DistributedDBToolsUnitTest::IsKvEntryExist(const DistributedDB::Entry &entr return isFound; } -int DistributedDBToolsUnitTest::ModifyDatabaseFile(const std::string &fileDir) +int DistributedDBToolsUnitTest::ModifyDatabaseFile(const std::string &fileDir, uint64_t modifyPos, + uint32_t modifyCnt, uint32_t value) { LOGI("Modify database file:%s", fileDir.c_str()); std::fstream dataFile(fileDir, std::fstream::binary | std::fstream::out | std::fstream::in); @@ -538,13 +541,15 @@ int DistributedDBToolsUnitTest::ModifyDatabaseFile(const std::string &fileDir) } } - const int sqliteCountPos = 0; - if (!dataFile.seekp(sqliteCountPos)) { + if (fileSize <= modifyPos) { + return E_OK; + } + + if (!dataFile.seekp(modifyPos)) { return -E_UNEXPECTED_DATA; } - uint32_t currentCount = 0x1F1F1F1F; // add the random value to corrupt the head. - for (int i = 0; i < 256; i++) { // 256 is 1024 / 4 times. - if (!dataFile.write(reinterpret_cast(¤tCount), sizeof(uint32_t))) { + for (uint32_t i = 0; i < modifyCnt; i++) { + if (!dataFile.write(reinterpret_cast(&value), sizeof(uint32_t))) { return -E_UNEXPECTED_DATA; } } @@ -758,4 +763,29 @@ void KvStoreCorruptInfo::Reset() { databaseInfoVect_.clear(); } + +int DistributedDBToolsUnitTest::GetRandInt(const int randMin, const int randMax) +{ + std::random_device randDev; + std::mt19937 genRand(randDev()); + std::uniform_int_distribution disRand(randMin, randMax); + return disRand(genRand); +} + +int64_t DistributedDBToolsUnitTest::GetRandInt64(const int64_t randMin, const int64_t randMax) +{ + std::random_device randDev; + std::mt19937_64 genRand(randDev()); + std::uniform_int_distribution disRand(randMin, randMax); + return disRand(genRand); +} + +void DistributedDBToolsUnitTest::PrintTestCaseInfo() +{ + testing::UnitTest *test = testing::UnitTest::GetInstance(); + ASSERT_NE(test, nullptr); + const testing::TestInfo *testInfo = test->current_test_info(); + ASSERT_NE(testInfo, nullptr); + LOGI("Start unit test: %s.%s", testInfo->test_case_name(), testInfo->name()); +} } // namespace DistributedDBUnitTest diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h index 0b01f496c..e1641da15 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h @@ -38,22 +38,23 @@ #include "sqlite_utils.h" #include "single_ver_kv_entry.h" #include "sqlite_single_ver_natural_store.h" +#include "query.h" namespace DistributedDBUnitTest { static const int DIR_PERMISSION = 0750; struct DatabaseInfo { - std::string appId; - std::string userId; - std::string storeId; - std::string dir; + std::string appId{}; + std::string userId{}; + std::string storeId{}; + std::string dir{}; int dbUserVersion = 0; }; struct SyncInputArg { - uint64_t begin_; - uint64_t end_; - uint32_t blockSize_; + uint64_t begin_{}; + uint64_t end_{}; + uint32_t blockSize_{}; SyncInputArg(uint64_t begin, uint64_t end, uint32_t blockSize) : begin_(begin), end_(end), blockSize_(blockSize) {} @@ -156,7 +157,8 @@ public: static int CreateMockMultiDb(DatabaseInfo &dbInfo, DistributedDB::OpenDbProperties &properties); - static int ModifyDatabaseFile(const std::string &fileDir); + static int ModifyDatabaseFile(const std::string &fileDir, uint64_t modifyPos = 0, + uint32_t modifyCnt = 256, uint32_t value = 0x1F1F1F1F); static int GetSyncDataTest(const SyncInputArg &syncInputArg, DistributedDB::SQLiteSingleVerNaturalStore *store, std::vector &dataItems, DistributedDB::ContinueToken &continueStmtToken); @@ -181,11 +183,16 @@ public: static int GetResourceDir(std::string& dir); + static int GetRandInt(const int randMin, const int randMax); + static int64_t GetRandInt64(const int64_t randMin, const int64_t randMax); + + static void PrintTestCaseInfo(); + private: static int OpenMockMultiDb(DatabaseInfo &dbInfo, DistributedDB::OpenDbProperties &properties); - std::mutex syncLock_; - std::condition_variable syncCondVar_; + std::mutex syncLock_{}; + std::condition_variable syncCondVar_{}; }; class KvStoreObserverUnitTest : public DistributedDB::KvStoreObserver { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp index 491612491..26ff8c32b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp @@ -53,6 +53,16 @@ uint32_t AdapterStub::GetMtuSize(const std::string &target) return GetMtuSize(); } +uint32_t AdapterStub::GetTimeout() +{ + return -E_TIMEOUT; +} + +uint32_t AdapterStub::GetTimeout(const std::string &target) +{ + return GetTimeout(); +} + int AdapterStub::GetLocalIdentity(std::string &outTarget) { outTarget = localTarget_; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h index 5b39c4048..ac758bc3b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h @@ -36,6 +36,8 @@ public: void StopAdapter() override; uint32_t GetMtuSize() override; uint32_t GetMtuSize(const std::string &target) override; + uint32_t GetTimeout() override; + uint32_t GetTimeout(const std::string &target) override; int GetLocalIdentity(std::string &outTarget) override; int SendBytes(const std::string &dstTarget, const uint8_t *bytes, uint32_t length) override; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_test.cpp index 6057dd90d..e9163cfed 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_test.cpp @@ -385,7 +385,7 @@ HWTEST_F(DistributedDBInterfacesDatabaseTest, GetKvStore001, TestSize.Level1) option = {false, true, false}; g_mgr.GetKvStore("distributed_db_test3", option, g_kvDelegateCallback); ASSERT_TRUE(g_kvDelegatePtr == nullptr); - EXPECT_TRUE(g_kvDelegateStatus == DB_ERROR); + EXPECT_NE(g_kvDelegateStatus, OK); /** * @tc.steps: step6. Obtain the kvStore through the GetKvStore interface of the delegate manager @@ -395,7 +395,7 @@ HWTEST_F(DistributedDBInterfacesDatabaseTest, GetKvStore001, TestSize.Level1) option = {false, false, false}; g_mgr.GetKvStore("distributed_db_test4", option, g_kvDelegateCallback); ASSERT_TRUE(g_kvDelegatePtr == nullptr); - EXPECT_TRUE(g_kvDelegateStatus == DB_ERROR); + EXPECT_NE(g_kvDelegateStatus, OK); /** * @tc.steps: step7. Obtain the kvStore through the GetKvStore interface of the delegate manager diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp index ed731588c..3e13c09d7 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp @@ -344,7 +344,7 @@ HWTEST_F(DistributedDBInterfacesImportAndExportTest, ExportParameterCheck001, Te * @tc.steps: step1. The filePath path does not exist. * @tc.expected: step1. Return INVALID_ARGS. */ - std::string invalidFileName = g_exportFileDir + "/jadaksdjadkjsa/" + "/ExportParameterCheck001.$$"; + std::string invalidFileName = g_exportFileDir + "/tempNotCreated/" + "/ExportParameterCheck001.$$"; CipherPassword passwd; EXPECT_EQ(g_kvNbDelegatePtr->Export(invalidFileName, passwd), INVALID_ARGS); @@ -411,7 +411,7 @@ HWTEST_F(DistributedDBInterfacesImportAndExportTest, ExportParameterCheck002, Te * @tc.steps: step1. The filePath path does not exist. * @tc.expected: step1. Return INVALID_ARGS. */ - std::string invalidExportFileName = g_exportFileDir + "/jadaksdjadkjsa/" + "/ExportParameterCheck002.$$"; + std::string invalidExportFileName = g_exportFileDir + "/tempNotCreated/" + "/ExportParameterCheck002.$$"; CipherPassword passwd; EXPECT_EQ(g_kvDelegatePtr->Export(invalidExportFileName, passwd), INVALID_ARGS); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp index 7eace5ee1..5a5d30576 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp @@ -18,10 +18,11 @@ #include "ability_sync.h" #include "version.h" #include "sync_types.h" -#include "vitural_communicator_aggregator.h" -#include "vitural_communicator.h" +#include "virtual_communicator_aggregator.h" +#include "virtual_communicator.h" #include "virtual_single_ver_sync_db_Interface.h" #include "single_ver_sync_task_context.h" +#include "distributeddb_tools_unit_test.h" using namespace std; using namespace testing::ext; @@ -35,8 +36,10 @@ namespace { VirtualSingleVerSyncDBInterface *g_syncInterface = nullptr; VirtualCommunicatorAggregator *g_communicatorAggregator = nullptr; + ICommunicator *g_communicatorA = nullptr; ICommunicator *g_communicatorB = nullptr; + std::shared_ptr g_meta = nullptr; } class DistributedDBAbilitySyncTest : public testing::Test { @@ -63,6 +66,7 @@ void DistributedDBAbilitySyncTest::TearDownTestCase(void) void DistributedDBAbilitySyncTest::SetUp(void) { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create the instance for virtual communicator, virtual storage */ @@ -76,6 +80,8 @@ void DistributedDBAbilitySyncTest::SetUp(void) ASSERT_TRUE(g_communicatorA != nullptr); g_communicatorB = g_communicatorAggregator->AllocCommunicator(DEVICE_B, errCode); ASSERT_TRUE(g_communicatorB != nullptr); + g_meta = std::make_shared(); + g_meta->Initialize(g_syncInterface); } void DistributedDBAbilitySyncTest::TearDown(void) @@ -115,10 +121,14 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest001, TestSize.Level0) * @tc.steps: step2. set version = ABILITY_SYNC_VERSION_V1. schema = TEST_SCHEMA. */ AbilitySyncRequestPacket packet1; + DbAbility ability1; + std::set localAlgorithmSet = {CompressAlgorithm::ZLIB}; + ability1.SetCompressAlgorithm(localAlgorithmSet); packet1.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); - packet1.SetSoftwareVersion(SOFTWARE_VERSION_BASE); + packet1.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); packet1.SetSchema(TEST_SCHEMA); packet1.SetSendCode(E_OK); + packet1.SetDbAbility(ability1); Message msg1(ABILITY_SYNC_MESSAGE); msg1.SetMessageType(TYPE_REQUEST); msg1.SetCopiedObject(packet1); @@ -146,10 +156,10 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest001, TestSize.Level0) * @tc.expected: step5. packet1 == packet2 */ EXPECT_TRUE(packet2->GetProtocolVersion() == ABILITY_SYNC_VERSION_V1); - EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_BASE); + EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(packet2->GetSendCode() == E_OK); - std::string schema; - packet2->GetSchema(schema); + EXPECT_TRUE(packet2->GetDbAbility() == ability1); + std::string schema = packet2->GetSchema(); EXPECT_EQ(schema, TEST_SCHEMA); } @@ -214,7 +224,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest003, TestSize.Level0) */ AbilitySyncRequestPacket packet1; packet1.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); - packet1.SetSoftwareVersion(SOFTWARE_VERSION_RELEASE_3_0); + packet1.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); packet1.SetSchema(TEST_SCHEMA); packet1.SetSendCode(E_OK); int secLabel = 3; // label 3 @@ -248,10 +258,9 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest003, TestSize.Level0) * @tc.expected: step5. packet1 == packet2 */ EXPECT_TRUE(packet2->GetProtocolVersion() == ABILITY_SYNC_VERSION_V1); - EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_RELEASE_3_0); + EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(packet2->GetSendCode() == E_OK); - std::string schema; - packet2->GetSchema(schema); + std::string schema = packet2->GetSchema(); EXPECT_EQ(schema, TEST_SCHEMA); EXPECT_TRUE(packet2->GetSecFlag() == secFlag); EXPECT_TRUE(packet2->GetSecLabel() == secLabel); @@ -308,8 +317,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest004, TestSize.Level0) EXPECT_TRUE(packet2->GetProtocolVersion() == ABILITY_SYNC_VERSION_V1); EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_RELEASE_2_0); EXPECT_TRUE(packet2->GetSendCode() == E_OK); - std::string schema; - packet2->GetSchema(schema); + std::string schema = packet2->GetSchema(); EXPECT_EQ(schema, TEST_SCHEMA); EXPECT_TRUE(packet2->GetSecFlag() == 0); EXPECT_TRUE(packet2->GetSecLabel() == 0); @@ -329,10 +337,14 @@ HWTEST_F(DistributedDBAbilitySyncTest, AckPacketTest001, TestSize.Level0) * @tc.steps: step2. set version = ABILITY_SYNC_VERSION_V1. schema = TEST_SCHEMA. */ AbilitySyncAckPacket packet1; + DbAbility ability1; + std::set localAlgorithmSet = {CompressAlgorithm::ZLIB}; + ability1.SetCompressAlgorithm(localAlgorithmSet); packet1.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); packet1.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); packet1.SetSchema(TEST_SCHEMA); packet1.SetAckCode(E_VERSION_NOT_SUPPORT); + packet1.SetDbAbility(ability1); Message msg1(ABILITY_SYNC_MESSAGE); msg1.SetMessageType(TYPE_RESPONSE); msg1.SetCopiedObject(packet1); @@ -362,8 +374,8 @@ HWTEST_F(DistributedDBAbilitySyncTest, AckPacketTest001, TestSize.Level0) EXPECT_TRUE(packet2->GetProtocolVersion() == ABILITY_SYNC_VERSION_V1); EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(packet2->GetAckCode() == E_VERSION_NOT_SUPPORT); - std::string schema; - packet2->GetSchema(schema); + EXPECT_TRUE(packet2->GetDbAbility() == ability1); + std::string schema = packet2->GetSchema(); ASSERT_TRUE(schema == TEST_SCHEMA); } @@ -380,7 +392,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, SyncStart001, TestSize.Level0) * @tc.steps: step1. create a AbilitySync */ AbilitySync async; - async.Initialize(g_communicatorB, g_syncInterface, DEVICE_A); + async.Initialize(g_communicatorB, g_syncInterface, g_meta, DEVICE_A); /** * @tc.steps: step2. call SyncStart @@ -413,7 +425,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestReceiveTest001, TestSize.Level0) * @tc.steps: step1. create a AbilitySync */ AbilitySync async; - async.Initialize(g_communicatorB, g_syncInterface, DEVICE_A); + async.Initialize(g_communicatorB, g_syncInterface, g_meta, DEVICE_A); /** * @tc.steps: step2. call RequestRecv, set inMsg nullptr or set context nullptr @@ -484,7 +496,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, AckReceiveTest001, TestSize.Level0) * @tc.steps: step1. create a AbilitySync */ AbilitySync async; - async.Initialize(g_communicatorB, g_syncInterface, DEVICE_A); + async.Initialize(g_communicatorB, g_syncInterface, g_meta, DEVICE_A); /** * @tc.steps: step2. call AckRecv, set inMsg nullptr or set context nullptr diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_anti_dos_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_anti_dos_sync_test.cpp index e3b5f9d86..473a91348 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_anti_dos_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_anti_dos_sync_test.cpp @@ -22,7 +22,7 @@ #include "ref_object.h" #include "single_ver_sync_engine.h" #include "single_ver_data_sync.h" -#include "vitural_communicator_aggregator.h" +#include "virtual_communicator_aggregator.h" #include "virtual_single_ver_sync_db_Interface.h" #include "version.h" #include "generic_single_ver_kv_entry.h" @@ -94,6 +94,7 @@ void DistributeddbAntiDosSyncTest::TearDownTestCase(void) void DistributeddbAntiDosSyncTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create VirtualCommunicator, VirtualSingleVerSyncDBInterface, SyncEngine, * and set maximum cache of queue. @@ -110,7 +111,7 @@ void DistributeddbAntiDosSyncTest::SetUp(void) ASSERT_TRUE(errCodeMetaData == E_OK); g_syncEngine = new (std::nothrow) SingleVerSyncEngine(); ASSERT_TRUE(g_syncEngine != nullptr); - int errCodeSyncEngine = g_syncEngine->Initialize(g_syncInterface, g_metaData, nullptr); + int errCodeSyncEngine = g_syncEngine->Initialize(g_syncInterface, g_metaData, nullptr, nullptr); ASSERT_TRUE(errCodeSyncEngine == E_OK); g_communicator = static_cast(g_communicatorAggregator->GetCommunicator(remoteDeviceId)); ASSERT_TRUE(g_communicator != nullptr); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_multi_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_multi_ver_p2p_sync_test.cpp index 920bd7fee..9cd129b0a 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_multi_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_multi_ver_p2p_sync_test.cpp @@ -20,9 +20,9 @@ #include "distributeddb_data_generate_unit_test.h" #include "kv_store_observer.h" #include "kv_store_delegate.h" -#include "vitural_communicator_aggregator.h" -#include "vitural_communicator.h" -#include "vitural_device.h" +#include "virtual_communicator_aggregator.h" +#include "virtual_communicator.h" +#include "virtual_device.h" #include "isyncer.h" #include "virtual_multi_ver_sync_db_interface.h" #include "time_sync.h" @@ -45,9 +45,9 @@ using namespace std; #ifndef LOW_LEVEL_MEM_DEV namespace { string g_testDir; - const string STORE_ID = "kv_stroe_sync_test"; - const string STORE_ID_A = "kv_stroe_sync_test_a"; - const string STORE_ID_B = "kv_stroe_sync_test_b"; + const string STORE_ID = "kv_store_sync_test"; + const string STORE_ID_A = "kv_store_sync_test_a"; + const string STORE_ID_B = "kv_store_sync_test_b"; const int WAIT_TIME_1 = 1000; const int WAIT_TIME_2 = 2000; const int WAIT_LONG_TIME = 10000; @@ -67,8 +67,8 @@ namespace { MultiVerNaturalStoreConnection *g_connectionA; MultiVerNaturalStoreConnection *g_connectionB; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; - VituralDevice* g_deviceB = nullptr; - VituralDevice* g_deviceC = nullptr; + VirtualDevice* g_deviceB = nullptr; + VirtualDevice* g_deviceC = nullptr; // the type of g_kvDelegateCallback is function auto g_kvDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreDelegateCallback, @@ -153,18 +153,19 @@ void DistributedDBMultiVerP2PSyncTest::TearDownTestCase(void) void DistributedDBMultiVerP2PSyncTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create virtual device B and C */ g_communicatorAggregator->Disable(); - g_deviceB = new (std::nothrow) VituralDevice(DEVICE_B); + g_deviceB = new (std::nothrow) VirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); VirtualMultiVerSyncDBInterface *syncInterfaceB = new (std::nothrow) VirtualMultiVerSyncDBInterface; ASSERT_TRUE(syncInterfaceB != nullptr); ASSERT_EQ(syncInterfaceB->Initialize(DEVICE_B), E_OK); ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); - g_deviceC = new (std::nothrow) VituralDevice(DEVICE_C); + g_deviceC = new (std::nothrow) VirtualDevice(DEVICE_C); ASSERT_TRUE(g_deviceC != nullptr); VirtualMultiVerSyncDBInterface *syncInterfaceC = new (std::nothrow) VirtualMultiVerSyncDBInterface; ASSERT_TRUE(syncInterfaceC != nullptr); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_check_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_check_test.cpp index 9b160250e..08df8a439 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_check_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_check_test.cpp @@ -14,12 +14,11 @@ */ #include +#include #include "distributeddb_tools_unit_test.h" #include "distributeddb_data_generate_unit_test.h" -#include "vitural_communicator_aggregator.h" -#include "vitural_device.h" -#include "virtual_single_ver_sync_db_Interface.h" +#include "virtual_communicator_aggregator.h" #include "platform_specific.h" #include "process_system_api_adapter_impl.h" @@ -40,8 +39,8 @@ namespace { DBStatus g_kvDelegateStatus = INVALID_ARGS; KvStoreNbDelegate* g_kvDelegatePtr = nullptr; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; - VituralDevice* g_deviceB = nullptr; - VituralDevice* g_deviceC = nullptr; + VirtualDevice* g_deviceB = nullptr; + VirtualDevice* g_deviceC = nullptr; VirtualSingleVerSyncDBInterface *g_syncInterfaceB = nullptr; VirtualSingleVerSyncDBInterface *g_syncInterfaceC = nullptr; @@ -51,7 +50,7 @@ namespace { #ifndef LOW_LEVEL_MEM_DEV const int KEY_LEN = 20; // 20 Bytes const int VALUE_LEN = 4 * 1024 * 1024; // 4MB - const int ENTRY_NUM = 6; // 6 entries + const int ENTRY_NUM = 2; // 16 entries #endif } @@ -103,6 +102,7 @@ void DistributedDBSingleVerP2PSyncCheckTest::TearDownTestCase(void) void DistributedDBSingleVerP2PSyncCheckTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create virtual device B and C, and get a KvStoreNbDelegate as deviceA */ @@ -112,13 +112,13 @@ void DistributedDBSingleVerP2PSyncCheckTest::SetUp(void) g_mgr.GetKvStore(STORE_ID, option, g_kvDelegateCallback); ASSERT_TRUE(g_kvDelegateStatus == OK); ASSERT_TRUE(g_kvDelegatePtr != nullptr); - g_deviceB = new (std::nothrow) VituralDevice(DEVICE_B); + g_deviceB = new (std::nothrow) VirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); g_syncInterfaceB = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(g_syncInterfaceB != nullptr); ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, g_syncInterfaceB), E_OK); - g_deviceC = new (std::nothrow) VituralDevice(DEVICE_C); + g_deviceC = new (std::nothrow) VirtualDevice(DEVICE_C); ASSERT_TRUE(g_deviceC != nullptr); g_syncInterfaceC = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(g_syncInterfaceC != nullptr); @@ -145,6 +145,9 @@ void DistributedDBSingleVerP2PSyncCheckTest::TearDown(void) delete g_deviceC; g_deviceC = nullptr; } + if (g_communicatorAggregator != nullptr) { + g_communicatorAggregator->RegOnDispatch(nullptr); + } } /** @@ -264,7 +267,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, BigDataSync001, TestSize.Level1 devices.push_back(g_deviceC->GetDeviceId()); /** - * @tc.steps: step1. deviceA put 6 bigData + * @tc.steps: step1. deviceA put 2 bigData */ std::vector entries; std::vector keys; @@ -375,11 +378,11 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, BigDataSync003, TestSize.Level1 DistributedDBUnitTest::GenerateRecords(ENTRY_NUM, entries, keys, KEY_LEN, VALUE_LEN); for (uint32_t i = 0; i < entries.size(); i++) { - if (i % 3 == 0) { // 0 3 for deivec B + if (i % 3 == 0) { // 0 3 6 9 12 15 for deivec B g_deviceB->PutData(entries[i].key, entries[i].value, 0, 0); - } else if (i % 3 == 1) { // 1 4 for device C + } else if (i % 3 == 1) { // 1 4 7 10 13 16 for device C g_deviceC->PutData(entries[i].key, entries[i].value, 0, 0); - } else { // 2 5 for device A + } else { // 2 5 8 11 14 for device A status = g_kvDelegatePtr->Put(entries[i].key, entries[i].value); ASSERT_TRUE(status == OK); } @@ -447,7 +450,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, PushFinishedNotify001, TestSize /** * @tc.steps: step2. deviceB put k2, v2, and deviceA pull from deviceB - * @tc.expected: step2. deviceA can not revice push finished notify + * @tc.expected: step2. deviceA can not receive push finished notify */ EXPECT_EQ(g_kvDelegatePtr->Put(KEY_2, VALUE_2), OK); std::map result; @@ -458,7 +461,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, PushFinishedNotify001, TestSize /** * @tc.steps: step3. deviceB put k3, v3, and deviceA push and pull to deviceB - * @tc.expected: step3. deviceA can not revice push finished notify + * @tc.expected: step3. deviceA can not receive push finished notify */ EXPECT_EQ(g_kvDelegatePtr->Put(KEY_3, VALUE_3), OK); status = g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_PULL, result); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp index 23706825d..aeeafd4d3 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp @@ -21,9 +21,9 @@ #include "distributeddb_data_generate_unit_test.h" #include "kv_store_observer.h" #include "kv_store_nb_delegate.h" -#include "vitural_communicator_aggregator.h" -#include "vitural_communicator.h" -#include "vitural_device.h" +#include "virtual_communicator_aggregator.h" +#include "virtual_communicator.h" +#include "virtual_device.h" #include "isyncer.h" #include "virtual_single_ver_sync_db_Interface.h" #include "time_sync.h" @@ -52,8 +52,8 @@ namespace { DBStatus g_kvDelegateStatus = INVALID_ARGS; KvStoreNbDelegate* g_kvDelegatePtr = nullptr; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; - VituralDevice* g_deviceB = nullptr; - VituralDevice* g_deviceC = nullptr; + VirtualDevice* g_deviceB = nullptr; + VirtualDevice* g_deviceC = nullptr; // the type of g_kvDelegateCallback is function auto g_kvDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, @@ -103,6 +103,7 @@ void DistributedDBSingleVerP2PSyncTest::TearDownTestCase(void) void DistributedDBSingleVerP2PSyncTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create virtual device B and C, and get a KvStoreNbDelegate as deviceA */ @@ -110,13 +111,13 @@ void DistributedDBSingleVerP2PSyncTest::SetUp(void) g_mgr.GetKvStore(STORE_ID, option, g_kvDelegateCallback); ASSERT_TRUE(g_kvDelegateStatus == OK); ASSERT_TRUE(g_kvDelegatePtr != nullptr); - g_deviceB = new (std::nothrow) VituralDevice(DEVICE_B); + g_deviceB = new (std::nothrow) VirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); VirtualSingleVerSyncDBInterface *syncInterfaceB = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(syncInterfaceB != nullptr); ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); - g_deviceC = new (std::nothrow) VituralDevice(DEVICE_C); + g_deviceC = new (std::nothrow) VirtualDevice(DEVICE_C); ASSERT_TRUE(g_deviceC != nullptr); VirtualSingleVerSyncDBInterface *syncInterfaceC = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(syncInterfaceC != nullptr); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_syncer_device_manager_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_syncer_device_manager_test.cpp index 12cadb291..61a6f0179 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_syncer_device_manager_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_syncer_device_manager_test.cpp @@ -16,14 +16,15 @@ #include #include -#include "vitural_communicator_aggregator.h" -#include "vitural_communicator.h" -#include "vitural_device.h" +#include "virtual_communicator_aggregator.h" +#include "virtual_communicator.h" +#include "virtual_device.h" #include "device_manager.h" #include "virtual_single_ver_sync_db_Interface.h" #include "log_print.h" #include "parcel.h" #include "sync_types.h" +#include "distributeddb_tools_unit_test.h" using namespace testing::ext; using namespace DistributedDB; @@ -34,8 +35,8 @@ namespace { const std::string DEVICE_C = "deviceC"; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; VirtualCommunicator* g_virtualCommunicator = nullptr; - VituralDevice* g_deviceB = nullptr; - VituralDevice* g_deviceC = nullptr; + VirtualDevice* g_deviceB = nullptr; + VirtualDevice* g_deviceC = nullptr; DeviceManager *g_deviceManager = nullptr; const int WAIT_TIME = 1000; } @@ -57,7 +58,7 @@ void DistributedDBSyncerDeviceManagerTest::SetUpTestCase(void) ASSERT_TRUE(g_communicatorAggregator != nullptr); RuntimeContext::GetInstance()->SetCommunicatorAggregator(g_communicatorAggregator); int errCode; - g_virtualCommunicator = static_cast(g_communicatorAggregator->AllocCommunicator(0, errCode)); + g_virtualCommunicator = static_cast(g_communicatorAggregator->AllocCommunicator(0, errCode)); ASSERT_TRUE(g_virtualCommunicator != nullptr); } @@ -72,6 +73,7 @@ void DistributedDBSyncerDeviceManagerTest::TearDownTestCase(void) void DistributedDBSyncerDeviceManagerTest::SetUp(void) { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: Init a DeviceManager and DeviceB, C */ @@ -82,13 +84,13 @@ void DistributedDBSyncerDeviceManagerTest::SetUp(void) std::bind(&DeviceManager::OnDeviceConnectCallback, g_deviceManager, std::placeholders::_1, std::placeholders::_2), nullptr); - g_deviceB = new (std::nothrow) VituralDevice(DEVICE_B); + g_deviceB = new (std::nothrow) VirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); VirtualSingleVerSyncDBInterface *syncInterfaceB = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(syncInterfaceB != nullptr); ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); - g_deviceC = new (std::nothrow) VituralDevice(DEVICE_C); + g_deviceC = new (std::nothrow) VirtualDevice(DEVICE_C); ASSERT_TRUE(g_deviceC != nullptr); VirtualSingleVerSyncDBInterface *syncInterfaceC = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(syncInterfaceC != nullptr); @@ -216,7 +218,7 @@ HWTEST_F(DistributedDBSyncerDeviceManagerTest, GetDevices001, TestSize.Level0) HWTEST_F(DistributedDBSyncerDeviceManagerTest, SendBroadCast001, TestSize.Level1) { bool deviceBReviced = false; -bool deviceCReviced = false; + bool deviceCReviced = false; /** * @tc.steps: step1. deviceB, C set OnRemoteDataChanged callback diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp index 434f89c50..34a481354 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp @@ -65,6 +65,7 @@ void DistributedDBTimeSyncTest::TearDownTestCase(void) void DistributedDBTimeSyncTest::SetUp(void) { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create the instance for virtual communicator, virtual storage component and time syncer */ @@ -170,7 +171,7 @@ HWTEST_F(DistributedDBTimeSyncTest, NormalSync001, TestSize.Level0) * @tc.expected: step4. (offsetB - offsetA ) - timeOffset < 100ms. */ TimeOffset timeOffset = 0; - g_timeSyncA->GetTimeOffset(timeOffset); + g_timeSyncA->GetTimeOffset(timeOffset, TIME_SYNC_WAIT_TIME); offsetB = g_metadataB->GetLocalTimeOffset(); offsetA = g_metadataA->GetLocalTimeOffset(); EXPECT_TRUE(abs(offsetB - offsetA - timeOffset) < NETWORK_DELAY); @@ -211,7 +212,7 @@ HWTEST_F(DistributedDBTimeSyncTest, NormalSync002, TestSize.Level0) * @tc.expected: step3. (offsetB - offsetA ) - timeOffset < 100ms. */ TimeOffset timeOffset; - g_timeSyncA->GetTimeOffset(timeOffset); + g_timeSyncA->GetTimeOffset(timeOffset, TIME_SYNC_WAIT_TIME); TimeOffset offsetB = g_metadataB->GetLocalTimeOffset(); TimeOffset offsetA = g_metadataA->GetLocalTimeOffset(); EXPECT_TRUE(abs(offsetB - offsetA - timeOffset) < NETWORK_DELAY); @@ -263,7 +264,7 @@ HWTEST_F(DistributedDBTimeSyncTest, NormalSync003, TestSize.Level0) * @tc.expected: step4. (offsetB - offsetA ) - timeOffset < 100ms. */ TimeOffset timeOffset = 0; - g_timeSyncA->GetTimeOffset(timeOffset); + g_timeSyncA->GetTimeOffset(timeOffset, TIME_SYNC_WAIT_TIME); TimeOffset absTimeOffset = abs(timeOffset); EXPECT_TRUE(abs(offsetB - offsetA - absTimeOffset) < NETWORK_DELAY); @@ -475,6 +476,6 @@ HWTEST_F(DistributedDBTimeSyncTest, SyncTimeout001, TestSize.Level2) * @tc.expected: step2. Start the time sync task return E_TIMEOUT */ TimeOffset offset; - errCode = g_timeSyncA->GetTimeOffset(offset); + errCode = g_timeSyncA->GetTimeOffset(offset, TIME_SYNC_WAIT_TIME); EXPECT_TRUE(errCode == -E_TIMEOUT); } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp similarity index 91% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.cpp rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp index 476816ea0..f8921e5ee 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp @@ -13,9 +13,9 @@ * limitations under the License. */ -#include "vitural_communicator.h" +#include "virtual_communicator.h" -#include "vitural_communicator_aggregator.h" +#include "virtual_communicator_aggregator.h" #include "sync_engine.h" #include "log_print.h" @@ -89,14 +89,14 @@ void VirtualCommunicator::CallbackOnMessage(const std::string &srcTarget, Messag void VirtualCommunicator::CallbackOnConnect(const std::string &target, bool isConnect) const { std::lock_guard lock(onConnectLock_); - if (isEnable_ && onConnect_ && (target != deviceId_)) { + if (isEnable_ && onConnect_) { onConnect_(target, isConnect); } } uint32_t VirtualCommunicator::GetCommunicatorMtuSize() const { - return 5 * 1024 * 1024; // 5M + return 5 * 1024 * 1024; // 5 * 1024 * 1024B } uint32_t VirtualCommunicator::GetCommunicatorMtuSize(const std::string &target) const @@ -104,6 +104,16 @@ uint32_t VirtualCommunicator::GetCommunicatorMtuSize(const std::string &target) return GetCommunicatorMtuSize(); } +uint32_t VirtualCommunicator::GetTimeout() const +{ + return 5 * 1000; // 5 * 1000ms +} + +uint32_t VirtualCommunicator::GetTimeout(const std::string &target) const +{ + return GetTimeout(); +} + int VirtualCommunicator::GetLocalIdentity(std::string &outTarget) const { outTarget = deviceId_; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h similarity index 94% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.h rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h index e19254870..59186c0f4 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h @@ -26,12 +26,12 @@ #include "ref_object.h" #include "serial_buffer.h" #include "icommunicator.h" -#include "vitural_device.h" +#include "virtual_device.h" namespace DistributedDB { class VirtualCommunicatorAggregator; -class VituralDevice; +class VirtualDevice; class VirtualCommunicator : public ICommunicator { public: @@ -48,6 +48,9 @@ public: uint32_t GetCommunicatorMtuSize() const override; uint32_t GetCommunicatorMtuSize(const std::string &target) const override; + + uint32_t GetTimeout() const override; + uint32_t GetTimeout(const std::string &target) const override; int GetLocalIdentity(std::string &outTarget) const override; int SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout) override; @@ -58,7 +61,7 @@ public: void CallbackOnMessage(const std::string &srcTarget, Message *inMsg) const; - void CallbackOnConnect(const std::string &target, bool isConnec) const; + void CallbackOnConnect(const std::string &target, bool isConnect) const; int GeneralVirtualSyncId(); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp similarity index 93% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.cpp rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp index 5884b95b7..61f334004 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp @@ -12,13 +12,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "vitural_communicator.h" +#include "virtual_communicator.h" #include #include +#include #include "db_errno.h" -#include "vitural_communicator_aggregator.h" +#include "virtual_communicator_aggregator.h" #include "log_print.h" namespace DistributedDB { @@ -173,7 +174,11 @@ void VirtualCommunicatorAggregator::DispatchMessage(const std::string &srcTarget Message *msg = const_cast(inMsg); msg->SetTarget(srcTarget); RefObject::IncObjRef(communicator); - std::thread thread([communicator, srcTarget, msg]() { + auto onDispatch = onDispatch_; + std::thread thread([communicator, srcTarget, dstTarget, msg, onDispatch]() { + if (onDispatch) { + onDispatch(dstTarget, msg); + } communicator->CallbackOnMessage(srcTarget, msg); RefObject::DecObjRef(communicator); }); @@ -220,5 +225,11 @@ void VirtualCommunicatorAggregator::CallSendEnd(int errCode, const OnSendEnd &on }); } } + +void VirtualCommunicatorAggregator::RegOnDispatch( + const std::function &onDispatch) +{ + onDispatch_ = onDispatch; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h similarity index 91% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.h rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h index 3d87d1747..29cfbc4dd 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h @@ -19,13 +19,13 @@ #include #include "icommunicator_aggregator.h" -#include "vitural_device.h" -#include "vitural_communicator.h" +#include "virtual_device.h" +#include "virtual_communicator.h" namespace DistributedDB { class ICommunicator; // Forward Declaration -class VituralDevice; +class VirtualDevice; class VirtualCommunicatorAggregator : public ICommunicatorAggregator { public: @@ -67,6 +67,8 @@ public: bool GetBlockValue() const; + void RegOnDispatch(const std::function &onDispatch); + ~VirtualCommunicatorAggregator() {}; VirtualCommunicatorAggregator() {}; @@ -82,6 +84,7 @@ private: bool isBlock_ = false; CommunicatorLackCallback onCommLack_; OnConnectCallback onConnect_; + std::function onDispatch_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp similarity index 79% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.cpp rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp index 3029a69a3..a0084f294 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp @@ -12,30 +12,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "vitural_device.h" +#include "virtual_device.h" #include #include #include "virtual_single_ver_sync_db_Interface.h" #include "virtual_multi_ver_sync_db_interface.h" -#include "vitural_communicator.h" -#include "vitural_communicator_aggregator.h" +#include "virtual_communicator.h" +#include "virtual_communicator_aggregator.h" #include "single_ver_sync_state_machine.h" #include "multi_ver_sync_state_machine.h" #include "device_manager.h" #include "log_print.h" namespace DistributedDB { -VituralDevice::VituralDevice(const std::string &deviceId) +VirtualDevice::VirtualDevice(const std::string &deviceId) : communicateHandle_(nullptr), + communicatorAggregator_(nullptr), storage_(nullptr), metadata_(nullptr), - deviceId_(deviceId) + deviceId_(deviceId), + remoteDeviceId_("real_device"), + context_(nullptr) { } -VituralDevice::~VituralDevice() +VirtualDevice::~VirtualDevice() { std::mutex cvMutex; std::condition_variable cv; @@ -72,7 +75,7 @@ VituralDevice::~VituralDevice() storage_ = nullptr; } -int VituralDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface) +int VirtualDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface) { if ((communicatorAggregator == nullptr) || (syncInterface == nullptr)) { return -E_INVALID_ARGS; @@ -100,68 +103,68 @@ int VituralDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggrega if (context_ == nullptr) { return -E_OUT_OF_MEMORY; } - communicateHandle_->RegOnMessageCallback(std::bind(&VituralDevice::MessageCallback, this, + communicateHandle_->RegOnMessageCallback(std::bind(&VirtualDevice::MessageCallback, this, std::placeholders::_1, std::placeholders::_2), []() {}); context_->Initialize(remoteDeviceId_, storage_, metadata_, communicateHandle_); context_->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); - context_->RegOnSyncTask(std::bind(&VituralDevice::StartResponseTask, this)); + context_->RegOnSyncTask(std::bind(&VirtualDevice::StartResponseTask, this)); return E_OK; } -void VituralDevice::SetDeviceId(const std::string &deviceId) +void VirtualDevice::SetDeviceId(const std::string &deviceId) { deviceId_ = deviceId; } -std::string VituralDevice::GetDeviceId() const +std::string VirtualDevice::GetDeviceId() const { return deviceId_; } -int VituralDevice::GetData(const Key &key, VirtualDataItem &item) +int VirtualDevice::GetData(const Key &key, VirtualDataItem &item) { VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); return syncAble->GetSyncData(key, item); } -int VituralDevice::GetData(const Key &key, Value &value) +int VirtualDevice::GetData(const Key &key, Value &value) { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->GetData(key, value); } -int VituralDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) +int VirtualDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) { VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); LOGI("dev %s put data time %llu", deviceId_.c_str(), time); return syncAble->PutData(key, value, time, flag); } -int VituralDevice::PutData(const Key &key, const Value &value) +int VirtualDevice::PutData(const Key &key, const Value &value) { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->PutData(key, value); } -int VituralDevice::DeleteData(const Key &key) +int VirtualDevice::DeleteData(const Key &key) { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->DeleteData(key); } -int VituralDevice::StartTransaction() +int VirtualDevice::StartTransaction() { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->StartTransaction(); } -int VituralDevice::Commit() +int VirtualDevice::Commit() { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->Commit(); } -int VituralDevice::MessageCallback(const std::string &deviceId, Message *inMsg) +int VirtualDevice::MessageCallback(const std::string &deviceId, Message *inMsg) { if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { if (onRemoteDataChanged_) { @@ -175,7 +178,7 @@ int VituralDevice::MessageCallback(const std::string &deviceId, Message *inMsg) return -E_INVALID_ARGS; } - LOGD("[VituralDevice] onMessag, src %s", deviceId.c_str()); + LOGD("[VirtualDevice] onMessag, src %s", deviceId.c_str()); RefObject::IncObjRef(context_); RefObject::IncObjRef(communicateHandle_); SyncTaskContext *context = context_; @@ -192,61 +195,61 @@ int VituralDevice::MessageCallback(const std::string &deviceId, Message *inMsg) return E_OK; } -void VituralDevice::OnRemoteDataChanged(const std::function &callback) +void VirtualDevice::OnRemoteDataChanged(const std::function &callback) { onRemoteDataChanged_ = callback; } -void VituralDevice::Online() +void VirtualDevice::Online() { static_cast(communicateHandle_)->Enable(); communicatorAggregator_->OnlineDevice(deviceId_); } -void VituralDevice::Offline() +void VirtualDevice::Offline() { static_cast(communicateHandle_)->Disable(); communicatorAggregator_->OfflineDevice(deviceId_); } -int VituralDevice::StartResponseTask() +int VirtualDevice::StartResponseTask() { - LOGD("[VituralDevice] StartResponseTask"); + LOGD("[VirtualDevice] StartResponseTask"); RefObject::AutoLock lockGuard(context_); int status = context_->GetTaskExecStatus(); if ((status == SyncTaskContext::RUNNING) || context_->IsKilled()) { - LOGD("[VituralDevice] StartResponseTask status:%d", status); + LOGD("[VirtualDevice] StartResponseTask status:%d", status); return -E_NOT_SUPPORT; } if (context_->IsTargetQueueEmpty()) { - LOGD("[VituralDevice] StartResponseTask IsTargetQueueEmpty is empty"); + LOGD("[VirtualDevice] StartResponseTask IsTargetQueueEmpty is empty"); return E_OK; } context_->SetTaskExecStatus(ISyncTaskContext::RUNNING); context_->MoveToNextTarget(); - LOGI("[VituralDevice] machine StartSync"); + LOGI("[VirtualDevice] machine StartSync"); context_->UnlockObj(); int errCode = context_->StartStateMachine(); context_->LockObj(); if (errCode != E_OK) { - LOGE("[VituralDevice] machine StartSync failed"); + LOGE("[VirtualDevice] machine StartSync failed"); context_->SetOperationStatus(SyncOperation::FAILED); } return errCode; } -TimeOffset VituralDevice::GetLocalTimeOffset() const +TimeOffset VirtualDevice::GetLocalTimeOffset() const { return metadata_->GetLocalTimeOffset(); } -void VituralDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) +void VirtualDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) { VirtualSingleVerSyncDBInterface *syncInterface = static_cast(storage_); syncInterface->SetSaveDataDelayTime(milliDelayTime); } -int VituralDevice::Sync(SyncMode mode, bool wait) +int VirtualDevice::Sync(SyncMode mode, bool wait) { auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); if (operation == nullptr) { @@ -255,10 +258,10 @@ int VituralDevice::Sync(SyncMode mode, bool wait) operation->Initialize(); operation->SetOnSyncFinished([operation](int id) { operation->NotifyIfNeed(); - RefObject::KillAndDecObjRef(operation); }); context_->AddSyncOperation(operation); operation->WaitIfNeed(); + RefObject::KillAndDecObjRef(operation); return E_OK; } } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h similarity index 88% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.h rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h index e520a0e74..9ca1b7806 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h @@ -25,10 +25,10 @@ namespace DistributedDB { class VirtualCommunicatorAggregator; -class VituralDevice { +class VirtualDevice { public: - explicit VituralDevice(const std::string &deviceId); - ~VituralDevice(); + explicit VirtualDevice(const std::string &deviceId); + ~VirtualDevice(); int Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface); void SetDeviceId(const std::string &deviceId); @@ -51,12 +51,12 @@ public: private: ICommunicator *communicateHandle_; - VirtualCommunicatorAggregator *communicatorAggregator_ = nullptr; + VirtualCommunicatorAggregator *communicatorAggregator_; IKvDBSyncInterface *storage_; std::shared_ptr metadata_; std::string deviceId_; - std::string remoteDeviceId_ = "real_device"; - SyncTaskContext *context_ = nullptr; + std::string remoteDeviceId_; + SyncTaskContext *context_; std::function onRemoteDataChanged_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp index d9ee109f9..fff954696 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp @@ -64,6 +64,11 @@ int VirtualMultiVerSyncDBInterface::PutMetaData(const Key &key, const Value &val return kvStore_->PutMetaData(key, value); } +int VirtualMultiVerSyncDBInterface::DeleteMetaData(const std::vector &keys) +{ + return kvStore_->DeleteMetaData(keys); +} + int VirtualMultiVerSyncDBInterface::GetAllMetaKeys(std::vector &keys) const { return kvStore_->GetAllMetaKeys(keys); @@ -224,5 +229,10 @@ const KvDBProperties &VirtualMultiVerSyncDBInterface::GetDbProperties() const { return properties_; } + +int VirtualMultiVerSyncDBInterface::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + return -E_NOT_SUPPORT; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.h index 17695c480..429376366 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.h @@ -41,6 +41,12 @@ public: int PutMetaData(const Key &key, const Value &value) override; + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys) override; + + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + int GetAllMetaKeys(std::vector &keys) const override; bool IsCommitExisted(const MultiVerCommitNode &) const override; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp index fd500bb87..d42a90ac4 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp @@ -24,8 +24,39 @@ #include "meta_data.h" #include "securec.h" #include "generic_single_ver_kv_entry.h" +#include "query_object.h" namespace DistributedDB { +namespace { + int GetEntriesFromItems(std::vector &entries, const std::vector &dataItems) + { + int errCode = E_OK; + for (auto &item : dataItems) { + auto entry = new (std::nothrow) GenericSingleVerKvEntry(); + if (entry == nullptr) { + LOGE("Create entry failed."); + errCode = -E_OUT_OF_MEMORY; + break; + } + DataItem storageItem; + storageItem.key = item.key; + storageItem.value = item.value; + storageItem.flag = item.flag; + storageItem.timeStamp = item.timeStamp; + storageItem.writeTimeStamp = item.writeTimeStamp; + entry->SetEntryData(std::move(storageItem)); + entries.push_back(entry); + } + if (errCode != E_OK) { + for (auto &kvEntry : entries) { + delete kvEntry; + kvEntry = nullptr; + } + entries.clear(); + } + return errCode; + } +} int VirtualSingleVerSyncDBInterface::GetInterfaceType() const { return SYNC_SVD; @@ -61,6 +92,14 @@ int VirtualSingleVerSyncDBInterface::PutMetaData(const Key &key, const Value &va return E_OK; } +int VirtualSingleVerSyncDBInterface::DeleteMetaData(const std::vector &keys) +{ + for (const auto &key : keys) { + (void)metadata_.erase(key); + } + return E_OK; +} + int VirtualSingleVerSyncDBInterface::GetAllMetaKeys(std::vector &keys) const { for (auto iter = metadata_.begin(); iter != metadata_.end(); ++iter) { @@ -140,17 +179,17 @@ int VirtualSingleVerSyncDBInterface::RemoveDeviceData(const std::string &deviceN return E_OK; } -int VirtualSingleVerSyncDBInterface::GetSyncData(const Key &key, VirtualDataItem &item) +int VirtualSingleVerSyncDBInterface::GetSyncData(const Key &key, VirtualDataItem &dataItem) { auto iter = std::find_if(dbData_.begin(), dbData_.end(), - [key](VirtualDataItem item) { return item.key == key; }); + [key](const VirtualDataItem& item) { return item.key == key; }); if (iter != dbData_.end()) { - item.key = iter->key; - item.value = iter->value; - item.timeStamp = iter->timeStamp; - item.writeTimeStamp = iter->writeTimeStamp; - item.flag = iter->flag; - item.isLocal = iter->isLocal; + dataItem.key = iter->key; + dataItem.value = iter->value; + dataItem.timeStamp = iter->timeStamp; + dataItem.writeTimeStamp = iter->writeTimeStamp; + dataItem.flag = iter->flag; + dataItem.isLocal = iter->isLocal; return E_OK; } return -E_NOT_FOUND; @@ -166,30 +205,7 @@ int VirtualSingleVerSyncDBInterface::GetSyncData(TimeStamp begin, TimeStamp end, LOGE("[VirtualSingleVerSyncDBInterface][GetSyncData] GetSyncData failed err %d", errCode); return errCode; } - for (auto item : dataItems) { - GenericSingleVerKvEntry *entry = new (std::nothrow) GenericSingleVerKvEntry(); - if (entry == nullptr) { - LOGE("Create entry failed."); - errCode = -E_OUT_OF_MEMORY; - break; - } - DataItem storageItem; - storageItem.key = item.key; - storageItem.value = item.value; - storageItem.flag = item.flag; - storageItem.timeStamp = item.timeStamp; - storageItem.writeTimeStamp = item.writeTimeStamp; - entry->SetEntryData(std::move(storageItem)); - entries.push_back(entry); - } - if (errCode != E_OK) { - for (auto kvEntry : entries) { - delete kvEntry; - kvEntry = nullptr; - } - entries.clear(); - } - return errCode; + return GetEntriesFromItems(entries, dataItems); } int VirtualSingleVerSyncDBInterface::GetSyncDataNext(std::vector &entries, @@ -312,4 +328,69 @@ void VirtualSingleVerSyncDBInterface::SetSecurityOption(SecurityOption &option) void VirtualSingleVerSyncDBInterface::NotifyRemotePushFinished(const std::string &targetId) const { } + +int VirtualSingleVerSyncDBInterface::GetDatabaseCreateTimeStamp(TimeStamp &outTime) const +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const +{ + const auto &startKey = query.GetPrefixKey(); + Key endKey = startKey; + endKey.resize(DBConstant::MAX_KEY_SIZE, UCHAR_MAX); + + std::vector dataItems; + for (const auto &data : dbData_) { + // Only get local data. + if (!data.isLocal) { + continue; + } + + if ((data.flag & VirtualDataItem::DELETE_FLAG) != 0) { + if (data.timeStamp >= timeRange.deleteBeginTime && data.timeStamp < timeRange.deleteEndTime) { + dataItems.push_back(data); + } + } else { + if (data.timeStamp >= timeRange.beginTime && data.timeStamp < timeRange.endTime && + data.key >= startKey && data.key <= endKey) { + dataItems.push_back(data); + } + } + } + + LOGD("dataItems size %d", dataItems.size()); + return GetEntriesFromItems(entries, dataItems); +} + +int VirtualSingleVerSyncDBInterface::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + size_t prefixKeySize = keyPrefix.size(); + for (auto iter = metadata_.begin(); iter != metadata_.end();) { + if (prefixKeySize <= iter->first.size() && + keyPrefix == Key(iter->first.begin(), std::next(iter->first.begin(), prefixKeySize))) { + iter = metadata_.erase(iter); + } else { + ++iter; + } + } + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::GetCompressionAlgo(std::set &algorithmSet) const +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::PutSyncData(const DataItem &item) +{ + return E_OK; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.h index 20b5215ec..a45cdc25c 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.h @@ -20,6 +20,7 @@ #include #include "single_ver_kvdb_sync_interface.h" +#include "query_object.h" #include "types.h" namespace DistributedDB { @@ -47,6 +48,10 @@ public: int PutMetaData(const Key& key, const Value& value) override; + int DeleteMetaData(const std::vector &keys) override; + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + int GetAllMetaKeys(std::vector& keys) const override; int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &dataItems, @@ -65,7 +70,7 @@ public: int RemoveDeviceData(const std::string &deviceName, bool isNeedNotify) override; - int GetSyncData(const Key& key, VirtualDataItem& item); + int GetSyncData(const Key& key, VirtualDataItem& dataItem); int PutSyncData(const DataItem& item); @@ -74,6 +79,9 @@ public: int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; + int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, const DataSizeSpecInfo &dataSizeInfo, + ContinueToken &continueStmtToken, std::vector &entries) const override; + int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; @@ -97,6 +105,11 @@ public: void NotifyRemotePushFinished(const std::string &targetId) const override; + int GetDatabaseCreateTimeStamp(TimeStamp &outTime) const override; + + int GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const override; + int GetCompressionAlgo(std::set &algorithmSet) const override; + private: int GetSyncData(TimeStamp begin, TimeStamp end, uint32_t blockSize, std::vector& dataItems, ContinueToken& continueStmtToken) const; @@ -106,7 +119,7 @@ private: int PutSyncData(std::vector& dataItems, const std::string &deviceName); - std::map, std::vector> metadata_; + mutable std::map, std::vector> metadata_; std::vector dbData_; std::string schema_; SchemaObject schemaObj_; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp index 21d10f445..48a2a6381 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp @@ -61,6 +61,16 @@ uint32_t VirtualTimeSyncCommunicator::GetCommunicatorMtuSize(const std::string & return GetCommunicatorMtuSize(); } +uint32_t VirtualTimeSyncCommunicator::GetTimeout() const +{ + return 0; +} + +uint32_t VirtualTimeSyncCommunicator::GetTimeout(const std::string &target) const +{ + return 0; +} + // Get local target name for identify self int VirtualTimeSyncCommunicator::GetLocalIdentity(std::string &outTarget) const { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h index ccc6bd7c1..1aa9bbdbb 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h @@ -28,7 +28,7 @@ #include "serial_buffer.h" #include "icommunicator.h" #include "communicator_aggregator.h" -#include "vitural_device.h" +#include "virtual_device.h" #include "time_sync.h" namespace DistributedDB { @@ -48,6 +48,11 @@ public: // return maximum allowed data size uint32_t GetCommunicatorMtuSize() const override; uint32_t GetCommunicatorMtuSize(const std::string &target) const override; + + // return timeout + uint32_t GetTimeout() const override; + uint32_t GetTimeout(const std::string &target) const override; + // Get local target name for identify self int GetLocalIdentity(std::string &outTarget) const override; diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_schema_test_tools.h b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_schema_test_tools.h index d70191032..48fa5274f 100755 --- a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_schema_test_tools.h +++ b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_schema_test_tools.h @@ -15,7 +15,10 @@ #ifndef DISTRIBUTED_DB_SCHEMA_TEST_TOOLS_H #define DISTRIBUTED_DB_SCHEMA_TEST_TOOLS_H +#ifdef RUNNING_ON_SIMULATED_ENV #include +#endif +#include #include "kv_store_delegate.h" #include "kv_store_delegate_manager.h" #include "types.h" @@ -109,8 +112,8 @@ static std::vector INVALID_ATTRIBUTES = { ATTRIBUTES_PRE1 + std::to_string(INT32_MIN) + "1\"", ATTRIBUTES_PRE2 + std::to_string(INT64_MAX) + "1\"", ATTRIBUTES_PRE2 + std::to_string(INT64_MIN) + "1\"", - ATTRIBUTES_PRE3 + "1" + std::to_string(DBL_MAX) + "\"", - ATTRIBUTES_PRE3 + "-1" + std::to_string(DBL_MAX) + "\"" + ATTRIBUTES_PRE3 + "1" + std::to_string(std::numeric_limits::max()) + "\"", + ATTRIBUTES_PRE3 + "-1" + std::to_string(std::numeric_limits::max()) + "\"" }; struct SchemaDefine { std::vector field; diff --git a/services/distributeddataservice/test/common/distributeddb/src/distributeddb_nb_test_tools.cpp b/services/distributeddataservice/test/common/distributeddb/src/distributeddb_nb_test_tools.cpp index 29fb2b05d..d929793b9 100755 --- a/services/distributeddataservice/test/common/distributeddb/src/distributeddb_nb_test_tools.cpp +++ b/services/distributeddataservice/test/common/distributeddb/src/distributeddb_nb_test_tools.cpp @@ -473,33 +473,33 @@ bool DistributedDBNbTestTools::MoveToNextFromBegin(KvStoreResultSet &resultSet, for (int position = -1; position < recordCnt; ++position) { // the first pos after getentries is -1. bool expectRes = resultSet.MoveToNext(); if (position < (recordCnt - 1)) { - result = result && expectRes; + result = result & expectRes; } else { - result = result && (!expectRes); + result = result & (!expectRes); } if (!result) { MST_LOG("resultSet.MoveToNext() doesn't meet expectations!!!"); break; } positionGot = position + 1; - result = result && (resultSet.GetPosition() == positionGot); + result = result & (resultSet.GetPosition() == positionGot); if (!result) { MST_LOG("resultSet.GetPosition() != positionGot!!!"); break; } if (position < (recordCnt - 1)) { - result = result && (resultSet.GetEntry(entry) == OK); + result = result & (resultSet.GetEntry(entry) == OK); if (!result) { MST_LOG("resultSet.GetEntry() != OK"); break; } - result = result && (entry.key == entries[positionGot].key) && (entry.value == entries[positionGot].value); + result = result & (entry.key == entries[positionGot].key) & (entry.value == entries[positionGot].value); if (!result) { MST_LOG("entry != entries[positionGot]"); break; } } else { - result = result && (resultSet.GetEntry(entry) == NOT_FOUND); + result = result & (resultSet.GetEntry(entry) == NOT_FOUND); } } return result; @@ -553,4 +553,4 @@ void KvStoreNbCorruptInfo::CorruptCallBackOfExport(const std::string &appId, con MST_LOG("[KvStoreNbCorruptInfo::CorruptCallBackOfExport] Export failed!"); pathResult.result = false; } -} +} \ No newline at end of file diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_kv_create_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_kv_create_test.cpp index c546abef1..4c80ef9c3 100755 --- a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_kv_create_test.cpp +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_kv_create_test.cpp @@ -202,7 +202,7 @@ HWTEST_F(DistributeddbKvCreateTest, Open003, TestSize.Level0) option.createIfNecessary = IS_NOT_NEED_CREATE; option.localOnly = IS_LOCAL_ONLY; status = DistributedTestTools::GetDelegateNotGood(manager, delegate, STORE_ID_1, APP_ID_1, USER_ID_1, option); - ASSERT_NE(status, DBStatus::OK); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); EXPECT_EQ(delegate, nullptr); EXPECT_EQ(manager->CloseKvStore(delegate), INVALID_ARGS); @@ -315,7 +315,7 @@ HWTEST_F(DistributeddbKvCreateTest, Open005, TestSize.Level1) option4.createIfNecessary = IS_NOT_NEED_CREATE; option4.localOnly = IS_LOCAL_ONLY; status = DistributedTestTools::GetDelegateNotGood(manager4, delegate4, STORE_ID_1, APP_ID_1, USER_ID_1, option4); - ASSERT_NE(status, DBStatus::OK); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); EXPECT_TRUE(delegate4 == nullptr); EXPECT_TRUE(manager4->CloseKvStore(delegate4) == INVALID_ARGS); @@ -1150,7 +1150,7 @@ HWTEST_F(DistributeddbKvCreateTest, PathException007, TestSize.Level2) STORE_ID_1, APP_ID_1, USER_ID_2, option); EXPECT_TRUE(delegate2 == nullptr); EXPECT_EQ(manager2->CloseKvStore(delegate2), INVALID_ARGS); - ASSERT_NE(status, OK); + ASSERT_EQ(status, INVALID_ARGS); delete manager2; manager2 = nullptr; diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp index 4511c646e..3a7827209 100755 --- a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp @@ -924,7 +924,7 @@ HWTEST_F(DistributeddbNbCreateTest, ManagerDb028, TestSize.Level0) KvStoreNbDelegate::Option option = DistributedDBNbTestTools::TransferNbOptionType(g_option); option.createIfNecessary = IS_NOT_NEED_CREATE; manager->GetKvStore(STORE_ID_1, option, Nbfunction1); - EXPECT_NE(delegateMgrCallback.GetStatus(), OK); + EXPECT_EQ(delegateMgrCallback.GetStatus(), INVALID_ARGS); KvStoreNbDelegate *delegate = const_cast(delegateMgrCallback.GetKvStore()); EXPECT_EQ(delegate, nullptr); EXPECT_EQ(manager->CloseKvStore(delegate), INVALID_ARGS); diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_enable_sync_by_closed_db_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_enable_sync_by_closed_db_test.cpp index 7d8a6fedc..14d6e3155 100755 --- a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_enable_sync_by_closed_db_test.cpp +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_enable_sync_by_closed_db_test.cpp @@ -172,7 +172,7 @@ HWTEST_F(DistributeddbNbEnableSyncByClosedDbTest, ParamCheck002, TestSize.Level0 option.createIfNecessary = false; option.dataDir = NB_DIRECTOR; option.observer = observer; - EXPECT_EQ(manager->EnableKvStoreAutoLaunch(USER_ID_1, APP_ID_1, STORE_ID_1, option, nullptr), DB_ERROR); + EXPECT_NE(manager->EnableKvStoreAutoLaunch(USER_ID_1, APP_ID_1, STORE_ID_1, option, nullptr), OK); /** * @tc.steps: step2. call EnableKvStoreAutoLaunch use the option with which createIfNecessary = true, * and other settings don't change; diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_schema_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_schema_test.cpp index ab482f83b..344707762 100755 --- a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_schema_test.cpp +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_schema_test.cpp @@ -127,6 +127,8 @@ vector GetInvalidDefine(SchemaDefine &validDefine, SchemaDefine &invalid return defineResult; } +DistributedDB::CipherPassword g_passwd1; + class DistributeddbNbSchemaDbTest : public testing::Test { public: static void SetUpTestCase(void); @@ -138,6 +140,7 @@ private: void DistributeddbNbSchemaDbTest::SetUpTestCase(void) { + (void)g_passwd1.SetValue(PASSWD_VECTOR_1.data(), PASSWD_VECTOR_1.size()); } void DistributeddbNbSchemaDbTest::TearDownTestCase(void) @@ -154,11 +157,16 @@ void DistributeddbNbSchemaDbTest::SetUp(void) ASSERT_NE(testinfo, nullptr); string testCaseName = string(testinfo->name()); MST_LOG("[SetUp] test case %s is start to run", testCaseName.c_str()); + + g_nbSchemaDelegate = DistributedDBNbTestTools::GetNbDelegateSuccess(g_manager, g_dbParameter1, g_option); + ASSERT_TRUE(g_manager != nullptr && g_nbSchemaDelegate != nullptr); } void DistributeddbNbSchemaDbTest::TearDown(void) { MST_LOG("TearDownTestCase after case."); + ASSERT_NE(g_manager, nullptr); + EXPECT_TRUE(EndCaseDeleteDB(g_manager, g_nbSchemaDelegate, STORE_ID_1, g_option.isMemoryDb)); RemoveDir(NB_DIRECTOR); } -- Gitee