From eb8dbaa770a57557d67c817c2839c64f536a6ce4 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 13 Sep 2023 16:22:03 -0400 Subject: Began re-architecting the project for concurrency support The project is now in a state where it builds, but it probably has a lot of bugs still. --- tests/internal_level_tests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index 9deb485..b0dfacb 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -37,11 +37,11 @@ START_TEST(t_memlevel_merge) ck_assert_int_eq(merging_level->get_record_count(), 100); auto old_level = base_level; - base_level = ILevel::merge_levels(old_level, merging_level); + auto new_level = ILevel::merge_levels(old_level, merging_level); delete old_level; delete merging_level; - ck_assert_int_eq(base_level->get_record_count(), 200); + ck_assert_int_eq(new_level->get_record_count(), 200); delete base_level; delete tbl1; -- cgit v1.2.3 From 7f56949bc847b56da69c9eb3ebe081d6cf9f61c6 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 18 Sep 2023 12:25:01 -0400 Subject: General bugfixes --- tests/internal_level_tests.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index b0dfacb..58369ff 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -36,10 +36,8 @@ START_TEST(t_memlevel_merge) merging_level->append_buffer(tbl2); ck_assert_int_eq(merging_level->get_record_count(), 100); - auto old_level = base_level; - auto new_level = ILevel::merge_levels(old_level, merging_level); + auto new_level = ILevel::merge_levels(base_level, merging_level); - delete old_level; delete merging_level; ck_assert_int_eq(new_level->get_record_count(), 200); -- cgit v1.2.3 From 754372aeccb74815cbb16f32ceacb04b4c5aaba9 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 20 Sep 2023 14:03:23 -0400 Subject: Bugfixes for tiering Fixed a few issues that manifested during the tiering tests, 1) When a version is copied, it now contains copies of the levels, not just pointers (the levels themselves still hold pointers to the shards, though). 2) Ensure that tasks are scheduled with the correct timestamp, they were originally being scheduled backwards. The get_merge_tasks() method already returns them in the correct order, so reversing them again put it in the wrong order. --- tests/dynamic_extension_tests.inc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'tests') diff --git a/tests/dynamic_extension_tests.inc b/tests/dynamic_extension_tests.inc index b9866c3..bee28f7 100644 --- a/tests/dynamic_extension_tests.inc +++ b/tests/dynamic_extension_tests.inc @@ -45,6 +45,25 @@ START_TEST(t_insert) END_TEST +START_TEST(t_debug_insert) +{ + auto ext_wirs = new DE(100, 2, 1); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<1000; i++) { + WRec r = {key, val, 1}; + ck_assert_int_eq(ext_wirs->insert(r), 1); + ck_assert_int_eq(ext_wirs->get_record_count(), i+1); + key++; + val++; + } + + delete ext_wirs; +} +END_TEST + + START_TEST(t_insert_with_mem_merges) { auto ext_wirs = new DE(100, 2, 1); @@ -329,8 +348,12 @@ START_TEST(t_static_structure) } size_t deletes = 0; + size_t t_reccnt = 0; + size_t k=0; for (auto rec : records) { + k++; ck_assert_int_eq(ext_wirs->insert(rec), 1); + t_reccnt++; if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { std::vector del_vec; @@ -378,6 +401,7 @@ Suite *unit_testing() TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); tcase_add_test(insert, t_insert); tcase_add_test(insert, t_insert_with_mem_merges); + tcase_add_test(insert, t_debug_insert); suite_add_tcase(unit, insert); TCase *sampling = tcase_create("de::DynamicExtension::range_sample Testing"); -- cgit v1.2.3 From 7c03d771475421c1d5a2bbc135242536af1a371c Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 25 Sep 2023 10:49:36 -0400 Subject: Re-structuring Project + scheduling updates This is a big one--probably should have split it apart, but I'm feeling lazy this morning. * Organized the mess of header files in include/framework by splitting them out into their own subdirectories, and renaming a few files to remove redundancies introduced by the directory structure. * Introduced a new framework/ShardRequirements.h header file for simpler shard development. This header simply contains the necessary includes from framework/* for creating shard files. This should help to remove structural dependencies from the framework file structure and shards, as well as centralizing the necessary framework files to make shard development easier. * Created a (currently dummy) SchedulerInterface, and make the scheduler implementation a template parameter of the dynamic extension for easier testing of various scheduling policies. There's still more work to be done to fully integrate the scheduler (queries, multiple buffers), but some more of the necessary framework code for this has been added as well. * Adjusted the Task interface setup for the scheduler. The task structures have been removed from ExtensionStructure and placed in their own header file. Additionally, I started experimenting with using std::variant, as opposed to inheritence, to implement subtype polymorphism on the Merge and Query tasks. The scheduler now has a general task queue that contains both, and std::variant, std::visit, and std::get are used to manipulate them without virtual functions. * Removed Alex.h, as it can't build anyway. There's a branch out there containing the Alex implementation stripped of the C++20 stuff. So there's no need to keep it here. --- tests/internal_level_tests.cpp | 8 ++++---- tests/mutable_buffer_tests.cpp | 2 +- tests/testing.h | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index 58369ff..056e458 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -10,10 +10,10 @@ * */ #include "shard/WIRS.h" -#include "framework/InternalLevel.h" -#include "framework/RecordInterface.h" -#include "framework/QueryInterface.h" -#include "framework/ShardInterface.h" +#include "framework/structure/InternalLevel.h" +#include "framework/interface/Record.h" +#include "framework/interface/Query.h" +#include "framework/interface/Shard.h" #include "testing.h" diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index 201fddb..a2561c8 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -15,7 +15,7 @@ #include #include "testing.h" -#include "framework/MutableBuffer.h" +#include "framework/structure/MutableBuffer.h" #include diff --git a/tests/testing.h b/tests/testing.h index bdf4869..023be7f 100644 --- a/tests/testing.h +++ b/tests/testing.h @@ -18,8 +18,8 @@ #include "util/types.h" #include "psu-util/alignment.h" -#include "framework/MutableBuffer.h" -#include "framework/RecordInterface.h" +#include "framework/structure/MutableBuffer.h" +#include "framework/interface/Record.h" typedef de::WeightedRecord WRec; typedef de::Record Rec; -- cgit v1.2.3 From 7ecfb22c32b7986ed1a2439c1abbeed298e4153a Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 20 Oct 2023 17:00:42 -0400 Subject: Initial pass w/ new scheduler setup currently there's a race condition of some type to sort out. --- tests/dynamic_extension_tests.inc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/dynamic_extension_tests.inc b/tests/dynamic_extension_tests.inc index bee28f7..2f82e92 100644 --- a/tests/dynamic_extension_tests.inc +++ b/tests/dynamic_extension_tests.inc @@ -213,10 +213,11 @@ START_TEST(t_range_sample_weighted) for (size_t i=0; i<1000; i++) { auto result = ext_wirs->query(&p); - total_samples += result.size(); + auto r = result.get(); + total_samples += r.size(); - for (size_t j=0; j Date: Tue, 31 Oct 2023 11:55:36 -0400 Subject: DE Tests: updated existing tests to use single threaded operation --- tests/de_level_tag.cpp | 2 +- tests/de_level_tomb.cpp | 2 +- tests/de_tier_tag.cpp | 3 ++- tests/de_tier_tomb.cpp | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index 91f158c..822ccc2 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -20,6 +20,6 @@ #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::LEVELING, DeletePolicy::TAGGING> DE; +typedef DynamicExtension, WIRSQuery, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; #include "dynamic_extension_tests.inc" diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index c3dc5df..264b5b0 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -20,6 +20,6 @@ #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE> DE; +typedef DynamicExtension, WIRSQuery, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "dynamic_extension_tests.inc" diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 9b6b5a4..2c263eb 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -15,11 +15,12 @@ #include "testing.h" #include "framework/DynamicExtension.h" +#include "framework/scheduling/SerialScheduler.h" #include "shard/WIRS.h" #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::TEIRING, DeletePolicy::TAGGING> DE; +typedef DynamicExtension, WIRSQuery, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; #include "dynamic_extension_tests.inc" diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index 82942fd..509494a 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -20,6 +20,6 @@ #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE> DE; +typedef DynamicExtension, WIRSQuery, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "dynamic_extension_tests.inc" -- cgit v1.2.3 From 83ca486048a5053d8c75bb5041091edb1b183a85 Mon Sep 17 00:00:00 2001 From: "Douglas B. Rumbaugh" Date: Tue, 31 Oct 2023 12:42:21 -0400 Subject: DE CC Testing: initial test file for FIFOScheduler --- tests/de_cc_isam_level.cpp | 461 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 461 insertions(+) create mode 100644 tests/de_cc_isam_level.cpp (limited to 'tests') diff --git a/tests/de_cc_isam_level.cpp b/tests/de_cc_isam_level.cpp new file mode 100644 index 0000000..e574578 --- /dev/null +++ b/tests/de_cc_isam_level.cpp @@ -0,0 +1,461 @@ +/* + * tests/dynamic_extension_tests.cpp + * + * Unit tests for Dynamic Extension Framework + * + * Copyright (C) 2023 Douglas B. Rumbaugh + * + * All rights reserved. Published under the Modified BSD License. + * + */ + +#include + +#include "framework/DynamicExtension.h" +#include "shard/MemISAM.h" + +#include + +using namespace de; + +typedef Record Rec; +typedef DynamicExtension, ISAMRangeQuery, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE> DE; + +START_TEST(t_create) +{ + auto de_isam = new DE(100, 2, 1); + + ck_assert_ptr_nonnull(de_isam); + ck_assert_int_eq(de_isam->get_record_count(), 0); + ck_assert_int_eq(de_isam->get_height(), 0); + + delete de_isam; +} +END_TEST + + +START_TEST(t_insert) +{ + auto ext_wirs = new DE(100, 2, 1); + + int32_t key = 0; + int32_t val = 0; + for (size_t i=0; i<100; i++) { + Rec r = {key, val}; + ck_assert_int_eq(ext_wirs->insert(r), 1); + key++; + val++; + } + + ck_assert_int_eq(ext_wirs->get_height(), 0); + ck_assert_int_eq(ext_wirs->get_record_count(), 100); + + delete ext_wirs; +} +END_TEST + + +START_TEST(t_debug_insert) +{ + auto ext_wirs = new DE(100, 2, 1); + + int32_t key = 0; + int32_t val = 0; + for (size_t i=0; i<1000; i++) { + Rec r = {key, val}; + ck_assert_int_eq(ext_wirs->insert(r), 1); + ck_assert_int_eq(ext_wirs->get_record_count(), i+1); + key++; + val++; + } + + delete ext_wirs; +} +END_TEST + + +START_TEST(t_insert_with_mem_merges) +{ + auto ext_wirs = new DE(100, 2, 1); + + int32_t key = 0; + int32_t val = 0; + for (size_t i=0; i<300; i++) { + Rec r = {key, val}; + ck_assert_int_eq(ext_wirs->insert(r), 1); + key++; + val++; + } + + ck_assert_int_eq(ext_wirs->get_record_count(), 300); + ck_assert_int_eq(ext_wirs->get_height(), 1); + + delete ext_wirs; +} +END_TEST + + +/* +START_TEST(t_range_sample_memtable) +{ + auto ext_wirs = new DE(100, 2, 1); + + int32_t key = 0; + int32_t val = 0; + for (size_t i=0; i<100; i++) { + Rec r = {key, val}; + ck_assert_int_eq(ext_wirs->insert(r), 1); + key++; + val++; + } + + int32_t lower_bound = 20; + int32_t upper_bound = 50; + + char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); + char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); + Rec sample_set[100]; + + ext_wirs->range_sample(sample_set, lower_bound, upper_bound, 100); + + for(size_t i=0; i<100; i++) { + ck_assert_int_le(sample_set[i].key, upper_bound); + ck_assert_int_ge(sample_set[i].key, lower_bound); + } + + free(buf); + free(util_buf); + + delete ext_wirs; +} +END_TEST + + +START_TEST(t_range_sample_memlevels) +{ + auto ext_wirs = new DE(100, 2, 1); + + int32_t key = 0; + int32_t val = 0; + for (size_t i=0; i<300; i++) { + Rec r = {key, val}; + ck_assert_int_eq(ext_wirs->insert(r), 1); + key++; + val++; + } + + int32_t lower_bound = 100; + int32_t upper_bound = 250; + + char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); + char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); + + Rec sample_set[100]; + ext_wirs->range_sample(sample_set, lower_bound, upper_bound, 100); + + for(size_t i=0; i<100; i++) { + ck_assert_int_le(sample_set[i].key, upper_bound); + ck_assert_int_ge(sample_set[i].key, lower_bound); + } + + free(buf); + free(util_buf); + + delete ext_wirs; +} +END_TEST + +START_TEST(t_range_sample_weighted) +{ + auto ext_wirs = new DE(100, 2, 1); + size_t n = 10000; + + std::vector keys; + + int32_t key = 1; + for (size_t i=0; i< n / 2; i++) { + keys.push_back(key); + } + + // put in a quarter of the count with weight two. + key = 2; + for (size_t i=0; i< n / 4; i++) { + keys.push_back(key); + } + + // the remaining quarter with weight four. + key = 3; + for (size_t i=0; i< n / 4; i++) { + keys.push_back(key); + } + + std::random_device rd; + std::mt19937 gen{rd()}; + std::shuffle(keys.begin(), keys.end(), gen); + + for (size_t i=0; iinsert(r); + } + size_t k = 1000; + int32_t lower_key = 0; + int32_t upper_key = 5; + + size_t cnt[3] = {0}; + size_t total_samples = 0; + + wirs_query_parms p; + p.lower_bound = lower_key; + p.upper_bound = upper_key; + p.sample_size = k; + p.rng = gsl_rng_alloc(gsl_rng_mt19937); + + for (size_t i=0; i<1000; i++) { + + auto result = ext_wirs->query(&p); + auto r = result.get(); + total_samples += r.size(); + + for (size_t j=0; j> records; + std::set> to_delete; + std::set> deleted; + + while (records.size() < reccnt) { + int32_t key = rand(); + int32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t cnt=0; + for (auto rec : records) { + Rec r = {rec.first, rec.second, 1}; + ck_assert_int_eq(ext_wirs->insert(r), 1); + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector> del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(dr); + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + + ck_assert(ext_wirs->validate_tombstone_proportion()); + } + + ck_assert(ext_wirs->validate_tombstone_proportion()); + + gsl_rng_free(rng); + delete ext_wirs; +} +END_TEST + +DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + auto ext_wirs = new DE(1000, 2, 1); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + int32_t key = rand(); + int32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + for (auto rec : records) { + ck_assert_int_eq(ext_wirs->insert(rec), 1); + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[i]); + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + gsl_rng_free(rng); + + return ext_wirs; +} + +START_TEST(t_static_structure) +{ + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t reccnt = 100000; + auto ext_wirs = new DE(100, 2, 1); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + int32_t key = rand(); + int32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t t_reccnt = 0; + size_t k=0; + for (auto rec : records) { + k++; + ck_assert_int_eq(ext_wirs->insert(rec), 1); + t_reccnt++; + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[i]), 1); + + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + auto flat = ext_wirs->create_static_structure(); + ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); + + int32_t prev_key = 0; + for (size_t i=0; iget_record_count(); i++) { + auto k = flat->get_record_at(i)->rec.key; + ck_assert_int_ge(k, prev_key); + prev_key = k; + } + + gsl_rng_free(rng); + delete flat; + delete ext_wirs; +} +END_TEST + + +Suite *unit_testing() +{ + Suite *unit = suite_create("de::DynamicExtension Unit Testing"); + + TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); + tcase_add_test(create, t_create); + suite_add_tcase(unit, create); + + TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); + tcase_add_test(insert, t_insert); + tcase_add_test(insert, t_insert_with_mem_merges); + tcase_add_test(insert, t_debug_insert); + suite_add_tcase(unit, insert); + + + /* + TCase *sampling = tcase_create("de::DynamicExtension::range_sample Testing"); + + tcase_add_test(sampling, t_range_sample_weighted); + suite_add_tcase(unit, sampling); + tcase_add_test(sampling, t_range_sample_memtable); + tcase_add_test(sampling, t_range_sample_memlevels); + */ + + TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); + tcase_add_test(ts, t_tombstone_merging_01); + tcase_set_timeout(ts, 500); + suite_add_tcase(unit, ts); + + TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); + tcase_add_test(flat, t_static_structure); + tcase_set_timeout(flat, 500); + suite_add_tcase(unit, flat); + + return unit; +} + +int run_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_runner = srunner_create(unit); + + srunner_run_all(unit_runner, CK_NORMAL); + failed = srunner_ntests_failed(unit_runner); + srunner_free(unit_runner); + + return failed; +} + + +int main() +{ + int unit_failed = run_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} -- cgit v1.2.3 From 4e4cf858122ca6c1ae6d5f635e839089769fee38 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 6 Nov 2023 10:01:23 -0500 Subject: Scheduling: Switched over to a thread pool model --- tests/de_level_tomb.cpp | 2 +- tests/dynamic_extension_tests.inc | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 264b5b0..ba3f784 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -20,6 +20,6 @@ #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef DynamicExtension, WIRSQuery, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; #include "dynamic_extension_tests.inc" diff --git a/tests/dynamic_extension_tests.inc b/tests/dynamic_extension_tests.inc index 2f82e92..df88d98 100644 --- a/tests/dynamic_extension_tests.inc +++ b/tests/dynamic_extension_tests.inc @@ -77,6 +77,8 @@ START_TEST(t_insert_with_mem_merges) val++; } + ext_wirs->await_next_epoch(); + ck_assert_int_eq(ext_wirs->get_record_count(), 300); ck_assert_int_eq(ext_wirs->get_height(), 1); @@ -197,6 +199,9 @@ START_TEST(t_range_sample_weighted) WRec r = {keys[i], (uint32_t) i, weight}; ext_wirs->insert(r); } + + ext_wirs->await_next_epoch(); + size_t k = 1000; uint64_t lower_key = 0; uint64_t upper_key = 5; @@ -277,6 +282,8 @@ START_TEST(t_tombstone_merging_01) ck_assert(ext_wirs->validate_tombstone_proportion()); } + ext_wirs->await_next_epoch(); + ck_assert(ext_wirs->validate_tombstone_proportion()); gsl_rng_free(rng); -- cgit v1.2.3 From e02742b07540dd5a9bcbb44dae14856bf10955ed Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 6 Nov 2023 15:18:53 -0500 Subject: Refactoring progress --- tests/alias_tests.cpp | 391 +++++++++++++++++++++++++++++++++++++++++++++ tests/memisam_tests.cpp | 69 ++++---- tests/pgm_tests.cpp | 33 ++-- tests/triespline_tests.cpp | 17 +- tests/wss_tests.cpp | 390 -------------------------------------------- 5 files changed, 452 insertions(+), 448 deletions(-) create mode 100644 tests/alias_tests.cpp delete mode 100644 tests/wss_tests.cpp (limited to 'tests') diff --git a/tests/alias_tests.cpp b/tests/alias_tests.cpp new file mode 100644 index 0000000..b9e678b --- /dev/null +++ b/tests/alias_tests.cpp @@ -0,0 +1,391 @@ +/* + * tests/alias_tests.cpp + * + * Unit tests for Alias shard + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * All rights reserved. Published under the Modified BSD License. + * + */ + +#include "shard/Alias.h" +#include "query/wss.h" +#include "testing.h" + +#include + +using namespace de; + +typedef Alias Shard; + +START_TEST(t_mbuffer_init) +{ + auto buffer = new MutableBuffer(1024, 1024); + for (uint64_t i = 512; i > 0; i--) { + uint32_t v = i; + buffer->append({i,v, 1}); + } + + for (uint64_t i = 1; i <= 256; ++i) { + uint32_t v = i; + buffer->append({i, v, 1}, true); + } + + for (uint64_t i = 257; i <= 512; ++i) { + uint32_t v = i + 1; + buffer->append({i, v, 1}); + } + + Shard* shard = new Shard(buffer); + ck_assert_uint_eq(shard->get_record_count(), 512); + + delete buffer; + delete shard; +} + + +START_TEST(t_alias_init) +{ + size_t n = 512; + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); + + auto shard1 = new Shard(mbuffer1); + auto shard2 = new Shard(mbuffer2); + auto shard3 = new Shard(mbuffer3); + + Shard* shards[3] = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards, 3); + + ck_assert_int_eq(shard4->get_record_count(), n * 3); + ck_assert_int_eq(shard4->get_tombstone_count(), 0); + + size_t total_cnt = 0; + size_t shard1_idx = 0; + size_t shard2_idx = 0; + size_t shard3_idx = 0; + + for (size_t i = 0; i < shard4->get_record_count(); ++i) { + auto rec1 = shard1->get_record_at(shard1_idx); + auto rec2 = shard2->get_record_at(shard2_idx); + auto rec3 = shard3->get_record_at(shard3_idx); + + auto cur_rec = shard4->get_record_at(i); + + if (shard1_idx < n && cur_rec->rec == rec1->rec) { + ++shard1_idx; + } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { + ++shard2_idx; + } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { + ++shard3_idx; + } else { + assert(false); + } + } + + delete mbuffer1; + delete mbuffer2; + delete mbuffer3; + + delete shard1; + delete shard2; + delete shard3; + delete shard4; +} + + +START_TEST(t_point_lookup) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto alias = Shard(buffer); + + for (size_t i=0; iget_data() + i); + r.key = rec->rec.key; + r.value = rec->rec.value; + + auto result = alias.point_lookup(r); + ck_assert_ptr_nonnull(result); + ck_assert_int_eq(result->rec.key, r.key); + ck_assert_int_eq(result->rec.value, r.value); + } + + delete buffer; +} +END_TEST + + +START_TEST(t_point_lookup_miss) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto alias = Shard(buffer); + + for (size_t i=n + 100; i<2*n; i++) { + WRec r; + r.key = i; + r.value = i; + + auto result = alias.point_lookup(r); + ck_assert_ptr_null(result); + } + + delete buffer; +} + +START_TEST(t_full_cancelation) +{ + size_t n = 100; + auto buffer = create_double_seq_mbuffer(n, false); + auto buffer_ts = create_double_seq_mbuffer(n, true); + + Shard* shard = new Shard(buffer); + Shard* shard_ts = new Shard(buffer_ts); + + ck_assert_int_eq(shard->get_record_count(), n); + ck_assert_int_eq(shard->get_tombstone_count(), 0); + ck_assert_int_eq(shard_ts->get_record_count(), n); + ck_assert_int_eq(shard_ts->get_tombstone_count(), n); + + Shard* shards[] = {shard, shard_ts}; + + Shard* merged = new Shard(shards, 2); + + ck_assert_int_eq(merged->get_tombstone_count(), 0); + ck_assert_int_eq(merged->get_record_count(), 0); + + delete buffer; + delete buffer_ts; + delete shard; + delete shard_ts; + delete merged; +} +END_TEST + + +START_TEST(t_alias_query) +{ + size_t n=1000; + auto buffer = create_weighted_mbuffer(n); + + Shard* shard = new Shard(buffer); + + size_t k = 1000; + + size_t cnt[3] = {0}; + wss:Parms parms = {k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = wss::Query::get_query_state(shard, &parms); + ((wss::State *) state)->sample_size = k; + auto result = wss::Query::query(shard, state, &parms); + + total_samples += result.size(); + + for (size_t j=0; j::delete_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); + + gsl_rng_free(parms.rng); + delete shard; + delete buffer; +} +END_TEST + + +START_TEST(t_alias_query_merge) +{ + size_t n=1000; + auto buffer = create_weighted_mbuffer(n); + + Shard* shard = new Shard(buffer); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wss:Parms parms = {k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + std::vector>> results(2); + + for (size_t i=0; i<1000; i++) { + auto state1 = wss::Query::get_query_state(shard, &parms); + ((wss::State *) state1)->sample_size = k; + results[0] = wss::Query::query(shard, state1, &parms); + + auto state2 = wss::Query::get_query_state(shard, &parms); + ((wss::State *) state2)->sample_size = k; + results[1] = wss::Query::query(shard, state2, &parms); + + wss::Query::delete_query_state(state1); + wss::Query::delete_query_state(state2); + } + + auto merged = wss::Query::merge(results, nullptr); + + ck_assert_int_eq(merged.size(), 2*k); + for (size_t i=0; i(n); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wss:Parms parms = {k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = wss::Query::get_buffer_query_state(buffer, &parms); + ((wss::BufferState *) state)->sample_size = k; + auto result = wss::Query::buffer_query(buffer, state, &parms); + total_samples += result.size(); + + for (size_t j=0; j::delete_buffer_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); + + gsl_rng_free(parms.rng); + delete buffer; +} +END_TEST + + +START_TEST(t_alias_buffer_query_rejection) +{ + size_t n=1000; + auto buffer = create_weighted_mbuffer(n); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wss:Parms parms = {k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = wss::Query::get_buffer_query_state(buffer, &parms); + ((wss::BufferState *) state)->sample_size = k; + auto result = wss::Query::buffer_query(buffer, state, &parms); + + total_samples += result.size(); + + for (size_t j=0; j::delete_buffer_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .1)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .1)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .1)); + + gsl_rng_free(parms.rng); + delete buffer; +} +END_TEST + + +Suite *unit_testing() +{ + Suite *unit = suite_create("Alias Shard Unit Testing"); + + TCase *create = tcase_create("de::Alias constructor Testing"); + tcase_add_test(create, t_mbuffer_init); + tcase_add_test(create, t_alias_init); + tcase_set_timeout(create, 100); + suite_add_tcase(unit, create); + + + TCase *tombstone = tcase_create("de:Alias::tombstone cancellation Testing"); + tcase_add_test(tombstone, t_full_cancelation); + suite_add_tcase(unit, tombstone); + + + TCase *lookup = tcase_create("de:Alias:point_lookup Testing"); + tcase_add_test(lookup, t_point_lookup); + tcase_add_test(lookup, t_point_lookup_miss); + suite_add_tcase(unit, lookup); + + + + TCase *sampling = tcase_create("de:Alias::AliasQuery Testing"); + tcase_add_test(sampling, t_alias_query); + tcase_add_test(sampling, t_alias_query_merge); + tcase_add_test(sampling, t_alias_buffer_query_rejection); + tcase_add_test(sampling, t_alias_buffer_query_scan); + suite_add_tcase(unit, sampling); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/memisam_tests.cpp b/tests/memisam_tests.cpp index 0ae97dc..d3b8087 100644 --- a/tests/memisam_tests.cpp +++ b/tests/memisam_tests.cpp @@ -1,7 +1,7 @@ /* - * tests/irs_tests.cpp + * tests/isam_tests.cpp * - * Unit tests for MemISAM (Augmented B+Tree) shard + * Unit tests for ISAM Tree shard * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie @@ -10,14 +10,15 @@ * */ -#include "shard/MemISAM.h" +#include "shard/ISAMTree.h" +#include "query/irs.h" #include "testing.h" #include using namespace de; -typedef MemISAM Shard; +typedef ISAMTree Shard; START_TEST(t_mbuffer_init) { @@ -181,15 +182,15 @@ START_TEST(t_irs_query) size_t k = 100; size_t cnt[3] = {0}; - irs_query_parms parms = {lower_key, upper_key, k}; + irs::Parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; for (size_t i=0; i<1000; i++) { - auto state = IRSQuery::get_query_state(&isam, &parms); - ((IRSState *) state)->sample_size = k; - auto result = IRSQuery::query(&isam, state, &parms); + auto state = irs::Query::get_query_state(&isam, &parms); + ((irs::State *) state)->sample_size = k; + auto result = irs::Query::query(&isam, state, &parms); ck_assert_int_eq(result.size(), k); @@ -198,7 +199,7 @@ START_TEST(t_irs_query) ck_assert_int_ge(rec.rec.key, lower_key); } - IRSQuery::delete_query_state(state); + irs::Query::delete_query_state(state); } gsl_rng_free(parms.rng); @@ -220,25 +221,25 @@ START_TEST(t_irs_query_merge) size_t k = 1000; size_t cnt[3] = {0}; - irs_query_parms parms = {lower_key, upper_key, k}; + irs::Parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); std::vector>> results(2); for (size_t i=0; i<1000; i++) { - auto state1 = IRSQuery::get_query_state(&shard, &parms); - ((IRSState *) state1)->sample_size = k; - results[0] = IRSQuery::query(&shard, state1, &parms); + auto state1 = irs::Query::get_query_state(&shard, &parms); + ((irs::State *) state1)->sample_size = k; + results[0] = irs::Query::query(&shard, state1, &parms); - auto state2 = IRSQuery::get_query_state(&shard, &parms); - ((IRSState *) state2)->sample_size = k; - results[1] = IRSQuery::query(&shard, state2, &parms); + auto state2 = irs::Query::get_query_state(&shard, &parms); + ((irs::State *) state2)->sample_size = k; + results[1] = irs::Query::query(&shard, state2, &parms); - IRSQuery::delete_query_state(state1); - IRSQuery::delete_query_state(state2); + irs::Query::delete_query_state(state1); + irs::Query::delete_query_state(state2); } - auto merged = IRSQuery::merge(results, nullptr); + auto merged = irs::Query::merge(results, nullptr); ck_assert_int_eq(merged.size(), 2*k); for (size_t i=0; i parms = {lower_key, upper_key, k}; + irs::Parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; for (size_t i=0; i<1000; i++) { - auto state = IRSQuery::get_buffer_query_state(buffer, &parms); - ((IRSBufferState *) state)->sample_size = k; - auto result = IRSQuery::buffer_query(buffer, state, &parms); + auto state = irs::Query::get_buffer_query_state(buffer, &parms); + ((irs::BufferState *) state)->sample_size = k; + auto result = irs::Query::buffer_query(buffer, state, &parms); ck_assert_int_eq(result.size(), k); @@ -280,7 +281,7 @@ START_TEST(t_irs_buffer_query_scan) ck_assert_int_ge(rec.rec.key, lower_key); } - IRSQuery::delete_buffer_query_state(state); + irs::Query::delete_buffer_query_state(state); } gsl_rng_free(parms.rng); @@ -300,15 +301,15 @@ START_TEST(t_irs_buffer_query_rejection) size_t k = 10000; size_t cnt[3] = {0}; - irs_query_parms parms = {lower_key, upper_key, k}; + irs::Parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; for (size_t i=0; i<1000; i++) { - auto state = IRSQuery::get_buffer_query_state(buffer, &parms); - ((IRSBufferState *) state)->sample_size = k; - auto result = IRSQuery::buffer_query(buffer, state, &parms); + auto state = irs::Query::get_buffer_query_state(buffer, &parms); + ((irs::BufferState *) state)->sample_size = k; + auto result = irs::Query::buffer_query(buffer, state, &parms); ck_assert_int_gt(result.size(), 0); ck_assert_int_le(result.size(), k); @@ -318,7 +319,7 @@ START_TEST(t_irs_buffer_query_rejection) ck_assert_int_ge(rec.rec.key, lower_key); } - IRSQuery::delete_buffer_query_state(state); + irs::Query::delete_buffer_query_state(state); } gsl_rng_free(parms.rng); @@ -329,27 +330,27 @@ END_TEST Suite *unit_testing() { - Suite *unit = suite_create("MemISAM Shard Unit Testing"); + Suite *unit = suite_create("ISAMTree Shard Unit Testing"); - TCase *create = tcase_create("de::MemISAM constructor Testing"); + TCase *create = tcase_create("de::ISAMTree constructor Testing"); tcase_add_test(create, t_mbuffer_init); tcase_add_test(create, t_irs_init); tcase_set_timeout(create, 100); suite_add_tcase(unit, create); - TCase *tombstone = tcase_create("de:MemISAM::tombstone cancellation Testing"); + TCase *tombstone = tcase_create("de:ISAMTree::tombstone cancellation Testing"); tcase_add_test(tombstone, t_full_cancelation); suite_add_tcase(unit, tombstone); - TCase *lookup = tcase_create("de:MemISAM:point_lookup Testing"); + TCase *lookup = tcase_create("de:ISAMTree:point_lookup Testing"); tcase_add_test(lookup, t_point_lookup); tcase_add_test(lookup, t_point_lookup_miss); suite_add_tcase(unit, lookup); - TCase *sampling = tcase_create("de:MemISAM::MemISAMQuery Testing"); + TCase *sampling = tcase_create("de:ISAMTree::ISAMTreeQuery Testing"); tcase_add_test(sampling, t_irs_query); tcase_add_test(sampling, t_irs_query_merge); tcase_add_test(sampling, t_irs_buffer_query_rejection); diff --git a/tests/pgm_tests.cpp b/tests/pgm_tests.cpp index 0552417..1565384 100644 --- a/tests/pgm_tests.cpp +++ b/tests/pgm_tests.cpp @@ -11,6 +11,7 @@ */ #include "shard/PGM.h" +#include "query/rangequery.h" #include "testing.h" #include @@ -144,13 +145,13 @@ START_TEST(t_range_query) auto buffer = create_sequential_mbuffer(100, 1000); auto shard = Shard(buffer); - pgm_range_query_parms parms; + rq::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; - auto state = PGMRangeQuery::get_query_state(&shard, &parms); - auto result = PGMRangeQuery::query(&shard, state, &parms); - PGMRangeQuery::delete_query_state(state); + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i(100, 1000); - pgm_range_query_parms parms; + rq::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; - auto state = PGMRangeQuery::get_buffer_query_state(buffer, &parms); - auto result = PGMRangeQuery::buffer_query(buffer, state, &parms); - PGMRangeQuery::delete_buffer_query_state(state); + auto state = rq::Query::get_buffer_query_state(buffer, &parms); + auto result = rq::Query::buffer_query(buffer, state, &parms); + rq::Query::delete_buffer_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i parms; + rq::Parms parms; parms.lower_bound = 150; parms.upper_bound = 500; size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; - auto state1 = PGMRangeQuery::get_query_state(&shard1, &parms); - auto state2 = PGMRangeQuery::get_query_state(&shard2, &parms); + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); std::vector>> results(2); - results[0] = PGMRangeQuery::query(&shard1, state1, &parms); - results[1] = PGMRangeQuery::query(&shard2, state2, &parms); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); - PGMRangeQuery::delete_query_state(state1); - PGMRangeQuery::delete_query_state(state2); + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); ck_assert_int_eq(results[0].size() + results[1].size(), result_size); @@ -221,7 +222,7 @@ START_TEST(t_range_query_merge) } } - auto result = PGMRangeQuery::merge(proc_results, nullptr); + auto result = rq::Query::merge(proc_results, nullptr); std::sort(result.begin(), result.end()); ck_assert_int_eq(result.size(), result_size); diff --git a/tests/triespline_tests.cpp b/tests/triespline_tests.cpp index 6f63961..101f143 100644 --- a/tests/triespline_tests.cpp +++ b/tests/triespline_tests.cpp @@ -13,6 +13,7 @@ #include #include "shard/TrieSpline.h" +#include "query/rangequery.h" #include "testing.h" #include @@ -176,13 +177,13 @@ START_TEST(t_range_query) auto buffer = create_sequential_mbuffer(100, 1000); auto shard = Shard(buffer); - ts_range_query_parms parms; + rq::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; - auto state = TrieSplineRangeQuery::get_query_state(&shard, &parms); - auto result = TrieSplineRangeQuery::query(&shard, state, &parms); - TrieSplineRangeQuery::delete_query_state(state); + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i(100, 1000); - ts_range_query_parms parms; + rq::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; - auto state = TrieSplineRangeQuery::get_buffer_query_state(buffer, &parms); - auto result = TrieSplineRangeQuery::buffer_query(buffer, state, &parms); - TrieSplineRangeQuery::delete_buffer_query_state(state); + auto state = rq::Query::get_buffer_query_state(buffer, &parms); + auto result = rq::Query::buffer_query(buffer, state, &parms); + rq::Query::delete_buffer_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i - * Dong Xie - * - * All rights reserved. Published under the Modified BSD License. - * - */ - -#include "shard/WSS.h" -#include "testing.h" - -#include - -using namespace de; - -typedef WSS Shard; - -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer(1024, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_wss_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); - - auto shard1 = new Shard(mbuffer1); - auto shard2 = new Shard(mbuffer2); - auto shard3 = new Shard(mbuffer3); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto wss = Shard(buffer); - - for (size_t i=0; iget_data() + i); - r.key = rec->rec.key; - r.value = rec->rec.value; - - auto result = wss.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto wss = Shard(buffer); - - for (size_t i=n + 100; i<2*n; i++) { - WRec r; - r.key = i; - r.value = i; - - auto result = wss.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); - - Shard* shard = new Shard(buffer); - Shard* shard_ts = new Shard(buffer_ts); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST - - -START_TEST(t_wss_query) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - Shard* shard = new Shard(buffer); - - size_t k = 1000; - - size_t cnt[3] = {0}; - wss_query_parms parms = {k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = WSSQuery::get_query_state(shard, &parms); - ((WSSState *) state)->sample_size = k; - auto result = WSSQuery::query(shard, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete shard; - delete buffer; -} -END_TEST - - -START_TEST(t_wss_query_merge) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - Shard* shard = new Shard(buffer); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wss_query_parms parms = {k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - std::vector>> results(2); - - for (size_t i=0; i<1000; i++) { - auto state1 = WSSQuery::get_query_state(shard, &parms); - ((WSSState *) state1)->sample_size = k; - results[0] = WSSQuery::query(shard, state1, &parms); - - auto state2 = WSSQuery::get_query_state(shard, &parms); - ((WSSState *) state2)->sample_size = k; - results[1] = WSSQuery::query(shard, state2, &parms); - - WSSQuery::delete_query_state(state1); - WSSQuery::delete_query_state(state2); - } - - auto merged = WSSQuery::merge(results, nullptr); - - ck_assert_int_eq(merged.size(), 2*k); - for (size_t i=0; i(n); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wss_query_parms parms = {k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = WSSQuery::get_buffer_query_state(buffer, &parms); - ((WSSBufferState *) state)->sample_size = k; - auto result = WSSQuery::buffer_query(buffer, state, &parms); - total_samples += result.size(); - - for (size_t j=0; j::delete_buffer_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_wss_buffer_query_rejection) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wss_query_parms parms = {k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = WSSQuery::get_buffer_query_state(buffer, &parms); - ((WSSBufferState *) state)->sample_size = k; - auto result = WSSQuery::buffer_query(buffer, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_buffer_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .1)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .1)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .1)); - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -Suite *unit_testing() -{ - Suite *unit = suite_create("WSS Shard Unit Testing"); - - TCase *create = tcase_create("de::WSS constructor Testing"); - tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_wss_init); - tcase_set_timeout(create, 100); - suite_add_tcase(unit, create); - - - TCase *tombstone = tcase_create("de:WSS::tombstone cancellation Testing"); - tcase_add_test(tombstone, t_full_cancelation); - suite_add_tcase(unit, tombstone); - - - TCase *lookup = tcase_create("de:WSS:point_lookup Testing"); - tcase_add_test(lookup, t_point_lookup); - tcase_add_test(lookup, t_point_lookup_miss); - suite_add_tcase(unit, lookup); - - - - TCase *sampling = tcase_create("de:WSS::WSSQuery Testing"); - tcase_add_test(sampling, t_wss_query); - tcase_add_test(sampling, t_wss_query_merge); - tcase_add_test(sampling, t_wss_buffer_query_rejection); - tcase_add_test(sampling, t_wss_buffer_query_scan); - suite_add_tcase(unit, sampling); - - return unit; -} - - -int shard_unit_tests() -{ - int failed = 0; - Suite *unit = unit_testing(); - SRunner *unit_shardner = srunner_create(unit); - - srunner_run_all(unit_shardner, CK_NORMAL); - failed = srunner_ntests_failed(unit_shardner); - srunner_free(unit_shardner); - - return failed; -} - - -int main() -{ - int unit_failed = shard_unit_tests(); - - return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; -} -- cgit v1.2.3 From a2fe4b1616a1b2318f70e842382818ee44aea9e6 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 7 Nov 2023 12:29:03 -0500 Subject: Alias shard fixes --- tests/alias_tests.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/alias_tests.cpp b/tests/alias_tests.cpp index b9e678b..c4a302d 100644 --- a/tests/alias_tests.cpp +++ b/tests/alias_tests.cpp @@ -180,7 +180,7 @@ START_TEST(t_alias_query) size_t k = 1000; size_t cnt[3] = {0}; - wss:Parms parms = {k}; + wss::Parms parms = {k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; @@ -223,7 +223,7 @@ START_TEST(t_alias_query_merge) size_t k = 1000; size_t cnt[3] = {0}; - wss:Parms parms = {k}; + wss::Parms parms = {k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); std::vector>> results(2); @@ -267,7 +267,7 @@ START_TEST(t_alias_buffer_query_scan) size_t k = 1000; size_t cnt[3] = {0}; - wss:Parms parms = {k}; + wss::Parms parms = {k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; @@ -306,7 +306,7 @@ START_TEST(t_alias_buffer_query_rejection) size_t k = 1000; size_t cnt[3] = {0}; - wss:Parms parms = {k}; + wss::Parms parms = {k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; -- cgit v1.2.3 From 9e1c1b1b930031896851b1ed4a15152508327d73 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 7 Nov 2023 13:35:54 -0500 Subject: Converted WIRS to the new interface --- tests/augbtree_tests.cpp | 395 +++++++++++++++++++++++++++++++++++++++++++++++ tests/wirs_tests.cpp | 394 ---------------------------------------------- 2 files changed, 395 insertions(+), 394 deletions(-) create mode 100644 tests/augbtree_tests.cpp delete mode 100644 tests/wirs_tests.cpp (limited to 'tests') diff --git a/tests/augbtree_tests.cpp b/tests/augbtree_tests.cpp new file mode 100644 index 0000000..878af82 --- /dev/null +++ b/tests/augbtree_tests.cpp @@ -0,0 +1,395 @@ +/* + * tests/wirs_tests.cpp + * + * Unit tests for AugBTree (Augmented B+Tree) shard + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * All rights reserved. Published under the Modified BSD License. + * + */ + +#include "shard/AugBTree.h" +#include "query/wirs.h" +#include "testing.h" + +#include + +using namespace de; + +typedef AugBTree Shard; + +START_TEST(t_mbuffer_init) +{ + auto buffer = new MutableBuffer(1024, 1024); + for (uint64_t i = 512; i > 0; i--) { + uint32_t v = i; + buffer->append({i,v, 1}); + } + + for (uint64_t i = 1; i <= 256; ++i) { + uint32_t v = i; + buffer->append({i, v, 1}, true); + } + + for (uint64_t i = 257; i <= 512; ++i) { + uint32_t v = i + 1; + buffer->append({i, v, 1}); + } + + Shard* shard = new Shard(buffer); + ck_assert_uint_eq(shard->get_record_count(), 512); + + delete buffer; + delete shard; +} + + +START_TEST(t_wirs_init) +{ + size_t n = 512; + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); + + auto shard1 = new Shard(mbuffer1); + auto shard2 = new Shard(mbuffer2); + auto shard3 = new Shard(mbuffer3); + + Shard* shards[3] = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards, 3); + + ck_assert_int_eq(shard4->get_record_count(), n * 3); + ck_assert_int_eq(shard4->get_tombstone_count(), 0); + + size_t total_cnt = 0; + size_t shard1_idx = 0; + size_t shard2_idx = 0; + size_t shard3_idx = 0; + + for (size_t i = 0; i < shard4->get_record_count(); ++i) { + auto rec1 = shard1->get_record_at(shard1_idx); + auto rec2 = shard2->get_record_at(shard2_idx); + auto rec3 = shard3->get_record_at(shard3_idx); + + auto cur_rec = shard4->get_record_at(i); + + if (shard1_idx < n && cur_rec->rec == rec1->rec) { + ++shard1_idx; + } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { + ++shard2_idx; + } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { + ++shard3_idx; + } else { + assert(false); + } + } + + delete mbuffer1; + delete mbuffer2; + delete mbuffer3; + + delete shard1; + delete shard2; + delete shard3; + delete shard4; +} + + +START_TEST(t_point_lookup) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto wirs = Shard(buffer); + + for (size_t i=0; iget_data() + i); + r.key = rec->rec.key; + r.value = rec->rec.value; + + auto result = wirs.point_lookup(r); + ck_assert_ptr_nonnull(result); + ck_assert_int_eq(result->rec.key, r.key); + ck_assert_int_eq(result->rec.value, r.value); + } + + delete buffer; +} +END_TEST + + +START_TEST(t_point_lookup_miss) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto wirs = Shard(buffer); + + for (size_t i=n + 100; i<2*n; i++) { + WRec r; + r.key = i; + r.value = i; + + auto result = wirs.point_lookup(r); + ck_assert_ptr_null(result); + } + + delete buffer; +} + + +START_TEST(t_full_cancelation) +{ + size_t n = 100; + auto buffer = create_double_seq_mbuffer(n, false); + auto buffer_ts = create_double_seq_mbuffer(n, true); + + Shard* shard = new Shard(buffer); + Shard* shard_ts = new Shard(buffer_ts); + + ck_assert_int_eq(shard->get_record_count(), n); + ck_assert_int_eq(shard->get_tombstone_count(), 0); + ck_assert_int_eq(shard_ts->get_record_count(), n); + ck_assert_int_eq(shard_ts->get_tombstone_count(), n); + + Shard* shards[] = {shard, shard_ts}; + + Shard* merged = new Shard(shards, 2); + + ck_assert_int_eq(merged->get_tombstone_count(), 0); + ck_assert_int_eq(merged->get_record_count(), 0); + + delete buffer; + delete buffer_ts; + delete shard; + delete shard_ts; + delete merged; +} +END_TEST + + +START_TEST(t_wirs_query) +{ + size_t n=1000; + auto buffer = create_weighted_mbuffer(n); + + Shard* shard = new Shard(buffer); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wirs::Parms parms = {lower_key, upper_key, k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = wirs::Query::get_query_state(shard, &parms); + ((wirs::State *) state)->sample_size = k; + auto result = wirs::Query::query(shard, state, &parms); + + total_samples += result.size(); + + for (size_t j=0; j::delete_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); + + gsl_rng_free(parms.rng); + delete shard; + delete buffer; +} +END_TEST + + +START_TEST(t_wirs_query_merge) +{ + size_t n=1000; + auto buffer = create_weighted_mbuffer(n); + + Shard* shard = new Shard(buffer); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wirs::Parms parms = {lower_key, upper_key, k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + std::vector>> results(2); + + for (size_t i=0; i<1000; i++) { + auto state1 = wirs::Query::get_query_state(shard, &parms); + ((wirs::State *) state1)->sample_size = k; + results[0] = wirs::Query::query(shard, state1, &parms); + + auto state2 = wirs::Query::get_query_state(shard, &parms); + ((wirs::State *) state2)->sample_size = k; + results[1] = wirs::Query::query(shard, state2, &parms); + + wirs::Query::delete_query_state(state1); + wirs::Query::delete_query_state(state2); + } + + auto merged = wirs::Query::merge(results, nullptr); + + ck_assert_int_eq(merged.size(), 2*k); + for (size_t i=0; i(n); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wirs::Parms parms = {lower_key, upper_key, k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = wirs::Query::get_buffer_query_state(buffer, &parms); + ((wirs::BufferState *) state)->sample_size = k; + auto result = wirs::Query::buffer_query(buffer, state, &parms); + + total_samples += result.size(); + + for (size_t j=0; j::delete_buffer_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); + + gsl_rng_free(parms.rng); + delete buffer; +} +END_TEST + + +START_TEST(t_wirs_buffer_query_rejection) +{ + size_t n=1000; + auto buffer = create_weighted_mbuffer(n); + + uint64_t lower_key = 0; + uint64_t upper_key = 5; + + size_t k = 1000; + + size_t cnt[3] = {0}; + wirs::Parms parms = {lower_key, upper_key, k}; + parms.rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t total_samples = 0; + + for (size_t i=0; i<1000; i++) { + auto state = wirs::Query::get_buffer_query_state(buffer, &parms); + ((wirs::BufferState *) state)->sample_size = k; + auto result = wirs::Query::buffer_query(buffer, state, &parms); + + total_samples += result.size(); + + for (size_t j=0; j::delete_buffer_query_state(state); + } + + ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); + ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); + + gsl_rng_free(parms.rng); + delete buffer; +} +END_TEST + + +Suite *unit_testing() +{ + Suite *unit = suite_create("AugBTree Shard Unit Testing"); + + TCase *create = tcase_create("de::AugBTree constructor Testing"); + tcase_add_test(create, t_mbuffer_init); + tcase_add_test(create, t_wirs_init); + tcase_set_timeout(create, 100); + suite_add_tcase(unit, create); + + + TCase *tombstone = tcase_create("de:AugBTree::tombstone cancellation Testing"); + tcase_add_test(tombstone, t_full_cancelation); + suite_add_tcase(unit, tombstone); + + + TCase *lookup = tcase_create("de:AugBTree:point_lookup Testing"); + tcase_add_test(lookup, t_point_lookup); + tcase_add_test(lookup, t_point_lookup_miss); + suite_add_tcase(unit, lookup); + + + TCase *sampling = tcase_create("de:AugBTree::AugBTreeQuery Testing"); + tcase_add_test(sampling, t_wirs_query); + tcase_add_test(sampling, t_wirs_query_merge); + tcase_add_test(sampling, t_wirs_buffer_query_rejection); + tcase_add_test(sampling, t_wirs_buffer_query_scan); + suite_add_tcase(unit, sampling); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/wirs_tests.cpp b/tests/wirs_tests.cpp deleted file mode 100644 index a72f950..0000000 --- a/tests/wirs_tests.cpp +++ /dev/null @@ -1,394 +0,0 @@ -/* - * tests/wirs_tests.cpp - * - * Unit tests for WIRS (Augmented B+Tree) shard - * - * Copyright (C) 2023 Douglas Rumbaugh - * Dong Xie - * - * All rights reserved. Published under the Modified BSD License. - * - */ - -#include "shard/WIRS.h" -#include "testing.h" - -#include - -using namespace de; - -typedef WIRS Shard; - -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer(1024, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_wirs_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); - - auto shard1 = new Shard(mbuffer1); - auto shard2 = new Shard(mbuffer2); - auto shard3 = new Shard(mbuffer3); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto wirs = Shard(buffer); - - for (size_t i=0; iget_data() + i); - r.key = rec->rec.key; - r.value = rec->rec.value; - - auto result = wirs.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto wirs = Shard(buffer); - - for (size_t i=n + 100; i<2*n; i++) { - WRec r; - r.key = i; - r.value = i; - - auto result = wirs.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); - - Shard* shard = new Shard(buffer); - Shard* shard_ts = new Shard(buffer_ts); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST - - -START_TEST(t_wirs_query) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - Shard* shard = new Shard(buffer); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wirs_query_parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = WIRSQuery::get_query_state(shard, &parms); - ((WIRSState *) state)->sample_size = k; - auto result = WIRSQuery::query(shard, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete shard; - delete buffer; -} -END_TEST - - -START_TEST(t_wirs_query_merge) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - Shard* shard = new Shard(buffer); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wirs_query_parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - std::vector>> results(2); - - for (size_t i=0; i<1000; i++) { - auto state1 = WIRSQuery::get_query_state(shard, &parms); - ((WIRSState *) state1)->sample_size = k; - results[0] = WIRSQuery::query(shard, state1, &parms); - - auto state2 = WIRSQuery::get_query_state(shard, &parms); - ((WIRSState *) state2)->sample_size = k; - results[1] = WIRSQuery::query(shard, state2, &parms); - - WIRSQuery::delete_query_state(state1); - WIRSQuery::delete_query_state(state2); - } - - auto merged = WIRSQuery::merge(results, nullptr); - - ck_assert_int_eq(merged.size(), 2*k); - for (size_t i=0; i(n); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wirs_query_parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = WIRSQuery::get_buffer_query_state(buffer, &parms); - ((WIRSBufferState *) state)->sample_size = k; - auto result = WIRSQuery::buffer_query(buffer, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_buffer_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_wirs_buffer_query_rejection) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wirs_query_parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = WIRSQuery::get_buffer_query_state(buffer, &parms); - ((WIRSBufferState *) state)->sample_size = k; - auto result = WIRSQuery::buffer_query(buffer, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_buffer_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -Suite *unit_testing() -{ - Suite *unit = suite_create("WIRS Shard Unit Testing"); - - TCase *create = tcase_create("de::WIRS constructor Testing"); - tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_wirs_init); - tcase_set_timeout(create, 100); - suite_add_tcase(unit, create); - - - TCase *tombstone = tcase_create("de:WIRS::tombstone cancellation Testing"); - tcase_add_test(tombstone, t_full_cancelation); - suite_add_tcase(unit, tombstone); - - - TCase *lookup = tcase_create("de:WIRS:point_lookup Testing"); - tcase_add_test(lookup, t_point_lookup); - tcase_add_test(lookup, t_point_lookup_miss); - suite_add_tcase(unit, lookup); - - - TCase *sampling = tcase_create("de:WIRS::WIRSQuery Testing"); - tcase_add_test(sampling, t_wirs_query); - tcase_add_test(sampling, t_wirs_query_merge); - tcase_add_test(sampling, t_wirs_buffer_query_rejection); - tcase_add_test(sampling, t_wirs_buffer_query_scan); - suite_add_tcase(unit, sampling); - - return unit; -} - - -int shard_unit_tests() -{ - int failed = 0; - Suite *unit = unit_testing(); - SRunner *unit_shardner = srunner_create(unit); - - srunner_run_all(unit_shardner, CK_NORMAL); - failed = srunner_ntests_failed(unit_shardner); - srunner_free(unit_shardner); - - return failed; -} - - -int main() -{ - int unit_failed = shard_unit_tests(); - - return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; -} -- cgit v1.2.3 From cc415c7c100a17c4e944915aeab01be99b14adb9 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 7 Nov 2023 13:36:11 -0500 Subject: Switched default DE test case over to ISAMTree --- tests/de_level_tag.cpp | 5 +- tests/de_level_tomb.cpp | 5 +- tests/de_tier_tag.cpp | 5 +- tests/de_tier_tomb.cpp | 5 +- tests/dynamic_extension_tests.inc | 206 ++++++++++++++++---------------------- tests/internal_level_tests.cpp | 13 +-- 6 files changed, 103 insertions(+), 136 deletions(-) (limited to 'tests') diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index 822ccc2..f3f099d 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -15,11 +15,12 @@ #include "testing.h" #include "framework/DynamicExtension.h" -#include "shard/WIRS.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; #include "dynamic_extension_tests.inc" diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 264b5b0..912743c 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -15,11 +15,12 @@ #include "testing.h" #include "framework/DynamicExtension.h" -#include "shard/WIRS.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "dynamic_extension_tests.inc" diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 2c263eb..712c203 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -16,11 +16,12 @@ #include "testing.h" #include "framework/DynamicExtension.h" #include "framework/scheduling/SerialScheduler.h" -#include "shard/WIRS.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; #include "dynamic_extension_tests.inc" diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index 509494a..f25ec34 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -15,11 +15,12 @@ #include "testing.h" #include "framework/DynamicExtension.h" -#include "shard/WIRS.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" #include using namespace de; -typedef DynamicExtension, WIRSQuery, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "dynamic_extension_tests.inc" diff --git a/tests/dynamic_extension_tests.inc b/tests/dynamic_extension_tests.inc index 2f82e92..aa12e31 100644 --- a/tests/dynamic_extension_tests.inc +++ b/tests/dynamic_extension_tests.inc @@ -12,75 +12,74 @@ START_TEST(t_create) { - auto ext_wirs = new DE(100, 2, 1); + auto test_de = new DE(100, 2, 1); + ck_assert_ptr_nonnull(test_de); + ck_assert_int_eq(test_de->get_record_count(), 0); + ck_assert_int_eq(test_de->get_height(), 0); - ck_assert_ptr_nonnull(ext_wirs); - ck_assert_int_eq(ext_wirs->get_record_count(), 0); - ck_assert_int_eq(ext_wirs->get_height(), 0); - - delete ext_wirs; + delete test_de; } END_TEST START_TEST(t_insert) { - auto ext_wirs = new DE(100, 2, 1); + auto test_de = new DE(100, 2, 1); uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<100; i++) { - WRec r = {key, val, 1}; - ck_assert_int_eq(ext_wirs->insert(r), 1); + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); key++; val++; } - ck_assert_int_eq(ext_wirs->get_height(), 0); - ck_assert_int_eq(ext_wirs->get_record_count(), 100); + ck_assert_int_eq(test_de->get_height(), 0); + ck_assert_int_eq(test_de->get_record_count(), 100); - delete ext_wirs; + delete test_de; } END_TEST START_TEST(t_debug_insert) { - auto ext_wirs = new DE(100, 2, 1); + auto test_de = new DE(100, 2, 1); uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<1000; i++) { - WRec r = {key, val, 1}; - ck_assert_int_eq(ext_wirs->insert(r), 1); - ck_assert_int_eq(ext_wirs->get_record_count(), i+1); + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + ck_assert_int_eq(test_de->get_record_count(), i+1); key++; val++; } - delete ext_wirs; + delete test_de; } END_TEST START_TEST(t_insert_with_mem_merges) { - auto ext_wirs = new DE(100, 2, 1); + auto test_de = new DE(100, 2, 1); uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<300; i++) { - WRec r = {key, val, 1}; - ck_assert_int_eq(ext_wirs->insert(r), 1); + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); key++; val++; } - ck_assert_int_eq(ext_wirs->get_record_count(), 300); - ck_assert_int_eq(ext_wirs->get_height(), 1); + ck_assert_int_eq(test_de->get_record_count(), 300); + ck_assert_int_eq(test_de->get_height(), 1); - delete ext_wirs; + delete test_de; } END_TEST @@ -88,13 +87,13 @@ END_TEST /* START_TEST(t_range_sample_memtable) { - auto ext_wirs = new DE(100, 2, 1); + auto test_de = new DE(100, 2, 1); uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<100; i++) { - WRec r = {key, val, 1}; - ck_assert_int_eq(ext_wirs->insert(r), 1); + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); key++; val++; } @@ -104,9 +103,9 @@ START_TEST(t_range_sample_memtable) char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - WRec sample_set[100]; + Rec sample_set[100]; - ext_wirs->range_sample(sample_set, lower_bound, upper_bound, 100); + test_de->range_sample(sample_set, lower_bound, upper_bound, 100); for(size_t i=0; i<100; i++) { ck_assert_int_le(sample_set[i].key, upper_bound); @@ -116,20 +115,20 @@ START_TEST(t_range_sample_memtable) free(buf); free(util_buf); - delete ext_wirs; + delete test_de; } END_TEST START_TEST(t_range_sample_memlevels) { - auto ext_wirs = new DE(100, 2, 1); + auto test_de = new DE(100, 2, 1); uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<300; i++) { - WRec r = {key, val, 1}; - ck_assert_int_eq(ext_wirs->insert(r), 1); + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); key++; val++; } @@ -140,8 +139,8 @@ START_TEST(t_range_sample_memlevels) char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - WRec sample_set[100]; - ext_wirs->range_sample(sample_set, lower_bound, upper_bound, 100); + Rec sample_set[100]; + test_de->range_sample(sample_set, lower_bound, upper_bound, 100); for(size_t i=0; i<100; i++) { ck_assert_int_le(sample_set[i].key, upper_bound); @@ -151,33 +150,19 @@ START_TEST(t_range_sample_memlevels) free(buf); free(util_buf); - delete ext_wirs; + delete test_de; } END_TEST */ -START_TEST(t_range_sample_weighted) +START_TEST(t_range_query) { - auto ext_wirs = new DE(100, 2, 1); + auto test_de = new DE(100, 2, 1); size_t n = 10000; std::vector keys; - - uint64_t key = 1; - for (size_t i=0; i< n / 2; i++) { - keys.push_back(key); - } - - // put in a quarter of the count with weight two. - key = 2; - for (size_t i=0; i< n / 4; i++) { - keys.push_back(key); - } - - // the remaining quarter with weight four. - key = 3; - for (size_t i=0; i< n / 4; i++) { - keys.push_back(key); + for (size_t i=0; iinsert(r); + Rec r = {keys[i], (uint32_t) i}; + ck_assert_int_eq(test_de->insert(r), 1); } - size_t k = 1000; - uint64_t lower_key = 0; - uint64_t upper_key = 5; - size_t cnt[3] = {0}; - size_t total_samples = 0; + std::sort(keys.begin(), keys.end()); + + auto idx = rand() % (keys.size() - 250); - wirs_query_parms p; + uint64_t lower_key = keys[idx]; + uint64_t upper_key = keys[idx + 250]; + + rq::Parms p; p.lower_bound = lower_key; p.upper_bound = upper_key; - p.sample_size = k; - p.rng = gsl_rng_alloc(gsl_rng_mt19937); - - for (size_t i=0; i<1000; i++) { - auto result = ext_wirs->query(&p); - auto r = result.get(); - total_samples += r.size(); + auto result = test_de->query(&p); + auto r = result.get(); + std::sort(r.begin(), r.end()); + ck_assert_int_eq(r.size(), 251); - for (size_t j=0; jinsert(r), 1); + Rec r = {rec.first, rec.second}; + ck_assert_int_eq(test_de->insert(r), 1); if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { std::vector> del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; ierase(dr); + Rec dr = {del_vec[i].first, del_vec[i].second}; + test_de->erase(dr); deletes++; to_delete.erase(del_vec[i]); deleted.insert(del_vec[i]); @@ -274,24 +242,24 @@ START_TEST(t_tombstone_merging_01) to_delete.insert(rec); } - ck_assert(ext_wirs->validate_tombstone_proportion()); + ck_assert(test_de->validate_tombstone_proportion()); } - ck_assert(ext_wirs->validate_tombstone_proportion()); + ck_assert(test_de->validate_tombstone_proportion()); gsl_rng_free(rng); - delete ext_wirs; + delete test_de; } END_TEST DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { auto rng = gsl_rng_alloc(gsl_rng_mt19937); - auto ext_wirs = new DE(1000, 2, 1); + auto test_de = new DE(1000, 2, 1); - std::set records; - std::set to_delete; - std::set deleted; + std::set records; + std::set to_delete; + std::set deleted; while (records.size() < reccnt) { uint64_t key = rand(); @@ -304,14 +272,14 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { size_t deletes = 0; for (auto rec : records) { - ck_assert_int_eq(ext_wirs->insert(rec), 1); + ck_assert_int_eq(test_de->insert(rec), 1); if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; + std::vector del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; ierase(del_vec[i]); + test_de->erase(del_vec[i]); deletes++; to_delete.erase(del_vec[i]); deleted.insert(del_vec[i]); @@ -325,7 +293,7 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { gsl_rng_free(rng); - return ext_wirs; + return test_de; } START_TEST(t_static_structure) @@ -333,11 +301,11 @@ START_TEST(t_static_structure) auto rng = gsl_rng_alloc(gsl_rng_mt19937); size_t reccnt = 100000; - auto ext_wirs = new DE(100, 2, 1); + auto test_de = new DE(100, 2, 1); - std::set records; - std::set to_delete; - std::set deleted; + std::set records; + std::set to_delete; + std::set deleted; while (records.size() < reccnt) { uint64_t key = rand(); @@ -345,7 +313,7 @@ START_TEST(t_static_structure) if (records.find({key, val}) != records.end()) continue; - records.insert({key, val, 1}); + records.insert({key, val}); } size_t deletes = 0; @@ -353,15 +321,15 @@ START_TEST(t_static_structure) size_t k=0; for (auto rec : records) { k++; - ck_assert_int_eq(ext_wirs->insert(rec), 1); + ck_assert_int_eq(test_de->insert(rec), 1); t_reccnt++; if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; + std::vector del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; ierase(del_vec[i]), 1); + ck_assert_int_eq(test_de->erase(del_vec[i]), 1); deletes++; to_delete.erase(del_vec[i]); @@ -374,7 +342,7 @@ START_TEST(t_static_structure) } } - auto flat = ext_wirs->create_static_structure(); + auto flat = test_de->create_static_structure(); ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); uint64_t prev_key = 0; @@ -386,7 +354,7 @@ START_TEST(t_static_structure) gsl_rng_free(rng); delete flat; - delete ext_wirs; + delete test_de; } END_TEST @@ -399,21 +367,15 @@ Suite *unit_testing() tcase_add_test(create, t_create); suite_add_tcase(unit, create); - TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); + TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); tcase_add_test(insert, t_insert); tcase_add_test(insert, t_insert_with_mem_merges); tcase_add_test(insert, t_debug_insert); suite_add_tcase(unit, insert); - TCase *sampling = tcase_create("de::DynamicExtension::range_sample Testing"); - - tcase_add_test(sampling, t_range_sample_weighted); - suite_add_tcase(unit, sampling); - - /* - tcase_add_test(sampling, t_range_sample_memtable); - tcase_add_test(sampling, t_range_sample_memlevels); - */ + TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); + tcase_add_test(query, t_range_query); + suite_add_tcase(unit, query); TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); tcase_add_test(ts, t_tombstone_merging_01); diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index 056e458..1009ea8 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -9,7 +9,8 @@ * All rights reserved. Published under the Modified BSD License. * */ -#include "shard/WIRS.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" #include "framework/structure/InternalLevel.h" #include "framework/interface/Record.h" #include "framework/interface/Query.h" @@ -21,12 +22,12 @@ using namespace de; -typedef InternalLevel, WIRSQuery> ILevel; +typedef InternalLevel, rq::Query, Rec>> ILevel; START_TEST(t_memlevel_merge) { - auto tbl1 = create_test_mbuffer(100); - auto tbl2 = create_test_mbuffer(100); + auto tbl1 = create_test_mbuffer(100); + auto tbl2 = create_test_mbuffer(100); auto base_level = new ILevel(1, 1); base_level->append_buffer(tbl1); @@ -48,8 +49,8 @@ START_TEST(t_memlevel_merge) ILevel *create_test_memlevel(size_t reccnt) { - auto tbl1 = create_test_mbuffer(reccnt/2); - auto tbl2 = create_test_mbuffer(reccnt/2); + auto tbl1 = create_test_mbuffer(reccnt/2); + auto tbl2 = create_test_mbuffer(reccnt/2); auto base_level = new ILevel(1, 2); base_level->append_buffer(tbl1); -- cgit v1.2.3 From 357cab549c2ed33970562b84ff6f83923742343d Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 7 Nov 2023 15:34:24 -0500 Subject: Comment and License updates --- tests/alias_tests.cpp | 2 +- tests/augbtree_tests.cpp | 4 ++-- tests/de_cc_isam_level.cpp | 4 ++-- tests/de_level_tag.cpp | 4 ++-- tests/de_level_tomb.cpp | 4 ++-- tests/de_tier_tag.cpp | 4 ++-- tests/de_tier_tomb.cpp | 4 ++-- tests/dynamic_extension_tests.inc | 4 ++-- tests/internal_level_tests.cpp | 2 +- tests/memisam_tests.cpp | 2 +- tests/mutable_buffer_tests.cpp | 2 +- tests/pgm_tests.cpp | 2 +- tests/testing.h | 2 +- tests/triespline_tests.cpp | 2 +- tests/vptree_tests.cpp | 2 +- 15 files changed, 22 insertions(+), 22 deletions(-) (limited to 'tests') diff --git a/tests/alias_tests.cpp b/tests/alias_tests.cpp index c4a302d..e3c736b 100644 --- a/tests/alias_tests.cpp +++ b/tests/alias_tests.cpp @@ -6,7 +6,7 @@ * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ diff --git a/tests/augbtree_tests.cpp b/tests/augbtree_tests.cpp index 878af82..1b24344 100644 --- a/tests/augbtree_tests.cpp +++ b/tests/augbtree_tests.cpp @@ -1,12 +1,12 @@ /* - * tests/wirs_tests.cpp + * tests/augbtree_tests.cpp * * Unit tests for AugBTree (Augmented B+Tree) shard * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ diff --git a/tests/de_cc_isam_level.cpp b/tests/de_cc_isam_level.cpp index e574578..4972fb5 100644 --- a/tests/de_cc_isam_level.cpp +++ b/tests/de_cc_isam_level.cpp @@ -1,11 +1,11 @@ /* - * tests/dynamic_extension_tests.cpp + * tests/de_cc_isam_level.cpp * * Unit tests for Dynamic Extension Framework * * Copyright (C) 2023 Douglas B. Rumbaugh * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index f3f099d..3a2f271 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -1,12 +1,12 @@ /* - * tests/dynamic_extension_tests.cpp + * tests/de_level_tag.cpp * * Unit tests for Dynamic Extension Framework * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #include diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 912743c..2412959 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -1,12 +1,12 @@ /* - * tests/dynamic_extension_tests.cpp + * tests/de_level_tomb.cpp * * Unit tests for Dynamic Extension Framework * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #include diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 712c203..46f2ee7 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -1,12 +1,12 @@ /* - * tests/dynamic_extension_tests.cpp + * tests/de_tier_tag.cpp * * Unit tests for Dynamic Extension Framework * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #include diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index f25ec34..e2b65a0 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -1,12 +1,12 @@ /* - * tests/dynamic_extension_tests.cpp + * tests/de_tier_tomb.cpp * * Unit tests for Dynamic Extension Framework * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #include diff --git a/tests/dynamic_extension_tests.inc b/tests/dynamic_extension_tests.inc index bcb5ae6..dac7d51 100644 --- a/tests/dynamic_extension_tests.inc +++ b/tests/dynamic_extension_tests.inc @@ -1,12 +1,12 @@ /* - * tests/dynamic_extension_tests.cpp + * tests/dynamic_extension_tests.inc * * Unit tests for Dynamic Extension Framework * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index 1009ea8..cb74bca 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -6,7 +6,7 @@ * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #include "shard/ISAMTree.h" diff --git a/tests/memisam_tests.cpp b/tests/memisam_tests.cpp index d3b8087..8c499e2 100644 --- a/tests/memisam_tests.cpp +++ b/tests/memisam_tests.cpp @@ -6,7 +6,7 @@ * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index a2561c8..8480f55 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -6,7 +6,7 @@ * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #include diff --git a/tests/pgm_tests.cpp b/tests/pgm_tests.cpp index 1565384..c7750ac 100644 --- a/tests/pgm_tests.cpp +++ b/tests/pgm_tests.cpp @@ -6,7 +6,7 @@ * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ diff --git a/tests/testing.h b/tests/testing.h index 023be7f..e1d3402 100644 --- a/tests/testing.h +++ b/tests/testing.h @@ -6,7 +6,7 @@ * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #pragma once diff --git a/tests/triespline_tests.cpp b/tests/triespline_tests.cpp index 101f143..14506be 100644 --- a/tests/triespline_tests.cpp +++ b/tests/triespline_tests.cpp @@ -6,7 +6,7 @@ * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ diff --git a/tests/vptree_tests.cpp b/tests/vptree_tests.cpp index 06f147b..fb568dd 100644 --- a/tests/vptree_tests.cpp +++ b/tests/vptree_tests.cpp @@ -5,7 +5,7 @@ * * Copyright (C) 2023 Douglas Rumbaugh * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ -- cgit v1.2.3 From 90bb0614fc1d8f1a185a778e31aaf9027c01aeb8 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 13 Nov 2023 11:44:09 -0500 Subject: Tombstone Compaction: re-enabled tombstone compaction Currently, proactive buffer tombstone compaction is disabled by forcing the buffer tombstone capacity to match its record capacity. It isn't clear how to best handle proactive buffer compactions in an environment where new buffers are spawned anyway. --- tests/dynamic_extension_tests.inc | 2 -- 1 file changed, 2 deletions(-) (limited to 'tests') diff --git a/tests/dynamic_extension_tests.inc b/tests/dynamic_extension_tests.inc index dac7d51..be82132 100644 --- a/tests/dynamic_extension_tests.inc +++ b/tests/dynamic_extension_tests.inc @@ -245,8 +245,6 @@ START_TEST(t_tombstone_merging_01) if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { to_delete.insert(rec); } - - ck_assert(test_de->validate_tombstone_proportion()); } test_de->await_next_epoch(); -- cgit v1.2.3 From 3c127eda69295cb306739bdd3c5ddccff6026a8d Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 13 Dec 2023 12:39:54 -0500 Subject: Refactoring: corrected a number of names and added more comments --- tests/internal_level_tests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index cb74bca..9299c07 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -37,7 +37,7 @@ START_TEST(t_memlevel_merge) merging_level->append_buffer(tbl2); ck_assert_int_eq(merging_level->get_record_count(), 100); - auto new_level = ILevel::merge_levels(base_level, merging_level); + auto new_level = ILevel::reconstruction(base_level, merging_level); delete merging_level; ck_assert_int_eq(new_level->get_record_count(), 200); @@ -66,7 +66,7 @@ Suite *unit_testing() { Suite *unit = suite_create("InternalLevel Unit Testing"); - TCase *merge = tcase_create("de::InternalLevel::merge_level Testing"); + TCase *merge = tcase_create("de::InternalLevel::reconstruction Testing"); tcase_add_test(merge, t_memlevel_merge); suite_add_tcase(unit, merge); -- cgit v1.2.3 From 53879a0d69f5e578710b7125e9b41e516c2371d4 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 10 Jan 2024 17:39:28 -0500 Subject: MutableBuffer+View: Implementation with unit tests --- tests/mutable_buffer_tests.cpp | 254 ++++++++++++++++++----------------------- 1 file changed, 113 insertions(+), 141 deletions(-) (limited to 'tests') diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index 8480f55..e714d3e 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -1,7 +1,7 @@ /* * tests/mutable_buffer_tests.cpp * - * Unit tests for MutableBuffer + * Unit tests for MutableBuffer and BufferView * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie @@ -9,7 +9,7 @@ * Distributed under the Modified BSD License. * */ -#include + #include #include #include @@ -25,15 +25,26 @@ using namespace de; START_TEST(t_create) { - auto buffer = new MutableBuffer(100, 50); + size_t lwm = 50, hwm = 100; + size_t cap = 2 * hwm; + + auto buffer = new MutableBuffer(lwm, hwm); ck_assert_ptr_nonnull(buffer); - ck_assert_int_eq(buffer->get_capacity(), 100); - ck_assert_int_eq(buffer->get_record_count(), 0); + ck_assert_int_eq(buffer->get_capacity(), cap); + ck_assert_int_eq(buffer->get_low_watermark(), lwm); + ck_assert_int_eq(buffer->get_high_watermark(), hwm); + ck_assert_int_eq(buffer->is_full(), false); - ck_assert_ptr_nonnull(buffer->get_data()); + ck_assert_int_eq(buffer->is_at_low_watermark(), false); + ck_assert_int_eq(buffer->get_record_count(), 0); ck_assert_int_eq(buffer->get_tombstone_count(), 0); - ck_assert_int_eq(buffer->get_tombstone_capacity(), 50); + + { + auto view = buffer->get_buffer_view(); + ck_assert_int_eq(view.get_tombstone_count(), 0); + ck_assert_int_eq(view.get_record_count(), 0); + } delete buffer; } @@ -42,74 +53,47 @@ END_TEST START_TEST(t_insert) { - auto buffer = new MutableBuffer(100, 50); + auto buffer = new MutableBuffer(50, 100); - uint64_t key = 0; - uint32_t val = 5; + Rec rec = {0, 5, 1}; - WRec rec = {0, 5, 1}; - - for (size_t i=0; i<99; i++) { + /* insert records up to the low watermark */ + size_t cnt = 0; + for (size_t i=0; i<50; i++) { ck_assert_int_eq(buffer->append(rec), 1); ck_assert_int_eq(buffer->check_tombstone(rec), 0); + ck_assert_int_eq(buffer->is_at_low_watermark(), false); rec.key++; rec.value++; + cnt++; - ck_assert_int_eq(buffer->get_record_count(), i+1); - ck_assert_int_eq(buffer->get_tombstone_count(), 0); - ck_assert_int_eq(buffer->is_full(), 0); + ck_assert_int_eq(buffer->get_record_count(), cnt); + ck_assert_int_eq(buffer->get_buffer_view().get_record_count(), cnt); + ck_assert_int_eq(buffer->get_tail(), cnt); } - ck_assert_int_eq(buffer->append(rec), 1); - - rec.key++; - rec.value++; - - ck_assert_int_eq(buffer->is_full(), 1); - ck_assert_int_eq(buffer->append(rec), 0); - - delete buffer; - -} -END_TEST - - -START_TEST(t_insert_tombstones) -{ - auto buffer = new MutableBuffer(100, 50); - - size_t ts_cnt = 0; - - Rec rec = {0, 5}; - - for (size_t i=0; i<99; i++) { - bool ts = false; - if (i % 2 == 0) { - ts_cnt++; - ts=true; - } - - ck_assert_int_eq(buffer->append(rec, ts), 1); - ck_assert_int_eq(buffer->check_tombstone(rec), ts); + /* insert records up to the high watermark */ + for (size_t i=0; i<50; i++) { + ck_assert_int_eq(buffer->is_full(), 0); + ck_assert_int_eq(buffer->append(rec), 1); + ck_assert_int_eq(buffer->check_tombstone(rec), 0); rec.key++; rec.value++; + cnt++; - ck_assert_int_eq(buffer->get_record_count(), i+1); - ck_assert_int_eq(buffer->get_tombstone_count(), ts_cnt); - ck_assert_int_eq(buffer->is_full(), 0); - } - - // inserting one more tombstone should not be possible - ck_assert_int_eq(buffer->append(rec, true), 0); + ck_assert_int_eq(buffer->get_record_count(), cnt); + ck_assert_int_eq(buffer->get_buffer_view().get_record_count(), cnt); + ck_assert_int_eq(buffer->get_tombstone_count(), 0); + ck_assert_int_eq(buffer->is_at_low_watermark(), true); + ck_assert_int_eq(buffer->get_tail(), cnt); + } - ck_assert_int_eq(buffer->append(rec), 1); - + /* further inserts should fail */ rec.key++; rec.value++; - ck_assert_int_eq(buffer->is_full(), 1); ck_assert_int_eq(buffer->append(rec), 0); @@ -118,88 +102,51 @@ START_TEST(t_insert_tombstones) END_TEST -START_TEST(t_truncate) +START_TEST(t_advance_head) { - auto buffer = new MutableBuffer(100, 100); + auto buffer = new MutableBuffer(50, 100); - size_t ts_cnt = 0; - Rec rec = {0, 5}; - - for (size_t i=0; i<100; i++) { - bool ts = false; - if (i % 2 == 0) { - ts_cnt++; - ts=true; - } - - ck_assert_int_eq(buffer->append(rec, ts), 1); - ck_assert_int_eq(buffer->check_tombstone(rec), ts); + /* insert 75 records and get tail when LWM is exceeded */ + size_t new_head = 0; + Rec rec = {1, 1}; + size_t cnt = 0; + for (size_t i=0; i<75; i++) { + ck_assert_int_eq(buffer->append(rec), 1); rec.key++; rec.value++; + cnt++; - ck_assert_int_eq(buffer->get_record_count(), i+1); - ck_assert_int_eq(buffer->get_tombstone_count(), ts_cnt); - } - - ck_assert_int_eq(buffer->is_full(), 1); - ck_assert_int_eq(buffer->append(rec), 0); - - ck_assert_int_eq(buffer->truncate(), 1); - - ck_assert_int_eq(buffer->is_full(), 0); - ck_assert_int_eq(buffer->get_record_count(), 0); - ck_assert_int_eq(buffer->get_tombstone_count(), 0); - ck_assert_int_eq(buffer->append(rec), 1); - - delete buffer; - -} -END_TEST - - -START_TEST(t_get_data) -{ - size_t cnt = 100; - - auto buffer = new MutableBuffer(cnt, cnt/2); - - - std::vector keys(cnt); - for (size_t i=0; iis_at_low_watermark() && new_head == 0) { + new_head = buffer->get_tail() - 1; + } } - // duplicate final two records for tombstone testing - // purposes - keys[cnt-2] = keys[cnt-3]; - keys[cnt-1] = keys[cnt-2]; - - uint32_t val = 12345; - for (size_t i=0; iappend(Rec {keys[i], val}); + ck_assert_int_eq(buffer->get_available_capacity(), 200 - cnt); + + Wrapped *view_records = new Wrapped[buffer->get_record_count()]; + { + /* get a view of the pre-advanced state */ + auto view = buffer->get_buffer_view(); + ck_assert_int_eq(view.get_record_count(), cnt); + view.copy_to_buffer((psudb::byte *) view_records); + + /* advance the head */ + buffer->advance_head(new_head); + ck_assert_int_eq(buffer->get_record_count(), 25); + ck_assert_int_eq(buffer->get_buffer_view().get_record_count(), 25); + ck_assert_int_eq(view.get_record_count(), cnt); + ck_assert_int_eq(buffer->get_available_capacity(), 200 - cnt); } - Rec r1 = {keys[cnt-2], val}; - buffer->append(r1, true); - - Rec r2 = {keys[cnt-1], val}; - buffer->append(r2, true); - - - auto *sorted_records = buffer->get_data(); - std::sort(keys.begin(), keys.end()); - std::sort(sorted_records, sorted_records + buffer->get_record_count(), std::less>()); - - for (size_t i=0; iget_available_capacity(), 175); delete buffer; + delete[] view_records; } END_TEST - void insert_records(std::vector> *values, size_t start, size_t stop, MutableBuffer *buffer) { for (size_t i=start; i> *values, size_t s } -#if DE_MT_TEST +/* START_TEST(t_multithreaded_insert) { size_t cnt = 10000; - auto buffer = new MutableBuffer(cnt, true, cnt/2); + auto buffer = new MutableBuffer(cnt/2, cnt); std::vector records(cnt); for (size_t i=0; i workers(thread_cnt); @@ -239,17 +186,49 @@ START_TEST(t_multithreaded_insert) ck_assert_int_eq(buffer->is_full(), 1); ck_assert_int_eq(buffer->get_record_count(), cnt); +} +END_TEST +*/ - std::sort(records.begin(), records.end()); - auto *sorted_records = buffer->sorted_output(); - for (size_t i=0; i(100, 100); + + size_t ts_cnt = 0; + Rec rec = {0, 5}; + + for (size_t i=0; i<100; i++) { + bool ts = false; + if (i % 2 == 0) { + ts_cnt++; + ts=true; + } + + ck_assert_int_eq(buffer->append(rec, ts), 1); + ck_assert_int_eq(buffer->check_tombstone(rec), ts); + + rec.key++; + rec.value++; + + ck_assert_int_eq(buffer->get_record_count(), i+1); + ck_assert_int_eq(buffer->get_tombstone_count(), ts_cnt); } + ck_assert_int_eq(buffer->is_full(), 1); + ck_assert_int_eq(buffer->append(rec), 0); + + ck_assert_int_eq(buffer->truncate(), 1); + + ck_assert_int_eq(buffer->is_full(), 0); + ck_assert_int_eq(buffer->get_record_count(), 0); + ck_assert_int_eq(buffer->get_tombstone_count(), 0); + ck_assert_int_eq(buffer->append(rec), 1); + delete buffer; + } END_TEST -#endif Suite *unit_testing() @@ -263,10 +242,8 @@ Suite *unit_testing() TCase *append = tcase_create("de::MutableBuffer::append Testing"); tcase_add_test(append, t_insert); - tcase_add_test(append, t_insert_tombstones); - #if DE_MT_TEST - tcase_add_test(append, t_multithreaded_insert); - #endif + tcase_add_test(append, t_advance_head); + //tcase_add_test(append, t_multithreaded_insert); suite_add_tcase(unit, append); @@ -277,11 +254,6 @@ Suite *unit_testing() suite_add_tcase(unit, truncate); - TCase *sorted_out = tcase_create("de::MutableBuffer::get_data"); - tcase_add_test(sorted_out, t_get_data); - - suite_add_tcase(unit, sorted_out); - return unit; } -- cgit v1.2.3 From eb19677340be6f0befe9da2199e5832af51eea0d Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 10 Jan 2024 18:01:30 -0500 Subject: MutableBuffer: multithreaded insert test + bugfixes --- tests/mutable_buffer_tests.cpp | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'tests') diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index e714d3e..0520097 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -12,15 +12,12 @@ #include #include -#include #include "testing.h" #include "framework/structure/MutableBuffer.h" #include -#define DE_MT_TEST 0 - using namespace de; START_TEST(t_create) @@ -60,9 +57,9 @@ START_TEST(t_insert) /* insert records up to the low watermark */ size_t cnt = 0; for (size_t i=0; i<50; i++) { + ck_assert_int_eq(buffer->is_at_low_watermark(), false); ck_assert_int_eq(buffer->append(rec), 1); ck_assert_int_eq(buffer->check_tombstone(rec), 0); - ck_assert_int_eq(buffer->is_at_low_watermark(), false); rec.key++; rec.value++; @@ -73,6 +70,8 @@ START_TEST(t_insert) ck_assert_int_eq(buffer->get_tail(), cnt); } + ck_assert_int_eq(buffer->is_at_low_watermark(), true); + /* insert records up to the high watermark */ for (size_t i=0; i<50; i++) { ck_assert_int_eq(buffer->is_full(), 0); @@ -118,7 +117,7 @@ START_TEST(t_advance_head) cnt++; if (buffer->is_at_low_watermark() && new_head == 0) { - new_head = buffer->get_tail() - 1; + new_head = buffer->get_tail(); } } @@ -132,30 +131,38 @@ START_TEST(t_advance_head) view.copy_to_buffer((psudb::byte *) view_records); /* advance the head */ - buffer->advance_head(new_head); + ck_assert_int_eq(buffer->advance_head(new_head), 1); ck_assert_int_eq(buffer->get_record_count(), 25); ck_assert_int_eq(buffer->get_buffer_view().get_record_count(), 25); ck_assert_int_eq(view.get_record_count(), cnt); ck_assert_int_eq(buffer->get_available_capacity(), 200 - cnt); + + /* refuse to advance head again while there remain references to the old one */ + ck_assert_int_eq(buffer->advance_head(buffer->get_tail() -1), 0); } /* once the buffer view falls out of scope, the capacity of the buffer should increase */ ck_assert_int_eq(buffer->get_available_capacity(), 175); + /* now the head should be able to be advanced */ + ck_assert_int_eq(buffer->advance_head(buffer->get_tail()), 1); + + /* and the buffer should be empty */ + ck_assert_int_eq(buffer->get_record_count(), 0); + delete buffer; delete[] view_records; } END_TEST -void insert_records(std::vector> *values, size_t start, size_t stop, MutableBuffer *buffer) +void insert_records(std::vector *values, size_t start, size_t stop, MutableBuffer *buffer) { for (size_t i=start; iappend({(*values)[i].first, (*values)[i].second}); + buffer->append((*values)[i]); } } -/* START_TEST(t_multithreaded_insert) { size_t cnt = 10000; @@ -166,7 +173,7 @@ START_TEST(t_multithreaded_insert) records[i] = Rec {(uint64_t) rand(), (uint32_t) rand()}; } - // perform a multithreaded insertion + /* perform a multithreaded insertion */ size_t thread_cnt = 8; size_t per_thread = cnt / thread_cnt; std::vector workers(thread_cnt); @@ -186,9 +193,10 @@ START_TEST(t_multithreaded_insert) ck_assert_int_eq(buffer->is_full(), 1); ck_assert_int_eq(buffer->get_record_count(), cnt); + + delete buffer; } END_TEST -*/ START_TEST(t_truncate) @@ -243,7 +251,7 @@ Suite *unit_testing() TCase *append = tcase_create("de::MutableBuffer::append Testing"); tcase_add_test(append, t_insert); tcase_add_test(append, t_advance_head); - //tcase_add_test(append, t_multithreaded_insert); + tcase_add_test(append, t_multithreaded_insert); suite_add_tcase(unit, append); -- cgit v1.2.3 From 5db0f96e9f3d2505b5f751abc133cbf7e13b5129 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 11 Jan 2024 11:31:33 -0500 Subject: Fixed some potential buffer-related concurrency bugs --- tests/mutable_buffer_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index 0520097..e61a832 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -201,7 +201,7 @@ END_TEST START_TEST(t_truncate) { - auto buffer = new MutableBuffer(100, 100); + auto buffer = new MutableBuffer(50, 100); size_t ts_cnt = 0; Rec rec = {0, 5}; -- cgit v1.2.3 From 5a2c378aad3f1a9923db3191ffaa3fb807d392b2 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 11 Jan 2024 15:29:51 -0500 Subject: Ported ISAMTree over to new buffer setup I may still play with the shard from shards constructor, and queries need some work yet too. --- tests/memisam_tests.cpp | 123 +++++++++++++++++++++++++----------------------- tests/testing.h | 14 +++--- 2 files changed, 72 insertions(+), 65 deletions(-) (limited to 'tests') diff --git a/tests/memisam_tests.cpp b/tests/memisam_tests.cpp index 8c499e2..919fd69 100644 --- a/tests/memisam_tests.cpp +++ b/tests/memisam_tests.cpp @@ -11,7 +11,7 @@ */ #include "shard/ISAMTree.h" -#include "query/irs.h" +#include "query/rangequery.h" #include "testing.h" #include @@ -22,7 +22,7 @@ typedef ISAMTree Shard; START_TEST(t_mbuffer_init) { - auto buffer = new MutableBuffer(1024, 1024); + auto buffer = new MutableBuffer(512, 1024); for (uint64_t i = 512; i > 0; i--) { uint32_t v = i; buffer->append({i,v, 1}); @@ -38,7 +38,7 @@ START_TEST(t_mbuffer_init) buffer->append({i, v, 1}); } - Shard* shard = new Shard(buffer); + Shard* shard = new Shard(buffer->get_buffer_view()); ck_assert_uint_eq(shard->get_record_count(), 512); delete buffer; @@ -46,16 +46,16 @@ START_TEST(t_mbuffer_init) } -START_TEST(t_irs_init) +START_TEST(t_rq_init) { size_t n = 512; auto mbuffer1 = create_test_mbuffer(n); auto mbuffer2 = create_test_mbuffer(n); auto mbuffer3 = create_test_mbuffer(n); - auto shard1 = new Shard(mbuffer1); - auto shard2 = new Shard(mbuffer2); - auto shard3 = new Shard(mbuffer3); + auto shard1 = new Shard(mbuffer1->get_buffer_view()); + auto shard2 = new Shard(mbuffer2->get_buffer_view()); + auto shard3 = new Shard(mbuffer3->get_buffer_view()); Shard* shards[3] = {shard1, shard2, shard3}; auto shard4 = new Shard(shards, 3); @@ -101,18 +101,22 @@ START_TEST(t_point_lookup) size_t n = 10000; auto buffer = create_double_seq_mbuffer(n, false); - auto isam = Shard(buffer); + auto isam = Shard(buffer->get_buffer_view()); - for (size_t i=0; iget_data() + i); - r.key = rec->rec.key; - r.value = rec->rec.value; + { + auto view = buffer->get_buffer_view(); - auto result = isam.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); + for (size_t i=0; irec.key; + r.value = rec->rec.value; + + auto result = isam.point_lookup(r); + ck_assert_ptr_nonnull(result); + ck_assert_int_eq(result->rec.key, r.key); + ck_assert_int_eq(result->rec.value, r.value); + } } delete buffer; @@ -125,7 +129,7 @@ START_TEST(t_point_lookup_miss) size_t n = 10000; auto buffer = create_double_seq_mbuffer(n, false); - auto isam = Shard(buffer); + auto isam = Shard(buffer->get_buffer_view()); for (size_t i=n + 100; i<2*n; i++) { Rec r; @@ -146,8 +150,8 @@ START_TEST(t_full_cancelation) auto buffer = create_double_seq_mbuffer(n, false); auto buffer_ts = create_double_seq_mbuffer(n, true); - Shard* shard = new Shard(buffer); - Shard* shard_ts = new Shard(buffer_ts); + Shard* shard = new Shard(buffer->get_buffer_view()); + Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); ck_assert_int_eq(shard->get_record_count(), n); ck_assert_int_eq(shard->get_tombstone_count(), 0); @@ -170,11 +174,12 @@ START_TEST(t_full_cancelation) END_TEST -START_TEST(t_irs_query) +/* +START_TEST(t_rq_query) { size_t n=1000; auto buffer = create_double_seq_mbuffer(n); - auto isam = Shard(buffer); + auto isam = Shard(buffer->get_buffer_view()); uint64_t lower_key = 100; uint64_t upper_key = 250; @@ -182,15 +187,15 @@ START_TEST(t_irs_query) size_t k = 100; size_t cnt[3] = {0}; - irs::Parms parms = {lower_key, upper_key, k}; + rq::Parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; for (size_t i=0; i<1000; i++) { - auto state = irs::Query::get_query_state(&isam, &parms); - ((irs::State *) state)->sample_size = k; - auto result = irs::Query::query(&isam, state, &parms); + auto state = rq::Query::get_query_state(&isam, &parms); + ((rq::State *) state)->sample_size = k; + auto result = rq::Query::query(&isam, state, &parms); ck_assert_int_eq(result.size(), k); @@ -199,7 +204,7 @@ START_TEST(t_irs_query) ck_assert_int_ge(rec.rec.key, lower_key); } - irs::Query::delete_query_state(state); + rq::Query::delete_query_state(state); } gsl_rng_free(parms.rng); @@ -208,12 +213,12 @@ START_TEST(t_irs_query) END_TEST -START_TEST(t_irs_query_merge) +START_TEST(t_rq_query_merge) { size_t n=1000; auto buffer = create_double_seq_mbuffer(n); - Shard shard = Shard(buffer); + Shard shard = Shard(buffer->get_buffer_view()); uint64_t lower_key = 100; uint64_t upper_key = 250; @@ -221,25 +226,25 @@ START_TEST(t_irs_query_merge) size_t k = 1000; size_t cnt[3] = {0}; - irs::Parms parms = {lower_key, upper_key, k}; + rq::Parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); std::vector>> results(2); for (size_t i=0; i<1000; i++) { - auto state1 = irs::Query::get_query_state(&shard, &parms); - ((irs::State *) state1)->sample_size = k; - results[0] = irs::Query::query(&shard, state1, &parms); + auto state1 = rq::Query::get_query_state(&shard, &parms); + ((rq::State *) state1)->sample_size = k; + results[0] = rq::Query::query(&shard, state1, &parms); - auto state2 = irs::Query::get_query_state(&shard, &parms); - ((irs::State *) state2)->sample_size = k; - results[1] = irs::Query::query(&shard, state2, &parms); + auto state2 = rq::Query::get_query_state(&shard, &parms); + ((rq::State *) state2)->sample_size = k; + results[1] = rq::Query::query(&shard, state2, &parms); - irs::Query::delete_query_state(state1); - irs::Query::delete_query_state(state2); + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); } - auto merged = irs::Query::merge(results, nullptr); + auto merged = rq::Query::merge(results, nullptr); ck_assert_int_eq(merged.size(), 2*k); for (size_t i=0; i(n); @@ -264,15 +269,15 @@ START_TEST(t_irs_buffer_query_scan) size_t k = 100; size_t cnt[3] = {0}; - irs::Parms parms = {lower_key, upper_key, k}; + rq::Parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; for (size_t i=0; i<1000; i++) { - auto state = irs::Query::get_buffer_query_state(buffer, &parms); - ((irs::BufferState *) state)->sample_size = k; - auto result = irs::Query::buffer_query(buffer, state, &parms); + auto state = rq::Query::get_buffer_query_state(buffer, &parms); + ((rq::BufferState *) state)->sample_size = k; + auto result = rq::Query::buffer_query(buffer, state, &parms); ck_assert_int_eq(result.size(), k); @@ -281,7 +286,7 @@ START_TEST(t_irs_buffer_query_scan) ck_assert_int_ge(rec.rec.key, lower_key); } - irs::Query::delete_buffer_query_state(state); + rq::Query::delete_buffer_query_state(state); } gsl_rng_free(parms.rng); @@ -290,7 +295,7 @@ START_TEST(t_irs_buffer_query_scan) END_TEST -START_TEST(t_irs_buffer_query_rejection) +START_TEST(t_rq_buffer_query_rejection) { size_t n=1000; auto buffer = create_double_seq_mbuffer(n); @@ -301,15 +306,15 @@ START_TEST(t_irs_buffer_query_rejection) size_t k = 10000; size_t cnt[3] = {0}; - irs::Parms parms = {lower_key, upper_key, k}; + rq::Parms parms = {lower_key, upper_key, k}; parms.rng = gsl_rng_alloc(gsl_rng_mt19937); size_t total_samples = 0; for (size_t i=0; i<1000; i++) { - auto state = irs::Query::get_buffer_query_state(buffer, &parms); - ((irs::BufferState *) state)->sample_size = k; - auto result = irs::Query::buffer_query(buffer, state, &parms); + auto state = rq::Query::get_buffer_query_state(buffer, &parms); + ((rq::BufferState *) state)->sample_size = k; + auto result = rq::Query::buffer_query(buffer, state, &parms); ck_assert_int_gt(result.size(), 0); ck_assert_int_le(result.size(), k); @@ -319,13 +324,14 @@ START_TEST(t_irs_buffer_query_rejection) ck_assert_int_ge(rec.rec.key, lower_key); } - irs::Query::delete_buffer_query_state(state); + rq::Query::delete_buffer_query_state(state); } gsl_rng_free(parms.rng); delete buffer; } END_TEST +*/ Suite *unit_testing() @@ -334,7 +340,7 @@ Suite *unit_testing() TCase *create = tcase_create("de::ISAMTree constructor Testing"); tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_irs_init); + tcase_add_test(create, t_rq_init); tcase_set_timeout(create, 100); suite_add_tcase(unit, create); @@ -349,14 +355,15 @@ Suite *unit_testing() tcase_add_test(lookup, t_point_lookup_miss); suite_add_tcase(unit, lookup); - - TCase *sampling = tcase_create("de:ISAMTree::ISAMTreeQuery Testing"); - tcase_add_test(sampling, t_irs_query); - tcase_add_test(sampling, t_irs_query_merge); - tcase_add_test(sampling, t_irs_buffer_query_rejection); - tcase_add_test(sampling, t_irs_buffer_query_scan); + /* + TCase *sampling = tcase_create("de:ISAMTree::IRS Testing"); + tcase_add_test(sampling, t_rq_query); + tcase_add_test(sampling, t_rq_query_merge); + tcase_add_test(sampling, t_rq_buffer_query_rejection); + tcase_add_test(sampling, t_rq_buffer_query_scan); tcase_set_timeout(sampling, 100); suite_add_tcase(unit, sampling); + */ return unit; } diff --git a/tests/testing.h b/tests/testing.h index e1d3402..4e660dd 100644 --- a/tests/testing.h +++ b/tests/testing.h @@ -77,7 +77,7 @@ static bool roughly_equal(int n1, int n2, size_t mag, double epsilon) { } static de::MutableBuffer *create_2d_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt, cnt); + auto buffer = new de::MutableBuffer(cnt/2, cnt); for (int64_t i=0; iappend({rand(), rand()}); @@ -87,7 +87,7 @@ static de::MutableBuffer *create_2d_mbuffer(size_t cnt) { } static de::MutableBuffer *create_2d_sequential_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt, cnt); + auto buffer = new de::MutableBuffer(cnt/2, cnt); for (int64_t i=0; iappend({i, i}); } @@ -98,7 +98,7 @@ static de::MutableBuffer *create_2d_sequential_mbuffer(size_t cnt) { template static de::MutableBuffer *create_test_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt, cnt); + auto buffer = new de::MutableBuffer(cnt/2, cnt); R rec; for (size_t i = 0; i < cnt; i++) { @@ -119,7 +119,7 @@ template static de::MutableBuffer *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop) { size_t cnt = stop - start; - auto buffer = new de::MutableBuffer(cnt, cnt); + auto buffer = new de::MutableBuffer(cnt/2, cnt); for (size_t i=start; i *create_sequential_mbuffer(decltype(R::key) start, d template static de::MutableBuffer *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) { - auto buffer = new de::MutableBuffer(cnt, ts_cnt); + auto buffer = new de::MutableBuffer(cnt/2, cnt); std::vector> tombstones; @@ -171,7 +171,7 @@ template requires de::WeightedRecordInterface && de::KVPInterface static de::MutableBuffer *create_weighted_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt, cnt); + auto buffer = new de::MutableBuffer(cnt/2, cnt); // Put in half of the count with weight one. for (uint32_t i=0; i< cnt / 2; i++) { @@ -194,7 +194,7 @@ static de::MutableBuffer *create_weighted_mbuffer(size_t cnt) template static de::MutableBuffer *create_double_seq_mbuffer(size_t cnt, bool ts=false) { - auto buffer = new de::MutableBuffer(cnt, cnt); + auto buffer = new de::MutableBuffer(cnt/2, cnt); for (size_t i = 0; i < cnt / 2; i++) { R rec; -- cgit v1.2.3 From 7e503464176adbd0880373325e30a6bfd58616f0 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 11 Jan 2024 16:31:24 -0500 Subject: InternalLevel update and tests Plus some assorted fixes for move semantics stuff in BufferView that accompanied these changes. --- tests/internal_level_tests.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index 9299c07..b8aa56f 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -30,11 +30,11 @@ START_TEST(t_memlevel_merge) auto tbl2 = create_test_mbuffer(100); auto base_level = new ILevel(1, 1); - base_level->append_buffer(tbl1); + base_level->append_buffer(tbl1->get_buffer_view()); ck_assert_int_eq(base_level->get_record_count(), 100); auto merging_level = new ILevel(0, 1); - merging_level->append_buffer(tbl2); + merging_level->append_buffer(tbl2->get_buffer_view()); ck_assert_int_eq(merging_level->get_record_count(), 100); auto new_level = ILevel::reconstruction(base_level, merging_level); @@ -53,8 +53,8 @@ ILevel *create_test_memlevel(size_t reccnt) { auto tbl2 = create_test_mbuffer(reccnt/2); auto base_level = new ILevel(1, 2); - base_level->append_buffer(tbl1); - base_level->append_buffer(tbl2); + base_level->append_buffer(tbl1->get_buffer_view()); + base_level->append_buffer(tbl2->get_buffer_view()); delete tbl1; delete tbl2; -- cgit v1.2.3 From c4514c2e62a711189cf3c914297885d97fb51a09 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 12 Jan 2024 14:08:33 -0500 Subject: Initial pass at unit test refactoring Restructured unit tests to be a bit more modular. I have some further plans to expand on this, particular for the query tests (including both shard and framework level test functions that can be injected at will). --- tests/de_level_tag.cpp | 35 +++- tests/de_level_tomb.cpp | 38 +++- tests/de_tier_tag.cpp | 35 +++- tests/de_tier_tomb.cpp | 36 +++- tests/dynamic_extension_tests.inc | 416 -------------------------------------- tests/include/dynamic_extension.h | 341 +++++++++++++++++++++++++++++++ tests/include/rangequery.h | 179 ++++++++++++++++ tests/include/shard_standard.h | 198 ++++++++++++++++++ tests/include/testing.h | 218 ++++++++++++++++++++ tests/internal_level_tests.cpp | 2 +- tests/memisam_tests.cpp | 348 +------------------------------ tests/mutable_buffer_tests.cpp | 2 +- tests/rangequery_tests.cpp | 195 ++++++++++++++++++ tests/testing.h | 218 -------------------- 14 files changed, 1275 insertions(+), 986 deletions(-) delete mode 100644 tests/dynamic_extension_tests.inc create mode 100644 tests/include/dynamic_extension.h create mode 100644 tests/include/rangequery.h create mode 100644 tests/include/shard_standard.h create mode 100644 tests/include/testing.h create mode 100644 tests/rangequery_tests.cpp delete mode 100644 tests/testing.h (limited to 'tests') diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index 3a2f271..5c95aa2 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -13,7 +13,7 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" @@ -23,4 +23,35 @@ using namespace de; typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tagged Leveling Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 2412959..91ee26d 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -13,14 +13,46 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" +#include "shard/TrieSpline.h" #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tombstone Leveling Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 46f2ee7..9d4fe7d 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -13,7 +13,7 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "framework/scheduling/SerialScheduler.h" #include "shard/ISAMTree.h" @@ -24,4 +24,35 @@ using namespace de; typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tagged Tiering Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index e2b65a0..7d8f144 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -13,9 +13,10 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" +#include "shard/TrieSpline.h" #include "query/rangequery.h" #include @@ -23,4 +24,35 @@ using namespace de; typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tombstone Tiering Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/dynamic_extension_tests.inc b/tests/dynamic_extension_tests.inc deleted file mode 100644 index be82132..0000000 --- a/tests/dynamic_extension_tests.inc +++ /dev/null @@ -1,416 +0,0 @@ -/* - * tests/dynamic_extension_tests.inc - * - * Unit tests for Dynamic Extension Framework - * - * Copyright (C) 2023 Douglas Rumbaugh - * Dong Xie - * - * Distributed under the Modified BSD License. - * - */ - -START_TEST(t_create) -{ - auto test_de = new DE(100, 2, 1); - - ck_assert_ptr_nonnull(test_de); - ck_assert_int_eq(test_de->get_record_count(), 0); - ck_assert_int_eq(test_de->get_height(), 0); - - delete test_de; -} -END_TEST - - -START_TEST(t_insert) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<100; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - ck_assert_int_eq(test_de->get_height(), 0); - ck_assert_int_eq(test_de->get_record_count(), 100); - - delete test_de; -} -END_TEST - - -START_TEST(t_debug_insert) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<1000; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - ck_assert_int_eq(test_de->get_record_count(), i+1); - key++; - val++; - } - - delete test_de; -} -END_TEST - - -START_TEST(t_insert_with_mem_merges) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<300; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - test_de->await_next_epoch(); - - ck_assert_int_eq(test_de->get_record_count(), 300); - ck_assert_int_eq(test_de->get_height(), 1); - - delete test_de; -} -END_TEST - - -/* -START_TEST(t_range_sample_memtable) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<100; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - uint64_t lower_bound = 20; - uint64_t upper_bound = 50; - - char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - Rec sample_set[100]; - - test_de->range_sample(sample_set, lower_bound, upper_bound, 100); - - for(size_t i=0; i<100; i++) { - ck_assert_int_le(sample_set[i].key, upper_bound); - ck_assert_int_ge(sample_set[i].key, lower_bound); - } - - free(buf); - free(util_buf); - - delete test_de; -} -END_TEST - - -START_TEST(t_range_sample_memlevels) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<300; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - uint64_t lower_bound = 100; - uint64_t upper_bound = 250; - - char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - - Rec sample_set[100]; - test_de->range_sample(sample_set, lower_bound, upper_bound, 100); - - for(size_t i=0; i<100; i++) { - ck_assert_int_le(sample_set[i].key, upper_bound); - ck_assert_int_ge(sample_set[i].key, lower_bound); - } - - free(buf); - free(util_buf); - - delete test_de; -} -END_TEST -*/ - -START_TEST(t_range_query) -{ - auto test_de = new DE(100, 2, 1); - size_t n = 10000; - - std::vector keys; - for (size_t i=0; iinsert(r), 1); - } - - test_de->await_next_epoch(); - - std::sort(keys.begin(), keys.end()); - - auto idx = rand() % (keys.size() - 250); - - uint64_t lower_key = keys[idx]; - uint64_t upper_key = keys[idx + 250]; - - rq::Parms p; - p.lower_bound = lower_key; - p.upper_bound = upper_key; - - auto result = test_de->query(&p); - auto r = result.get(); - std::sort(r.begin(), r.end()); - ck_assert_int_eq(r.size(), 251); - - for (size_t i=0; i> records; - std::set> to_delete; - std::set> deleted; - - while (records.size() < reccnt) { - uint64_t key = rand(); - uint32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - size_t cnt=0; - for (auto rec : records) { - Rec r = {rec.first, rec.second}; - ck_assert_int_eq(test_de->insert(r), 1); - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector> del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(dr); - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - test_de->await_next_epoch(); - - ck_assert(test_de->validate_tombstone_proportion()); - - gsl_rng_free(rng); - delete test_de; -} -END_TEST - -DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { - auto rng = gsl_rng_alloc(gsl_rng_mt19937); - - auto test_de = new DE(1000, 2, 1); - - std::set records; - std::set to_delete; - std::set deleted; - - while (records.size() < reccnt) { - uint64_t key = rand(); - uint32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - for (auto rec : records) { - ck_assert_int_eq(test_de->insert(rec), 1); - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(del_vec[i]); - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - gsl_rng_free(rng); - - return test_de; -} - -START_TEST(t_static_structure) -{ - auto rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t reccnt = 100000; - auto test_de = new DE(100, 2, 1); - - std::set records; - std::set to_delete; - std::set deleted; - - while (records.size() < reccnt) { - uint64_t key = rand(); - uint32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - size_t t_reccnt = 0; - size_t k=0; - for (auto rec : records) { - k++; - ck_assert_int_eq(test_de->insert(rec), 1); - t_reccnt++; - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(del_vec[i]), 1); - - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - auto flat = test_de->create_static_structure(); - ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); - - uint64_t prev_key = 0; - for (size_t i=0; iget_record_count(); i++) { - auto k = flat->get_record_at(i)->rec.key; - ck_assert_int_ge(k, prev_key); - prev_key = k; - } - - gsl_rng_free(rng); - delete flat; - delete test_de; -} -END_TEST - - -Suite *unit_testing() -{ - Suite *unit = suite_create("de::DynamicExtension Unit Testing"); - - TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); - tcase_add_test(create, t_create); - suite_add_tcase(unit, create); - - TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); - tcase_add_test(insert, t_insert); - tcase_add_test(insert, t_insert_with_mem_merges); - tcase_add_test(insert, t_debug_insert); - suite_add_tcase(unit, insert); - - TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); - tcase_add_test(query, t_range_query); - suite_add_tcase(unit, query); - - TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); - tcase_add_test(ts, t_tombstone_merging_01); - tcase_set_timeout(ts, 500); - suite_add_tcase(unit, ts); - - TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); - tcase_add_test(flat, t_static_structure); - tcase_set_timeout(flat, 500); - suite_add_tcase(unit, flat); - - return unit; -} - -int run_unit_tests() -{ - int failed = 0; - Suite *unit = unit_testing(); - SRunner *unit_runner = srunner_create(unit); - - srunner_run_all(unit_runner, CK_NORMAL); - failed = srunner_ntests_failed(unit_runner); - srunner_free(unit_runner); - - return failed; -} - - -int main() -{ - int unit_failed = run_unit_tests(); - - return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; -} diff --git a/tests/include/dynamic_extension.h b/tests/include/dynamic_extension.h new file mode 100644 index 0000000..5a08f5a --- /dev/null +++ b/tests/include/dynamic_extension.h @@ -0,0 +1,341 @@ +/* + * tests/include/dynamic_extension.h + * + * Standardized unit tests for DynamicExtension objects + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard, Query, and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "testing.h" +//#include "framework/DynamicExtension.h" +//#include "framework/scheduling/SerialScheduler.h" +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include +//using namespace de; +//typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; + + +START_TEST(t_create) +{ + auto test_de = new DE(100, 1000, 2); + + ck_assert_ptr_nonnull(test_de); + ck_assert_int_eq(test_de->get_record_count(), 0); + ck_assert_int_eq(test_de->get_height(), 0); + + delete test_de; +} +END_TEST + + +START_TEST(t_insert) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<100; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + key++; + val++; + } + + ck_assert_int_eq(test_de->get_height(), 0); + ck_assert_int_eq(test_de->get_record_count(), 100); + + delete test_de; +} +END_TEST + + +START_TEST(t_debug_insert) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<1000; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + ck_assert_int_eq(test_de->get_record_count(), i+1); + key++; + val++; + } + + delete test_de; +} +END_TEST + + +START_TEST(t_insert_with_mem_merges) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<300; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + key++; + val++; + } + + test_de->await_next_epoch(); + + ck_assert_int_eq(test_de->get_record_count(), 300); + ck_assert_int_eq(test_de->get_height(), 1); + + delete test_de; +} +END_TEST + + +START_TEST(t_range_query) +{ + auto test_de = new DE(100, 1000, 2); + size_t n = 10000; + + std::vector keys; + for (size_t i=0; iinsert(r), 1); + } + + test_de->await_next_epoch(); + + std::sort(keys.begin(), keys.end()); + + auto idx = rand() % (keys.size() - 250); + + uint64_t lower_key = keys[idx]; + uint64_t upper_key = keys[idx + 250]; + + rq::Parms p; + p.lower_bound = lower_key; + p.upper_bound = upper_key; + + auto result = test_de->query(&p); + auto r = result.get(); + std::sort(r.begin(), r.end()); + ck_assert_int_eq(r.size(), 251); + + for (size_t i=0; i> records; + std::set> to_delete; + std::set> deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t cnt=0; + for (auto rec : records) { + Rec r = {rec.first, rec.second}; + ck_assert_int_eq(test_de->insert(r), 1); + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector> del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(dr); + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + test_de->await_next_epoch(); + + ck_assert(test_de->validate_tombstone_proportion()); + + gsl_rng_free(rng); + delete test_de; +} +END_TEST + +DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + auto test_de = new DE(1000, 10000, 2); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + for (auto rec : records) { + ck_assert_int_eq(test_de->insert(rec), 1); + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[i]); + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + gsl_rng_free(rng); + + return test_de; +} + +START_TEST(t_static_structure) +{ + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t reccnt = 100000; + auto test_de = new DE(100, 1000, 2); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t t_reccnt = 0; + size_t k=0; + for (auto rec : records) { + k++; + ck_assert_int_eq(test_de->insert(rec), 1); + t_reccnt++; + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[i]), 1); + + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + auto flat = test_de->create_static_structure(); + ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); + + uint64_t prev_key = 0; + for (size_t i=0; iget_record_count(); i++) { + auto k = flat->get_record_at(i)->rec.key; + ck_assert_int_ge(k, prev_key); + prev_key = k; + } + + gsl_rng_free(rng); + delete flat; + delete test_de; +} +END_TEST + + +static void inject_dynamic_extension_tests(Suite *suite) { + TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); + tcase_add_test(create, t_create); + suite_add_tcase(suite, create); + + TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); + tcase_add_test(insert, t_insert); + tcase_add_test(insert, t_insert_with_mem_merges); + tcase_add_test(insert, t_debug_insert); + suite_add_tcase(suite, insert); + + TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); + tcase_add_test(query, t_range_query); + suite_add_tcase(suite, query); + + TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); + tcase_add_test(ts, t_tombstone_merging_01); + tcase_set_timeout(ts, 500); + suite_add_tcase(suite, ts); + + TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); + tcase_add_test(flat, t_static_structure); + tcase_set_timeout(flat, 500); + suite_add_tcase(suite, flat); +} diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h new file mode 100644 index 0000000..3c7e7e0 --- /dev/null +++ b/tests/include/rangequery.h @@ -0,0 +1,179 @@ +/* + * tests/include/rangequery.h + * + * Standardized unit tests for range queries against supporting + * shard types + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface and Shard needs to support lower_bound. + * For other types of record and shard, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; + + +START_TEST(t_range_query) +{ + auto buffer = create_sequential_mbuffer(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 1000); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); + + std::vector>> results(2); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); + + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector>> proc_results; + + for (size_t j=0; j>()); + for (size_t i=0; i::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + Shard *shards[2]; + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + shards[0] = &shard1; + shards[1] = &shard2; + + auto merged = Shard(shards, 2); + + for (size_t i=100; i<1000; i++) { + Rec r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + +static void inject_rangequery_tests(Suite *suite) { + TCase *range_query = tcase_create("Range Query Testing"); + tcase_add_test(range_query, t_range_query); + tcase_add_test(range_query, t_buffer_range_query); + tcase_add_test(range_query, t_range_query_merge); + suite_add_tcase(suite, range_query); +} diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h new file mode 100644 index 0000000..047a7b5 --- /dev/null +++ b/tests/include/shard_standard.h @@ -0,0 +1,198 @@ +/* + * tests/include/shard_standard.h + * + * Standardized unit tests for Shard objects + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; + +START_TEST(t_mbuffer_init) +{ + auto buffer = new MutableBuffer(512, 1024); + for (uint64_t i = 512; i > 0; i--) { + uint32_t v = i; + buffer->append({i,v, 1}); + } + + for (uint64_t i = 1; i <= 256; ++i) { + uint32_t v = i; + buffer->append({i, v, 1}, true); + } + + for (uint64_t i = 257; i <= 512; ++i) { + uint32_t v = i + 1; + buffer->append({i, v, 1}); + } + + Shard* shard = new Shard(buffer->get_buffer_view()); + ck_assert_uint_eq(shard->get_record_count(), 512); + + delete buffer; + delete shard; +} + + +START_TEST(t_shard_init) +{ + size_t n = 512; + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); + + auto shard1 = new Shard(mbuffer1->get_buffer_view()); + auto shard2 = new Shard(mbuffer2->get_buffer_view()); + auto shard3 = new Shard(mbuffer3->get_buffer_view()); + + Shard* shards[3] = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards, 3); + + ck_assert_int_eq(shard4->get_record_count(), n * 3); + ck_assert_int_eq(shard4->get_tombstone_count(), 0); + + size_t total_cnt = 0; + size_t shard1_idx = 0; + size_t shard2_idx = 0; + size_t shard3_idx = 0; + + for (size_t i = 0; i < shard4->get_record_count(); ++i) { + auto rec1 = shard1->get_record_at(shard1_idx); + auto rec2 = shard2->get_record_at(shard2_idx); + auto rec3 = shard3->get_record_at(shard3_idx); + + auto cur_rec = shard4->get_record_at(i); + + if (shard1_idx < n && cur_rec->rec == rec1->rec) { + ++shard1_idx; + } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { + ++shard2_idx; + } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { + ++shard3_idx; + } else { + assert(false); + } + } + + delete mbuffer1; + delete mbuffer2; + delete mbuffer3; + + delete shard1; + delete shard2; + delete shard3; + delete shard4; +} + + +START_TEST(t_full_cancelation) +{ + size_t n = 100; + auto buffer = create_double_seq_mbuffer(n, false); + auto buffer_ts = create_double_seq_mbuffer(n, true); + + Shard* shard = new Shard(buffer->get_buffer_view()); + Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); + + ck_assert_int_eq(shard->get_record_count(), n); + ck_assert_int_eq(shard->get_tombstone_count(), 0); + ck_assert_int_eq(shard_ts->get_record_count(), n); + ck_assert_int_eq(shard_ts->get_tombstone_count(), n); + + Shard* shards[] = {shard, shard_ts}; + + Shard* merged = new Shard(shards, 2); + + ck_assert_int_eq(merged->get_tombstone_count(), 0); + ck_assert_int_eq(merged->get_record_count(), 0); + + delete buffer; + delete buffer_ts; + delete shard; + delete shard_ts; + delete merged; +} +END_TEST + + +START_TEST(t_point_lookup) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto isam = Shard(buffer->get_buffer_view()); + + { + auto view = buffer->get_buffer_view(); + + for (size_t i=0; irec.key; + r.value = rec->rec.value; + + auto result = isam.point_lookup(r); + ck_assert_ptr_nonnull(result); + ck_assert_int_eq(result->rec.key, r.key); + ck_assert_int_eq(result->rec.value, r.value); + } + } + + delete buffer; +} +END_TEST + + +START_TEST(t_point_lookup_miss) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto isam = Shard(buffer->get_buffer_view()); + + for (size_t i=n + 100; i<2*n; i++) { + Rec r; + r.key = i; + r.value = i; + + auto result = isam.point_lookup(r); + ck_assert_ptr_null(result); + } + + delete buffer; +} + +static void inject_shard_tests(Suite *suite) { + TCase *create = tcase_create("Shard constructor Testing"); + tcase_add_test(create, t_mbuffer_init); + tcase_add_test(create, t_shard_init); + tcase_set_timeout(create, 100); + suite_add_tcase(suite, create); + TCase *tombstone = tcase_create("Shard tombstone cancellation Testing"); + tcase_add_test(tombstone, t_full_cancelation); + suite_add_tcase(suite, tombstone); + TCase *pointlookup = tcase_create("Shard point lookup Testing"); + tcase_add_test(pointlookup, t_point_lookup); + tcase_add_test(pointlookup, t_point_lookup_miss); + suite_add_tcase(suite, pointlookup); +} diff --git a/tests/include/testing.h b/tests/include/testing.h new file mode 100644 index 0000000..4e660dd --- /dev/null +++ b/tests/include/testing.h @@ -0,0 +1,218 @@ +/* + * tests/testing.h + * + * Unit test utility functions/definitions + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * Distributed under the Modified BSD License. + * + */ +#pragma once + +#include + +#include +#include + +#include "util/types.h" +#include "psu-util/alignment.h" +#include "framework/structure/MutableBuffer.h" +#include "framework/interface/Record.h" + +typedef de::WeightedRecord WRec; +typedef de::Record Rec; +typedef de::EuclidPoint PRec; + +template +std::vector strip_wrapping(std::vector> vec) { + std::vector out(vec.size()); + for (size_t i=0; i *create_2d_mbuffer(size_t cnt) { + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + for (int64_t i=0; iappend({rand(), rand()}); + } + + return buffer; +} + +static de::MutableBuffer *create_2d_sequential_mbuffer(size_t cnt) { + auto buffer = new de::MutableBuffer(cnt/2, cnt); + for (int64_t i=0; iappend({i, i}); + } + + return buffer; +} + +template +static de::MutableBuffer *create_test_mbuffer(size_t cnt) +{ + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + R rec; + for (size_t i = 0; i < cnt; i++) { + rec.key = rand(); + rec.value = rand(); + + if constexpr (de::WeightedRecordInterface) { + rec.weight = 1; + } + + buffer->append(rec); + } + + return buffer; +} + +template +static de::MutableBuffer *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop) +{ + size_t cnt = stop - start; + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + for (size_t i=start; i) { + rec.weight = 1; + } + + buffer->append(rec); + } + + return buffer; +} + +template +static de::MutableBuffer *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) +{ + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + std::vector> tombstones; + + R rec; + for (size_t i = 0; i < cnt; i++) { + rec.key = rand(); + rec.value = rand(); + + if constexpr (de::WeightedRecordInterface) { + rec.weight = 1; + } + + if (i < ts_cnt) { + tombstones.push_back({rec.key, rec.value}); + } + + buffer->append(rec); + } + + rec.set_tombstone(); + for (size_t i=0; iappend(rec); + } + + return buffer; +} + +template +requires de::WeightedRecordInterface && de::KVPInterface +static de::MutableBuffer *create_weighted_mbuffer(size_t cnt) +{ + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + // Put in half of the count with weight one. + for (uint32_t i=0; i< cnt / 2; i++) { + buffer->append(R {1, i, 2}); + } + + // put in a quarter of the count with weight four. + for (uint32_t i=0; i< cnt / 4; i++) { + buffer->append(R {2, i, 4}); + } + + // the remaining quarter with weight eight. + for (uint32_t i=0; i< cnt / 4; i++) { + buffer->append(R {3, i, 8}); + } + + return buffer; +} + +template +static de::MutableBuffer *create_double_seq_mbuffer(size_t cnt, bool ts=false) +{ + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + for (size_t i = 0; i < cnt / 2; i++) { + R rec; + rec.key = i; + rec.value = i; + + buffer->append(rec, ts); + } + + for (size_t i = 0; i < cnt / 2; i++) { + R rec; + rec.key = i; + rec.value = i + 1; + + buffer->append(rec, ts); + } + + return buffer; +} + + diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index b8aa56f..79b9c21 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -16,7 +16,7 @@ #include "framework/interface/Query.h" #include "framework/interface/Shard.h" -#include "testing.h" +#include "include/testing.h" #include diff --git a/tests/memisam_tests.cpp b/tests/memisam_tests.cpp index 919fd69..b398524 100644 --- a/tests/memisam_tests.cpp +++ b/tests/memisam_tests.cpp @@ -11,359 +11,25 @@ */ #include "shard/ISAMTree.h" +#include "include/testing.h" #include "query/rangequery.h" -#include "testing.h" - #include -using namespace de; - -typedef ISAMTree Shard; - -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer(512, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer->get_buffer_view()); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_rq_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); - - auto shard1 = new Shard(mbuffer1->get_buffer_view()); - auto shard2 = new Shard(mbuffer2->get_buffer_view()); - auto shard3 = new Shard(mbuffer3->get_buffer_view()); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto isam = Shard(buffer->get_buffer_view()); - - { - auto view = buffer->get_buffer_view(); - - for (size_t i=0; irec.key; - r.value = rec->rec.value; - - auto result = isam.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto isam = Shard(buffer->get_buffer_view()); - - for (size_t i=n + 100; i<2*n; i++) { - Rec r; - r.key = i; - r.value = i; - - auto result = isam.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); - - Shard* shard = new Shard(buffer->get_buffer_view()); - Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST - - -/* -START_TEST(t_rq_query) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer(n); - auto isam = Shard(buffer->get_buffer_view()); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - size_t k = 100; - size_t cnt[3] = {0}; - rq::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query::get_query_state(&isam, &parms); - ((rq::State *) state)->sample_size = k; - auto result = rq::Query::query(&isam, state, &parms); - - ck_assert_int_eq(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query::delete_query_state(state); - } - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_rq_query_merge) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer(n); - - Shard shard = Shard(buffer->get_buffer_view()); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 1000; - - size_t cnt[3] = {0}; - rq::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - std::vector>> results(2); - - for (size_t i=0; i<1000; i++) { - auto state1 = rq::Query::get_query_state(&shard, &parms); - ((rq::State *) state1)->sample_size = k; - results[0] = rq::Query::query(&shard, state1, &parms); - - auto state2 = rq::Query::get_query_state(&shard, &parms); - ((rq::State *) state2)->sample_size = k; - results[1] = rq::Query::query(&shard, state2, &parms); - - rq::Query::delete_query_state(state1); - rq::Query::delete_query_state(state2); - } - - auto merged = rq::Query::merge(results, nullptr); - - ck_assert_int_eq(merged.size(), 2*k); - for (size_t i=0; i(n); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 100; - - size_t cnt[3] = {0}; - rq::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query::get_buffer_query_state(buffer, &parms); - ((rq::BufferState *) state)->sample_size = k; - auto result = rq::Query::buffer_query(buffer, state, &parms); - - ck_assert_int_eq(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query::delete_buffer_query_state(state); - } - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_rq_buffer_query_rejection) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer(n); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 10000; - - size_t cnt[3] = {0}; - rq::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query::get_buffer_query_state(buffer, &parms); - ((rq::BufferState *) state)->sample_size = k; - auto result = rq::Query::buffer_query(buffer, state, &parms); - - ck_assert_int_gt(result.size(), 0); - ck_assert_int_le(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query::delete_buffer_query_state(state); - } +using namespace de; - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST -*/ +typedef ISAMTree Shard; +#include "include/shard_standard.h" +#include "include/rangequery.h" Suite *unit_testing() { Suite *unit = suite_create("ISAMTree Shard Unit Testing"); - TCase *create = tcase_create("de::ISAMTree constructor Testing"); - tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_rq_init); - tcase_set_timeout(create, 100); - suite_add_tcase(unit, create); - - - TCase *tombstone = tcase_create("de:ISAMTree::tombstone cancellation Testing"); - tcase_add_test(tombstone, t_full_cancelation); - suite_add_tcase(unit, tombstone); - - - TCase *lookup = tcase_create("de:ISAMTree:point_lookup Testing"); - tcase_add_test(lookup, t_point_lookup); - tcase_add_test(lookup, t_point_lookup_miss); - suite_add_tcase(unit, lookup); - - /* - TCase *sampling = tcase_create("de:ISAMTree::IRS Testing"); - tcase_add_test(sampling, t_rq_query); - tcase_add_test(sampling, t_rq_query_merge); - tcase_add_test(sampling, t_rq_buffer_query_rejection); - tcase_add_test(sampling, t_rq_buffer_query_scan); - tcase_set_timeout(sampling, 100); - suite_add_tcase(unit, sampling); - */ + inject_rangequery_tests(unit); + inject_shard_tests(unit); return unit; } diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index e61a832..4064412 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -13,7 +13,7 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/structure/MutableBuffer.h" #include diff --git a/tests/rangequery_tests.cpp b/tests/rangequery_tests.cpp new file mode 100644 index 0000000..6a00f5a --- /dev/null +++ b/tests/rangequery_tests.cpp @@ -0,0 +1,195 @@ +/* + * tests/rangequery_tests.cpp + * + * Unit tests for Range Queries across several different + * shards + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * Distributed under the Modified BSD License. + * + */ + +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include "include/testing.h" + +#include + +using namespace de; + +typedef ISAMTree Shard; + +START_TEST(t_range_query) +{ + auto buffer = create_sequential_mbuffer(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 1000); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); + + std::vector>> results(2); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); + + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector>> proc_results; + + for (size_t j=0; j>()); + for (size_t i=0; i::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + Shard *shards[2]; + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + shards[0] = &shard1; + shards[1] = &shard2; + + auto merged = Shard(shards, 2); + + for (size_t i=100; i<1000; i++) { + Rec r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + + +Suite *unit_testing() +{ + Suite *unit = suite_create("Range Query Unit Testing"); + + TCase *range_query = tcase_create("de:PGM::range_query Testing"); + tcase_add_test(range_query, t_range_query); + tcase_add_test(range_query, t_buffer_range_query); + tcase_add_test(range_query, t_range_query_merge); + suite_add_tcase(unit, range_query); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/testing.h b/tests/testing.h deleted file mode 100644 index 4e660dd..0000000 --- a/tests/testing.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - * tests/testing.h - * - * Unit test utility functions/definitions - * - * Copyright (C) 2023 Douglas Rumbaugh - * Dong Xie - * - * Distributed under the Modified BSD License. - * - */ -#pragma once - -#include - -#include -#include - -#include "util/types.h" -#include "psu-util/alignment.h" -#include "framework/structure/MutableBuffer.h" -#include "framework/interface/Record.h" - -typedef de::WeightedRecord WRec; -typedef de::Record Rec; -typedef de::EuclidPoint PRec; - -template -std::vector strip_wrapping(std::vector> vec) { - std::vector out(vec.size()); - for (size_t i=0; i *create_2d_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - for (int64_t i=0; iappend({rand(), rand()}); - } - - return buffer; -} - -static de::MutableBuffer *create_2d_sequential_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt/2, cnt); - for (int64_t i=0; iappend({i, i}); - } - - return buffer; -} - -template -static de::MutableBuffer *create_test_mbuffer(size_t cnt) -{ - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - R rec; - for (size_t i = 0; i < cnt; i++) { - rec.key = rand(); - rec.value = rand(); - - if constexpr (de::WeightedRecordInterface) { - rec.weight = 1; - } - - buffer->append(rec); - } - - return buffer; -} - -template -static de::MutableBuffer *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop) -{ - size_t cnt = stop - start; - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - for (size_t i=start; i) { - rec.weight = 1; - } - - buffer->append(rec); - } - - return buffer; -} - -template -static de::MutableBuffer *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) -{ - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - std::vector> tombstones; - - R rec; - for (size_t i = 0; i < cnt; i++) { - rec.key = rand(); - rec.value = rand(); - - if constexpr (de::WeightedRecordInterface) { - rec.weight = 1; - } - - if (i < ts_cnt) { - tombstones.push_back({rec.key, rec.value}); - } - - buffer->append(rec); - } - - rec.set_tombstone(); - for (size_t i=0; iappend(rec); - } - - return buffer; -} - -template -requires de::WeightedRecordInterface && de::KVPInterface -static de::MutableBuffer *create_weighted_mbuffer(size_t cnt) -{ - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - // Put in half of the count with weight one. - for (uint32_t i=0; i< cnt / 2; i++) { - buffer->append(R {1, i, 2}); - } - - // put in a quarter of the count with weight four. - for (uint32_t i=0; i< cnt / 4; i++) { - buffer->append(R {2, i, 4}); - } - - // the remaining quarter with weight eight. - for (uint32_t i=0; i< cnt / 4; i++) { - buffer->append(R {3, i, 8}); - } - - return buffer; -} - -template -static de::MutableBuffer *create_double_seq_mbuffer(size_t cnt, bool ts=false) -{ - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - for (size_t i = 0; i < cnt / 2; i++) { - R rec; - rec.key = i; - rec.value = i; - - buffer->append(rec, ts); - } - - for (size_t i = 0; i < cnt / 2; i++) { - R rec; - rec.key = i; - rec.value = i + 1; - - buffer->append(rec, ts); - } - - return buffer; -} - - -- cgit v1.2.3 From 2117935e85412f3733ee0bcb1830c7fd0b129b29 Mon Sep 17 00:00:00 2001 From: "Douglas B. Rumbaugh" Date: Mon, 15 Jan 2024 17:23:57 -0500 Subject: Concurrency testing and bug fixes --- tests/de_level_concurrent.cpp | 58 ++++++ tests/de_level_tomb.cpp | 2 +- tests/include/concurrent_extension.h | 379 +++++++++++++++++++++++++++++++++++ 3 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 tests/de_level_concurrent.cpp create mode 100644 tests/include/concurrent_extension.h (limited to 'tests') diff --git a/tests/de_level_concurrent.cpp b/tests/de_level_concurrent.cpp new file mode 100644 index 0000000..b52fdd9 --- /dev/null +++ b/tests/de_level_concurrent.cpp @@ -0,0 +1,58 @@ +/* + * tests/de_level_tomb.cpp + * + * Unit tests for Dynamic Extension Framework + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * Distributed under the Modified BSD License. + * + */ +#include +#include +#include + +#include "include/testing.h" +#include "framework/DynamicExtension.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include "shard/TrieSpline.h" + +#include +using namespace de; + +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; + +#include "include/concurrent_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tombstone Leveling Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 91ee26d..44a0759 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -22,7 +22,7 @@ #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h new file mode 100644 index 0000000..86f8e12 --- /dev/null +++ b/tests/include/concurrent_extension.h @@ -0,0 +1,379 @@ +/* + * tests/include/dynamic_extension.h + * + * Standardized unit tests for DynamicExtension objects + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard, Query, and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +#include "testing.h" +#include "framework/DynamicExtension.h" +#include "framework/scheduling/FIFOScheduler.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include +using namespace de; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; + + +START_TEST(t_create) +{ + auto test_de = new DE(100, 1000, 2); + + ck_assert_ptr_nonnull(test_de); + ck_assert_int_eq(test_de->get_record_count(), 0); + ck_assert_int_eq(test_de->get_height(), 0); + + delete test_de; +} +END_TEST + + +START_TEST(t_insert) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<100; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + key++; + val++; + } + + ck_assert_int_eq(test_de->get_height(), 0); + ck_assert_int_eq(test_de->get_record_count(), 100); + + delete test_de; +} +END_TEST + + +START_TEST(t_debug_insert) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<1000; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + //ck_assert_int_eq(test_de->get_record_count(), i+1); + key++; + val++; + } + + delete test_de; +} +END_TEST + + +START_TEST(t_insert_with_mem_merges) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + + Rec r = {key, val}; + for (size_t i=0; i<1000; i++) { + ck_assert_int_eq(test_de->insert(r), 1); + r.key++; + r.value++; + } + + ck_assert_int_eq(test_de->get_record_count(), 1000); + + test_de->await_next_epoch(); + + ck_assert_int_eq(test_de->get_record_count(), 1000); + + /* + * verify that we can fill past the high water mark, potentially + * stalling to allow merges to finish as needed. + */ + size_t cnt = 0; + do { + if (test_de->insert(r)) { + r.key++; + r.value++; + cnt++; + } else { + sleep(1); + } + } while (cnt < 10000); + + test_de->await_next_epoch(); + + ck_assert_int_eq(test_de->get_record_count(), 11000); + + delete test_de; +} +END_TEST + + +START_TEST(t_range_query) +{ + auto test_de = new DE(100, 1000, 2); + size_t n = 10000; + + std::vector keys; + for (size_t i=0; iinsert(r)) { + i++; + } else { + sleep(1); + } + } + + test_de->await_next_epoch(); + + std::sort(keys.begin(), keys.end()); + + auto idx = rand() % (keys.size() - 250); + + uint64_t lower_key = keys[idx]; + uint64_t upper_key = keys[idx + 250]; + + rq::Parms p; + p.lower_bound = lower_key; + p.upper_bound = upper_key; + + auto result = test_de->query(&p); + auto r = result.get(); + std::sort(r.begin(), r.end()); + ck_assert_int_eq(r.size(), 251); + + for (size_t i=0; i> records; + std::set> to_delete; + std::set> deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t cnt=0; + for (auto rec : records) { + Rec r = {rec.first, rec.second}; + while (!test_de->insert(r)) { + sleep(1); + } + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector> del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(dr)) { + sleep(1); + } + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + test_de->await_next_epoch(); + + ck_assert(test_de->validate_tombstone_proportion()); + + gsl_rng_free(rng); + delete test_de; +} +END_TEST + +DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + auto test_de = new DE(1000, 10000, 2); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + for (auto rec : records) { + ck_assert_int_eq(test_de->insert(rec), 1); + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[i]); + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + gsl_rng_free(rng); + + return test_de; +} + +START_TEST(t_static_structure) +{ + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t reccnt = 100000; + auto test_de = new DE(100, 1000, 2); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t t_reccnt = 0; + size_t k=0; + for (auto rec : records) { + k++; + while (!test_de->insert(rec)) { + sleep(1); + } + t_reccnt++; + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[1])) { + sleep(1); + } + + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + auto flat = test_de->create_static_structure(); + ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); + + uint64_t prev_key = 0; + for (size_t i=0; iget_record_count(); i++) { + auto k = flat->get_record_at(i)->rec.key; + ck_assert_int_ge(k, prev_key); + prev_key = k; + } + + gsl_rng_free(rng); + delete flat; + delete test_de; +} +END_TEST + + +static void inject_dynamic_extension_tests(Suite *suite) { + TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); + tcase_add_test(create, t_create); + suite_add_tcase(suite, create); + + TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); + tcase_add_test(insert, t_insert); + tcase_add_test(insert, t_insert_with_mem_merges); + tcase_add_test(insert, t_debug_insert); + tcase_set_timeout(insert, 500); + suite_add_tcase(suite, insert); + + /* + TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); + tcase_add_test(query, t_range_query); + suite_add_tcase(suite, query); + + + TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); + tcase_add_test(ts, t_tombstone_merging_01); + tcase_set_timeout(ts, 500); + suite_add_tcase(suite, ts); + + TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); + tcase_add_test(flat, t_static_structure); + tcase_set_timeout(flat, 500); + suite_add_tcase(suite, flat); + */ +} -- cgit v1.2.3 From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 17 Jan 2024 18:22:00 -0500 Subject: Multiple concurrency bug fixes A poorly organized commit with fixes for a variety of bugs that were causing missing records. The core problems all appear to be fixed, though there is an outstanding problem with tombstones not being completely canceled. A very small number are appearing in the wrong order during the static structure test. --- tests/de_tier_concurrent.cpp | 57 ++++++++++++++++++++++++++++++++++++ tests/include/concurrent_extension.h | 54 ++++++++++++++++++++++------------ tests/include/rangequery.h | 25 ++++++++-------- tests/include/shard_standard.h | 8 ++--- tests/rangequery_tests.cpp | 7 ++--- 5 files changed, 110 insertions(+), 41 deletions(-) create mode 100644 tests/de_tier_concurrent.cpp (limited to 'tests') diff --git a/tests/de_tier_concurrent.cpp b/tests/de_tier_concurrent.cpp new file mode 100644 index 0000000..9387b21 --- /dev/null +++ b/tests/de_tier_concurrent.cpp @@ -0,0 +1,57 @@ +/* + * tests/de_level_tomb.cpp + * + * Unit tests for Dynamic Extension Framework + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * Distributed under the Modified BSD License. + * + */ +#include +#include +#include + +#include "include/testing.h" +#include "framework/DynamicExtension.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" + +#include +using namespace de; + +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; + +#include "include/concurrent_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tombstone Leveling Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h index 86f8e12..a0e71c9 100644 --- a/tests/include/concurrent_extension.h +++ b/tests/include/concurrent_extension.h @@ -28,8 +28,9 @@ #include "shard/ISAMTree.h" #include "query/rangequery.h" #include -using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; + +//using namespace de; +//typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; START_TEST(t_create) @@ -75,7 +76,7 @@ START_TEST(t_debug_insert) for (size_t i=0; i<1000; i++) { Rec r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); - //ck_assert_int_eq(test_de->get_record_count(), i+1); + ck_assert_int_eq(test_de->get_record_count(), i+1); key++; val++; } @@ -115,14 +116,15 @@ START_TEST(t_insert_with_mem_merges) r.key++; r.value++; cnt++; + ck_assert_int_eq(test_de->get_record_count(), cnt + 1000); } else { - sleep(1); + _mm_pause(); } - } while (cnt < 10000); + } while (cnt < 100000); test_de->await_next_epoch(); - ck_assert_int_eq(test_de->get_record_count(), 11000); + ck_assert_int_eq(test_de->get_record_count(), 101000); delete test_de; } @@ -131,12 +133,12 @@ END_TEST START_TEST(t_range_query) { - auto test_de = new DE(100, 1000, 2); - size_t n = 10000; + auto test_de = new DE(1000, 10000, 4); + size_t n = 10000000; std::vector keys; for (size_t i=0; iinsert(r)) { i++; } else { - sleep(1); + _mm_pause(); } } + test_de->await_next_epoch(); std::sort(keys.begin(), keys.end()); @@ -166,9 +169,12 @@ START_TEST(t_range_query) p.lower_bound = lower_key; p.upper_bound = upper_key; + fprintf(stderr, "query start\n"); auto result = test_de->query(&p); auto r = result.get(); + fprintf(stderr, "query stop\n"); std::sort(r.begin(), r.end()); + ck_assert_int_eq(r.size(), 251); for (size_t i=0; iinsert(r)) { - sleep(1); + _mm_pause(); } if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { @@ -215,7 +221,7 @@ START_TEST(t_tombstone_merging_01) for (size_t i=0; ierase(dr)) { - sleep(1); + _mm_pause(); } deletes++; to_delete.erase(del_vec[i]); @@ -307,7 +313,7 @@ START_TEST(t_static_structure) for (auto rec : records) { k++; while (!test_de->insert(rec)) { - sleep(1); + _mm_pause(); } t_reccnt++; @@ -316,8 +322,8 @@ START_TEST(t_static_structure) std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; ierase(del_vec[1])) { - sleep(1); + while (!test_de->erase(del_vec[i])) { + _mm_pause(); } deletes++; @@ -331,12 +337,23 @@ START_TEST(t_static_structure) } } - auto flat = test_de->create_static_structure(); - ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); + + //fprintf(stderr, "Tombstones: %ld\tRecords: %ld\n", test_de->get_tombstone_count(), test_de->get_record_count()); + //fprintf(stderr, "Inserts: %ld\tDeletes:%ld\tNet:%ld\n", reccnt, deletes, reccnt - deletes); + + auto flat = test_de->create_static_structure(true); + //fprintf(stderr, "Flat: Tombstones: %ld\tRecords %ld\n", flat->get_tombstone_count(), flat->get_record_count()); + //ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); uint64_t prev_key = 0; for (size_t i=0; iget_record_count(); i++) { auto k = flat->get_record_at(i)->rec.key; + if (flat->get_record_at(i)->is_tombstone()) { + fprintf(stderr, "%ld %ld %ld\n", flat->get_record_at(i-1)->rec.key, + flat->get_record_at(i)->rec.key, + flat->get_record_at(i+1)->rec.key); + } + // ck_assert(!flat->get_record_at(i)->is_tombstone()); ck_assert_int_ge(k, prev_key); prev_key = k; } @@ -360,9 +377,9 @@ static void inject_dynamic_extension_tests(Suite *suite) { tcase_set_timeout(insert, 500); suite_add_tcase(suite, insert); - /* TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); tcase_add_test(query, t_range_query); + tcase_set_timeout(query, 500); suite_add_tcase(suite, query); @@ -375,5 +392,4 @@ static void inject_dynamic_extension_tests(Suite *suite) { tcase_add_test(flat, t_static_structure); tcase_set_timeout(flat, 500); suite_add_tcase(suite, flat); - */ } diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index 3c7e7e0..e45de57 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -24,12 +24,12 @@ * should be included in the source file that includes this one, above the * include statement. */ -//#include "shard/ISAMTree.h" -//#include "query/rangequery.h" -//#include "testing.h" -//#include -//using namespace de; -//typedef ISAMTree Shard; +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include "testing.h" +#include +using namespace de; +typedef ISAMTree Shard; START_TEST(t_range_query) @@ -137,15 +137,12 @@ START_TEST(t_lower_bound) auto buffer1 = create_sequential_mbuffer(100, 200); auto buffer2 = create_sequential_mbuffer(400, 1000); - Shard *shards[2]; + auto shard1 = new Shard(buffer1->get_buffer_view()); + auto shard2 = new Shard(buffer2->get_buffer_view()); - auto shard1 = Shard(buffer1->get_buffer_view()); - auto shard2 = Shard(buffer2->get_buffer_view()); - - shards[0] = &shard1; - shards[1] = &shard2; + std::vector shards = {shard1, shard2}; - auto merged = Shard(shards, 2); + auto merged = Shard(shards); for (size_t i=100; i<1000; i++) { Rec r; @@ -167,6 +164,8 @@ START_TEST(t_lower_bound) delete buffer1; delete buffer2; + delete shard1; + delete shard2; } END_TEST diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h index 047a7b5..ddd7614 100644 --- a/tests/include/shard_standard.h +++ b/tests/include/shard_standard.h @@ -65,8 +65,8 @@ START_TEST(t_shard_init) auto shard2 = new Shard(mbuffer2->get_buffer_view()); auto shard3 = new Shard(mbuffer3->get_buffer_view()); - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); + std::vector shards = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards); ck_assert_int_eq(shard4->get_record_count(), n * 3); ck_assert_int_eq(shard4->get_tombstone_count(), 0); @@ -119,9 +119,9 @@ START_TEST(t_full_cancelation) ck_assert_int_eq(shard_ts->get_record_count(), n); ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - Shard* shards[] = {shard, shard_ts}; + std::vector shards = {shard, shard_ts}; - Shard* merged = new Shard(shards, 2); + Shard* merged = new Shard(shards); ck_assert_int_eq(merged->get_tombstone_count(), 0); ck_assert_int_eq(merged->get_record_count(), 0); diff --git a/tests/rangequery_tests.cpp b/tests/rangequery_tests.cpp index 6a00f5a..78a4e72 100644 --- a/tests/rangequery_tests.cpp +++ b/tests/rangequery_tests.cpp @@ -125,15 +125,12 @@ START_TEST(t_lower_bound) auto buffer1 = create_sequential_mbuffer(100, 200); auto buffer2 = create_sequential_mbuffer(400, 1000); - Shard *shards[2]; - auto shard1 = Shard(buffer1->get_buffer_view()); auto shard2 = Shard(buffer2->get_buffer_view()); - shards[0] = &shard1; - shards[1] = &shard2; + std::vector shards = {&shard1, &shard2}; - auto merged = Shard(shards, 2); + auto merged = Shard(shards); for (size_t i=100; i<1000; i++) { Rec r; -- cgit v1.2.3 From 4ac2e14d24a1fdd3f9bf777775b16bf6a677f487 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 22 Jan 2024 10:14:05 -0500 Subject: Added RangeCount query --- tests/include/rangecount.h | 155 +++++++++++++++++++++++++++++++++++++++++++++ tests/rangecount_tests.cpp | 55 ++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 tests/include/rangecount.h create mode 100644 tests/rangecount_tests.cpp (limited to 'tests') diff --git a/tests/include/rangecount.h b/tests/include/rangecount.h new file mode 100644 index 0000000..83bf4d4 --- /dev/null +++ b/tests/include/rangecount.h @@ -0,0 +1,155 @@ +/* + * tests/include/rangecount.h + * + * Standardized unit tests for range queries against supporting + * shard types + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface and Shard needs to support lower_bound. + * For other types of record and shard, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "query/rangecount.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; + +START_TEST(t_range_count) +{ + auto buffer = create_sequential_mbuffer(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rc::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rc::Query::get_query_state(&shard, &parms); + auto result = rc::Query::query(&shard, state, &parms); + rc::Query::delete_query_state(state); + + ck_assert_int_eq(result.size(), 1); + ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1); + + delete buffer; +} +END_TEST + + +START_TEST(t_buffer_range_count) +{ + auto buffer = create_sequential_mbuffer(100, 1000); + + rc::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rc::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); + auto result = rc::Query::buffer_query(state, &parms); + rc::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), 1); + ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1); + + delete buffer; +} +END_TEST + + +START_TEST(t_range_count_merge) +{ + auto buffer1 = create_sequential_mbuffer(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rc::Parms parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rc::Query::get_query_state(&shard1, &parms); + auto state2 = rc::Query::get_query_state(&shard2, &parms); + + std::vector>> results(2); + results[0] = rc::Query::query(&shard1, state1, &parms); + results[1] = rc::Query::query(&shard2, state2, &parms); + + rc::Query::delete_query_state(state1); + rc::Query::delete_query_state(state2); + + ck_assert_int_eq(results[0].size(), 1); + ck_assert_int_eq(results[1].size(), 1); + + auto result = rc::Query::merge(results, nullptr); + + ck_assert_int_eq(result[0].key, result_size); + + delete buffer1; + delete buffer2; +} +END_TEST + + +START_TEST(t_lower_bound) +{ + auto buffer1 = create_sequential_mbuffer(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = new Shard(buffer1->get_buffer_view()); + auto shard2 = new Shard(buffer2->get_buffer_view()); + + std::vector shards = {shard1, shard2}; + + auto merged = Shard(shards); + + for (size_t i=100; i<1000; i++) { + Rec r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; + delete shard1; + delete shard2; +} +END_TEST + +static void inject_rangecount_tests(Suite *suite) { + TCase *range_count = tcase_create("Range Query Testing"); + tcase_add_test(range_count, t_range_count); + tcase_add_test(range_count, t_buffer_range_count); + tcase_add_test(range_count, t_range_count_merge); + suite_add_tcase(suite, range_count); +} diff --git a/tests/rangecount_tests.cpp b/tests/rangecount_tests.cpp new file mode 100644 index 0000000..fe3a587 --- /dev/null +++ b/tests/rangecount_tests.cpp @@ -0,0 +1,55 @@ +/* + * tests/rangequery_tests.cpp + * + * Unit tests for Range Queries across several different + * shards + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * Distributed under the Modified BSD License. + * + */ + +#include "shard/ISAMTree.h" +#include "query/rangecount.h" +#include "include/testing.h" + +#include + +using namespace de; + +typedef ISAMTree Shard; + +#include "include/rangecount.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("Range Count Query Testing"); + inject_rangecount_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} -- cgit v1.2.3 From 51a85013236f4b2bd596caf179d90e67c848963c Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 30 Jan 2024 15:31:34 -0500 Subject: TrieSpline + tests --- tests/include/rangequery.h | 12 +-- tests/triespline_tests.cpp | 242 ++------------------------------------------- 2 files changed, 14 insertions(+), 240 deletions(-) (limited to 'tests') diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index e45de57..1ac0891 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -24,12 +24,12 @@ * should be included in the source file that includes this one, above the * include statement. */ -#include "shard/ISAMTree.h" -#include "query/rangequery.h" -#include "testing.h" -#include -using namespace de; -typedef ISAMTree Shard; +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; START_TEST(t_range_query) diff --git a/tests/triespline_tests.cpp b/tests/triespline_tests.cpp index 14506be..9df278f 100644 --- a/tests/triespline_tests.cpp +++ b/tests/triespline_tests.cpp @@ -1,7 +1,7 @@ /* - * tests/triespline_tests.cpp + * tests/isam_tests.cpp * - * Unit tests for TrieSpline (Augmented B+Tree) shard + * Unit tests for ISAM Tree shard * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie @@ -10,250 +10,24 @@ * */ -#include - #include "shard/TrieSpline.h" +#include "include/testing.h" #include "query/rangequery.h" -#include "testing.h" - #include using namespace de; typedef TrieSpline Shard; -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer(1024, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); - - auto shard1 = new Shard(mbuffer1); - auto shard2 = new Shard(mbuffer2); - auto shard3 = new Shard(mbuffer3); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto shard = Shard(buffer); - - for (size_t i=0; iget_data() + i); - r.key = rec->rec.key; - r.value = rec->rec.value; - - auto result = shard.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto isam = Shard(buffer); - - for (size_t i=n + 100; i<2*n; i++) { - Rec r; - r.key = i; - r.value = i; - - auto result = isam.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); - - Shard* shard = new Shard(buffer); - Shard* shard_ts = new Shard(buffer_ts); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST - - -START_TEST(t_range_query) -{ - auto buffer = create_sequential_mbuffer(100, 1000); - auto shard = Shard(buffer); - - rq::Parms parms; - parms.lower_bound = 300; - parms.upper_bound = 500; - - auto state = rq::Query::get_query_state(&shard, &parms); - auto result = rq::Query::query(&shard, state, &parms); - rq::Query::delete_query_state(state); - - ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); - for (size_t i=0; i(100, 1000); - - rq::Parms parms; - parms.lower_bound = 300; - parms.upper_bound = 500; - - auto state = rq::Query::get_buffer_query_state(buffer, &parms); - auto result = rq::Query::buffer_query(buffer, state, &parms); - rq::Query::delete_buffer_query_state(state); - - ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); - for (size_t i=0; i Date: Mon, 5 Feb 2024 15:17:25 -0500 Subject: Set up tombstone deletes properly --- tests/include/rangecount.h | 14 +++++++++----- tests/include/rangequery.h | 19 +++++++++++-------- tests/rangequery_tests.cpp | 19 +++++++++++-------- 3 files changed, 31 insertions(+), 21 deletions(-) (limited to 'tests') diff --git a/tests/include/rangecount.h b/tests/include/rangecount.h index 83bf4d4..e09ab12 100644 --- a/tests/include/rangecount.h +++ b/tests/include/rangecount.h @@ -33,6 +33,7 @@ START_TEST(t_range_count) { + auto buffer = create_sequential_mbuffer(100, 1000); auto shard = Shard(buffer->get_buffer_view()); @@ -60,12 +61,15 @@ START_TEST(t_buffer_range_count) parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rc::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); - auto result = rc::Query::buffer_query(state, &parms); - rc::Query::delete_buffer_query_state(state); + { + auto view = buffer->get_buffer_view(); + auto state = rc::Query::get_buffer_query_state(&view, &parms); + auto result = rc::Query::buffer_query(state, &parms); + rc::Query::delete_buffer_query_state(state); - ck_assert_int_eq(result.size(), 1); - ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1); + ck_assert_int_eq(result.size(), 1); + ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1); + } delete buffer; } diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index 1ac0891..b9694a4 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -64,14 +64,17 @@ START_TEST(t_buffer_range_query) parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rq::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); - auto result = rq::Query::buffer_query(state, &parms); - rq::Query::delete_buffer_query_state(state); - - ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); - for (size_t i=0; iget_buffer_view(); + auto state = rq::Query::get_buffer_query_state(&view, &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i::get_buffer_query_state(buffer->get_buffer_view(), &parms); - auto result = rq::Query::buffer_query(state, &parms); - rq::Query::delete_buffer_query_state(state); + { + auto view = buffer->get_buffer_view(); + auto state = rq::Query::get_buffer_query_state(&view, &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i Date: Mon, 5 Feb 2024 15:18:33 -0500 Subject: BufferView: Adjusted BV to avoid repeated modulus operations --- tests/mutable_buffer_tests.cpp | 142 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) (limited to 'tests') diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index 4064412..31c16dc 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -238,6 +238,143 @@ START_TEST(t_truncate) } END_TEST +START_TEST(t_bview_get) +{ + auto buffer = new MutableBuffer(50, 100); + + /* insert 75 records and get tail when LWM is exceeded */ + size_t new_head = 0; + Rec rec = {1, 1}; + size_t cnt = 0; + for (size_t i=0; i<75; i++) { + ck_assert_int_eq(buffer->append(rec), 1); + + rec.key++; + rec.value++; + cnt++; + + if (buffer->is_at_low_watermark() && new_head == 0) { + new_head = buffer->get_tail(); + } + } + + ck_assert_int_eq(buffer->get_available_capacity(), 200 - cnt); + + { + /* get a view of the pre-advanced state */ + auto view = buffer->get_buffer_view(); + auto reccnt = view.get_record_count(); + + /* scan the records in the view */ + for (size_t i=0; irec.key, i+1); + } + + /* advance the head */ + buffer->advance_head(new_head); + + /* scan the records in the view again -- should be unchanged */ + for (size_t i=0; irec.key, i+1); + } + } + + { + /* get a new view (should have fewer records) */ + auto view = buffer->get_buffer_view(); + auto reccnt = view.get_record_count(); + + /* verify the scan again */ + for (size_t i=0; irec.key, i + 51); + } + } + + /* insert more records (to trigger a wrap-around) */ + for (size_t i=0; i<75; i++) { + ck_assert_int_eq(buffer->append(rec), 1); + + rec.key++; + rec.value++; + cnt++; + } + + { + /* get a new view (should have fewer records) */ + auto view = buffer->get_buffer_view(); + auto reccnt = view.get_record_count(); + + /* verify the scan again */ + for (size_t i=0; irec.key, i + 51); + } + } + + delete buffer; +} +END_TEST + + +START_TEST(t_bview_delete) +{ + + auto buffer = new MutableBuffer(50, 100); + + /* insert 75 records and get tail when LWM is exceeded */ + size_t new_head = 0; + Rec rec = {1, 1}; + size_t cnt = 0; + for (size_t i=0; i<75; i++) { + ck_assert_int_eq(buffer->append(rec), 1); + + rec.key++; + rec.value++; + cnt++; + + if (buffer->is_at_low_watermark() && new_head == 0) { + new_head = buffer->get_tail(); + } + } + + buffer->advance_head(new_head); + + for (size_t i=0; i<75; i++) { + ck_assert_int_eq(buffer->append(rec), 1); + + rec.key++; + rec.value++; + cnt++; + } + + Rec dr1 = {67, 67}; + Rec dr2 = {89, 89}; + Rec dr3 = {103, 103}; + + Rec fdr1 = {5, 5}; + Rec fdr2 = {300, 300}; + { + /* get a new view (should have fewer records) */ + auto view = buffer->get_buffer_view(); + ck_assert_int_eq(view.delete_record(dr1), 1); + ck_assert_int_eq(view.delete_record(dr2), 1); + ck_assert_int_eq(view.delete_record(dr3), 1); + ck_assert_int_eq(view.delete_record(fdr1), 0); + ck_assert_int_eq(view.delete_record(fdr2), 0); + + for (size_t i=0; irec == dr1 || view.get(i)->rec == dr2 + || view.get(i)->rec == dr3) { + ck_assert_int_eq(view.get(i)->is_deleted(), 1); + } else { + ck_assert_int_eq(view.get(i)->is_deleted(), 0); + } + } + } + + delete buffer; +} +END_TEST + Suite *unit_testing() { @@ -255,6 +392,11 @@ Suite *unit_testing() suite_add_tcase(unit, append); + TCase *view = tcase_create("de::BufferView Testing"); + tcase_add_test(view, t_bview_get); + tcase_add_test(view, t_bview_delete); + + suite_add_tcase(unit, view); TCase *truncate = tcase_create("de::MutableBuffer::truncate Testing"); tcase_add_test(truncate, t_truncate); -- cgit v1.2.3 From 10b4425e842d10b7fbfa85978969ed4591d6b98e Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 7 Feb 2024 10:56:52 -0500 Subject: Fully implemented Query concept and adjusted queries to use it --- tests/include/concurrent_extension.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h index a0e71c9..24cb2ce 100644 --- a/tests/include/concurrent_extension.h +++ b/tests/include/concurrent_extension.h @@ -22,7 +22,7 @@ * should be included in the source file that includes this one, above the * include statement. */ -#include "testing.h" +/*#include "testing.h" #include "framework/DynamicExtension.h" #include "framework/scheduling/FIFOScheduler.h" #include "shard/ISAMTree.h" @@ -31,6 +31,7 @@ //using namespace de; //typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +*/ START_TEST(t_create) @@ -169,10 +170,10 @@ START_TEST(t_range_query) p.lower_bound = lower_key; p.upper_bound = upper_key; - fprintf(stderr, "query start\n"); + //fprintf(stderr, "query start\n"); auto result = test_de->query(&p); auto r = result.get(); - fprintf(stderr, "query stop\n"); + //fprintf(stderr, "query stop\n"); std::sort(r.begin(), r.end()); ck_assert_int_eq(r.size(), 251); -- cgit v1.2.3 From 2c5d549b3618b9ea72e6eece4cb4f3da5a6811a8 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 7 Feb 2024 13:42:34 -0500 Subject: Fully realized shard concept interface --- tests/de_cc_isam_level.cpp | 461 ----------------------------------------- tests/de_level_concurrent.cpp | 2 +- tests/de_level_tag.cpp | 2 +- tests/de_level_tomb.cpp | 2 +- tests/de_tier_concurrent.cpp | 2 +- tests/de_tier_tag.cpp | 2 +- tests/de_tier_tomb.cpp | 2 +- tests/include/rangecount.h | 26 +-- tests/include/rangequery.h | 26 +-- tests/internal_level_tests.cpp | 2 +- tests/rangequery_tests.cpp | 147 +------------ 11 files changed, 36 insertions(+), 638 deletions(-) delete mode 100644 tests/de_cc_isam_level.cpp (limited to 'tests') diff --git a/tests/de_cc_isam_level.cpp b/tests/de_cc_isam_level.cpp deleted file mode 100644 index 4972fb5..0000000 --- a/tests/de_cc_isam_level.cpp +++ /dev/null @@ -1,461 +0,0 @@ -/* - * tests/de_cc_isam_level.cpp - * - * Unit tests for Dynamic Extension Framework - * - * Copyright (C) 2023 Douglas B. Rumbaugh - * - * Distributed under the Modified BSD License. - * - */ - -#include - -#include "framework/DynamicExtension.h" -#include "shard/MemISAM.h" - -#include - -using namespace de; - -typedef Record Rec; -typedef DynamicExtension, ISAMRangeQuery, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE> DE; - -START_TEST(t_create) -{ - auto de_isam = new DE(100, 2, 1); - - ck_assert_ptr_nonnull(de_isam); - ck_assert_int_eq(de_isam->get_record_count(), 0); - ck_assert_int_eq(de_isam->get_height(), 0); - - delete de_isam; -} -END_TEST - - -START_TEST(t_insert) -{ - auto ext_wirs = new DE(100, 2, 1); - - int32_t key = 0; - int32_t val = 0; - for (size_t i=0; i<100; i++) { - Rec r = {key, val}; - ck_assert_int_eq(ext_wirs->insert(r), 1); - key++; - val++; - } - - ck_assert_int_eq(ext_wirs->get_height(), 0); - ck_assert_int_eq(ext_wirs->get_record_count(), 100); - - delete ext_wirs; -} -END_TEST - - -START_TEST(t_debug_insert) -{ - auto ext_wirs = new DE(100, 2, 1); - - int32_t key = 0; - int32_t val = 0; - for (size_t i=0; i<1000; i++) { - Rec r = {key, val}; - ck_assert_int_eq(ext_wirs->insert(r), 1); - ck_assert_int_eq(ext_wirs->get_record_count(), i+1); - key++; - val++; - } - - delete ext_wirs; -} -END_TEST - - -START_TEST(t_insert_with_mem_merges) -{ - auto ext_wirs = new DE(100, 2, 1); - - int32_t key = 0; - int32_t val = 0; - for (size_t i=0; i<300; i++) { - Rec r = {key, val}; - ck_assert_int_eq(ext_wirs->insert(r), 1); - key++; - val++; - } - - ck_assert_int_eq(ext_wirs->get_record_count(), 300); - ck_assert_int_eq(ext_wirs->get_height(), 1); - - delete ext_wirs; -} -END_TEST - - -/* -START_TEST(t_range_sample_memtable) -{ - auto ext_wirs = new DE(100, 2, 1); - - int32_t key = 0; - int32_t val = 0; - for (size_t i=0; i<100; i++) { - Rec r = {key, val}; - ck_assert_int_eq(ext_wirs->insert(r), 1); - key++; - val++; - } - - int32_t lower_bound = 20; - int32_t upper_bound = 50; - - char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - Rec sample_set[100]; - - ext_wirs->range_sample(sample_set, lower_bound, upper_bound, 100); - - for(size_t i=0; i<100; i++) { - ck_assert_int_le(sample_set[i].key, upper_bound); - ck_assert_int_ge(sample_set[i].key, lower_bound); - } - - free(buf); - free(util_buf); - - delete ext_wirs; -} -END_TEST - - -START_TEST(t_range_sample_memlevels) -{ - auto ext_wirs = new DE(100, 2, 1); - - int32_t key = 0; - int32_t val = 0; - for (size_t i=0; i<300; i++) { - Rec r = {key, val}; - ck_assert_int_eq(ext_wirs->insert(r), 1); - key++; - val++; - } - - int32_t lower_bound = 100; - int32_t upper_bound = 250; - - char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - - Rec sample_set[100]; - ext_wirs->range_sample(sample_set, lower_bound, upper_bound, 100); - - for(size_t i=0; i<100; i++) { - ck_assert_int_le(sample_set[i].key, upper_bound); - ck_assert_int_ge(sample_set[i].key, lower_bound); - } - - free(buf); - free(util_buf); - - delete ext_wirs; -} -END_TEST - -START_TEST(t_range_sample_weighted) -{ - auto ext_wirs = new DE(100, 2, 1); - size_t n = 10000; - - std::vector keys; - - int32_t key = 1; - for (size_t i=0; i< n / 2; i++) { - keys.push_back(key); - } - - // put in a quarter of the count with weight two. - key = 2; - for (size_t i=0; i< n / 4; i++) { - keys.push_back(key); - } - - // the remaining quarter with weight four. - key = 3; - for (size_t i=0; i< n / 4; i++) { - keys.push_back(key); - } - - std::random_device rd; - std::mt19937 gen{rd()}; - std::shuffle(keys.begin(), keys.end(), gen); - - for (size_t i=0; iinsert(r); - } - size_t k = 1000; - int32_t lower_key = 0; - int32_t upper_key = 5; - - size_t cnt[3] = {0}; - size_t total_samples = 0; - - wirs_query_parms p; - p.lower_bound = lower_key; - p.upper_bound = upper_key; - p.sample_size = k; - p.rng = gsl_rng_alloc(gsl_rng_mt19937); - - for (size_t i=0; i<1000; i++) { - - auto result = ext_wirs->query(&p); - auto r = result.get(); - total_samples += r.size(); - - for (size_t j=0; j> records; - std::set> to_delete; - std::set> deleted; - - while (records.size() < reccnt) { - int32_t key = rand(); - int32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - size_t cnt=0; - for (auto rec : records) { - Rec r = {rec.first, rec.second, 1}; - ck_assert_int_eq(ext_wirs->insert(r), 1); - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector> del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(dr); - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - - ck_assert(ext_wirs->validate_tombstone_proportion()); - } - - ck_assert(ext_wirs->validate_tombstone_proportion()); - - gsl_rng_free(rng); - delete ext_wirs; -} -END_TEST - -DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { - auto rng = gsl_rng_alloc(gsl_rng_mt19937); - - auto ext_wirs = new DE(1000, 2, 1); - - std::set records; - std::set to_delete; - std::set deleted; - - while (records.size() < reccnt) { - int32_t key = rand(); - int32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - for (auto rec : records) { - ck_assert_int_eq(ext_wirs->insert(rec), 1); - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(del_vec[i]); - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - gsl_rng_free(rng); - - return ext_wirs; -} - -START_TEST(t_static_structure) -{ - auto rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t reccnt = 100000; - auto ext_wirs = new DE(100, 2, 1); - - std::set records; - std::set to_delete; - std::set deleted; - - while (records.size() < reccnt) { - int32_t key = rand(); - int32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - size_t t_reccnt = 0; - size_t k=0; - for (auto rec : records) { - k++; - ck_assert_int_eq(ext_wirs->insert(rec), 1); - t_reccnt++; - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(del_vec[i]), 1); - - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - auto flat = ext_wirs->create_static_structure(); - ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); - - int32_t prev_key = 0; - for (size_t i=0; iget_record_count(); i++) { - auto k = flat->get_record_at(i)->rec.key; - ck_assert_int_ge(k, prev_key); - prev_key = k; - } - - gsl_rng_free(rng); - delete flat; - delete ext_wirs; -} -END_TEST - - -Suite *unit_testing() -{ - Suite *unit = suite_create("de::DynamicExtension Unit Testing"); - - TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); - tcase_add_test(create, t_create); - suite_add_tcase(unit, create); - - TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); - tcase_add_test(insert, t_insert); - tcase_add_test(insert, t_insert_with_mem_merges); - tcase_add_test(insert, t_debug_insert); - suite_add_tcase(unit, insert); - - - /* - TCase *sampling = tcase_create("de::DynamicExtension::range_sample Testing"); - - tcase_add_test(sampling, t_range_sample_weighted); - suite_add_tcase(unit, sampling); - tcase_add_test(sampling, t_range_sample_memtable); - tcase_add_test(sampling, t_range_sample_memlevels); - */ - - TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); - tcase_add_test(ts, t_tombstone_merging_01); - tcase_set_timeout(ts, 500); - suite_add_tcase(unit, ts); - - TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); - tcase_add_test(flat, t_static_structure); - tcase_set_timeout(flat, 500); - suite_add_tcase(unit, flat); - - return unit; -} - -int run_unit_tests() -{ - int failed = 0; - Suite *unit = unit_testing(); - SRunner *unit_runner = srunner_create(unit); - - srunner_run_all(unit_runner, CK_NORMAL); - failed = srunner_ntests_failed(unit_runner); - srunner_free(unit_runner); - - return failed; -} - - -int main() -{ - int unit_failed = run_unit_tests(); - - return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; -} diff --git a/tests/de_level_concurrent.cpp b/tests/de_level_concurrent.cpp index b52fdd9..40605c4 100644 --- a/tests/de_level_concurrent.cpp +++ b/tests/de_level_concurrent.cpp @@ -22,7 +22,7 @@ #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; #include "include/concurrent_extension.h" diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index 5c95aa2..2ff2d26 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -21,7 +21,7 @@ #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; +typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 44a0759..9b30ac0 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -22,7 +22,7 @@ #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/de_tier_concurrent.cpp b/tests/de_tier_concurrent.cpp index 9387b21..418332b 100644 --- a/tests/de_tier_concurrent.cpp +++ b/tests/de_tier_concurrent.cpp @@ -21,7 +21,7 @@ #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; #include "include/concurrent_extension.h" diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 9d4fe7d..83c37af 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -22,7 +22,7 @@ #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; +typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index 7d8f144..58a7a0f 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -22,7 +22,7 @@ #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/include/rangecount.h b/tests/include/rangecount.h index e09ab12..471af27 100644 --- a/tests/include/rangecount.h +++ b/tests/include/rangecount.h @@ -41,9 +41,9 @@ START_TEST(t_range_count) parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rc::Query::get_query_state(&shard, &parms); - auto result = rc::Query::query(&shard, state, &parms); - rc::Query::delete_query_state(state); + auto state = rc::Query::get_query_state(&shard, &parms); + auto result = rc::Query::query(&shard, state, &parms); + rc::Query::delete_query_state(state); ck_assert_int_eq(result.size(), 1); ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1); @@ -63,9 +63,9 @@ START_TEST(t_buffer_range_count) { auto view = buffer->get_buffer_view(); - auto state = rc::Query::get_buffer_query_state(&view, &parms); - auto result = rc::Query::buffer_query(state, &parms); - rc::Query::delete_buffer_query_state(state); + auto state = rc::Query::get_buffer_query_state(&view, &parms); + auto result = rc::Query::buffer_query(state, &parms); + rc::Query::delete_buffer_query_state(state); ck_assert_int_eq(result.size(), 1); ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1); @@ -90,20 +90,20 @@ START_TEST(t_range_count_merge) size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; - auto state1 = rc::Query::get_query_state(&shard1, &parms); - auto state2 = rc::Query::get_query_state(&shard2, &parms); + auto state1 = rc::Query::get_query_state(&shard1, &parms); + auto state2 = rc::Query::get_query_state(&shard2, &parms); std::vector>> results(2); - results[0] = rc::Query::query(&shard1, state1, &parms); - results[1] = rc::Query::query(&shard2, state2, &parms); + results[0] = rc::Query::query(&shard1, state1, &parms); + results[1] = rc::Query::query(&shard2, state2, &parms); - rc::Query::delete_query_state(state1); - rc::Query::delete_query_state(state2); + rc::Query::delete_query_state(state1); + rc::Query::delete_query_state(state2); ck_assert_int_eq(results[0].size(), 1); ck_assert_int_eq(results[1].size(), 1); - auto result = rc::Query::merge(results, nullptr); + auto result = rc::Query::merge(results, nullptr); ck_assert_int_eq(result[0].key, result_size); diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index b9694a4..dbb71db 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -41,9 +41,9 @@ START_TEST(t_range_query) parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rq::Query::get_query_state(&shard, &parms); - auto result = rq::Query::query(&shard, state, &parms); - rq::Query::delete_query_state(state); + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; iget_buffer_view(); - auto state = rq::Query::get_buffer_query_state(&view, &parms); - auto result = rq::Query::buffer_query(state, &parms); - rq::Query::delete_buffer_query_state(state); + auto state = rq::Query::get_buffer_query_state(&view, &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i::get_query_state(&shard1, &parms); - auto state2 = rq::Query::get_query_state(&shard2, &parms); + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); std::vector>> results(2); - results[0] = rq::Query::query(&shard1, state1, &parms); - results[1] = rq::Query::query(&shard2, state2, &parms); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); - rq::Query::delete_query_state(state1); - rq::Query::delete_query_state(state2); + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); ck_assert_int_eq(results[0].size() + results[1].size(), result_size); @@ -117,7 +117,7 @@ START_TEST(t_range_query_merge) } } - auto result = rq::Query::merge(proc_results, nullptr); + auto result = rq::Query::merge(proc_results, nullptr); std::sort(result.begin(), result.end()); ck_assert_int_eq(result.size(), result_size); diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index 79b9c21..06b0bab 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -22,7 +22,7 @@ using namespace de; -typedef InternalLevel, rq::Query, Rec>> ILevel; +typedef InternalLevel, rq::Query>> ILevel; START_TEST(t_memlevel_merge) { diff --git a/tests/rangequery_tests.cpp b/tests/rangequery_tests.cpp index c78571c..49c73d3 100644 --- a/tests/rangequery_tests.cpp +++ b/tests/rangequery_tests.cpp @@ -21,153 +21,12 @@ using namespace de; typedef ISAMTree Shard; -START_TEST(t_range_query) -{ - auto buffer = create_sequential_mbuffer(100, 1000); - auto shard = Shard(buffer->get_buffer_view()); - - rq::Parms parms; - parms.lower_bound = 300; - parms.upper_bound = 500; - - auto state = rq::Query::get_query_state(&shard, &parms); - auto result = rq::Query::query(&shard, state, &parms); - rq::Query::delete_query_state(state); - - ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); - for (size_t i=0; i(100, 1000); - - rq::Parms parms; - parms.lower_bound = 300; - parms.upper_bound = 500; - - { - auto view = buffer->get_buffer_view(); - auto state = rq::Query::get_buffer_query_state(&view, &parms); - auto result = rq::Query::buffer_query(state, &parms); - rq::Query::delete_buffer_query_state(state); - - ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); - for (size_t i=0; i(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); - - auto shard1 = Shard(buffer1->get_buffer_view()); - auto shard2 = Shard(buffer2->get_buffer_view()); - - rq::Parms parms; - parms.lower_bound = 150; - parms.upper_bound = 500; - - size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; - - auto state1 = rq::Query::get_query_state(&shard1, &parms); - auto state2 = rq::Query::get_query_state(&shard2, &parms); - - std::vector>> results(2); - results[0] = rq::Query::query(&shard1, state1, &parms); - results[1] = rq::Query::query(&shard2, state2, &parms); - - rq::Query::delete_query_state(state1); - rq::Query::delete_query_state(state2); - - ck_assert_int_eq(results[0].size() + results[1].size(), result_size); - - std::vector>> proc_results; - - for (size_t j=0; j>()); - for (size_t i=0; i::merge(proc_results, nullptr); - std::sort(result.begin(), result.end()); - - ck_assert_int_eq(result.size(), result_size); - auto key = parms.lower_bound; - for (size_t i=0; i(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); - - auto shard1 = Shard(buffer1->get_buffer_view()); - auto shard2 = Shard(buffer2->get_buffer_view()); - - std::vector shards = {&shard1, &shard2}; - - auto merged = Shard(shards); - - for (size_t i=100; i<1000; i++) { - Rec r; - r.key = i; - r.value = i; - - auto idx = merged.get_lower_bound(i); - - assert(idx < merged.get_record_count()); - - auto res = merged.get_record_at(idx); - - if (i >=200 && i <400) { - ck_assert_int_lt(res->rec.key, i); - } else { - ck_assert_int_eq(res->rec.key, i); - } - } - - delete buffer1; - delete buffer2; -} -END_TEST - +#include "include/rangequery.h" Suite *unit_testing() { - Suite *unit = suite_create("Range Query Unit Testing"); - - TCase *range_query = tcase_create("de:PGM::range_query Testing"); - tcase_add_test(range_query, t_range_query); - tcase_add_test(range_query, t_buffer_range_query); - tcase_add_test(range_query, t_range_query_merge); - suite_add_tcase(unit, range_query); + Suite *unit = suite_create("Range Count Query Testing"); + inject_rangequery_tests(unit); return unit; } -- cgit v1.2.3 From bd74e27b28bd95267ce50d2e4b6f12b51d9b6aae Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 7 Feb 2024 17:23:23 -0500 Subject: Cleaned up shard files (except VPTree) Cleaned up shard implementations, fixed a few bugs, and set up some tests. There's still some work to be done in creating tests for the weighted sampling operations for the alias and aug btree shards. --- tests/alias_tests.cpp | 352 ++-------------------------------- tests/augbtree_tests.cpp | 360 +---------------------------------- tests/de_level_concurrent.cpp | 4 +- tests/de_level_tag.cpp | 3 +- tests/de_level_tomb.cpp | 3 +- tests/de_tier_concurrent.cpp | 3 +- tests/de_tier_tag.cpp | 3 +- tests/de_tier_tomb.cpp | 3 +- tests/include/concurrent_extension.h | 40 ++-- tests/include/dynamic_extension.h | 36 ++-- tests/include/rangecount.h | 57 +++--- tests/include/rangequery.h | 60 +++--- tests/include/shard_standard.h | 26 +-- tests/include/wirs.h | 181 ++++++++++++++++++ tests/include/wss.h | 144 ++++++++++++++ tests/memisam_tests.cpp | 8 +- tests/pgm_tests.cpp | 325 +------------------------------ tests/rangecount_tests.cpp | 1 + tests/rangequery_tests.cpp | 3 +- tests/triespline_tests.cpp | 6 +- 20 files changed, 488 insertions(+), 1130 deletions(-) create mode 100644 tests/include/wirs.h create mode 100644 tests/include/wss.h (limited to 'tests') diff --git a/tests/alias_tests.cpp b/tests/alias_tests.cpp index e3c736b..98d0c63 100644 --- a/tests/alias_tests.cpp +++ b/tests/alias_tests.cpp @@ -12,358 +12,27 @@ #include "shard/Alias.h" #include "query/wss.h" -#include "testing.h" +#include "framework/structure/MutableBuffer.h" +#include "include/testing.h" + #include using namespace de; -typedef Alias Shard; - -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer(1024, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_alias_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); - - auto shard1 = new Shard(mbuffer1); - auto shard2 = new Shard(mbuffer2); - auto shard3 = new Shard(mbuffer3); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto alias = Shard(buffer); - - for (size_t i=0; iget_data() + i); - r.key = rec->rec.key; - r.value = rec->rec.value; - - auto result = alias.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto alias = Shard(buffer); - - for (size_t i=n + 100; i<2*n; i++) { - WRec r; - r.key = i; - r.value = i; - - auto result = alias.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); - - Shard* shard = new Shard(buffer); - Shard* shard_ts = new Shard(buffer_ts); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST - - -START_TEST(t_alias_query) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - Shard* shard = new Shard(buffer); - - size_t k = 1000; - - size_t cnt[3] = {0}; - wss::Parms parms = {k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = wss::Query::get_query_state(shard, &parms); - ((wss::State *) state)->sample_size = k; - auto result = wss::Query::query(shard, state, &parms); - - total_samples += result.size(); +typedef WRec R; +typedef Alias Shard; - for (size_t j=0; j::delete_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete shard; - delete buffer; -} -END_TEST - - -START_TEST(t_alias_query_merge) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - Shard* shard = new Shard(buffer); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wss::Parms parms = {k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - std::vector>> results(2); - - for (size_t i=0; i<1000; i++) { - auto state1 = wss::Query::get_query_state(shard, &parms); - ((wss::State *) state1)->sample_size = k; - results[0] = wss::Query::query(shard, state1, &parms); - - auto state2 = wss::Query::get_query_state(shard, &parms); - ((wss::State *) state2)->sample_size = k; - results[1] = wss::Query::query(shard, state2, &parms); - - wss::Query::delete_query_state(state1); - wss::Query::delete_query_state(state2); - } - - auto merged = wss::Query::merge(results, nullptr); - - ck_assert_int_eq(merged.size(), 2*k); - for (size_t i=0; i(n); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wss::Parms parms = {k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = wss::Query::get_buffer_query_state(buffer, &parms); - ((wss::BufferState *) state)->sample_size = k; - auto result = wss::Query::buffer_query(buffer, state, &parms); - total_samples += result.size(); - - for (size_t j=0; j::delete_buffer_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_alias_buffer_query_rejection) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wss::Parms parms = {k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = wss::Query::get_buffer_query_state(buffer, &parms); - ((wss::BufferState *) state)->sample_size = k; - auto result = wss::Query::buffer_query(buffer, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_buffer_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .1)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .1)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .1)); - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST +#include "include/shard_standard.h" +#include "include/rangequery.h" Suite *unit_testing() { - Suite *unit = suite_create("Alias Shard Unit Testing"); - - TCase *create = tcase_create("de::Alias constructor Testing"); - tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_alias_init); - tcase_set_timeout(create, 100); - suite_add_tcase(unit, create); - + Suite *unit = suite_create("ISAMTree Shard Unit Testing"); - TCase *tombstone = tcase_create("de:Alias::tombstone cancellation Testing"); - tcase_add_test(tombstone, t_full_cancelation); - suite_add_tcase(unit, tombstone); - - - TCase *lookup = tcase_create("de:Alias:point_lookup Testing"); - tcase_add_test(lookup, t_point_lookup); - tcase_add_test(lookup, t_point_lookup_miss); - suite_add_tcase(unit, lookup); - - - - TCase *sampling = tcase_create("de:Alias::AliasQuery Testing"); - tcase_add_test(sampling, t_alias_query); - tcase_add_test(sampling, t_alias_query_merge); - tcase_add_test(sampling, t_alias_buffer_query_rejection); - tcase_add_test(sampling, t_alias_buffer_query_scan); - suite_add_tcase(unit, sampling); + inject_rangequery_tests(unit); + inject_shard_tests(unit); return unit; } @@ -389,3 +58,4 @@ int main() return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; } + diff --git a/tests/augbtree_tests.cpp b/tests/augbtree_tests.cpp index 1b24344..c7a0885 100644 --- a/tests/augbtree_tests.cpp +++ b/tests/augbtree_tests.cpp @@ -1,7 +1,7 @@ /* - * tests/augbtree_tests.cpp + * tests/isam_tests.cpp * - * Unit tests for AugBTree (Augmented B+Tree) shard + * Unit tests for ISAM Tree shard * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie @@ -11,363 +11,23 @@ */ #include "shard/AugBTree.h" -#include "query/wirs.h" -#include "testing.h" - +#include "include/testing.h" #include using namespace de; -typedef AugBTree Shard; - -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer(1024, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_wirs_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); - - auto shard1 = new Shard(mbuffer1); - auto shard2 = new Shard(mbuffer2); - auto shard3 = new Shard(mbuffer3); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto wirs = Shard(buffer); - - for (size_t i=0; iget_data() + i); - r.key = rec->rec.key; - r.value = rec->rec.value; - - auto result = wirs.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto wirs = Shard(buffer); - - for (size_t i=n + 100; i<2*n; i++) { - WRec r; - r.key = i; - r.value = i; - - auto result = wirs.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); - - Shard* shard = new Shard(buffer); - Shard* shard_ts = new Shard(buffer_ts); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST - - -START_TEST(t_wirs_query) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - Shard* shard = new Shard(buffer); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wirs::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = wirs::Query::get_query_state(shard, &parms); - ((wirs::State *) state)->sample_size = k; - auto result = wirs::Query::query(shard, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete shard; - delete buffer; -} -END_TEST - - -START_TEST(t_wirs_query_merge) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - Shard* shard = new Shard(buffer); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wirs::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - std::vector>> results(2); - - for (size_t i=0; i<1000; i++) { - auto state1 = wirs::Query::get_query_state(shard, &parms); - ((wirs::State *) state1)->sample_size = k; - results[0] = wirs::Query::query(shard, state1, &parms); - - auto state2 = wirs::Query::get_query_state(shard, &parms); - ((wirs::State *) state2)->sample_size = k; - results[1] = wirs::Query::query(shard, state2, &parms); - - wirs::Query::delete_query_state(state1); - wirs::Query::delete_query_state(state2); - } - - auto merged = wirs::Query::merge(results, nullptr); - - ck_assert_int_eq(merged.size(), 2*k); - for (size_t i=0; i(n); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wirs::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = wirs::Query::get_buffer_query_state(buffer, &parms); - ((wirs::BufferState *) state)->sample_size = k; - auto result = wirs::Query::buffer_query(buffer, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_buffer_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_wirs_buffer_query_rejection) -{ - size_t n=1000; - auto buffer = create_weighted_mbuffer(n); - - uint64_t lower_key = 0; - uint64_t upper_key = 5; - - size_t k = 1000; - - size_t cnt[3] = {0}; - wirs::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = wirs::Query::get_buffer_query_state(buffer, &parms); - ((wirs::BufferState *) state)->sample_size = k; - auto result = wirs::Query::buffer_query(buffer, state, &parms); - - total_samples += result.size(); - - for (size_t j=0; j::delete_buffer_query_state(state); - } - - ck_assert(roughly_equal(cnt[0], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[1], (double) total_samples/4.0, total_samples, .05)); - ck_assert(roughly_equal(cnt[2], (double) total_samples/2.0, total_samples, .05)); - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST +typedef WRec R; +typedef AugBTree Shard; +#include "include/shard_standard.h" +#include "include/rangequery.h" Suite *unit_testing() { - Suite *unit = suite_create("AugBTree Shard Unit Testing"); - - TCase *create = tcase_create("de::AugBTree constructor Testing"); - tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_wirs_init); - tcase_set_timeout(create, 100); - suite_add_tcase(unit, create); - - - TCase *tombstone = tcase_create("de:AugBTree::tombstone cancellation Testing"); - tcase_add_test(tombstone, t_full_cancelation); - suite_add_tcase(unit, tombstone); - - - TCase *lookup = tcase_create("de:AugBTree:point_lookup Testing"); - tcase_add_test(lookup, t_point_lookup); - tcase_add_test(lookup, t_point_lookup_miss); - suite_add_tcase(unit, lookup); - + Suite *unit = suite_create("Alias-augmented B+Tree Shard Unit Testing"); - TCase *sampling = tcase_create("de:AugBTree::AugBTreeQuery Testing"); - tcase_add_test(sampling, t_wirs_query); - tcase_add_test(sampling, t_wirs_query_merge); - tcase_add_test(sampling, t_wirs_buffer_query_rejection); - tcase_add_test(sampling, t_wirs_buffer_query_scan); - suite_add_tcase(unit, sampling); + inject_rangequery_tests(unit); + inject_shard_tests(unit); return unit; } diff --git a/tests/de_level_concurrent.cpp b/tests/de_level_concurrent.cpp index 40605c4..2039efb 100644 --- a/tests/de_level_concurrent.cpp +++ b/tests/de_level_concurrent.cpp @@ -17,12 +17,12 @@ #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" -#include "shard/TrieSpline.h" #include using namespace de; -typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +typedef Rec R; +typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; #include "include/concurrent_extension.h" diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index 2ff2d26..75131c4 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -21,7 +21,8 @@ #include using namespace de; -typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; +typedef Rec R; +typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 9b30ac0..6da211d 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -22,7 +22,8 @@ #include using namespace de; -typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef Rec R; +typedef DynamicExtension, rq::Query>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/de_tier_concurrent.cpp b/tests/de_tier_concurrent.cpp index 418332b..722b9bd 100644 --- a/tests/de_tier_concurrent.cpp +++ b/tests/de_tier_concurrent.cpp @@ -21,7 +21,8 @@ #include using namespace de; -typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +typedef Rec R; +typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; #include "include/concurrent_extension.h" diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 83c37af..79bb7bf 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -22,7 +22,8 @@ #include using namespace de; -typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; +typedef Rec R; +typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index 58a7a0f..b1387bb 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -22,7 +22,8 @@ #include using namespace de; -typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef Rec R; +typedef DynamicExtension, rq::Query>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; #include "include/dynamic_extension.h" diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h index 24cb2ce..0993fac 100644 --- a/tests/include/concurrent_extension.h +++ b/tests/include/concurrent_extension.h @@ -8,8 +8,8 @@ * Distributed under the Modified BSD License. * * WARNING: This file must be included in the main unit test set - * after the definition of an appropriate Shard, Query, and Rec - * type. In particular, Rec needs to implement the key-value + * after the definition of an appropriate Shard, Query, and R + * type. In particular, R needs to implement the key-value * pair interface. For other types of record, you'll need to * use a different set of unit tests. */ @@ -30,7 +30,7 @@ #include //using namespace de; -//typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +//typedef DynamicExtension, rq::Query, R>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; */ @@ -54,7 +54,7 @@ START_TEST(t_insert) uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<100; i++) { - Rec r = {key, val}; + R r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); key++; val++; @@ -75,7 +75,7 @@ START_TEST(t_debug_insert) uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<1000; i++) { - Rec r = {key, val}; + R r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); ck_assert_int_eq(test_de->get_record_count(), i+1); key++; @@ -94,7 +94,7 @@ START_TEST(t_insert_with_mem_merges) uint64_t key = 0; uint32_t val = 0; - Rec r = {key, val}; + R r = {key, val}; for (size_t i=0; i<1000; i++) { ck_assert_int_eq(test_de->insert(r), 1); r.key++; @@ -148,7 +148,7 @@ START_TEST(t_range_query) size_t i=0; while ( i < keys.size()) { - Rec r = {keys[i], (uint32_t) i}; + R r = {keys[i], (uint32_t) i}; if (test_de->insert(r)) { i++; } else { @@ -166,7 +166,7 @@ START_TEST(t_range_query) uint64_t lower_key = keys[idx]; uint64_t upper_key = keys[idx + 250]; - rq::Parms p; + rq::Parms p; p.lower_bound = lower_key; p.upper_bound = upper_key; @@ -210,7 +210,7 @@ START_TEST(t_tombstone_merging_01) size_t deletes = 0; size_t cnt=0; for (auto rec : records) { - Rec r = {rec.first, rec.second}; + R r = {rec.first, rec.second}; while (!test_de->insert(r)) { _mm_pause(); } @@ -220,7 +220,7 @@ START_TEST(t_tombstone_merging_01) std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; ierase(dr)) { _mm_pause(); } @@ -249,9 +249,9 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { auto test_de = new DE(1000, 10000, 2); - std::set records; - std::set to_delete; - std::set deleted; + std::set records; + std::set to_delete; + std::set deleted; while (records.size() < reccnt) { uint64_t key = rand(); @@ -267,7 +267,7 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { ck_assert_int_eq(test_de->insert(rec), 1); if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; + std::vector del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; i records; - std::set to_delete; - std::set deleted; + std::set records; + std::set to_delete; + std::set deleted; while (records.size() < reccnt) { uint64_t key = rand(); @@ -319,7 +319,7 @@ START_TEST(t_static_structure) t_reccnt++; if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; + std::vector del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; iget_tombstone_count(), test_de->get_record_count()); + //fprintf(stderr, "Tombstones: %ld\tRords: %ld\n", test_de->get_tombstone_count(), test_de->get_record_count()); //fprintf(stderr, "Inserts: %ld\tDeletes:%ld\tNet:%ld\n", reccnt, deletes, reccnt - deletes); auto flat = test_de->create_static_structure(true); - //fprintf(stderr, "Flat: Tombstones: %ld\tRecords %ld\n", flat->get_tombstone_count(), flat->get_record_count()); + //fprintf(stderr, "Flat: Tombstones: %ld\tRords %ld\n", flat->get_tombstone_count(), flat->get_record_count()); //ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); uint64_t prev_key = 0; diff --git a/tests/include/dynamic_extension.h b/tests/include/dynamic_extension.h index 5a08f5a..f0f13dd 100644 --- a/tests/include/dynamic_extension.h +++ b/tests/include/dynamic_extension.h @@ -8,8 +8,8 @@ * Distributed under the Modified BSD License. * * WARNING: This file must be included in the main unit test set - * after the definition of an appropriate Shard, Query, and Rec - * type. In particular, Rec needs to implement the key-value + * after the definition of an appropriate Shard, Query, and R + * type. In particular, R needs to implement the key-value * pair interface. For other types of record, you'll need to * use a different set of unit tests. */ @@ -29,7 +29,7 @@ //#include "query/rangequery.h" //#include //using namespace de; -//typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; +//typedef DynamicExtension, rq::Query, R>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; START_TEST(t_create) @@ -52,7 +52,7 @@ START_TEST(t_insert) uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<100; i++) { - Rec r = {key, val}; + R r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); key++; val++; @@ -73,7 +73,7 @@ START_TEST(t_debug_insert) uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<1000; i++) { - Rec r = {key, val}; + R r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); ck_assert_int_eq(test_de->get_record_count(), i+1); key++; @@ -92,7 +92,7 @@ START_TEST(t_insert_with_mem_merges) uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<300; i++) { - Rec r = {key, val}; + R r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); key++; val++; @@ -123,7 +123,7 @@ START_TEST(t_range_query) std::shuffle(keys.begin(), keys.end(), gen); for (size_t i=0; iinsert(r), 1); } @@ -136,7 +136,7 @@ START_TEST(t_range_query) uint64_t lower_key = keys[idx]; uint64_t upper_key = keys[idx + 250]; - rq::Parms p; + rq::Parms p; p.lower_bound = lower_key; p.upper_bound = upper_key; @@ -177,7 +177,7 @@ START_TEST(t_tombstone_merging_01) size_t deletes = 0; size_t cnt=0; for (auto rec : records) { - Rec r = {rec.first, rec.second}; + R r = {rec.first, rec.second}; ck_assert_int_eq(test_de->insert(r), 1); if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { @@ -185,7 +185,7 @@ START_TEST(t_tombstone_merging_01) std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; ierase(dr); deletes++; to_delete.erase(del_vec[i]); @@ -212,9 +212,9 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { auto test_de = new DE(1000, 10000, 2); - std::set records; - std::set to_delete; - std::set deleted; + std::set records; + std::set to_delete; + std::set deleted; while (records.size() < reccnt) { uint64_t key = rand(); @@ -230,7 +230,7 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { ck_assert_int_eq(test_de->insert(rec), 1); if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; + std::vector del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; i records; - std::set to_delete; - std::set deleted; + std::set records; + std::set to_delete; + std::set deleted; while (records.size() < reccnt) { uint64_t key = rand(); @@ -280,7 +280,7 @@ START_TEST(t_static_structure) t_reccnt++; if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; + std::vector del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; i //using namespace de; -//typedef ISAMTree Shard; +//typedef ISAMTree Shard; + + +#include "query/rangecount.h" START_TEST(t_range_count) { - auto buffer = create_sequential_mbuffer(100, 1000); + auto buffer = create_sequential_mbuffer(100, 1000); auto shard = Shard(buffer->get_buffer_view()); - rc::Parms parms; + rc::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rc::Query::get_query_state(&shard, &parms); - auto result = rc::Query::query(&shard, state, &parms); - rc::Query::delete_query_state(state); + auto state = rc::Query::get_query_state(&shard, &parms); + auto result = rc::Query::query(&shard, state, &parms); + rc::Query::delete_query_state(state); ck_assert_int_eq(result.size(), 1); ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1); @@ -55,17 +58,17 @@ END_TEST START_TEST(t_buffer_range_count) { - auto buffer = create_sequential_mbuffer(100, 1000); + auto buffer = create_sequential_mbuffer(100, 1000); - rc::Parms parms; + rc::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; { auto view = buffer->get_buffer_view(); - auto state = rc::Query::get_buffer_query_state(&view, &parms); - auto result = rc::Query::buffer_query(state, &parms); - rc::Query::delete_buffer_query_state(state); + auto state = rc::Query::get_buffer_query_state(&view, &parms); + auto result = rc::Query::buffer_query(state, &parms); + rc::Query::delete_buffer_query_state(state); ck_assert_int_eq(result.size(), 1); ck_assert_int_eq(result[0].rec.key, parms.upper_bound - parms.lower_bound + 1); @@ -78,32 +81,32 @@ END_TEST START_TEST(t_range_count_merge) { - auto buffer1 = create_sequential_mbuffer(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); + auto buffer1 = create_sequential_mbuffer(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); auto shard1 = Shard(buffer1->get_buffer_view()); auto shard2 = Shard(buffer2->get_buffer_view()); - rc::Parms parms; + rc::Parms parms; parms.lower_bound = 150; parms.upper_bound = 500; size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; - auto state1 = rc::Query::get_query_state(&shard1, &parms); - auto state2 = rc::Query::get_query_state(&shard2, &parms); + auto state1 = rc::Query::get_query_state(&shard1, &parms); + auto state2 = rc::Query::get_query_state(&shard2, &parms); - std::vector>> results(2); - results[0] = rc::Query::query(&shard1, state1, &parms); - results[1] = rc::Query::query(&shard2, state2, &parms); + std::vector>> results(2); + results[0] = rc::Query::query(&shard1, state1, &parms); + results[1] = rc::Query::query(&shard2, state2, &parms); - rc::Query::delete_query_state(state1); - rc::Query::delete_query_state(state2); + rc::Query::delete_query_state(state1); + rc::Query::delete_query_state(state2); ck_assert_int_eq(results[0].size(), 1); ck_assert_int_eq(results[1].size(), 1); - auto result = rc::Query::merge(results, nullptr); + auto result = rc::Query::merge(results, nullptr); ck_assert_int_eq(result[0].key, result_size); @@ -115,8 +118,8 @@ END_TEST START_TEST(t_lower_bound) { - auto buffer1 = create_sequential_mbuffer(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); + auto buffer1 = create_sequential_mbuffer(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); auto shard1 = new Shard(buffer1->get_buffer_view()); auto shard2 = new Shard(buffer2->get_buffer_view()); @@ -126,7 +129,7 @@ START_TEST(t_lower_bound) auto merged = Shard(shards); for (size_t i=100; i<1000; i++) { - Rec r; + R r; r.key = i; r.value = i; diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index dbb71db..a8a73f7 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -9,8 +9,8 @@ * Distributed under the Modified BSD License. * * WARNING: This file must be included in the main unit test set - * after the definition of an appropriate Shard and Rec - * type. In particular, Rec needs to implement the key-value + * after the definition of an appropriate Shard and R + * type. In particular, R needs to implement the key-value * pair interface and Shard needs to support lower_bound. * For other types of record and shard, you'll need to * use a different set of unit tests. @@ -29,21 +29,23 @@ //#include "testing.h" //#include //using namespace de; -//typedef ISAMTree Shard; +//typedef ISAMTree Shard; + +#include "query/rangequery.h" START_TEST(t_range_query) { - auto buffer = create_sequential_mbuffer(100, 1000); + auto buffer = create_sequential_mbuffer(100, 1000); auto shard = Shard(buffer->get_buffer_view()); - rq::Parms parms; + rq::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rq::Query::get_query_state(&shard, &parms); - auto result = rq::Query::query(&shard, state, &parms); - rq::Query::delete_query_state(state); + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i(100, 1000); + auto buffer = create_sequential_mbuffer(100, 1000); - rq::Parms parms; + rq::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; { auto view = buffer->get_buffer_view(); - auto state = rq::Query::get_buffer_query_state(&view, &parms); - auto result = rq::Query::buffer_query(state, &parms); - rq::Query::delete_buffer_query_state(state); + auto state = rq::Query::get_buffer_query_state(&view, &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); + auto buffer1 = create_sequential_mbuffer(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); auto shard1 = Shard(buffer1->get_buffer_view()); auto shard2 = Shard(buffer2->get_buffer_view()); - rq::Parms parms; + rq::Parms parms; parms.lower_bound = 150; parms.upper_bound = 500; size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; - auto state1 = rq::Query::get_query_state(&shard1, &parms); - auto state2 = rq::Query::get_query_state(&shard2, &parms); + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); - std::vector>> results(2); - results[0] = rq::Query::query(&shard1, state1, &parms); - results[1] = rq::Query::query(&shard2, state2, &parms); + std::vector>> results(2); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); - rq::Query::delete_query_state(state1); - rq::Query::delete_query_state(state2); + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); ck_assert_int_eq(results[0].size() + results[1].size(), result_size); - std::vector>> proc_results; + std::vector>> proc_results; for (size_t j=0; j>()); + proc_results.emplace_back(std::vector>()); for (size_t i=0; i::merge(proc_results, nullptr); + auto result = rq::Query::merge(proc_results, nullptr); std::sort(result.begin(), result.end()); ck_assert_int_eq(result.size(), result_size); @@ -137,8 +139,8 @@ END_TEST START_TEST(t_lower_bound) { - auto buffer1 = create_sequential_mbuffer(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); + auto buffer1 = create_sequential_mbuffer(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); auto shard1 = new Shard(buffer1->get_buffer_view()); auto shard2 = new Shard(buffer2->get_buffer_view()); @@ -148,7 +150,7 @@ START_TEST(t_lower_bound) auto merged = Shard(shards); for (size_t i=100; i<1000; i++) { - Rec r; + R r; r.key = i; r.value = i; diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h index ddd7614..f50c1cb 100644 --- a/tests/include/shard_standard.h +++ b/tests/include/shard_standard.h @@ -8,8 +8,8 @@ * Distributed under the Modified BSD License. * * WARNING: This file must be included in the main unit test set - * after the definition of an appropriate Shard and Rec - * type. In particular, Rec needs to implement the key-value + * after the definition of an appropriate Shard and R + * type. In particular, R needs to implement the key-value * pair interface. For other types of record, you'll need to * use a different set of unit tests. */ @@ -26,11 +26,11 @@ //#include "testing.h" //#include //using namespace de; -//typedef ISAMTree Shard; +//typedef ISAMTree Shard; START_TEST(t_mbuffer_init) { - auto buffer = new MutableBuffer(512, 1024); + auto buffer = new MutableBuffer(512, 1024); for (uint64_t i = 512; i > 0; i--) { uint32_t v = i; buffer->append({i,v, 1}); @@ -57,9 +57,9 @@ START_TEST(t_mbuffer_init) START_TEST(t_shard_init) { size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); auto shard1 = new Shard(mbuffer1->get_buffer_view()); auto shard2 = new Shard(mbuffer2->get_buffer_view()); @@ -108,8 +108,8 @@ START_TEST(t_shard_init) START_TEST(t_full_cancelation) { size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); + auto buffer = create_double_seq_mbuffer(n, false); + auto buffer_ts = create_double_seq_mbuffer(n, true); Shard* shard = new Shard(buffer->get_buffer_view()); Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); @@ -139,14 +139,14 @@ START_TEST(t_point_lookup) { size_t n = 10000; - auto buffer = create_double_seq_mbuffer(n, false); + auto buffer = create_double_seq_mbuffer(n, false); auto isam = Shard(buffer->get_buffer_view()); { auto view = buffer->get_buffer_view(); for (size_t i=0; irec.key; r.value = rec->rec.value; @@ -167,11 +167,11 @@ START_TEST(t_point_lookup_miss) { size_t n = 10000; - auto buffer = create_double_seq_mbuffer(n, false); + auto buffer = create_double_seq_mbuffer(n, false); auto isam = Shard(buffer->get_buffer_view()); for (size_t i=n + 100; i<2*n; i++) { - Rec r; + R r; r.key = i; r.value = i; diff --git a/tests/include/wirs.h b/tests/include/wirs.h new file mode 100644 index 0000000..90cd22d --- /dev/null +++ b/tests/include/wirs.h @@ -0,0 +1,181 @@ +/* + * tests/include/rangequery.h + * + * Standardized unit tests for range queries against supporting + * shard types + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and R + * type. In particular, R needs to implement the key-value + * pair interface and Shard needs to support lower_bound. + * For other types of record and shard, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; + + +START_TEST(t_range_query) +{ + auto buffer = create_sequential_mbuffer(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 1000); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + { + auto view = buffer->get_buffer_view(); + auto state = rq::Query::get_buffer_query_state(&view, &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); + + std::vector>> results(2); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); + + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector>> proc_results; + + for (size_t j=0; j>()); + for (size_t i=0; i::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = new Shard(buffer1->get_buffer_view()); + auto shard2 = new Shard(buffer2->get_buffer_view()); + + std::vector shards = {shard1, shard2}; + + auto merged = Shard(shards); + + for (size_t i=100; i<1000; i++) { + R r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; + delete shard1; + delete shard2; +} +END_TEST + +static void inject_rangequery_tests(Suite *suite) { + TCase *range_query = tcase_create("Range Query Testing"); + tcase_add_test(range_query, t_range_query); + tcase_add_test(range_query, t_buffer_range_query); + tcase_add_test(range_query, t_range_query_merge); + suite_add_tcase(suite, range_query); +} diff --git a/tests/include/wss.h b/tests/include/wss.h new file mode 100644 index 0000000..f0ac74c --- /dev/null +++ b/tests/include/wss.h @@ -0,0 +1,144 @@ +/* + * tests/include/rangequery.h + * + * Standardized unit tests for range queries against supporting + * shard types + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and R + * type. In particular, R needs to implement the key-value + * pair interface and Shard needs to support lower_bound. + * For other types of record and shard, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +#include "shard/Alias.h" +#include "testing.h" +#include +using namespace de; +typedef Alias Shard; + +#include "query/wss.h" + +START_TEST(t_wss_query) +{ + auto buffer = create_weighted_mbuffer(1000); + auto shard = Shard(buffer->get_buffer_view()); + + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + wss::Parms parms; + parms.rng = rng; + parms.sample_size = 20; + + auto state = wss::Query::get_query_state(&shard, &parms); + auto result = wss::Query::query(&shard, state, &parms); + wss::Query::delete_query_state(state); + + delete buffer; + gsl_rng_free(rng); +} +END_TEST + + +START_TEST(t_buffer_wss_query) +{ + auto buffer = create_weighted_mbuffer(1000); + + + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + wss::Parms parms; + parms.rng = rng; + + { + auto view = buffer->get_buffer_view(); + auto state = wss::Query::get_buffer_query_state(&view, &parms); + auto result = wss::Query::buffer_query(state, &parms); + wss::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.sample_size); + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + wss::Parms parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = wss::Query::get_query_state(&shard1, &parms); + auto state2 = wss::Query::get_query_state(&shard2, &parms); + + std::vector>> results(2); + results[0] = wss::Query::query(&shard1, state1, &parms); + results[1] = wss::Query::query(&shard2, state2, &parms); + + wss::Query::delete_query_state(state1); + wss::Query::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector>> proc_results; + + for (size_t j=0; j>()); + for (size_t i=0; i::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i - - using namespace de; -typedef ISAMTree Shard; +typedef Rec R; +typedef ISAMTree Shard; #include "include/shard_standard.h" #include "include/rangequery.h" Suite *unit_testing() { - Suite *unit = suite_create("ISAMTree Shard Unit Testing"); + Suite *unit = suite_create("Alias-augmented B+Tree Shard Unit Testing"); inject_rangequery_tests(unit); inject_shard_tests(unit); diff --git a/tests/pgm_tests.cpp b/tests/pgm_tests.cpp index c7750ac..ee350de 100644 --- a/tests/pgm_tests.cpp +++ b/tests/pgm_tests.cpp @@ -1,7 +1,7 @@ /* - * tests/irs_tests.cpp + * tests/isam_tests.cpp * - * Unit tests for PGM (Augmented B+Tree) shard + * Unit tests for ISAM Tree shard * * Copyright (C) 2023 Douglas Rumbaugh * Dong Xie @@ -11,330 +11,23 @@ */ #include "shard/PGM.h" -#include "query/rangequery.h" -#include "testing.h" - +#include "include/testing.h" #include using namespace de; -typedef PGM Shard; - -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer(1024, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_irs_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); - - auto shard1 = new Shard(mbuffer1); - auto shard2 = new Shard(mbuffer2); - auto shard3 = new Shard(mbuffer3); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto shard = Shard(buffer); - - for (size_t i=0; iget_data() + i); - r.key = rec->rec.key; - r.value = rec->rec.value; - - auto result = shard.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto isam = Shard(buffer); - - for (size_t i=n + 100; i<2*n; i++) { - Rec r; - r.key = i; - r.value = i; - - auto result = isam.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - - -START_TEST(t_range_query) -{ - auto buffer = create_sequential_mbuffer(100, 1000); - auto shard = Shard(buffer); - - rq::Parms parms; - parms.lower_bound = 300; - parms.upper_bound = 500; - - auto state = rq::Query::get_query_state(&shard, &parms); - auto result = rq::Query::query(&shard, state, &parms); - rq::Query::delete_query_state(state); - - ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); - for (size_t i=0; i(100, 1000); - - rq::Parms parms; - parms.lower_bound = 300; - parms.upper_bound = 500; - - auto state = rq::Query::get_buffer_query_state(buffer, &parms); - auto result = rq::Query::buffer_query(buffer, state, &parms); - rq::Query::delete_buffer_query_state(state); - - ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); - for (size_t i=0; i(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); - - auto shard1 = Shard(buffer1); - auto shard2 = Shard(buffer2); - - rq::Parms parms; - parms.lower_bound = 150; - parms.upper_bound = 500; - - size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; - - auto state1 = rq::Query::get_query_state(&shard1, &parms); - auto state2 = rq::Query::get_query_state(&shard2, &parms); - - std::vector>> results(2); - results[0] = rq::Query::query(&shard1, state1, &parms); - results[1] = rq::Query::query(&shard2, state2, &parms); - - rq::Query::delete_query_state(state1); - rq::Query::delete_query_state(state2); - - ck_assert_int_eq(results[0].size() + results[1].size(), result_size); - - std::vector>> proc_results; - - for (size_t j=0; j>()); - for (size_t i=0; i::merge(proc_results, nullptr); - std::sort(result.begin(), result.end()); - - ck_assert_int_eq(result.size(), result_size); - auto key = parms.lower_bound; - for (size_t i=0; i(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); - - de::PGM *shards[2]; - - auto shard1 = Shard(buffer1); - auto shard2 = Shard(buffer2); - - shards[0] = &shard1; - shards[1] = &shard2; - - auto merged = Shard(shards, 2); - - for (size_t i=100; i<1000; i++) { - Rec r; - r.key = i; - r.value = i; - - auto idx = merged.get_lower_bound(i); - - assert(idx < merged.get_record_count()); - - auto res = merged.get_record_at(idx); - - if (i >=200 && i <400) { - ck_assert_int_lt(res->rec.key, i); - } else { - ck_assert_int_eq(res->rec.key, i); - } - } - - delete buffer1; - delete buffer2; -} -END_TEST - - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); - - Shard* shard = new Shard(buffer); - Shard* shard_ts = new Shard(buffer_ts); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST +typedef Rec R; +typedef PGM Shard; +#include "include/shard_standard.h" +#include "include/rangequery.h" Suite *unit_testing() { Suite *unit = suite_create("PGM Shard Unit Testing"); - TCase *create = tcase_create("de::PGM constructor Testing"); - tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_irs_init); - tcase_set_timeout(create, 100); - suite_add_tcase(unit, create); - - - TCase *tombstone = tcase_create("de:PGM::tombstone cancellation Testing"); - tcase_add_test(tombstone, t_full_cancelation); - suite_add_tcase(unit, tombstone); - - - TCase *lookup = tcase_create("de:PGM:point_lookup Testing"); - tcase_add_test(lookup, t_point_lookup); - tcase_add_test(lookup, t_point_lookup_miss); - tcase_add_test(lookup, t_lower_bound); - suite_add_tcase(unit, lookup); - - TCase *range_query = tcase_create("de:PGM::range_query Testing"); - tcase_add_test(range_query, t_range_query); - tcase_add_test(range_query, t_buffer_range_query); - tcase_add_test(range_query, t_range_query_merge); - suite_add_tcase(unit, range_query); + inject_rangequery_tests(unit); + inject_shard_tests(unit); return unit; } diff --git a/tests/rangecount_tests.cpp b/tests/rangecount_tests.cpp index fe3a587..3be8234 100644 --- a/tests/rangecount_tests.cpp +++ b/tests/rangecount_tests.cpp @@ -19,6 +19,7 @@ using namespace de; +typedef Rec R; typedef ISAMTree Shard; #include "include/rangecount.h" diff --git a/tests/rangequery_tests.cpp b/tests/rangequery_tests.cpp index 49c73d3..bf5fb5e 100644 --- a/tests/rangequery_tests.cpp +++ b/tests/rangequery_tests.cpp @@ -19,7 +19,8 @@ using namespace de; -typedef ISAMTree Shard; +typedef Rec R; +typedef ISAMTree Shard; #include "include/rangequery.h" diff --git a/tests/triespline_tests.cpp b/tests/triespline_tests.cpp index 9df278f..e884360 100644 --- a/tests/triespline_tests.cpp +++ b/tests/triespline_tests.cpp @@ -12,19 +12,19 @@ #include "shard/TrieSpline.h" #include "include/testing.h" -#include "query/rangequery.h" #include using namespace de; -typedef TrieSpline Shard; +typedef Rec R; +typedef TrieSpline Shard; #include "include/shard_standard.h" #include "include/rangequery.h" Suite *unit_testing() { - Suite *unit = suite_create("ISAMTree Shard Unit Testing"); + Suite *unit = suite_create("Triespline Shard Unit Testing"); inject_rangequery_tests(unit); inject_shard_tests(unit); -- cgit v1.2.3 From 711769574e647839677739192698e400529efe75 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 8 Feb 2024 16:38:44 -0500 Subject: Updated VPTree to new shard/query interfaces --- tests/include/shard_standard.h | 16 ++++--- tests/include/testing.h | 59 ++++++++++++-------------- tests/vptree_tests.cpp | 94 +++++++++++++++++++++++------------------- 3 files changed, 87 insertions(+), 82 deletions(-) (limited to 'tests') diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h index f50c1cb..7d17dcb 100644 --- a/tests/include/shard_standard.h +++ b/tests/include/shard_standard.h @@ -22,18 +22,22 @@ * should be included in the source file that includes this one, above the * include statement. */ -//#include "shard/ISAMTree.h" -//#include "testing.h" -//#include -//using namespace de; -//typedef ISAMTree Shard; +/* +#include "shard/ISAMTree.h" +#include "shard/ISAMTree.h" +#include "testing.h" +#include +using namespace de; +typedef Rec R; +typedef ISAMTree Shard; +*/ START_TEST(t_mbuffer_init) { auto buffer = new MutableBuffer(512, 1024); for (uint64_t i = 512; i > 0; i--) { uint32_t v = i; - buffer->append({i,v, 1}); + buffer->append({i, v, 1}); } for (uint64_t i = 1; i <= 256; ++i) { diff --git a/tests/include/testing.h b/tests/include/testing.h index 4e660dd..f935b53 100644 --- a/tests/include/testing.h +++ b/tests/include/testing.h @@ -23,7 +23,7 @@ typedef de::WeightedRecord WRec; typedef de::Record Rec; -typedef de::EuclidPoint PRec; +typedef de::EuclidPoint PRec; template std::vector strip_wrapping(std::vector> vec) { @@ -76,55 +76,48 @@ static bool roughly_equal(int n1, int n2, size_t mag, double epsilon) { return ((double) std::abs(n1 - n2) / (double) mag) < epsilon; } -static de::MutableBuffer *create_2d_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - for (int64_t i=0; iappend({rand(), rand()}); - } - - return buffer; -} - -static de::MutableBuffer *create_2d_sequential_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt/2, cnt); - for (int64_t i=0; iappend({i, i}); - } - - return buffer; -} - -template +template static de::MutableBuffer *create_test_mbuffer(size_t cnt) { auto buffer = new de::MutableBuffer(cnt/2, cnt); R rec; - for (size_t i = 0; i < cnt; i++) { - rec.key = rand(); - rec.value = rand(); + if constexpr (de::KVPInterface) { + for (size_t i = 0; i < cnt; i++) { + rec.key = rand(); + rec.value = rand(); - if constexpr (de::WeightedRecordInterface) { - rec.weight = 1; - } + if constexpr (de::WeightedRecordInterface) { + rec.weight = 1; + } - buffer->append(rec); - } + buffer->append(rec); + } + } else if constexpr (de::NDRecordInterface) { + for (size_t i=0; iappend({a, b}); + } + } return buffer; } -template -static de::MutableBuffer *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop) +template +static de::MutableBuffer *create_sequential_mbuffer(size_t start, size_t stop) { size_t cnt = stop - start; auto buffer = new de::MutableBuffer(cnt/2, cnt); for (size_t i=start; i) { + rec.key = i; + rec.value = i; + } else if constexpr (de::NDRecordInterface) { + rec = {i, i}; + } if constexpr (de::WeightedRecordInterface) { rec.weight = 1; diff --git a/tests/vptree_tests.cpp b/tests/vptree_tests.cpp index fb568dd..ff99ba6 100644 --- a/tests/vptree_tests.cpp +++ b/tests/vptree_tests.cpp @@ -9,27 +9,28 @@ * */ + +#include "include/testing.h" #include "shard/VPTree.h" -#include "testing.h" -#include "vptree.hpp" +#include "query/knn.h" #include using namespace de; - -typedef VPTree Shard; +typedef PRec R; +typedef VPTree Shard; START_TEST(t_mbuffer_init) { size_t n= 24; - auto buffer = new MutableBuffer(n, n); + auto buffer = new MutableBuffer(n/2, n); for (int64_t i=0; iappend({i, i}); + buffer->append({(uint64_t) i, (uint64_t) i}); } - Shard* shard = new Shard(buffer); + Shard* shard = new Shard(buffer->get_buffer_view()); ck_assert_uint_eq(shard->get_record_count(), n); delete buffer; @@ -40,16 +41,16 @@ START_TEST(t_mbuffer_init) START_TEST(t_wss_init) { size_t n = 512; - auto mbuffer1 = create_2d_mbuffer(n); - auto mbuffer2 = create_2d_mbuffer(n); - auto mbuffer3 = create_2d_mbuffer(n); + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); - auto shard1 = new Shard(mbuffer1); - auto shard2 = new Shard(mbuffer2); - auto shard3 = new Shard(mbuffer3); + auto shard1 = new Shard(mbuffer1->get_buffer_view()); + auto shard2 = new Shard(mbuffer2->get_buffer_view()); + auto shard3 = new Shard(mbuffer3->get_buffer_view()); - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); + std::vector shards = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards); ck_assert_int_eq(shard4->get_record_count(), n * 3); ck_assert_int_eq(shard4->get_tombstone_count(), 0); @@ -69,19 +70,23 @@ START_TEST(t_point_lookup) { size_t n = 16; - auto buffer = create_2d_sequential_mbuffer(n); - auto wss = Shard(buffer); + auto buffer = create_sequential_mbuffer(0, n); + auto wss = Shard(buffer->get_buffer_view()); - for (size_t i=0; iget_data() + i); - r.data[0] = rec->rec.data[0]; - r.data[1] = rec->rec.data[1]; + { + auto bv = buffer->get_buffer_view(); - auto result = wss.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.data[0], r.data[0]); - ck_assert_int_eq(result->rec.data[1], r.data[1]); + for (size_t i=0; irec.data[0]; + r.data[1] = rec->rec.data[1]; + + auto result = wss.point_lookup(r); + ck_assert_ptr_nonnull(result); + ck_assert_int_eq(result->rec.data[0], r.data[0]); + ck_assert_int_eq(result->rec.data[1], r.data[1]); + } } delete buffer; @@ -93,8 +98,8 @@ START_TEST(t_point_lookup_miss) { size_t n = 10000; - auto buffer = create_2d_sequential_mbuffer(n); - auto wss = Shard(buffer); + auto buffer = create_sequential_mbuffer(0, n); + auto wss = Shard(buffer->get_buffer_view()); for (size_t i=n + 100; i<2*n; i++) { PRec r; @@ -112,24 +117,27 @@ START_TEST(t_point_lookup_miss) START_TEST(t_buffer_query) { size_t n = 10000; - auto buffer = create_2d_sequential_mbuffer(n); + auto buffer = create_sequential_mbuffer(0, n); PRec target; target.data[0] = 120; target.data[1] = 120; - KNNQueryParms p; + knn::Parms p; p.k = 10; p.point = target; - auto state = KNNQuery::get_buffer_query_state(buffer, &p); - auto result = KNNQuery::buffer_query(buffer, state, &p); - KNNQuery::delete_buffer_query_state(state); + { + auto bv = buffer->get_buffer_view(); + auto state = knn::Query::get_buffer_query_state(&bv, &p); + auto result = knn::Query::buffer_query(state, &p); + knn::Query::delete_buffer_query_state(state); - std::sort(result.begin(), result.end()); - size_t start = 120 - 5; - for (size_t i=0; i(0, n); - auto vptree = VPTree(buffer); + auto vptree = VPTree(buffer->get_buffer_view()); - KNNQueryParms p; + knn::Parms p; for (size_t i=0; i<100; i++) { p.k = rand() % 150; p.point.data[0] = rand() % (n-p.k); p.point.data[1] = p.point.data[0]; - auto state = KNNQuery::get_query_state(&vptree, &p); - auto results = KNNQuery::query(&vptree, state, &p); - KNNQuery::delete_query_state(state); + auto state = knn::Query::get_query_state(&vptree, &p); + auto results = knn::Query::query(&vptree, state, &p); + knn::Query::delete_query_state(state); ck_assert_int_eq(results.size(), p.k); -- cgit v1.2.3 From aa1b40e9249afc03bf1a2f35de4cbf67c7f9b47e Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 9 Feb 2024 12:42:55 -0500 Subject: Framework: Fixed a bug where tagged deletes didn't release the epoch --- tests/include/dynamic_extension.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'tests') diff --git a/tests/include/dynamic_extension.h b/tests/include/dynamic_extension.h index f0f13dd..6e9b16c 100644 --- a/tests/include/dynamic_extension.h +++ b/tests/include/dynamic_extension.h @@ -22,14 +22,16 @@ * should be included in the source file that includes this one, above the * include statement. */ -//#include "testing.h" -//#include "framework/DynamicExtension.h" -//#include "framework/scheduling/SerialScheduler.h" -//#include "shard/ISAMTree.h" -//#include "query/rangequery.h" -//#include -//using namespace de; -//typedef DynamicExtension, rq::Query, R>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; +/* +#include "testing.h" +#include "framework/DynamicExtension.h" +#include "framework/scheduling/SerialScheduler.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include +using namespace de; +typedef DynamicExtension, rq::Query, R>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; +*/ START_TEST(t_create) -- cgit v1.2.3