From ba65c8976f54d4da2467074235a12f5be0bd5ebc Mon Sep 17 00:00:00 2001 From: "Douglas B. Rumbaugh" Date: Sun, 22 Dec 2024 15:12:13 -0500 Subject: Continued development --- tests/de_bsm_tag.cpp | 5 +- tests/de_bsm_tomb.cpp | 5 +- tests/de_level_concurrent.cpp | 5 +- tests/de_level_tag.cpp | 4 +- tests/de_level_tomb.cpp | 4 +- tests/de_tier_concurrent.cpp | 5 +- tests/de_tier_tag.cpp | 4 +- tests/de_tier_tomb.cpp | 4 +- tests/include/concurrent_extension.h | 78 ++++++++--------------------- tests/include/dynamic_extension.h | 81 ++++++++---------------------- tests/include/shard_standard.h | 4 +- tests/include/shard_string.h | 2 +- tests/internal_level_tests.cpp | 95 ------------------------------------ tests/vptree_tests.cpp | 2 +- 14 files changed, 73 insertions(+), 225 deletions(-) delete mode 100644 tests/internal_level_tests.cpp (limited to 'tests') diff --git a/tests/de_bsm_tag.cpp b/tests/de_bsm_tag.cpp index 4063cfe..cc76f05 100644 --- a/tests/de_bsm_tag.cpp +++ b/tests/de_bsm_tag.cpp @@ -18,6 +18,8 @@ #include "shard/ISAMTree.h" #include "query/rangequery.h" +#include "framework/reconstruction/BSMPolicy.h" + #include using namespace de; @@ -25,7 +27,8 @@ typedef Rec R; typedef ISAMTree S; typedef rq::Query Q; -typedef DynamicExtension DE; +typedef DynamicExtension DE; +ReconstructionPolicy *recon = new BSMPolicy(1000); #include "include/dynamic_extension.h" diff --git a/tests/de_bsm_tomb.cpp b/tests/de_bsm_tomb.cpp index 3a24e87..205de4d 100644 --- a/tests/de_bsm_tomb.cpp +++ b/tests/de_bsm_tomb.cpp @@ -18,6 +18,8 @@ #include "shard/ISAMTree.h" #include "query/rangequery.h" +#include "framework/reconstruction/BSMPolicy.h" + #include using namespace de; @@ -25,7 +27,8 @@ typedef Rec R; typedef ISAMTree S; typedef rq::Query Q; -typedef DynamicExtension DE; +typedef DynamicExtension DE; +ReconstructionPolicy *recon = new BSMPolicy(1000); #include "include/dynamic_extension.h" diff --git a/tests/de_level_concurrent.cpp b/tests/de_level_concurrent.cpp index afd1af2..a76285b 100644 --- a/tests/de_level_concurrent.cpp +++ b/tests/de_level_concurrent.cpp @@ -17,6 +17,7 @@ #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" +#include "framework/reconstruction/LevelingPolicy.h" #include using namespace de; @@ -25,7 +26,9 @@ typedef Rec R; typedef ISAMTree S; typedef rq::Query Q; -typedef DynamicExtension DE; +typedef DynamicExtension DE; +ReconstructionPolicy *recon = new LevelingPolicy(2, 1000); +ReconstructionPolicy *recon2 = new LevelingPolicy(4, 10000); #include "include/concurrent_extension.h" diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index c175357..6fead54 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -17,6 +17,7 @@ #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" +#include "framework/reconstruction/LevelingPolicy.h" #include using namespace de; @@ -25,7 +26,8 @@ typedef Rec R; typedef ISAMTree S; typedef rq::Query Q; -typedef DynamicExtension DE; +typedef DynamicExtension DE; +ReconstructionPolicy *recon = new LevelingPolicy(2, 1000); #include "include/dynamic_extension.h" diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index e587817..31e87a0 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -18,6 +18,7 @@ #include "shard/ISAMTree.h" #include "query/rangequery.h" #include "shard/TrieSpline.h" +#include "framework/reconstruction/LevelingPolicy.h" #include using namespace de; @@ -26,7 +27,8 @@ typedef Rec R; typedef ISAMTree S; typedef rq::Query Q; -typedef DynamicExtension DE; +typedef DynamicExtension DE; +ReconstructionPolicy *recon = new LevelingPolicy(2, 1000); #include "include/dynamic_extension.h" diff --git a/tests/de_tier_concurrent.cpp b/tests/de_tier_concurrent.cpp index ce41dbc..cba84f1 100644 --- a/tests/de_tier_concurrent.cpp +++ b/tests/de_tier_concurrent.cpp @@ -18,6 +18,7 @@ #include "shard/ISAMTree.h" #include "query/rangequery.h" #include "framework/scheduling//FIFOScheduler.h" +#include "framework/reconstruction/TieringPolicy.h" #include using namespace de; @@ -26,7 +27,9 @@ typedef Rec R; typedef ISAMTree S; typedef rq::Query Q; -typedef DynamicExtension DE; +typedef DynamicExtension DE; +ReconstructionPolicy *recon = new TieringPolicy(2, 1000); +ReconstructionPolicy *recon2 = new TieringPolicy(4, 10000); #include "include/concurrent_extension.h" diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 97a5299..d640d10 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -18,6 +18,7 @@ #include "framework/scheduling/SerialScheduler.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" +#include "framework/reconstruction/TieringPolicy.h" #include using namespace de; @@ -26,7 +27,8 @@ typedef Rec R; typedef ISAMTree S; typedef rq::Query Q; -typedef DynamicExtension DE; +typedef DynamicExtension DE; +ReconstructionPolicy *recon = new TieringPolicy(2, 1000); #include "include/dynamic_extension.h" diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index 930d0d5..42b0625 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -18,6 +18,7 @@ #include "shard/ISAMTree.h" #include "shard/TrieSpline.h" #include "query/rangequery.h" +#include "framework/reconstruction/TieringPolicy.h" #include using namespace de; @@ -25,7 +26,8 @@ using namespace de; typedef Rec R; typedef ISAMTree S; typedef rq::Query Q; -typedef DynamicExtension DE; +typedef DynamicExtension DE; +ReconstructionPolicy *recon = new TieringPolicy(2, 1000); #include "include/dynamic_extension.h" diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h index d99cd23..84f816d 100644 --- a/tests/include/concurrent_extension.h +++ b/tests/include/concurrent_extension.h @@ -22,24 +22,28 @@ * should be included in the source file that includes this one, above the * include statement. */ -// #include "testing.h" -// #include "framework/DynamicExtension.h" -// //#include "framework/scheduling/FIFOScheduler.h" -// #include "shard/ISAMTree.h" -// #include "query/rangequery.h" -// #include -// #include -// #include +#include "framework/reconstruction/ReconstructionPolicy.h" +#include "framework/reconstruction/TieringPolicy.h" +#include "testing.h" +#include "framework/DynamicExtension.h" +#include "framework/scheduling/FIFOScheduler.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include +#include +#include // using namespace de; // typedef Rec R; // typedef ISAMTree S; // typedef rq::Query Q; -// typedef DynamicExtension DE; //, FIFOScheduler> DE; +// typedef DynamicExtension DE; +// ReconstructionPolicy *recon = new TieringPolicy(2, 1000); +// ReconstructionPolicy *recon2 = new TieringPolicy(4, 10000); START_TEST(t_create) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100, 1000); ck_assert_ptr_nonnull(test_de); ck_assert_int_eq(test_de->get_record_count(), 0); @@ -52,7 +56,7 @@ END_TEST START_TEST(t_insert) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100, 1000); uint64_t key = 0; uint32_t val = 0; @@ -73,7 +77,7 @@ END_TEST START_TEST(t_debug_insert) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100, 1000); uint64_t key = 0; uint32_t val = 0; @@ -92,7 +96,7 @@ END_TEST START_TEST(t_insert_with_mem_merges) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100, 1000); uint64_t key = 0; uint32_t val = 0; @@ -135,7 +139,7 @@ END_TEST START_TEST(t_range_query) { - auto test_de = new DE(1000, 10000, 4); + auto test_de = new DE(recon2, 1000, 10000); size_t n = 10000000; std::vector keys; @@ -189,7 +193,7 @@ END_TEST START_TEST(t_tombstone_merging_01) { size_t reccnt = 100000; - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100, 1000); auto rng = gsl_rng_alloc(gsl_rng_mt19937); @@ -242,54 +246,12 @@ START_TEST(t_tombstone_merging_01) } END_TEST -DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { - auto rng = gsl_rng_alloc(gsl_rng_mt19937); - - auto test_de = new DE(1000, 10000, 2); - - std::set records; - std::set to_delete; - std::set deleted; - - while (records.size() < reccnt) { - uint64_t key = rand(); - uint32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - for (auto rec : records) { - ck_assert_int_eq(test_de->insert(rec), 1); - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(del_vec[i]); - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - gsl_rng_free(rng); - - return test_de; -} - START_TEST(t_static_structure) { auto rng = gsl_rng_alloc(gsl_rng_mt19937); size_t reccnt = 100000; - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100, 1000); std::set records; std::set to_delete; diff --git a/tests/include/dynamic_extension.h b/tests/include/dynamic_extension.h index 326bb72..a1ab20a 100644 --- a/tests/include/dynamic_extension.h +++ b/tests/include/dynamic_extension.h @@ -23,26 +23,28 @@ * include statement. */ -// #include "testing.h" -// #include "framework/DynamicExtension.h" -// #include "framework/scheduling/SerialScheduler.h" -// #include "shard/ISAMTree.h" -// #include "query/rangequery.h" -// #include -// #include -// #include +#include "framework/reconstruction/BSMPolicy.h" +#include "framework/reconstruction/TieringPolicy.h" +#include "testing.h" +#include "framework/DynamicExtension.h" +#include "framework/scheduling/SerialScheduler.h" +#include "framework/reconstruction/ReconstructionPolicy.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include +#include +#include // using namespace de; // typedef Rec R; // typedef ISAMTree S; // typedef rq::Query Q; -// typedef DynamicExtension DE; +// typedef DynamicExtension DE; +// ReconstructionPolicy *recon = new TieringPolicy(1000, 2); - -#include "framework/util/Configuration.h" START_TEST(t_create) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100); ck_assert_ptr_nonnull(test_de); ck_assert_int_eq(test_de->get_record_count(), 0); @@ -55,7 +57,7 @@ END_TEST START_TEST(t_insert) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100); uint64_t key = 0; uint32_t val = 0; @@ -76,7 +78,7 @@ END_TEST START_TEST(t_debug_insert) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100); uint64_t key = 0; uint32_t val = 0; @@ -95,7 +97,7 @@ END_TEST START_TEST(t_insert_with_mem_merges) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100); uint64_t key = 0; uint32_t val = 0; @@ -114,7 +116,7 @@ START_TEST(t_insert_with_mem_merges) * BSM grows on every flush, so the height will be different than * normal layout policies */ - if (test_de->Layout == de::LayoutPolicy::BSM) { + if (dynamic_cast*>(recon)) { ck_assert_int_eq(test_de->get_height(), 2); } else { ck_assert_int_eq(test_de->get_height(), 1); @@ -127,7 +129,7 @@ END_TEST START_TEST(t_range_query) { - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100); size_t n = 10000; std::vector keys; @@ -175,7 +177,7 @@ END_TEST START_TEST(t_tombstone_merging_01) { size_t reccnt = 100000; - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100); auto rng = gsl_rng_alloc(gsl_rng_mt19937); @@ -224,54 +226,13 @@ START_TEST(t_tombstone_merging_01) } END_TEST -[[maybe_unused]] static DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { - auto rng = gsl_rng_alloc(gsl_rng_mt19937); - - auto test_de = new DE(1000, 10000, 2); - - std::set records; - std::set to_delete; - std::set deleted; - - while (records.size() < reccnt) { - uint64_t key = rand(); - uint32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - for (auto rec : records) { - ck_assert_int_eq(test_de->insert(rec), 1); - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(del_vec[i]); - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - gsl_rng_free(rng); - - return test_de; -} START_TEST(t_static_structure) { auto rng = gsl_rng_alloc(gsl_rng_mt19937); size_t reccnt = 100000; - auto test_de = new DE(100, 1000, 2); + auto test_de = new DE(recon, 100); std::set records; std::set to_delete; diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h index de43edc..0b5ab00 100644 --- a/tests/include/shard_standard.h +++ b/tests/include/shard_standard.h @@ -77,7 +77,7 @@ START_TEST(t_shard_init) auto shard2 = new Shard(mbuffer2->get_buffer_view()); auto shard3 = new Shard(mbuffer3->get_buffer_view()); - std::vector shards = {shard1, shard2, shard3}; + std::vector shards = {shard1, shard2, shard3}; auto shard4 = new Shard(shards); ck_assert_int_eq(shard4->get_record_count(), n * 3); @@ -130,7 +130,7 @@ START_TEST(t_full_cancelation) ck_assert_int_eq(shard_ts->get_record_count(), n); ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - std::vector shards = {shard, shard_ts}; + std::vector shards = {shard, shard_ts}; Shard* merged = new Shard(shards); diff --git a/tests/include/shard_string.h b/tests/include/shard_string.h index 2ef4cec..7a3d761 100644 --- a/tests/include/shard_string.h +++ b/tests/include/shard_string.h @@ -67,7 +67,7 @@ START_TEST(t_shard_init) auto shard2 = new Shard(mbuffer2->get_buffer_view()); auto shard3 = new Shard(mbuffer3->get_buffer_view()); - std::vector shards = {shard1, shard2, shard3}; + std::vector shards = {shard1, shard2, shard3}; auto shard4 = new Shard(shards); ck_assert_int_eq(shard4->get_record_count(), n * 3); diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp deleted file mode 100644 index e11b7c7..0000000 --- a/tests/internal_level_tests.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/* - * tests/internal_level_tests.cpp - * - * Unit tests for InternalLevel - * - * Copyright (C) 2023 Douglas Rumbaugh - * Dong Xie - * - * Distributed under the Modified BSD License. - * - */ -#include "shard/ISAMTree.h" -#include "query/rangequery.h" -#include "framework/structure/InternalLevel.h" -#include "framework/interface/Record.h" -#include "framework/interface/Query.h" -#include "framework/interface/Shard.h" - -#include "include/testing.h" - -#include - -using namespace de; - -typedef InternalLevel, rq::Query>> ILevel; - -START_TEST(t_memlevel_merge) -{ - auto tbl1 = create_test_mbuffer(100); - auto tbl2 = create_test_mbuffer(100); - - auto base_level = new ILevel(1, 1); - base_level->append_buffer(tbl1->get_buffer_view()); - ck_assert_int_eq(base_level->get_record_count(), 100); - - auto merging_level = new ILevel(0, 1); - merging_level->append_buffer(tbl2->get_buffer_view()); - ck_assert_int_eq(merging_level->get_record_count(), 100); - - auto new_level = ILevel::reconstruction(base_level, merging_level); - - delete merging_level; - ck_assert_int_eq(new_level->get_record_count(), 200); - - delete base_level; - delete tbl1; - delete tbl2; -} - - -ILevel *create_test_memlevel(size_t reccnt) { - auto tbl1 = create_test_mbuffer(reccnt/2); - auto tbl2 = create_test_mbuffer(reccnt/2); - - auto base_level = new ILevel(1, 2); - base_level->append_buffer(tbl1->get_buffer_view()); - base_level->append_buffer(tbl2->get_buffer_view()); - - delete tbl1; - delete tbl2; - - return base_level; -} - -Suite *unit_testing() -{ - Suite *unit = suite_create("InternalLevel Unit Testing"); - - TCase *merge = tcase_create("de::InternalLevel::reconstruction Testing"); - tcase_add_test(merge, t_memlevel_merge); - suite_add_tcase(unit, merge); - - return unit; -} - -int run_unit_tests() -{ - int failed = 0; - Suite *unit = unit_testing(); - SRunner *unit_runner = srunner_create(unit); - - srunner_run_all(unit_runner, CK_NORMAL); - failed = srunner_ntests_failed(unit_runner); - srunner_free(unit_runner); - - return failed; -} - - -int main() -{ - int unit_failed = run_unit_tests(); - - return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; -} diff --git a/tests/vptree_tests.cpp b/tests/vptree_tests.cpp index 53bb526..49964e5 100644 --- a/tests/vptree_tests.cpp +++ b/tests/vptree_tests.cpp @@ -52,7 +52,7 @@ START_TEST(t_wss_init) auto shard2 = new Shard(mbuffer2->get_buffer_view()); auto shard3 = new Shard(mbuffer3->get_buffer_view()); - std::vector shards = {shard1, shard2, shard3}; + std::vector shards = {shard1, shard2, shard3}; auto shard4 = new Shard(shards); ck_assert_int_eq(shard4->get_record_count(), n * 3); -- cgit v1.2.3