From c4514c2e62a711189cf3c914297885d97fb51a09 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 12 Jan 2024 14:08:33 -0500 Subject: Initial pass at unit test refactoring Restructured unit tests to be a bit more modular. I have some further plans to expand on this, particular for the query tests (including both shard and framework level test functions that can be injected at will). --- tests/de_level_tag.cpp | 35 +++- tests/de_level_tomb.cpp | 38 +++- tests/de_tier_tag.cpp | 35 +++- tests/de_tier_tomb.cpp | 36 +++- tests/dynamic_extension_tests.inc | 416 -------------------------------------- tests/include/dynamic_extension.h | 341 +++++++++++++++++++++++++++++++ tests/include/rangequery.h | 179 ++++++++++++++++ tests/include/shard_standard.h | 198 ++++++++++++++++++ tests/include/testing.h | 218 ++++++++++++++++++++ tests/internal_level_tests.cpp | 2 +- tests/memisam_tests.cpp | 348 +------------------------------ tests/mutable_buffer_tests.cpp | 2 +- tests/rangequery_tests.cpp | 195 ++++++++++++++++++ tests/testing.h | 218 -------------------- 14 files changed, 1275 insertions(+), 986 deletions(-) delete mode 100644 tests/dynamic_extension_tests.inc create mode 100644 tests/include/dynamic_extension.h create mode 100644 tests/include/rangequery.h create mode 100644 tests/include/shard_standard.h create mode 100644 tests/include/testing.h create mode 100644 tests/rangequery_tests.cpp delete mode 100644 tests/testing.h (limited to 'tests') diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index 3a2f271..5c95aa2 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -13,7 +13,7 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" @@ -23,4 +23,35 @@ using namespace de; typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tagged Leveling Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 2412959..91ee26d 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -13,14 +13,46 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" +#include "shard/TrieSpline.h" #include using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tombstone Leveling Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 46f2ee7..9d4fe7d 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -13,7 +13,7 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "framework/scheduling/SerialScheduler.h" #include "shard/ISAMTree.h" @@ -24,4 +24,35 @@ using namespace de; typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tagged Tiering Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index e2b65a0..7d8f144 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -13,9 +13,10 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" +#include "shard/TrieSpline.h" #include "query/rangequery.h" #include @@ -23,4 +24,35 @@ using namespace de; typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tombstone Tiering Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/dynamic_extension_tests.inc b/tests/dynamic_extension_tests.inc deleted file mode 100644 index be82132..0000000 --- a/tests/dynamic_extension_tests.inc +++ /dev/null @@ -1,416 +0,0 @@ -/* - * tests/dynamic_extension_tests.inc - * - * Unit tests for Dynamic Extension Framework - * - * Copyright (C) 2023 Douglas Rumbaugh - * Dong Xie - * - * Distributed under the Modified BSD License. - * - */ - -START_TEST(t_create) -{ - auto test_de = new DE(100, 2, 1); - - ck_assert_ptr_nonnull(test_de); - ck_assert_int_eq(test_de->get_record_count(), 0); - ck_assert_int_eq(test_de->get_height(), 0); - - delete test_de; -} -END_TEST - - -START_TEST(t_insert) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<100; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - ck_assert_int_eq(test_de->get_height(), 0); - ck_assert_int_eq(test_de->get_record_count(), 100); - - delete test_de; -} -END_TEST - - -START_TEST(t_debug_insert) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<1000; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - ck_assert_int_eq(test_de->get_record_count(), i+1); - key++; - val++; - } - - delete test_de; -} -END_TEST - - -START_TEST(t_insert_with_mem_merges) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<300; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - test_de->await_next_epoch(); - - ck_assert_int_eq(test_de->get_record_count(), 300); - ck_assert_int_eq(test_de->get_height(), 1); - - delete test_de; -} -END_TEST - - -/* -START_TEST(t_range_sample_memtable) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<100; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - uint64_t lower_bound = 20; - uint64_t upper_bound = 50; - - char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - Rec sample_set[100]; - - test_de->range_sample(sample_set, lower_bound, upper_bound, 100); - - for(size_t i=0; i<100; i++) { - ck_assert_int_le(sample_set[i].key, upper_bound); - ck_assert_int_ge(sample_set[i].key, lower_bound); - } - - free(buf); - free(util_buf); - - delete test_de; -} -END_TEST - - -START_TEST(t_range_sample_memlevels) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<300; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - uint64_t lower_bound = 100; - uint64_t upper_bound = 250; - - char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - - Rec sample_set[100]; - test_de->range_sample(sample_set, lower_bound, upper_bound, 100); - - for(size_t i=0; i<100; i++) { - ck_assert_int_le(sample_set[i].key, upper_bound); - ck_assert_int_ge(sample_set[i].key, lower_bound); - } - - free(buf); - free(util_buf); - - delete test_de; -} -END_TEST -*/ - -START_TEST(t_range_query) -{ - auto test_de = new DE(100, 2, 1); - size_t n = 10000; - - std::vector keys; - for (size_t i=0; iinsert(r), 1); - } - - test_de->await_next_epoch(); - - std::sort(keys.begin(), keys.end()); - - auto idx = rand() % (keys.size() - 250); - - uint64_t lower_key = keys[idx]; - uint64_t upper_key = keys[idx + 250]; - - rq::Parms p; - p.lower_bound = lower_key; - p.upper_bound = upper_key; - - auto result = test_de->query(&p); - auto r = result.get(); - std::sort(r.begin(), r.end()); - ck_assert_int_eq(r.size(), 251); - - for (size_t i=0; i> records; - std::set> to_delete; - std::set> deleted; - - while (records.size() < reccnt) { - uint64_t key = rand(); - uint32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - size_t cnt=0; - for (auto rec : records) { - Rec r = {rec.first, rec.second}; - ck_assert_int_eq(test_de->insert(r), 1); - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector> del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(dr); - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - test_de->await_next_epoch(); - - ck_assert(test_de->validate_tombstone_proportion()); - - gsl_rng_free(rng); - delete test_de; -} -END_TEST - -DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { - auto rng = gsl_rng_alloc(gsl_rng_mt19937); - - auto test_de = new DE(1000, 2, 1); - - std::set records; - std::set to_delete; - std::set deleted; - - while (records.size() < reccnt) { - uint64_t key = rand(); - uint32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - for (auto rec : records) { - ck_assert_int_eq(test_de->insert(rec), 1); - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(del_vec[i]); - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - gsl_rng_free(rng); - - return test_de; -} - -START_TEST(t_static_structure) -{ - auto rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t reccnt = 100000; - auto test_de = new DE(100, 2, 1); - - std::set records; - std::set to_delete; - std::set deleted; - - while (records.size() < reccnt) { - uint64_t key = rand(); - uint32_t val = rand(); - - if (records.find({key, val}) != records.end()) continue; - - records.insert({key, val}); - } - - size_t deletes = 0; - size_t t_reccnt = 0; - size_t k=0; - for (auto rec : records) { - k++; - ck_assert_int_eq(test_de->insert(rec), 1); - t_reccnt++; - - if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; - std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); - - for (size_t i=0; ierase(del_vec[i]), 1); - - deletes++; - to_delete.erase(del_vec[i]); - deleted.insert(del_vec[i]); - } - } - - if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { - to_delete.insert(rec); - } - } - - auto flat = test_de->create_static_structure(); - ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); - - uint64_t prev_key = 0; - for (size_t i=0; iget_record_count(); i++) { - auto k = flat->get_record_at(i)->rec.key; - ck_assert_int_ge(k, prev_key); - prev_key = k; - } - - gsl_rng_free(rng); - delete flat; - delete test_de; -} -END_TEST - - -Suite *unit_testing() -{ - Suite *unit = suite_create("de::DynamicExtension Unit Testing"); - - TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); - tcase_add_test(create, t_create); - suite_add_tcase(unit, create); - - TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); - tcase_add_test(insert, t_insert); - tcase_add_test(insert, t_insert_with_mem_merges); - tcase_add_test(insert, t_debug_insert); - suite_add_tcase(unit, insert); - - TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); - tcase_add_test(query, t_range_query); - suite_add_tcase(unit, query); - - TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); - tcase_add_test(ts, t_tombstone_merging_01); - tcase_set_timeout(ts, 500); - suite_add_tcase(unit, ts); - - TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); - tcase_add_test(flat, t_static_structure); - tcase_set_timeout(flat, 500); - suite_add_tcase(unit, flat); - - return unit; -} - -int run_unit_tests() -{ - int failed = 0; - Suite *unit = unit_testing(); - SRunner *unit_runner = srunner_create(unit); - - srunner_run_all(unit_runner, CK_NORMAL); - failed = srunner_ntests_failed(unit_runner); - srunner_free(unit_runner); - - return failed; -} - - -int main() -{ - int unit_failed = run_unit_tests(); - - return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; -} diff --git a/tests/include/dynamic_extension.h b/tests/include/dynamic_extension.h new file mode 100644 index 0000000..5a08f5a --- /dev/null +++ b/tests/include/dynamic_extension.h @@ -0,0 +1,341 @@ +/* + * tests/include/dynamic_extension.h + * + * Standardized unit tests for DynamicExtension objects + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard, Query, and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "testing.h" +//#include "framework/DynamicExtension.h" +//#include "framework/scheduling/SerialScheduler.h" +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include +//using namespace de; +//typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; + + +START_TEST(t_create) +{ + auto test_de = new DE(100, 1000, 2); + + ck_assert_ptr_nonnull(test_de); + ck_assert_int_eq(test_de->get_record_count(), 0); + ck_assert_int_eq(test_de->get_height(), 0); + + delete test_de; +} +END_TEST + + +START_TEST(t_insert) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<100; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + key++; + val++; + } + + ck_assert_int_eq(test_de->get_height(), 0); + ck_assert_int_eq(test_de->get_record_count(), 100); + + delete test_de; +} +END_TEST + + +START_TEST(t_debug_insert) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<1000; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + ck_assert_int_eq(test_de->get_record_count(), i+1); + key++; + val++; + } + + delete test_de; +} +END_TEST + + +START_TEST(t_insert_with_mem_merges) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<300; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + key++; + val++; + } + + test_de->await_next_epoch(); + + ck_assert_int_eq(test_de->get_record_count(), 300); + ck_assert_int_eq(test_de->get_height(), 1); + + delete test_de; +} +END_TEST + + +START_TEST(t_range_query) +{ + auto test_de = new DE(100, 1000, 2); + size_t n = 10000; + + std::vector keys; + for (size_t i=0; iinsert(r), 1); + } + + test_de->await_next_epoch(); + + std::sort(keys.begin(), keys.end()); + + auto idx = rand() % (keys.size() - 250); + + uint64_t lower_key = keys[idx]; + uint64_t upper_key = keys[idx + 250]; + + rq::Parms p; + p.lower_bound = lower_key; + p.upper_bound = upper_key; + + auto result = test_de->query(&p); + auto r = result.get(); + std::sort(r.begin(), r.end()); + ck_assert_int_eq(r.size(), 251); + + for (size_t i=0; i> records; + std::set> to_delete; + std::set> deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t cnt=0; + for (auto rec : records) { + Rec r = {rec.first, rec.second}; + ck_assert_int_eq(test_de->insert(r), 1); + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector> del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(dr); + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + test_de->await_next_epoch(); + + ck_assert(test_de->validate_tombstone_proportion()); + + gsl_rng_free(rng); + delete test_de; +} +END_TEST + +DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + auto test_de = new DE(1000, 10000, 2); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + for (auto rec : records) { + ck_assert_int_eq(test_de->insert(rec), 1); + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[i]); + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + gsl_rng_free(rng); + + return test_de; +} + +START_TEST(t_static_structure) +{ + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t reccnt = 100000; + auto test_de = new DE(100, 1000, 2); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t t_reccnt = 0; + size_t k=0; + for (auto rec : records) { + k++; + ck_assert_int_eq(test_de->insert(rec), 1); + t_reccnt++; + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[i]), 1); + + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + auto flat = test_de->create_static_structure(); + ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); + + uint64_t prev_key = 0; + for (size_t i=0; iget_record_count(); i++) { + auto k = flat->get_record_at(i)->rec.key; + ck_assert_int_ge(k, prev_key); + prev_key = k; + } + + gsl_rng_free(rng); + delete flat; + delete test_de; +} +END_TEST + + +static void inject_dynamic_extension_tests(Suite *suite) { + TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); + tcase_add_test(create, t_create); + suite_add_tcase(suite, create); + + TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); + tcase_add_test(insert, t_insert); + tcase_add_test(insert, t_insert_with_mem_merges); + tcase_add_test(insert, t_debug_insert); + suite_add_tcase(suite, insert); + + TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); + tcase_add_test(query, t_range_query); + suite_add_tcase(suite, query); + + TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); + tcase_add_test(ts, t_tombstone_merging_01); + tcase_set_timeout(ts, 500); + suite_add_tcase(suite, ts); + + TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); + tcase_add_test(flat, t_static_structure); + tcase_set_timeout(flat, 500); + suite_add_tcase(suite, flat); +} diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h new file mode 100644 index 0000000..3c7e7e0 --- /dev/null +++ b/tests/include/rangequery.h @@ -0,0 +1,179 @@ +/* + * tests/include/rangequery.h + * + * Standardized unit tests for range queries against supporting + * shard types + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface and Shard needs to support lower_bound. + * For other types of record and shard, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; + + +START_TEST(t_range_query) +{ + auto buffer = create_sequential_mbuffer(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 1000); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); + + std::vector>> results(2); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); + + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector>> proc_results; + + for (size_t j=0; j>()); + for (size_t i=0; i::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + Shard *shards[2]; + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + shards[0] = &shard1; + shards[1] = &shard2; + + auto merged = Shard(shards, 2); + + for (size_t i=100; i<1000; i++) { + Rec r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + +static void inject_rangequery_tests(Suite *suite) { + TCase *range_query = tcase_create("Range Query Testing"); + tcase_add_test(range_query, t_range_query); + tcase_add_test(range_query, t_buffer_range_query); + tcase_add_test(range_query, t_range_query_merge); + suite_add_tcase(suite, range_query); +} diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h new file mode 100644 index 0000000..047a7b5 --- /dev/null +++ b/tests/include/shard_standard.h @@ -0,0 +1,198 @@ +/* + * tests/include/shard_standard.h + * + * Standardized unit tests for Shard objects + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; + +START_TEST(t_mbuffer_init) +{ + auto buffer = new MutableBuffer(512, 1024); + for (uint64_t i = 512; i > 0; i--) { + uint32_t v = i; + buffer->append({i,v, 1}); + } + + for (uint64_t i = 1; i <= 256; ++i) { + uint32_t v = i; + buffer->append({i, v, 1}, true); + } + + for (uint64_t i = 257; i <= 512; ++i) { + uint32_t v = i + 1; + buffer->append({i, v, 1}); + } + + Shard* shard = new Shard(buffer->get_buffer_view()); + ck_assert_uint_eq(shard->get_record_count(), 512); + + delete buffer; + delete shard; +} + + +START_TEST(t_shard_init) +{ + size_t n = 512; + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); + + auto shard1 = new Shard(mbuffer1->get_buffer_view()); + auto shard2 = new Shard(mbuffer2->get_buffer_view()); + auto shard3 = new Shard(mbuffer3->get_buffer_view()); + + Shard* shards[3] = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards, 3); + + ck_assert_int_eq(shard4->get_record_count(), n * 3); + ck_assert_int_eq(shard4->get_tombstone_count(), 0); + + size_t total_cnt = 0; + size_t shard1_idx = 0; + size_t shard2_idx = 0; + size_t shard3_idx = 0; + + for (size_t i = 0; i < shard4->get_record_count(); ++i) { + auto rec1 = shard1->get_record_at(shard1_idx); + auto rec2 = shard2->get_record_at(shard2_idx); + auto rec3 = shard3->get_record_at(shard3_idx); + + auto cur_rec = shard4->get_record_at(i); + + if (shard1_idx < n && cur_rec->rec == rec1->rec) { + ++shard1_idx; + } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { + ++shard2_idx; + } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { + ++shard3_idx; + } else { + assert(false); + } + } + + delete mbuffer1; + delete mbuffer2; + delete mbuffer3; + + delete shard1; + delete shard2; + delete shard3; + delete shard4; +} + + +START_TEST(t_full_cancelation) +{ + size_t n = 100; + auto buffer = create_double_seq_mbuffer(n, false); + auto buffer_ts = create_double_seq_mbuffer(n, true); + + Shard* shard = new Shard(buffer->get_buffer_view()); + Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); + + ck_assert_int_eq(shard->get_record_count(), n); + ck_assert_int_eq(shard->get_tombstone_count(), 0); + ck_assert_int_eq(shard_ts->get_record_count(), n); + ck_assert_int_eq(shard_ts->get_tombstone_count(), n); + + Shard* shards[] = {shard, shard_ts}; + + Shard* merged = new Shard(shards, 2); + + ck_assert_int_eq(merged->get_tombstone_count(), 0); + ck_assert_int_eq(merged->get_record_count(), 0); + + delete buffer; + delete buffer_ts; + delete shard; + delete shard_ts; + delete merged; +} +END_TEST + + +START_TEST(t_point_lookup) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto isam = Shard(buffer->get_buffer_view()); + + { + auto view = buffer->get_buffer_view(); + + for (size_t i=0; irec.key; + r.value = rec->rec.value; + + auto result = isam.point_lookup(r); + ck_assert_ptr_nonnull(result); + ck_assert_int_eq(result->rec.key, r.key); + ck_assert_int_eq(result->rec.value, r.value); + } + } + + delete buffer; +} +END_TEST + + +START_TEST(t_point_lookup_miss) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto isam = Shard(buffer->get_buffer_view()); + + for (size_t i=n + 100; i<2*n; i++) { + Rec r; + r.key = i; + r.value = i; + + auto result = isam.point_lookup(r); + ck_assert_ptr_null(result); + } + + delete buffer; +} + +static void inject_shard_tests(Suite *suite) { + TCase *create = tcase_create("Shard constructor Testing"); + tcase_add_test(create, t_mbuffer_init); + tcase_add_test(create, t_shard_init); + tcase_set_timeout(create, 100); + suite_add_tcase(suite, create); + TCase *tombstone = tcase_create("Shard tombstone cancellation Testing"); + tcase_add_test(tombstone, t_full_cancelation); + suite_add_tcase(suite, tombstone); + TCase *pointlookup = tcase_create("Shard point lookup Testing"); + tcase_add_test(pointlookup, t_point_lookup); + tcase_add_test(pointlookup, t_point_lookup_miss); + suite_add_tcase(suite, pointlookup); +} diff --git a/tests/include/testing.h b/tests/include/testing.h new file mode 100644 index 0000000..4e660dd --- /dev/null +++ b/tests/include/testing.h @@ -0,0 +1,218 @@ +/* + * tests/testing.h + * + * Unit test utility functions/definitions + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * Distributed under the Modified BSD License. + * + */ +#pragma once + +#include + +#include +#include + +#include "util/types.h" +#include "psu-util/alignment.h" +#include "framework/structure/MutableBuffer.h" +#include "framework/interface/Record.h" + +typedef de::WeightedRecord WRec; +typedef de::Record Rec; +typedef de::EuclidPoint PRec; + +template +std::vector strip_wrapping(std::vector> vec) { + std::vector out(vec.size()); + for (size_t i=0; i *create_2d_mbuffer(size_t cnt) { + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + for (int64_t i=0; iappend({rand(), rand()}); + } + + return buffer; +} + +static de::MutableBuffer *create_2d_sequential_mbuffer(size_t cnt) { + auto buffer = new de::MutableBuffer(cnt/2, cnt); + for (int64_t i=0; iappend({i, i}); + } + + return buffer; +} + +template +static de::MutableBuffer *create_test_mbuffer(size_t cnt) +{ + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + R rec; + for (size_t i = 0; i < cnt; i++) { + rec.key = rand(); + rec.value = rand(); + + if constexpr (de::WeightedRecordInterface) { + rec.weight = 1; + } + + buffer->append(rec); + } + + return buffer; +} + +template +static de::MutableBuffer *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop) +{ + size_t cnt = stop - start; + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + for (size_t i=start; i) { + rec.weight = 1; + } + + buffer->append(rec); + } + + return buffer; +} + +template +static de::MutableBuffer *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) +{ + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + std::vector> tombstones; + + R rec; + for (size_t i = 0; i < cnt; i++) { + rec.key = rand(); + rec.value = rand(); + + if constexpr (de::WeightedRecordInterface) { + rec.weight = 1; + } + + if (i < ts_cnt) { + tombstones.push_back({rec.key, rec.value}); + } + + buffer->append(rec); + } + + rec.set_tombstone(); + for (size_t i=0; iappend(rec); + } + + return buffer; +} + +template +requires de::WeightedRecordInterface && de::KVPInterface +static de::MutableBuffer *create_weighted_mbuffer(size_t cnt) +{ + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + // Put in half of the count with weight one. + for (uint32_t i=0; i< cnt / 2; i++) { + buffer->append(R {1, i, 2}); + } + + // put in a quarter of the count with weight four. + for (uint32_t i=0; i< cnt / 4; i++) { + buffer->append(R {2, i, 4}); + } + + // the remaining quarter with weight eight. + for (uint32_t i=0; i< cnt / 4; i++) { + buffer->append(R {3, i, 8}); + } + + return buffer; +} + +template +static de::MutableBuffer *create_double_seq_mbuffer(size_t cnt, bool ts=false) +{ + auto buffer = new de::MutableBuffer(cnt/2, cnt); + + for (size_t i = 0; i < cnt / 2; i++) { + R rec; + rec.key = i; + rec.value = i; + + buffer->append(rec, ts); + } + + for (size_t i = 0; i < cnt / 2; i++) { + R rec; + rec.key = i; + rec.value = i + 1; + + buffer->append(rec, ts); + } + + return buffer; +} + + diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index b8aa56f..79b9c21 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -16,7 +16,7 @@ #include "framework/interface/Query.h" #include "framework/interface/Shard.h" -#include "testing.h" +#include "include/testing.h" #include diff --git a/tests/memisam_tests.cpp b/tests/memisam_tests.cpp index 919fd69..b398524 100644 --- a/tests/memisam_tests.cpp +++ b/tests/memisam_tests.cpp @@ -11,359 +11,25 @@ */ #include "shard/ISAMTree.h" +#include "include/testing.h" #include "query/rangequery.h" -#include "testing.h" - #include -using namespace de; - -typedef ISAMTree Shard; - -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer(512, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer->get_buffer_view()); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_rq_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); - - auto shard1 = new Shard(mbuffer1->get_buffer_view()); - auto shard2 = new Shard(mbuffer2->get_buffer_view()); - auto shard3 = new Shard(mbuffer3->get_buffer_view()); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto isam = Shard(buffer->get_buffer_view()); - - { - auto view = buffer->get_buffer_view(); - - for (size_t i=0; irec.key; - r.value = rec->rec.value; - - auto result = isam.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer(n, false); - auto isam = Shard(buffer->get_buffer_view()); - - for (size_t i=n + 100; i<2*n; i++) { - Rec r; - r.key = i; - r.value = i; - - auto result = isam.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); - - Shard* shard = new Shard(buffer->get_buffer_view()); - Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST - - -/* -START_TEST(t_rq_query) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer(n); - auto isam = Shard(buffer->get_buffer_view()); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - size_t k = 100; - size_t cnt[3] = {0}; - rq::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query::get_query_state(&isam, &parms); - ((rq::State *) state)->sample_size = k; - auto result = rq::Query::query(&isam, state, &parms); - - ck_assert_int_eq(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query::delete_query_state(state); - } - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_rq_query_merge) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer(n); - - Shard shard = Shard(buffer->get_buffer_view()); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 1000; - - size_t cnt[3] = {0}; - rq::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - std::vector>> results(2); - - for (size_t i=0; i<1000; i++) { - auto state1 = rq::Query::get_query_state(&shard, &parms); - ((rq::State *) state1)->sample_size = k; - results[0] = rq::Query::query(&shard, state1, &parms); - - auto state2 = rq::Query::get_query_state(&shard, &parms); - ((rq::State *) state2)->sample_size = k; - results[1] = rq::Query::query(&shard, state2, &parms); - - rq::Query::delete_query_state(state1); - rq::Query::delete_query_state(state2); - } - - auto merged = rq::Query::merge(results, nullptr); - - ck_assert_int_eq(merged.size(), 2*k); - for (size_t i=0; i(n); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 100; - - size_t cnt[3] = {0}; - rq::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query::get_buffer_query_state(buffer, &parms); - ((rq::BufferState *) state)->sample_size = k; - auto result = rq::Query::buffer_query(buffer, state, &parms); - - ck_assert_int_eq(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query::delete_buffer_query_state(state); - } - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_rq_buffer_query_rejection) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer(n); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 10000; - - size_t cnt[3] = {0}; - rq::Parms parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query::get_buffer_query_state(buffer, &parms); - ((rq::BufferState *) state)->sample_size = k; - auto result = rq::Query::buffer_query(buffer, state, &parms); - - ck_assert_int_gt(result.size(), 0); - ck_assert_int_le(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query::delete_buffer_query_state(state); - } +using namespace de; - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST -*/ +typedef ISAMTree Shard; +#include "include/shard_standard.h" +#include "include/rangequery.h" Suite *unit_testing() { Suite *unit = suite_create("ISAMTree Shard Unit Testing"); - TCase *create = tcase_create("de::ISAMTree constructor Testing"); - tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_rq_init); - tcase_set_timeout(create, 100); - suite_add_tcase(unit, create); - - - TCase *tombstone = tcase_create("de:ISAMTree::tombstone cancellation Testing"); - tcase_add_test(tombstone, t_full_cancelation); - suite_add_tcase(unit, tombstone); - - - TCase *lookup = tcase_create("de:ISAMTree:point_lookup Testing"); - tcase_add_test(lookup, t_point_lookup); - tcase_add_test(lookup, t_point_lookup_miss); - suite_add_tcase(unit, lookup); - - /* - TCase *sampling = tcase_create("de:ISAMTree::IRS Testing"); - tcase_add_test(sampling, t_rq_query); - tcase_add_test(sampling, t_rq_query_merge); - tcase_add_test(sampling, t_rq_buffer_query_rejection); - tcase_add_test(sampling, t_rq_buffer_query_scan); - tcase_set_timeout(sampling, 100); - suite_add_tcase(unit, sampling); - */ + inject_rangequery_tests(unit); + inject_shard_tests(unit); return unit; } diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index e61a832..4064412 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -13,7 +13,7 @@ #include #include -#include "testing.h" +#include "include/testing.h" #include "framework/structure/MutableBuffer.h" #include diff --git a/tests/rangequery_tests.cpp b/tests/rangequery_tests.cpp new file mode 100644 index 0000000..6a00f5a --- /dev/null +++ b/tests/rangequery_tests.cpp @@ -0,0 +1,195 @@ +/* + * tests/rangequery_tests.cpp + * + * Unit tests for Range Queries across several different + * shards + * + * Copyright (C) 2023 Douglas Rumbaugh + * Dong Xie + * + * Distributed under the Modified BSD License. + * + */ + +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include "include/testing.h" + +#include + +using namespace de; + +typedef ISAMTree Shard; + +START_TEST(t_range_query) +{ + auto buffer = create_sequential_mbuffer(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 1000); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); + + std::vector>> results(2); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); + + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector>> proc_results; + + for (size_t j=0; j>()); + for (size_t i=0; i::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + Shard *shards[2]; + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + shards[0] = &shard1; + shards[1] = &shard2; + + auto merged = Shard(shards, 2); + + for (size_t i=100; i<1000; i++) { + Rec r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + + +Suite *unit_testing() +{ + Suite *unit = suite_create("Range Query Unit Testing"); + + TCase *range_query = tcase_create("de:PGM::range_query Testing"); + tcase_add_test(range_query, t_range_query); + tcase_add_test(range_query, t_buffer_range_query); + tcase_add_test(range_query, t_range_query_merge); + suite_add_tcase(unit, range_query); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/testing.h b/tests/testing.h deleted file mode 100644 index 4e660dd..0000000 --- a/tests/testing.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - * tests/testing.h - * - * Unit test utility functions/definitions - * - * Copyright (C) 2023 Douglas Rumbaugh - * Dong Xie - * - * Distributed under the Modified BSD License. - * - */ -#pragma once - -#include - -#include -#include - -#include "util/types.h" -#include "psu-util/alignment.h" -#include "framework/structure/MutableBuffer.h" -#include "framework/interface/Record.h" - -typedef de::WeightedRecord WRec; -typedef de::Record Rec; -typedef de::EuclidPoint PRec; - -template -std::vector strip_wrapping(std::vector> vec) { - std::vector out(vec.size()); - for (size_t i=0; i *create_2d_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - for (int64_t i=0; iappend({rand(), rand()}); - } - - return buffer; -} - -static de::MutableBuffer *create_2d_sequential_mbuffer(size_t cnt) { - auto buffer = new de::MutableBuffer(cnt/2, cnt); - for (int64_t i=0; iappend({i, i}); - } - - return buffer; -} - -template -static de::MutableBuffer *create_test_mbuffer(size_t cnt) -{ - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - R rec; - for (size_t i = 0; i < cnt; i++) { - rec.key = rand(); - rec.value = rand(); - - if constexpr (de::WeightedRecordInterface) { - rec.weight = 1; - } - - buffer->append(rec); - } - - return buffer; -} - -template -static de::MutableBuffer *create_sequential_mbuffer(decltype(R::key) start, decltype(R::key) stop) -{ - size_t cnt = stop - start; - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - for (size_t i=start; i) { - rec.weight = 1; - } - - buffer->append(rec); - } - - return buffer; -} - -template -static de::MutableBuffer *create_test_mbuffer_tombstones(size_t cnt, size_t ts_cnt) -{ - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - std::vector> tombstones; - - R rec; - for (size_t i = 0; i < cnt; i++) { - rec.key = rand(); - rec.value = rand(); - - if constexpr (de::WeightedRecordInterface) { - rec.weight = 1; - } - - if (i < ts_cnt) { - tombstones.push_back({rec.key, rec.value}); - } - - buffer->append(rec); - } - - rec.set_tombstone(); - for (size_t i=0; iappend(rec); - } - - return buffer; -} - -template -requires de::WeightedRecordInterface && de::KVPInterface -static de::MutableBuffer *create_weighted_mbuffer(size_t cnt) -{ - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - // Put in half of the count with weight one. - for (uint32_t i=0; i< cnt / 2; i++) { - buffer->append(R {1, i, 2}); - } - - // put in a quarter of the count with weight four. - for (uint32_t i=0; i< cnt / 4; i++) { - buffer->append(R {2, i, 4}); - } - - // the remaining quarter with weight eight. - for (uint32_t i=0; i< cnt / 4; i++) { - buffer->append(R {3, i, 8}); - } - - return buffer; -} - -template -static de::MutableBuffer *create_double_seq_mbuffer(size_t cnt, bool ts=false) -{ - auto buffer = new de::MutableBuffer(cnt/2, cnt); - - for (size_t i = 0; i < cnt / 2; i++) { - R rec; - rec.key = i; - rec.value = i; - - buffer->append(rec, ts); - } - - for (size_t i = 0; i < cnt / 2; i++) { - R rec; - rec.key = i; - rec.value = i + 1; - - buffer->append(rec, ts); - } - - return buffer; -} - - -- cgit v1.2.3