diff options
| author | Douglas Rumbaugh <dbr4@psu.edu> | 2024-01-12 14:08:33 -0500 |
|---|---|---|
| committer | Douglas Rumbaugh <dbr4@psu.edu> | 2024-01-12 14:09:45 -0500 |
| commit | c4514c2e62a711189cf3c914297885d97fb51a09 (patch) | |
| tree | 9cea57c0ce23c6fdaf627495c7bd4f4f8dc9019d | |
| parent | 3a89d7f6ea2679ff7b9bb1e3c37da9480be6c115 (diff) | |
| download | dynamic-extension-c4514c2e62a711189cf3c914297885d97fb51a09.tar.gz | |
Initial pass at unit test refactoring
Restructured unit tests to be a bit more modular. I have some further
plans to expand on this, particular for the query tests (including both
shard and framework level test functions that can be injected at will).
| -rw-r--r-- | tests/de_level_tag.cpp | 35 | ||||
| -rw-r--r-- | tests/de_level_tomb.cpp | 38 | ||||
| -rw-r--r-- | tests/de_tier_tag.cpp | 35 | ||||
| -rw-r--r-- | tests/de_tier_tomb.cpp | 36 | ||||
| -rw-r--r-- | tests/include/dynamic_extension.h (renamed from tests/dynamic_extension_tests.inc) | 157 | ||||
| -rw-r--r-- | tests/include/rangequery.h | 179 | ||||
| -rw-r--r-- | tests/include/shard_standard.h | 198 | ||||
| -rw-r--r-- | tests/include/testing.h (renamed from tests/testing.h) | 0 | ||||
| -rw-r--r-- | tests/internal_level_tests.cpp | 2 | ||||
| -rw-r--r-- | tests/memisam_tests.cpp | 348 | ||||
| -rw-r--r-- | tests/mutable_buffer_tests.cpp | 2 | ||||
| -rw-r--r-- | tests/rangequery_tests.cpp | 195 |
12 files changed, 757 insertions, 468 deletions
diff --git a/tests/de_level_tag.cpp b/tests/de_level_tag.cpp index 3a2f271..5c95aa2 100644 --- a/tests/de_level_tag.cpp +++ b/tests/de_level_tag.cpp @@ -13,7 +13,7 @@ #include <random> #include <algorithm> -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" @@ -23,4 +23,35 @@ using namespace de; typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TAGGING, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tagged Leveling Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_level_tomb.cpp b/tests/de_level_tomb.cpp index 2412959..91ee26d 100644 --- a/tests/de_level_tomb.cpp +++ b/tests/de_level_tomb.cpp @@ -13,14 +13,46 @@ #include <random> #include <algorithm> -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" #include "query/rangequery.h" +#include "shard/TrieSpline.h" #include <check.h> using namespace de; -typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; +typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tombstone Leveling Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_tier_tag.cpp b/tests/de_tier_tag.cpp index 46f2ee7..9d4fe7d 100644 --- a/tests/de_tier_tag.cpp +++ b/tests/de_tier_tag.cpp @@ -13,7 +13,7 @@ #include <random> #include <algorithm> -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "framework/scheduling/SerialScheduler.h" #include "shard/ISAMTree.h" @@ -24,4 +24,35 @@ using namespace de; typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tagged Tiering Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/de_tier_tomb.cpp b/tests/de_tier_tomb.cpp index e2b65a0..7d8f144 100644 --- a/tests/de_tier_tomb.cpp +++ b/tests/de_tier_tomb.cpp @@ -13,9 +13,10 @@ #include <random> #include <algorithm> -#include "testing.h" +#include "include/testing.h" #include "framework/DynamicExtension.h" #include "shard/ISAMTree.h" +#include "shard/TrieSpline.h" #include "query/rangequery.h" #include <check.h> @@ -23,4 +24,35 @@ using namespace de; typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TOMBSTONE, SerialScheduler> DE; -#include "dynamic_extension_tests.inc" +#include "include/dynamic_extension.h" + + +Suite *unit_testing() +{ + Suite *unit = suite_create("DynamicExtension: Tombstone Tiering Testing"); + inject_dynamic_extension_tests(unit); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/dynamic_extension_tests.inc b/tests/include/dynamic_extension.h index be82132..5a08f5a 100644 --- a/tests/dynamic_extension_tests.inc +++ b/tests/include/dynamic_extension.h @@ -1,18 +1,40 @@ /* - * tests/dynamic_extension_tests.inc + * tests/include/dynamic_extension.h * - * Unit tests for Dynamic Extension Framework + * Standardized unit tests for DynamicExtension objects * * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> - * Dong Xie <dongx@psu.edu> * * Distributed under the Modified BSD License. * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard, Query, and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "testing.h" +//#include "framework/DynamicExtension.h" +//#include "framework/scheduling/SerialScheduler.h" +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include <check.h> +//using namespace de; +//typedef DynamicExtension<Rec, ISAMTree<Rec>, rq::Query<ISAMTree<Rec>, Rec>, LayoutPolicy::TEIRING, DeletePolicy::TAGGING, SerialScheduler> DE; + START_TEST(t_create) { - auto test_de = new DE(100, 2, 1); + auto test_de = new DE(100, 1000, 2); ck_assert_ptr_nonnull(test_de); ck_assert_int_eq(test_de->get_record_count(), 0); @@ -25,7 +47,7 @@ END_TEST START_TEST(t_insert) { - auto test_de = new DE(100, 2, 1); + auto test_de = new DE(100, 1000, 2); uint64_t key = 0; uint32_t val = 0; @@ -46,7 +68,7 @@ END_TEST START_TEST(t_debug_insert) { - auto test_de = new DE(100, 2, 1); + auto test_de = new DE(100, 1000, 2); uint64_t key = 0; uint32_t val = 0; @@ -65,7 +87,7 @@ END_TEST START_TEST(t_insert_with_mem_merges) { - auto test_de = new DE(100, 2, 1); + auto test_de = new DE(100, 1000, 2); uint64_t key = 0; uint32_t val = 0; @@ -86,80 +108,9 @@ START_TEST(t_insert_with_mem_merges) END_TEST -/* -START_TEST(t_range_sample_memtable) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<100; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - uint64_t lower_bound = 20; - uint64_t upper_bound = 50; - - char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - Rec sample_set[100]; - - test_de->range_sample(sample_set, lower_bound, upper_bound, 100); - - for(size_t i=0; i<100; i++) { - ck_assert_int_le(sample_set[i].key, upper_bound); - ck_assert_int_ge(sample_set[i].key, lower_bound); - } - - free(buf); - free(util_buf); - - delete test_de; -} -END_TEST - - -START_TEST(t_range_sample_memlevels) -{ - auto test_de = new DE(100, 2, 1); - - uint64_t key = 0; - uint32_t val = 0; - for (size_t i=0; i<300; i++) { - Rec r = {key, val}; - ck_assert_int_eq(test_de->insert(r), 1); - key++; - val++; - } - - uint64_t lower_bound = 100; - uint64_t upper_bound = 250; - - char *buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - char *util_buf = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - - Rec sample_set[100]; - test_de->range_sample(sample_set, lower_bound, upper_bound, 100); - - for(size_t i=0; i<100; i++) { - ck_assert_int_le(sample_set[i].key, upper_bound); - ck_assert_int_ge(sample_set[i].key, lower_bound); - } - - free(buf); - free(util_buf); - - delete test_de; -} -END_TEST -*/ - START_TEST(t_range_query) { - auto test_de = new DE(100, 2, 1); + auto test_de = new DE(100, 1000, 2); size_t n = 10000; std::vector<uint64_t> keys; @@ -206,7 +157,7 @@ END_TEST START_TEST(t_tombstone_merging_01) { size_t reccnt = 100000; - auto test_de = new DE(100, 2, .01); + auto test_de = new DE(100, 1000, 2); auto rng = gsl_rng_alloc(gsl_rng_mt19937); @@ -259,7 +210,7 @@ END_TEST DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { auto rng = gsl_rng_alloc(gsl_rng_mt19937); - auto test_de = new DE(1000, 2, 1); + auto test_de = new DE(1000, 10000, 2); std::set<Rec> records; std::set<Rec> to_delete; @@ -305,7 +256,7 @@ START_TEST(t_static_structure) auto rng = gsl_rng_alloc(gsl_rng_mt19937); size_t reccnt = 100000; - auto test_de = new DE(100, 2, 1); + auto test_de = new DE(100, 1000, 2); std::set<Rec> records; std::set<Rec> to_delete; @@ -363,54 +314,28 @@ START_TEST(t_static_structure) END_TEST -Suite *unit_testing() -{ - Suite *unit = suite_create("de::DynamicExtension Unit Testing"); - +static void inject_dynamic_extension_tests(Suite *suite) { TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); tcase_add_test(create, t_create); - suite_add_tcase(unit, create); + suite_add_tcase(suite, create); - TCase *insert = tcase_create("de::DynamicExtension<ISAMTree>::insert Testing"); + TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); tcase_add_test(insert, t_insert); tcase_add_test(insert, t_insert_with_mem_merges); tcase_add_test(insert, t_debug_insert); - suite_add_tcase(unit, insert); + suite_add_tcase(suite, insert); - TCase *query = tcase_create("de::DynamicExtension<ISAMTree>::range_query Testing"); + TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); tcase_add_test(query, t_range_query); - suite_add_tcase(unit, query); + suite_add_tcase(suite, query); TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); tcase_add_test(ts, t_tombstone_merging_01); tcase_set_timeout(ts, 500); - suite_add_tcase(unit, ts); + suite_add_tcase(suite, ts); TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); tcase_add_test(flat, t_static_structure); tcase_set_timeout(flat, 500); - suite_add_tcase(unit, flat); - - return unit; -} - -int run_unit_tests() -{ - int failed = 0; - Suite *unit = unit_testing(); - SRunner *unit_runner = srunner_create(unit); - - srunner_run_all(unit_runner, CK_NORMAL); - failed = srunner_ntests_failed(unit_runner); - srunner_free(unit_runner); - - return failed; -} - - -int main() -{ - int unit_failed = run_unit_tests(); - - return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; + suite_add_tcase(suite, flat); } diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h new file mode 100644 index 0000000..3c7e7e0 --- /dev/null +++ b/tests/include/rangequery.h @@ -0,0 +1,179 @@ +/* + * tests/include/rangequery.h + * + * Standardized unit tests for range queries against supporting + * shard types + * + * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface and Shard needs to support lower_bound. + * For other types of record and shard, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include "testing.h" +//#include <check.h> +//using namespace de; +//typedef ISAMTree<Rec> Shard; + + +START_TEST(t_range_query) +{ + auto buffer = create_sequential_mbuffer<Rec>(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rq::Parms<Rec> parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query<Shard, Rec>::get_query_state(&shard, &parms); + auto result = rq::Query<Shard, Rec>::query(&shard, state, &parms); + rq::Query<Shard, Rec>::delete_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i<result.size(); i++) { + ck_assert_int_le(result[i].rec.key, parms.upper_bound); + ck_assert_int_ge(result[i].rec.key, parms.lower_bound); + } + + delete buffer; +} +END_TEST + + +START_TEST(t_buffer_range_query) +{ + auto buffer = create_sequential_mbuffer<Rec>(100, 1000); + + rq::Parms<Rec> parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query<Shard, Rec>::get_buffer_query_state(buffer->get_buffer_view(), &parms); + auto result = rq::Query<Shard, Rec>::buffer_query(state, &parms); + rq::Query<Shard, Rec>::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i<result.size(); i++) { + ck_assert_int_le(result[i].rec.key, parms.upper_bound); + ck_assert_int_ge(result[i].rec.key, parms.lower_bound); + } + + delete buffer; +} +END_TEST + + +START_TEST(t_range_query_merge) +{ + auto buffer1 = create_sequential_mbuffer<Rec>(100, 200); + auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rq::Parms<Rec> parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rq::Query<Shard, Rec>::get_query_state(&shard1, &parms); + auto state2 = rq::Query<Shard, Rec>::get_query_state(&shard2, &parms); + + std::vector<std::vector<de::Wrapped<Rec>>> results(2); + results[0] = rq::Query<Shard, Rec>::query(&shard1, state1, &parms); + results[1] = rq::Query<Shard, Rec>::query(&shard2, state2, &parms); + + rq::Query<Shard, Rec>::delete_query_state(state1); + rq::Query<Shard, Rec>::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector<std::vector<Wrapped<Rec>>> proc_results; + + for (size_t j=0; j<results.size(); j++) { + proc_results.emplace_back(std::vector<Wrapped<Rec>>()); + for (size_t i=0; i<results[j].size(); i++) { + proc_results[j].emplace_back(results[j][i]); + } + } + + auto result = rq::Query<Shard, Rec>::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i<result.size(); i++) { + ck_assert_int_eq(key++, result[i].key); + if (key == 200) { + key = 400; + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + + +START_TEST(t_lower_bound) +{ + auto buffer1 = create_sequential_mbuffer<Rec>(100, 200); + auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000); + + Shard *shards[2]; + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + shards[0] = &shard1; + shards[1] = &shard2; + + auto merged = Shard(shards, 2); + + for (size_t i=100; i<1000; i++) { + Rec r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + +static void inject_rangequery_tests(Suite *suite) { + TCase *range_query = tcase_create("Range Query Testing"); + tcase_add_test(range_query, t_range_query); + tcase_add_test(range_query, t_buffer_range_query); + tcase_add_test(range_query, t_range_query_merge); + suite_add_tcase(suite, range_query); +} diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h new file mode 100644 index 0000000..047a7b5 --- /dev/null +++ b/tests/include/shard_standard.h @@ -0,0 +1,198 @@ +/* + * tests/include/shard_standard.h + * + * Standardized unit tests for Shard objects + * + * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "testing.h" +//#include <check.h> +//using namespace de; +//typedef ISAMTree<Rec> Shard; + +START_TEST(t_mbuffer_init) +{ + auto buffer = new MutableBuffer<Rec>(512, 1024); + for (uint64_t i = 512; i > 0; i--) { + uint32_t v = i; + buffer->append({i,v, 1}); + } + + for (uint64_t i = 1; i <= 256; ++i) { + uint32_t v = i; + buffer->append({i, v, 1}, true); + } + + for (uint64_t i = 257; i <= 512; ++i) { + uint32_t v = i + 1; + buffer->append({i, v, 1}); + } + + Shard* shard = new Shard(buffer->get_buffer_view()); + ck_assert_uint_eq(shard->get_record_count(), 512); + + delete buffer; + delete shard; +} + + +START_TEST(t_shard_init) +{ + size_t n = 512; + auto mbuffer1 = create_test_mbuffer<Rec>(n); + auto mbuffer2 = create_test_mbuffer<Rec>(n); + auto mbuffer3 = create_test_mbuffer<Rec>(n); + + auto shard1 = new Shard(mbuffer1->get_buffer_view()); + auto shard2 = new Shard(mbuffer2->get_buffer_view()); + auto shard3 = new Shard(mbuffer3->get_buffer_view()); + + Shard* shards[3] = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards, 3); + + ck_assert_int_eq(shard4->get_record_count(), n * 3); + ck_assert_int_eq(shard4->get_tombstone_count(), 0); + + size_t total_cnt = 0; + size_t shard1_idx = 0; + size_t shard2_idx = 0; + size_t shard3_idx = 0; + + for (size_t i = 0; i < shard4->get_record_count(); ++i) { + auto rec1 = shard1->get_record_at(shard1_idx); + auto rec2 = shard2->get_record_at(shard2_idx); + auto rec3 = shard3->get_record_at(shard3_idx); + + auto cur_rec = shard4->get_record_at(i); + + if (shard1_idx < n && cur_rec->rec == rec1->rec) { + ++shard1_idx; + } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { + ++shard2_idx; + } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { + ++shard3_idx; + } else { + assert(false); + } + } + + delete mbuffer1; + delete mbuffer2; + delete mbuffer3; + + delete shard1; + delete shard2; + delete shard3; + delete shard4; +} + + +START_TEST(t_full_cancelation) +{ + size_t n = 100; + auto buffer = create_double_seq_mbuffer<Rec>(n, false); + auto buffer_ts = create_double_seq_mbuffer<Rec>(n, true); + + Shard* shard = new Shard(buffer->get_buffer_view()); + Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); + + ck_assert_int_eq(shard->get_record_count(), n); + ck_assert_int_eq(shard->get_tombstone_count(), 0); + ck_assert_int_eq(shard_ts->get_record_count(), n); + ck_assert_int_eq(shard_ts->get_tombstone_count(), n); + + Shard* shards[] = {shard, shard_ts}; + + Shard* merged = new Shard(shards, 2); + + ck_assert_int_eq(merged->get_tombstone_count(), 0); + ck_assert_int_eq(merged->get_record_count(), 0); + + delete buffer; + delete buffer_ts; + delete shard; + delete shard_ts; + delete merged; +} +END_TEST + + +START_TEST(t_point_lookup) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer<Rec>(n, false); + auto isam = Shard(buffer->get_buffer_view()); + + { + auto view = buffer->get_buffer_view(); + + for (size_t i=0; i<n; i++) { + Rec r; + auto rec = view.get(i); + r.key = rec->rec.key; + r.value = rec->rec.value; + + auto result = isam.point_lookup(r); + ck_assert_ptr_nonnull(result); + ck_assert_int_eq(result->rec.key, r.key); + ck_assert_int_eq(result->rec.value, r.value); + } + } + + delete buffer; +} +END_TEST + + +START_TEST(t_point_lookup_miss) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer<Rec>(n, false); + auto isam = Shard(buffer->get_buffer_view()); + + for (size_t i=n + 100; i<2*n; i++) { + Rec r; + r.key = i; + r.value = i; + + auto result = isam.point_lookup(r); + ck_assert_ptr_null(result); + } + + delete buffer; +} + +static void inject_shard_tests(Suite *suite) { + TCase *create = tcase_create("Shard constructor Testing"); + tcase_add_test(create, t_mbuffer_init); + tcase_add_test(create, t_shard_init); + tcase_set_timeout(create, 100); + suite_add_tcase(suite, create); + TCase *tombstone = tcase_create("Shard tombstone cancellation Testing"); + tcase_add_test(tombstone, t_full_cancelation); + suite_add_tcase(suite, tombstone); + TCase *pointlookup = tcase_create("Shard point lookup Testing"); + tcase_add_test(pointlookup, t_point_lookup); + tcase_add_test(pointlookup, t_point_lookup_miss); + suite_add_tcase(suite, pointlookup); +} diff --git a/tests/testing.h b/tests/include/testing.h index 4e660dd..4e660dd 100644 --- a/tests/testing.h +++ b/tests/include/testing.h diff --git a/tests/internal_level_tests.cpp b/tests/internal_level_tests.cpp index b8aa56f..79b9c21 100644 --- a/tests/internal_level_tests.cpp +++ b/tests/internal_level_tests.cpp @@ -16,7 +16,7 @@ #include "framework/interface/Query.h" #include "framework/interface/Shard.h" -#include "testing.h" +#include "include/testing.h" #include <check.h> diff --git a/tests/memisam_tests.cpp b/tests/memisam_tests.cpp index 919fd69..b398524 100644 --- a/tests/memisam_tests.cpp +++ b/tests/memisam_tests.cpp @@ -11,359 +11,25 @@ */ #include "shard/ISAMTree.h" +#include "include/testing.h" #include "query/rangequery.h" -#include "testing.h" - #include <check.h> -using namespace de; - -typedef ISAMTree<Rec> Shard; - -START_TEST(t_mbuffer_init) -{ - auto buffer = new MutableBuffer<Rec>(512, 1024); - for (uint64_t i = 512; i > 0; i--) { - uint32_t v = i; - buffer->append({i,v, 1}); - } - - for (uint64_t i = 1; i <= 256; ++i) { - uint32_t v = i; - buffer->append({i, v, 1}, true); - } - - for (uint64_t i = 257; i <= 512; ++i) { - uint32_t v = i + 1; - buffer->append({i, v, 1}); - } - - Shard* shard = new Shard(buffer->get_buffer_view()); - ck_assert_uint_eq(shard->get_record_count(), 512); - - delete buffer; - delete shard; -} - - -START_TEST(t_rq_init) -{ - size_t n = 512; - auto mbuffer1 = create_test_mbuffer<Rec>(n); - auto mbuffer2 = create_test_mbuffer<Rec>(n); - auto mbuffer3 = create_test_mbuffer<Rec>(n); - - auto shard1 = new Shard(mbuffer1->get_buffer_view()); - auto shard2 = new Shard(mbuffer2->get_buffer_view()); - auto shard3 = new Shard(mbuffer3->get_buffer_view()); - - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); - - ck_assert_int_eq(shard4->get_record_count(), n * 3); - ck_assert_int_eq(shard4->get_tombstone_count(), 0); - - size_t total_cnt = 0; - size_t shard1_idx = 0; - size_t shard2_idx = 0; - size_t shard3_idx = 0; - - for (size_t i = 0; i < shard4->get_record_count(); ++i) { - auto rec1 = shard1->get_record_at(shard1_idx); - auto rec2 = shard2->get_record_at(shard2_idx); - auto rec3 = shard3->get_record_at(shard3_idx); - - auto cur_rec = shard4->get_record_at(i); - - if (shard1_idx < n && cur_rec->rec == rec1->rec) { - ++shard1_idx; - } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { - ++shard2_idx; - } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { - ++shard3_idx; - } else { - assert(false); - } - } - - delete mbuffer1; - delete mbuffer2; - delete mbuffer3; - - delete shard1; - delete shard2; - delete shard3; - delete shard4; -} - -START_TEST(t_point_lookup) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer<Rec>(n, false); - auto isam = Shard(buffer->get_buffer_view()); - - { - auto view = buffer->get_buffer_view(); - - for (size_t i=0; i<n; i++) { - Rec r; - auto rec = view.get(i); - r.key = rec->rec.key; - r.value = rec->rec.value; - - auto result = isam.point_lookup(r); - ck_assert_ptr_nonnull(result); - ck_assert_int_eq(result->rec.key, r.key); - ck_assert_int_eq(result->rec.value, r.value); - } - } - - delete buffer; -} -END_TEST - - -START_TEST(t_point_lookup_miss) -{ - size_t n = 10000; - - auto buffer = create_double_seq_mbuffer<Rec>(n, false); - auto isam = Shard(buffer->get_buffer_view()); - - for (size_t i=n + 100; i<2*n; i++) { - Rec r; - r.key = i; - r.value = i; - - auto result = isam.point_lookup(r); - ck_assert_ptr_null(result); - } - - delete buffer; -} - - -START_TEST(t_full_cancelation) -{ - size_t n = 100; - auto buffer = create_double_seq_mbuffer<Rec>(n, false); - auto buffer_ts = create_double_seq_mbuffer<Rec>(n, true); - - Shard* shard = new Shard(buffer->get_buffer_view()); - Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); - - ck_assert_int_eq(shard->get_record_count(), n); - ck_assert_int_eq(shard->get_tombstone_count(), 0); - ck_assert_int_eq(shard_ts->get_record_count(), n); - ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - - Shard* shards[] = {shard, shard_ts}; - - Shard* merged = new Shard(shards, 2); - - ck_assert_int_eq(merged->get_tombstone_count(), 0); - ck_assert_int_eq(merged->get_record_count(), 0); - - delete buffer; - delete buffer_ts; - delete shard; - delete shard_ts; - delete merged; -} -END_TEST - - -/* -START_TEST(t_rq_query) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer<Rec>(n); - auto isam = Shard(buffer->get_buffer_view()); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - size_t k = 100; - size_t cnt[3] = {0}; - rq::Parms<Rec> parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query<Shard, Rec, false>::get_query_state(&isam, &parms); - ((rq::State<WRec> *) state)->sample_size = k; - auto result = rq::Query<Shard, Rec, false>::query(&isam, state, &parms); - - ck_assert_int_eq(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query<Shard, Rec, false>::delete_query_state(state); - } - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_rq_query_merge) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer<Rec>(n); - - Shard shard = Shard(buffer->get_buffer_view()); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 1000; - - size_t cnt[3] = {0}; - rq::Parms<Rec> parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - std::vector<std::vector<de::Wrapped<Rec>>> results(2); - - for (size_t i=0; i<1000; i++) { - auto state1 = rq::Query<Shard, Rec>::get_query_state(&shard, &parms); - ((rq::State<WRec> *) state1)->sample_size = k; - results[0] = rq::Query<Shard, Rec>::query(&shard, state1, &parms); - - auto state2 = rq::Query<Shard, Rec>::get_query_state(&shard, &parms); - ((rq::State<WRec> *) state2)->sample_size = k; - results[1] = rq::Query<Shard, Rec>::query(&shard, state2, &parms); - - rq::Query<Shard, Rec>::delete_query_state(state1); - rq::Query<Shard, Rec>::delete_query_state(state2); - } - - auto merged = rq::Query<Shard, Rec>::merge(results, nullptr); - - ck_assert_int_eq(merged.size(), 2*k); - for (size_t i=0; i<merged.size(); i++) { - ck_assert_int_ge(merged[i].key, lower_key); - ck_assert_int_le(merged[i].key, upper_key); - } - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_rq_buffer_query_scan) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer<Rec>(n); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 100; - - size_t cnt[3] = {0}; - rq::Parms<Rec> parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query<Shard, Rec, false>::get_buffer_query_state(buffer, &parms); - ((rq::BufferState<WRec> *) state)->sample_size = k; - auto result = rq::Query<Shard, Rec, false>::buffer_query(buffer, state, &parms); - - ck_assert_int_eq(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query<Shard, Rec, false>::delete_buffer_query_state(state); - } - - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST - - -START_TEST(t_rq_buffer_query_rejection) -{ - size_t n=1000; - auto buffer = create_double_seq_mbuffer<Rec>(n); - - uint64_t lower_key = 100; - uint64_t upper_key = 250; - - size_t k = 10000; - - size_t cnt[3] = {0}; - rq::Parms<Rec> parms = {lower_key, upper_key, k}; - parms.rng = gsl_rng_alloc(gsl_rng_mt19937); - - size_t total_samples = 0; - - for (size_t i=0; i<1000; i++) { - auto state = rq::Query<Shard, Rec>::get_buffer_query_state(buffer, &parms); - ((rq::BufferState<WRec> *) state)->sample_size = k; - auto result = rq::Query<Shard, Rec>::buffer_query(buffer, state, &parms); - - ck_assert_int_gt(result.size(), 0); - ck_assert_int_le(result.size(), k); - - for (auto &rec : result) { - ck_assert_int_le(rec.rec.key, upper_key); - ck_assert_int_ge(rec.rec.key, lower_key); - } - - rq::Query<Shard, Rec>::delete_buffer_query_state(state); - } +using namespace de; - gsl_rng_free(parms.rng); - delete buffer; -} -END_TEST -*/ +typedef ISAMTree<Rec> Shard; +#include "include/shard_standard.h" +#include "include/rangequery.h" Suite *unit_testing() { Suite *unit = suite_create("ISAMTree Shard Unit Testing"); - TCase *create = tcase_create("de::ISAMTree constructor Testing"); - tcase_add_test(create, t_mbuffer_init); - tcase_add_test(create, t_rq_init); - tcase_set_timeout(create, 100); - suite_add_tcase(unit, create); - - - TCase *tombstone = tcase_create("de:ISAMTree::tombstone cancellation Testing"); - tcase_add_test(tombstone, t_full_cancelation); - suite_add_tcase(unit, tombstone); - - - TCase *lookup = tcase_create("de:ISAMTree:point_lookup Testing"); - tcase_add_test(lookup, t_point_lookup); - tcase_add_test(lookup, t_point_lookup_miss); - suite_add_tcase(unit, lookup); - - /* - TCase *sampling = tcase_create("de:ISAMTree::IRS Testing"); - tcase_add_test(sampling, t_rq_query); - tcase_add_test(sampling, t_rq_query_merge); - tcase_add_test(sampling, t_rq_buffer_query_rejection); - tcase_add_test(sampling, t_rq_buffer_query_scan); - tcase_set_timeout(sampling, 100); - suite_add_tcase(unit, sampling); - */ + inject_rangequery_tests(unit); + inject_shard_tests(unit); return unit; } diff --git a/tests/mutable_buffer_tests.cpp b/tests/mutable_buffer_tests.cpp index e61a832..4064412 100644 --- a/tests/mutable_buffer_tests.cpp +++ b/tests/mutable_buffer_tests.cpp @@ -13,7 +13,7 @@ #include <thread> #include <vector> -#include "testing.h" +#include "include/testing.h" #include "framework/structure/MutableBuffer.h" #include <check.h> diff --git a/tests/rangequery_tests.cpp b/tests/rangequery_tests.cpp new file mode 100644 index 0000000..6a00f5a --- /dev/null +++ b/tests/rangequery_tests.cpp @@ -0,0 +1,195 @@ +/* + * tests/rangequery_tests.cpp + * + * Unit tests for Range Queries across several different + * shards + * + * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> + * Dong Xie <dongx@psu.edu> + * + * Distributed under the Modified BSD License. + * + */ + +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include "include/testing.h" + +#include <check.h> + +using namespace de; + +typedef ISAMTree<Rec> Shard; + +START_TEST(t_range_query) +{ + auto buffer = create_sequential_mbuffer<Rec>(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rq::Parms<Rec> parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query<Shard, Rec>::get_query_state(&shard, &parms); + auto result = rq::Query<Shard, Rec>::query(&shard, state, &parms); + rq::Query<Shard, Rec>::delete_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i<result.size(); i++) { + ck_assert_int_le(result[i].rec.key, parms.upper_bound); + ck_assert_int_ge(result[i].rec.key, parms.lower_bound); + } + + delete buffer; +} +END_TEST + + +START_TEST(t_buffer_range_query) +{ + auto buffer = create_sequential_mbuffer<Rec>(100, 1000); + + rq::Parms<Rec> parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query<Shard, Rec>::get_buffer_query_state(buffer->get_buffer_view(), &parms); + auto result = rq::Query<Shard, Rec>::buffer_query(state, &parms); + rq::Query<Shard, Rec>::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i<result.size(); i++) { + ck_assert_int_le(result[i].rec.key, parms.upper_bound); + ck_assert_int_ge(result[i].rec.key, parms.lower_bound); + } + + delete buffer; +} +END_TEST + + +START_TEST(t_range_query_merge) +{ + auto buffer1 = create_sequential_mbuffer<Rec>(100, 200); + auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rq::Parms<Rec> parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rq::Query<Shard, Rec>::get_query_state(&shard1, &parms); + auto state2 = rq::Query<Shard, Rec>::get_query_state(&shard2, &parms); + + std::vector<std::vector<de::Wrapped<Rec>>> results(2); + results[0] = rq::Query<Shard, Rec>::query(&shard1, state1, &parms); + results[1] = rq::Query<Shard, Rec>::query(&shard2, state2, &parms); + + rq::Query<Shard, Rec>::delete_query_state(state1); + rq::Query<Shard, Rec>::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector<std::vector<Wrapped<Rec>>> proc_results; + + for (size_t j=0; j<results.size(); j++) { + proc_results.emplace_back(std::vector<Wrapped<Rec>>()); + for (size_t i=0; i<results[j].size(); i++) { + proc_results[j].emplace_back(results[j][i]); + } + } + + auto result = rq::Query<Shard, Rec>::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i<result.size(); i++) { + ck_assert_int_eq(key++, result[i].key); + if (key == 200) { + key = 400; + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + +START_TEST(t_lower_bound) +{ + auto buffer1 = create_sequential_mbuffer<Rec>(100, 200); + auto buffer2 = create_sequential_mbuffer<Rec>(400, 1000); + + Shard *shards[2]; + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + shards[0] = &shard1; + shards[1] = &shard2; + + auto merged = Shard(shards, 2); + + for (size_t i=100; i<1000; i++) { + Rec r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + + +Suite *unit_testing() +{ + Suite *unit = suite_create("Range Query Unit Testing"); + + TCase *range_query = tcase_create("de:PGM::range_query Testing"); + tcase_add_test(range_query, t_range_query); + tcase_add_test(range_query, t_buffer_range_query); + tcase_add_test(range_query, t_range_query_merge); + suite_add_tcase(unit, range_query); + + return unit; +} + + +int shard_unit_tests() +{ + int failed = 0; + Suite *unit = unit_testing(); + SRunner *unit_shardner = srunner_create(unit); + + srunner_run_all(unit_shardner, CK_NORMAL); + failed = srunner_ntests_failed(unit_shardner); + srunner_free(unit_shardner); + + return failed; +} + + +int main() +{ + int unit_failed = shard_unit_tests(); + + return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} |