From c4514c2e62a711189cf3c914297885d97fb51a09 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 12 Jan 2024 14:08:33 -0500 Subject: Initial pass at unit test refactoring Restructured unit tests to be a bit more modular. I have some further plans to expand on this, particular for the query tests (including both shard and framework level test functions that can be injected at will). --- tests/include/rangequery.h | 179 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 tests/include/rangequery.h (limited to 'tests/include/rangequery.h') diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h new file mode 100644 index 0000000..3c7e7e0 --- /dev/null +++ b/tests/include/rangequery.h @@ -0,0 +1,179 @@ +/* + * tests/include/rangequery.h + * + * Standardized unit tests for range queries against supporting + * shard types + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface and Shard needs to support lower_bound. + * For other types of record and shard, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; + + +START_TEST(t_range_query) +{ + auto buffer = create_sequential_mbuffer(100, 1000); + auto shard = Shard(buffer->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 1000); + + rq::Parms parms; + parms.lower_bound = 300; + parms.upper_bound = 500; + + auto state = rq::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + rq::Parms parms; + parms.lower_bound = 150; + parms.upper_bound = 500; + + size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; + + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); + + std::vector>> results(2); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); + + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); + + ck_assert_int_eq(results[0].size() + results[1].size(), result_size); + + std::vector>> proc_results; + + for (size_t j=0; j>()); + for (size_t i=0; i::merge(proc_results, nullptr); + std::sort(result.begin(), result.end()); + + ck_assert_int_eq(result.size(), result_size); + auto key = parms.lower_bound; + for (size_t i=0; i(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); + + Shard *shards[2]; + + auto shard1 = Shard(buffer1->get_buffer_view()); + auto shard2 = Shard(buffer2->get_buffer_view()); + + shards[0] = &shard1; + shards[1] = &shard2; + + auto merged = Shard(shards, 2); + + for (size_t i=100; i<1000; i++) { + Rec r; + r.key = i; + r.value = i; + + auto idx = merged.get_lower_bound(i); + + assert(idx < merged.get_record_count()); + + auto res = merged.get_record_at(idx); + + if (i >=200 && i <400) { + ck_assert_int_lt(res->rec.key, i); + } else { + ck_assert_int_eq(res->rec.key, i); + } + } + + delete buffer1; + delete buffer2; +} +END_TEST + +static void inject_rangequery_tests(Suite *suite) { + TCase *range_query = tcase_create("Range Query Testing"); + tcase_add_test(range_query, t_range_query); + tcase_add_test(range_query, t_buffer_range_query); + tcase_add_test(range_query, t_range_query_merge); + suite_add_tcase(suite, range_query); +} -- cgit v1.2.3 From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 17 Jan 2024 18:22:00 -0500 Subject: Multiple concurrency bug fixes A poorly organized commit with fixes for a variety of bugs that were causing missing records. The core problems all appear to be fixed, though there is an outstanding problem with tombstones not being completely canceled. A very small number are appearing in the wrong order during the static structure test. --- tests/include/rangequery.h | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) (limited to 'tests/include/rangequery.h') diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index 3c7e7e0..e45de57 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -24,12 +24,12 @@ * should be included in the source file that includes this one, above the * include statement. */ -//#include "shard/ISAMTree.h" -//#include "query/rangequery.h" -//#include "testing.h" -//#include -//using namespace de; -//typedef ISAMTree Shard; +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include "testing.h" +#include +using namespace de; +typedef ISAMTree Shard; START_TEST(t_range_query) @@ -137,15 +137,12 @@ START_TEST(t_lower_bound) auto buffer1 = create_sequential_mbuffer(100, 200); auto buffer2 = create_sequential_mbuffer(400, 1000); - Shard *shards[2]; + auto shard1 = new Shard(buffer1->get_buffer_view()); + auto shard2 = new Shard(buffer2->get_buffer_view()); - auto shard1 = Shard(buffer1->get_buffer_view()); - auto shard2 = Shard(buffer2->get_buffer_view()); - - shards[0] = &shard1; - shards[1] = &shard2; + std::vector shards = {shard1, shard2}; - auto merged = Shard(shards, 2); + auto merged = Shard(shards); for (size_t i=100; i<1000; i++) { Rec r; @@ -167,6 +164,8 @@ START_TEST(t_lower_bound) delete buffer1; delete buffer2; + delete shard1; + delete shard2; } END_TEST -- cgit v1.2.3 From 51a85013236f4b2bd596caf179d90e67c848963c Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 30 Jan 2024 15:31:34 -0500 Subject: TrieSpline + tests --- tests/include/rangequery.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'tests/include/rangequery.h') diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index e45de57..1ac0891 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -24,12 +24,12 @@ * should be included in the source file that includes this one, above the * include statement. */ -#include "shard/ISAMTree.h" -#include "query/rangequery.h" -#include "testing.h" -#include -using namespace de; -typedef ISAMTree Shard; +//#include "shard/ISAMTree.h" +//#include "query/rangequery.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; START_TEST(t_range_query) -- cgit v1.2.3 From db4806d9dd9757273a14e6c3ea92e5a087239145 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 5 Feb 2024 15:17:25 -0500 Subject: Set up tombstone deletes properly --- tests/include/rangequery.h | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'tests/include/rangequery.h') diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index 1ac0891..b9694a4 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -64,14 +64,17 @@ START_TEST(t_buffer_range_query) parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rq::Query::get_buffer_query_state(buffer->get_buffer_view(), &parms); - auto result = rq::Query::buffer_query(state, &parms); - rq::Query::delete_buffer_query_state(state); - - ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); - for (size_t i=0; iget_buffer_view(); + auto state = rq::Query::get_buffer_query_state(&view, &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); + + ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); + for (size_t i=0; i Date: Wed, 7 Feb 2024 13:42:34 -0500 Subject: Fully realized shard concept interface --- tests/include/rangequery.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'tests/include/rangequery.h') diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index b9694a4..dbb71db 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -41,9 +41,9 @@ START_TEST(t_range_query) parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rq::Query::get_query_state(&shard, &parms); - auto result = rq::Query::query(&shard, state, &parms); - rq::Query::delete_query_state(state); + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; iget_buffer_view(); - auto state = rq::Query::get_buffer_query_state(&view, &parms); - auto result = rq::Query::buffer_query(state, &parms); - rq::Query::delete_buffer_query_state(state); + auto state = rq::Query::get_buffer_query_state(&view, &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i::get_query_state(&shard1, &parms); - auto state2 = rq::Query::get_query_state(&shard2, &parms); + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); std::vector>> results(2); - results[0] = rq::Query::query(&shard1, state1, &parms); - results[1] = rq::Query::query(&shard2, state2, &parms); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); - rq::Query::delete_query_state(state1); - rq::Query::delete_query_state(state2); + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); ck_assert_int_eq(results[0].size() + results[1].size(), result_size); @@ -117,7 +117,7 @@ START_TEST(t_range_query_merge) } } - auto result = rq::Query::merge(proc_results, nullptr); + auto result = rq::Query::merge(proc_results, nullptr); std::sort(result.begin(), result.end()); ck_assert_int_eq(result.size(), result_size); -- cgit v1.2.3 From bd74e27b28bd95267ce50d2e4b6f12b51d9b6aae Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 7 Feb 2024 17:23:23 -0500 Subject: Cleaned up shard files (except VPTree) Cleaned up shard implementations, fixed a few bugs, and set up some tests. There's still some work to be done in creating tests for the weighted sampling operations for the alias and aug btree shards. --- tests/include/rangequery.h | 60 ++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 29 deletions(-) (limited to 'tests/include/rangequery.h') diff --git a/tests/include/rangequery.h b/tests/include/rangequery.h index dbb71db..a8a73f7 100644 --- a/tests/include/rangequery.h +++ b/tests/include/rangequery.h @@ -9,8 +9,8 @@ * Distributed under the Modified BSD License. * * WARNING: This file must be included in the main unit test set - * after the definition of an appropriate Shard and Rec - * type. In particular, Rec needs to implement the key-value + * after the definition of an appropriate Shard and R + * type. In particular, R needs to implement the key-value * pair interface and Shard needs to support lower_bound. * For other types of record and shard, you'll need to * use a different set of unit tests. @@ -29,21 +29,23 @@ //#include "testing.h" //#include //using namespace de; -//typedef ISAMTree Shard; +//typedef ISAMTree Shard; + +#include "query/rangequery.h" START_TEST(t_range_query) { - auto buffer = create_sequential_mbuffer(100, 1000); + auto buffer = create_sequential_mbuffer(100, 1000); auto shard = Shard(buffer->get_buffer_view()); - rq::Parms parms; + rq::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; - auto state = rq::Query::get_query_state(&shard, &parms); - auto result = rq::Query::query(&shard, state, &parms); - rq::Query::delete_query_state(state); + auto state = rq::Query::get_query_state(&shard, &parms); + auto result = rq::Query::query(&shard, state, &parms); + rq::Query::delete_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i(100, 1000); + auto buffer = create_sequential_mbuffer(100, 1000); - rq::Parms parms; + rq::Parms parms; parms.lower_bound = 300; parms.upper_bound = 500; { auto view = buffer->get_buffer_view(); - auto state = rq::Query::get_buffer_query_state(&view, &parms); - auto result = rq::Query::buffer_query(state, &parms); - rq::Query::delete_buffer_query_state(state); + auto state = rq::Query::get_buffer_query_state(&view, &parms); + auto result = rq::Query::buffer_query(state, &parms); + rq::Query::delete_buffer_query_state(state); ck_assert_int_eq(result.size(), parms.upper_bound - parms.lower_bound + 1); for (size_t i=0; i(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); + auto buffer1 = create_sequential_mbuffer(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); auto shard1 = Shard(buffer1->get_buffer_view()); auto shard2 = Shard(buffer2->get_buffer_view()); - rq::Parms parms; + rq::Parms parms; parms.lower_bound = 150; parms.upper_bound = 500; size_t result_size = parms.upper_bound - parms.lower_bound + 1 - 200; - auto state1 = rq::Query::get_query_state(&shard1, &parms); - auto state2 = rq::Query::get_query_state(&shard2, &parms); + auto state1 = rq::Query::get_query_state(&shard1, &parms); + auto state2 = rq::Query::get_query_state(&shard2, &parms); - std::vector>> results(2); - results[0] = rq::Query::query(&shard1, state1, &parms); - results[1] = rq::Query::query(&shard2, state2, &parms); + std::vector>> results(2); + results[0] = rq::Query::query(&shard1, state1, &parms); + results[1] = rq::Query::query(&shard2, state2, &parms); - rq::Query::delete_query_state(state1); - rq::Query::delete_query_state(state2); + rq::Query::delete_query_state(state1); + rq::Query::delete_query_state(state2); ck_assert_int_eq(results[0].size() + results[1].size(), result_size); - std::vector>> proc_results; + std::vector>> proc_results; for (size_t j=0; j>()); + proc_results.emplace_back(std::vector>()); for (size_t i=0; i::merge(proc_results, nullptr); + auto result = rq::Query::merge(proc_results, nullptr); std::sort(result.begin(), result.end()); ck_assert_int_eq(result.size(), result_size); @@ -137,8 +139,8 @@ END_TEST START_TEST(t_lower_bound) { - auto buffer1 = create_sequential_mbuffer(100, 200); - auto buffer2 = create_sequential_mbuffer(400, 1000); + auto buffer1 = create_sequential_mbuffer(100, 200); + auto buffer2 = create_sequential_mbuffer(400, 1000); auto shard1 = new Shard(buffer1->get_buffer_view()); auto shard2 = new Shard(buffer2->get_buffer_view()); @@ -148,7 +150,7 @@ START_TEST(t_lower_bound) auto merged = Shard(shards); for (size_t i=100; i<1000; i++) { - Rec r; + R r; r.key = i; r.value = i; -- cgit v1.2.3