From c4514c2e62a711189cf3c914297885d97fb51a09 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 12 Jan 2024 14:08:33 -0500 Subject: Initial pass at unit test refactoring Restructured unit tests to be a bit more modular. I have some further plans to expand on this, particular for the query tests (including both shard and framework level test functions that can be injected at will). --- tests/include/shard_standard.h | 198 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 tests/include/shard_standard.h (limited to 'tests/include/shard_standard.h') diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h new file mode 100644 index 0000000..047a7b5 --- /dev/null +++ b/tests/include/shard_standard.h @@ -0,0 +1,198 @@ +/* + * tests/include/shard_standard.h + * + * Standardized unit tests for Shard objects + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +//#include "shard/ISAMTree.h" +//#include "testing.h" +//#include +//using namespace de; +//typedef ISAMTree Shard; + +START_TEST(t_mbuffer_init) +{ + auto buffer = new MutableBuffer(512, 1024); + for (uint64_t i = 512; i > 0; i--) { + uint32_t v = i; + buffer->append({i,v, 1}); + } + + for (uint64_t i = 1; i <= 256; ++i) { + uint32_t v = i; + buffer->append({i, v, 1}, true); + } + + for (uint64_t i = 257; i <= 512; ++i) { + uint32_t v = i + 1; + buffer->append({i, v, 1}); + } + + Shard* shard = new Shard(buffer->get_buffer_view()); + ck_assert_uint_eq(shard->get_record_count(), 512); + + delete buffer; + delete shard; +} + + +START_TEST(t_shard_init) +{ + size_t n = 512; + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); + + auto shard1 = new Shard(mbuffer1->get_buffer_view()); + auto shard2 = new Shard(mbuffer2->get_buffer_view()); + auto shard3 = new Shard(mbuffer3->get_buffer_view()); + + Shard* shards[3] = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards, 3); + + ck_assert_int_eq(shard4->get_record_count(), n * 3); + ck_assert_int_eq(shard4->get_tombstone_count(), 0); + + size_t total_cnt = 0; + size_t shard1_idx = 0; + size_t shard2_idx = 0; + size_t shard3_idx = 0; + + for (size_t i = 0; i < shard4->get_record_count(); ++i) { + auto rec1 = shard1->get_record_at(shard1_idx); + auto rec2 = shard2->get_record_at(shard2_idx); + auto rec3 = shard3->get_record_at(shard3_idx); + + auto cur_rec = shard4->get_record_at(i); + + if (shard1_idx < n && cur_rec->rec == rec1->rec) { + ++shard1_idx; + } else if (shard2_idx < n && cur_rec->rec == rec2->rec) { + ++shard2_idx; + } else if (shard3_idx < n && cur_rec->rec == rec3->rec) { + ++shard3_idx; + } else { + assert(false); + } + } + + delete mbuffer1; + delete mbuffer2; + delete mbuffer3; + + delete shard1; + delete shard2; + delete shard3; + delete shard4; +} + + +START_TEST(t_full_cancelation) +{ + size_t n = 100; + auto buffer = create_double_seq_mbuffer(n, false); + auto buffer_ts = create_double_seq_mbuffer(n, true); + + Shard* shard = new Shard(buffer->get_buffer_view()); + Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); + + ck_assert_int_eq(shard->get_record_count(), n); + ck_assert_int_eq(shard->get_tombstone_count(), 0); + ck_assert_int_eq(shard_ts->get_record_count(), n); + ck_assert_int_eq(shard_ts->get_tombstone_count(), n); + + Shard* shards[] = {shard, shard_ts}; + + Shard* merged = new Shard(shards, 2); + + ck_assert_int_eq(merged->get_tombstone_count(), 0); + ck_assert_int_eq(merged->get_record_count(), 0); + + delete buffer; + delete buffer_ts; + delete shard; + delete shard_ts; + delete merged; +} +END_TEST + + +START_TEST(t_point_lookup) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto isam = Shard(buffer->get_buffer_view()); + + { + auto view = buffer->get_buffer_view(); + + for (size_t i=0; irec.key; + r.value = rec->rec.value; + + auto result = isam.point_lookup(r); + ck_assert_ptr_nonnull(result); + ck_assert_int_eq(result->rec.key, r.key); + ck_assert_int_eq(result->rec.value, r.value); + } + } + + delete buffer; +} +END_TEST + + +START_TEST(t_point_lookup_miss) +{ + size_t n = 10000; + + auto buffer = create_double_seq_mbuffer(n, false); + auto isam = Shard(buffer->get_buffer_view()); + + for (size_t i=n + 100; i<2*n; i++) { + Rec r; + r.key = i; + r.value = i; + + auto result = isam.point_lookup(r); + ck_assert_ptr_null(result); + } + + delete buffer; +} + +static void inject_shard_tests(Suite *suite) { + TCase *create = tcase_create("Shard constructor Testing"); + tcase_add_test(create, t_mbuffer_init); + tcase_add_test(create, t_shard_init); + tcase_set_timeout(create, 100); + suite_add_tcase(suite, create); + TCase *tombstone = tcase_create("Shard tombstone cancellation Testing"); + tcase_add_test(tombstone, t_full_cancelation); + suite_add_tcase(suite, tombstone); + TCase *pointlookup = tcase_create("Shard point lookup Testing"); + tcase_add_test(pointlookup, t_point_lookup); + tcase_add_test(pointlookup, t_point_lookup_miss); + suite_add_tcase(suite, pointlookup); +} -- cgit v1.2.3 From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 17 Jan 2024 18:22:00 -0500 Subject: Multiple concurrency bug fixes A poorly organized commit with fixes for a variety of bugs that were causing missing records. The core problems all appear to be fixed, though there is an outstanding problem with tombstones not being completely canceled. A very small number are appearing in the wrong order during the static structure test. --- tests/include/shard_standard.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'tests/include/shard_standard.h') diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h index 047a7b5..ddd7614 100644 --- a/tests/include/shard_standard.h +++ b/tests/include/shard_standard.h @@ -65,8 +65,8 @@ START_TEST(t_shard_init) auto shard2 = new Shard(mbuffer2->get_buffer_view()); auto shard3 = new Shard(mbuffer3->get_buffer_view()); - Shard* shards[3] = {shard1, shard2, shard3}; - auto shard4 = new Shard(shards, 3); + std::vector shards = {shard1, shard2, shard3}; + auto shard4 = new Shard(shards); ck_assert_int_eq(shard4->get_record_count(), n * 3); ck_assert_int_eq(shard4->get_tombstone_count(), 0); @@ -119,9 +119,9 @@ START_TEST(t_full_cancelation) ck_assert_int_eq(shard_ts->get_record_count(), n); ck_assert_int_eq(shard_ts->get_tombstone_count(), n); - Shard* shards[] = {shard, shard_ts}; + std::vector shards = {shard, shard_ts}; - Shard* merged = new Shard(shards, 2); + Shard* merged = new Shard(shards); ck_assert_int_eq(merged->get_tombstone_count(), 0); ck_assert_int_eq(merged->get_record_count(), 0); -- cgit v1.2.3 From bd74e27b28bd95267ce50d2e4b6f12b51d9b6aae Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 7 Feb 2024 17:23:23 -0500 Subject: Cleaned up shard files (except VPTree) Cleaned up shard implementations, fixed a few bugs, and set up some tests. There's still some work to be done in creating tests for the weighted sampling operations for the alias and aug btree shards. --- tests/include/shard_standard.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'tests/include/shard_standard.h') diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h index ddd7614..f50c1cb 100644 --- a/tests/include/shard_standard.h +++ b/tests/include/shard_standard.h @@ -8,8 +8,8 @@ * Distributed under the Modified BSD License. * * WARNING: This file must be included in the main unit test set - * after the definition of an appropriate Shard and Rec - * type. In particular, Rec needs to implement the key-value + * after the definition of an appropriate Shard and R + * type. In particular, R needs to implement the key-value * pair interface. For other types of record, you'll need to * use a different set of unit tests. */ @@ -26,11 +26,11 @@ //#include "testing.h" //#include //using namespace de; -//typedef ISAMTree Shard; +//typedef ISAMTree Shard; START_TEST(t_mbuffer_init) { - auto buffer = new MutableBuffer(512, 1024); + auto buffer = new MutableBuffer(512, 1024); for (uint64_t i = 512; i > 0; i--) { uint32_t v = i; buffer->append({i,v, 1}); @@ -57,9 +57,9 @@ START_TEST(t_mbuffer_init) START_TEST(t_shard_init) { size_t n = 512; - auto mbuffer1 = create_test_mbuffer(n); - auto mbuffer2 = create_test_mbuffer(n); - auto mbuffer3 = create_test_mbuffer(n); + auto mbuffer1 = create_test_mbuffer(n); + auto mbuffer2 = create_test_mbuffer(n); + auto mbuffer3 = create_test_mbuffer(n); auto shard1 = new Shard(mbuffer1->get_buffer_view()); auto shard2 = new Shard(mbuffer2->get_buffer_view()); @@ -108,8 +108,8 @@ START_TEST(t_shard_init) START_TEST(t_full_cancelation) { size_t n = 100; - auto buffer = create_double_seq_mbuffer(n, false); - auto buffer_ts = create_double_seq_mbuffer(n, true); + auto buffer = create_double_seq_mbuffer(n, false); + auto buffer_ts = create_double_seq_mbuffer(n, true); Shard* shard = new Shard(buffer->get_buffer_view()); Shard* shard_ts = new Shard(buffer_ts->get_buffer_view()); @@ -139,14 +139,14 @@ START_TEST(t_point_lookup) { size_t n = 10000; - auto buffer = create_double_seq_mbuffer(n, false); + auto buffer = create_double_seq_mbuffer(n, false); auto isam = Shard(buffer->get_buffer_view()); { auto view = buffer->get_buffer_view(); for (size_t i=0; irec.key; r.value = rec->rec.value; @@ -167,11 +167,11 @@ START_TEST(t_point_lookup_miss) { size_t n = 10000; - auto buffer = create_double_seq_mbuffer(n, false); + auto buffer = create_double_seq_mbuffer(n, false); auto isam = Shard(buffer->get_buffer_view()); for (size_t i=n + 100; i<2*n; i++) { - Rec r; + R r; r.key = i; r.value = i; -- cgit v1.2.3 From 711769574e647839677739192698e400529efe75 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 8 Feb 2024 16:38:44 -0500 Subject: Updated VPTree to new shard/query interfaces --- tests/include/shard_standard.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'tests/include/shard_standard.h') diff --git a/tests/include/shard_standard.h b/tests/include/shard_standard.h index f50c1cb..7d17dcb 100644 --- a/tests/include/shard_standard.h +++ b/tests/include/shard_standard.h @@ -22,18 +22,22 @@ * should be included in the source file that includes this one, above the * include statement. */ -//#include "shard/ISAMTree.h" -//#include "testing.h" -//#include -//using namespace de; -//typedef ISAMTree Shard; +/* +#include "shard/ISAMTree.h" +#include "shard/ISAMTree.h" +#include "testing.h" +#include +using namespace de; +typedef Rec R; +typedef ISAMTree Shard; +*/ START_TEST(t_mbuffer_init) { auto buffer = new MutableBuffer(512, 1024); for (uint64_t i = 512; i > 0; i--) { uint32_t v = i; - buffer->append({i,v, 1}); + buffer->append({i, v, 1}); } for (uint64_t i = 1; i <= 256; ++i) { -- cgit v1.2.3