From 2117935e85412f3733ee0bcb1830c7fd0b129b29 Mon Sep 17 00:00:00 2001 From: "Douglas B. Rumbaugh" Date: Mon, 15 Jan 2024 17:23:57 -0500 Subject: Concurrency testing and bug fixes --- tests/include/concurrent_extension.h | 379 +++++++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 tests/include/concurrent_extension.h (limited to 'tests/include/concurrent_extension.h') diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h new file mode 100644 index 0000000..86f8e12 --- /dev/null +++ b/tests/include/concurrent_extension.h @@ -0,0 +1,379 @@ +/* + * tests/include/dynamic_extension.h + * + * Standardized unit tests for DynamicExtension objects + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * Distributed under the Modified BSD License. + * + * WARNING: This file must be included in the main unit test set + * after the definition of an appropriate Shard, Query, and Rec + * type. In particular, Rec needs to implement the key-value + * pair interface. For other types of record, you'll need to + * use a different set of unit tests. + */ +#pragma once + +/* + * Uncomment these lines temporarily to remove errors in this file + * temporarily for development purposes. They should be removed prior + * to building, to ensure no duplicate definitions. These includes/defines + * should be included in the source file that includes this one, above the + * include statement. + */ +#include "testing.h" +#include "framework/DynamicExtension.h" +#include "framework/scheduling/FIFOScheduler.h" +#include "shard/ISAMTree.h" +#include "query/rangequery.h" +#include +using namespace de; +typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; + + +START_TEST(t_create) +{ + auto test_de = new DE(100, 1000, 2); + + ck_assert_ptr_nonnull(test_de); + ck_assert_int_eq(test_de->get_record_count(), 0); + ck_assert_int_eq(test_de->get_height(), 0); + + delete test_de; +} +END_TEST + + +START_TEST(t_insert) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<100; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + key++; + val++; + } + + ck_assert_int_eq(test_de->get_height(), 0); + ck_assert_int_eq(test_de->get_record_count(), 100); + + delete test_de; +} +END_TEST + + +START_TEST(t_debug_insert) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + for (size_t i=0; i<1000; i++) { + Rec r = {key, val}; + ck_assert_int_eq(test_de->insert(r), 1); + //ck_assert_int_eq(test_de->get_record_count(), i+1); + key++; + val++; + } + + delete test_de; +} +END_TEST + + +START_TEST(t_insert_with_mem_merges) +{ + auto test_de = new DE(100, 1000, 2); + + uint64_t key = 0; + uint32_t val = 0; + + Rec r = {key, val}; + for (size_t i=0; i<1000; i++) { + ck_assert_int_eq(test_de->insert(r), 1); + r.key++; + r.value++; + } + + ck_assert_int_eq(test_de->get_record_count(), 1000); + + test_de->await_next_epoch(); + + ck_assert_int_eq(test_de->get_record_count(), 1000); + + /* + * verify that we can fill past the high water mark, potentially + * stalling to allow merges to finish as needed. + */ + size_t cnt = 0; + do { + if (test_de->insert(r)) { + r.key++; + r.value++; + cnt++; + } else { + sleep(1); + } + } while (cnt < 10000); + + test_de->await_next_epoch(); + + ck_assert_int_eq(test_de->get_record_count(), 11000); + + delete test_de; +} +END_TEST + + +START_TEST(t_range_query) +{ + auto test_de = new DE(100, 1000, 2); + size_t n = 10000; + + std::vector keys; + for (size_t i=0; iinsert(r)) { + i++; + } else { + sleep(1); + } + } + + test_de->await_next_epoch(); + + std::sort(keys.begin(), keys.end()); + + auto idx = rand() % (keys.size() - 250); + + uint64_t lower_key = keys[idx]; + uint64_t upper_key = keys[idx + 250]; + + rq::Parms p; + p.lower_bound = lower_key; + p.upper_bound = upper_key; + + auto result = test_de->query(&p); + auto r = result.get(); + std::sort(r.begin(), r.end()); + ck_assert_int_eq(r.size(), 251); + + for (size_t i=0; i> records; + std::set> to_delete; + std::set> deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t cnt=0; + for (auto rec : records) { + Rec r = {rec.first, rec.second}; + while (!test_de->insert(r)) { + sleep(1); + } + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector> del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(dr)) { + sleep(1); + } + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + test_de->await_next_epoch(); + + ck_assert(test_de->validate_tombstone_proportion()); + + gsl_rng_free(rng); + delete test_de; +} +END_TEST + +DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + auto test_de = new DE(1000, 10000, 2); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + for (auto rec : records) { + ck_assert_int_eq(test_de->insert(rec), 1); + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[i]); + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + gsl_rng_free(rng); + + return test_de; +} + +START_TEST(t_static_structure) +{ + auto rng = gsl_rng_alloc(gsl_rng_mt19937); + + size_t reccnt = 100000; + auto test_de = new DE(100, 1000, 2); + + std::set records; + std::set to_delete; + std::set deleted; + + while (records.size() < reccnt) { + uint64_t key = rand(); + uint32_t val = rand(); + + if (records.find({key, val}) != records.end()) continue; + + records.insert({key, val}); + } + + size_t deletes = 0; + size_t t_reccnt = 0; + size_t k=0; + for (auto rec : records) { + k++; + while (!test_de->insert(rec)) { + sleep(1); + } + t_reccnt++; + + if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { + std::vector del_vec; + std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); + + for (size_t i=0; ierase(del_vec[1])) { + sleep(1); + } + + deletes++; + to_delete.erase(del_vec[i]); + deleted.insert(del_vec[i]); + } + } + + if (gsl_rng_uniform(rng) < 0.25 && deleted.find(rec) == deleted.end()) { + to_delete.insert(rec); + } + } + + auto flat = test_de->create_static_structure(); + ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); + + uint64_t prev_key = 0; + for (size_t i=0; iget_record_count(); i++) { + auto k = flat->get_record_at(i)->rec.key; + ck_assert_int_ge(k, prev_key); + prev_key = k; + } + + gsl_rng_free(rng); + delete flat; + delete test_de; +} +END_TEST + + +static void inject_dynamic_extension_tests(Suite *suite) { + TCase *create = tcase_create("de::DynamicExtension::constructor Testing"); + tcase_add_test(create, t_create); + suite_add_tcase(suite, create); + + TCase *insert = tcase_create("de::DynamicExtension::insert Testing"); + tcase_add_test(insert, t_insert); + tcase_add_test(insert, t_insert_with_mem_merges); + tcase_add_test(insert, t_debug_insert); + tcase_set_timeout(insert, 500); + suite_add_tcase(suite, insert); + + /* + TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); + tcase_add_test(query, t_range_query); + suite_add_tcase(suite, query); + + + TCase *ts = tcase_create("de::DynamicExtension::tombstone_compaction Testing"); + tcase_add_test(ts, t_tombstone_merging_01); + tcase_set_timeout(ts, 500); + suite_add_tcase(suite, ts); + + TCase *flat = tcase_create("de::DynamicExtension::create_static_structure Testing"); + tcase_add_test(flat, t_static_structure); + tcase_set_timeout(flat, 500); + suite_add_tcase(suite, flat); + */ +} -- cgit v1.2.3 From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 17 Jan 2024 18:22:00 -0500 Subject: Multiple concurrency bug fixes A poorly organized commit with fixes for a variety of bugs that were causing missing records. The core problems all appear to be fixed, though there is an outstanding problem with tombstones not being completely canceled. A very small number are appearing in the wrong order during the static structure test. --- tests/include/concurrent_extension.h | 54 +++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 19 deletions(-) (limited to 'tests/include/concurrent_extension.h') diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h index 86f8e12..a0e71c9 100644 --- a/tests/include/concurrent_extension.h +++ b/tests/include/concurrent_extension.h @@ -28,8 +28,9 @@ #include "shard/ISAMTree.h" #include "query/rangequery.h" #include -using namespace de; -typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; + +//using namespace de; +//typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; START_TEST(t_create) @@ -75,7 +76,7 @@ START_TEST(t_debug_insert) for (size_t i=0; i<1000; i++) { Rec r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); - //ck_assert_int_eq(test_de->get_record_count(), i+1); + ck_assert_int_eq(test_de->get_record_count(), i+1); key++; val++; } @@ -115,14 +116,15 @@ START_TEST(t_insert_with_mem_merges) r.key++; r.value++; cnt++; + ck_assert_int_eq(test_de->get_record_count(), cnt + 1000); } else { - sleep(1); + _mm_pause(); } - } while (cnt < 10000); + } while (cnt < 100000); test_de->await_next_epoch(); - ck_assert_int_eq(test_de->get_record_count(), 11000); + ck_assert_int_eq(test_de->get_record_count(), 101000); delete test_de; } @@ -131,12 +133,12 @@ END_TEST START_TEST(t_range_query) { - auto test_de = new DE(100, 1000, 2); - size_t n = 10000; + auto test_de = new DE(1000, 10000, 4); + size_t n = 10000000; std::vector keys; for (size_t i=0; iinsert(r)) { i++; } else { - sleep(1); + _mm_pause(); } } + test_de->await_next_epoch(); std::sort(keys.begin(), keys.end()); @@ -166,9 +169,12 @@ START_TEST(t_range_query) p.lower_bound = lower_key; p.upper_bound = upper_key; + fprintf(stderr, "query start\n"); auto result = test_de->query(&p); auto r = result.get(); + fprintf(stderr, "query stop\n"); std::sort(r.begin(), r.end()); + ck_assert_int_eq(r.size(), 251); for (size_t i=0; iinsert(r)) { - sleep(1); + _mm_pause(); } if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { @@ -215,7 +221,7 @@ START_TEST(t_tombstone_merging_01) for (size_t i=0; ierase(dr)) { - sleep(1); + _mm_pause(); } deletes++; to_delete.erase(del_vec[i]); @@ -307,7 +313,7 @@ START_TEST(t_static_structure) for (auto rec : records) { k++; while (!test_de->insert(rec)) { - sleep(1); + _mm_pause(); } t_reccnt++; @@ -316,8 +322,8 @@ START_TEST(t_static_structure) std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; ierase(del_vec[1])) { - sleep(1); + while (!test_de->erase(del_vec[i])) { + _mm_pause(); } deletes++; @@ -331,12 +337,23 @@ START_TEST(t_static_structure) } } - auto flat = test_de->create_static_structure(); - ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); + + //fprintf(stderr, "Tombstones: %ld\tRecords: %ld\n", test_de->get_tombstone_count(), test_de->get_record_count()); + //fprintf(stderr, "Inserts: %ld\tDeletes:%ld\tNet:%ld\n", reccnt, deletes, reccnt - deletes); + + auto flat = test_de->create_static_structure(true); + //fprintf(stderr, "Flat: Tombstones: %ld\tRecords %ld\n", flat->get_tombstone_count(), flat->get_record_count()); + //ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); uint64_t prev_key = 0; for (size_t i=0; iget_record_count(); i++) { auto k = flat->get_record_at(i)->rec.key; + if (flat->get_record_at(i)->is_tombstone()) { + fprintf(stderr, "%ld %ld %ld\n", flat->get_record_at(i-1)->rec.key, + flat->get_record_at(i)->rec.key, + flat->get_record_at(i+1)->rec.key); + } + // ck_assert(!flat->get_record_at(i)->is_tombstone()); ck_assert_int_ge(k, prev_key); prev_key = k; } @@ -360,9 +377,9 @@ static void inject_dynamic_extension_tests(Suite *suite) { tcase_set_timeout(insert, 500); suite_add_tcase(suite, insert); - /* TCase *query = tcase_create("de::DynamicExtension::range_query Testing"); tcase_add_test(query, t_range_query); + tcase_set_timeout(query, 500); suite_add_tcase(suite, query); @@ -375,5 +392,4 @@ static void inject_dynamic_extension_tests(Suite *suite) { tcase_add_test(flat, t_static_structure); tcase_set_timeout(flat, 500); suite_add_tcase(suite, flat); - */ } -- cgit v1.2.3 From 10b4425e842d10b7fbfa85978969ed4591d6b98e Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 7 Feb 2024 10:56:52 -0500 Subject: Fully implemented Query concept and adjusted queries to use it --- tests/include/concurrent_extension.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'tests/include/concurrent_extension.h') diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h index a0e71c9..24cb2ce 100644 --- a/tests/include/concurrent_extension.h +++ b/tests/include/concurrent_extension.h @@ -22,7 +22,7 @@ * should be included in the source file that includes this one, above the * include statement. */ -#include "testing.h" +/*#include "testing.h" #include "framework/DynamicExtension.h" #include "framework/scheduling/FIFOScheduler.h" #include "shard/ISAMTree.h" @@ -31,6 +31,7 @@ //using namespace de; //typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +*/ START_TEST(t_create) @@ -169,10 +170,10 @@ START_TEST(t_range_query) p.lower_bound = lower_key; p.upper_bound = upper_key; - fprintf(stderr, "query start\n"); + //fprintf(stderr, "query start\n"); auto result = test_de->query(&p); auto r = result.get(); - fprintf(stderr, "query stop\n"); + //fprintf(stderr, "query stop\n"); std::sort(r.begin(), r.end()); ck_assert_int_eq(r.size(), 251); -- cgit v1.2.3 From bd74e27b28bd95267ce50d2e4b6f12b51d9b6aae Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 7 Feb 2024 17:23:23 -0500 Subject: Cleaned up shard files (except VPTree) Cleaned up shard implementations, fixed a few bugs, and set up some tests. There's still some work to be done in creating tests for the weighted sampling operations for the alias and aug btree shards. --- tests/include/concurrent_extension.h | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'tests/include/concurrent_extension.h') diff --git a/tests/include/concurrent_extension.h b/tests/include/concurrent_extension.h index 24cb2ce..0993fac 100644 --- a/tests/include/concurrent_extension.h +++ b/tests/include/concurrent_extension.h @@ -8,8 +8,8 @@ * Distributed under the Modified BSD License. * * WARNING: This file must be included in the main unit test set - * after the definition of an appropriate Shard, Query, and Rec - * type. In particular, Rec needs to implement the key-value + * after the definition of an appropriate Shard, Query, and R + * type. In particular, R needs to implement the key-value * pair interface. For other types of record, you'll need to * use a different set of unit tests. */ @@ -30,7 +30,7 @@ #include //using namespace de; -//typedef DynamicExtension, rq::Query, Rec>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; +//typedef DynamicExtension, rq::Query, R>, LayoutPolicy::LEVELING, DeletePolicy::TOMBSTONE, FIFOScheduler> DE; */ @@ -54,7 +54,7 @@ START_TEST(t_insert) uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<100; i++) { - Rec r = {key, val}; + R r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); key++; val++; @@ -75,7 +75,7 @@ START_TEST(t_debug_insert) uint64_t key = 0; uint32_t val = 0; for (size_t i=0; i<1000; i++) { - Rec r = {key, val}; + R r = {key, val}; ck_assert_int_eq(test_de->insert(r), 1); ck_assert_int_eq(test_de->get_record_count(), i+1); key++; @@ -94,7 +94,7 @@ START_TEST(t_insert_with_mem_merges) uint64_t key = 0; uint32_t val = 0; - Rec r = {key, val}; + R r = {key, val}; for (size_t i=0; i<1000; i++) { ck_assert_int_eq(test_de->insert(r), 1); r.key++; @@ -148,7 +148,7 @@ START_TEST(t_range_query) size_t i=0; while ( i < keys.size()) { - Rec r = {keys[i], (uint32_t) i}; + R r = {keys[i], (uint32_t) i}; if (test_de->insert(r)) { i++; } else { @@ -166,7 +166,7 @@ START_TEST(t_range_query) uint64_t lower_key = keys[idx]; uint64_t upper_key = keys[idx + 250]; - rq::Parms p; + rq::Parms p; p.lower_bound = lower_key; p.upper_bound = upper_key; @@ -210,7 +210,7 @@ START_TEST(t_tombstone_merging_01) size_t deletes = 0; size_t cnt=0; for (auto rec : records) { - Rec r = {rec.first, rec.second}; + R r = {rec.first, rec.second}; while (!test_de->insert(r)) { _mm_pause(); } @@ -220,7 +220,7 @@ START_TEST(t_tombstone_merging_01) std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; ierase(dr)) { _mm_pause(); } @@ -249,9 +249,9 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { auto test_de = new DE(1000, 10000, 2); - std::set records; - std::set to_delete; - std::set deleted; + std::set records; + std::set to_delete; + std::set deleted; while (records.size() < reccnt) { uint64_t key = rand(); @@ -267,7 +267,7 @@ DE *create_test_tree(size_t reccnt, size_t memlevel_cnt) { ck_assert_int_eq(test_de->insert(rec), 1); if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; + std::vector del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; i records; - std::set to_delete; - std::set deleted; + std::set records; + std::set to_delete; + std::set deleted; while (records.size() < reccnt) { uint64_t key = rand(); @@ -319,7 +319,7 @@ START_TEST(t_static_structure) t_reccnt++; if (gsl_rng_uniform(rng) < 0.05 && !to_delete.empty()) { - std::vector del_vec; + std::vector del_vec; std::sample(to_delete.begin(), to_delete.end(), std::back_inserter(del_vec), 3, std::mt19937{std::random_device{}()}); for (size_t i=0; iget_tombstone_count(), test_de->get_record_count()); + //fprintf(stderr, "Tombstones: %ld\tRords: %ld\n", test_de->get_tombstone_count(), test_de->get_record_count()); //fprintf(stderr, "Inserts: %ld\tDeletes:%ld\tNet:%ld\n", reccnt, deletes, reccnt - deletes); auto flat = test_de->create_static_structure(true); - //fprintf(stderr, "Flat: Tombstones: %ld\tRecords %ld\n", flat->get_tombstone_count(), flat->get_record_count()); + //fprintf(stderr, "Flat: Tombstones: %ld\tRords %ld\n", flat->get_tombstone_count(), flat->get_record_count()); //ck_assert_int_eq(flat->get_record_count(), reccnt - deletes); uint64_t prev_key = 0; -- cgit v1.2.3