From 7c03d771475421c1d5a2bbc135242536af1a371c Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 25 Sep 2023 10:49:36 -0400 Subject: Re-structuring Project + scheduling updates This is a big one--probably should have split it apart, but I'm feeling lazy this morning. * Organized the mess of header files in include/framework by splitting them out into their own subdirectories, and renaming a few files to remove redundancies introduced by the directory structure. * Introduced a new framework/ShardRequirements.h header file for simpler shard development. This header simply contains the necessary includes from framework/* for creating shard files. This should help to remove structural dependencies from the framework file structure and shards, as well as centralizing the necessary framework files to make shard development easier. * Created a (currently dummy) SchedulerInterface, and make the scheduler implementation a template parameter of the dynamic extension for easier testing of various scheduling policies. There's still more work to be done to fully integrate the scheduler (queries, multiple buffers), but some more of the necessary framework code for this has been added as well. * Adjusted the Task interface setup for the scheduler. The task structures have been removed from ExtensionStructure and placed in their own header file. Additionally, I started experimenting with using std::variant, as opposed to inheritence, to implement subtype polymorphism on the Merge and Query tasks. The scheduler now has a general task queue that contains both, and std::variant, std::visit, and std::get are used to manipulate them without virtual functions. * Removed Alex.h, as it can't build anyway. There's a branch out there containing the Alex implementation stripped of the C++20 stuff. So there's no need to keep it here. --- include/framework/interface/Shard.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 include/framework/interface/Shard.h (limited to 'include/framework/interface/Shard.h') diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h new file mode 100644 index 0000000..ea58b2a --- /dev/null +++ b/include/framework/interface/Shard.h @@ -0,0 +1,26 @@ +/* + * include/framework/ShardInterface.h + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * All rights reserved. Published under the Modified BSD License. + * + */ +#pragma once + +#include + +#include "util/types.h" +#include "framework/interface/Record.h" + +namespace de { + +//template typename S, typename R> +template +concept ShardInterface = requires(S s, void *p, bool b) { + //{s.point_lookup(r, b) } -> std::same_as; + {s.get_record_count()} -> std::convertible_to; + {s.get_memory_usage()} -> std::convertible_to; +}; + +} -- cgit v1.2.3 From d2279e1b96d352a0af1d425dcaaf93e8a26a8d52 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 30 Oct 2023 17:15:05 -0400 Subject: General Comment + Consistency updates --- include/framework/interface/Shard.h | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'include/framework/interface/Shard.h') diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h index ea58b2a..d3a6cf8 100644 --- a/include/framework/interface/Shard.h +++ b/include/framework/interface/Shard.h @@ -1,7 +1,7 @@ /* - * include/framework/ShardInterface.h + * include/framework/interface/Shard.h * - * Copyright (C) 2023 Douglas Rumbaugh + * Copyright (C) 2023 Douglas B. Rumbaugh * * All rights reserved. Published under the Modified BSD License. * @@ -15,12 +15,22 @@ namespace de { -//template typename S, typename R> +// FIXME: The interface is not completely specified yet, as it is pending +// determining a good way to handle additional template arguments +// to get the Record type into play template -concept ShardInterface = requires(S s, void *p, bool b) { - //{s.point_lookup(r, b) } -> std::same_as; +concept ShardInterface = requires(S s, S **spp, void *p, bool b, size_t i) { + {S(spp, i)}; + /* + {S(mutable buffer)} + {s.point_lookup(r, b) } -> std::convertible_to + */ + {s.get_data()} -> std::convertible_to; + {s.get_record_count()} -> std::convertible_to; + {s.get_tombstone_count()} -> std::convertible_to; {s.get_memory_usage()} -> std::convertible_to; + {s.get_aux_memory_usage()} -> std::convertible_to; }; } -- cgit v1.2.3 From 0b723322a611de83872dd83b55d2e10e8886a283 Mon Sep 17 00:00:00 2001 From: "Douglas B. Rumbaugh" Date: Thu, 2 Nov 2023 08:01:45 -0400 Subject: started refactoring queries interface --- include/framework/interface/Shard.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/framework/interface/Shard.h') diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h index d3a6cf8..40a696b 100644 --- a/include/framework/interface/Shard.h +++ b/include/framework/interface/Shard.h @@ -33,4 +33,10 @@ concept ShardInterface = requires(S s, S **spp, void *p, bool b, size_t i) { {s.get_aux_memory_usage()} -> std::convertible_to; }; +template +concept SortedShardInterface = ShardInterface && requires(S s, R r, R *rp) { + {s.lower_bound(r)} -> std::convertible_to; + {s.upper_bound(r)} -> std::convertible_to; +} + } -- cgit v1.2.3 From e02742b07540dd5a9bcbb44dae14856bf10955ed Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 6 Nov 2023 15:18:53 -0500 Subject: Refactoring progress --- include/framework/interface/Shard.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/framework/interface/Shard.h') diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h index 40a696b..92cdca0 100644 --- a/include/framework/interface/Shard.h +++ b/include/framework/interface/Shard.h @@ -37,6 +37,6 @@ template concept SortedShardInterface = ShardInterface && requires(S s, R r, R *rp) { {s.lower_bound(r)} -> std::convertible_to; {s.upper_bound(r)} -> std::convertible_to; -} +}; } -- cgit v1.2.3 From 357cab549c2ed33970562b84ff6f83923742343d Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 7 Nov 2023 15:34:24 -0500 Subject: Comment and License updates --- include/framework/interface/Shard.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/framework/interface/Shard.h') diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h index 92cdca0..2357795 100644 --- a/include/framework/interface/Shard.h +++ b/include/framework/interface/Shard.h @@ -3,7 +3,7 @@ * * Copyright (C) 2023 Douglas B. Rumbaugh * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #pragma once -- cgit v1.2.3 From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 17 Jan 2024 18:22:00 -0500 Subject: Multiple concurrency bug fixes A poorly organized commit with fixes for a variety of bugs that were causing missing records. The core problems all appear to be fixed, though there is an outstanding problem with tombstones not being completely canceled. A very small number are appearing in the wrong order during the static structure test. --- include/framework/interface/Shard.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/framework/interface/Shard.h') diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h index 2357795..8c4db34 100644 --- a/include/framework/interface/Shard.h +++ b/include/framework/interface/Shard.h @@ -12,6 +12,7 @@ #include "util/types.h" #include "framework/interface/Record.h" +#include namespace de { @@ -19,8 +20,8 @@ namespace de { // determining a good way to handle additional template arguments // to get the Record type into play template -concept ShardInterface = requires(S s, S **spp, void *p, bool b, size_t i) { - {S(spp, i)}; +concept ShardInterface = requires(S s, std::vector spp, void *p, bool b, size_t i) { + {S(spp)}; /* {S(mutable buffer)} {s.point_lookup(r, b) } -> std::convertible_to -- cgit v1.2.3 From 2c5d549b3618b9ea72e6eece4cb4f3da5a6811a8 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 7 Feb 2024 13:42:34 -0500 Subject: Fully realized shard concept interface --- include/framework/interface/Shard.h | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) (limited to 'include/framework/interface/Shard.h') diff --git a/include/framework/interface/Shard.h b/include/framework/interface/Shard.h index 8c4db34..c4a9180 100644 --- a/include/framework/interface/Shard.h +++ b/include/framework/interface/Shard.h @@ -8,25 +8,17 @@ */ #pragma once -#include - -#include "util/types.h" -#include "framework/interface/Record.h" -#include +#include "framework/ShardRequirements.h" namespace de { -// FIXME: The interface is not completely specified yet, as it is pending -// determining a good way to handle additional template arguments -// to get the Record type into play -template -concept ShardInterface = requires(S s, std::vector spp, void *p, bool b, size_t i) { +template +concept ShardInterface = RecordInterface && requires(S s, std::vector spp, void *p, bool b, size_t i, BufferView bv, R r) { {S(spp)}; - /* - {S(mutable buffer)} - {s.point_lookup(r, b) } -> std::convertible_to - */ - {s.get_data()} -> std::convertible_to; + {S(std::move(bv))}; + + {s.point_lookup(r, b) } -> std::same_as*>; + {s.get_data()} -> std::same_as*>; {s.get_record_count()} -> std::convertible_to; {s.get_tombstone_count()} -> std::convertible_to; @@ -35,9 +27,10 @@ concept ShardInterface = requires(S s, std::vector spp, void *p, bool b, siz }; template -concept SortedShardInterface = ShardInterface && requires(S s, R r, R *rp) { +concept SortedShardInterface = ShardInterface && requires(S s, R r, R *rp, size_t i) { {s.lower_bound(r)} -> std::convertible_to; {s.upper_bound(r)} -> std::convertible_to; + {s.get_record_at(i)} -> std::same_as*>; }; } -- cgit v1.2.3