From 3afacb7702e6d8fa67749a2a41dc776d315e02a9 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 23 Oct 2023 17:43:22 -0400 Subject: Began moving to an explicit epoch-based system I started moving over to an explicit Epoch based system, which has necessitated a ton of changes throughout the code base. This will ultimately allow for a much cleaner set of abstractions for managing concurrency. --- include/framework/structure/BufferView.h | 124 +++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 include/framework/structure/BufferView.h (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h new file mode 100644 index 0000000..1efc1ac --- /dev/null +++ b/include/framework/structure/BufferView.h @@ -0,0 +1,124 @@ +/* + * include/framework/structure/BufferView.h + * + * Copyright (C) 2023 Douglas Rumbaugh + * + * All rights reserved. Published under the Modified BSD License. + * + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "psu-util/alignment.h" +#include "util/bf_config.h" +#include "psu-ds/BloomFilter.h" +#include "psu-ds/Alias.h" +#include "psu-util/timer.h" +#include "framework/interface/Record.h" +#include "framework/structure/MutableBuffer.h" +#include "framework/interface/Query.h" + +namespace de { + +template +class BufferView { + typedef MutableBuffer Buffer; +public: + BufferView() = default; + + BufferView(std::vector buffers) + : m_buffers(buffers) + , m_cutoff(buffers[buffers->size()-1]->get_record_count()) + {} + + ~BufferView() = default; + + bool delete_record(const R& rec) { + auto res = false; + for (auto buf : m_buffers) { + res = buf->delete_record(rec); + if (res) return true; + } + return false; + } + + bool check_tombstone(const R& rec) { + auto res = false; + for (auto buf : m_buffers) { + res = buf->check_tombstone(rec); + if (res) return true; + } + return false; + } + + size_t get_record_count() { + size_t reccnt = 0; + for (auto buf : m_buffers) { + reccnt += buf->get_record_count(); + } + return reccnt; + } + + size_t get_capacity() { + return m_buffers[0]->get_capacity(); + } + + bool is_full() { + return m_buffers[m_buffers.size() - 1]->is_full(); + } + + size_t get_tombstone_count() { + size_t tscnt = 0; + for (auto buf : m_buffers) { + tscnt += buf->get_tombstone_count(); + } + return tscnt; + } + + size_t get_memory_usage() { + size_t mem = 0; + for (auto buf : m_buffers) { + mem += buf->get_memory_usage(); + } + return mem; + } + + size_t get_aux_memory_usage() { + size_t mem = 0; + for (auto buf : m_buffers) { + mem += buf->get_aux_memory_usage(); + } + return mem; + } + + size_t get_tombstone_capacity() { + return m_buffers[0]->get_tombstone_capacity(); + } + + std::vector get_buffer_states(void *parms) { + std::vector states; + + for (auto buf : m_buffers) { + states.push_back(Q::get_buffer_query_state(buf, parms)); + } + + return states; + } + + std::vector &get_buffers() { + return m_buffers; + } + +private: + std::vector m_buffers; + size_t m_cutoff; +}; + +} -- cgit v1.2.3 From 39ae3e0441d8297a09197aba98bd494b5ada12c1 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 30 Oct 2023 14:17:59 -0400 Subject: Concurrency updates + fixes for compile errors --- include/framework/structure/BufferView.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 1efc1ac..14abedc 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -35,7 +35,7 @@ public: BufferView(std::vector buffers) : m_buffers(buffers) - , m_cutoff(buffers[buffers->size()-1]->get_record_count()) + , m_cutoff(buffers[buffers.size()-1]->get_record_count()) {} ~BufferView() = default; @@ -102,7 +102,7 @@ public: return m_buffers[0]->get_tombstone_capacity(); } - std::vector get_buffer_states(void *parms) { + std::vector get_query_states(void *parms) { std::vector states; for (auto buf : m_buffers) { -- cgit v1.2.3 From ceffd8caf5e4e827e2cc4d6975507a66d88f77a9 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 30 Oct 2023 14:25:28 -0400 Subject: DynamicExtension: adjusted a few operations to ensure conistency get_memory_usage, get_aux_memory_usage, get_record_count, get_tombstone_count, and create_static_structure have been adjusted to ensure that they pull from a consistent epoch, even if a change-over occurs midway through the function. These functions also now register with the epoch as a job, to ensure that the epoch they are operating own isn't retired midway through the function. Probably not a big issue for the accessors, but I could see it being very important for create_static_structure. --- include/framework/structure/BufferView.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 14abedc..8dff2ef 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -116,6 +116,10 @@ public: return m_buffers; } + size_t size() { + return m_buffers.size(); + } + private: std::vector m_buffers; size_t m_cutoff; -- cgit v1.2.3 From d2279e1b96d352a0af1d425dcaaf93e8a26a8d52 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 30 Oct 2023 17:15:05 -0400 Subject: General Comment + Consistency updates --- include/framework/structure/BufferView.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 8dff2ef..ccd3dac 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -1,7 +1,7 @@ /* * include/framework/structure/BufferView.h * - * Copyright (C) 2023 Douglas Rumbaugh + * Copyright (C) 2023 Douglas B. Rumbaugh * * All rights reserved. Published under the Modified BSD License. * -- cgit v1.2.3 From 357cab549c2ed33970562b84ff6f83923742343d Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Tue, 7 Nov 2023 15:34:24 -0500 Subject: Comment and License updates --- include/framework/structure/BufferView.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index ccd3dac..651e430 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -3,7 +3,7 @@ * * Copyright (C) 2023 Douglas B. Rumbaugh * - * All rights reserved. Published under the Modified BSD License. + * Distributed under the Modified BSD License. * */ #pragma once -- cgit v1.2.3 From 3c2e6b3b456867d7155b158432b891b84e4e1dd6 Mon Sep 17 00:00:00 2001 From: "Douglas B. Rumbaugh" Date: Tue, 9 Jan 2024 14:47:48 -0500 Subject: Initial update of buffer to new specifications There are a few minor issues that this introduces, however. Global tracking of a lot of secondary information, such as weights for WIRS/WSS, or the exact number of tombstones will need to be approached differently than they have been historically with this new approach. I've also removed most of the tombstone capacity related code. We had decided not to bother enforcing this at the buffer level anyway, and it would greatly increase the complexity of the problem of predicting when the next compaction will be. On the whole this new approach seems like it'll simplify a lot. This commit actually removes significantly more code than it adds. One minor issue: the currently implementation will have problems in the circular array indexes once more than 2^64 records have been inserted. This doesn't seem like a realistic problem at the moment. --- include/framework/structure/BufferView.h | 98 ++++++++------------------------ 1 file changed, 25 insertions(+), 73 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 651e430..8a5f50f 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -22,107 +22,59 @@ #include "psu-ds/Alias.h" #include "psu-util/timer.h" #include "framework/interface/Record.h" -#include "framework/structure/MutableBuffer.h" #include "framework/interface/Query.h" namespace de { -template +template class BufferView { - typedef MutableBuffer Buffer; public: BufferView() = default; - BufferView(std::vector buffers) - : m_buffers(buffers) - , m_cutoff(buffers[buffers.size()-1]->get_record_count()) - {} + BufferView(const Wrapped *buffer, size_t head, size_t tail, psudb::BloomFilter *filter) + : m_buffer(buffer), m_head(head), m_tail(tail), m_tombstone_filter(filter) {} ~BufferView() = default; - bool delete_record(const R& rec) { - auto res = false; - for (auto buf : m_buffers) { - res = buf->delete_record(rec); - if (res) return true; - } - return false; - } - bool check_tombstone(const R& rec) { - auto res = false; - for (auto buf : m_buffers) { - res = buf->check_tombstone(rec); - if (res) return true; + if (m_tombstone_filter && !m_tombstone_filter->lookup(rec)) return false; + + for (size_t i=0; iget_record_count(); - } - return reccnt; + return m_tail - m_head; } - size_t get_capacity() { - return m_buffers[0]->get_capacity(); - } - - bool is_full() { - return m_buffers[m_buffers.size() - 1]->is_full(); - } - size_t get_tombstone_count() { - size_t tscnt = 0; - for (auto buf : m_buffers) { - tscnt += buf->get_tombstone_count(); - } - return tscnt; + // FIXME: tombstone count + return 0; } - size_t get_memory_usage() { - size_t mem = 0; - for (auto buf : m_buffers) { - mem += buf->get_memory_usage(); - } - return mem; + Wrapped *get(size_t i) { + assert(i < get_record_count()); + return m_buffer + to_idx(i); } - size_t get_aux_memory_usage() { - size_t mem = 0; - for (auto buf : m_buffers) { - mem += buf->get_aux_memory_usage(); - } - return mem; - } - - size_t get_tombstone_capacity() { - return m_buffers[0]->get_tombstone_capacity(); - } - - std::vector get_query_states(void *parms) { - std::vector states; - - for (auto buf : m_buffers) { - states.push_back(Q::get_buffer_query_state(buf, parms)); - } - - return states; + void copy_to_buffer(byte *buffer) { + memcpy(buffer, m_buffer, get_record_count() * sizeof(Wrapped)); } - std::vector &get_buffers() { - return m_buffers; - } +private: + const Wrapped* m_buffer; + size_t m_head; + size_t m_tail; + psudb::BloomFilter *m_tombstone_filter; - size_t size() { - return m_buffers.size(); + size_t to_idx(size_t i) { + return (m_head + i) % m_buffer->get_capacity(); } - -private: - std::vector m_buffers; - size_t m_cutoff; }; } -- cgit v1.2.3 From 53879a0d69f5e578710b7125e9b41e516c2371d4 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 10 Jan 2024 17:39:28 -0500 Subject: MutableBuffer+View: Implementation with unit tests --- include/framework/structure/BufferView.h | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 8a5f50f..7e8af45 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -9,32 +9,34 @@ #pragma once #include -#include -#include #include -#include -#include -#include +#include #include "psu-util/alignment.h" -#include "util/bf_config.h" #include "psu-ds/BloomFilter.h" -#include "psu-ds/Alias.h" -#include "psu-util/timer.h" #include "framework/interface/Record.h" -#include "framework/interface/Query.h" namespace de { +typedef std::function ReleaseFunction; + template class BufferView { public: BufferView() = default; - BufferView(const Wrapped *buffer, size_t head, size_t tail, psudb::BloomFilter *filter) - : m_buffer(buffer), m_head(head), m_tail(tail), m_tombstone_filter(filter) {} + BufferView(const Wrapped *buffer, size_t head, size_t tail, psudb::BloomFilter *filter, + void *parent_buffer, ReleaseFunction release) + : m_buffer(buffer) + , m_release(release) + , m_parent_buffer(parent_buffer) + , m_head(head) + , m_tail(tail) + , m_tombstone_filter(filter) {} - ~BufferView() = default; + ~BufferView() { + m_release(m_parent_buffer, m_head); + } bool check_tombstone(const R& rec) { if (m_tombstone_filter && !m_tombstone_filter->lookup(rec)) return false; @@ -62,12 +64,14 @@ public: return m_buffer + to_idx(i); } - void copy_to_buffer(byte *buffer) { - memcpy(buffer, m_buffer, get_record_count() * sizeof(Wrapped)); + void copy_to_buffer(psudb::byte *buffer) { + memcpy(buffer, (std::byte*) (m_buffer + m_head), get_record_count() * sizeof(Wrapped)); } private: const Wrapped* m_buffer; + void *m_parent_buffer; + ReleaseFunction m_release; size_t m_head; size_t m_tail; psudb::BloomFilter *m_tombstone_filter; -- cgit v1.2.3 From 5db0f96e9f3d2505b5f751abc133cbf7e13b5129 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 11 Jan 2024 11:31:33 -0500 Subject: Fixed some potential buffer-related concurrency bugs --- include/framework/structure/BufferView.h | 45 ++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 14 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 7e8af45..00b6101 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -18,31 +18,43 @@ namespace de { -typedef std::function ReleaseFunction; +typedef std::_Bind ReleaseFunction; template class BufferView { public: BufferView() = default; - BufferView(const Wrapped *buffer, size_t head, size_t tail, psudb::BloomFilter *filter, - void *parent_buffer, ReleaseFunction release) - : m_buffer(buffer) + BufferView(const Wrapped *buffer, size_t cap, size_t head, size_t tail, size_t tombstone_cnt, psudb::BloomFilter *filter, + ReleaseFunction release) + : m_data(buffer) , m_release(release) - , m_parent_buffer(parent_buffer) , m_head(head) , m_tail(tail) + , m_cap(cap) + , m_approx_ts_cnt(tombstone_cnt) , m_tombstone_filter(filter) {} ~BufferView() { - m_release(m_parent_buffer, m_head); + m_release(); } bool check_tombstone(const R& rec) { if (m_tombstone_filter && !m_tombstone_filter->lookup(rec)) return false; for (size_t i=0; i *get(size_t i) { assert(i < get_record_count()); - return m_buffer + to_idx(i); + return m_data + to_idx(i); } void copy_to_buffer(psudb::byte *buffer) { - memcpy(buffer, (std::byte*) (m_buffer + m_head), get_record_count() * sizeof(Wrapped)); + memcpy(buffer, (std::byte*) (m_data + m_head), get_record_count() * sizeof(Wrapped)); } private: - const Wrapped* m_buffer; - void *m_parent_buffer; + const Wrapped* m_data; ReleaseFunction m_release; size_t m_head; size_t m_tail; + size_t m_cap; + size_t m_approx_ts_cnt; psudb::BloomFilter *m_tombstone_filter; size_t to_idx(size_t i) { - return (m_head + i) % m_buffer->get_capacity(); + return (m_head + i) % m_cap; } }; -- cgit v1.2.3 From c596ed468c2279f959b04d83d7f2e9692db84bae Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 11 Jan 2024 15:28:00 -0500 Subject: BufferView: enforce move semantics Because a BufferView's lifetime is so tightly linked to the lifetime of regions of the buffer, it can't be copied without potentially breaking things. --- include/framework/structure/BufferView.h | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 00b6101..98e41dd 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -25,7 +25,24 @@ class BufferView { public: BufferView() = default; - BufferView(const Wrapped *buffer, size_t cap, size_t head, size_t tail, size_t tombstone_cnt, psudb::BloomFilter *filter, + /* + * the BufferView's lifetime is tightly linked to buffer versioning, and so + * copying and assignment are disabled. + */ + BufferView(const BufferView&) = delete; + BufferView &operator=(BufferView &) = delete; + + void operator=(BufferView &&src) { + m_data = src.m_data; + m_release = src.m_release; + m_head = src.m_head; + m_tail = src.m_tail; + m_cap = src.m_cap; + m_approx_ts_cnt = src.m_approx_ts_cnt; + m_tombstone_filter = src.filter; + } + + BufferView(Wrapped *buffer, size_t cap, size_t head, size_t tail, size_t tombstone_cnt, psudb::BloomFilter *filter, ReleaseFunction release) : m_data(buffer) , m_release(release) @@ -85,7 +102,7 @@ public: } private: - const Wrapped* m_data; + Wrapped* m_data; ReleaseFunction m_release; size_t m_head; size_t m_tail; -- cgit v1.2.3 From 7e503464176adbd0880373325e30a6bfd58616f0 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 11 Jan 2024 16:31:24 -0500 Subject: InternalLevel update and tests Plus some assorted fixes for move semantics stuff in BufferView that accompanied these changes. --- include/framework/structure/BufferView.h | 44 ++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 10 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 98e41dd..d058714 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -18,12 +18,25 @@ namespace de { -typedef std::_Bind ReleaseFunction; +typedef std::_Bind ReleaseFunction; + +static void noop_func(void *arg1, size_t arg2) { + return; +} + +constexpr auto noop_bind = std::bind(noop_func, (void*) nullptr, 0ul); template class BufferView { public: - BufferView() = default; + BufferView() + : m_data(nullptr) + , m_release(noop_bind) + , m_head(0) + , m_tail(0) + , m_cap(0) + , m_approx_ts_cnt(0) + , m_tombstone_filter(nullptr) {} /* * the BufferView's lifetime is tightly linked to buffer versioning, and so @@ -32,14 +45,25 @@ public: BufferView(const BufferView&) = delete; BufferView &operator=(BufferView &) = delete; - void operator=(BufferView &&src) { - m_data = src.m_data; - m_release = src.m_release; - m_head = src.m_head; - m_tail = src.m_tail; - m_cap = src.m_cap; - m_approx_ts_cnt = src.m_approx_ts_cnt; - m_tombstone_filter = src.filter; + BufferView(BufferView &&other) + : m_data(std::exchange(other.m_data, nullptr)) + , m_release(std::move(other.m_release)) + , m_head(std::exchange(other.m_head, 0)) + , m_tail(std::exchange(other.m_tail, 0)) + , m_cap(std::exchange(other.m_cap, 0)) + , m_approx_ts_cnt(std::exchange(other.m_approx_ts_cnt, 0)) + , m_tombstone_filter(std::exchange(other.m_tombstone_filter, nullptr)) {} + + BufferView &operator=(BufferView &&other) noexcept { + std::swap(m_data, other.m_data); + m_release = std::move(other.m_release); + std::swap(m_head, other.m_head); + std::swap(m_tail, other.m_tail); + std::swap(m_cap, other.m_cap); + std::swap(m_approx_ts_cnt, other.m_approx_ts_cnt); + std::swap(m_tombstone_filter, other.m_tombstone_filter); + + return *this; } BufferView(Wrapped *buffer, size_t cap, size_t head, size_t tail, size_t tombstone_cnt, psudb::BloomFilter *filter, -- cgit v1.2.3 From 3a89d7f6ea2679ff7b9bb1e3c37da9480be6c115 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 12 Jan 2024 11:29:37 -0500 Subject: BufferView.h: Hopefully the last necessary tweak to the move semantics stuff You can't move assign an std::Bind, but you can move construct it. So I had to disable the move assignment operator. This means that when you change the BufferView ownership over to, say, a QueryBufferState object, you need to do it by passing std::move(buffview) into a constructor call only--you cannot assign it. --- include/framework/structure/BufferView.h | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index d058714..47c7b9b 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -20,23 +20,10 @@ namespace de { typedef std::_Bind ReleaseFunction; -static void noop_func(void *arg1, size_t arg2) { - return; -} - -constexpr auto noop_bind = std::bind(noop_func, (void*) nullptr, 0ul); - template class BufferView { public: - BufferView() - : m_data(nullptr) - , m_release(noop_bind) - , m_head(0) - , m_tail(0) - , m_cap(0) - , m_approx_ts_cnt(0) - , m_tombstone_filter(nullptr) {} + BufferView() = default; /* * the BufferView's lifetime is tightly linked to buffer versioning, and so @@ -54,17 +41,8 @@ public: , m_approx_ts_cnt(std::exchange(other.m_approx_ts_cnt, 0)) , m_tombstone_filter(std::exchange(other.m_tombstone_filter, nullptr)) {} - BufferView &operator=(BufferView &&other) noexcept { - std::swap(m_data, other.m_data); - m_release = std::move(other.m_release); - std::swap(m_head, other.m_head); - std::swap(m_tail, other.m_tail); - std::swap(m_cap, other.m_cap); - std::swap(m_approx_ts_cnt, other.m_approx_ts_cnt); - std::swap(m_tombstone_filter, other.m_tombstone_filter); + BufferView &operator=(BufferView &&other) = delete; - return *this; - } BufferView(Wrapped *buffer, size_t cap, size_t head, size_t tail, size_t tombstone_cnt, psudb::BloomFilter *filter, ReleaseFunction release) -- cgit v1.2.3 From aac0bb661af8fae38d3ce08d6078cb4d9dfcb575 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 12 Jan 2024 14:10:11 -0500 Subject: Initial integration of new buffering scheme into framework It isn't working right now (lotsa test failures), but we're to the debugging phase now. --- include/framework/structure/BufferView.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 47c7b9b..ba5e693 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -103,6 +103,10 @@ public: memcpy(buffer, (std::byte*) (m_data + m_head), get_record_count() * sizeof(Wrapped)); } + size_t get_tail() { + return m_tail; + } + private: Wrapped* m_data; ReleaseFunction m_release; -- cgit v1.2.3 From cf178ae74a76b780b655a447531d2114f9f81d98 Mon Sep 17 00:00:00 2001 From: "Douglas B. Rumbaugh" Date: Mon, 15 Jan 2024 14:01:36 -0500 Subject: Various single-threaded bug fixes --- include/framework/structure/BufferView.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index ba5e693..c751786 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -11,6 +11,7 @@ #include #include #include +#include #include "psu-util/alignment.h" #include "psu-ds/BloomFilter.h" @@ -39,7 +40,8 @@ public: , m_tail(std::exchange(other.m_tail, 0)) , m_cap(std::exchange(other.m_cap, 0)) , m_approx_ts_cnt(std::exchange(other.m_approx_ts_cnt, 0)) - , m_tombstone_filter(std::exchange(other.m_tombstone_filter, nullptr)) {} + , m_tombstone_filter(std::exchange(other.m_tombstone_filter, nullptr)) + , m_active(std::exchange(other.m_active, false)) {} BufferView &operator=(BufferView &&other) = delete; @@ -52,10 +54,13 @@ public: , m_tail(tail) , m_cap(cap) , m_approx_ts_cnt(tombstone_cnt) - , m_tombstone_filter(filter) {} + , m_tombstone_filter(filter) + , m_active(true) {} ~BufferView() { - m_release(); + if (m_active) { + m_release(); + } } bool check_tombstone(const R& rec) { @@ -100,13 +105,17 @@ public: } void copy_to_buffer(psudb::byte *buffer) { - memcpy(buffer, (std::byte*) (m_data + m_head), get_record_count() * sizeof(Wrapped)); + memcpy(buffer, (std::byte*) (m_data + (m_head % m_cap)), get_record_count() * sizeof(Wrapped)); } size_t get_tail() { return m_tail; } + size_t get_head() { + return m_head; + } + private: Wrapped* m_data; ReleaseFunction m_release; @@ -115,6 +124,7 @@ private: size_t m_cap; size_t m_approx_ts_cnt; psudb::BloomFilter *m_tombstone_filter; + bool m_active; size_t to_idx(size_t i) { return (m_head + i) % m_cap; -- cgit v1.2.3 From 2117935e85412f3733ee0bcb1830c7fd0b129b29 Mon Sep 17 00:00:00 2001 From: "Douglas B. Rumbaugh" Date: Mon, 15 Jan 2024 17:23:57 -0500 Subject: Concurrency testing and bug fixes --- include/framework/structure/BufferView.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index c751786..099b7a2 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -105,7 +105,15 @@ public: } void copy_to_buffer(psudb::byte *buffer) { - memcpy(buffer, (std::byte*) (m_data + (m_head % m_cap)), get_record_count() * sizeof(Wrapped)); + /* check if the region to be copied circles back to start. If so, do it in two steps */ + if ((m_head % m_cap) + get_record_count() > m_cap) { + size_t split_idx = m_cap - (m_head % m_cap); + + memcpy(buffer, (std::byte*) (m_data + (m_head % m_cap)), split_idx* sizeof(Wrapped)); + memcpy(buffer + split_idx, (std::byte*) m_data, (get_record_count() - split_idx) * sizeof(Wrapped)); + } else { + memcpy(buffer, (std::byte*) (m_data + (m_head % m_cap)), get_record_count() * sizeof(Wrapped)); + } } size_t get_tail() { -- cgit v1.2.3 From 138c793b0a58577713d98c98bb140cf1d9c79bee Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Wed, 17 Jan 2024 18:22:00 -0500 Subject: Multiple concurrency bug fixes A poorly organized commit with fixes for a variety of bugs that were causing missing records. The core problems all appear to be fixed, though there is an outstanding problem with tombstones not being completely canceled. A very small number are appearing in the wrong order during the static structure test. --- include/framework/structure/BufferView.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 099b7a2..30fffed 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -110,7 +110,7 @@ public: size_t split_idx = m_cap - (m_head % m_cap); memcpy(buffer, (std::byte*) (m_data + (m_head % m_cap)), split_idx* sizeof(Wrapped)); - memcpy(buffer + split_idx, (std::byte*) m_data, (get_record_count() - split_idx) * sizeof(Wrapped)); + memcpy(buffer + (split_idx * sizeof(Wrapped)), (std::byte*) m_data, (get_record_count() - split_idx) * sizeof(Wrapped)); } else { memcpy(buffer, (std::byte*) (m_data + (m_head % m_cap)), get_record_count() * sizeof(Wrapped)); } -- cgit v1.2.3 From 0ff3cedf5df9c27bccd3053ce6339e317f87ff76 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Mon, 5 Feb 2024 15:18:33 -0500 Subject: BufferView: Adjusted BV to avoid repeated modulus operations --- include/framework/structure/BufferView.h | 50 +++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 10 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 30fffed..edf6707 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -38,6 +38,8 @@ public: , m_release(std::move(other.m_release)) , m_head(std::exchange(other.m_head, 0)) , m_tail(std::exchange(other.m_tail, 0)) + , m_start(std::exchange(other.m_start, 0)) + , m_stop(std::exchange(other.m_stop, 0)) , m_cap(std::exchange(other.m_cap, 0)) , m_approx_ts_cnt(std::exchange(other.m_approx_ts_cnt, 0)) , m_tombstone_filter(std::exchange(other.m_tombstone_filter, nullptr)) @@ -52,6 +54,8 @@ public: , m_release(release) , m_head(head) , m_tail(tail) + , m_start(m_head % cap) + , m_stop(m_tail % cap) , m_cap(cap) , m_approx_ts_cnt(tombstone_cnt) , m_tombstone_filter(filter) @@ -76,11 +80,29 @@ public: } bool delete_record(const R& rec) { - for (size_t i=0; i *get(size_t i) { assert(i < get_record_count()); + m_total += (m_data + to_idx(i))->rec.key; return m_data + to_idx(i); } void copy_to_buffer(psudb::byte *buffer) { /* check if the region to be copied circles back to start. If so, do it in two steps */ - if ((m_head % m_cap) + get_record_count() > m_cap) { - size_t split_idx = m_cap - (m_head % m_cap); + if (m_start > m_stop) { + size_t split_idx = m_cap - m_start; - memcpy(buffer, (std::byte*) (m_data + (m_head % m_cap)), split_idx* sizeof(Wrapped)); - memcpy(buffer + (split_idx * sizeof(Wrapped)), (std::byte*) m_data, (get_record_count() - split_idx) * sizeof(Wrapped)); + memcpy(buffer, (std::byte*) (m_data + m_start), split_idx* sizeof(Wrapped)); + memcpy(buffer + (split_idx * sizeof(Wrapped)), (std::byte*) m_data, m_stop * sizeof(Wrapped)); } else { - memcpy(buffer, (std::byte*) (m_data + (m_head % m_cap)), get_record_count() * sizeof(Wrapped)); + memcpy(buffer, (std::byte*) (m_data + m_start), get_record_count() * sizeof(Wrapped)); } } @@ -129,13 +152,20 @@ private: ReleaseFunction m_release; size_t m_head; size_t m_tail; + size_t m_start; + size_t m_stop; size_t m_cap; size_t m_approx_ts_cnt; psudb::BloomFilter *m_tombstone_filter; bool m_active; + size_t m_total; + size_t to_idx(size_t i) { - return (m_head + i) % m_cap; + size_t idx = (m_start + i >= m_cap) ? i = (m_cap - m_start) + : m_start + i; + assert(idx < m_cap); + return idx; } }; -- cgit v1.2.3 From 711769574e647839677739192698e400529efe75 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Thu, 8 Feb 2024 16:38:44 -0500 Subject: Updated VPTree to new shard/query interfaces --- include/framework/structure/BufferView.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index edf6707..4e3de25 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -123,7 +123,6 @@ public: Wrapped *get(size_t i) { assert(i < get_record_count()); - m_total += (m_data + to_idx(i))->rec.key; return m_data + to_idx(i); } @@ -159,8 +158,6 @@ private: psudb::BloomFilter *m_tombstone_filter; bool m_active; - size_t m_total; - size_t to_idx(size_t i) { size_t idx = (m_start + i >= m_cap) ? i = (m_cap - m_start) : m_start + i; -- cgit v1.2.3 From 402fc269c0aaa671d84a6d15918735ad4b90e6b2 Mon Sep 17 00:00:00 2001 From: Douglas Rumbaugh Date: Fri, 9 Feb 2024 12:30:21 -0500 Subject: Comment updates/fixes --- include/framework/structure/BufferView.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/framework/structure/BufferView.h') diff --git a/include/framework/structure/BufferView.h b/include/framework/structure/BufferView.h index 4e3de25..9e0872b 100644 --- a/include/framework/structure/BufferView.h +++ b/include/framework/structure/BufferView.h @@ -5,6 +5,7 @@ * * Distributed under the Modified BSD License. * + * TODO: This file is very poorly commented. */ #pragma once -- cgit v1.2.3