1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
|
/*
* include/framework/structure/InternalLevel.h
*
* Copyright (C) 2023-2024 Douglas B. Rumbaugh <drumbaugh@psu.edu>
* Dong Xie <dongx@psu.edu>
*
* Distributed under the Modified BSD License.
*
* The word `Internal` in this class's name refers to memory. The current
* model, inherited from the framework in Practical Dynamic Extension for
* Sampling Indexes, would use a different ExternalLevel for shards stored
* on external storage. This is a distinction that can probably be avoided
* with some more thought being put into interface design.
*
*/
#pragma once
#include <memory>
#include <vector>
#include "framework/interface/Query.h"
#include "framework/interface/Shard.h"
#include "util/types.h"
namespace de {
template <ShardInterface ShardType, QueryInterface<ShardType> QueryType>
class InternalLevel {
typedef typename ShardType::RECORD RecordType;
typedef BufferView<RecordType> BuffView;
public:
InternalLevel(ssize_t level_no) : m_level_no(level_no) {}
~InternalLevel() = default;
/*
* Create a new shard containing the combined records
* from all shards on this level and return it.
*
* No changes are made to this level.
*/
ShardType *get_combined_shard() {
if (m_shards.size() == 0) {
return nullptr;
}
std::vector<ShardType *> shards;
for (auto shard : m_shards) {
if (shard)
shards.emplace_back(shard.get());
}
return new ShardType(shards);
}
void get_local_queries(
std::vector<std::pair<ShardID, ShardType *>> &shards,
std::vector<typename QueryType::LocalQuery *> &local_queries,
typename QueryType::Parameters *query_parms) {
for (size_t i = 0; i < m_shards.size(); i++) {
if (m_shards[i]) {
auto local_query =
QueryType::local_preproc(m_shards[i].get(), query_parms);
shards.push_back({{m_level_no, (ssize_t)i}, m_shards[i].get()});
local_queries.emplace_back(local_query);
}
}
}
bool check_tombstone(size_t shard_stop, const RecordType &rec) {
if (m_shards.size() == 0)
return false;
for (int i = m_shards.size() - 1; i >= (ssize_t)shard_stop; i--) {
if (m_shards[i]) {
auto res = m_shards[i]->point_lookup(rec, true);
if (res && res->is_tombstone()) {
return true;
}
}
}
return false;
}
bool delete_record(const RecordType &rec) {
if (m_shards.size() == 0)
return false;
for (size_t i = 0; i < m_shards.size(); ++i) {
if (m_shards[i]) {
auto res = m_shards[i]->point_lookup(rec);
if (res) {
res->set_delete();
return true;
}
}
}
return false;
}
ShardType *get_shard(size_t idx) {
if (idx >= m_shards.size()) {
return nullptr;
}
return m_shards[idx].get();
}
size_t get_shard_count() { return m_shards.size(); }
size_t get_record_count() {
size_t cnt = 0;
for (size_t i = 0; i < m_shards.size(); i++) {
if (m_shards[i]) {
cnt += m_shards[i]->get_record_count();
}
}
return cnt;
}
size_t get_tombstone_count() {
size_t res = 0;
for (size_t i = 0; i < m_shards.size(); ++i) {
if (m_shards[i]) {
res += m_shards[i]->get_tombstone_count();
}
}
return res;
}
size_t get_aux_memory_usage() {
size_t cnt = 0;
for (size_t i = 0; i < m_shards.size(); i++) {
if (m_shards[i]) {
cnt += m_shards[i]->get_aux_memory_usage();
}
}
return cnt;
}
size_t get_memory_usage() {
size_t cnt = 0;
for (size_t i = 0; i < m_shards.size(); i++) {
if (m_shards[i]) {
cnt += m_shards[i]->get_memory_usage();
}
}
return cnt;
}
double get_tombstone_prop() {
size_t tscnt = 0;
size_t reccnt = 0;
for (size_t i = 0; i < m_shards.size(); i++) {
if (m_shards[i]) {
tscnt += m_shards[i]->get_tombstone_count();
reccnt += m_shards[i]->get_record_count();
}
}
return (double)tscnt / (double)(tscnt + reccnt);
}
std::shared_ptr<InternalLevel> clone() {
auto new_level = std::make_shared<InternalLevel>(m_level_no);
for (size_t i = 0; i < m_shards.size(); i++) {
new_level->m_shards[i] = m_shards[i];
}
return new_level;
}
void truncate() { m_shards.erase(m_shards.begin(), m_shards.end()); }
void delete_shard(shard_index shard) {
m_shards.erase(m_shards.begin() + shard);
}
bool append(std::shared_ptr<ShardType> shard) {
m_shards.emplace_back(shard);
}
private:
ssize_t m_level_no;
std::vector<std::shared_ptr<ShardType>> m_shards;
};
} // namespace de
|