diff options
| author | Douglas Rumbaugh <dbr4@psu.edu> | 2023-08-24 16:54:41 -0400 |
|---|---|---|
| committer | Douglas Rumbaugh <dbr4@psu.edu> | 2023-08-24 16:54:41 -0400 |
| commit | 1cb522b36382381ef3f1494f24b0c6a98f8843a9 (patch) | |
| tree | eca4180e91be5e505dd7e2240a5a784679f5a20b | |
| parent | d7a4b84a14bb7ee5988f4b1a00743ed645a2825b (diff) | |
| download | dynamic-extension-1cb522b36382381ef3f1494f24b0c6a98f8843a9.tar.gz | |
Removed unused pagedfile header
| -rw-r--r-- | CMakeLists.txt | 4 | ||||
| -rw-r--r-- | include/io/PagedFile.h | 434 | ||||
| -rw-r--r-- | tests/pagedfile_tests.cpp | 363 |
3 files changed, 0 insertions, 801 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index ca9afa1..1416420 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,10 +29,6 @@ if (tests) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin/tests") file(MAKE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests/data") - add_executable(pagedfile_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/pagedfile_tests.cpp) - target_link_libraries(pagedfile_tests PUBLIC gsl check subunit pthread) - target_include_directories(pagedfile_tests PRIVATE include) - add_executable(wirs_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/wirs_tests.cpp) target_link_libraries(wirs_tests PUBLIC gsl check subunit pthread) target_include_directories(wirs_tests PRIVATE include) diff --git a/include/io/PagedFile.h b/include/io/PagedFile.h deleted file mode 100644 index 516c0d9..0000000 --- a/include/io/PagedFile.h +++ /dev/null @@ -1,434 +0,0 @@ -/* - * include/io/PagedFile.h - * - * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> - * - * All rights reserved. Published under the Modified BSD License. - * - */ -#pragma once -#define __GNU_SOURCE - -#include <string> -#include <memory> -#include <cassert> -#include <vector> -#include <algorithm> -#include <unistd.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <sys/uio.h> -#include <fcntl.h> - -#include "util/types.h" -#include "util/base.h" - -#define PF_COUNT_IO - -#ifdef PF_COUNT_IO - #define INC_READ() de::pf_read_cnt++ - #define INC_WRITE() de::pf_write_cnt++ - #define RESET_IO_CNT() \ - de::pf_read_cnt = 0; \ - de::pf_write_cnt = 0 -#else - #define INC_READ() do {} while (0) - #define INC_WRITE() do {} while (0) - #define RESET_IO_CNT() do {} while (0) -#endif - -namespace de { - -static thread_local size_t pf_read_cnt = 0; -static thread_local size_t pf_write_cnt = 0; - -class PagedFileIterator; -class PagedFile; -static PagedFileIterator *create_pfile_iter(PagedFile *pfile, PageNum start_page=0, PageNum stop_page=0); - -class PagedFile { -public: - static PagedFile *create(const std::string fname, bool new_file=true) { - auto flags = O_RDWR; - mode_t mode = 0640; - off_t size = 0; - - if (new_file) { - flags |= O_CREAT | O_TRUNC; - } - - int fd = open(fname.c_str(), flags, mode); - if (fd == -1) { - return nullptr; - } - - if (new_file) { - if(fallocate(fd, 0, 0, PAGE_SIZE)) { - return nullptr; - } - - size = PAGE_SIZE; - } else { - struct stat buf; - if (fstat(fd, &buf) == -1) { - return nullptr; - } - - size = buf.st_size; - } - - if (fd) { - return new PagedFile(fd, fname, size, mode); - } - - return nullptr; - - } - - ~PagedFile() { - if (m_file_open) { - close(m_fd); - } - } - - /* - * Add new_page_count new pages to the file in bulk, and returns the - * PageId of the first page in the new range. - * - * If the allocation fails, returns INVALID_PID. Also returns INVALID_PID - * if bulk allocation is not supported by the implementation. This can be - * iter1. checked via the supports_allocation method. - */ - PageNum allocate_pages(PageNum count=1) { - PageNum new_first = get_page_count() + 1; - size_t alloc_size = count * PAGE_SIZE; - - if (raw_allocate(alloc_size)) { - return new_first; - } - - return INVALID_PNUM; - } - - /* - * Reads data from the specified page into a buffer pointed to by - * buffer_ptr. It is necessary for buffer_ptr to be parm::SECTOR_SIZE - * aligned, and also for it to be large enough to accommodate - * parm::PAGE_SIZE chars. If the read succeeds, returns 1. Otherwise - * returns 0. The contents of the input buffer are undefined in the case of - * an error. - */ - int read_page(PageNum pnum, char *buffer_ptr) { - - return (check_pnum(pnum)) ? raw_read(buffer_ptr, PAGE_SIZE, PagedFile::pnum_to_offset(pnum)) - : 0; - } - - /* - * Reads several pages into associated buffers. It is necessary for the - * buffer referred to by each pointer to be parm::SECTOR_SIZE aligned and - * large enough to accommodate parm::PAGE_SIZE chars. If possible, - * vectorized IO may be used to read adjacent pages. If the reads succeed, - * returns 1. If a read fails, returns 0. The contents of all the buffers - * are undefined in the case of an error. - */ - int read_pages(std::vector<std::pair<PageNum, char*>> pages) { - if (pages.size() == 0) { - return 0; - } - - if (pages.size() == 1) { - read_page(pages[0].first, pages[0].second); - } - - std::sort(pages.begin(), pages.end()); - - PageNum range_start = pages[0].first; - PageNum prev_pnum = range_start; - - std::vector<char *> buffers; - buffers.push_back(pages[0].second); - - for (size_t i=1; i<pages.size(); i++) { - if (pages[i].first == prev_pnum + 1) { - buffers.push_back(pages[i].second); - prev_pnum = pages[i].first; - } else { - if (!raw_readv(buffers, PAGE_SIZE, PagedFile::pnum_to_offset(range_start))) { - return 0; - } - - range_start = pages[i].first; - prev_pnum = range_start; - - buffers.clear(); - buffers.push_back(pages[i].second); - } - } - - return raw_readv(buffers, PAGE_SIZE, PagedFile::pnum_to_offset(range_start)); - } - - /* - * Reads several pages stored contiguously into a single buffer. It is - * necessary that buffer_ptr be SECTOR_SIZE aligned and also at least - * page_cnt * PAGE_SIZE chars large. - */ - int read_pages(PageNum first_page, size_t page_cnt, char *buffer_ptr) { - if (check_pnum(first_page) && check_pnum(first_page + page_cnt - 1)) { - return raw_read(buffer_ptr, page_cnt * PAGE_SIZE, first_page * PAGE_SIZE); - } - - return 0; - } - - /* - * Writes data from the provided buffer into the specified page within the - * file. It is necessary for buffer_ptr to be parm::SECTOR_SIZE aligned, - * and also for it to be at least parm::PAGE_SIZE chars large. If it is - * larger, only the first parm::PAGE_SIZE chars will be written. If it is - * smaller, the result is undefined. - * - * If the write succeeds, returns 1. Otherwise returns 0. The contents of - * the specified page within the file are undefined in the case of an error. - */ - int write_page(PageNum pnum, const char *buffer_ptr) { - if (check_pnum(pnum)) { - return raw_write(buffer_ptr, PAGE_SIZE, PagedFile::pnum_to_offset(pnum)); - } - - return 0; - } - - /* - * Writes multiple pages stored sequentially in the provided buffer into - * a contiguous region of the file, starting at first_page. If the write - * would overrun the allocated space in the file, no data is written. - * - * It is necessary for buffer_ptr to be SECTOR_SIZE aligned, and at - * least PAGE_SIZE * page_cnt chars large. - * - * Returns the number of complete pages successfully written. - */ - int write_pages(PageNum first_page, size_t page_cnt, const char *buffer_ptr) { - if (check_pnum(first_page) && check_pnum(first_page + page_cnt - 1)) { - return raw_write(buffer_ptr, page_cnt * PAGE_SIZE, first_page * PAGE_SIZE); - } - - return 0; - } - - /* - * Returns the number of allocated paged in the file. - */ - PageNum get_page_count() { - return m_size / PAGE_SIZE - 1; - } - - /* - * Delete this file from the underlying filesystem. Once this has been called, - * this object will be closed, and all operations other than destructing it are - * undefined. Returns 1 on successful removal of the file, and 0 on failure. - */ - int remove_file() { - if (m_file_open) { - close(m_fd); - } - - if (unlink(m_fname.c_str())) { - return 0; - } - - m_file_open = false; - - return 1; - } - - /* - * Returns the raw number of bytes allocated in the backing file. - */ - off_t get_file_size() const { - return m_size; - } - - PagedFileIterator *start_scan(PageNum start_page=1, PageNum end_page=0) { - if (end_page == INVALID_PNUM) { - end_page = get_page_count(); - } - - if (check_pnum(start_page) && check_pnum(end_page)) { - return create_pfile_iter(this, start_page, end_page); - } - - return nullptr; - } - - std::string get_fname() { - return m_fname; - } - - void rename_file(std::string new_fname) { - if(rename(m_fname.c_str(), new_fname.c_str())) { - fprintf(stderr, "%s -> %s\n", m_fname.c_str(), new_fname.c_str()); - perror("IN RENAME:"); - assert(false); - } - - m_fname = new_fname; - } - -private: - PagedFile(int fd, std::string fname, off_t size, mode_t mode) { - m_file_open = true; - m_fd = fd; - m_fname = fname; - m_size = size; - m_mode = mode; - m_flags = O_RDWR | O_DIRECT; - } - - static off_t pnum_to_offset(PageNum pnum) { - return pnum * PAGE_SIZE; - } - - bool check_pnum(PageNum pnum) const { - return pnum != INVALID_PNUM && pnum < (get_file_size() / PAGE_SIZE); - } - - int raw_read(char *buffer, off_t amount, off_t offset) { - if (!verify_io_parms(amount, offset)) { - return 0; - } - - if (pread(m_fd, buffer, amount, offset) != amount) { - return 0; - } - - INC_READ(); - - return 1; - } - - int raw_readv(std::vector<char *> buffers, off_t buffer_size, off_t initial_offset) { - size_t buffer_cnt = buffers.size(); - - off_t amount = buffer_size * buffer_cnt; - if (!verify_io_parms(amount, initial_offset)) { - return 0; - } - - auto iov = new iovec[buffer_cnt]; - for (size_t i=0; i<buffer_cnt; i++) { - iov[i].iov_base = buffers[i]; - iov[i].iov_len = buffer_size; - } - - if (preadv(m_fd, iov, buffer_cnt, initial_offset) != amount) { - return 0; - } - - INC_READ(); - - delete[] iov; - - return 1; - } - - int raw_write(const char *buffer, off_t amount, off_t offset) { - if (!verify_io_parms(amount, offset)) { - return 0; - } - - if (pwrite(m_fd, buffer, amount, offset) != amount) { - return 0; - } - - INC_WRITE(); - - return 1; - } - - int raw_allocate(size_t amount) { - if (!m_file_open || (amount % SECTOR_SIZE != 0)) { - return 0; - } - - int alloc_mode = 0; - if (fallocate(m_fd, alloc_mode, m_size, amount)) { - return 0; - } - - m_size += amount; - return 1; - } - - bool verify_io_parms(off_t amount, off_t offset) { - if (!m_file_open || amount + offset > m_size) { - return false; - } - - if (amount % SECTOR_SIZE != 0) { - return false; - } - - if (offset % SECTOR_SIZE != 0) { - return false; - } - - return true; - } - - int m_fd; - bool m_file_open; - off_t m_size; - mode_t m_mode; - std::string m_fname; - int m_flags; -}; - - -class PagedFileIterator { -public: - PagedFileIterator(PagedFile *pfile, PageNum start_page = 0, - PageNum stop_page = 0) - : m_pfile(pfile), - m_current_pnum((start_page == INVALID_PNUM) ? 0 : start_page - 1), - m_start_pnum(start_page), m_stop_pnum(stop_page), - m_buffer((char *)aligned_alloc(SECTOR_SIZE, PAGE_SIZE)) {} - - bool next() { - while (m_current_pnum < m_stop_pnum) { - if (m_pfile->read_page(++m_current_pnum, m_buffer)) { - return true; - } - - // IO error of some kind - return false; - } - - // no more pages to read - return false; - } - - char *get_item() { - return m_buffer; - } - - ~PagedFileIterator() { - free(m_buffer); - } - -private: - PagedFile *m_pfile; - PageNum m_current_pnum; - PageNum m_start_pnum; - PageNum m_stop_pnum; - - char *m_buffer; -}; - -static PagedFileIterator *create_pfile_iter(PagedFile *pfile, PageNum start_page, PageNum stop_page) { - return new PagedFileIterator(pfile, start_page, stop_page); -} - -} diff --git a/tests/pagedfile_tests.cpp b/tests/pagedfile_tests.cpp deleted file mode 100644 index 5c5bd2f..0000000 --- a/tests/pagedfile_tests.cpp +++ /dev/null @@ -1,363 +0,0 @@ -/* - * tests/pagedfile_Tests.cpp - * - * Unit tests for PagedFile - * - * Copyright (C) 2023 Douglas Rumbaugh <drumbaugh@psu.edu> - * Dong Xie <dongx@psu.edu> - * - * All rights reserved. Published under the Modified BSD License. - * - */ -#include <string> - -#include "testing.h" -#include "io/PagedFile.h" - -#include <check.h> - -using namespace de; - -std::string existing_file1 = "tests/data/test_file1.dat"; -std::string nonexisting_file = "tests/data/nonexisting_file.dat"; -std::string new_file = "tests/data/new_file.dat"; - - -START_TEST(t_create) -{ - auto pfile = PagedFile::create(new_file, true); - - ck_assert_ptr_nonnull(pfile); - ck_assert_int_eq(pfile->get_page_count(), 0); - ck_assert_int_eq(pfile->get_file_size(), PAGE_SIZE); - - std::string fname = pfile->get_fname(); - ck_assert_str_eq(fname.c_str(), new_file.c_str()); - - delete pfile; -} -END_TEST - - -START_TEST(t_create_fail) -{ - auto pfile = PagedFile::create(nonexisting_file, false); - - ck_assert_ptr_null(pfile); -} -END_TEST - - -START_TEST(t_create_open) -{ - size_t pg_cnt = 10; - ck_assert(initialize_test_file(existing_file1, pg_cnt)); - - auto pfile = PagedFile::create(existing_file1, false); - ck_assert_ptr_nonnull(pfile); - ck_assert_int_eq(pfile->get_page_count(), pg_cnt); - ck_assert_int_eq(pfile->get_file_size(), PAGE_SIZE * (pg_cnt + 1)); - - std::string fname = pfile->get_fname(); - ck_assert_str_eq(fname.c_str(), existing_file1.c_str()); - - delete pfile; -} -END_TEST - - -START_TEST(t_read_page) -{ - size_t pg_cnt = 10; - ck_assert(initialize_test_file(existing_file1, pg_cnt)); - auto pfile = PagedFile::create(existing_file1, false); - ck_assert_ptr_nonnull(pfile); - - char *buffer = (char *) aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - - for (size_t i=1; i<=pg_cnt; i++) { - ck_assert_int_eq(pfile->read_page(i, buffer), 1); - ck_assert_int_eq(*((int *) buffer), i); - } - - ck_assert_int_eq(pfile->read_page(0, buffer), 0); - ck_assert_int_eq(pfile->read_page(11, buffer), 0); - - free(buffer); - delete pfile; -} -END_TEST - - -START_TEST(t_read_pages_sg) -{ - size_t pg_cnt = 20; - ck_assert(initialize_test_file(existing_file1, pg_cnt)); - auto pfile = PagedFile::create(existing_file1, false); - ck_assert_ptr_nonnull(pfile); - - size_t buf_cnt = 10; - std::vector<char *> buffers(buf_cnt); - std::vector<std::pair<PageNum, char*>> reads(buf_cnt); - std::vector<PageNum> to_read = {1, 2, 3, 5, 9, 7, 8, 11, 12, 15}; - for (size_t i=0; i<buf_cnt; i++) { - buffers[i] = (char *) std::aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - reads[i] = {to_read[i], buffers[i]}; - } - - ck_assert_int_eq(pfile->read_pages(reads), 1); - - for (size_t i=0; i<buf_cnt; i++) { - ck_assert_int_eq(*((int*) buffers[i]), to_read[i]); - } - - for (size_t i=0; i<buf_cnt; i++) { - free(buffers[i]); - } - - delete pfile; -} -END_TEST - - -START_TEST(t_read_pages_seq) -{ - size_t pg_cnt = 20; - ck_assert(initialize_test_file(existing_file1, pg_cnt)); - auto pfile = PagedFile::create(existing_file1, false); - ck_assert_ptr_nonnull(pfile); - - PageNum read_cnt = 10; - PageNum start_pg = 5; - char *buffer = (char *) aligned_alloc(SECTOR_SIZE, PAGE_SIZE*read_cnt); - ck_assert_ptr_nonnull(buffer); - - ck_assert_int_eq(pfile->read_pages(start_pg, read_cnt, buffer), 1); - for (size_t i=0; i<read_cnt; i++) { - ck_assert_int_eq(i + start_pg, *((int*) (buffer + PAGE_SIZE * i))); - } - - read_cnt = 25; - ck_assert_int_eq(pfile->read_pages(start_pg, start_pg + read_cnt, buffer), 0); - - // FIXME: This throws a double free error, but otherwise works. - // I'm definitely not freeing this anywhere else, so not sure - // what to make of that. - free(buffer); - delete pfile; -} -END_TEST - -START_TEST(t_allocate_pages) -{ - auto pfile = PagedFile::create(new_file, true); - ck_assert_ptr_nonnull(pfile); - - ck_assert_int_eq(pfile->get_page_count(), 0); - - ck_assert_int_eq(pfile->allocate_pages(1), 1); - ck_assert_int_eq(pfile->get_page_count(), 1); - ck_assert_int_eq(pfile->get_file_size(), 2*PAGE_SIZE); - - ck_assert_int_eq(pfile->allocate_pages(10), 2); - ck_assert_int_eq(pfile->get_page_count(), 11); - ck_assert_int_eq(pfile->get_file_size(), 12*PAGE_SIZE); - - delete pfile; -} -END_TEST - -START_TEST(t_write) -{ - auto pfile = PagedFile::create(new_file, true); - ck_assert_ptr_nonnull(pfile); - - char *buffer = (char *) aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - *((int*) buffer) = 123; - - // writing to an unallocated page fails - ck_assert_int_eq(pfile->write_page(1, buffer), 0); - - pfile->allocate_pages(1); - - ck_assert_int_eq(pfile->write_page(1, buffer), 1); - ck_assert_int_eq(pfile->write_page(2, buffer), 0); - - delete pfile; - - auto pfile2 = PagedFile::create(new_file, false); - ck_assert_int_eq(pfile2->get_page_count(), 1); - - char *buffer2 = (char *) aligned_alloc(SECTOR_SIZE, PAGE_SIZE); - ck_assert_int_eq(pfile2->read_page(1, buffer2), 1); - ck_assert_int_eq(123, *((int*) buffer2)); - - free(buffer); - free(buffer2); - delete pfile2; -} -END_TEST - - -START_TEST(t_write_pages) -{ - auto pfile = PagedFile::create(new_file, true); - ck_assert_ptr_nonnull(pfile); - - PageNum page_cnt = 13; - char *buffer = (char *) aligned_alloc(SECTOR_SIZE, page_cnt*PAGE_SIZE); - - for (size_t i=0; i<page_cnt; i++) { - *((int*) (buffer + PAGE_SIZE * i)) = i; - } - - ck_assert_int_eq(pfile->write_pages(1, 13, buffer), 0); - - pfile->allocate_pages(20); - - PageNum start_pg = 3; - ck_assert_int_eq(pfile->write_pages(start_pg, page_cnt, buffer), 1); - - delete pfile; - free(buffer); - - auto pfile2 = PagedFile::create(new_file, false); - buffer = (char *) aligned_alloc(SECTOR_SIZE, page_cnt*PAGE_SIZE); - - ck_assert_int_eq(pfile2->read_pages(start_pg, page_cnt, buffer), 1); - for (size_t i=0; i<page_cnt; i++) { - ck_assert_int_eq(*((int*) (buffer + PAGE_SIZE * i)), i); - } - - free(buffer); - delete pfile2; -} -END_TEST - - -START_TEST(t_remove) -{ - auto pfile = PagedFile::create(new_file, true); - ck_assert_ptr_nonnull(pfile); - - delete pfile; - - pfile = PagedFile::create(new_file, false); - ck_assert_ptr_nonnull(pfile); - - ck_assert_int_eq(pfile->remove_file(), 1); - delete pfile; - - pfile = PagedFile::create(new_file, false); - ck_assert_ptr_null(pfile); -} -END_TEST - - -START_TEST(t_iterator) -{ - size_t pg_cnt = 20; - ck_assert(initialize_test_file(existing_file1, pg_cnt)); - auto pfile = PagedFile::create(existing_file1, false); - ck_assert_ptr_nonnull(pfile); - - auto iter = pfile->start_scan(); - ck_assert_ptr_nonnull(iter); - size_t i=0; - while (iter->next()) { - i++; - ck_assert_int_eq(i, *((int*) iter->get_item())); - } - - ck_assert_int_eq(i, pg_cnt); - - delete iter; - delete pfile; -} -END_TEST - - -START_TEST(t_iterator_page_range) -{ - size_t pg_cnt = 20; - ck_assert(initialize_test_file(existing_file1, pg_cnt)); - auto pfile = PagedFile::create(existing_file1, false); - ck_assert_ptr_nonnull(pfile); - - auto iter = pfile->start_scan(30, 45); - ck_assert_ptr_null(iter); - - iter = pfile->start_scan(5, 13); - - size_t i=4; - while (iter->next()) { - i++; - ck_assert_int_eq(i, *((int*) iter->get_item())); - } - - ck_assert_int_eq(i, 13); - - delete iter; - delete pfile; -} -END_TEST - - -Suite *unit_testing() -{ - Suite *unit = suite_create("PagedFile Unit Testing"); - TCase *initialize = tcase_create("de::PagedFile::create Testing"); - tcase_add_test(initialize, t_create); - tcase_add_test(initialize, t_create_fail); - tcase_add_test(initialize, t_create_open); - suite_add_tcase(unit, initialize); - - TCase *read = tcase_create("de::PagedFile::read_page(s) Testing"); - tcase_add_test(read, t_read_page); - tcase_add_test(read, t_read_pages_sg); - tcase_add_test(read, t_read_pages_seq); - suite_add_tcase(unit, read); - - TCase *allocate = tcase_create("de::PagedFile::allocate_pages Testing"); - tcase_add_test(allocate, t_allocate_pages); - suite_add_tcase(unit, allocate); - - TCase *write = tcase_create("de::PagedFile::write_page(s) Testing"); - tcase_add_test(write, t_write); - tcase_add_test(write, t_write_pages); - suite_add_tcase(unit, write); - - TCase *remove = tcase_create("de::PagedFile::remove_file Testing"); - tcase_add_test(remove, t_remove); - suite_add_tcase(unit, remove); - - TCase *iter = tcase_create("de::PagedFile::start_scan Testing"); - tcase_add_test(iter, t_iterator); - tcase_add_test(iter, t_iterator_page_range); - suite_add_tcase(unit, iter); - - return unit; -} - - -int run_unit_tests() -{ - int failed = 0; - Suite *unit = unit_testing(); - SRunner *unit_runner = srunner_create(unit); - - srunner_run_all(unit_runner, CK_NORMAL); - failed = srunner_ntests_failed(unit_runner); - srunner_free(unit_runner); - - return failed; -} - - -int main() -{ - int unit_failed = run_unit_tests(); - - return (unit_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; -} - |