Skip to content

GitLab

Commit cf18d1ad authored by Philipp Götze's avatar Philipp Götze
Browse files

Made pfabric use ptable as library

parent 1d242627
Hide whitespace changes
Inline Side-by-side
cmake/Download3rdParty.cmake
View file @ cf18d1ad
......@@ -122,34 +122,26 @@ if(USE_NVML_TABLE)
# Non-Volatile Memory Library (pmem.io)
download_project(PROJ nvml
GIT_REPOSITORY https://github.com/pmem/nvml.git
GIT_TAG master
GIT_TAG 1.3.1-rc2
UPDATE_DISCONNECTED 1
QUIET
)
add_custom_command(
OUTPUT ${THIRD_PARTY_DIR}/nvml
COMMAND ${CMAKE_COMMAND} -E chdir ${nvml_SOURCE_DIR} $(MAKE)
COMMAND ${CMAKE_COMMAND} -E chdir ${nvml_SOURCE_DIR} $(MAKE) install prefix=${THIRD_PARTY_DIR}/nvml
#COMMAND ${CMAKE_COMMAND} -E make_directory ${THIRD_PARTY_DIR}/nvml/include
#COMMAND ${CMAKE_COMMAND} -E copy_directory
# ${nvml_SOURCE_DIR}/src/include
# ${THIRD_PARTY_DIR}/nvml/include
COMMAND ${CMAKE_COMMAND} -E chdir ${nvml_SOURCE_DIR} $(MAKE) install prefix=${THIRD_PARTY_DIR}/nvml
)
# Key/Value Store for Non-Volatile Memory(pmem.io)
if(FALSE)
download_project(PROJ pmemkv
GIT_REPOSITORY https://github.com/pmem/pmemkv.git
# PTable (internal gitlab project) for NVM
download_project(PROJ ptable
GIT_REPOSITORY https://dbgit.prakinf.tu-ilmenau.de/code/PTable.git
GIT_TAG master
UPDATE_DISCONNECTED 1
QUIET
)
add_custom_command(
OUTPUT ${THIRD_PARTY_DIR}/pmemkv
COMMAND ${CMAKE_COMMAND} -E chdir ${pmemkv_SOURCE_DIR} $(MAKE) configure
COMMAND ${CMAKE_COMMAND} -E make_directory ${THIRD_PARTY_DIR}/pmemkv/include
COMMAND ${CMAKE_COMMAND} -E make_directory ${THIRD_PARTY_DIR}/pmemkv/lib
COMMAND ${CMAKE_COMMAND} -E chdir ${pmemkv_SOURCE_DIR} $(MAKE) install prefix=${THIRD_PARTY_DIR}/pmemkv
OUTPUT ${THIRD_PARTY_DIR}/ptable
COMMAND ${CMAKE_COMMAND} -E chdir ${ptable_SOURCE_DIR} cmake -DPTABLE_DIR=${THIRD_PARTY_DIR}/ptable src
COMMAND ${CMAKE_COMMAND} -E chdir ${ptable_SOURCE_DIR} $(MAKE) install
)
endif()
endif()
src/CMakeLists.txt
View file @ cf18d1ad
......@@ -52,7 +52,7 @@ option(BUILD_ONLY_LIBS
# If switched to off, no tests will be build
option(BUILD_TEST_CASES
"build tests for pipefabric functionality"
OFF
ON
)
#Build google benchmark library
......@@ -64,7 +64,7 @@ option(BUILD_GOOGLE_BENCH
# Build benchmark test
option(BUILD_BENCHMARKS
"build benchmarks for pipefabric"
ON
OFF
)
# Benchmark test requires benchmark library
......@@ -78,7 +78,7 @@ endif()
#CMAKE_FORCE_CXX_COMPILER(icpc "Intel C++ Compiler")
# C++ compiler flags
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -Wno-deprecated -g -O2 -Wsign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -Wno-deprecated -g -O3 -Wsign-compare")
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedefs -Wno-#pragma-messages")
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
......@@ -168,17 +168,19 @@ if (USE_NVML_TABLE)
message(STATUS "using NVML based persistent table")
add_definitions(-DUSE_NVML_TABLE)
set (NVML_LIBRARIES
"${THIRD_PARTY_DIR}/nvml/lib/libpmemblk.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmemlog.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmemobj.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmempool.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmem.a"
# "${THIRD_PARTY_DIR}/nvml/lib/libvmem.a"
# "${THIRD_PARTY_DIR}/nvml/lib/libvmmalloc.a"
${DYLIB_LIBRARY}
)
include_directories("${THIRD_PARTY_DIR}/nvml/include")
else()
"${THIRD_PARTY_DIR}/nvml/lib/libpmemblk.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmemlog.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmemobj.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmempool.a"
"${THIRD_PARTY_DIR}/nvml/lib/libpmem.a"
${DYLIB_LIBRARY}
"${THIRD_PARTY_DIR}/ptable/lib/libptable.so"
)
include_directories(
"${THIRD_PARTY_DIR}/nvml/include"
"${THIRD_PARTY_DIR}/ptable/include"
)
else ()
message(STATUS "don't use NVML based persistent table")
set (NVML_LIBRARIES "")
endif()
......@@ -303,9 +305,8 @@ endif()
if(USE_NVML_TABLE)
set(core_sources
${core_sources}
nvm/PTableInfo.cpp
${THIRD_PARTY_DIR}/nvml
# ${THIRD_PARTY_DIR}/pmemkv
${THIRD_PARTY_DIR}/ptable
)
add_library(pfabric_core SHARED
${core_sources}
......
src/table/NVMTable.hpp
View file @ cf18d1ad
......@@ -31,31 +31,32 @@
#include <stdint.h>
#include <unistd.h>
#include <cstdio>
#include <type_traits>
#include <boost/array.hpp>
#include <boost/fusion/algorithm/iteration/for_each.hpp>
#include <boost/fusion/include/for_each.hpp>
#include <boost/signals2.hpp>
#include <libpmemobj++/make_persistent.hpp>
#include <libpmemobj++/p.hpp>
#include <libpmemobj++/persistent_ptr.hpp>
#include <libpmemobj++/pool.hpp>
#include <libpmemobj++/transaction.hpp>
#include <libpmemobj++/utils.hpp>
#include <libpmempool.h>
#include "fmt/format.h"
#include "PTable.hpp"
#include "core/Tuple.hpp"
#include "nvm/PTable.hpp"
#include "nvm/PTableInfo.hpp"
#include "nvm/PTuple.hpp"
#include "table/TableException.hpp"
#include "table/BaseTable.hpp"
#include "table/TableInfo.hpp"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/p.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
#include "nvml/include/libpmemobj++/pool.hpp"
#include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmempool.h"
namespace pfabric {
//TODO: Maybe the pmem device path prefix should be a CMake variable?
const std::string pathPrefix = "/mnt/pmem/test/";
namespace detail {
struct GetType {
......@@ -111,6 +112,14 @@ TableInfo constructSchema(const std::string &tableName) {
tInfo.setColumns(cols);
return tInfo;
}
template<typename T>
struct is_tuple_impl : std::false_type {};
template<typename... Ts>
struct is_tuple_impl<pfabric::Tuple<Ts...>> : std::true_type {};
template<typename T>
struct is_tuple : is_tuple_impl<std::decay_t<T>> {};
} /* namespace detail */
using nvml::obj::delete_persistent;
......@@ -119,13 +128,16 @@ using nvml::obj::p;
using nvml::obj::persistent_ptr;
using nvml::obj::pool;
using nvml::obj::transaction;
using pfabric::nvm::PTable;
using ptable::PTable;
using ptable::PTuple;
template<typename RecordType, typename KeyType>
class NVMIterator {
public:
typedef std::function<bool(const nvm::PTuple<RecordType, KeyType> &)> Predicate;
typedef PTable<RecordType, KeyType> PTableType;
static_assert(detail::is_tuple<RecordType>::value, "Value type must be a pfabric::Tuple");
using TupleType = typename RecordType::Base;
using Predicate = std::function<bool(const PTuple<TupleType, KeyType> &)>;
using PTableType = PTable<TupleType, KeyType>;
explicit NVMIterator() {
}
......@@ -155,7 +167,8 @@ class NVMIterator {
}
SmartPtr<RecordType> operator*() {
return (*iter).createTuple(); //TODO: Is this to expensive?
SmartPtr<RecordType> tptr(new RecordType(*(*iter).createTuple()));
return tptr; //TODO: Is this to expensive?
}
protected:
......@@ -168,8 +181,8 @@ class NVMIterator {
template<typename RecordType, typename KeyType>
inline NVMIterator<RecordType, KeyType> makeNVMIterator(
typename PTable<RecordType, KeyType>::iterator &&iter,
typename PTable<RecordType, KeyType>::iterator &&end,
typename PTable<typename RecordType::Base, KeyType>::iterator &&iter,
typename PTable<typename RecordType::Base, KeyType>::iterator &&end,
typename NVMIterator<RecordType, KeyType>::Predicate pred) {
return NVMIterator<RecordType, KeyType>(std::move(iter), std::move(end), pred);
}
......@@ -190,28 +203,31 @@ inline NVMIterator<RecordType, KeyType> makeNVMIterator(
template<typename RecordType, typename KeyType = DefaultKeyType>
class NVMTable : public BaseTable {
public:
typedef nvm::PTable<RecordType, KeyType> PTableType;
static_assert(detail::is_tuple<RecordType>::value, "Value type must be a pfabric::Tuple");
using TupleType = typename RecordType::Base;
using PTableType = PTable<TupleType, KeyType>;
struct root {
persistent_ptr<PTableType> pTable;
};
//< typedef for a updater function which returns a modification of the parameter tuple
typedef std::function<void(RecordType &)> UpdaterFunc;
/** typedef for a updater function which returns a modification of the parameter tuple */
using UpdaterFunc = std::function<void(RecordType &)>;
//< typedefs for a function performing updates + deletes. Similar to UpdaterFunc
//< it allows to update the tuple, but also to delete it (indictated by the
//< setting the bool component of \c UpdateResult to false)
typedef std::function<bool(RecordType &)> UpdelFunc;
/** typedefs for a function performing updates + deletes. Similar to UpdaterFunc
* it allows to update the tuple, but also to delete it (indictated by the
* setting the bool component of \c UpdateResult to false)
**/
using UpdelFunc = std::function<bool(RecordType &)>;
//< typedef for a callback function which is invoked when the table was updated
typedef boost::signals2::signal<void(const RecordType &, TableParams::ModificationMode)> ObserverCallback;
/** typedef for a callback function which is invoked when the table was updated */
using ObserverCallback = boost::signals2::signal<void(const RecordType &, TableParams::ModificationMode)>;
//< typedef for an iterator to scan the table
typedef NVMIterator<RecordType, KeyType> TableIterator;
/** typedef for an iterator to scan the table */
using TableIterator = NVMIterator<RecordType, KeyType>;
//< typedef for a predicate evaluated using a scan: see \TableIterator for details
typedef typename TableIterator::Predicate Predicate;
/** typedef for a predicate evaluated using a scan: see \TableIterator for details */
using Predicate = typename TableIterator::Predicate;
/************************************************************************//**
* \brief Constructor for creating an empty table with only a given name.
......@@ -244,9 +260,9 @@ class NVMTable : public BaseTable {
* \param key the key value of the tuple
* \param rec the actual tuple
*****************************************************************************/
void insert(KeyType key, const RecordType &rec) throw(TableException) {
pop.get_root()->pTable->insert(key, rec);
notifyObservers(rec, TableParams::Insert, TableParams::Immediate);
void insert(KeyType key, const RecordType &rec) noexcept(false) {
pTable->insert(key, rec.data());
notifyObservers(rec, TableParams::Insert, TableParams::Immediate);
}
/************************************************************************//**
......@@ -259,11 +275,7 @@ class NVMTable : public BaseTable {
* \return the number of deleted tuples
*****************************************************************************/
unsigned long deleteByKey(KeyType key) {
unsigned long nres = 0;
{
//TODO:
}
return nres;
return pTable->deleteByKey(key);
}
/************************************************************************//**
......@@ -340,8 +352,11 @@ class NVMTable : public BaseTable {
* \param key the key value
* \return the tuple associated with the given key
*****************************************************************************/
const SmartPtr<RecordType> getByKey(KeyType key) throw(TableException) {
return pTable->getByKey(key).createTuple();
const SmartPtr<RecordType> getByKey(KeyType key) noexcept(false) {
//TODO: ugly, can we do better?
SmartPtr<RecordType> tptr(new RecordType(*pTable->getByKey(key).createTuple()));
return tptr;
}
/************************************************************************//**
......@@ -380,7 +395,7 @@ class NVMTable : public BaseTable {
* \return a pair of iterators
*****************************************************************************/
TableIterator select() {
auto alwaysTrue = [](const nvm::PTuple<RecordType, KeyType> &) { return true; };
auto alwaysTrue = [](const PTuple<TupleType, KeyType> &) { return true; };
return makeNVMIterator<RecordType, KeyType>(std::move(pTable->begin()), std::move(pTable->end()), alwaysTrue);
}
......@@ -413,6 +428,7 @@ class NVMTable : public BaseTable {
}
void drop() {
auto pop = pool_by_pptr(q);
transaction::exec_tx(pop, [&] {
delete_persistent<PTableType>(pTable);
pTable = nullptr;
......@@ -420,7 +436,7 @@ class NVMTable : public BaseTable {
q = nullptr;
});
pop.close();
pmempool_rm((BaseTable::mTableInfo->tableName() + ".db").c_str(), 1);
pmempool_rm((pathPrefix + BaseTable::mTableInfo->tableName() + ".db").c_str(), 1);
//std::remove((BaseTable::mTableInfo->tableName()+".db").c_str());
}
......@@ -431,15 +447,30 @@ class NVMTable : public BaseTable {
private:
void openOrCreateTable(const TableInfo &tableInfo) /*throw(TableException)*/
{
std::string path = tableInfo.tableName() + ".db";
std::string path = pathPrefix + tableInfo.tableName() + ".db";
pool<root> pop;
if (access(path.c_str(), F_OK) != 0) {
pop = pool<root>::create(path, nvm::LAYOUT, 16 * 1024 * 1024); //, (size_t)blockSize, 0666);
pop = pool<root>::create(path, ptable::LAYOUT, 64 * 1024 * 1024); //, (size_t)blockSize, 0666);
transaction::exec_tx(pop, [&] {
auto tbl = make_persistent<PTableType>(tableInfo);
ptable::ColumnVector cVector;
for (auto &c : tableInfo) {
switch (c.getType()) {
case ColumnInfo::Void_Type:cVector.push_back(ptable::Column(c.getName(), ptable::Void_Type));
break;
case ColumnInfo::Int_Type:cVector.push_back(ptable::Column(c.getName(), ptable::Int_Type));
break;
case ColumnInfo::Double_Type:cVector.push_back(ptable::Column(c.getName(), ptable::Double_Type));
break;
case ColumnInfo::String_Type:cVector.push_back(ptable::Column(c.getName(), ptable::String_Type));
};
}
ptable::VTableInfo vTableInfo(tableInfo.tableName(), cVector);
auto tbl = make_persistent<PTableType>(vTableInfo);
pop.get_root()->pTable = tbl;
});
} else {
pop = pool<root>::open(path, nvm::LAYOUT);
pop = pool<root>::open(path, ptable::LAYOUT);
}
q = pop.get_root();
pTable = q->pTable;
......@@ -464,7 +495,6 @@ class NVMTable : public BaseTable {
}
}
pool<root> pop;
persistent_ptr<struct root> q;
persistent_ptr<PTableType> pTable;
ObserverCallback mImmediateObservers, mDeferredObservers;
......
src/test/CMakeLists.txt
View file @ cf18d1ad
......@@ -49,7 +49,7 @@ if (BUILD_TEST_CASES)
do_test(RocksDBTest)
do_test(RDBTableTest)
elseif (USE_NVML_TABLE)
do_test(PTableTest)
do_test(NVMTableTest)
else ()
do_test(HashMapTableTest)
endif ()
......
src/test/NVMTableTest.cpp
View file @ cf18d1ad
......@@ -3,75 +3,32 @@
#include "catch.hpp"
#include <unistd.h>
#include "fmt/format.h"
#include "nvm/BDCCInfo.hpp"
#include "nvm/PTable.hpp"
#include "nvm/PTableInfo.hpp"
#include "table/TableInfo.hpp"
#include "core/Tuple.hpp"
#include "core/serialize.hpp"
#include "pfabric.hpp"
#include "table/NVMTable.hpp"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/p.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
#include "nvml/include/libpmemobj++/pool.hpp"
using namespace pfabric;
using MyTuple = Tuple<int, int, string, double>;
using TableType = NVMTable<MyTuple, int>;
using namespace pfabric::nvm;
using nvml::obj::make_persistent;
using nvml::obj::p;
using nvml::obj::persistent_ptr;
using nvml::obj::pool;
using nvml::obj::transaction;
typedef pfabric::Tuple<int, int, string, double> MyTuple;
typedef PTable<MyTuple, int> PTableType;
TEST_CASE("Testing storing tuples in PTable", "[PTable]") {
struct root {
persistent_ptr<PTableType> pTable;
};
pool<root> pop;
const std::string path = "/mnt/pmem/tests/testdb.db";
//std::remove(path.c_str());
if (access(path.c_str(), F_OK) != 0) {
pop = pool<root>::create(path, LAYOUT, 16*1024*1024);
transaction::exec_tx(pop, [&] {
using namespace pfabric;
auto tInfo = TableInfo("MyTable", {
ColumnInfo("a", ColumnInfo::Int_Type),
ColumnInfo("b", ColumnInfo::Int_Type),
ColumnInfo("c", ColumnInfo::String_Type),
ColumnInfo("d", ColumnInfo::Double_Type)
});
pop.get_root()->pTable = make_persistent<PTableType>(tInfo, BDCCInfo::ColumnBitsMap({{0,4},{3,6}}));
});
} else {
std::cerr << "WARNING: Table already exists" << std::endl;
pop = pool<root>::open(path, LAYOUT);
TEST_CASE("Creating a table with a given schema and inserting data",
"[NVMTable]") {
auto testTable = std::make_shared<TableType>("MyTestTable1");
for (int i = 0; i < 10000; i++) {
auto tp = MyTuple((unsigned long) i, i + 100, fmt::format("String#{}", i), i / 100.0);
testTable->insert(i, tp);
}
auto pTable = pop.get_root()->pTable;
for (unsigned int i = 0; i < 10; i++) {
auto tup = MyTuple(i + 1,
(i + 1) * 100,
fmt::format("String #{0}", i),
i * 12.345);
pTable->insert(i+1, tup);
REQUIRE(testTable->size() == 10000);
for (unsigned long i = 0; i < 10000; i++) {
auto tp = testTable->getByKey(i);
REQUIRE(get<0>(tp) == i);
REQUIRE(get<1>(tp) == (int) (i + 100));
REQUIRE(get<2>(tp) == fmt::format("String#{}", i));
REQUIRE(get<3>(tp) == i / 100.0);
}
auto ptp = pTable->getByKey(5);
/* Clean up */
//transaction::exec_tx(pop, [&] {delete_persistent<PTableType>(q->pTable);});
pop.close();
testTable->drop();
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment