Commit e2d7a4fd authored by Philipp Götze's avatar Philipp Götze
Browse files

Cleaned some implementation details + and some fixes

parent 3f4e53b3
...@@ -28,13 +28,13 @@ option(USE_BOOST_SPIRIT_PARSER ...@@ -28,13 +28,13 @@ option(USE_BOOST_SPIRIT_PARSER
# If switched to off, it will not be downloaded, saving initial building time. # If switched to off, it will not be downloaded, saving initial building time.
option(USE_ROCKSDB_TABLE option(USE_ROCKSDB_TABLE
"use RocksDB for implementing persistent tables" "use RocksDB for implementing persistent tables"
OFF OFF
) )
# Use Non-Volatile Memory Library for implementing tables # Use Non-Volatile Memory Library for implementing tables
option(USE_NVML_TABLE option(USE_NVML_TABLE
"use nvml for implementing persistent memory tables" "use nvml for implementing persistent memory tables"
ON ON
) )
# Build only pipefabric libraries (cep and core) # Build only pipefabric libraries (cep and core)
...@@ -55,7 +55,7 @@ option(BUILD_TEST_CASES ...@@ -55,7 +55,7 @@ option(BUILD_TEST_CASES
#Build google benchmark library #Build google benchmark library
option(BUILD_GOOGLE_BENCH option(BUILD_GOOGLE_BENCH
"build google benchmark" "build google benchmark"
ON OFF
) )
# Build benchmark test # Build benchmark test
...@@ -75,7 +75,7 @@ endif() ...@@ -75,7 +75,7 @@ endif()
#CMAKE_FORCE_CXX_COMPILER(icpc "Intel C++ Compiler") #CMAKE_FORCE_CXX_COMPILER(icpc "Intel C++ Compiler")
# C++ compiler flags # C++ compiler flags
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -Wno-deprecated -g -O3 -Wsign-compare") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -Wno-deprecated -g -O2 -Wsign-compare")
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedefs -Wno-#pragma-messages") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedefs -Wno-#pragma-messages")
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
...@@ -278,37 +278,43 @@ set(core_sources ...@@ -278,37 +278,43 @@ set(core_sources
${THIRD_PARTY_DIR}/catch ${THIRD_PARTY_DIR}/catch
) )
if(USE_ROCKSDB_TABLE) if(BUILD_GOOGLE_BENCH)
set(core_sources set(core_sources
${core_sources} ${core_sources}
${THIRD_PARTY_DIR}/rocksdb ${THIRD_PARTY_DIR}/benchmark
) )
endif() endif()
if(BUILD_GOOGLE_BENCH) if(USE_ROCKSDB_TABLE)
set(core_sources set(core_sources
${core_sources} ${core_sources}
${THIRD_PARTY_DIR}/benchmark ${THIRD_PARTY_DIR}/rocksdb)
) add_library(pfabric_core SHARED
${core_sources})
target_link_libraries(pfabric_core
${BOOST_LIBRARIES}
${ZEROMQ_LIBRARIES}
${ROCKSDB_LIB})
add_executable(rock_test rock_test.cpp)
target_link_libraries(rock_test pfabric_core ${ROCKSDB_LIB})
endif() endif()
if(USE_NVML_TABLE) if(USE_NVML_TABLE)
set(core_sources set(core_sources
${core_sources} ${core_sources}
nvm/PTableInfo.cpp nvm/PTableInfo.cpp
${THIRD_PARTY_DIR}/nvml ${THIRD_PARTY_DIR}/nvml)
) add_library(pfabric_core SHARED
endif()
add_library(pfabric_core SHARED
${core_sources} ${core_sources}
) )
target_link_libraries(pfabric_core
${BOOST_LIBRARIES}
${ZEROMQ_LIBRARIES}
${NVML_LIBRARIES})
target_link_libraries(pfabric_core add_executable(nvm_test nvm_test.cpp)
${BOOST_LIBRARIES} target_link_libraries(nvm_test pfabric_core)
${ZEROMQ_LIBRARIES} endif()
${NVML_LIBRARIES}
)
#----------------------------------------------------------------------------------------- #-----------------------------------------------------------------------------------------
# #
......
...@@ -23,8 +23,6 @@ ...@@ -23,8 +23,6 @@
#include "dsl/PFabricContext.hpp" #include "dsl/PFabricContext.hpp"
#include "dsl/Topology.hpp" #include "dsl/Topology.hpp"
#include "table/BaseTable.hpp"
#include "table/TableInfo.hpp"
using namespace pfabric; using namespace pfabric;
......
...@@ -35,50 +35,57 @@ ...@@ -35,50 +35,57 @@
#include "nvml/include/libpmemobj++/transaction.hpp" #include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmemobj++/utils.hpp" #include "nvml/include/libpmemobj++/utils.hpp"
namespace pfabric { namespace nvm { namespace pfabric {
namespace nvm {
using nvml::obj::allocator; using nvml::obj::allocator;
using nvml::obj::delete_persistent; using nvml::obj::delete_persistent;
using nvml::obj::make_persistent; using nvml::obj::make_persistent;
using nvml::obj::p; using nvml::obj::p;
using nvml::obj::persistent_ptr; using nvml::obj::persistent_ptr;
using nvml::obj::pool_by_vptr; using nvml::obj::pool_by_vptr;
using nvml::obj::transaction; using nvml::obj::transaction;
/**************************************************************************//** /**************************************************************************//**
* \brief Info structure about the BDCC meta data. * \brief Info structure about the BDCC meta data.
* *
* It is used in persistent tables to store the BDCC meta data and statistics. * It is used in persistent tables to store the BDCC meta data and statistics.
*****************************************************************************/ *****************************************************************************/
struct BDCCInfo { class BDCCInfo {
typedef std::unordered_map<uint16_t, uint16_t> ColumnBitsMap; //<mapping from column id to number of bits using pColumnBitsMap = const std::vector<std::pair<uint16_t, uint16_t>, nvml::obj::allocator<std::pair<uint16_t, uint16_t>>>;
typedef std::vector<std::pair<uint16_t,uint16_t>, nvml::obj::allocator<std::pair<uint16_t, uint16_t>>> pColumnBitsMap; public:
using ColumnBitsMap = const std::unordered_map<uint16_t, uint16_t>; //<mapping from column id to number of bits
explicit BDCCInfo(const ColumnBitsMap &_bitMap) :
bitMap(_bitMap.cbegin(), _bitMap.cend()),
numberOfBins(std::accumulate(_bitMap.begin(), _bitMap.end(), 0,
[](const size_t sum, decltype(*_bitMap.begin()) p) { return sum + p.second; })) {}
explicit BDCCInfo(const ColumnBitsMap &_bitMap) { const pColumnBitsMap::const_iterator find(uint16_t item) const {
auto pop = pool_by_vptr(this); for (auto it = bitMap.cbegin(); it != bitMap.cend(); it++) {
transaction::exec_tx(pop, [&] { if (it->first == item) return it;
//bitMap = make_persistent<pColumnBitsMap>(); }
for (const auto& c : _bitMap) { return bitMap.cend();
bitMap.insert(bitMap.cbegin(), std::make_pair(c.first, c.second));
} }
numBins = std::accumulate(_bitMap.begin(), _bitMap.end(), 0,
[](const size_t sum, decltype(*_bitMap.begin()) p) { return sum + p.second; });
});
}
pColumnBitsMap::const_iterator find(uint16_t item) { const size_t numColumns() const {
for (auto it = bitMap.cbegin(); it!= bitMap.cend(); it++) { return bitMap.size();
if (it->first == item) return it; }
}
return bitMap.cend();
}
pColumnBitsMap bitMap; const size_t numBins() const {
p<size_t> numBins; return numberOfBins.get_ro();
//std::map<uint32_t, std::size_t> histogram; }
};/* struct BDCCInfo */ const pColumnBitsMap::const_iterator cend() const {
return bitMap.cend();
}
// private:
const pColumnBitsMap bitMap;
p<const size_t> numberOfBins;
//std::map<uint32_t, std::size_t> histogram;
};/* struct BDCCInfo */
}} /* namespace pfabric::nvm */ }
} /* namespace pfabric::nvm */
#endif /* PTuple_hpp_ */ #endif /* PTuple_hpp_ */
/*
* Copyright (c) 2014-17 The PipeFabric team,
* All Rights Reserved.
*
* This file is part of the PipeFabric package.
*
* PipeFabric is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License (GPL) as
* published by the Free Software Foundation; either version 2 of
* the License, or (at your option) any later version.
*
* This package is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; see the file LICENSE.
* If not you can find the GPL at http://www.gnu.org/copyleft/gpl.html
*/
#ifndef NVM_Block_hpp_
#define NVM_Block_hpp_
#include <array>
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
namespace pfabric {
namespace nvm {
using nvml::obj::persistent_ptr;
using nvml::obj::make_persistent;
using nvml::obj::delete_persistent;
/** Positions in NVM_Block */
const int gDDCRangePos1 = 0;
const int gDDCRangePos2 = 4;
const int gCountPos = 8;
const int gFreeSpacePos = 12;
const int gSmaOffsetPos = 14;
const int gDataOffsetPos = 16;
/** Sizes/Lengths in NVM_Block */
const int gFixedHeaderSize = 14;
const int gDDCValueSize = 4;
const int gAttrOffsetSize = 4;
const int gOffsetSize = 2;
/** The size of a single block in persistent memory */
static constexpr uint16_t gBlockSize = 1 << 15; // 32KB
/**
* \brief This type represents a byte array used for persistent structures.
*
* A BDCC_Block is a PAX oriented data block with the following structure for 32KB:
* <ddc_range><ddc_cnt><sma_offset_0><data_offset_0> ...<sma_offset_n><data_offset_n>
* <sma_min_0><sma_max_0><data_vector_0> ... <sma_min_n><sma_max_n><data_vector_n>
* 0 ddc_range -> long (x2) - 8 Byte
* 8 ddc_cnt -> long - 4 Byte
* 12 free_space -> unsigned short
* for each attribute:
* 14 sma_offset_x -> unsigned short - 2 Byte (depends on block size)
* 16 data_offset_x -> unsigned short
* ...
*
* for each attribute (int, double):
* . sma_min_x -> size of attributes data type
* . sma_max_x -> size of attributes data type
* . data_vector -> size of attributes data type * ddc_cnt
* ...
*
* for each attribute (string - data starts at the end of the minipage):
* . sma_min_offset_x -> unsigned short
* . sma_max_offset_x -> unsigned short
* . data_offset_vector -> unsigned short * ddc_cnt
* . ...
* . data -> size of all strings + ddc_cnt (Nul termination)
*/
using BDCC_Block = typename std::array<uint8_t, gBlockSize>;
template<typename KeyType>
struct DataNode {
using KeyVector = std::array<KeyType, 8192>; // <KeyType, nvml::obj::allocator<KeyType>>;
DataNode() : next(nullptr), block(nullptr) {
keys = make_persistent<KeyVector>();
}
DataNode(BDCC_Block _block) : next(nullptr) {
block = make_persistent<BDCC_Block>(_block);
keys = make_persistent<KeyVector>();
}
persistent_ptr<struct DataNode> next;
persistent_ptr<BDCC_Block> block;
persistent_ptr<KeyVector> keys;
void clear() {
if (next) {
//delete_persistent<struct DataNode>(next);
next = nullptr;
}
if (block) {
delete_persistent<BDCC_Block>(block);
block = nullptr;
}
delete_persistent<struct DataNode>(this);
}
};
} /* end namespace nvm */
} /* end namespace pfabric */
#endif /* NVM_Block_hpp_ */
This diff is collapsed.
This diff is collapsed.
...@@ -14,39 +14,45 @@ using nvml::obj::delete_persistent; ...@@ -14,39 +14,45 @@ using nvml::obj::delete_persistent;
using nvml::obj::make_persistent; using nvml::obj::make_persistent;
using nvml::obj::persistent_ptr; using nvml::obj::persistent_ptr;
namespace pfabric { namespace nvm {
#define SSO_CHARS 15 #define SSO_CHARS 15
#define SSO_SIZE (SSO_CHARS + 1) #define SSO_SIZE (SSO_CHARS + 1)
class PString { class PString {
public: public:
char* data() const { return str ? str.get() : const_cast<char*>(sso); } char *data() const { return str ? str.get() : const_cast<char *>(sso); }
void reset();
void set(std::string* value); void reset();
private:
char sso[SSO_SIZE]; void set(std::string *value);
persistent_ptr<char[]> str;
}; private:
char sso[SSO_SIZE];
void PString::reset() { persistent_ptr<char[]> str;
pmemobj_tx_add_range_direct(sso, 1); };
sso[0] = 0;
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1); inline void PString::reset() {
} pmemobj_tx_add_range_direct(sso, 1);
sso[0] = 0;
void PString::set(std::string* value) { if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
unsigned long length = value->length(); }
if (length <= SSO_CHARS) {
if (str) { inline void PString::set(std::string *value) {
delete_persistent<char[]>(str, strlen(str.get()) + 1); unsigned long length = value->length();
str = nullptr; if (length <= SSO_CHARS) {
if (str) {
delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = nullptr;
}
pmemobj_tx_add_range_direct(sso, SSO_SIZE);
strcpy(sso, value->c_str());
} else {
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = make_persistent<char[]>(length + 1);
strcpy(str.get(), value->c_str());
}
} }
pmemobj_tx_add_range_direct(sso, SSO_SIZE); }} /* namespace pfabric::nvm */
strcpy(sso, value->c_str());
} else {
if (str) delete_persistent<char[]>(str, strlen(str.get()) + 1);
str = make_persistent<char[]>(length + 1);
strcpy(str.get(), value->c_str());
}
}
#endif #endif
This diff is collapsed.
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
#include <vector> #include <vector>
#include "table/TableInfo.hpp" #include "table/TableInfo.hpp"
#include "nvm/PString.hpp"
#include "nvml/include/libpmemobj++/allocator.hpp" #include "nvml/include/libpmemobj++/allocator.hpp"
#include "nvml/include/libpmemobj++/detail/persistent_ptr_base.hpp" #include "nvml/include/libpmemobj++/detail/persistent_ptr_base.hpp"
...@@ -51,20 +52,20 @@ private: ...@@ -51,20 +52,20 @@ private:
p<ColumnInfo::ColumnType> mColType; p<ColumnInfo::ColumnType> mColType;
}; };
typedef std::initializer_list<std::pair<std::string, ColumnInfo::ColumnType>> ColumnInitList; using ColumnInitList = std::initializer_list<std::pair<std::string, ColumnInfo::ColumnType>>;
class PTableInfo { class PTableInfo {
public: public:
typedef persistent_ptr<PTableInfo> PTableInfoPtr; using PTableInfoPtr = persistent_ptr<PTableInfo>;
typedef std::vector<PColumnInfo, nvml::obj::allocator<PColumnInfo>> ColumnVector; using ColumnVector = std::vector<PColumnInfo, nvml::obj::allocator<PColumnInfo>>;
typedef ColumnVector::const_iterator ColumnIterator; using ColumnIterator = ColumnVector::const_iterator;
PTableInfo(){} PTableInfo(){}
PTableInfo(const TableInfo& _tInfo, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) PTableInfo(const TableInfo& _tInfo, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) : mKeyType(keyType){
: mName(_tInfo.tableName().c_str()), mKeyType(keyType){
auto pop = pool_by_vptr(this); auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] { transaction::exec_tx(pop, [&] {
mName.set(const_cast<std::string*>(& _tInfo.tableName()));
mColumns = make_persistent<ColumnVector>(); mColumns = make_persistent<ColumnVector>();
for (const auto &c : _tInfo) for (const auto &c : _tInfo)
mColumns->push_back(PColumnInfo(pop, c.getName(), c.getType())); mColumns->push_back(PColumnInfo(pop, c.getName(), c.getType()));
...@@ -72,10 +73,10 @@ class PTableInfo { ...@@ -72,10 +73,10 @@ class PTableInfo {
} }
PTableInfo(const std::string& name, ColumnInitList columns, PTableInfo(const std::string& name, ColumnInitList columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) : ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) : mKeyType(keyType) {
mName(name.c_str()), mKeyType(keyType) {
auto pop = pool_by_vptr(this); auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] { transaction::exec_tx(pop, [&] {
mName.set(const_cast<std::string*>(&name));
mColumns = make_persistent<ColumnVector>(); mColumns = make_persistent<ColumnVector>();
for (const auto &c : columns) for (const auto &c : columns)
mColumns->push_back(PColumnInfo(pop, c.first, c.second)); mColumns->push_back(PColumnInfo(pop, c.first, c.second));
...@@ -83,15 +84,15 @@ class PTableInfo { ...@@ -83,15 +84,15 @@ class PTableInfo {
} }
PTableInfo(const std::string& name, const ColumnVector& columns, PTableInfo(const std::string& name, const ColumnVector& columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) : ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) : mKeyType(keyType) {
mName(name.c_str()), mKeyType(keyType) {
auto pop = pool_by_vptr(this); auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] { transaction::exec_tx(pop, [&] {
mName.set(const_cast<std::string*>(&name));
mColumns = make_persistent<ColumnVector>(columns.begin(), columns.end()); mColumns = make_persistent<ColumnVector>(columns.begin(), columns.end());
}); });
} }
const std::string tableName() const { return std::string(mName.get()); } const std::string tableName() const { return std::string(mName.data()); }
std::string typeSignature() const; std::string typeSignature() const;
...@@ -117,7 +118,7 @@ class PTableInfo { ...@@ -117,7 +118,7 @@ class PTableInfo {
auto pop = pool_by_vptr(this); auto pop = pool_by_vptr(this);
PTableInfoPtr tInfo_ptr = nullptr; PTableInfoPtr tInfo_ptr = nullptr;
transaction::exec_tx(pop, [&] { transaction::exec_tx(pop, [&] {
tInfo_ptr = make_persistent<PTableInfo>(std::string(mName.get()), *mColumns, mKeyType.get_ro()); tInfo_ptr = make_persistent<PTableInfo>(std::string(mName.data()), *mColumns, mKeyType.get_ro());
}); });
return tInfo_ptr; return tInfo_ptr;
} }
...@@ -125,14 +126,14 @@ class PTableInfo { ...@@ -125,14 +126,14 @@ class PTableInfo {
ColumnIterator begin() const { return mColumns->begin(); } ColumnIterator begin() const { return mColumns->begin(); }
ColumnIterator end() const { return mColumns->end(); } ColumnIterator end() const { return mColumns->end(); }
private: // private:
persistent_ptr<const char[]> mName; PString mName;
persistent_ptr<ColumnVector> mColumns; persistent_ptr<ColumnVector> mColumns;
p<ColumnInfo::ColumnType> mKeyType; p<ColumnInfo::ColumnType> mKeyType;
}; };
} }
typedef persistent_ptr<nvm::PTableInfo> PTableInfoPtr; using PTableInfoPtr = persistent_ptr<nvm::PTableInfo>;
} /* namespace pfabric::nvm */ } /* namespace pfabric::nvm */
......
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include "core/PFabricTypes.hpp" #include "core/PFabricTypes.hpp"
#include "core/Tuple.hpp" #include "core/Tuple.hpp"
#include "nvm/DataNode.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp" #include "nvml/include/libpmemobj++/persistent_ptr.hpp"
...@@ -40,53 +41,6 @@ namespace pfabric { ...@@ -40,53 +41,6 @@ namespace pfabric {
namespace nvm { namespace nvm {
//TODO: Find a more suitable position for these constants
/* Positions in NVM_Block */
const int gDDCRangePos1 = 0;
const int gDDCRangePos2 = 4;
const int gCountPos = 8;
const int gFreeSpacePos = 12;
const int gSmaOffsetPos = 14;
const int gDataOffsetPos = 16;
/* Sizes/Lengths in NVM_Block */
const int gFixedHeaderSize = 14;
const int gDDCValueSize = 4;
const int gAttrOffsetSize = 4;
const int gOffsetSize = 2;
/** The size of a single block in persistent memory */
static constexpr uint16_t gBlockSize = 1 << 15; // 32KB
/**
* \brief This type represents a byte array used for persistent structures.
*
* A NVM_Block is a PAX oriented data block with the following structure for 32KB: