Skip to content
GitLab
Commit 2beb9320 authored by Philipp Götze's avatar Philipp Götze
Browse files

Made BDCC insert work even after 'power loss'

parent 718ebb63
Hide whitespace changes
Inline Side-by-side
src/nvm/BDCCInfo.hpp
View file @ 2beb9320
......@@ -25,24 +25,57 @@
#include <nvm/PTableInfo.hpp>
#include <map>
#include <unordered_map>
#include <algorithm>
#include "nvml/include/libpmemobj++/allocator.hpp"
#include "nvml/include/libpmemobj++/detail/persistent_ptr_base.hpp"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/p.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
#include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmemobj++/utils.hpp"
namespace pfabric { namespace nvm {
using nvml::obj::allocator;
using nvml::obj::delete_persistent;
using nvml::obj::make_persistent;
using nvml::obj::p;
using nvml::obj::persistent_ptr;
using nvml::obj::pool_by_vptr;
using nvml::obj::transaction;
/**************************************************************************//**
* \brief Info structure about the BDCC meta data.
*
* It is used in persistent tables to store the BDCC meta data and statistics.
*****************************************************************************/
struct BDCCInfo {
typedef std::unordered_map<ColumnInfo, uint16_t> ColumnBitsMap;
typedef std::unordered_map<uint16_t, uint16_t> ColumnBitsMap; //<mapping from column id to number of bits
typedef std::vector<std::pair<uint16_t,uint16_t>, nvml::obj::allocator<std::pair<uint16_t, uint16_t>>> pColumnBitsMap;
explicit BDCCInfo(const ColumnBitsMap &_bitMap) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
//bitMap = make_persistent<pColumnBitsMap>();
for (const auto& c : _bitMap) {
bitMap.insert(bitMap.cbegin(), std::make_pair(c.first, c.second));
}
numBins = std::accumulate(_bitMap.begin(), _bitMap.end(), 0,
[](const size_t sum, decltype(*_bitMap.begin()) p) { return sum + p.second; });
});
}
BDCCInfo(ColumnBitsMap _bitMap) : bitMap(_bitMap),
numBins(std::accumulate(_bitMap.begin(), _bitMap.end(), 0,
[](const size_t sum, decltype(*_bitMap.begin()) p) { return sum + p.second; })) {}
pColumnBitsMap::const_iterator find(uint16_t item) {
for (auto it = bitMap.cbegin(); it!= bitMap.cend(); it++) {
if (it->first == item) return it;
}
return bitMap.cend();
}
const ColumnBitsMap bitMap;
const size_t numBins;
std::map<uint32_t, std::size_t> histogram;
pColumnBitsMap bitMap;
p<size_t> numBins;
//std::map<uint32_t, std::size_t> histogram;
};/* struct BDCCInfo */
......
src/nvm/PTableInfo.hpp
View file @ 2beb9320
#ifndef PTableInfo_hpp_
#define PTableInfo_hpp_
#include <cstddef>
#include <cstring>
#include <initializer_list>
#include <iostream>
#include <memory>
#include <string>
#include <array>
#include <utility>
#include <vector>
#include "nvml/include/libpmemobj++/allocator.hpp"
#include "nvml/include/libpmemobj++/detail/persistent_ptr_base.hpp"
#include "nvml/include/libpmemobj++/make_persistent.hpp"
#include "nvml/include/libpmemobj++/make_persistent_array.hpp"
#include "nvml/include/libpmemobj++/p.hpp"
#include "nvml/include/libpmemobj++/pool.hpp"
#include "nvml/include/libpmemobj++/persistent_ptr.hpp"
#include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmemobj++/utils.hpp"
......@@ -19,6 +23,7 @@ using nvml::obj::delete_persistent;
using nvml::obj::make_persistent;
using nvml::obj::p;
using nvml::obj::persistent_ptr;
using nvml::obj::pool_base;
using nvml::obj::pool_by_vptr;
using nvml::obj::transaction;
......@@ -28,9 +33,14 @@ class ColumnInfo {
public:
enum ColumnType { Void_Type, Int_Type, Double_Type, String_Type };
ColumnInfo() : mColName(""), mColType(Void_Type) {}
ColumnInfo(pool_base pop) : ColumnInfo(pop, "", ColumnType::Void_Type) {}
ColumnInfo(const std::string& n, ColumnType ct) : mColName(n.c_str()), mColType(ct) {}
ColumnInfo(pool_base pop, const std::string& n, ColumnType ct) : mColType(ct) {
transaction::exec_tx(pop, [&] {
mColName = make_persistent<char[]>(n.length() +1);
strcpy(mColName.get(), n.c_str());
});
}
const std::string getName() const { return mColName.get(); }
......@@ -45,10 +55,12 @@ public:
}
private:
persistent_ptr<const char[]> mColName;
persistent_ptr<char[]> mColName;
p<ColumnType> mColType;
};
typedef std::initializer_list<std::pair<std::string, ColumnInfo::ColumnType>> ColumnInitList;
/** TODO: For later, template implementation
template< bool isFixedSize >
struct IsFixedSize {
......@@ -97,12 +109,14 @@ class PTableInfo {
PTableInfo(const std::string& name, ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type)
: mName(name.c_str()), mKeyType(keyType) {}
PTableInfo(const std::string& name, std::initializer_list<ColumnInfo> columns,
PTableInfo(const std::string& name, ColumnInitList columns,
ColumnInfo::ColumnType keyType = ColumnInfo::Void_Type) :
mName(name.c_str()), mKeyType(keyType) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {
mColumns = make_persistent<ColumnVector>(columns);
mColumns = make_persistent<ColumnVector>();
for (const auto &c : columns)
mColumns->push_back(ColumnInfo(pop, c.first, c.second));
});
}
......
src/nvm/persistent_table.hpp
View file @ 2beb9320
......@@ -52,6 +52,8 @@
#include "nvml/include/libpmemobj++/transaction.hpp"
#include "nvml/include/libpmemobj++/utils.hpp"
namespace pfabric { namespace nvm {
using nvml::obj::pool;
using nvml::obj::pool_by_vptr;
using nvml::obj::pool_base;
......@@ -61,7 +63,7 @@ using nvml::obj::make_persistent;
using nvml::obj::delete_persistent;
using nvml::obj::transaction;
namespace pfabric { namespace nvm {
const std::string LAYOUT = "PTable";
/**************************************************************************//**
* \brief A persistent table used for PMEM technologies or emulations.
......@@ -73,30 +75,31 @@ class persistent_table {
public:
typedef persistent_ptr<Tuple> RecordType;
typedef std::unordered_map<ColumnInfo, uint16_t> ColumnIntMap;
typedef std::unordered_map<uint16_t, uint16_t> ColumnIntMap;
/************************************************************************//**
* \brief Default Constructor.
***************************************************************************/
persistent_table() {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {init(TableInfo(), ColumnIntMap());});
transaction::exec_tx(pop, [&] {init("", {}, ColumnIntMap());});
}
/************************************************************************//**
* \brief Constructor for a given schema (TableInfo).
***************************************************************************/
persistent_table(const TableInfo &_tInfo) {
persistent_table(const std::string& tName, ColumnInitList columns) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {init(_tInfo, ColumnIntMap());});
transaction::exec_tx(pop, [&] {init(tName, columns, ColumnIntMap());});
}
/************************************************************************//**
* \brief Constructor for a given schema and dimension clustering.
***************************************************************************/
persistent_table(const TableInfo &_tInfo, const ColumnIntMap& _bdccInfo) {
persistent_table(const std::string& tName, ColumnInitList columns, const ColumnIntMap& _bdccInfo) {
auto pop = pool_by_vptr(this);
transaction::exec_tx(pop, [&] {init(_tInfo, _bdccInfo);});
transaction::exec_tx(pop, [&] {init(tName, columns, _bdccInfo);});
}
/************************************************************************//**
......@@ -256,10 +259,10 @@ public:
void print(bool raw = false) {
auto dest_block = root->block_list;
auto tInfo = root->tInfo;
auto& tInfo = *root->tInfo;
size_t colCnt = 0;
for (auto &c : *tInfo)
for (auto &c : tInfo)
colCnt++;
do {
......@@ -297,7 +300,7 @@ public:
/* Body/Column/Minipage data */
if (cnt > 0) {
size_t idx = 0;
for (auto &c : *tInfo) {
for (auto &c : tInfo) {
std::cout << "Column Info: " << c.getName() << ": " << c.getType() << std::endl;
const auto& sma_pos = reinterpret_cast<const uint16_t&>(b->at(nvm::gSmaOffsetPos + idx * nvm::gAttrOffsetSize));
const auto& data_pos = reinterpret_cast<const uint16_t&>(b->at(nvm::gDataOffsetPos + idx * nvm::gAttrOffsetSize));
......@@ -438,8 +441,9 @@ private:
size_t portions = 0;
ColumnIntMap mp_sizes = ColumnIntMap();
/* Get the sum of all column portitions */
for (auto &c : *root->tInfo) {
if (customizations.find(c) == customizations.end()) {
for (auto i = 0u; i < (*root->tInfo).numColumns(); i++) {
const auto &c = (*root->tInfo).columnInfo(i);
if (customizations.find(i) == customizations.end()) {
switch (c.getType()) {
case ColumnInfo::Int_Type:
portions += 1;
......@@ -455,27 +459,28 @@ private:
break;
}
} else
portions += customizations[c];
portions += customizations[i];
}
/* Calculate and save the minipage sizes for all columns */
for (auto &c : *root->tInfo) {
if (customizations.find(c) == customizations.end()) {
for (auto i = 0u; i < (*root->tInfo).numColumns(); i++) {
const auto &c = (*root->tInfo).columnInfo(i);
if (customizations.find(i) == customizations.end()) {
switch (c.getType()) {
case ColumnInfo::Int_Type:
mp_sizes[c] = 1 * totalSize / portions;
mp_sizes[i] = 1 * totalSize / portions;
break;
case ColumnInfo::Double_Type:
mp_sizes[c] = 2 * totalSize / portions;
mp_sizes[i] = 2 * totalSize / portions;
break;
case ColumnInfo::String_Type:
mp_sizes[c] = 5 * totalSize / portions;
mp_sizes[i] = 5 * totalSize / portions;
break;
default:
throw TableException("unsupported column type\n");
break;
}
} else
mp_sizes[c] = customizations[c] * totalSize / portions;
mp_sizes[i] = customizations[i] * totalSize / portions;
}
return mp_sizes;
}
......@@ -487,15 +492,14 @@ private:
* \param[in] _tInfo
* the underlying schema to use
***************************************************************************/
void init(const TableInfo& _tInfo, const ColumnIntMap& _bdccInfo) {
void init(const std::string& _tName, const ColumnInitList& _columns, const ColumnIntMap& _bdccInfo) {
this->root = make_persistent<struct root>();
this->root->tInfo = make_persistent<TableInfo>(_tInfo);
this->root->bdccInfo = make_persistent<BDCCInfo>(BDCCInfo(_bdccInfo));
this->root->tInfo = make_persistent<TableInfo>(_tName, _columns);
this->root->bdccInfo = make_persistent<BDCCInfo>(_bdccInfo);
this->root->index = make_persistent<examples::ctree_map_p<KeyType, persistent_ptr<nvm::PTuple<Tuple>>>>();
this->root->block_list = make_persistent<struct nvm_block>();
auto block_ptr = make_persistent<nvm::NVM_Block>(initBlock(0, ((1L << this->root->bdccInfo->numBins) - 1)));
this->root->block_list->block = block_ptr;
}
/************************************************************************//**
......@@ -527,7 +531,8 @@ private:
size_t idx = 0;
size_t sma_size = 0;
uint16_t current_offset = header_size;
for (auto &c : *this->root->tInfo) {
for (auto i = 0u; i < (*this->root->tInfo).numColumns(); i++) {
const auto &c = (*this->root->tInfo).columnInfo(i);
uint16_t sma_offset;
uint16_t data_offset;
switch (c.getType()) {
......@@ -535,21 +540,21 @@ private:
sma_offset = current_offset;
data_offset = current_offset + 2 * sizeof(uint32_t);
sma_size += 2 * sizeof(uint32_t);
current_offset += sizes[c];
current_offset += sizes[i];
}
break;
case ColumnInfo::Double_Type: {
sma_offset = current_offset;
data_offset = current_offset + 2 * sizeof(uint64_t);
sma_size += 2 * sizeof(uint64_t);
current_offset += sizes[c];
current_offset += sizes[i];
}
break;
case ColumnInfo::String_Type: {
sma_offset = current_offset;
data_offset = current_offset + nvm::gAttrOffsetSize;
sma_size += nvm::gAttrOffsetSize;
current_offset += sizes[c];
current_offset += sizes[i];
}
break;
default: {
......@@ -934,17 +939,19 @@ private:
std::bitset<32> getBDCCFromTuple (Tuple tp) {
//TODO: Just use integers and shift operations instead of bitset
auto bdccInfo = *this->root->bdccInfo;
auto tInfo = *this->root->tInfo;
const auto& bdccInfo = this->root->bdccInfo;
const auto& tInfo = *this->root->tInfo;
auto colCnt = 0; // total column counter
auto dimCnt = 0; // dimension column counter
auto bdccSize = 0; // sum of bits used for bdcc
std::bitset<32> xtr = 0; // extracted bdcc value
std::pair<int, std::bitset<32>> dimsBDCC[bdccInfo.bitMap.size()]; // (no. of bits, binned value)
std::pair<int, std::bitset<32>> dimsBDCC[bdccInfo->bitMap.size()]; // (no. of bits, binned value)
for (const auto &c : tInfo) {
if (bdccInfo.bitMap.find(c) != bdccInfo.bitMap.end()) {
auto nBits = bdccInfo.bitMap.at(c);
for (auto i = 0u; i < tInfo.numColumns(); i++) {
if(bdccInfo->find(i) != bdccInfo->bitMap.cend()) {
//if (bdccInfo->bitMap.find(i) != bdccInfo->bitMap.end()) {
auto nBits = bdccInfo->find(i)->second; //bitMap.at(i);
//get value
auto value = ns_types::dynamic_get(colCnt, tp);
//get bin
......@@ -966,7 +973,7 @@ private:
/* Round robin the bins */
while (bdccSize > 0) {
for (std::size_t k = 0; k < bdccInfo.bitMap.size(); ++k) {
for (auto k = 0u; k < bdccInfo->bitMap.size(); ++k) {
if (dimsBDCC[k].first > 0) {
xtr[bdccSize - 1] = dimsBDCC[k].second[dimsBDCC[k].first - 1];
bdccSize--;
......
src/table/NVMTable.hpp
View file @ 2beb9320
......@@ -47,8 +47,6 @@
#include "nvml/include/libpmemobj++/pool.hpp"
#include "nvml/include/libpmemobj++/transaction.hpp"
#define LAYOUT "NVMTable"
namespace pfabric {
using nvml::obj::delete_persistent;
......@@ -398,12 +396,12 @@ private:
std::string path = tableInfo.tableName() + ".db";
if (access(path.c_str(), F_OK) != 0)
{
pop = pool<root>::create(path, LAYOUT);//, (size_t)blockSize, 0666);
pop = pool<root>::create(path, nvm::LAYOUT);//, (size_t)blockSize, 0666);
//throw TableException("failed to create pool\n");
}
else
{
pop = pool<root>::open(path, LAYOUT);
pop = pool<root>::open(path, nvm::LAYOUT);
//throw TableException("failed to open pool\n");
}
q = pop.get_root();
......
src/test/persistent_tableTest.cpp
View file @ 2beb9320
......@@ -39,12 +39,12 @@ TEST_CASE("Testing storing tuples in persistent_table", "[persistent_table]") {
std::vector<typename std::chrono::duration<int64_t,micro>::rep> measures;
pool<root> pop;
const std::string path = "/tmp/testdb.db";
const std::string path = "/mnt/pmem/tests/testdb.db";
std::remove(path.c_str());
//std::remove(path.c_str());
if (access(path.c_str(), F_OK) != 0) {
pop = pool<root>::create(path, LAYOUT);
pop = pool<root>::create(path, LAYOUT, 16*1024*1024);
} else {
pop = pool<root>::open(path, LAYOUT);
}
......@@ -52,25 +52,33 @@ TEST_CASE("Testing storing tuples in persistent_table", "[persistent_table]") {
auto q = pop.get_root();
if (!q->pTable) {
transaction::exec_tx(pop, [&] {
auto tInfo = make_persistent<pfabric::TableInfo, std::string, std::initializer_list<ColumnInfo>>("MyTable", {
/*auto tInfo = make_persistent<pfabric::TableInfo, std::string, std::initializer_list<ColumnInfo>>("MyTable", {
ColumnInfo("a", ColumnInfo::Int_Type),
ColumnInfo("b", ColumnInfo::Int_Type),
ColumnInfo("c", ColumnInfo::String_Type),
ColumnInfo("d", ColumnInfo::Double_Type)
});
auto dimInfo = BDCCInfo::ColumnBitsMap( {
});*/
/*auto dimInfo = BDCCInfo::ColumnBitsMap( {
{ ColumnInfo("a", ColumnInfo::Int_Type), 4},
{ ColumnInfo("d", ColumnInfo::Double_Type), 6}
});
q->pTable = make_persistent<pTable_type>(*tInfo, dimInfo);
//TODO: Rewrite constuctor
});*/
q->pTable = make_persistent<pTable_type, std::string, ColumnInitList, BDCCInfo::ColumnBitsMap> (
"MyTable",
{
std::make_pair("a", ColumnInfo::Int_Type),
std::make_pair("b", ColumnInfo::Int_Type),
std::make_pair("c", ColumnInfo::String_Type),
std::make_pair("d", ColumnInfo::Double_Type)
},
BDCCInfo::ColumnBitsMap({{0,4},{3,6}})
);
});
} else {
std::cerr << "WARNING: Table already exists" << std::endl;
}
for (unsigned int i = 0; i < 500; i++) {
for (unsigned int i = 0; i < 10; i++) {
auto tup = MyTuple(i + 1,
(i + 1) * 100,
fmt::format("String #{0}", i),
......@@ -95,6 +103,6 @@ TEST_CASE("Testing storing tuples in persistent_table", "[persistent_table]") {
/* Clean up */
transaction::exec_tx(pop, [&] {delete_persistent<pTable_type>(q->pTable);});
//transaction::exec_tx(pop, [&] {delete_persistent<pTable_type>(q->pTable);});
pop.close();
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment