Merge branch 'master' into more_data_structures

.gitlab-ci.yml

@@ -26,7 +26,7 @@ test:
   tags:
     - grouprunner
   script:
-    - echo pass | sudo -S mkdir -m 777 -p /mnt/pmem0/test
+    - mkdir -m 777 -p /mnt/pmem0/test
     - cd build
     - ctest
   dependencies:
@@ -43,7 +43,7 @@ coverage:
     - cd build
     - cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-g -O0 -fprofile-arcs -ftest-coverage" ..
     - make -j 3
-    - echo pass | sudo -S mkdir -m 777 -p /mnt/pmem0/test
+    - mkdir -m 777 -p /mnt/pmem0/test
     - ctest
     - ~/.local/bin/gcovr -r .. --exclude _deps
   coverage: '/^TOTAL.*\s+(\d+\%)$/'

CMakeLists.txt

@@ -17,8 +17,9 @@ endif()
 
 option(ENABLE_LOG         "enables log output for e.g. debugging"        OFF)
 option(BUILD_TEST_CASES   "build tests for functionality of structures"  ON )
-option(BUILD_GOOGLE_BENCH "build google benchmark"                       OFF)
+option(BUILD_GOOGLE_BENCH "build google benchmark"                       ON )
 option(BUILD_BENCHMARKS   "build benchmarks for structures"              ON )
+option(ENABLE_PCM         "enables performance counter monitor"          OFF)
 
 ################################################################################
 # End of customization section                                                 #
@@ -27,6 +28,9 @@ option(BUILD_BENCHMARKS   "build benchmarks for structures"              ON )
 if(ENABLE_LOG)
   add_definitions("-DENABLE_LOG")
 endif()
+if(ENABLE_PCM)
+  add_definitions("-DENABLE_PCM")
+endif()
 
 # Benchmark test requires benchmark library
 if (BUILD_BENCHMARKS)
@@ -102,7 +106,7 @@ add_subdirectory(src/pbptrees) # Persistent versions of B⁺-Tree, contains also
                                # - Remake of the wB+Tree
 add_subdirectory(src/ptable)  # BDCC-based analytical table structure
 
-add_subdirectory(src/pskiplists) 
+add_subdirectory(src/pskiplists)
 #########################
 # Unit test using Catch #
 #########################

README.md

-NVM-based Data Strucutres
+PMem-based Data Strucutres
 =========================
 
 [![pipeline status](https://dbgit.prakinf.tu-ilmenau.de/code/nvm-based_data_structures/badges/master/pipeline.svg)](https://dbgit.prakinf.tu-ilmenau.de/code/nvm-based_data_structures/commits/master)
 [![coverage report](https://dbgit.prakinf.tu-ilmenau.de/code/nvm-based_data_structures/badges/master/coverage.svg?job=coverage)](https://dbgit.prakinf.tu-ilmenau.de/code/nvm-based_data_structures/commits/master)
 
-This is a repository for persistent data structures based on non-volatile memory (NVM).
+This is a repository for persistent data structures based on persistent memory (PMem).
 
 :sparkles: TODOs
 ----------------
@@ -13,8 +13,8 @@ This is a repository for persistent data structures based on non-volatile memory
 - [x] ~~PTable (BDCC + NVM)~~
 - [x] ~~Add Skip-Lists (Alex)~~
 - [ ] Add LSM Trees (Arun)
-- [ ] Add Tries (Leret)
-- [ ] Data structure common benchmark
+- [x] ~~Add Tries (Leret)~~
+- [ ] ~~Data structure common benchmark~~
 - [ ] Documentation
 
 :copyright: License

bench/trees/common.hpp

@@ -56,6 +56,7 @@ const std::string path = dbis::gPmemPath + "tree_bench.data";
 constexpr auto POOL_SIZE = 1024 * 1024 * 1024 * 4ull;  //< 4GB
 constexpr auto LAYOUT = "Tree";
 constexpr auto NODE_PTR_SIZE = 24;  ///< 24 Byte (8-tag + 16-pptr)
+constexpr auto L3 = 14080 * 1024;
 
 /* wBPTree pre-calculations */
 template <unsigned int KEYS>
@@ -239,7 +240,7 @@ class HybridWrapper {
   inline void recover(T &tree) const { recover(tree, Int2Type<isHybrid>()); }
 };
 
-auto hybridWrapperPtr = new HybridWrapper<TreeType, IS_HYBRID>();
+auto hybridWrapperPtr = std::make_unique<HybridWrapper<TreeType, IS_HYBRID>>();
 auto &hybridWrapper = *hybridWrapperPtr;
 
 /*=== Insert Function ===*/

bench/trees/tree_balance.cpp

@@ -17,8 +17,13 @@
 
 #include "common.hpp"
 #include "utils/PersistEmulation.hpp"
+#ifdef ENABLE_PCM
+#include <cpucounters.h>
+#endif
 
-constexpr auto L3 = 30 * 1024 * 1024;
+uint64_t pmmwrites = 0;
+uint64_t pmmreads = 0;
+uint64_t modBytes = 0;  ///< modified Bytes
 constexpr auto ArraySize = L3 / TARGET_LEAF_SIZE;
 using ArrayType = pmem::obj::array<pptr<TreeType::LeafNode>, ArraySize>;
 
@@ -40,6 +45,18 @@ void prepare(const pptr<TreeType> &tree, const pptr<TreeType::LeafNode> &leaf2);
 
 /* Get benchmarks on Tree */
 static void BM_TreeBalance(benchmark::State &state) {
+#ifdef ENABLE_PCM
+  PCM *pcm_ = PCM::getInstance();
+  auto s = pcm_->program();
+  if (s != PCM::Success) {
+    std::cerr << "Error creating PCM instance: " << s << std::endl;
+    if (s == PCM::PMUBusy)
+      pcm_->resetPMU();
+    else
+      exit(0);
+  }
+#endif
+
   std::cout << "BRANCHKEYS: " << BRANCHKEYS << "\nLEAFKEYS: " << LEAFKEYS << "\n";
 
   pool<root> pop;
@@ -73,25 +90,23 @@ static void BM_TreeBalance(benchmark::State &state) {
   leaf2 = root->receiver;
   pop.drain();
 
-  /* BENCHMARKING */
-  for (auto _ : state) {
-    state.PauseTiming();
-    std::cout.setstate(std::ios_base::failbit);
-    const auto leafNodePos = std::rand() % ArraySize;
-    auto donorNode = (*donorArray)[leafNodePos];
-    auto receiverNode = (*receiverArray)[leafNodePos];
+#ifdef ENABLE_PCM
+  SocketCounterState before_sstate;
+  SocketCounterState after_sstate;
+#endif
+
+  /// Lambda function for measured part (needed twice)
+  auto benchmark = [&pop, &treeRef](
+      pptr<TreeType::LeafNode> &donorNode,
+      pptr<TreeType::LeafNode> &receiverNode) {
     auto &donorRef = *donorNode;
     auto &receiverRef = *receiverNode;
-    dbis::PersistEmulation::getBytesWritten();
-    state.ResumeTiming();
 
     benchmark::DoNotOptimize(*donorNode);
     benchmark::DoNotOptimize(*receiverNode);
     // transaction::run(pop, [&] {
     treeRef.balanceLeafNodes(donorNode, receiverNode);
     // });
-    benchmark::DoNotOptimize(*donorNode);
-    benchmark::DoNotOptimize(*receiverNode);
 
     constexpr auto Balanced = ((LEAFKEYS - 1) / 2 + LEAFKEYS) / 2;
     constexpr auto SortedMoves = (LEAFKEYS - 1) / 2 + LEAFKEYS - Balanced;
@@ -100,45 +115,93 @@ static void BM_TreeBalance(benchmark::State &state) {
     // pop.flush(donorRef.bits);
     // pop.flush(&donorRef.slot.get_ro(),
     //           sizeof(donorRef.slot.get_ro()) + sizeof(donorRef.bits.get_ro()));
-    // pop.flush(&donorRef.keys.get_ro(), sizeof(MyKey) * UnsortedMoves);
-    // pop.flush(&donorRef.values.get_ro(), sizeof(MyTuple) * UnsortedMoves);
+    // pop.flush(&donorRef.keys.get_ro(), sizeof(MyKey) * SortedMoves);
+    // pop.flush(&donorRef.values.get_ro(), sizeof(MyTuple) * SortedMoves);
     // pop.flush(&donorRef.keys.get_ro()[(LEAFKEYS - 1) / 2], sizeof(MyKey) * UnsortedMoves);
     // pop.flush(&donorRef.values.get_ro()[(LEAFKEYS - 1) / 2], sizeof(MyTuple) * UnsortedMoves);
+    // donorNode.flush(pop);
 
     pop.flush(receiverRef.numKeys);
     // pop.flush(receiverRef.bits);
     // pop.flush(&receiverRef.bits.get_ro(),
-    //           sizeof(receiverRef.bits.get_ro()) + sizeof(receiverRef.fp.get_ro()));
+    //          sizeof(receiverRef.bits.get_ro()) + sizeof(receiverRef.fp.get_ro()));
     // pop.flush(&receiverRef.slot.get_ro(),
     //           sizeof(receiverRef.slot.get_ro()) + sizeof(receiverRef.bits.get_ro()));
-
-    pop.flush(&receiverRef.keys.get_ro()[(LEAFKEYS - 1) / 2], sizeof(MyKey) * SortedMoves);
-    pop.flush(&receiverRef.values.get_ro()[(LEAFKEYS - 1) / 2], sizeof(MyTuple) * SortedMoves);
+    pop.flush(&receiverRef.keys.get_ro(), sizeof(MyKey) * SortedMoves);
+    pop.flush(&receiverRef.values.get_ro(), sizeof(MyTuple) * SortedMoves);
     // pop.flush(&receiverRef.keys.get_ro()[(LEAFKEYS - 1) / 2], sizeof(MyKey) * UnsortedMoves);
     // pop.flush(&receiverRef.values.get_ro()[(LEAFKEYS - 1) / 2], sizeof(MyTuple) * UnsortedMoves);
     // receiverNode.flush(pop);
     pop.drain();
 
-    state.PauseTiming();
+    benchmark::DoNotOptimize(*donorNode);
+    benchmark::DoNotOptimize(*receiverNode);
+    benchmark::ClobberMemory();
+  };
+
+  /// BENCHMARK Writes
+  if (pmmwrites == 0) {
+    auto donorNode = (*donorArray)[0];
+    auto receiverNode = (*receiverArray)[0];
+
+    dbis::PersistEmulation::getBytesWritten();
+#ifdef ENABLE_PCM
+    before_sstate = getSocketCounterState(1);
+#endif
+    benchmark(donorNode, receiverNode);
+#ifdef ENABLE_PCM
+    after_sstate = getSocketCounterState(1);
+    pmmreads = getBytesReadFromPMM(before_sstate, after_sstate);
+    pmmwrites = getBytesWrittenToPMM(before_sstate, after_sstate);
+#endif
+    modBytes = dbis::PersistEmulation::getBytesWritten();
     *donorNode = *leaf;      ///< reset the modified node
     *receiverNode = *leaf2;  ///< reset the modified node
-    state.ResumeTiming();
+    donorNode.flush(pop);
+    receiverNode.flush(pop);
+    pop.drain();
+  }
+
+  /// BENCHMARKING
+  std::cout.setstate(std::ios_base::failbit);
+  std::srand(std::time(nullptr));
+  for (auto _ : state) {
+    const auto leafNodePos = std::rand() % ArraySize;
+    auto donorNode = (*donorArray)[leafNodePos];
+    auto receiverNode = (*receiverArray)[leafNodePos];
+    auto &donorRef = *donorNode;
+    auto &receiverRef = *receiverNode;
+
+    auto start = std::chrono::high_resolution_clock::now();
+    benchmark(donorNode, receiverNode);
+    auto end = std::chrono::high_resolution_clock::now();
+    // treeRef.printLeafNode(0, donorNode);
+    // treeRef.printLeafNode(0, receiverNode);
+
+    donorRef = *leaf;      ///< reset the modified node
+    receiverRef = *leaf2;  ///< reset the modified node
+    donorNode.flush(pop);
+    receiverNode.flush(pop);
+    pop.drain();
+    auto elapsed_seconds =
+      std::chrono::duration_cast<std::chrono::duration<double>>(end - start);
+    state.SetIterationTime(elapsed_seconds.count());
   }
 
-  // treeRef.printLeafNode(0, leaf);
   std::cout.clear();
-  std::cout << "Writes:" << dbis::PersistEmulation::getBytesWritten() << '\n';
+  // treeRef.printLeafNode(0, leaf);
+  // treeRef.printLeafNode(0, leaf2);
+  std::cout << "Iterations:" << state.iterations() << '\n';
+  std::cout << "PMM Reads:" << pmmreads << '\n';
+  std::cout << "Writes:" << modBytes << '\n';
+  std::cout << "PMM Writes:" << pmmwrites << '\n';
   std::cout << "Elements:" << ELEMENTS << '\n';
-  transaction::run(pop, [&] {
-    delete_persistent<TreeType>(tree);
-    delete_persistent<ArrayType>(donorArray);
-    delete_persistent<ArrayType>(receiverArray);
-  });
+
   pop.close();
   pmempool_rm(path.c_str(), 0);
 }
 
-BENCHMARK(BM_TreeBalance);
+BENCHMARK(BM_TreeBalance)->UseManualTime();
 BENCHMARK_MAIN();
 
 void prepare(pmem::obj::pool<root> &pop, pobj_alloc_class_desc alloc_class) {

bench/trees/tree_erase.cpp

@@ -18,11 +18,31 @@
 #include <libpmemobj++/container/array.hpp>
 #include "common.hpp"
 #include "utils/PersistEmulation.hpp"
+#ifdef ENABLE_PCM
+#include <cpucounters.h>
+#endif
+
+uint64_t pmmwrites = 0;
+uint64_t pmmreads = 0;
+uint64_t modBytes = 0;  ///< modified Bytes
+constexpr auto ArraySize = 4 * L3 / 256;//TARGET_LEAF_SIZE;
 
 void prepare(const persistent_ptr<TreeType> tree);
 
 /* Get benchmarks on Tree */
 static void BM_TreeErase(benchmark::State &state) {
+    #ifdef ENABLE_PCM
+    PCM *pcm_ = PCM::getInstance();
+    auto s = pcm_->program();
+    if (s != PCM::Success) {
+      std::cerr << "Error creating PCM instance: " << s << std::endl;
+      if (s == PCM::PMUBusy)
+        pcm_->resetPMU();
+      else
+        exit(0);
+    }
+  #endif
+
   std::cout << "BRANCHKEYS: " << BRANCHKEYS << " -> " << sizeof(TreeType::BranchNode)
             << "\nLEAFKEYS: " << LEAFKEYS << " -> " << sizeof(TreeType::LeafNode) << "\n";
 
@@ -47,12 +67,12 @@ static void BM_TreeErase(benchmark::State &state) {
     transaction::run(pop, [&] { pop.root()->tree = make_persistent<TreeType>(alloc_class); });
   }
 
-  const auto tree = pop.root()->tree;
+  auto tree = pop.root()->tree;
   prepare(tree);
   auto &treeRef = *tree;
+  tree.flush(pop);
   auto leaf = treeRef.rootNode.leaf;
-  constexpr auto L3 = 30 * 1024 * 1024;
-  constexpr auto ArraySize = L3 / TARGET_LEAF_SIZE;
+  leaf.flush(pop);
 
   pptr<pmem::obj::array<pptr<TreeType::LeafNode>, ArraySize>> leafArray;
   transaction::run(pop, [&] {
@@ -65,17 +85,17 @@ static void BM_TreeErase(benchmark::State &state) {
   });
   pop.drain();
 
-  /* BENCHMARKING */
-  for (auto _ : state) {
-    state.PauseTiming();
-    std::cout.setstate(std::ios_base::failbit);
-    const auto leafNodePos = std::rand() % ArraySize;
-    auto leafNode = (*leafArray)[leafNodePos];
-    const auto pos = treeRef.lookupPositionInLeafNode(leafNode, KEYPOS);
-    // const auto pos = dbis::BitOperations::getFreeZero(leafNode->bits.get_ro());
+  const auto pos = treeRef.lookupPositionInLeafNode(leaf, KEYPOS);
+  // const auto pos = dbis::BitOperations::getFreeZero(leaf->bits.get_ro());
+
+#ifdef ENABLE_PCM
+  SocketCounterState before_sstate;
+  SocketCounterState after_sstate;
+#endif
+
+  /// Lambda function for measured part (needed twice)
+  auto benchmark = [&pop, &treeRef, &pos](pptr<TreeType::LeafNode> &leafNode) {
     auto &leafRef = *leafNode;
-    dbis::PersistEmulation::getBytesWritten();
-    state.ResumeTiming();
     benchmark::DoNotOptimize(*leafNode);
 
     // transaction::run(pop, [&] {
@@ -86,22 +106,69 @@ static void BM_TreeErase(benchmark::State &state) {
     // pop.flush(leafRef.bits);
     // pop.flush(&leafRef.slot.get_ro(),
     //           sizeof(leafRef.slot.get_ro()) + sizeof(leafRef.bits.get_ro()));
+    // pop.flush(&leafRef.bits.get_ro(),
+    //           sizeof(leafRef.bits.get_ro()) + sizeof(leafRef.fp.get_ro()));
     // pop.flush(&leafRef.keys.get_ro()[pos], sizeof(MyKey));
     // pop.flush(&leafRef.values.get_ro()[pos], sizeof(MyTuple));
     pop.flush(&leafRef.keys.get_ro()[pos], (LEAFKEYS - 1 - pos) * sizeof(MyKey));
     pop.flush(&leafRef.values.get_ro()[pos], (LEAFKEYS - 1 - pos) * sizeof(MyTuple));
     pop.drain();
-    // leaf.persist(pop);
+    // leafNode.persist(pop);
+
+    benchmark::DoNotOptimize(*leafNode);
+    benchmark::ClobberMemory();
+  };
+
+ /// BENCHMARK Writes
+  if (pmmwrites == 0) {
+    auto &leafNode = (*leafArray)[0];
+    // auto &leafRef = *leafNode;
 
-    benchmark::DoNotOptimize(*leaf);
-    state.PauseTiming();
+    dbis::PersistEmulation::getBytesWritten();
+#ifdef ENABLE_PCM
+    before_sstate = getSocketCounterState(1);
+#endif
+    benchmark(leafNode);
+#ifdef ENABLE_PCM
+    after_sstate = getSocketCounterState(1);
+    pmmreads = getBytesReadFromPMM(before_sstate, after_sstate);
+    pmmwrites = getBytesWrittenToPMM(before_sstate, after_sstate);
+#endif
+    modBytes = dbis::PersistEmulation::getBytesWritten();
     *leafNode = *leaf;  ///< reset the modified node
-    state.ResumeTiming();
+    leafNode.flush(pop);
+    // pop.flush((char *)((uintptr_t)&*leafNode - 64), 64); //< flush header
+    pop.drain();
+  }
+
+  /// BENCHMARK Timing
+  /// avoid output to stdout during benchmark
+  std::cout.setstate(std::ios_base::failbit);
+  std::srand(std::time(nullptr));
+  for (auto _ : state) {
+    const auto leafNodePos = std::rand() % ArraySize;
+    auto leafNode = (*leafArray)[leafNodePos];
+    auto &leafRef = *leafNode;
+
+    auto start = std::chrono::high_resolution_clock::now();
+    benchmark(leafNode);
+    auto end = std::chrono::high_resolution_clock::now();
+
+    leafRef = *leaf;  ///< reset the modified node
+    leafNode.flush(pop);
+    pop.drain();
+
+    auto elapsed_seconds =
+        std::chrono::duration_cast<std::chrono::duration<double>>(end - start);
+    state.SetIterationTime(elapsed_seconds.count());
   }
 
   // treeRef.printLeafNode(0, leaf);
   std::cout.clear();
-  std::cout << "Writes:" << dbis::PersistEmulation::getBytesWritten() << '\n';
+  std::cout << "Iterations:" << state.iterations() << '\n';
+  std::cout << "PMM Reads:" << pmmreads << '\n';
+  std::cout << "Writes:" << modBytes << '\n';
+  std::cout << "PMM Writes:" << pmmwrites << '\n';
   std::cout << "Elements:" << ELEMENTS << '\n';
 
   transaction::run(pop, [&] { delete_persistent<TreeType>(tree); });
@@ -109,7 +176,7 @@ static void BM_TreeErase(benchmark::State &state) {
   pmempool_rm(path.c_str(), 0);
 }
 
-BENCHMARK(BM_TreeErase);
+BENCHMARK(BM_TreeErase)->UseManualTime();
 BENCHMARK_MAIN();
 
 /* preparing inserts */

bench/trees/tree_get.cpp

@@ -15,21 +15,23 @@
  * If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "common.hpp"
 #include <cstdlib>
 #include <ctime>
 #include <libpmemobj++/container/array.hpp>
+#include <thread>
+#include "common.hpp"
 
-constexpr auto L3 = 30 * 1024 * 1024;
-constexpr auto ArraySize = L3 / TARGET_LEAF_SIZE;
+constexpr auto ArraySize = 4 * L3 / 256; //TARGET_LEAF_SIZE;
 
 /* Get benchmarks on Tree */
 static void BM_TreeGet(benchmark::State &state) {
   std::cout << "BRANCHKEYS: " << BRANCHKEYS << " - " << sizeof(TreeType::BranchNode)
             << "\nLEAFKEYS: " << LEAFKEYS << " - " << sizeof(TreeType::LeafNode) << "\n";
 
+  using LeafArray = pmem::obj::array<pptr<TreeType::LeafNode>, ArraySize>;
   struct root {
-    persistent_ptr<TreeType> tree;
+    pptr<TreeType> tree;
+    pptr<LeafArray> leafArray;
   };
 
   pool<root> pop;
@@ -38,19 +40,19 @@ static void BM_TreeGet(benchmark::State &state) {
   if (access(path.c_str(), F_OK) != 0) {
     pop = pool<root>::create(path, LAYOUT, POOL_SIZE);
     alloc_class =
-        pop.ctl_set<struct pobj_alloc_class_desc>("heap.alloc_class.128.desc", TreeType::AllocClass);
+        pop.ctl_set<struct pobj_alloc_class_desc>("heap.alloc_class.new.desc", TreeType::AllocClass);
     transaction::run(pop, [&] { pop.root()->tree = make_persistent<TreeType>(alloc_class); });
   } else {
     LOG("Warning: " << path << " already exists");
     pmempool_rm(path.c_str(), 0);
     pop = pool<root>::create(path, LAYOUT, POOL_SIZE);
     alloc_class =
-        pop.ctl_set<struct pobj_alloc_class_desc>("heap.alloc_class.128.desc", TreeType::AllocClass);
+        pop.ctl_set<struct pobj_alloc_class_desc>("heap.alloc_class.new.desc", TreeType::AllocClass);
     transaction::run(pop, [&] { pop.root()->tree = make_persistent<TreeType>(alloc_class); });
   }
 
-
-/*
+  /// DRAM considerations
+  /*
   struct alignas(64) DRAMLeafNode {
     DRAMLeafNode() : numKeys(0), nextLeaf(nullptr), prevLeaf(nullptr) {}
     DRAMLeafNode(const TreeType::LeafNode &other)
@@ -61,22 +63,59 @@ static void BM_TreeGet(benchmark::State &state) {
     // values(other.values.get_ro()),
     {}
 
-    unsigned int numKeys;             ///< the number of currently stored keys - 0x10
-    DRAMLeafNode *nextLeaf;               ///< pointer to the subsequent sibling   - 0x18
-    DRAMLeafNode *prevLeaf;               ///< pointer to the preceding sibling    - 0x28
-    char padding[44];                 ///< padding to align keys to 64 byte    - 0x38
+    size_t numKeys;                        ///< the number of currently stored keys - 0x00
+    DRAMLeafNode *nextLeaf;                ///< pointer to the subsequent sibling   - 0x08
+    DRAMLeafNode *prevLeaf;                ///< pointer to the preceding sibling    - 0x10
+    char padding[40];                      ///< padding to align keys to 64 byte    - 0x18
     std::array<MyKey, LEAFKEYS> keys;      ///< the actual keys                     - 0x40
     std::array<MyTuple, LEAFKEYS> values;  ///< the actual values
-  };*/
+  }; */
 
-  const auto tree = pop.root()->tree;
+  /*
+  static constexpr auto SlotSize = ((LEAFKEYS+1 + 7) / 8) * 8;
+  static constexpr auto HashSize = ((LEAFKEYS + 7) / 8) * 8;
+  static constexpr auto BitsetSize = ((LEAFKEYS + 63) / 64) * 8;
+  static constexpr auto SearchSize = BitsetSize + HashSize + 16;
+  static constexpr auto PaddingSize = (64 - SearchSize % 64) % 64;
+
+  struct alignas(64) DRAMLeafNode {
+    DRAMLeafNode() : nextLeaf(nullptr), prevLeaf(nullptr) {
+      //slot[0] = 0;
+    }
+    DRAMLeafNode(const TreeType::LeafNode &other)
+      : slot(other.slot.get_ro()),
+        bits(other.bits.get_ro()),
+        // fp(other.fp.get_ro()),
+        nextLeaf(nullptr),
+        prevLeaf(nullptr),
+        keys(other.keys.get_ro())
+        // values(other.values.get_ro()),
+    {}
+
+    std::array<uint8_t, LEAFKEYS+1> slot;
+    std::bitset<LEAFKEYS>           bits;
+    // std::array<uint8_t, LEAFKEYS>     fp;
+    DRAMLeafNode               *nextLeaf;
+    DRAMLeafNode               *prevLeaf;
+    char            padding[PaddingSize];
+    std::array<MyKey, LEAFKEYS>     keys;
+    std::array<MyTuple, LEAFKEYS> values;
+  };
+  */
+
+  /// Both PMem and DRAM
+  auto &tree = pop.root()->tree;
   insert(tree, 0);
   auto &treeRef = *tree;
-  auto leaf = treeRef.rootNode.leaf;
+  tree.flush(pop);
+  auto &leafArray = pop.root()->leafArray;  ///< PMem
+  auto &leaf = treeRef.rootNode.leaf;
+  leaf.flush(pop);
 
-  pptr<pmem::obj::array<pptr<TreeType::LeafNode>, ArraySize>> leafArray;
+  /// PMem considerations
+  ///*
   transaction::run(pop, [&] {
-    leafArray = make_persistent<pmem::obj::array<pptr<TreeType::LeafNode>, ArraySize>>();
+    leafArray = make_persistent<LeafArray>();
     for (int i = 0; i < ArraySize; ++i) {
       auto &a = *leafArray;
       a[i] = make_persistent<TreeType::LeafNode>(allocation_flag::class_id(alloc_class.class_id),
@@ -84,37 +123,63 @@ static void BM_TreeGet(benchmark::State &state) {
     }
   });
   pop.drain();
+  //*/
 
+  /// DRAM considerations
   /*
   std::unique_ptr<std::array<DRAMLeafNode *, ArraySize>> leafArray(
       new std::array<DRAMLeafNode *, ArraySize>);
   for (int i = 0; i < ArraySize; ++i) {
     auto &a = *leafArray;
     a[i] =  new DRAMLeafNode(*leaf);
-  }*/
+  }
+  */
 
   std::srand(std::time(nullptr));
+  using namespace std::chrono_literals;
+  std::this_thread::sleep_for(2s);
+
+/* BENCHMARKING */
+for (auto _ : state) {
+  const auto leafNodePos = std::rand() % ArraySize;
+  const auto leafNode = (*leafArray)[leafNodePos];
+  const auto key = (long long unsigned int)KEYPOS;
+
+  auto start = std::chrono::high_resolution_clock::now();
+  benchmark::DoNotOptimize(*leafNode);
+  treeRef.lookupPositionInLeafNode(leafNode, KEYPOS); ///< PMem variant
+  /// DRAM considerations
+  /*
+  // auto pos = 0u;
+  const auto &nodeRef = *leafNode;
+  const auto &keys = nodeRef.keys;
+  // const auto num = nodeRef.numKeys;
+  const auto &slots = nodeRef.slot;
+  // const auto &bits = nodeRef.bits;
+  // const auto hash = (uint8_t)(key & 0xFF);
+  // const auto &hashs = nodeRef.fp;
+  // for (; pos < LEAFKEYS; ++pos)
+    // if (//hashs[pos] == hash &&
+    // bits.test(pos) && keys[pos] == key) break;
+  // for (; pos < num && keys[pos] != key; ++pos);
+  // auto pos = dbis::binarySearch<false>(keys, 0, num-1, key);
+  auto pos = dbis::binarySearch<false>(keys, slots, 1, slots[0], key);
+  benchmark::DoNotOptimize(pos);
+  */
+  benchmark::DoNotOptimize(*leafNode);
+  auto end = std::chrono::high_resolution_clock::now();
+  auto elapsed_seconds =
+      std::chrono::duration_cast<std::chrono::duration<double>>(end - start);
+  state.SetIterationTime(elapsed_seconds.count());
+}
 
-  /* BENCHMARKING */
-  for (auto _ : state) {
-    state.PauseTiming();
-    const auto leafNodePos = std::rand() % ArraySize;
-    const auto leafNode = (*leafArray)[leafNodePos];
-    const auto key = (long long unsigned int)KEYPOS;
-    state.ResumeTiming();
-    benchmark::DoNotOptimize(*leafNode);
-    treeRef.lookupPositionInLeafNode(leafNode, KEYPOS);
-    // dbis::binarySearch<false>(leaf->keys.get_ro(),