<commit_before>// Copyright 2014, University of Freiburg,
// Chair of Algorithms and Data Structures.
// Author: <NAME> (<EMAIL>)

#include <algorithm>
#include <unordered_set>
#include <stxxl/algorithm>
#include "../parser/TsvParser.h"
#include "./Index.h"
#include "../parser/NTriplesParser.h"
#include "../util/Conversions.h"

using std::array;

// _____________________________________________________________________________
void Index::createFromTsvFile(const string& tsvFile, const string& onDiskBase,
                              bool allPermutations) {
  _onDiskBase = onDiskBase;
  string indexFilename = _onDiskBase + ".index";
  size_t nofLines = passTsvFileForVocabulary(tsvFile);
  _vocab.writeToFile(onDiskBase + ".vocabulary");
  ExtVec v(nofLines);
  passTsvFileIntoIdVector(tsvFile, v);
  // PSO permutation
  LOG(INFO) << "Sorting for PSO permutation..." << std::endl;
  stxxl::sort(begin(v), end(v), SortByPSO(), STXXL_MEMORY_TO_USE);
  LOG(INFO) << "Sort done." << std::endl;
  createPermutation(indexFilename + ".pso", v, _psoMeta, 1, 0, 2);
  // POS permutation
  LOG(INFO) << "Sorting for POS permutation..." << std::endl;;
  stxxl::sort(begin(v), end(v), SortByPOS(), STXXL_MEMORY_TO_USE);
  LOG(INFO) << "Sort done." << std::endl;
  createPermutation(indexFilename + ".pos", v, _posMeta, 1, 2, 0);
  if (allPermutations) {
    // SPO permutation
    LOG(INFO) << "Sorting for SPO permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortBySPO(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".spo", v, _spoMeta, 0, 1, 2);
    // SOP permutation
    LOG(INFO) << "Sorting for SOP permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortBySOP(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".sop", v, _sopMeta, 0, 2, 1);
    // OSP permutation
    LOG(INFO) << "Sorting for OSP permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortByOSP(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".osp", v, _ospMeta, 2, 0, 1);
    // OPS permutation
    LOG(INFO) << "Sorting for OPS permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortByOPS(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".ops", v, _opsMeta, 2, 1, 0);
  }
  openFileHandles();
}

// _____________________________________________________________________________
void Index::createFromNTriplesFile(const string& ntFile,
                                   const string& onDiskBase,
                                   bool allPermutations) {
  _onDiskBase = onDiskBase;
  string indexFilename = _onDiskBase + ".index";
  size_t nofLines = passNTriplesFileForVocabulary(ntFile);
  _vocab.writeToFile(onDiskBase + ".vocabulary");
  ExtVec v(nofLines);
  passNTriplesFileIntoIdVector(ntFile, v);
  LOG(INFO) << "Sorting for PSO permutation..." << std::endl;
  stxxl::sort(begin(v), end(v), SortByPSO(), STXXL_MEMORY_TO_USE);
  LOG(INFO) << "Sort done." << std::endl;
  createPermutation(indexFilename + ".pso", v, _psoMeta, 1, 0, 2);
  LOG(INFO) << "Sorting for POS permutation..." << std::endl;;
  stxxl::sort(begin(v), end(v), SortByPOS(), STXXL_MEMORY_TO_USE);
  LOG(INFO) << "Sort done." << std::endl;;
  createPermutation(indexFilename + ".pos", v, _posMeta, 1, 2, 0);
  if (allPermutations) {
    // SPO permutation
    LOG(INFO) << "Sorting for SPO permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortBySPO(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".spo", v, _spoMeta, 0, 1, 2);
    // SOP permutation
    LOG(INFO) << "Sorting for SOP permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortBySOP(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".sop", v, _sopMeta, 0, 2, 1);
    // OSP permutation
    LOG(INFO) << "Sorting for OSP permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortByOSP(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".osp", v, _ospMeta, 2, 0, 1);
    // OPS permutation
    LOG(INFO) << "Sorting for OPS permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortByOPS(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".ops", v, _opsMeta, 2, 1, 0);
  }
  openFileHandles();
}

// _____________________________________________________________________________
size_t Index::passTsvFileForVocabulary(const string& tsvFile) {
  LOG(INFO) << "Making pass over TsvFile " << tsvFile << " for vocabulary."
            << std::endl;
  array<string, 3> spo;
  TsvParser p(tsvFile);
  std::unordered_set<string> items;
  size_t i = 0;
  while (p.getLine(spo)) {
    if (ad_utility::isXsdValue(spo[2])) {
      spo[2] = ad_utility::convertValueLiteralToIndexWord(spo[2]);
    }
    items.insert(spo[0]);
    items.insert(spo[1]);
    items.insert(spo[2]);
    ++i;
    if (i % 10000000 == 0) {
      LOG(INFO) << "Lines processed: " << i << '\n';
    }
  }
  LOG(INFO) << "Pass done.\n";
  _vocab.createFromSet(items);
  return i;
}

// _____________________________________________________________________________
void Index::passTsvFileIntoIdVector(const string& tsvFile, ExtVec& data) {
  LOG(INFO) << "Making pass over TsvFile " << tsvFile
            << " and creating stxxl vector.\n";
  array<string, 3> spo;
  TsvParser p(tsvFile);
  std::unordered_map<string, Id> vocabMap = _vocab.asMap();
  size_t i = 0;
  // write using vector_bufwriter
  ExtVec::bufwriter_type writer(data);
  while (p.getLine(spo)) {
    if (ad_utility::isXsdValue(spo[2])) {
      spo[2] = ad_utility::convertValueLiteralToIndexWord(spo[2]);
    }
    writer << array<Id, 3>{{
                               vocabMap.find(spo[0])->second,
                               vocabMap.find(spo[1])->second,
                               vocabMap.find(spo[2])->second
                           }};
    ++i;
    if (i % 10000000 == 0) {
      LOG(INFO) << "Lines processed: " << i << '\n';
    }
  }
  writer.finish();
  LOG(INFO) << "Pass done.\n";
}

// _____________________________________________________________________________
size_t Index::passNTriplesFileForVocabulary(const string& ntFile) {
  LOG(INFO) << "Making pass over NTriples " << ntFile << " for vocabulary."
            << std::endl;
  array<string, 3> spo;
  NTriplesParser p(ntFile);
  std::unordered_set<string> items;
  size_t i = 0;
  while (p.getLine(spo)) {
    if (ad_utility::isXsdValue(spo[2])) {
      spo[2] = ad_utility::convertValueLiteralToIndexWord(spo[2]);
    }
    items.insert(spo[0]);
    items.insert(spo[1]);
    items.insert(spo[2]);
    ++i;
    if (i % 10000000 == 0) {
      LOG(INFO) << "Lines processed: " << i << '\n';
    }
  }
  LOG(INFO) << "Pass done.\n";
  _vocab.createFromSet(items);
  return i;
}

// _____________________________________________________________________________
void Index::passNTriplesFileIntoIdVector(const string& ntFile, ExtVec& data) {
  LOG(INFO) << "Making pass over NTriples " << ntFile
            << " and creating stxxl vector.\n";
  array<string, 3> spo;
  NTriplesParser p(ntFile);
  std::unordered_map<string, Id> vocabMap = _vocab.asMap();
  size_t i = 0;
  // write using vector_bufwriter
  ExtVec::bufwriter_type writer(data);
  while (p.getLine(spo)) {
    if (ad_utility::isXsdValue(spo[2])) {
      spo[2] = ad_utility::convertValueLiteralToIndexWord(spo[2]);
    }
    writer << array<Id, 3>{{
                               vocabMap.find(spo[0])->second,
                               vocabMap.find(spo[1])->second,
                               vocabMap.find(spo[2])->second
                           }};
    ++i;
    if (i % 10000000 == 0) {
      LOG(INFO) << "Lines processed: " << i << '\n';
    }
  }
  writer.finish();
  LOG(INFO) << "Pass done.\n";
}

// _____________________________________________________________________________
void Index::createPermutation(const string& fileName, Index::ExtVec const& vec,
                              IndexMetaData& metaData, size_t c0, size_t c1,
                              size_t c2) {
  if (vec.size() == 0) {
    LOG(WARN) << "Attempt to write an empty index!" << std::endl;
    return;
  }
  ad_utility::File out(fileName.c_str(), "w");
  LOG(INFO) << "Creating an on-disk index permutation of " << vec.size()
            << " elements / facts." << std::endl;
  // Iterate over the vector and identify relation boundaries
  size_t from = 0;
  Id currentRel = vec[0][c0];
  off_t lastOffset = 0;
  vector<array<Id, 2>> buffer;
  bool functional = true;
  Id lastLhs = std::numeric_limits<Id>::max();
  for (ExtVec::bufreader_type reader(vec); !reader.empty(); ++reader) {
    if ((*reader)[c0] != currentRel) {
      metaData.add(writeRel(out, lastOffset, currentRel, buffer, functional));
      buffer.clear();
      lastOffset = metaData.getOffsetAfter();
      currentRel = (*reader)[c0];
      functional = true;
    } else {
      if ((*reader)[c1] == lastLhs) {
        functional = false;
      }
    }
    buffer.emplace_back(array<Id, 2>{{(*reader)[c1], (*reader)[c2]}});
    lastLhs = (*reader)[c1];
  }
  if (from < vec.size()) {
    metaData.add(writeRel(out, lastOffset, currentRel, buffer, functional));
  }

  LOG(INFO) << "Done creating index permutation." << std::endl;
  LOG(INFO) << "Writing statistics for this permutation:\n"
            << metaData.statistics() << std::endl;

  LOG(INFO) << "Writing Meta data to index file...\n";
  out << metaData;
  off_t startOfMeta = metaData.getOffsetAfter();
  out.write(&startOfMeta, sizeof(startOfMeta));
  out.close();
  LOG(INFO) << "Permutation done.\n";
}

// _____________________________________________________________________________
RelationMetaData Index::writeRel(ad_utility::File& out, off_t currentOffset,
                                 Id relId, const vector<array<Id, 2>>& data,
                                 bool functional) {
  LOG(TRACE) << "Writing a relation ...\n";
  AD_CHECK_GT(data.size(), 0);
  RelationMetaData rmd;
  rmd._relId = relId;
  rmd._startFullIndex = currentOffset;
  rmd._nofElements = data.size();

  // Write the full pair index.
  out.write(data.data(), data.size() * 2 * sizeof(Id));

  if (functional) {
    rmd = writeFunctionalRelation(data, rmd);
  } else {
    rmd = writeNonFunctionalRelation(out, data, rmd);
  };
  rmd._nofBlocks = rmd._blocks.size();
  LOG(TRACE) << "Done writing relation.\n";
  return rmd;
}

// _____________________________________________________________________________
RelationMetaData& Index::writeFunctionalRelation(
    const vector<array<Id, 2>>& data, RelationMetaData& rmd) {
  LOG(TRACE) << "Writing part for functional relation ...\n";
  // Do not write extra LHS and RHS lists.
  rmd._startRhs = rmd._startFullIndex + data.size() * 2 * sizeof(Id);
  rmd._offsetAfter = rmd._startRhs;
  // Create the block data for the RelationMetaData.
  // Blocks are offsets into the full pair index for functional relations.
  size_t nofDistinctLhs = 0;
  Id lastLhs = std::numeric_limits<Id>::max();
  for (size_t i = 0; i < data.size(); ++i) {
    if (data[i][0] != lastLhs) {
      if (nofDistinctLhs % DISTINCT_LHS_PER_BLOCK == 0) {
        rmd._blocks.emplace_back(
            BlockMetaData(data[i][0],
                          rmd._startFullIndex + i * 2 * sizeof(Id)));
      }
      ++nofDistinctLhs;
    }
  }
  return rmd;
}

// _____________________________________________________________________________
RelationMetaData& Index::writeNonFunctionalRelation(ad_utility::File& out,
                                                    const vector<array<Id, 2>>& data,
                                                    RelationMetaData& rmd) {
  LOG(TRACE) << "Writing part for non-functional relation ...\n";
  // Make a pass over the data and extract a RHS list for each LHS.
  // Prepare both in buffers.
  // TODO: add compression - at least to RHS.
  pair<Id, off_t> *bufLhs = new pair<Id, off_t>[data.size()];
  Id *bufRhs = new Id[data.size()];
  size_t nofDistinctLhs = 0;
  Id lastLhs = std::numeric_limits<Id>::max();
  size_t nofRhsDone = 0;
  for (; nofRhsDone < data.size(); ++nofRhsDone) {
    if (data[nofRhsDone][0] != lastLhs) {
      bufLhs[nofDistinctLhs++] =
          pair<Id, off_t>(data[nofRhsDone][0], nofRhsDone * sizeof(Id));
      lastLhs = data[nofRhsDone][0];
    }
    bufRhs[nofRhsDone] = data[nofRhsDone][1];
  }

  // Go over the Lhs data once more and adjust the offsets.
  off_t startRhs = rmd.getStartOfLhs()
                   + nofDistinctLhs * (sizeof(Id) + sizeof(off_t));

  for (size_t i = 0; i < nofDistinctLhs; ++i) {
    bufLhs[i].second += startRhs;
  }

  // Write to file.
  out.write(bufLhs, nofDistinctLhs * (sizeof(Id) + sizeof(off_t)));
  out.write(bufRhs, data.size() * sizeof(Id));


  // Update meta data.
  rmd._startRhs = startRhs;
  rmd._offsetAfter = startRhs + rmd._nofElements * sizeof(Id);

  // Create the block data for the RelationMetaData.
  // Block are offsets into the LHS list for non-functional relations.
  for (size_t i = 0; i < nofDistinctLhs; ++i) {
    if (i % DISTINCT_LHS_PER_BLOCK == 0) {
      rmd._blocks.emplace_back(BlockMetaData(bufLhs[i].first,
                                             rmd.getStartOfLhs() +
                                             i * (sizeof(Id) + sizeof(off_t))));
    }
  }

  delete[] bufLhs;
  delete[] bufRhs;
  return rmd;
}

// _____________________________________________________________________________
void Index::createFromOnDiskIndex(const string& onDiskBase) {
  _onDiskBase = onDiskBase;
  _vocab.readFromFile(onDiskBase + ".vocabulary");
  _psoFile.open(string(_onDiskBase + ".index.pso").c_str(), "r");
  _posFile.open(string(_onDiskBase + ".index.pos").c_str(), "r");
  AD_CHECK(_psoFile.isOpen() && _posFile.isOpen());
  // PSO
  off_t metaFrom;
  off_t metaTo = _psoFile.getLastOffset(&metaFrom);
  unsigned char *buf = new unsigned char[metaTo - metaFrom];
  _psoFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
  _psoMeta.createFromByteBuffer(buf);
  delete[] buf;
  LOG(INFO) << "Registered PSO permutation: " << _psoMeta.statistics()
            << std::endl;
  // POS
  metaTo = _posFile.getLastOffset(&metaFrom);
  buf = new unsigned char[metaTo - metaFrom];
  _posFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
  _posMeta.createFromByteBuffer(buf);
  delete[] buf;
  LOG(INFO) << "Registered POS permutation: " << _posMeta.statistics()
            << std::endl;
  if (ad_utility::File::exists(_onDiskBase + ".index.spo")) {
    _spoFile.open(string(_onDiskBase + ".index.spo").c_str(), "r");
    _sopFile.open(string(_onDiskBase + ".index.sop").c_str(), "r");
    _ospFile.open(string(_onDiskBase + ".index.osp").c_str(), "r");
    _opsFile.open(string(_onDiskBase + ".index.ops").c_str(), "r");
    AD_CHECK(_spoFile.isOpen() && _sopFile.isOpen() && _ospFile.isOpen() &&
             _opsFile.isOpen());
    // SPO
    metaTo = _spoFile.getLastOffset(&metaFrom);
    buf = new unsigned char[metaTo - metaFrom];
    _spoFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
    _spoMeta.createFromByteBuffer(buf);
    delete[] buf;
    LOG(INFO) << "Registered SPO permutation: " << _spoMeta.statistics()
              << std::endl;
    // SOP
    metaTo = _sopFile.getLastOffset(&metaFrom);
    buf = new unsigned char[metaTo - metaFrom];
    _sopFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
    _sopMeta.createFromByteBuffer(buf);
    delete[] buf;
    LOG(INFO) << "Registered SOP permutation: " << _psoMeta.statistics()
              << std::endl;
    // OSP
    metaTo = _ospFile.getLastOffset(&metaFrom);
    buf = new unsigned char[metaTo - metaFrom];
    _ospFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
    _ospMeta.createFromByteBuffer(buf);
    delete[] buf;
    LOG(INFO) << "Registered OSP permutation: " << _ospMeta.statistics()
              << std::endl;
    // OPS
    metaTo = _opsFile.getLastOffset(&metaFrom);
    buf = new unsigned char[metaTo - metaFrom];
    _opsFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
    _opsMeta.createFromByteBuffer(buf);
    delete[] buf;
    LOG(INFO) << "Registered OPS permutation: " << _opsMeta.statistics()
              << std::endl;
  }<commit_msg>Fixed statistics printed for SOP permutation<commit_after>// Copyright 2014, University of Freiburg,
// Chair of Algorithms and Data Structures.
// Author: <NAME> (<EMAIL>)

#include <algorithm>
#include <unordered_set>
#include <stxxl/algorithm>
#include "../parser/TsvParser.h"
#include "./Index.h"
#include "../parser/NTriplesParser.h"
#include "../util/Conversions.h"

using std::array;

// _____________________________________________________________________________
void Index::createFromTsvFile(const string& tsvFile, const string& onDiskBase,
                              bool allPermutations) {
  _onDiskBase = onDiskBase;
  string indexFilename = _onDiskBase + ".index";
  size_t nofLines = passTsvFileForVocabulary(tsvFile);
  _vocab.writeToFile(onDiskBase + ".vocabulary");
  ExtVec v(nofLines);
  passTsvFileIntoIdVector(tsvFile, v);
  // PSO permutation
  LOG(INFO) << "Sorting for PSO permutation..." << std::endl;
  stxxl::sort(begin(v), end(v), SortByPSO(), STXXL_MEMORY_TO_USE);
  LOG(INFO) << "Sort done." << std::endl;
  createPermutation(indexFilename + ".pso", v, _psoMeta, 1, 0, 2);
  // POS permutation
  LOG(INFO) << "Sorting for POS permutation..." << std::endl;;
  stxxl::sort(begin(v), end(v), SortByPOS(), STXXL_MEMORY_TO_USE);
  LOG(INFO) << "Sort done." << std::endl;
  createPermutation(indexFilename + ".pos", v, _posMeta, 1, 2, 0);
  if (allPermutations) {
    // SPO permutation
    LOG(INFO) << "Sorting for SPO permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortBySPO(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".spo", v, _spoMeta, 0, 1, 2);
    // SOP permutation
    LOG(INFO) << "Sorting for SOP permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortBySOP(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".sop", v, _sopMeta, 0, 2, 1);
    // OSP permutation
    LOG(INFO) << "Sorting for OSP permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortByOSP(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".osp", v, _ospMeta, 2, 0, 1);
    // OPS permutation
    LOG(INFO) << "Sorting for OPS permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortByOPS(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".ops", v, _opsMeta, 2, 1, 0);
  }
  openFileHandles();
}

// _____________________________________________________________________________
void Index::createFromNTriplesFile(const string& ntFile,
                                   const string& onDiskBase,
                                   bool allPermutations) {
  _onDiskBase = onDiskBase;
  string indexFilename = _onDiskBase + ".index";
  size_t nofLines = passNTriplesFileForVocabulary(ntFile);
  _vocab.writeToFile(onDiskBase + ".vocabulary");
  ExtVec v(nofLines);
  passNTriplesFileIntoIdVector(ntFile, v);
  LOG(INFO) << "Sorting for PSO permutation..." << std::endl;
  stxxl::sort(begin(v), end(v), SortByPSO(), STXXL_MEMORY_TO_USE);
  LOG(INFO) << "Sort done." << std::endl;
  createPermutation(indexFilename + ".pso", v, _psoMeta, 1, 0, 2);
  LOG(INFO) << "Sorting for POS permutation..." << std::endl;;
  stxxl::sort(begin(v), end(v), SortByPOS(), STXXL_MEMORY_TO_USE);
  LOG(INFO) << "Sort done." << std::endl;;
  createPermutation(indexFilename + ".pos", v, _posMeta, 1, 2, 0);
  if (allPermutations) {
    // SPO permutation
    LOG(INFO) << "Sorting for SPO permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortBySPO(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".spo", v, _spoMeta, 0, 1, 2);
    // SOP permutation
    LOG(INFO) << "Sorting for SOP permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortBySOP(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".sop", v, _sopMeta, 0, 2, 1);
    // OSP permutation
    LOG(INFO) << "Sorting for OSP permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortByOSP(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".osp", v, _ospMeta, 2, 0, 1);
    // OPS permutation
    LOG(INFO) << "Sorting for OPS permutation..." << std::endl;
    stxxl::sort(begin(v), end(v), SortByOPS(), STXXL_MEMORY_TO_USE);
    LOG(INFO) << "Sort done." << std::endl;
    createPermutation(indexFilename + ".ops", v, _opsMeta, 2, 1, 0);
  }
  openFileHandles();
}

// _____________________________________________________________________________
size_t Index::passTsvFileForVocabulary(const string& tsvFile) {
  LOG(INFO) << "Making pass over TsvFile " << tsvFile << " for vocabulary."
            << std::endl;
  array<string, 3> spo;
  TsvParser p(tsvFile);
  std::unordered_set<string> items;
  size_t i = 0;
  while (p.getLine(spo)) {
    if (ad_utility::isXsdValue(spo[2])) {
      spo[2] = ad_utility::convertValueLiteralToIndexWord(spo[2]);
    }
    items.insert(spo[0]);
    items.insert(spo[1]);
    items.insert(spo[2]);
    ++i;
    if (i % 10000000 == 0) {
      LOG(INFO) << "Lines processed: " << i << '\n';
    }
  }
  LOG(INFO) << "Pass done.\n";
  _vocab.createFromSet(items);
  return i;
}

// _____________________________________________________________________________
void Index::passTsvFileIntoIdVector(const string& tsvFile, ExtVec& data) {
  LOG(INFO) << "Making pass over TsvFile " << tsvFile
            << " and creating stxxl vector.\n";
  array<string, 3> spo;
  TsvParser p(tsvFile);
  std::unordered_map<string, Id> vocabMap = _vocab.asMap();
  size_t i = 0;
  // write using vector_bufwriter
  ExtVec::bufwriter_type writer(data);
  while (p.getLine(spo)) {
    if (ad_utility::isXsdValue(spo[2])) {
      spo[2] = ad_utility::convertValueLiteralToIndexWord(spo[2]);
    }
    writer << array<Id, 3>{{
                               vocabMap.find(spo[0])->second,
                               vocabMap.find(spo[1])->second,
                               vocabMap.find(spo[2])->second
                           }};
    ++i;
    if (i % 10000000 == 0) {
      LOG(INFO) << "Lines processed: " << i << '\n';
    }
  }
  writer.finish();
  LOG(INFO) << "Pass done.\n";
}

// _____________________________________________________________________________
size_t Index::passNTriplesFileForVocabulary(const string& ntFile) {
  LOG(INFO) << "Making pass over NTriples " << ntFile << " for vocabulary."
            << std::endl;
  array<string, 3> spo;
  NTriplesParser p(ntFile);
  std::unordered_set<string> items;
  size_t i = 0;
  while (p.getLine(spo)) {
    if (ad_utility::isXsdValue(spo[2])) {
      spo[2] = ad_utility::convertValueLiteralToIndexWord(spo[2]);
    }
    items.insert(spo[0]);
    items.insert(spo[1]);
    items.insert(spo[2]);
    ++i;
    if (i % 10000000 == 0) {
      LOG(INFO) << "Lines processed: " << i << '\n';
    }
  }
  LOG(INFO) << "Pass done.\n";
  _vocab.createFromSet(items);
  return i;
}

// _____________________________________________________________________________
void Index::passNTriplesFileIntoIdVector(const string& ntFile, ExtVec& data) {
  LOG(INFO) << "Making pass over NTriples " << ntFile
            << " and creating stxxl vector.\n";
  array<string, 3> spo;
  NTriplesParser p(ntFile);
  std::unordered_map<string, Id> vocabMap = _vocab.asMap();
  size_t i = 0;
  // write using vector_bufwriter
  ExtVec::bufwriter_type writer(data);
  while (p.getLine(spo)) {
    if (ad_utility::isXsdValue(spo[2])) {
      spo[2] = ad_utility::convertValueLiteralToIndexWord(spo[2]);
    }
    writer << array<Id, 3>{{
                               vocabMap.find(spo[0])->second,
                               vocabMap.find(spo[1])->second,
                               vocabMap.find(spo[2])->second
                           }};
    ++i;
    if (i % 10000000 == 0) {
      LOG(INFO) << "Lines processed: " << i << '\n';
    }
  }
  writer.finish();
  LOG(INFO) << "Pass done.\n";
}

// _____________________________________________________________________________
void Index::createPermutation(const string& fileName, Index::ExtVec const& vec,
                              IndexMetaData& metaData, size_t c0, size_t c1,
                              size_t c2) {
  if (vec.size() == 0) {
    LOG(WARN) << "Attempt to write an empty index!" << std::endl;
    return;
  }
  ad_utility::File out(fileName.c_str(), "w");
  LOG(INFO) << "Creating an on-disk index permutation of " << vec.size()
            << " elements / facts." << std::endl;
  // Iterate over the vector and identify relation boundaries
  size_t from = 0;
  Id currentRel = vec[0][c0];
  off_t lastOffset = 0;
  vector<array<Id, 2>> buffer;
  bool functional = true;
  Id lastLhs = std::numeric_limits<Id>::max();
  for (ExtVec::bufreader_type reader(vec); !reader.empty(); ++reader) {
    if ((*reader)[c0] != currentRel) {
      metaData.add(writeRel(out, lastOffset, currentRel, buffer, functional));
      buffer.clear();
      lastOffset = metaData.getOffsetAfter();
      currentRel = (*reader)[c0];
      functional = true;
    } else {
      if ((*reader)[c1] == lastLhs) {
        functional = false;
      }
    }
    buffer.emplace_back(array<Id, 2>{{(*reader)[c1], (*reader)[c2]}});
    lastLhs = (*reader)[c1];
  }
  if (from < vec.size()) {
    metaData.add(writeRel(out, lastOffset, currentRel, buffer, functional));
  }

  LOG(INFO) << "Done creating index permutation." << std::endl;
  LOG(INFO) << "Writing statistics for this permutation:\n"
            << metaData.statistics() << std::endl;

  LOG(INFO) << "Writing Meta data to index file...\n";
  out << metaData;
  off_t startOfMeta = metaData.getOffsetAfter();
  out.write(&startOfMeta, sizeof(startOfMeta));
  out.close();
  LOG(INFO) << "Permutation done.\n";
}

// _____________________________________________________________________________
RelationMetaData Index::writeRel(ad_utility::File& out, off_t currentOffset,
                                 Id relId, const vector<array<Id, 2>>& data,
                                 bool functional) {
  LOG(TRACE) << "Writing a relation ...\n";
  AD_CHECK_GT(data.size(), 0);
  RelationMetaData rmd;
  rmd._relId = relId;
  rmd._startFullIndex = currentOffset;
  rmd._nofElements = data.size();

  // Write the full pair index.
  out.write(data.data(), data.size() * 2 * sizeof(Id));

  if (functional) {
    rmd = writeFunctionalRelation(data, rmd);
  } else {
    rmd = writeNonFunctionalRelation(out, data, rmd);
  };
  rmd._nofBlocks = rmd._blocks.size();
  LOG(TRACE) << "Done writing relation.\n";
  return rmd;
}

// _____________________________________________________________________________
RelationMetaData& Index::writeFunctionalRelation(
    const vector<array<Id, 2>>& data, RelationMetaData& rmd) {
  LOG(TRACE) << "Writing part for functional relation ...\n";
  // Do not write extra LHS and RHS lists.
  rmd._startRhs = rmd._startFullIndex + data.size() * 2 * sizeof(Id);
  rmd._offsetAfter = rmd._startRhs;
  // Create the block data for the RelationMetaData.
  // Blocks are offsets into the full pair index for functional relations.
  size_t nofDistinctLhs = 0;
  Id lastLhs = std::numeric_limits<Id>::max();
  for (size_t i = 0; i < data.size(); ++i) {
    if (data[i][0] != lastLhs) {
      if (nofDistinctLhs % DISTINCT_LHS_PER_BLOCK == 0) {
        rmd._blocks.emplace_back(
            BlockMetaData(data[i][0],
                          rmd._startFullIndex + i * 2 * sizeof(Id)));
      }
      ++nofDistinctLhs;
    }
  }
  return rmd;
}

// _____________________________________________________________________________
RelationMetaData& Index::writeNonFunctionalRelation(ad_utility::File& out,
                                                    const vector<array<Id, 2>>& data,
                                                    RelationMetaData& rmd) {
  LOG(TRACE) << "Writing part for non-functional relation ...\n";
  // Make a pass over the data and extract a RHS list for each LHS.
  // Prepare both in buffers.
  // TODO: add compression - at least to RHS.
  pair<Id, off_t> *bufLhs = new pair<Id, off_t>[data.size()];
  Id *bufRhs = new Id[data.size()];
  size_t nofDistinctLhs = 0;
  Id lastLhs = std::numeric_limits<Id>::max();
  size_t nofRhsDone = 0;
  for (; nofRhsDone < data.size(); ++nofRhsDone) {
    if (data[nofRhsDone][0] != lastLhs) {
      bufLhs[nofDistinctLhs++] =
          pair<Id, off_t>(data[nofRhsDone][0], nofRhsDone * sizeof(Id));
      lastLhs = data[nofRhsDone][0];
    }
    bufRhs[nofRhsDone] = data[nofRhsDone][1];
  }

  // Go over the Lhs data once more and adjust the offsets.
  off_t startRhs = rmd.getStartOfLhs()
                   + nofDistinctLhs * (sizeof(Id) + sizeof(off_t));

  for (size_t i = 0; i < nofDistinctLhs; ++i) {
    bufLhs[i].second += startRhs;
  }

  // Write to file.
  out.write(bufLhs, nofDistinctLhs * (sizeof(Id) + sizeof(off_t)));
  out.write(bufRhs, data.size() * sizeof(Id));


  // Update meta data.
  rmd._startRhs = startRhs;
  rmd._offsetAfter = startRhs + rmd._nofElements * sizeof(Id);

  // Create the block data for the RelationMetaData.
  // Block are offsets into the LHS list for non-functional relations.
  for (size_t i = 0; i < nofDistinctLhs; ++i) {
    if (i % DISTINCT_LHS_PER_BLOCK == 0) {
      rmd._blocks.emplace_back(BlockMetaData(bufLhs[i].first,
                                             rmd.getStartOfLhs() +
                                             i * (sizeof(Id) + sizeof(off_t))));
    }
  }

  delete[] bufLhs;
  delete[] bufRhs;
  return rmd;
}

// _____________________________________________________________________________
void Index::createFromOnDiskIndex(const string& onDiskBase) {
  _onDiskBase = onDiskBase;
  _vocab.readFromFile(onDiskBase + ".vocabulary");
  _psoFile.open(string(_onDiskBase + ".index.pso").c_str(), "r");
  _posFile.open(string(_onDiskBase + ".index.pos").c_str(), "r");
  AD_CHECK(_psoFile.isOpen() && _posFile.isOpen());
  // PSO
  off_t metaFrom;
  off_t metaTo = _psoFile.getLastOffset(&metaFrom);
  unsigned char *buf = new unsigned char[metaTo - metaFrom];
  _psoFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
  _psoMeta.createFromByteBuffer(buf);
  delete[] buf;
  LOG(INFO) << "Registered PSO permutation: " << _psoMeta.statistics()
            << std::endl;
  // POS
  metaTo = _posFile.getLastOffset(&metaFrom);
  buf = new unsigned char[metaTo - metaFrom];
  _posFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
  _posMeta.createFromByteBuffer(buf);
  delete[] buf;
  LOG(INFO) << "Registered POS permutation: " << _posMeta.statistics()
            << std::endl;
  if (ad_utility::File::exists(_onDiskBase + ".index.spo")) {
    _spoFile.open(string(_onDiskBase + ".index.spo").c_str(), "r");
    _sopFile.open(string(_onDiskBase + ".index.sop").c_str(), "r");
    _ospFile.open(string(_onDiskBase + ".index.osp").c_str(), "r");
    _opsFile.open(string(_onDiskBase + ".index.ops").c_str(), "r");
    AD_CHECK(_spoFile.isOpen() && _sopFile.isOpen() && _ospFile.isOpen() &&
             _opsFile.isOpen());
    // SPO
    metaTo = _spoFile.getLastOffset(&metaFrom);
    buf = new unsigned char[metaTo - metaFrom];
    _spoFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
    _spoMeta.createFromByteBuffer(buf);
    delete[] buf;
    LOG(INFO) << "Registered SPO permutation: " << _spoMeta.statistics()
              << std::endl;
    // SOP
    metaTo = _sopFile.getLastOffset(&metaFrom);
    buf = new unsigned char[metaTo - metaFrom];
    _sopFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
    _sopMeta.createFromByteBuffer(buf);
    delete[] buf;
    LOG(INFO) << "Registered SOP permutation: " << _sopMeta.statistics()
              << std::endl;
    // OSP
    metaTo = _ospFile.getLastOffset(&metaFrom);
    buf = new unsigned char[metaTo - metaFrom];
    _ospFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
    _ospMeta.createFromByteBuffer(buf);
    delete[] buf;
    LOG(INFO) << "Registered OSP permutation: " << _ospMeta.statistics()
              << std::endl;
    // OPS
    metaTo = _opsFile.getLastOffset(&metaFrom);
    buf = new unsigned char[metaTo - metaFrom];
    _opsFile.read(buf, static_cast<size_t>(metaTo - metaFrom), metaFrom);
    _opsMeta.createFromByteBuffer(buf);
    delete[] buf;
    LOG(INFO) << "Registered OPS permutation: " << _opsMeta.statistics()
              << std::endl;
  }
{'fraction_non_alphanumeric': 0.15576117423671598, 'fraction_numerical': 0.008861026552940113, 'mean_word_length': 3.2912571132953956, 'pattern_counts': {'":': 0, '<': 549, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7515747', 'n_tokens_mistral': 11599, 'n_tokens_neox': 11163, 'n_words': 2339}