<commit_before>#include <list>
#include <stdexcept>
#include <string>
#include <utility>

#include <tr1/unordered_map>

#include <boost/filesystem.hpp>

#include <AlpinoCorpus/CorpusReader.hh>
#include <AlpinoCorpus/Error.hh>
#include <AlpinoCorpus/RecursiveCorpusReader.hh>

#include <iostream>

namespace bf = boost::filesystem;

namespace alpinocorpus {

struct ReaderIter
{
  ReaderIter(std::string newName, CorpusReader *newReader,
      CorpusReader::EntryIterator newIter) :
    name(newName), reader(newReader), iter(newIter) {}

  std::string name;
  CorpusReader *reader;
  CorpusReader::EntryIterator iter;
};

bool operator==(ReaderIter const &left, ReaderIter const &right)
{
  return left.name == right.name && left.reader == right.reader &&
    left.iter == right.iter;
}

class RecursiveCorpusReaderPrivate : public CorpusReader
{
  class RecursiveIter : public CorpusReader::IterImpl
  {
  public:
    RecursiveIter(std::tr1::unordered_map<std::string, CorpusReader *> const &readers);
    RecursiveIter(std::tr1::unordered_map<std::string, CorpusReader *> const &readers,
      std::string const &query);
    ~RecursiveIter();
    std::string current() const;
    bool equals(IterImpl const &other) const;
    void next();
  private:

    std::list<ReaderIter> d_iters;
  };
public:
  RecursiveCorpusReaderPrivate(std::string const &directory);
  virtual ~RecursiveCorpusReaderPrivate();

  EntryIterator getBegin() const;
  EntryIterator getEnd() const;
  std::string getName() const;
  size_t getSize() const;
  void push_back(std::string const &name, CorpusReader *reader);
  std::string readEntry(std::string const &) const;
  std::string readEntryMarkQueries(std::string const &entry, std::list<MarkerQuery> const &queries) const;
  EntryIterator runXPath(std::string const &query) const;
  bool validQuery(QueryDialect d, bool variables, std::string const &query) const;

private:
  CorpusReader const *corpusReaderFromPath(std::string const &path) const;
  std::string entryFromPath(std::string const &path) const;

  bf::path d_directory;
  std::list<CorpusReader *> d_corpusReaders;
  std::tr1::unordered_map<std::string, CorpusReader *> d_corpusReaderMap;
};


// Implementation of the public interface.
RecursiveCorpusReader::RecursiveCorpusReader(std::string const &directory) :
  d_private(new RecursiveCorpusReaderPrivate(directory))
{
}

RecursiveCorpusReader::~RecursiveCorpusReader()
{
}

CorpusReader::EntryIterator RecursiveCorpusReader::getBegin() const
{
  return d_private->getBegin();
}

CorpusReader::EntryIterator RecursiveCorpusReader::getEnd() const
{
  return d_private->getEnd();
}

std::string RecursiveCorpusReader::getName() const
{
  return d_private->getName();
}

size_t RecursiveCorpusReader::getSize() const
{
  return d_private->getSize();
}
    
bool RecursiveCorpusReader::validQuery(QueryDialect d, bool variables, std::string const &query) const
{
  return d_private->isValidQuery(d, variables, query);
}

std::string RecursiveCorpusReader::readEntry(std::string const &entry) const
{
  return d_private->readEntry(entry);
}
    
std::string RecursiveCorpusReader::readEntryMarkQueries(std::string const &entry, 
    std::list<MarkerQuery> const &queries) const
{
  return d_private->readEntryMarkQueries(entry, queries);
}

CorpusReader::EntryIterator RecursiveCorpusReader::runXPath(std::string const &query) const
{
  return d_private->query(XPATH, query);
}

// Implementation of the private interface

RecursiveCorpusReaderPrivate::RecursiveCorpusReaderPrivate(std::string const &directory)
{
  if (directory[directory.size() - 1] == '/')
    d_directory = bf::path(directory).parent_path();
  else
    d_directory = bf::path(directory);
  
  if (!bf::exists(d_directory) ||
    !bf::is_directory(d_directory))
    throw OpenError(directory, "non-existent or not a directory");

  // Pick up loose XML files.
  CorpusReader *dirCorpusReader = CorpusReader::open(d_directory.native());
  push_back("", dirCorpusReader);

  for (bf::recursive_directory_iterator iter(d_directory, bf::symlink_option::recurse);
       iter != bf::recursive_directory_iterator();
       ++iter)
  {
    if (iter->path().extension() != ".dact" &&
        iter->path().extension() != ".data.dz")
      continue;

    CorpusReader *reader(CorpusReader::open(iter->path().native()));

    std::string name = iter->path().native();
    name.erase(0, d_directory.native().size() + 1);
    
    push_back(name, reader);
  }
}

RecursiveCorpusReaderPrivate::~RecursiveCorpusReaderPrivate()
{
  for (std::list<CorpusReader *>::iterator iter = d_corpusReaders.begin();
      iter != d_corpusReaders.end(); ++iter)
    delete *iter;
}

CorpusReader::EntryIterator RecursiveCorpusReaderPrivate::getBegin() const
{
  return EntryIterator(new RecursiveIter(d_corpusReaderMap));
}

CorpusReader::EntryIterator RecursiveCorpusReaderPrivate::getEnd() const
{
  return EntryIterator(new RecursiveIter(
    std::tr1::unordered_map<std::string, CorpusReader *>()));
}

std::string RecursiveCorpusReaderPrivate::getName() const
{
  return "<multi>";
}

size_t RecursiveCorpusReaderPrivate::getSize() const
{
  size_t size = 0;

  for (std::list<CorpusReader *>::const_iterator iter =
      d_corpusReaders.begin(); iter != d_corpusReaders.end(); ++iter)
    size += (*iter)->size();

  return size;
}

void RecursiveCorpusReaderPrivate::push_back(std::string const &name,
    CorpusReader *reader)
{
  // Ignore empty corpus readers, simplifies assumptions.
  if (reader->size() == 0) {
    delete reader;
    return;
  }

  d_corpusReaders.push_back(reader);
  d_corpusReaderMap[name] = reader; // XXX - exists check?
}

CorpusReader const *RecursiveCorpusReaderPrivate::corpusReaderFromPath(
    std::string const &path) const
{
  for (std::tr1::unordered_map<std::string, CorpusReader *>::const_iterator iter =
      d_corpusReaderMap.begin(); iter != d_corpusReaderMap.end(); ++iter)
    if (path.find(iter->first) == 0)
      return iter->second;
  
  throw std::runtime_error(std::string("Could not find corpus for: " + path));
}

std::string RecursiveCorpusReaderPrivate::entryFromPath(
    std::string const &path) const
{
  for (std::tr1::unordered_map<std::string, CorpusReader *>::const_iterator iter =
      d_corpusReaderMap.begin(); iter != d_corpusReaderMap.end(); ++iter)
    if (path.find(iter->first) == 0)
      return path.substr(iter->first.size() + 1);

  throw std::runtime_error(std::string("Could not find corpus for: " + path));
}

std::string RecursiveCorpusReaderPrivate::readEntry(std::string const &path) const
{
  CorpusReader const *reader = corpusReaderFromPath(path);
  return reader->read(entryFromPath(path));
}

std::string RecursiveCorpusReaderPrivate::readEntryMarkQueries(
    std::string const &path, std::list<MarkerQuery> const &queries) const
{
  CorpusReader const *reader = corpusReaderFromPath(path);
  return reader->readMarkQueries(entryFromPath(path), queries);
}

CorpusReader::EntryIterator RecursiveCorpusReaderPrivate::runXPath(
    std::string const &query) const
{
  return EntryIterator(new RecursiveIter(d_corpusReaderMap, query));
}

bool RecursiveCorpusReaderPrivate::validQuery(QueryDialect d, bool variables, std::string const &query) const
{
  if (d_corpusReaders.size() > 0)
    return d_corpusReaders.front()->isValidQuery(d, variables, query);

  return false;
}

// Iteration over RecursiveCorpusReaders

RecursiveCorpusReaderPrivate::RecursiveIter::RecursiveIter(
  std::tr1::unordered_map<std::string, CorpusReader *> const &readers)
{
  for (std::tr1::unordered_map<std::string, CorpusReader *>::const_iterator
      iter = readers.begin();
      iter != readers.end(); ++iter)
    d_iters.push_back(ReaderIter(iter->first, iter->second,
          (iter->second->begin())));
}

RecursiveCorpusReaderPrivate::RecursiveIter::RecursiveIter(
  std::tr1::unordered_map<std::string, CorpusReader *> const &readers,
  std::string const &query)
{
  for (std::tr1::unordered_map<std::string, CorpusReader *>::const_iterator
      iter = readers.begin();
      iter != readers.end(); ++iter)
    d_iters.push_back(ReaderIter(iter->first, iter->second,
          (iter->second->query(XPATH, query))));
}

RecursiveCorpusReaderPrivate::RecursiveIter::~RecursiveIter() {}

std::string RecursiveCorpusReaderPrivate::RecursiveIter::current() const
{
  if (d_iters.size() == 0)
    throw std::runtime_error("Cannot dereference an end iterator!");

  // If there is no corpus name/path, we do not need a directory separator.
  // This should only be used by DirectoryCorpusReader(s).
  if (d_iters.front().name.size() == 0)
    return *d_iters.front().iter;

  return d_iters.front().name + "/" + *d_iters.front().iter;
}

bool RecursiveCorpusReaderPrivate::RecursiveIter::equals(IterImpl const &other) const
{
  try {
    RecursiveIter &that = const_cast<RecursiveIter &>(dynamic_cast<RecursiveIter const&>(other));
    return that.d_iters == d_iters;
  } catch (std::bad_cast const &e) {
    return false;
  }
}

void RecursiveCorpusReaderPrivate::RecursiveIter::next() {
  // Iteration is done. Yay.
  if (d_iters.size() == 0)
    return;

  // Move the iterator over the current corpus .
  if (++d_iters.front().iter != d_iters.front().reader->end())
    return;

  // Ok, we are at the end of the current corpus, so we'll remove the
  // corpus from the iteration list, and get started on the next corpus.
  d_iters.pop_front();
}

}

<commit_msg>RecursiveCorpusReader: remove extensions in corpus names.<commit_after>#include <list>
#include <stdexcept>
#include <string>
#include <utility>

#include <tr1/unordered_map>

#include <boost/filesystem.hpp>

#include <AlpinoCorpus/CorpusReader.hh>
#include <AlpinoCorpus/Error.hh>
#include <AlpinoCorpus/RecursiveCorpusReader.hh>

#include <iostream>

namespace bf = boost::filesystem;

namespace alpinocorpus {

struct ReaderIter
{
  ReaderIter(std::string newName, CorpusReader *newReader,
      CorpusReader::EntryIterator newIter) :
    name(newName), reader(newReader), iter(newIter) {}

  std::string name;
  CorpusReader *reader;
  CorpusReader::EntryIterator iter;
};

bool operator==(ReaderIter const &left, ReaderIter const &right)
{
  return left.name == right.name && left.reader == right.reader &&
    left.iter == right.iter;
}

class RecursiveCorpusReaderPrivate : public CorpusReader
{
  class RecursiveIter : public CorpusReader::IterImpl
  {
  public:
    RecursiveIter(std::tr1::unordered_map<std::string, CorpusReader *> const &readers);
    RecursiveIter(std::tr1::unordered_map<std::string, CorpusReader *> const &readers,
      std::string const &query);
    ~RecursiveIter();
    std::string current() const;
    bool equals(IterImpl const &other) const;
    void next();
  private:

    std::list<ReaderIter> d_iters;
  };
public:
  RecursiveCorpusReaderPrivate(std::string const &directory);
  virtual ~RecursiveCorpusReaderPrivate();

  EntryIterator getBegin() const;
  EntryIterator getEnd() const;
  std::string getName() const;
  size_t getSize() const;
  void push_back(std::string const &name, CorpusReader *reader);
  std::string readEntry(std::string const &) const;
  std::string readEntryMarkQueries(std::string const &entry, std::list<MarkerQuery> const &queries) const;
  EntryIterator runXPath(std::string const &query) const;
  bool validQuery(QueryDialect d, bool variables, std::string const &query) const;

private:
  CorpusReader const *corpusReaderFromPath(std::string const &path) const;
  std::string entryFromPath(std::string const &path) const;

  bf::path d_directory;
  std::list<CorpusReader *> d_corpusReaders;
  std::tr1::unordered_map<std::string, CorpusReader *> d_corpusReaderMap;
};


// Implementation of the public interface.
RecursiveCorpusReader::RecursiveCorpusReader(std::string const &directory) :
  d_private(new RecursiveCorpusReaderPrivate(directory))
{
}

RecursiveCorpusReader::~RecursiveCorpusReader()
{
}

CorpusReader::EntryIterator RecursiveCorpusReader::getBegin() const
{
  return d_private->getBegin();
}

CorpusReader::EntryIterator RecursiveCorpusReader::getEnd() const
{
  return d_private->getEnd();
}

std::string RecursiveCorpusReader::getName() const
{
  return d_private->getName();
}

size_t RecursiveCorpusReader::getSize() const
{
  return d_private->getSize();
}
    
bool RecursiveCorpusReader::validQuery(QueryDialect d, bool variables, std::string const &query) const
{
  return d_private->isValidQuery(d, variables, query);
}

std::string RecursiveCorpusReader::readEntry(std::string const &entry) const
{
  return d_private->readEntry(entry);
}
    
std::string RecursiveCorpusReader::readEntryMarkQueries(std::string const &entry, 
    std::list<MarkerQuery> const &queries) const
{
  return d_private->readEntryMarkQueries(entry, queries);
}

CorpusReader::EntryIterator RecursiveCorpusReader::runXPath(std::string const &query) const
{
  return d_private->query(XPATH, query);
}

// Implementation of the private interface

RecursiveCorpusReaderPrivate::RecursiveCorpusReaderPrivate(std::string const &directory)
{
  if (directory[directory.size() - 1] == '/')
    d_directory = bf::path(directory).parent_path();
  else
    d_directory = bf::path(directory);
  
  if (!bf::exists(d_directory) ||
    !bf::is_directory(d_directory))
    throw OpenError(directory, "non-existent or not a directory");

  // Pick up loose XML files.
  CorpusReader *dirCorpusReader = CorpusReader::open(d_directory.native());
  push_back("", dirCorpusReader);

  for (bf::recursive_directory_iterator iter(d_directory, bf::symlink_option::recurse);
       iter != bf::recursive_directory_iterator();
       ++iter)
  {
    if (iter->path().extension() != ".dact" &&
        iter->path().extension() != ".index")
      continue;

    CorpusReader *reader(CorpusReader::open(iter->path().native()));

    bf::path namePath = iter->path();
    namePath.replace_extension("");
    std::string name = namePath.native();

    name.erase(0, d_directory.native().size() + 1);
    
    push_back(name, reader);
  }
}

RecursiveCorpusReaderPrivate::~RecursiveCorpusReaderPrivate()
{
  for (std::list<CorpusReader *>::iterator iter = d_corpusReaders.begin();
      iter != d_corpusReaders.end(); ++iter)
    delete *iter;
}

CorpusReader::EntryIterator RecursiveCorpusReaderPrivate::getBegin() const
{
  return EntryIterator(new RecursiveIter(d_corpusReaderMap));
}

CorpusReader::EntryIterator RecursiveCorpusReaderPrivate::getEnd() const
{
  return EntryIterator(new RecursiveIter(
    std::tr1::unordered_map<std::string, CorpusReader *>()));
}

std::string RecursiveCorpusReaderPrivate::getName() const
{
  return "<multi>";
}

size_t RecursiveCorpusReaderPrivate::getSize() const
{
  size_t size = 0;

  for (std::list<CorpusReader *>::const_iterator iter =
      d_corpusReaders.begin(); iter != d_corpusReaders.end(); ++iter)
    size += (*iter)->size();

  return size;
}

void RecursiveCorpusReaderPrivate::push_back(std::string const &name,
    CorpusReader *reader)
{
  // Ignore empty corpus readers, simplifies assumptions.
  if (reader->size() == 0) {
    delete reader;
    return;
  }

  d_corpusReaders.push_back(reader);
  d_corpusReaderMap[name] = reader; // XXX - exists check?
}

CorpusReader const *RecursiveCorpusReaderPrivate::corpusReaderFromPath(
    std::string const &path) const
{
  for (std::tr1::unordered_map<std::string, CorpusReader *>::const_iterator iter =
      d_corpusReaderMap.begin(); iter != d_corpusReaderMap.end(); ++iter)
    if (path.find(iter->first) == 0)
      return iter->second;
  
  throw std::runtime_error(std::string("Could not find corpus for: " + path));
}

std::string RecursiveCorpusReaderPrivate::entryFromPath(
    std::string const &path) const
{
  for (std::tr1::unordered_map<std::string, CorpusReader *>::const_iterator iter =
      d_corpusReaderMap.begin(); iter != d_corpusReaderMap.end(); ++iter)
    if (path.find(iter->first) == 0)
      return path.substr(iter->first.size() + 1);

  throw std::runtime_error(std::string("Could not find corpus for: " + path));
}

std::string RecursiveCorpusReaderPrivate::readEntry(std::string const &path) const
{
  CorpusReader const *reader = corpusReaderFromPath(path);
  return reader->read(entryFromPath(path));
}

std::string RecursiveCorpusReaderPrivate::readEntryMarkQueries(
    std::string const &path, std::list<MarkerQuery> const &queries) const
{
  CorpusReader const *reader = corpusReaderFromPath(path);
  return reader->readMarkQueries(entryFromPath(path), queries);
}

CorpusReader::EntryIterator RecursiveCorpusReaderPrivate::runXPath(
    std::string const &query) const
{
  return EntryIterator(new RecursiveIter(d_corpusReaderMap, query));
}

bool RecursiveCorpusReaderPrivate::validQuery(QueryDialect d, bool variables, std::string const &query) const
{
  if (d_corpusReaders.size() > 0)
    return d_corpusReaders.front()->isValidQuery(d, variables, query);

  return false;
}

// Iteration over RecursiveCorpusReaders

RecursiveCorpusReaderPrivate::RecursiveIter::RecursiveIter(
  std::tr1::unordered_map<std::string, CorpusReader *> const &readers)
{
  for (std::tr1::unordered_map<std::string, CorpusReader *>::const_iterator
      iter = readers.begin();
      iter != readers.end(); ++iter)
    d_iters.push_back(ReaderIter(iter->first, iter->second,
          (iter->second->begin())));
}

RecursiveCorpusReaderPrivate::RecursiveIter::RecursiveIter(
  std::tr1::unordered_map<std::string, CorpusReader *> const &readers,
  std::string const &query)
{
  for (std::tr1::unordered_map<std::string, CorpusReader *>::const_iterator
      iter = readers.begin();
      iter != readers.end(); ++iter)
    d_iters.push_back(ReaderIter(iter->first, iter->second,
          (iter->second->query(XPATH, query))));
}

RecursiveCorpusReaderPrivate::RecursiveIter::~RecursiveIter() {}

std::string RecursiveCorpusReaderPrivate::RecursiveIter::current() const
{
  if (d_iters.size() == 0)
    throw std::runtime_error("Cannot dereference an end iterator!");

  // If there is no corpus name/path, we do not need a directory separator.
  // This should only be used by DirectoryCorpusReader(s).
  if (d_iters.front().name.size() == 0)
    return *d_iters.front().iter;

  return d_iters.front().name + "/" + *d_iters.front().iter;
}

bool RecursiveCorpusReaderPrivate::RecursiveIter::equals(IterImpl const &other) const
{
  try {
    RecursiveIter &that = const_cast<RecursiveIter &>(dynamic_cast<RecursiveIter const&>(other));
    return that.d_iters == d_iters;
  } catch (std::bad_cast const &e) {
    return false;
  }
}

void RecursiveCorpusReaderPrivate::RecursiveIter::next() {
  // Iteration is done. Yay.
  if (d_iters.size() == 0)
    return;

  // Move the iterator over the current corpus .
  if (++d_iters.front().iter != d_iters.front().reader->end())
    return;

  // Ok, we are at the end of the current corpus, so we'll remove the
  // corpus from the iteration list, and get started on the next corpus.
  d_iters.pop_front();
}

}


{'fraction_non_alphanumeric': 0.13711845642891032, 'fraction_numerical': 0.0024250092255785755, 'mean_word_length': 5.489907629148136, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 125, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2620297', 'n_tokens_mistral': 6263, 'n_tokens_neox': 6167, 'n_words': 1251}