<commit_before>#include <stdexcept>
#include "retriever.h"
#include "nexus_retriever.h"

#ifdef IPNS_RETRIEVER
#include "IPNS_CPP/ipns_retriever.h"
#endif
#ifdef TEXT_PLAIN_RETRIEVER
#include "text_plain/retriever.h"
#endif
#ifdef TEXT_XML_RETRIEVER
#include "text_xml/retriever.h"
#endif
#ifdef TEXT_COLLIST_RETRIEVER
#include "text_collist/collist_retriever.h"
#endif
#ifdef DYNAMIC_RETRIEVER
#include "dynamic_retriever.h"
#endif
#ifdef SNS_HISTOGRAM_RETRIEVER
#include "sns_histogram/retriever.h"
#endif

using std::string;
using std::invalid_argument;

// Implementation of a pure virtual destructor. This makes the compiler happy.
Retriever::~Retriever(){}

// factory method
Retriever::RetrieverPtr Retriever::getInstance(const string & type, const string &source){
  // return appropriate retriever based on type
  if(type==NexusRetriever::MIME_TYPE){
    RetrieverPtr ptr(new NexusRetriever(source));
    return ptr;
#ifdef IPNS_RETRIEVER
  }else if(type==IpnsRetriever::MIME_TYPE){
    RetrieverPtr ptr(new IpnsRetriever(source));
    return ptr;
#endif
#ifdef TEXT_PLAIN_RETRIEVER
  }else if(type==TextPlainRetriever::MIME_TYPE){
    RetrieverPtr ptr(new TextPlainRetriever(source));
    return ptr;
#endif
#ifdef TEXT_COLLIST_RETRIEVER
  }else if(type==TextCollistRetriever::MIME_TYPE){
    RetrieverPtr ptr(new TextCollistRetriever(source));
    return ptr;
#endif
#ifdef TEXT_XML_RETRIEVER
  }else if(type==TextXmlRetriever::MIME_TYPE){
    RetrieverPtr ptr(new TextXmlRetriever(source));
    return ptr;
#endif
#ifdef DYNAMIC_RETRIEVER
  }else if(type.substr(0,8) == "dynamic/") {
    RetrieverPtr ptr(new DynamicRetriever(source, type));
    return ptr;
#endif
#ifdef SNS_HISTOGRAM_RETRIEVER
  }else if(type==SnsHistogramRetriever::MIME_TYPE){
    RetrieverPtr ptr(new SnsHistogramRetriever(source);
    return ptr;
#endif
  }

  // if it gets this far the type is not understood
  throw invalid_argument("do not understand mime_type ("+type+") in retriever_factory");
}<commit_msg>Added missing  ')' for sns_histogram loading.<commit_after>#include <stdexcept>
#include "retriever.h"
#include "nexus_retriever.h"

#ifdef IPNS_RETRIEVER
#include "IPNS_CPP/ipns_retriever.h"
#endif
#ifdef TEXT_PLAIN_RETRIEVER
#include "text_plain/retriever.h"
#endif
#ifdef TEXT_XML_RETRIEVER
#include "text_xml/retriever.h"
#endif
#ifdef TEXT_COLLIST_RETRIEVER
#include "text_collist/collist_retriever.h"
#endif
#ifdef DYNAMIC_RETRIEVER
#include "dynamic_retriever.h"
#endif
#ifdef SNS_HISTOGRAM_RETRIEVER
#include "sns_histogram/retriever.h"
#endif

using std::string;
using std::invalid_argument;

// Implementation of a pure virtual destructor. This makes the compiler happy.
Retriever::~Retriever(){}

// factory method
Retriever::RetrieverPtr Retriever::getInstance(const string & type, const string &source){
  // return appropriate retriever based on type
  if(type==NexusRetriever::MIME_TYPE){
    RetrieverPtr ptr(new NexusRetriever(source));
    return ptr;
#ifdef IPNS_RETRIEVER
  }else if(type==IpnsRetriever::MIME_TYPE){
    RetrieverPtr ptr(new IpnsRetriever(source));
    return ptr;
#endif
#ifdef TEXT_PLAIN_RETRIEVER
  }else if(type==TextPlainRetriever::MIME_TYPE){
    RetrieverPtr ptr(new TextPlainRetriever(source));
    return ptr;
#endif
#ifdef TEXT_COLLIST_RETRIEVER
  }else if(type==TextCollistRetriever::MIME_TYPE){
    RetrieverPtr ptr(new TextCollistRetriever(source));
    return ptr;
#endif
#ifdef TEXT_XML_RETRIEVER
  }else if(type==TextXmlRetriever::MIME_TYPE){
    RetrieverPtr ptr(new TextXmlRetriever(source));
    return ptr;
#endif
#ifdef DYNAMIC_RETRIEVER
  }else if(type.substr(0,8) == "dynamic/") {
    RetrieverPtr ptr(new DynamicRetriever(source, type));
    return ptr;
#endif
#ifdef SNS_HISTOGRAM_RETRIEVER
  }else if(type==SnsHistogramRetriever::MIME_TYPE){
    RetrieverPtr ptr(new SnsHistogramRetriever(source));
    return ptr;
#endif
  }

  // if it gets this far the type is not understood
  throw invalid_argument("do not understand mime_type ("+type+") in retriever_factory");
}
{'fraction_non_alphanumeric': 0.10796723752792256, 'fraction_numerical': 0.0009928021841648052, 'mean_word_length': 6.380952380952381, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7305084', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1484, 'n_words': 310}