<commit_before>#include "extract_subset.h"
#include "SimpleMoab.h"

bool extract_subset::extract(std::string fin, std::string fout,
                             std::vector<std::string> const& remove_material)
{
  smoab::Interface interface(fin);
  smoab::MaterialTag metTag;

  smoab::EntityHandle rootHandle = interface.getRoot();
  smoab::Range parents = interface.findEntityRootParents(rootHandle);
  smoab::Range dimEnts = interface.findEntitiesWithTag( metTag,
                                                        rootHandle);

  smoab::Range geomParents = smoab::intersect(parents,dimEnts);

  parents.clear(); //remove this range as it is unneeded
  dimEnts.clear();

  //now each item in range can be extracted into a different grid
  typedef smoab::Range::iterator iterator;
  for(iterator i=geomParents.begin(); i != geomParents.end(); ++i)
  {
    std::string name = interface.name(*i);
    if(name.empty())
      name = "!!!!UnknownMaterial!!!!";
    else
      name = name.substr(2);
    bool remove = false;
    for(unsigned int j = 0; j < remove_material.size(); ++j)
    {
      remove = remove || name == remove_material[j];
    }
    if(remove)
    {
      interface.remove(*i);
    }<commit_msg>sometimes the material has a case difference from moab file<commit_after>#include "extract_subset.h"
#include "SimpleMoab.h"

#include <algorithm>

bool extract_subset::extract(std::string fin, std::string fout,
                             std::vector<std::string> const& remove_material)
{
  smoab::Interface interface(fin);
  smoab::MaterialTag metTag;

  smoab::EntityHandle rootHandle = interface.getRoot();
  smoab::Range parents = interface.findEntityRootParents(rootHandle);
  smoab::Range dimEnts = interface.findEntitiesWithTag( metTag,
                                                        rootHandle);

  smoab::Range geomParents = smoab::intersect(parents,dimEnts);

  parents.clear(); //remove this range as it is unneeded
  dimEnts.clear();

  //now each item in range can be extracted into a different grid
  typedef smoab::Range::iterator iterator;
  for(iterator i=geomParents.begin(); i != geomParents.end(); ++i)
  {
    std::string name = interface.name(*i);
    if(name.empty())
      name = "!!!!UnknownMaterial!!!!";
    else
      name = name.substr(2);
    bool remove = false;
    std::transform(name.begin(), name.end(), name.begin(), ::tolower);
    for(unsigned int j = 0; j < remove_material.size(); ++j)
    {
      //NOTE: If this becomes a performance issue, this can be cached
      std::string rmMat = remove_material[j];
      std::transform(rmMat.begin(), rmMat.end(), rmMat.begin(), ::tolower);
      remove = remove || name == rmMat;
    }
    if(remove)
    {
      interface.remove(*i);
    }
{'fraction_non_alphanumeric': 0.13616557734204793, 'fraction_numerical': 0.0014524328249818446, 'mean_word_length': 3.1490963855421685, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '402104', 'n_tokens_mistral': 836, 'n_tokens_neox': 816, 'n_words': 201}