<commit_before>//===-- Core/IncludeExcludeInfo.cpp - IncludeExclude class impl -*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief This file provides the implemention of the IncludeExcludeInfo class
/// to handle the include and exclude command line options.
///
//===----------------------------------------------------------------------===//

#include "IncludeExcludeInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"

using namespace llvm;

namespace {
/// \brief Helper function to determine whether a file has the same path
/// prefix as \a Path.
///
/// \a Path must be an absolute path.
bool fileHasPathPrefix(StringRef File, StringRef Path) {
  // Converts File to its absolute path.
  SmallString<64> AbsoluteFile = File;
  sys::fs::make_absolute(AbsoluteFile);

  // Convert path strings to sys::path to iterate over each of its directories.
  sys::path::const_iterator FileI = sys::path::begin(AbsoluteFile),
                            FileE = sys::path::end(AbsoluteFile),
                            PathI = sys::path::begin(Path),
                            PathE = sys::path::end(Path);
  while (FileI != FileE && PathI != PathE) {
    // If the strings aren't equal then the two paths aren't contained within
    // each other.
    if (!FileI->equals(*PathI))
      return false;
    ++FileI;
    ++PathI;
  }
  return true;
}

/// \brief Helper function to tokenize a string of paths and populate
/// the vector.
error_code parseCLInput(StringRef Line, std::vector<std::string> &List,
                        StringRef Separator) {
  SmallVector<StringRef, 32> Tokens;
  Line.split(Tokens, Separator, /*MaxSplit=*/ -1, /*KeepEmpty=*/ false);
  for (SmallVectorImpl<StringRef>::iterator I = Tokens.begin(),
                                            E = Tokens.end();
       I != E; ++I) {
    // Convert each path to its absolute path.
    SmallString<64> AbsolutePath = I->rtrim();
    if (error_code Err = sys::fs::make_absolute(AbsolutePath))
      return Err;
    List.push_back(std::string(AbsolutePath.str()));
  }
  return error_code::success();
}
} // end anonymous namespace

error_code IncludeExcludeInfo::readListFromString(StringRef IncludeString,
                                                  StringRef ExcludeString) {
  if (error_code Err = parseCLInput(IncludeString, IncludeList,
                                    /*Separator=*/ ","))
    return Err;
  if (error_code Err = parseCLInput(ExcludeString, ExcludeList,
                                    /*Separator=*/ ","))
    return Err;
  return error_code::success();
}

error_code IncludeExcludeInfo::readListFromFile(StringRef IncludeListFile,
                                                StringRef ExcludeListFile) {
  if (!IncludeListFile.empty()) {
    OwningPtr<MemoryBuffer> FileBuf;
    if (error_code Err = MemoryBuffer::getFile(IncludeListFile, FileBuf)) {
      errs() << "Unable to read from include file.\n";
      return Err;
    }
    if (error_code Err = parseCLInput(FileBuf->getBuffer(), IncludeList,
                                      /*Separator=*/ "\n"))
      return Err;
  }
  if (!ExcludeListFile.empty()) {
    OwningPtr<MemoryBuffer> FileBuf;
    if (error_code Err = MemoryBuffer::getFile(ExcludeListFile, FileBuf)) {<commit_msg>cpp11-migrate: Fix parsing of '.' for header include/excludes<commit_after>//===-- Core/IncludeExcludeInfo.cpp - IncludeExclude class impl -*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief This file provides the implemention of the IncludeExcludeInfo class
/// to handle the include and exclude command line options.
///
//===----------------------------------------------------------------------===//

#include "IncludeExcludeInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"

using namespace llvm;

namespace {
/// \brief Helper function to determine whether a file has the same path
/// prefix as \a Path.
///
/// \a Path must be an absolute path.
bool fileHasPathPrefix(StringRef File, StringRef Path) {
  // Converts File to its absolute path.
  SmallString<64> AbsoluteFile = File;
  sys::fs::make_absolute(AbsoluteFile);

  // Convert path strings to sys::path to iterate over each of its directories.
  sys::path::const_iterator FileI = sys::path::begin(AbsoluteFile),
                            FileE = sys::path::end(AbsoluteFile),
                            PathI = sys::path::begin(Path),
                            PathE = sys::path::end(Path);
  while (FileI != FileE && PathI != PathE) {
    // If the strings aren't equal then the two paths aren't contained within
    // each other.
    if (!FileI->equals(*PathI))
      return false;
    ++FileI;
    ++PathI;
  }
  return true;
}

/// \brief Helper function to tokenize a string of paths and populate
/// the vector.
error_code parseCLInput(StringRef Line, std::vector<std::string> &List,
                        StringRef Separator) {
  SmallVector<StringRef, 32> Tokens;
  Line.split(Tokens, Separator, /*MaxSplit=*/ -1, /*KeepEmpty=*/ false);
  for (SmallVectorImpl<StringRef>::iterator I = Tokens.begin(),
                                            E = Tokens.end();
       I != E; ++I) {
    // Convert each path to its absolute path.
    SmallString<64> Path = I->rtrim();
    if (error_code Err = sys::fs::make_absolute(Path))
      return Err;

    // sys::fs::make_absolute will turn "." into "<pwd>/.". Need to strip "/."
    // off or it interferes with prefix checking.
    if (Path.endswith("/."))
      List.push_back(Path.slice(0, Path.size() - 2).str());
    else
      List.push_back(Path.str());

    llvm::errs() << "Parse: " <<List.back() << "\n";
  }
  return error_code::success();
}
} // end anonymous namespace

error_code IncludeExcludeInfo::readListFromString(StringRef IncludeString,
                                                  StringRef ExcludeString) {
  if (error_code Err = parseCLInput(IncludeString, IncludeList,
                                    /*Separator=*/ ","))
    return Err;
  if (error_code Err = parseCLInput(ExcludeString, ExcludeList,
                                    /*Separator=*/ ","))
    return Err;
  return error_code::success();
}

error_code IncludeExcludeInfo::readListFromFile(StringRef IncludeListFile,
                                                StringRef ExcludeListFile) {
  if (!IncludeListFile.empty()) {
    OwningPtr<MemoryBuffer> FileBuf;
    if (error_code Err = MemoryBuffer::getFile(IncludeListFile, FileBuf)) {
      errs() << "Unable to read from include file.\n";
      return Err;
    }
    if (error_code Err = parseCLInput(FileBuf->getBuffer(), IncludeList,
                                      /*Separator=*/ "\n"))
      return Err;
  }
  if (!ExcludeListFile.empty()) {
    OwningPtr<MemoryBuffer> FileBuf;
    if (error_code Err = MemoryBuffer::getFile(ExcludeListFile, FileBuf)) {
{'fraction_non_alphanumeric': 0.16484385294505205, 'fraction_numerical': 0.002371853999209382, 'mean_word_length': 2.1850608476710027, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4428397', 'n_tokens_mistral': 2093, 'n_tokens_neox': 2056, 'n_words': 575}