<commit_before>//===- fpcmp.cpp - A fuzzy "cmp" that permits floating point noise --------===//
// 
//                     The LLVM Compiler Infrastructure
//
// This file was developed by the LLVM research group and is distributed under
// the University of Illinois Open Source License. See LICENSE.TXT for details.
// 
//===----------------------------------------------------------------------===//
//
// fpcmp is a tool that basically works like the 'cmp' tool, except that it can
// tolerate errors due to floating point noise, with the -r option.
//
//===----------------------------------------------------------------------===//

#include "Support/CommandLine.h"
#include "Support/FileUtilities.h"
#include "Config/fcntl.h"
#include "Config/sys/mman.h"
#include <iostream>
#include <cmath>

using namespace llvm;

namespace {
  cl::opt<std::string>
  File1(cl::Positional, cl::desc("<input file #1>"), cl::Required);
  cl::opt<std::string>
  File2(cl::Positional, cl::desc("<input file #2>"), cl::Required);

  cl::opt<double>
  RelTolerance("r", cl::desc("Relative error tolerated"), cl::init(0));
  cl::opt<double>
  AbsTolerance("a", cl::desc("Absolute error tolerated"), cl::init(0));
}


/// OpenFile - mmap the specified file into the address space for reading, and
/// return the length and address of the buffer.
static void OpenFile(const std::string &Filename, unsigned &Len, char* &BufPtr){
  int FD = open(Filename.c_str(), O_RDONLY);
  if (FD == -1 || (Len = getFileSize(Filename)) == ~0U) {
    std::cerr << "Error: cannot open file '" << Filename << "'\n";
    exit(2);
  }

  // mmap in the file all at once...
  BufPtr = (char*)mmap(0, Len, PROT_READ, MAP_PRIVATE, FD, 0);
  
  if (BufPtr == (char*)MAP_FAILED) {
    std::cerr << "Error: cannot open file '" << Filename << "'\n";
    exit(2);
  }
}

static bool isNumberChar(char C) {
  switch (C) {
  case '0': case '1': case '2': case '3': case '4':
  case '5': case '6': case '7': case '8': case '9': 
  case '.': case '+': case '-':
  case 'e':
  case 'E': return true;
  default: return false;
  }
}<commit_msg>mmap of a zero length file returns null on some platforms, so hack around it.<commit_after>//===- fpcmp.cpp - A fuzzy "cmp" that permits floating point noise --------===//
// 
//                     The LLVM Compiler Infrastructure
//
// This file was developed by the LLVM research group and is distributed under
// the University of Illinois Open Source License. See LICENSE.TXT for details.
// 
//===----------------------------------------------------------------------===//
//
// fpcmp is a tool that basically works like the 'cmp' tool, except that it can
// tolerate errors due to floating point noise, with the -r option.
//
//===----------------------------------------------------------------------===//

#include "Support/CommandLine.h"
#include "Support/FileUtilities.h"
#include "Config/fcntl.h"
#include "Config/sys/mman.h"
#include <iostream>
#include <cmath>

using namespace llvm;

namespace {
  cl::opt<std::string>
  File1(cl::Positional, cl::desc("<input file #1>"), cl::Required);
  cl::opt<std::string>
  File2(cl::Positional, cl::desc("<input file #2>"), cl::Required);

  cl::opt<double>
  RelTolerance("r", cl::desc("Relative error tolerated"), cl::init(0));
  cl::opt<double>
  AbsTolerance("a", cl::desc("Absolute error tolerated"), cl::init(0));
}


/// OpenFile - mmap the specified file into the address space for reading, and
/// return the length and address of the buffer.
static void OpenFile(const std::string &Filename, unsigned &Len, char* &BufPtr){
  int FD = open(Filename.c_str(), O_RDONLY);
  if (FD == -1 || (Len = getFileSize(Filename)) == ~0U) {
    std::cerr << "Error: cannot open file '" << Filename << "'\n";
    exit(2);
  }

  // mmap in the file all at once...
  BufPtr = (char*)mmap(0, Len, PROT_READ, MAP_PRIVATE, FD, 0);
  
  if (BufPtr == (char*)MAP_FAILED) {
    std::cerr << "Error: cannot open file '" << Filename << "'\n";
    exit(2);
  }

  // If mmap decided that the files were empty, it might have returned a
  // null pointer. If so, make a new, fake pointer -- it shouldn't matter
  // what it contains, because Len is 0, and it should never be read.
  if (BufPtr == 0 && Len == 0)
    BufPtr = new char[1];
}

static bool isNumberChar(char C) {
  switch (C) {
  case '0': case '1': case '2': case '3': case '4':
  case '5': case '6': case '7': case '8': case '9': 
  case '.': case '+': case '-':
  case 'e':
  case 'E': return true;
  default: return false;
  }
}
{'fraction_non_alphanumeric': 0.23037806765421182, 'fraction_numerical': 0.010612425381384037, 'mean_word_length': 2.6572352465642686, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '198348', 'n_tokens_mistral': 1457, 'n_tokens_neox': 1413, 'n_words': 492}