<commit_before>//===-- ArchiveReader.cpp - Read LLVM archive files -------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Builds up standard unix archive files (.a) containing LLVM bitcode.
//
//===----------------------------------------------------------------------===//

#include "ArchiveInternals.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Module.h"
#include <cstdlib>
#include <memory>
using namespace llvm;

/// Read a variable-bit-rate encoded unsigned integer
static inline unsigned readInteger(const char*&At, const char*End) {
  unsigned Shift = 0;
  unsigned Result = 0;

  do {
    if (At == End)
      return Result;
    Result |= (unsigned)((*At++) & 0x7F) << Shift;
    Shift += 7;
  } while (At[-1] & 0x80);
  return Result;
}

// Completely parse the Archive's symbol table and populate symTab member var.
bool
Archive::parseSymbolTable(const void* data, unsigned size, std::string* error) {
  const char* At = (const char*) data;
  const char* End = At + size;
  while (At < End) {
    unsigned offset = readInteger(At, End);
    if (At == End) {
      if (error)
        *error = "Ran out of data reading vbr_uint for symtab offset!";
      return false;
    }
    unsigned length = readInteger(At, End);
    if (At == End) {
      if (error)
        *error = "Ran out of data reading vbr_uint for symtab length!";
      return false;
    }
    if (At + length > End) {
      if (error)
        *error = "Malformed symbol table: length not consistent with size";
      return false;
    }
    // we don't care if it can't be inserted (duplicate entry)
    symTab.insert(std::make_pair(std::string(At, length), offset));
    At += length;
  }
  symTabSize = size;
  return true;
}

// This member parses an ArchiveMemberHeader that is presumed to be pointed to
// by At. The At pointer is updated to the byte just after the header, which
// can be variable in size.
ArchiveMember*
Archive::parseMemberHeader(const char*& At, const char* End, std::string* error)
{
  if (At + sizeof(ArchiveMemberHeader) >= End) {
    if (error)
      *error = "Unexpected end of file";
    return 0;
  }

  // Cast archive member header
  ArchiveMemberHeader* Hdr = (ArchiveMemberHeader*)At;
  At += sizeof(ArchiveMemberHeader);

  // Extract the size and determine if the file is
  // compressed or not (negative length).
  int flags = 0;
  int MemberSize = atoi(Hdr->size);
  if (MemberSize < 0) {
    flags |= ArchiveMember::CompressedFlag;
    MemberSize = -MemberSize;
  }

  // Check the size of the member for sanity
  if (At + MemberSize > End) {
    if (error)
      *error = "invalid member length in archive file";
    return 0;
  }

  // Check the member signature
  if (!Hdr->checkSignature()) {
    if (error)
      *error = "invalid file member signature";
    return 0;
  }

  // Convert and check the member name
  // The empty name ( '/' and 15 blanks) is for a foreign (non-LLVM) symbol
  // table. The special name "//" and 14 blanks is for a string table, used
  // for long file names. This library doesn't generate either of those but
  // it will accept them. If the name starts with #1/ and the remainder is
  // digits, then those digits specify the length of the name that is
  // stored immediately following the header. The special name
  // __LLVM_SYM_TAB__ identifies the symbol table for LLVM bitcode.
  // Anything else is a regular, short filename that is terminated with
  // a '/' and blanks.

  std::string pathname;
  switch (Hdr->name[0]) {
    case '#':
      if (Hdr->name[1] == '1' && Hdr->name[2] == '/') {
        if (isdigit(Hdr->name[3])) {
          unsigned len = atoi(&Hdr->name[3]);
          const char *nulp = (const char *)memchr(At, '\0', len);
          pathname.assign(At, nulp != 0 ? nulp - At : len);
          At += len;
          MemberSize -= len;
          flags |= ArchiveMember::HasLongFilenameFlag;
        } else {
          if (error)
            *error = "invalid long filename";
          return 0;
        }
      } else if (Hdr->name[1] == '_' &&
                 (0 == memcmp(Hdr->name, ARFILE_LLVM_SYMTAB_NAME, 16))) {
        // The member is using a long file name (>15 chars) format.
        // This format is standard for 4.4BSD and Mac OSX operating
        // systems. LLVM uses it similarly. In this format, the
        // remainder of the name field (after #1/) specifies the
        // length of the file name which occupy the first bytes of
        // the member's data. The pathname already has the #1/ stripped.
        pathname.assign(ARFILE_LLVM_SYMTAB_NAME);
        flags |= ArchiveMember::LLVMSymbolTableFlag;
      }
      break;
    case '/':
      if (Hdr->name[1]== '/') {
        if (0 == memcmp(Hdr->name, ARFILE_STRTAB_NAME, 16)) {
          pathname.assign(ARFILE_STRTAB_NAME);
          flags |= ArchiveMember::StringTableFlag;
        } else {
          if (error)
            *error = "invalid string table name";
          return 0;
        }
      } else if (Hdr->name[1] == ' ') {<commit_msg>silence a warning, patch by "mike".<commit_after>//===-- ArchiveReader.cpp - Read LLVM archive files -------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Builds up standard unix archive files (.a) containing LLVM bitcode.
//
//===----------------------------------------------------------------------===//

#include "ArchiveInternals.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Module.h"
#include <cstdlib>
#include <memory>
using namespace llvm;

/// Read a variable-bit-rate encoded unsigned integer
static inline unsigned readInteger(const char*&At, const char*End) {
  unsigned Shift = 0;
  unsigned Result = 0;

  do {
    if (At == End)
      return Result;
    Result |= (unsigned)((*At++) & 0x7F) << Shift;
    Shift += 7;
  } while (At[-1] & 0x80);
  return Result;
}

// Completely parse the Archive's symbol table and populate symTab member var.
bool
Archive::parseSymbolTable(const void* data, unsigned size, std::string* error) {
  const char* At = (const char*) data;
  const char* End = At + size;
  while (At < End) {
    unsigned offset = readInteger(At, End);
    if (At == End) {
      if (error)
        *error = "Ran out of data reading vbr_uint for symtab offset!";
      return false;
    }
    unsigned length = readInteger(At, End);
    if (At == End) {
      if (error)
        *error = "Ran out of data reading vbr_uint for symtab length!";
      return false;
    }
    if (At + length > End) {
      if (error)
        *error = "Malformed symbol table: length not consistent with size";
      return false;
    }
    // we don't care if it can't be inserted (duplicate entry)
    symTab.insert(std::make_pair(std::string(At, length), offset));
    At += length;
  }
  symTabSize = size;
  return true;
}

// This member parses an ArchiveMemberHeader that is presumed to be pointed to
// by At. The At pointer is updated to the byte just after the header, which
// can be variable in size.
ArchiveMember*
Archive::parseMemberHeader(const char*& At, const char* End, std::string* error)
{
  if (At + sizeof(ArchiveMemberHeader) >= End) {
    if (error)
      *error = "Unexpected end of file";
    return 0;
  }

  // Cast archive member header
  ArchiveMemberHeader* Hdr = (ArchiveMemberHeader*)At;
  At += sizeof(ArchiveMemberHeader);

  // Extract the size and determine if the file is
  // compressed or not (negative length).
  int flags = 0;
  int MemberSize = atoi(Hdr->size);
  if (MemberSize < 0) {
    flags |= ArchiveMember::CompressedFlag;
    MemberSize = -MemberSize;
  }

  // Check the size of the member for sanity
  if (At + MemberSize > End) {
    if (error)
      *error = "invalid member length in archive file";
    return 0;
  }

  // Check the member signature
  if (!Hdr->checkSignature()) {
    if (error)
      *error = "invalid file member signature";
    return 0;
  }

  // Convert and check the member name
  // The empty name ( '/' and 15 blanks) is for a foreign (non-LLVM) symbol
  // table. The special name "//" and 14 blanks is for a string table, used
  // for long file names. This library doesn't generate either of those but
  // it will accept them. If the name starts with #1/ and the remainder is
  // digits, then those digits specify the length of the name that is
  // stored immediately following the header. The special name
  // __LLVM_SYM_TAB__ identifies the symbol table for LLVM bitcode.
  // Anything else is a regular, short filename that is terminated with
  // a '/' and blanks.

  std::string pathname;
  switch (Hdr->name[0]) {
    case '#':
      if (Hdr->name[1] == '1' && Hdr->name[2] == '/') {
        if (isdigit(Hdr->name[3])) {
          unsigned len = atoi(&Hdr->name[3]);
          const char *nulp = (const char *)memchr(At, '\0', len);
          pathname.assign(At, nulp != 0 ? (uintptr_t)(nulp - At) : len);
          At += len;
          MemberSize -= len;
          flags |= ArchiveMember::HasLongFilenameFlag;
        } else {
          if (error)
            *error = "invalid long filename";
          return 0;
        }
      } else if (Hdr->name[1] == '_' &&
                 (0 == memcmp(Hdr->name, ARFILE_LLVM_SYMTAB_NAME, 16))) {
        // The member is using a long file name (>15 chars) format.
        // This format is standard for 4.4BSD and Mac OSX operating
        // systems. LLVM uses it similarly. In this format, the
        // remainder of the name field (after #1/) specifies the
        // length of the file name which occupy the first bytes of
        // the member's data. The pathname already has the #1/ stripped.
        pathname.assign(ARFILE_LLVM_SYMTAB_NAME);
        flags |= ArchiveMember::LLVMSymbolTableFlag;
      }
      break;
    case '/':
      if (Hdr->name[1]== '/') {
        if (0 == memcmp(Hdr->name, ARFILE_STRTAB_NAME, 16)) {
          pathname.assign(ARFILE_STRTAB_NAME);
          flags |= ArchiveMember::StringTableFlag;
        } else {
          if (error)
            *error = "invalid string table name";
          return 0;
        }
      } else if (Hdr->name[1] == ' ') {
{'fraction_non_alphanumeric': 0.15089008194405198, 'fraction_numerical': 0.008288593764716964, 'mean_word_length': 2.268082486919052, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1528582', 'n_tokens_mistral': 3250, 'n_tokens_neox': 3109, 'n_words': 1133}