<commit_before>//===-- CXLoadedDiagnostic.cpp - Handling of persisent diags ----*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Implements handling of persisent diagnostics.
//
//===----------------------------------------------------------------------===//

#include "CXLoadedDiagnostic.h"
#include "CXString.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/FileManager.h"
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/Optional.h"
#include "clang/Basic/LLVM.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include "llvm/Support/MemoryBuffer.h"
#include <assert.h>

using namespace clang;
using namespace clang::cxstring;

//===----------------------------------------------------------------------===//
// Extend CXDiagnosticSetImpl which contains strings for diagnostics.
//===----------------------------------------------------------------------===//

typedef llvm::DenseMap<unsigned, llvm::StringRef> Strings;

namespace {
class CXLoadedDiagnosticSetImpl : public CXDiagnosticSetImpl {
public:
  CXLoadedDiagnosticSetImpl() : CXDiagnosticSetImpl(true), FakeFiles(FO) {}
  virtual ~CXLoadedDiagnosticSetImpl() {}  

  llvm::StringRef makeString(const char *blob, unsigned blobLen);
  
  llvm::BumpPtrAllocator Alloc;
  Strings Categories;
  Strings WarningFlags;
  Strings FileNames;
  
  FileSystemOptions FO;
  FileManager FakeFiles;
  llvm::DenseMap<unsigned, const FileEntry *> Files;
};
}

llvm::StringRef CXLoadedDiagnosticSetImpl::makeString(const char *blob,
                                                      unsigned bloblen) {
  char *mem = Alloc.Allocate<char>(bloblen + 1);
  memcpy(mem, blob, bloblen);
  // Add a null terminator for those clients accessing the buffer
  // like a c-string.
  mem[bloblen] = '\0';
  return llvm::StringRef(mem, bloblen);
}

//===----------------------------------------------------------------------===//
// Cleanup.
//===----------------------------------------------------------------------===//

CXLoadedDiagnostic::~CXLoadedDiagnostic() {}

//===----------------------------------------------------------------------===//
// Public CXLoadedDiagnostic methods.
//===----------------------------------------------------------------------===//

CXDiagnosticSeverity CXLoadedDiagnostic::getSeverity() const {
  // FIXME: possibly refactor with logic in CXStoredDiagnostic.
  switch (severity) {
    case DiagnosticsEngine::Ignored: return CXDiagnostic_Ignored;
    case DiagnosticsEngine::Note:    return CXDiagnostic_Note;
    case DiagnosticsEngine::Warning: return CXDiagnostic_Warning;
    case DiagnosticsEngine::Error:   return CXDiagnostic_Error;
    case DiagnosticsEngine::Fatal:   return CXDiagnostic_Fatal;
  }
  
  llvm_unreachable("Invalid diagnostic level");
}

static CXSourceLocation makeLocation(const CXLoadedDiagnostic::Location *DLoc) {
  // The lowest bit of ptr_data[0] is always set to 1 to indicate this
  // is a persistent diagnostic.
  uintptr_t V = (uintptr_t) DLoc;
  V |= 0x1;
  CXSourceLocation Loc = { {  (void*) V, 0 }, 0 };
  return Loc;
}  

CXSourceLocation CXLoadedDiagnostic::getLocation() const {
  // The lowest bit of ptr_data[0] is always set to 1 to indicate this
  // is a persistent diagnostic.
  return makeLocation(&DiagLoc);
}

CXString CXLoadedDiagnostic::getSpelling() const {
  return cxstring::createCXString(Spelling, false);
}

CXString CXLoadedDiagnostic::getDiagnosticOption(CXString *Disable) const {
  if (DiagOption.empty())
    return createCXString("");

  // FIXME: possibly refactor with logic in CXStoredDiagnostic.
  if (Disable)
    *Disable = createCXString((Twine("-Wno-") + DiagOption).str());
  return createCXString((Twine("-W") + DiagOption).str());
}

unsigned CXLoadedDiagnostic::getCategory() const {
  return category;
}

CXString CXLoadedDiagnostic::getCategoryText() const {
  return cxstring::createCXString(CategoryText);
}

unsigned CXLoadedDiagnostic::getNumRanges() const {
  return Ranges.size();
}

CXSourceRange CXLoadedDiagnostic::getRange(unsigned Range) const {
  assert(Range < Ranges.size());
  return Ranges[Range];
}

unsigned CXLoadedDiagnostic::getNumFixIts() const {
  return FixIts.size();
}

CXString CXLoadedDiagnostic::getFixIt(unsigned FixIt,
                                      CXSourceRange *ReplacementRange) const {
  assert(FixIt < FixIts.size());
  if (ReplacementRange)
    *ReplacementRange = FixIts[FixIt].first;
  return FixIts[FixIt].second;
}

void CXLoadedDiagnostic::decodeLocation(CXSourceLocation location,
                                        CXFile *file,
                                        unsigned int *line,
                                        unsigned int *column,
                                        unsigned int *offset) {
  
  
  // CXSourceLocation consists of the following fields:
  //
  //   void *ptr_data[2];
  //   unsigned int_data;
  //
  // The lowest bit of ptr_data[0] is always set to 1 to indicate this
  // is a persistent diagnostic.
  //
  // For now, do the unoptimized approach and store the data in a side
  // data structure.  We can optimize this case later.
  
  uintptr_t V = (uintptr_t) location.ptr_data[0];
  assert((V & 0x1) == 1);
  V &= ~(uintptr_t)1;
  
  const Location &Loc = *((Location*)V);
  
  if (file)
    *file = Loc.file;  
  if (line)
    *line = Loc.line;
  if (column)
    *column = Loc.column;
  if (offset)
    *offset = Loc.offset;
}

//===----------------------------------------------------------------------===//
// Deserialize diagnostics.
//===----------------------------------------------------------------------===//

enum { MaxSupportedVersion = 1 };
typedef SmallVector<uint64_t, 64> RecordData;
enum LoadResult { Failure = 1, Success = 0 };
enum StreamResult { Read_EndOfStream,
                    Read_BlockBegin,
                    Read_Failure,
                    Read_Record,
                    Read_BlockEnd };

namespace {
class DiagLoader {
  enum CXLoadDiag_Error *error;
  CXString *errorString;
  
  void reportBad(enum CXLoadDiag_Error code, llvm::StringRef err) {
    if (error)
      *error = code;
    if (errorString)
      *errorString = createCXString(err);
  }
  
  void reportInvalidFile(llvm::StringRef err) {
    return reportBad(CXLoadDiag_InvalidFile, err);
  }

  LoadResult readMetaBlock(llvm::BitstreamCursor &Stream);
  
  LoadResult readDiagnosticBlock(llvm::BitstreamCursor &Stream,
                                 CXDiagnosticSetImpl &Diags,
                                 CXLoadedDiagnosticSetImpl &TopDiags);

  StreamResult readToNextRecordOrBlock(llvm::BitstreamCursor &Stream,
                                       llvm::StringRef errorContext,
                                       unsigned &BlockOrRecordID,
                                       const bool atTopLevel = false);
  
  
  LoadResult readString(CXLoadedDiagnosticSetImpl &TopDiags,
                        Strings &strings, llvm::StringRef errorContext,
                        RecordData &Record,
                        const char *BlobStart,
                        unsigned BlobLen,
                        bool allowEmptyString = false);

  LoadResult readString(CXLoadedDiagnosticSetImpl &TopDiags,
                        llvm::StringRef &RetStr,
                        llvm::StringRef errorContext,
                        RecordData &Record,
                        const char *BlobStart,
                        unsigned BlobLen,
                        bool allowEmptyString = false);

  LoadResult readRange(CXLoadedDiagnosticSetImpl &TopDiags,
                       RecordData &Record, unsigned RecStartIdx,
                       CXSourceRange &SR);
  
  LoadResult readLocation(CXLoadedDiagnosticSetImpl &TopDiags,
                          RecordData &Record, unsigned &offset,
                          CXLoadedDiagnostic::Location &Loc);
                       
public:
  DiagLoader(enum CXLoadDiag_Error *e, CXString *es)
    : error(e), errorString(es) {
      if (error)
        *error = CXLoadDiag_None;
      if (errorString)
        *errorString = createCXString("");
    }

  CXDiagnosticSet load(const char *file);
};
}

CXDiagnosticSet DiagLoader::load(const char *file) {
  // Open the diagnostics file.
  std::string ErrStr;
  FileSystemOptions FO;
  FileManager FileMgr(FO);

  OwningPtr<llvm::MemoryBuffer> Buffer;
  Buffer.reset(FileMgr.getBufferForFile(file));

  if (!Buffer) {
    reportBad(CXLoadDiag_CannotLoad, ErrStr);
    return 0;
  }

  llvm::BitstreamReader StreamFile;
  StreamFile.init((const unsigned char *)Buffer->getBufferStart(),
                  (const unsigned char *)Buffer->getBufferEnd());

  llvm::BitstreamCursor Stream;
  Stream.init(StreamFile);

  // Sniff for the signature.
  if (Stream.Read(8) != 'D' ||
      Stream.Read(8) != 'I' ||
      Stream.Read(8) != 'A' ||
      Stream.Read(8) != 'G') {
    reportBad(CXLoadDiag_InvalidFile,
              "Bad header in diagnostics file");
    return 0;
  }

  OwningPtr<CXLoadedDiagnosticSetImpl>
    Diags(new CXLoadedDiagnosticSetImpl());

  while (true) {
    unsigned BlockID = 0;
    StreamResult Res = readToNextRecordOrBlock(Stream, "Top-level", 
                                               BlockID, true);
    switch (Res) {
      case Read_EndOfStream:
        return (CXDiagnosticSet) Diags.take();
      case Read_Failure:
        return 0;
      case Read_Record:
        llvm_unreachable("Top-level does not have records");
      case Read_BlockEnd:
        continue;
      case Read_BlockBegin:
        break;
    }
    
    switch (BlockID) {
      case serialized_diags::BLOCK_META:
        if (readMetaBlock(Stream))
          return 0;
        break;
      case serialized_diags::BLOCK_DIAG:
        if (readDiagnosticBlock(Stream, *Diags.get(), *Diags.get()))
          return 0;
        break;
      default:
        if (!Stream.SkipBlock()) {
          reportInvalidFile("Malformed block at top-level of diagnostics file");
          return 0;
        }
        break;
    }
  }
}

StreamResult DiagLoader::readToNextRecordOrBlock(llvm::BitstreamCursor &Stream,
                                                 llvm::StringRef errorContext,
                                                 unsigned &blockOrRecordID,
                                                 const bool atTopLevel) {
  
  blockOrRecordID = 0;

  while (!Stream.AtEndOfStream()) {
    unsigned Code = Stream.ReadCode();

    // Handle the top-level specially.
    if (atTopLevel) {
      if (Code == llvm::bitc::ENTER_SUBBLOCK) {
        unsigned BlockID = Stream.ReadSubBlockID();
        if (BlockID == llvm::bitc::BLOCKINFO_BLOCK_ID) {
          if (Stream.ReadBlockInfoBlock()) {
            reportInvalidFile("Malformed BlockInfoBlock in diagnostics file");
            return Read_Failure;
          }
          continue;
        }
        blockOrRecordID = BlockID;
        return Read_BlockBegin;
      }
      reportInvalidFile("Only blocks can appear at the top of a "
                        "diagnostic file");<commit_msg>random tidying<commit_after>//===-- CXLoadedDiagnostic.cpp - Handling of persisent diags ----*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Implements handling of persisent diagnostics.
//
//===----------------------------------------------------------------------===//

#include "CXLoadedDiagnostic.h"
#include "CXString.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/FileManager.h"
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/Optional.h"
#include "clang/Basic/LLVM.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include "llvm/Support/MemoryBuffer.h"
using namespace clang;
using namespace clang::cxstring;

//===----------------------------------------------------------------------===//
// Extend CXDiagnosticSetImpl which contains strings for diagnostics.
//===----------------------------------------------------------------------===//

typedef llvm::DenseMap<unsigned, llvm::StringRef> Strings;

namespace {
class CXLoadedDiagnosticSetImpl : public CXDiagnosticSetImpl {
public:
  CXLoadedDiagnosticSetImpl() : CXDiagnosticSetImpl(true), FakeFiles(FO) {}
  virtual ~CXLoadedDiagnosticSetImpl() {}  

  llvm::StringRef makeString(const char *blob, unsigned blobLen);
  
  llvm::BumpPtrAllocator Alloc;
  Strings Categories;
  Strings WarningFlags;
  Strings FileNames;
  
  FileSystemOptions FO;
  FileManager FakeFiles;
  llvm::DenseMap<unsigned, const FileEntry *> Files;
};
}

llvm::StringRef CXLoadedDiagnosticSetImpl::makeString(const char *blob,
                                                      unsigned bloblen) {
  char *mem = Alloc.Allocate<char>(bloblen + 1);
  memcpy(mem, blob, bloblen);
  // Add a null terminator for those clients accessing the buffer
  // like a c-string.
  mem[bloblen] = '\0';
  return llvm::StringRef(mem, bloblen);
}

//===----------------------------------------------------------------------===//
// Cleanup.
//===----------------------------------------------------------------------===//

CXLoadedDiagnostic::~CXLoadedDiagnostic() {}

//===----------------------------------------------------------------------===//
// Public CXLoadedDiagnostic methods.
//===----------------------------------------------------------------------===//

CXDiagnosticSeverity CXLoadedDiagnostic::getSeverity() const {
  // FIXME: possibly refactor with logic in CXStoredDiagnostic.
  switch (severity) {
    case DiagnosticsEngine::Ignored: return CXDiagnostic_Ignored;
    case DiagnosticsEngine::Note:    return CXDiagnostic_Note;
    case DiagnosticsEngine::Warning: return CXDiagnostic_Warning;
    case DiagnosticsEngine::Error:   return CXDiagnostic_Error;
    case DiagnosticsEngine::Fatal:   return CXDiagnostic_Fatal;
  }
  
  llvm_unreachable("Invalid diagnostic level");
}

static CXSourceLocation makeLocation(const CXLoadedDiagnostic::Location *DLoc) {
  // The lowest bit of ptr_data[0] is always set to 1 to indicate this
  // is a persistent diagnostic.
  uintptr_t V = (uintptr_t) DLoc;
  V |= 0x1;
  CXSourceLocation Loc = { {  (void*) V, 0 }, 0 };
  return Loc;
}  

CXSourceLocation CXLoadedDiagnostic::getLocation() const {
  // The lowest bit of ptr_data[0] is always set to 1 to indicate this
  // is a persistent diagnostic.
  return makeLocation(&DiagLoc);
}

CXString CXLoadedDiagnostic::getSpelling() const {
  return cxstring::createCXString(Spelling, false);
}

CXString CXLoadedDiagnostic::getDiagnosticOption(CXString *Disable) const {
  if (DiagOption.empty())
    return createCXString("");

  // FIXME: possibly refactor with logic in CXStoredDiagnostic.
  if (Disable)
    *Disable = createCXString((Twine("-Wno-") + DiagOption).str());
  return createCXString((Twine("-W") + DiagOption).str());
}

unsigned CXLoadedDiagnostic::getCategory() const {
  return category;
}

CXString CXLoadedDiagnostic::getCategoryText() const {
  return cxstring::createCXString(CategoryText);
}

unsigned CXLoadedDiagnostic::getNumRanges() const {
  return Ranges.size();
}

CXSourceRange CXLoadedDiagnostic::getRange(unsigned Range) const {
  assert(Range < Ranges.size());
  return Ranges[Range];
}

unsigned CXLoadedDiagnostic::getNumFixIts() const {
  return FixIts.size();
}

CXString CXLoadedDiagnostic::getFixIt(unsigned FixIt,
                                      CXSourceRange *ReplacementRange) const {
  assert(FixIt < FixIts.size());
  if (ReplacementRange)
    *ReplacementRange = FixIts[FixIt].first;
  return FixIts[FixIt].second;
}

void CXLoadedDiagnostic::decodeLocation(CXSourceLocation location,
                                        CXFile *file,
                                        unsigned int *line,
                                        unsigned int *column,
                                        unsigned int *offset) {
  
  
  // CXSourceLocation consists of the following fields:
  //
  //   void *ptr_data[2];
  //   unsigned int_data;
  //
  // The lowest bit of ptr_data[0] is always set to 1 to indicate this
  // is a persistent diagnostic.
  //
  // For now, do the unoptimized approach and store the data in a side
  // data structure.  We can optimize this case later.
  
  uintptr_t V = (uintptr_t) location.ptr_data[0];
  assert((V & 0x1) == 1);
  V &= ~(uintptr_t)1;
  
  const Location &Loc = *((Location*)V);
  
  if (file)
    *file = Loc.file;  
  if (line)
    *line = Loc.line;
  if (column)
    *column = Loc.column;
  if (offset)
    *offset = Loc.offset;
}

//===----------------------------------------------------------------------===//
// Deserialize diagnostics.
//===----------------------------------------------------------------------===//

enum { MaxSupportedVersion = 1 };
typedef SmallVector<uint64_t, 64> RecordData;
enum LoadResult { Failure = 1, Success = 0 };
enum StreamResult { Read_EndOfStream,
                    Read_BlockBegin,
                    Read_Failure,
                    Read_Record,
                    Read_BlockEnd };

namespace {
class DiagLoader {
  enum CXLoadDiag_Error *error;
  CXString *errorString;
  
  void reportBad(enum CXLoadDiag_Error code, llvm::StringRef err) {
    if (error)
      *error = code;
    if (errorString)
      *errorString = createCXString(err);
  }
  
  void reportInvalidFile(llvm::StringRef err) {
    return reportBad(CXLoadDiag_InvalidFile, err);
  }

  LoadResult readMetaBlock(llvm::BitstreamCursor &Stream);
  
  LoadResult readDiagnosticBlock(llvm::BitstreamCursor &Stream,
                                 CXDiagnosticSetImpl &Diags,
                                 CXLoadedDiagnosticSetImpl &TopDiags);

  StreamResult readToNextRecordOrBlock(llvm::BitstreamCursor &Stream,
                                       llvm::StringRef errorContext,
                                       unsigned &BlockOrRecordID,
                                       bool atTopLevel = false);
  
  
  LoadResult readString(CXLoadedDiagnosticSetImpl &TopDiags,
                        Strings &strings, llvm::StringRef errorContext,
                        RecordData &Record,
                        const char *BlobStart,
                        unsigned BlobLen,
                        bool allowEmptyString = false);

  LoadResult readString(CXLoadedDiagnosticSetImpl &TopDiags,
                        llvm::StringRef &RetStr,
                        llvm::StringRef errorContext,
                        RecordData &Record,
                        const char *BlobStart,
                        unsigned BlobLen,
                        bool allowEmptyString = false);

  LoadResult readRange(CXLoadedDiagnosticSetImpl &TopDiags,
                       RecordData &Record, unsigned RecStartIdx,
                       CXSourceRange &SR);
  
  LoadResult readLocation(CXLoadedDiagnosticSetImpl &TopDiags,
                          RecordData &Record, unsigned &offset,
                          CXLoadedDiagnostic::Location &Loc);
                       
public:
  DiagLoader(enum CXLoadDiag_Error *e, CXString *es)
    : error(e), errorString(es) {
      if (error)
        *error = CXLoadDiag_None;
      if (errorString)
        *errorString = createCXString("");
    }

  CXDiagnosticSet load(const char *file);
};
}

CXDiagnosticSet DiagLoader::load(const char *file) {
  // Open the diagnostics file.
  std::string ErrStr;
  FileSystemOptions FO;
  FileManager FileMgr(FO);

  OwningPtr<llvm::MemoryBuffer> Buffer;
  Buffer.reset(FileMgr.getBufferForFile(file));

  if (!Buffer) {
    reportBad(CXLoadDiag_CannotLoad, ErrStr);
    return 0;
  }

  llvm::BitstreamReader StreamFile;
  StreamFile.init((const unsigned char *)Buffer->getBufferStart(),
                  (const unsigned char *)Buffer->getBufferEnd());

  llvm::BitstreamCursor Stream;
  Stream.init(StreamFile);

  // Sniff for the signature.
  if (Stream.Read(8) != 'D' ||
      Stream.Read(8) != 'I' ||
      Stream.Read(8) != 'A' ||
      Stream.Read(8) != 'G') {
    reportBad(CXLoadDiag_InvalidFile,
              "Bad header in diagnostics file");
    return 0;
  }

  OwningPtr<CXLoadedDiagnosticSetImpl> Diags(new CXLoadedDiagnosticSetImpl());

  while (true) {
    unsigned BlockID = 0;
    StreamResult Res = readToNextRecordOrBlock(Stream, "Top-level", 
                                               BlockID, true);
    switch (Res) {
      case Read_EndOfStream:
        return (CXDiagnosticSet) Diags.take();
      case Read_Failure:
        return 0;
      case Read_Record:
        llvm_unreachable("Top-level does not have records");
      case Read_BlockEnd:
        continue;
      case Read_BlockBegin:
        break;
    }
    
    switch (BlockID) {
      case serialized_diags::BLOCK_META:
        if (readMetaBlock(Stream))
          return 0;
        break;
      case serialized_diags::BLOCK_DIAG:
        if (readDiagnosticBlock(Stream, *Diags.get(), *Diags.get()))
          return 0;
        break;
      default:
        if (!Stream.SkipBlock()) {
          reportInvalidFile("Malformed block at top-level of diagnostics file");
          return 0;
        }
        break;
    }
  }
}

StreamResult DiagLoader::readToNextRecordOrBlock(llvm::BitstreamCursor &Stream,
                                                 llvm::StringRef errorContext,
                                                 unsigned &blockOrRecordID,
                                                 bool atTopLevel) {
  
  blockOrRecordID = 0;

  while (!Stream.AtEndOfStream()) {
    unsigned Code = Stream.ReadCode();

    // Handle the top-level specially.
    if (atTopLevel) {
      if (Code == llvm::bitc::ENTER_SUBBLOCK) {
        unsigned BlockID = Stream.ReadSubBlockID();
        if (BlockID == llvm::bitc::BLOCKINFO_BLOCK_ID) {
          if (Stream.ReadBlockInfoBlock()) {
            reportInvalidFile("Malformed BlockInfoBlock in diagnostics file");
            return Read_Failure;
          }
          continue;
        }
        blockOrRecordID = BlockID;
        return Read_BlockBegin;
      }
      reportInvalidFile("Only blocks can appear at the top of a "
                        "diagnostic file");
{'fraction_non_alphanumeric': 0.16236909266918947, 'fraction_numerical': 0.003256182346211388, 'mean_word_length': 2.017392458842273, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 41, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5884731', 'n_tokens_mistral': 6368, 'n_tokens_neox': 6059, 'n_words': 1446}