<commit_before>// csabbg_nonascii.cpp                                                -*-C++-*-
// ----------------------------------------------------------------------------

#include <csabase_analyser.h>
#include <csabase_location.h>
#include <csabase_ppobserver.h>
#include <csabase_registercheck.h>
#include <string>

#ident "$Id$"

// ----------------------------------------------------------------------------

static std::string const check_name("nonascii");

// ----------------------------------------------------------------------------

using clang::SourceLocation;
using clang::SourceManager;
using clang::SourceRange;
using cool::csabase::Analyser;
using cool::csabase::Location;
using cool::csabase::PPObserver;
using cool::csabase::Range;
using cool::csabase::Visitor;

namespace
{

struct comments
    // Callback object for inspecting comments.
{
    Analyser& d_analyser;                   // Analyser object.

    comments(Analyser& analyser);
        // Create a 'comments' object, accessing the specified 'analyser'.

    void operator()(SourceRange range);
        // The specified 'range', representing a comment, is examined for
        // non-ascii characters.
};

comments::comments(Analyser& analyser)
: d_analyser(analyser)
{
}

void comments::operator()(SourceRange range)
{
    const std::string s(d_analyser.get_source(range));

    int begin = -1;
    for (size_t i = 0; i <= s.length(); ++i) {
        if (!(s[i] & 0x80)) {
            if (begin >= 0) {
                SourceRange bad(range.getBegin().getLocWithOffset(begin),
                                range.getBegin().getLocWithOffset(i - 1));
                d_analyser.report(bad.getBegin(),
                                  check_name, "NA01: "
                                  "Non-ASCII comment characters")
                    << bad;
                begin = -1;
            }
        } else {
            if (begin == -1) {
                begin = i;
            }
        }
    }
}

void subscribe(Analyser& analyser, Visitor&, PPObserver& observer)
    // Hook up the callback functions.
{
    observer.onComment += comments(analyser);
}

}  // close anonymous namespace

// ----------------------------------------------------------------------------

static cool::csabase::RegisterCheck c1(check_name, &subscribe);<commit_msg>Do whole-file check for non-ASCII chars.<commit_after>// csabbg_nonascii.cpp                                                -*-C++-*-
// ----------------------------------------------------------------------------

#include <csabase_analyser.h>
#include <csabase_debug.h>
#include <csabase_location.h>
#include <csabase_ppobserver.h>
#include <csabase_registercheck.h>
#include <string>

#ident "$Id$"

// ----------------------------------------------------------------------------

static std::string const check_name("nonascii");

// ----------------------------------------------------------------------------

using clang::SourceLocation;
using clang::SourceManager;
using clang::SourceRange;
using cool::csabase::Analyser;
using cool::csabase::Debug;
using cool::csabase::Location;
using cool::csabase::PPObserver;
using cool::csabase::Range;
using cool::csabase::Visitor;

namespace
{

struct files
    // Callback object for inspecting files.
{
    Analyser& d_analyser;                   // Analyser object.

    files(Analyser& analyser);
        // Create a 'files' object, accessing the specified 'analyser'.

    void operator()(SourceLocation loc,
                    std::string const&,
                    std::string const&);
        // The file specified by 'loc' is examined for non-ascii characters.
};

files::files(Analyser& analyser)
: d_analyser(analyser)
{
}

void files::operator()(SourceLocation loc,
                       std::string const&,
                       std::string const&)
{
    const SourceManager &m = d_analyser.manager();
    const llvm::MemoryBuffer *buf = m.getBuffer(m.getFileID(loc));
    const char *b = buf->getBufferStart();
    const char *e = buf->getBufferEnd();

    const char *begin = 0;
    for (const char *s = b; s <= e; ++s) {
        if (!(*s & 0x80)) {
            if (begin != 0) {
                SourceRange bad(loc.getLocWithOffset(begin - b),
                                loc.getLocWithOffset(s - b - 1));
                d_analyser.report(bad.getBegin(),
                                  check_name, "NA01: "
                                  "Non-ASCII comment characters",
                                  true)
                    << bad;
                begin = 0;
            }
        } else {
            if (begin == 0) {
                begin = s;
            }
        }
    }
}

void subscribe(Analyser& analyser, Visitor&, PPObserver& observer)
    // Hook up the callback functions.
{
    observer.onOpenFile += files(analyser);
}

}  // close anonymous namespace

// ----------------------------------------------------------------------------

static cool::csabase::RegisterCheck c1(check_name, &subscribe);
{'fraction_non_alphanumeric': 0.23324716643467885, 'fraction_numerical': 0.00457347385166037, 'mean_word_length': 1.4536585365853658, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4316425', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1308, 'n_words': 303}