<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#ifndef JSONNET_LEXER_H
#define JSONNET_LEXER_H

#include <cstdlib>
#include <cassert>

#include <iostream>
#include <string>
#include <list>
#include <sstream>
#include <vector>

#include "unicode.h"
#include "static_error.h"

/** Whitespace and comments.
 *
 * "Fodder" (as in cannon fodder) implies this data is expendable.
 */
struct FodderElement {
    enum Kind {
        LINE_END,
        INTERSTITIAL,
        PARAGRAPH,
    };
    Kind kind;
    unsigned blanks;
    unsigned indent;
    std::vector<std::string> comment;
    FodderElement(Kind kind, unsigned blanks, unsigned indent,
                  const std::vector<std::string> &comment)
      : kind(kind), blanks(blanks), indent(indent), comment(comment)
    {
        assert(kind != LINE_END || comment.size() <= 1);
        assert(kind != INTERSTITIAL || (blanks == 0 && indent == 0 && comment.size() == 1));
        assert(kind != PARAGRAPH || comment.size() >= 1);
    }
};
static inline std::ostream &operator<<(std::ostream &o, const FodderElement &f)
{
    switch (f.kind) {
        case FodderElement::LINE_END:
        o << "END(" << f.blanks << ", " << f.indent << ", " << f.comment[0] << ")";
        break;
        case FodderElement::INTERSTITIAL:
        o << "INT(" << f.blanks << ", " << f.indent << ", " << f.comment[0] << ")";
        break;
        case FodderElement::PARAGRAPH:
        o << "PAR(" << f.blanks << ", " << f.indent << ", " << f.comment[0] << "...)";
        break;
    }
    return o;
}
typedef std::vector<FodderElement> Fodder;

static inline std::ostream &operator<<(std::ostream &o, const Fodder &fodder)
{
    bool first = true;
    for (const auto &f : fodder) {
        o << (first ? "[" : ", ");
        first = false;
        o << f;
    }
    o << (first ? "[]" : "]");
    return o;
}

struct Token {
    enum Kind {
        // Symbols
        BRACE_L,
        BRACE_R,
        BRACKET_L,
        BRACKET_R,
        COMMA,
        DOLLAR,
        DOT,
        PAREN_L,
        PAREN_R,
        SEMICOLON,

        // Arbitrary length lexemes<commit_msg>Add comments giving more detail on fodder representation<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#ifndef JSONNET_LEXER_H
#define JSONNET_LEXER_H

#include <cstdlib>
#include <cassert>

#include <iostream>
#include <string>
#include <list>
#include <sstream>
#include <vector>

#include "unicode.h"
#include "static_error.h"

/** Whitespace and comments.
 *
 * "Fodder" (as in cannon fodder) implies this data is expendable.
 */
struct FodderElement {
    enum Kind {
        /** The next token, paragraph, or interstitial should be on a new line.
         *
         * A single comment string is allowed, which flows before the new line.
         *
         * The LINE_END fodder specifies the indentation level and vertical spacing before whatever
         * comes next.
         */
        LINE_END,

        /** A C-style comment that begins and ends on the same line.
         *
         * If it follows a token (i.e., it is the first fodder element) then it appears after the
         * token on the same line.  If it follows another interstitial, it will also flow after it
         * on the same line.  If it follows a new line or a paragraph, it is the first thing on the
         * following line, after the blank lines and indentation specified by the previous fodder.
         *
         * There is exactly one comment string.
         */
        INTERSTITIAL,

        /** A comment consisting of at least one line.
         *
         * // and # style commes have exactly one line.  C-style comments can have more than one
         * line.
         *
         * All lines of the comment are indented according to the indentation level of the previous new line
         * / paragraph fodder.
         *
         * The PARAGRAPH fodder specifies the indentation level and vertical spacing before whatever
         * comes next.
         */
        PARAGRAPH,
    };
    Kind kind;

    /** How many blank lines (vertical space) before the next fodder / token. */
    unsigned blanks;

    /** How far the next fodder / token should be indented. */
    unsigned indent;

    /** Whatever comments are part of this fodder.
     *
     * Constraints apply.  See Kind, above.
     *
     * The strings include any delimiting characters, e.g. // # and C-style comment delimiters but
     * not newline characters or indentation.
     */
    std::vector<std::string> comment;

    FodderElement(Kind kind, unsigned blanks, unsigned indent,
                  const std::vector<std::string> &comment)
      : kind(kind), blanks(blanks), indent(indent), comment(comment)
    {
        assert(kind != LINE_END || comment.size() <= 1);
        assert(kind != INTERSTITIAL || (blanks == 0 && indent == 0 && comment.size() == 1));
        assert(kind != PARAGRAPH || comment.size() >= 1);
    }
};

static inline std::ostream &operator<<(std::ostream &o, const FodderElement &f)
{
    switch (f.kind) {
        case FodderElement::LINE_END:
        o << "END(" << f.blanks << ", " << f.indent << ", " << f.comment[0] << ")";
        break;
        case FodderElement::INTERSTITIAL:
        o << "INT(" << f.blanks << ", " << f.indent << ", " << f.comment[0] << ")";
        break;
        case FodderElement::PARAGRAPH:
        o << "PAR(" << f.blanks << ", " << f.indent << ", " << f.comment[0] << "...)";
        break;
    }
    return o;
}

/** A sequence of fodder elements, typically between two tokens.
 *
 * A LINE_END is not allowed to follow a PARAGRAPH or a LINE_END.  This can be represented by
 * replacing the indent of the prior fodder and increasing the number of blank lines if necessary.
 * If there was a comment, it can be represented by changing the LINE_END to a paragraph containing
 * the same single comment string.
 */
typedef std::vector<FodderElement> Fodder;

static inline std::ostream &operator<<(std::ostream &o, const Fodder &fodder)
{
    bool first = true;
    for (const auto &f : fodder) {
        o << (first ? "[" : ", ");
        first = false;
        o << f;
    }
    o << (first ? "[]" : "]");
    return o;
}

struct Token {
    enum Kind {
        // Symbols
        BRACE_L,
        BRACE_R,
        BRACKET_L,
        BRACKET_R,
        COMMA,
        DOLLAR,
        DOT,
        PAREN_L,
        PAREN_R,
        SEMICOLON,

        // Arbitrary length lexemes
{'fraction_non_alphanumeric': 0.11564535424697595, 'fraction_numerical': 0.004253622225176126, 'mean_word_length': 2.476894639556377, 'pattern_counts': {'":': 0, '<': 129, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5643188', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2219, 'n_words': 792}