<commit_before>//------------------------------------------------------------------------------
// rewriter.cpp
// Simple tool that parses an input file and writes it back out; used
// for verifying the round-trip nature of the parse tree.
//
// File is under the MIT license; see LICENSE for details
//------------------------------------------------------------------------------

#include <cstdio>

#include "parsing/SyntaxTree.h"

using namespace slang;

int main(int argc, char** argv) {
    if (argc != 2) {
        fprintf(stderr, "usage: rewriter file");
        return 1;
    }

    SmallVectorSized<char, 8> buffer;
    auto tree = SyntaxTree::fromFile(argv[1]);
    tree->root().writeTo(buffer, SyntaxToStringFlags::IncludeTrivia | SyntaxToStringFlags::IncludeDirectives |
                         SyntaxToStringFlags::IncludeSkipped);

    buffer.append('\0');
    puts(buffer.data());

    return 0;
}<commit_msg>Fix line endings output by rewriter tool<commit_after>//------------------------------------------------------------------------------
// rewriter.cpp
// Simple tool that parses an input file and writes it back out; used
// for verifying the round-trip nature of the parse tree.
//
// File is under the MIT license; see LICENSE for details
//------------------------------------------------------------------------------

#include <cstdio>
#if defined(_WIN32)
  #include <fcntl.h>
  #include <io.h>
#endif

#include "parsing/SyntaxTree.h"

using namespace slang;

int main(int argc, char** argv) {
    if (argc != 2) {
        fprintf(stderr, "usage: rewriter file");
        return 1;
    }

    // Make sure we reproduce newlines correct on Windows:
#if defined(_WIN32)
    _setmode(fileno(stdout), _O_BINARY);
#endif

    SmallVectorSized<char, 8> buffer;
    auto tree = SyntaxTree::fromFile(argv[1]);
    tree->root().writeTo(buffer, SyntaxToStringFlags::IncludeTrivia | SyntaxToStringFlags::IncludeDirectives |
                         SyntaxToStringFlags::IncludeSkipped);

    buffer.append('\0');
    printf("%s", buffer.data());

    return 0;
}
{'fraction_non_alphanumeric': 0.25516578567996157, 'fraction_numerical': 0.0076886112445939455, 'mean_word_length': 1.8520547945205479, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '896735', 'n_tokens_mistral': 566, 'n_tokens_neox': 547, 'n_words': 165}