<commit_before>#!/usr/bin/env lua
-- vim: set tabstop=2 shiftwidth=2 expandtab cc=80:

-- Writes a transformation of a C source code file where all the string
-- literals have been moved to the beginning of the code, to stdout.
--
-- Requires lua (probably 5.2 or 5.3) and LPeg

lpeg = require"lpeg"

-- minimum length of string to be replaced
MINLEN = 4

-- symbol prefix for #define statements
SYMPREFIX = "STRSYM_"

function build_parser(syms)
  local P  = lpeg.P    -- literal pattern, matches a string
  local V  = lpeg.V    -- non-terminal, references a grammar rule
  local S  = lpeg.S    -- set, matches a set of bytes
  local B  = lpeg.B    -- look-behind
  local C  = lpeg.C    -- capture, captures the match of its argument pattern
  local Cs = lpeg.Cs   -- capture substitution
  local Ct = lpeg.Ct   -- table capture, creates a table of its captures

  local function cap(str)
    if #str >= MINLEN then
      local label = SYMPREFIX..str:gsub("%W", "_"):upper()
      syms[label] = str
      return label
    else
      return "\""..str.."\""
    end
  end

  -- NB: currently, this grammar captures all tokens to a table. Tokens
  -- that are not comments, preprocessor macros or string literals are one
  -- byte. That probably means a lot of overhead and can most likely be
  -- improved on. It shouldn't matter that much though, unless your source
  -- files are huge.
  return P{
    "code";
    lit_esc  = P"\\"*1,      -- matches a \ followed by any byte
    lit_cont = 1-S"\n\r\"",  -- matches any byte not in the given set

    -- matches lit_esc, or if lit_esc doesn't match, matches lit_cont,
    -- zero or more times. The matched string is passed to cap using the
    -- function capture operator '/', which produces a capture that
    -- will be added to the table in the code rule below
    in_lit   = ((V"lit_esc" + V"lit_cont")^0)/cap,

    -- matches " followed by the in_lit rule, followed by "
    literal  = P"\"" * V"in_lit" * P"\"",

    -- matches (\r at most 1 time followed by \n), or end of data
    newline  = (P"\r"^-1 * P"\n") + -1,

    -- matches the newline rule if it is preceded by something that's not
    -- a backslash
    endmacro = -B"\\" * V"newline",

    -- matches the literal patterns for macro directives
    macrodir = P"#define" + P"#error" + P"#warning" + P"#undef" + P"#ifdef" +
               P"#ifndef" + P"#if" + P"#else" + P"#elif" + P"#endif" +
               P"#pragma",

    -- matches the rule for macro directives followed by zero or more
    -- bytes where the endmacro rule does not match, followed by the endmacro
    -- rule
    macro    = V"macrodir" * (1-V"endmacro")^0 * V"endmacro",

    -- matches C single line comments; // followed by zero or more bytes where
    -- the newline rule does not match, followed by the newline rule
    scomment = P"//" * (1-V"newline")^0 * V"newline",
<commit_msg>Added #include to macrodir rule<commit_after>#!/usr/bin/env lua
-- vim: set tabstop=2 shiftwidth=2 expandtab cc=80:

-- Writes a transformation of a C source code file where all the string
-- literals have been moved to the beginning of the code, to stdout.
--
-- Requires lua (probably 5.2 or 5.3) and LPeg

lpeg = require"lpeg"

-- minimum length of string to be replaced
MINLEN = 4

-- symbol prefix for #define statements
SYMPREFIX = "STRSYM_"

function build_parser(syms)
  local P  = lpeg.P    -- literal pattern, matches a string
  local V  = lpeg.V    -- non-terminal, references a grammar rule
  local S  = lpeg.S    -- set, matches a set of bytes
  local B  = lpeg.B    -- look-behind
  local C  = lpeg.C    -- capture, captures the match of its argument pattern
  local Cs = lpeg.Cs   -- capture substitution
  local Ct = lpeg.Ct   -- table capture, creates a table of its captures

  local function cap(str)
    if #str >= MINLEN then
      local label = SYMPREFIX..str:gsub("%W", "_"):upper()
      syms[label] = str
      return label
    else
      return "\""..str.."\""
    end
  end

  -- NB: currently, this grammar captures all tokens to a table. Tokens
  -- that are not comments, preprocessor macros or string literals are one
  -- byte. That probably means a lot of overhead and can most likely be
  -- improved on. It shouldn't matter that much though, unless your source
  -- files are huge.
  return P{
    "code";
    lit_esc  = P"\\"*1,      -- matches a \ followed by any byte
    lit_cont = 1-S"\n\r\"",  -- matches any byte not in the given set

    -- matches lit_esc, or if lit_esc doesn't match, matches lit_cont,
    -- zero or more times. The matched string is passed to cap using the
    -- function capture operator '/', which produces a capture that
    -- will be added to the table in the code rule below
    in_lit   = ((V"lit_esc" + V"lit_cont")^0)/cap,

    -- matches " followed by the in_lit rule, followed by "
    literal  = P"\"" * V"in_lit" * P"\"",

    -- matches (\r at most 1 time followed by \n), or end of data
    newline  = (P"\r"^-1 * P"\n") + -1,

    -- matches the newline rule if it is preceded by something that's not
    -- a backslash
    endmacro = -B"\\" * V"newline",

    -- matches the literal patterns for macro directives
    macrodir = P"#define" + P"#error" + P"#warning" + P"#undef" + P"#ifdef" +
               P"#ifndef" + P"#if" + P"#else" + P"#elif" + P"#endif" +
               P"#pragma" + P"#include",

    -- matches the rule for macro directives followed by zero or more
    -- bytes where the endmacro rule does not match, followed by the endmacro
    -- rule
    macro    = V"macrodir" * (1-V"endmacro")^0 * V"endmacro",

    -- matches C single line comments; // followed by zero or more bytes where
    -- the newline rule does not match, followed by the newline rule
    scomment = P"//" * (1-V"newline")^0 * V"newline",

{'fraction_non_alphanumeric': 0.11658031088082901, 'fraction_numerical': 0.0065630397236614854, 'mean_word_length': 2.5118253486961795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '575231', 'n_tokens_mistral': 1889, 'n_tokens_neox': 1798, 'n_words': 747}