<commit_before># Theming constants
THEME_NAME = ENV_TOKENS['THEME_NAME']
USE_CUSTOM_THEME = !(THEME_NAME.nil? || THEME_NAME.empty?)
if USE_CUSTOM_THEME
    THEME_ROOT = File.join(ENV_ROOT, "themes", THEME_NAME)
    THEME_SASS = File.join(THEME_ROOT, "static", "sass")
end

# Run the specified file through the Mako templating engine, providing
# the ENV_TOKENS to the templating context.
def preprocess_with_mako(filename)
    # simple command-line invocation of Mako engine
    mako = "from mako.template import Template;" +
           "print Template(filename=\"#{filename}\")" +
           # Total hack. It works because a Python dict literal has
           # the same format as a JSON object.
           ".render(env=#{ENV_TOKENS.to_json});"

    # strip off the .mako extension
    output_filename = filename.chomp(File.extname(filename))

    # just pipe from stdout into the new file
    File.open(output_filename, 'w') do |file|
      file.write(`python -c '#{mako}'`)
    end
end<commit_msg>Capture Mako exit code and fail fast if necessary<commit_after># Theming constants
THEME_NAME = ENV_TOKENS['THEME_NAME']
USE_CUSTOM_THEME = !(THEME_NAME.nil? || THEME_NAME.empty?)
if USE_CUSTOM_THEME
    THEME_ROOT = File.join(ENV_ROOT, "themes", THEME_NAME)
    THEME_SASS = File.join(THEME_ROOT, "static", "sass")
end

# Run the specified file through the Mako templating engine, providing
# the ENV_TOKENS to the templating context.
def preprocess_with_mako(filename)
    # simple command-line invocation of Mako engine
    mako = "from mako.template import Template;" +
           "print Template(filename=\"#{filename}\")" +
           # Total hack. It works because a Python dict literal has
           # the same format as a JSON object.
           ".render(env=#{ENV_TOKENS.to_json});"

    # strip off the .mako extension
    output_filename = filename.chomp(File.extname(filename))

    # just pipe from stdout into the new file, exiting on failure
    File.open(output_filename, 'w') do |file|
      file.write(`python -c '#{mako}'`)
      exit_code = $?.to_i
      abort "#{mako} failed with #{exit_code}" if exit_code.to_i != 0
    end
end
{'fraction_non_alphanumeric': 0.10780669144981413, 'fraction_numerical': 0.00046468401486988845, 'mean_word_length': 3.6004273504273505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6848535', 'n_tokens_mistral': 700, 'n_tokens_neox': 698, 'n_words': 204}