<commit_before>#!/usr/bin/env python
#
# transform_sql.py -- create a header file with the appropriate SQL variables
# from an SQL file
#


import os
import re
import sys


DEFINE_END = '  ""\n'


def usage_and_exit(msg):
  if msg:
    sys.stderr.write('%s\n\n' % msg)
  sys.stderr.write(
    'USAGE: %s SQLITE_FILE [OUTPUT_FILE]\n'
    '  stdout will be used if OUTPUT_FILE is not provided.\n'
    % os.path.basename(sys.argv[0]))
  sys.stderr.flush()
  sys.exit(1)


def main(input, output, filename):
  input = input.read()

  var_name = re.sub('[-.]', '_', filename).upper()

  output.write(
    '/* This file is automatically generated from %s.\n'
    ' * Do not edit this file -- edit the source and rerun gen-make.py */\n'
    '\n'
    % (filename,))

  var_printed = False
  stmt_count = 0

  # only used once, but we need the flags. so re.compile it is!
  re_comments = re.compile(r'/\*.*?\*/', re.MULTILINE|re.DOTALL)<commit_msg>* build/transform_sql.py:  Add an empty line to make the generated *.h files     a bit more readable.<commit_after>#!/usr/bin/env python
#
# transform_sql.py -- create a header file with the appropriate SQL variables
# from an SQL file
#


import os
import re
import sys


DEFINE_END = '  ""\n\n'


def usage_and_exit(msg):
  if msg:
    sys.stderr.write('%s\n\n' % msg)
  sys.stderr.write(
    'USAGE: %s SQLITE_FILE [OUTPUT_FILE]\n'
    '  stdout will be used if OUTPUT_FILE is not provided.\n'
    % os.path.basename(sys.argv[0]))
  sys.stderr.flush()
  sys.exit(1)


def main(input, output, filename):
  input = input.read()

  var_name = re.sub('[-.]', '_', filename).upper()

  output.write(
    '/* This file is automatically generated from %s.\n'
    ' * Do not edit this file -- edit the source and rerun gen-make.py */\n'
    '\n'
    % (filename,))

  var_printed = False
  stmt_count = 0

  # only used once, but we need the flags. so re.compile it is!
  re_comments = re.compile(r'/\*.*?\*/', re.MULTILINE|re.DOTALL)
{'fraction_non_alphanumeric': 0.15040650406504066, 'fraction_numerical': 0.003048780487804878, 'mean_word_length': 2.98582995951417, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2604790', 'n_tokens_mistral': 736, 'n_tokens_neox': 700, 'n_words': 208}