<commit_before>#!/usr/bin/env python
#
# transform_sql.py -- create a header file with the appropriate SQL variables
# from an SQL file
#


import os
import re
import sys


def usage_and_exit(msg):
  if msg:
    sys.stderr.write("%s\n\n" % msg)
  sys.stderr.write("usage: %s [sqlite_file]\n" % \
    os.path.basename(sys.argv[0]))
  sys.stderr.flush()
  sys.exit(1)


def main(input_filename, output_filename):
  input = open(input_filename, "r")
  output = open(output_filename, "w")

  var_name = os.path.basename(input_filename).replace('.', '_')
  var_name = var_name.replace('-', '_')

  output.write('#define %s \\\n' % var_name.upper())

  in_comment = False
  for line in input:
    line = line.replace('\n', '')
    line = line.replace('"', '\\"')

    if line:
      output.write('  "' + line + '"\\\n')
    else:
      output.write('  APR_EOL_STR\\\n')

  output.write('  ""\n')

  input.close()
  output.close()


if __name__ == '__main__':
  if len(sys.argv) < 2:
    usage_and_exit("Incorrect number of arguments")
  main(sys.argv[1], sys.argv[1] + ".h")<commit_msg>A couple of cleanups for transform_sql.py.<commit_after>#!/usr/bin/env python
#
# transform_sql.py -- create a header file with the appropriate SQL variables
# from an SQL file
#


import os
import re
import sys


def usage_and_exit(msg):
  if msg:
    sys.stderr.write("%s\n\n" % msg)
  sys.stderr.write("usage: %s [sqlite_file]\n" % \
    os.path.basename(sys.argv[0]))
  sys.stderr.flush()
  sys.exit(1)


def main(input_filename, output_filename):
  input = open(input_filename, "r").read()
  output = open(output_filename, "w")

  var_name = os.path.basename(input_filename).replace('.', '_')
  var_name = var_name.replace('-', '_')

  output.write('/* This file is automatically generated from %s.\n' %
                                               os.path.basename(input_filename))
  output.write(' * Do not edit it directly, but edit the source file and ')
  output.write('rerun autogen.sh */\n\n')

  output.write('#define %s \\\n' % var_name.upper())

  regex = re.compile(r"/\*.*?\*/", re.MULTILINE|re.DOTALL)
  input = regex.sub('', input)
  for line in input.split('\n'):
    line = line.replace('\n', '')
    line = line.replace('"', '\\"')

    if line and not len(line.strip()) == 0:
      output.write('  "' + line + '"\\\n')

  output.write('  ""\n')

  output.close()


if __name__ == '__main__':
  if len(sys.argv) < 2:
    usage_and_exit("Incorrect number of arguments")
  main(sys.argv[1], sys.argv[1] + ".h")
{'fraction_non_alphanumeric': 0.18431528662420382, 'fraction_numerical': 0.004378980891719745, 'mean_word_length': 3.133223684210526, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '198971', 'n_tokens_mistral': 925, 'n_tokens_neox': 896, 'n_words': 204}