<commit_before>#!/usr/bin/env python

#
# Import Modules
#
import os, sys

def TrimPreprocessedFile (source, target, ConvertHex):
  f = open (source,'r')
  lines = f.readlines()
  f.close()
  for index in range (len(lines) - 1, -1, -1):
    if lines[index].strip().find('#line') >= 0:
      index += 1
      break
  else:
    index = 0
  f = open (target,'w')
  for index in range (index, len(lines)):
    if ConvertHex:
      while lines[index].lower().find('0x') >= 0:
        foo=lines[index].lower().find('0x')
        bar = foo + 2
        while lines[index][bar].lower() in '0123456789abcdef':
          bar += 1
        if lines[index][foo+2].lower() in 'abcdef': 
          lines[index] = lines[index][0:foo] + '0' + lines[index][foo+2:bar] + 'h' + lines[index][bar:]
        else:
          lines[index] = lines[index][0:foo] + lines[index][foo+2:bar] + 'h' + lines[index][bar:]
    f.write(lines[index])
  f.close()

if __name__ == '__main__':
  for arg in sys.argv[2:]:
    if sys.argv[1] == '-CONVERT':
      TrimPreprocessedFile(arg, os.path.splitext(arg)[0] + '.iii', True)
    else:
      TrimPreprocessedFile(arg, os.path.splitext(arg)[0] + '.iii', False)<commit_msg>Added functionality to process preprocessed VFR file<commit_after>#!/usr/bin/env python

#
# Import Modules
#
import os, sys, re

cTypedefPattern = re.compile("^\s*typedef\s+struct\s+\{", re.MULTILINE)
cPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)

def TrimPreprocessedFile (source, target, Convert, Vfr=False):
    f = open (source,'r')
    lines = f.readlines()
    f.close()

    for index in range (len(lines) - 1, -1, -1):
        if lines[index].strip().find('#line') >= 0:
            endOfCode = index + 1
            break
    else:
        index = 0

    f = open (target,'w')
    if Vfr:
        TrimVfr(lines, 0, endOfCode)
        f.writelines(lines)
    else:
        if Convert:
            ConvertHex(lines, endOfCode, len(lines))
        f.writelines(lines[endOfCode:])
    f.close()

def TrimVfr(lines, start, end):
    foundTypedef = False
    brace = 0
    for index in range (start, end):
        if lines[index].strip().find('#line') == 0:
            continue

        if foundTypedef == False and cTypedefPattern.search(lines[index]) == None:
            if cPragmaPattern.search(lines[index]) == None:
                lines[index] = "\n"
            continue
        elif foundTypedef == False:
            foundTypedef = True

        if lines[index].find("{") >= 0:
            brace += 1
        elif lines[index].find("}") >= 0:
            brace -= 1

        if brace == 0 and lines[index].find(";") >= 0:
##            if lines[index].find("typedef") >= 0:
##                lines[index] = "\n"
            foundTypedef = False

def ConvertHex(lines, start, end):
    for index in range (start, end):
        while lines[index].lower().find('0x') >= 0:
            foo=lines[index].lower().find('0x')
            bar = foo + 2
            while lines[index][bar].lower() in '0123456789abcdef':
                bar += 1
            if lines[index][foo+2].lower() in 'abcdef': 
                lines[index] = lines[index][0:foo] + '0' + lines[index][foo+2:bar] + 'h' + lines[index][bar:]
            else:
                lines[index] = lines[index][0:foo] + lines[index][foo+2:bar] + 'h' + lines[index][bar:]

if __name__ == '__main__':
    if sys.argv[1] == '-CONVERT':
        TrimPreprocessedFile(sys.argv[2], os.path.splitext(sys.argv[2])[0] + '.iii', True)
    elif sys.argv[1] == '-VFR':
        TrimPreprocessedFile(sys.argv[2], os.path.splitext(sys.argv[2])[0] + '.iii', False, True)
    else:
        TrimPreprocessedFile(sys.argv[1], os.path.splitext(sys.argv[1])[0] + '.iii', False)
{'fraction_non_alphanumeric': 0.17691275167785234, 'fraction_numerical': 0.021208053691275167, 'mean_word_length': 2.49203373945642, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4212169', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1264, 'n_words': 282}