<commit_before>#!/usr/bin/python2

import os
import argparse
import re
import sys

# Argument parsers
argParser = argparse.ArgumentParser('Sort the declarations in a source file')
argParser.add_argument('inputFile')
argParser.add_argument('outputFile')

args = argParser.parse_args()

# Read entire file in a single string
with open(args.inputFile, 'r') as inputFile:
	sourceText = inputFile.read()
	if not sourceText.endswith('\n'):
		sourceText.append('\n')


# Block class used to store and various fragments of the file
class Block:
	methodRegex = re.compile(r'.*\)(\s|const|override)*\s*(=\s*\w+)?(\s|\n)*{(\s|\n)*$', re.DOTALL)
	
	def __init__(self):
		self.prefix = ""
		self.suffix = ""
		self.children = []
		self.finalized = False
	
	def add(self, char):
		if self.finalized:
			self.suffix += char
		else:
			self.prefix += char
			
	def removeLast(self, char):
		if self.finalized:
			assert self.suffix.endswith(char)
			self.prefix = self.prefix[:-1]
		else:
			assert self.prefix.endswith(char)
			self.prefix = self.prefix[:-1]
	
	def deepen(self):
		assert not self.finalized<commit_msg>filter input in source-sort script<commit_after>#!/usr/bin/python2

import os
import argparse
import re
import sys

# Input filtering
def initialComment(line):
	return line.startswith("/***") or line.startswith(" **") or line.startswith("**")

def includeOrForwardDeclaration(line):
	forwardDeclarationRegex = re.compile('^class \w+;$')
	return line.startswith("#include ") or forwardDeclarationRegex.match(line);

def emptyLine(line):
	emptyLineRegex = re.compile('^\s*$')
	return emptyLineRegex.match(line)

# Argument parsers
argParser = argparse.ArgumentParser('Sort the declarations in a source file')
argParser.add_argument('inputFile')
argParser.add_argument('outputFile')

args = argParser.parse_args()
sourceText = ''

# Read entire file in a single string
with open(args.inputFile, 'r') as inputFile:
	for line in inputFile:
		if not initialComment(line) and not includeOrForwardDeclaration(line) and not emptyLine(line):
			if not sourceText.endswith('\n'):
				sourceText += '\n'
			sourceText += line


# Block class used to store and various fragments of the file
class Block:
	methodRegex = re.compile(r'.*\)(\s|const|override)*\s*(=\s*\w+)?(\s|\n)*{(\s|\n)*$', re.DOTALL)
	
	def __init__(self):
		self.prefix = ""
		self.suffix = ""
		self.children = []
		self.finalized = False
	
	def add(self, char):
		if self.finalized:
			self.suffix += char
		else:
			self.prefix += char
			
	def removeLast(self, char):
		if self.finalized:
			assert self.suffix.endswith(char)
			self.prefix = self.prefix[:-1]
		else:
			assert self.prefix.endswith(char)
			self.prefix = self.prefix[:-1]
	
	def deepen(self):
		assert not self.finalized
{'fraction_non_alphanumeric': 0.14472727272727273, 'fraction_numerical': 0.002181818181818182, 'mean_word_length': 4.394117647058824, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6321416', 'n_tokens_mistral': 996, 'n_tokens_neox': 906, 'n_words': 229}