<commit_before># 
# A simple tool to help you easily analayse TODOs spread across a whole project.
# 
# Author(s):     <NAME>
# License:       MIT
# Last Modified: 8/6/2016
# 

# TODO Allow user to customise number of lines visible and trigger char.

import codecs
import sys
import os


# Define global variable ARGS which we then parse.
ARGS = sys.argv[1:]

CONTEXT = True
WRITE   = False

# Define global variables which manage whether or not the user is choosing to search/ignore specific filetypes and directories.
EXCLUDE_DIRS  = True
EXCLUDE_FILES = True

# Define global variables to hold the filetypes and directories that should be searched/ignored later in the parse.
DIRS  = []
FILES = []

# Allow user to search a flat hierarchy of specified dirs. I.e. if "." is provided, just search cwd not any subdirs.
FLAT = False

VALID = ["--no-context", "--write", "+df", "+d", "-d", "+f", "-f"]


# This is where the program kicks off.
def parse():
	global ARGS
	global CONTEXT
	global DIRS
	global EXCLUDE_FILES
	global EXCLUDE_DIRS
	global FILES
	global FLAT
	global WRITE


	command = "todo " + " ".join(ARGS) + "\n\n"

	# We first parse the params passed to the program which allows the user to specify directories to search/avoid and similarly with file types.
	parse_args()
	
	# Get the current working directory so we can use relative paths.
	cwd = os.getcwd()

	# Apply a bit of manipulation to allow user to specify current directory (relative root dir).
	DIRS = [cwd + d if d != "." else cwd for d in DIRS]
	DIRS = [d.replace('\\', '/') for d in DIRS]

	output = ""

	# Now traverse whole tree.
	for path, dirs, files in os.walk(cwd):
		for name in files:
			f = os.path.join(path, name)
			f = f.replace('\\', '/')

			# Boolean storing the value of whether or this file is valid based on the params the user provided.
			proceed = True

			# TODO Can speed these checks up, once false, we can break immediately.
			# First check if valid filetype.
			if EXCLUDE_FILES:
				proceed = proceed and not any(f.endswith(x) for x in FILES)
			else:
				proceed = proceed and any(f.endswith(x) for x in FILES)

			# Now check if in valid directory.
			if EXCLUDE_DIRS:
				proceed = proceed and not any(f.startswith(x) for x in DIRS)
			else:
				if FLAT:
					proceed = proceed and any(x + "/" + name == f for x in DIRS)
				else:
					proceed = proceed and any(f.startswith(x) for x in DIRS)

			# Only search the file for a todo comment if it is a valid file.
			if proceed:
				# TODO Improve the error handling here.
				try:
					content = [line.rstrip('\n').rstrip('\r') for line in codecs.open(f, 'r', encoding='utf-8')]
				except:
					content = []
					pass

				# If a todo exists we continue.
				if len([x for x in content if "TODO" in x]) > 0:
					# Print out filename.
					output += "\n### " + f + " ###"

					# Now we find the todos and print them out.
					for i in range (0, len(content)):
						line = content[i]

						if "TODO" in line:
							index = len(line.split("TODO")[0])
							comment = line[index + 4:].strip()

							output += "\n\n  > TODO " + comment + "\n\n"
							
							if CONTEXT:
								# Try and print line above and below as well as line.
								if i < len(content) - 1 and i > 0:
									output += "    " + str(i) + " " + content[i - 1] + "\n"
									output += "    " + str(i + 1) + " " + line + "\n"
									output += "    " + str(i + 2) + " " + content[i + 1] + "\n\n"
								# Try to print line below and line.
								elif i > 0:
									output += "    " + str(i) + " " + content[i - 1] + "\n"
									output += "    " + str(i + 1) + " " + line + "\n\n"
								# Try to print line above and line.
								elif i < len(content) - 1:
									output += "    " + str(i + 1) + " " + line + "\n"
									output += "    " + str(i + 2) + " " + content[i + 1] + "\n\n"
								# Otherwise we simpyl print the line.
								else:
									output += "    " + str(i + 1) + " " + line + "\n\n"
					
					output += "\n"

	if WRITE:
		# Write output to a TODO.txt file
		f = open("TODO.mdown", "w")
		f.write(command + output);
		f.close()
	else:
		print(output)


# Parses the command line args and updates the global vars accordingly.
def parse_args():
	global ARGS
	global CONTEXT
	global EXCLUDE_FILES
	global EXCLUDE_DIRS
	global DIRS
	global FILES
	global FLAT
	global VALID
	global WRITE


	# Allow the shorthand notation for searching within the current dir of 'todo .' instead of 'todo +df .'
	if len(ARGS) > 0:
		if ARGS[0] == ".":
			ARGS = ["+df", "."] + ARGS[1:]

	d_index = -1
	f_index = -1

	p = []

	for c in VALID:
		if c in ARGS:
			start = ARGS.index(c)
			if c in ["--no-context", "--write"]:
				p.append((c, []))
			else:
				params = get_params(ARGS[start + 1:])
				p.append((c, params))

	set_args(p)


def get_params(l):
	global VALID

	x = []

	for i in l:
		if i in VALID:
			return x
		else:
			x.append(i)

	return x


def set_args(p):
	global CONTEXT
	global EXCLUDE_FILES
	global EXCLUDE_DIRS
	global DIRS
	global FILES
	global FLAT
	global WRITE

	for x in p:
		if x[0] == "--no-context":
			CONTEXT = False
		if x[0] == "--write":
			WRITE = True
		if x[0] == "+df":
			FLAT = True
			EXCLUDE_DIRS = False
			DIRS = x[1]
		if x[0] == "+d":
			EXCLUDE_DIRS = False
			DIRS = x[1]
		if x[0] == "-d":
			EXCLUDE_DIRS = True
			DIRS = x[1]
		if x[0] == "+f":
			EXCLUDE_FILES = False
			FILES = x[1]
		if x[0] == "-f":
			EXCLUDE_FILES = True
			FILES = x[1]


# Call the main parse function.
parse()
<commit_msg>Don't search through existing the TODO file as this just adds more apparent TODOs.<commit_after># 
# A simple tool to help you easily analayse TODOs spread across a whole project.
# 
# Author(s):     <NAME>
# License:       MIT
# Last Modified: 8/6/2016
# 

# TODO Allow user to customise number of lines visible and trigger char.

import codecs
import sys
import os


# Define global variable ARGS which we then parse.
ARGS = sys.argv[1:]

CONTEXT = True
WRITE   = False

# Define global variables which manage whether or not the user is choosing to search/ignore specific filetypes and directories.
EXCLUDE_DIRS  = True
EXCLUDE_FILES = True

# Define global variables to hold the filetypes and directories that should be searched/ignored later in the parse.
DIRS  = []
FILES = []

# Allow user to search a flat hierarchy of specified dirs. I.e. if "." is provided, just search cwd not any subdirs.
FLAT = False

VALID = ["--no-context", "--write", "+df", "+d", "-d", "+f", "-f"]


# This is where the program kicks off.
def parse():
	global ARGS
	global CONTEXT
	global DIRS
	global EXCLUDE_FILES
	global EXCLUDE_DIRS
	global FILES
	global FLAT
	global WRITE


	command = "todo " + " ".join(ARGS) + "\n\n"

	# We first parse the params passed to the program which allows the user to specify directories to search/avoid and similarly with file types.
	parse_args()
	
	# Get the current working directory so we can use relative paths.
	cwd = os.getcwd()

	# Apply a bit of manipulation to allow user to specify current directory (relative root dir).
	DIRS = [cwd + d if d != "." else cwd for d in DIRS]
	DIRS = [d.replace('\\', '/') for d in DIRS]

	output = ""

	# Now traverse whole tree.
	for path, dirs, files in os.walk(cwd):
		for name in files:
			if name != "TODO.mdown":
				f = os.path.join(path, name)
				f = f.replace('\\', '/')

				# Boolean storing the value of whether or this file is valid based on the params the user provided.
				proceed = True

				# TODO Can speed these checks up, once false, we can break immediately.
				# First check if valid filetype.
				if EXCLUDE_FILES:
					proceed = proceed and not any(f.endswith(x) for x in FILES)
				else:
					proceed = proceed and any(f.endswith(x) for x in FILES)

				# Now check if in valid directory.
				if EXCLUDE_DIRS:
					proceed = proceed and not any(f.startswith(x) for x in DIRS)
				else:
					if FLAT:
						proceed = proceed and any(x + "/" + name == f for x in DIRS)
					else:
						proceed = proceed and any(f.startswith(x) for x in DIRS)

				# Only search the file for a todo comment if it is a valid file.
				if proceed:
					# TODO Improve the error handling here.
					try:
						content = [line.rstrip('\n').rstrip('\r') for line in codecs.open(f, 'r', encoding='utf-8')]
					except:
						content = []
						pass

					# If a todo exists we continue.
					if len([x for x in content if "TODO" in x]) > 0:
						# Print out filename.
						output += "\n### " + f + " ###"

						# Now we find the todos and print them out.
						for i in range (0, len(content)):
							line = content[i]

							if "TODO" in line:
								index = len(line.split("TODO")[0])
								comment = line[index + 4:].strip()

								output += "\n\n  > TODO " + comment + "\n\n"
								
								if CONTEXT:
									# Try and print line above and below as well as line.
									if i < len(content) - 1 and i > 0:
										output += "    " + str(i) + " " + content[i - 1] + "\n"
										output += "    " + str(i + 1) + " " + line + "\n"
										output += "    " + str(i + 2) + " " + content[i + 1] + "\n\n"
									# Try to print line below and line.
									elif i > 0:
										output += "    " + str(i) + " " + content[i - 1] + "\n"
										output += "    " + str(i + 1) + " " + line + "\n\n"
									# Try to print line above and line.
									elif i < len(content) - 1:
										output += "    " + str(i + 1) + " " + line + "\n"
										output += "    " + str(i + 2) + " " + content[i + 1] + "\n\n"
									# Otherwise we simpyl print the line.
									else:
										output += "    " + str(i + 1) + " " + line + "\n\n"
						
						output += "\n"

	if WRITE:
		# Write output to a TODO.txt file
		f = open("TODO.mdown", "w")
		f.write(command + output);
		f.close()
	else:
		print(output)


# Parses the command line args and updates the global vars accordingly.
def parse_args():
	global ARGS
	global CONTEXT
	global EXCLUDE_FILES
	global EXCLUDE_DIRS
	global DIRS
	global FILES
	global FLAT
	global VALID
	global WRITE


	# Allow the shorthand notation for searching within the current dir of 'todo .' instead of 'todo +df .'
	if len(ARGS) > 0:
		if ARGS[0] == ".":
			ARGS = ["+df", "."] + ARGS[1:]

	d_index = -1
	f_index = -1

	p = []

	for c in VALID:
		if c in ARGS:
			start = ARGS.index(c)
			if c in ["--no-context", "--write"]:
				p.append((c, []))
			else:
				params = get_params(ARGS[start + 1:])
				p.append((c, params))

	set_args(p)


def get_params(l):
	global VALID

	x = []

	for i in l:
		if i in VALID:
			return x
		else:
			x.append(i)

	return x


def set_args(p):
	global CONTEXT
	global EXCLUDE_FILES
	global EXCLUDE_DIRS
	global DIRS
	global FILES
	global FLAT
	global WRITE

	for x in p:
		if x[0] == "--no-context":
			CONTEXT = False
		if x[0] == "--write":
			WRITE = True
		if x[0] == "+df":
			FLAT = True
			EXCLUDE_DIRS = False
			DIRS = x[1]
		if x[0] == "+d":
			EXCLUDE_DIRS = False
			DIRS = x[1]
		if x[0] == "-d":
			EXCLUDE_DIRS = True
			DIRS = x[1]
		if x[0] == "+f":
			EXCLUDE_FILES = False
			FILES = x[1]
		if x[0] == "-f":
			EXCLUDE_FILES = True
			FILES = x[1]


# Call the main parse function.
parse()

{'fraction_non_alphanumeric': 0.12939068100358422, 'fraction_numerical': 0.007885304659498209, 'mean_word_length': 2.3841722255912674, 'pattern_counts': {'":': 17, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '991481', 'n_tokens_mistral': 4754, 'n_tokens_neox': 3842, 'n_words': 1436}