<commit_before># -*- coding: utf-8 -*-
import re, subprocess, os

TEXCOUNT_PATH = "TeXcount_3_0_1/texcount.pl"

def parseOutput(data, regex):
	s, s2 = None, None
	if "Subcounts:" in data:
		s, s2 = data.split("Subcounts:\n")
	else:
		s = data

	res = {
		"chapters": []
	}
	for l in re.findall(regex, s, re.U):
		# (sumc, text, headers, outside, headersN, floatsN, mathsI, mathsD) = l
		res["total"] = l

	# print re.search(regex, s, re.U).group(1,2,3,4,5,6)

	if s2:
		for l in re.findall(r"^  ([0-9]+)\+([0-9]+)\+([0-9]+) \(([0-9]+)\/([0-9]+)\/([0-9]+)\/([0-9]+)\) ([\w: äöü]+)", s2, re.U|re.M):
			# (text, headers, captions, headersH, floatsH, inlinesH, displayedH, name) = l
			res["chapters"].append((l[7],) + l[:7])

	return res

def correctLetters(letters, words):
	for k, v in letters.items():
		if k in words:
			if k == "chapters":
				for ci, cv in enumerate(v):
					for cci, ccv in enumerate(cv):
						letters[k][ci][cci] = int(ccv) + int(words[k][ci][cci])
			else:
				for ci, cv in enumerate(v):
					letters[k][ci] = int(cv) + int(words[k][ci])
	# return {k: (v if k not in words else
	# 				(correctLettersB(letters[k], words[k]) if isinstance(v, dict) else<commit_msg>List vs tuple !!<commit_after># -*- coding: utf-8 -*-
import re, subprocess, os

TEXCOUNT_PATH = "TeXcount_3_0_1/texcount.pl"

def parseOutput(data, regex):
	s, s2 = None, None
	if "Subcounts:" in data:
		s, s2 = data.split("Subcounts:\n")
	else:
		s = data

	res = {
		"chapters": []
	}
	for l in re.findall(regex, s, re.U):
		# (sumc, text, headers, outside, headersN, floatsN, mathsI, mathsD) = l
		res["total"] = l

	# print re.search(regex, s, re.U).group(1,2,3,4,5,6)

	if s2:
		for l in re.findall(r"^  ([0-9]+)\+([0-9]+)\+([0-9]+) \(([0-9]+)\/([0-9]+)\/([0-9]+)\/([0-9]+)\) ([\w: äöü]+)", s2, re.U|re.M):
			# (text, headers, captions, headersH, floatsH, inlinesH, displayedH, name) = l
			res["chapters"].append((l[7],) + l[:7])

	return res

def correctLetters(letters, words):
	for k, v in letters.items():
		if k in words:
			if k == "chapters":
				for ci, cv in enumerate(v):
					letters[k][ci] = tuple((int(ccv) + int(words[k][ci][cci]) if not isinstance(ccv, basestring) else ccv) for cci, ccv in enumerate(cv))
			else:
				letters[k] = tuple(int(cv) + int(words[k][ci]) for ci, cv in enumerate(v))
	# return {k: (v if k not in words else
	# 				(correctLettersB(letters[k], words[k]) if isinstance(v, dict) else
{'fraction_non_alphanumeric': 0.22199255890863995, 'fraction_numerical': 0.024803637866887142, 'mean_word_length': 3.6628131021194603, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5404049', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1044, 'n_words': 260}