<commit_before># -*- coding: UTF-8 -*-
from __future__ import division

import unittest

import os
test_dir = os.path.dirname(__file__)
latex_dir = os.path.join(test_dir, 'latex')
tmp_dir = os.path.join(test_dir, '.tmp')

import tempfile


bin_path = os.path.join(test_dir, os.path.pardir, 'bin', 'pydflatex')

## mod_path = os.path.join(test_dir, os.path.pardir)
## import sys
## sys.path.insert(0, mod_path)
## print sys.path

from subprocess import Popen, PIPE

import re




from pydflatex import Runner, Cleaner, LaTeXError, LogProcessor, Typesetter
from pydflatex.latex_logger import LaTeXLoggerColour

import termstyle
termstyle.enable()
colours = dict([(style, LaTeXLoggerColour.styled('', style)[:-8]) for style in LaTeXLoggerColour.colours])

class Harness(unittest.TestCase):


	def setup_logger(self):
		import logging
		self.logfile = tempfile.NamedTemporaryFile()
		self.handler = logging.FileHandler(self.logfile.name)
		self.t.logger = self.t.setup_logger([self.handler])

	def typeset(self, file_name, with_binary=False, ):
		tex_path = os.path.join(latex_dir, file_name)
		if with_binary:
			self.output = Popen([bin_path, tex_path], stderr=PIPE).communicate()[1]
		else:
			try:
				self.t.tex_path = tex_path
				self.t.run()
			finally:
				self.output = self.logfile.read()

	def assert_contains(self, match, line=None, regexp=False):
		all_out = self.output
		if line is None:
			out = all_out
		else:
			out = all_out.splitlines()[line]
		if not regexp:
			does_match = match in out
		else:
			does_match = re.search(match, self.output, re.MULTILINE)
		if not does_match:
			raise AssertionError("'%s' not in\n%s" % (match, out))

	def assert_success(self):
		self.assert_contains('Typesetting of')
		self.assert_contains('completed')
		self.assert_contains(colours['success'])

class TestLogParse(Harness):
	def setUp(self):
		self.t = LogProcessor(options={'colour':True, 'debug':False})
		self.setup_logger()

	def process_log(self, name):
		path = os.path.join(test_dir, 'latex', name)
		try:
			self.t.process_log(path + os.path.extsep + 'testlog')
		finally:
			self.output = self.logfile.read()

	def test_error(self):
		self.process_log('error')<commit_msg>str decoding<commit_after># -*- coding: UTF-8 -*-
from __future__ import division

import unittest

import os
test_dir = os.path.dirname(__file__)
latex_dir = os.path.join(test_dir, 'latex')
tmp_dir = os.path.join(test_dir, '.tmp')

import tempfile


bin_path = os.path.join(test_dir, os.path.pardir, 'bin', 'pydflatex')

## mod_path = os.path.join(test_dir, os.path.pardir)
## import sys
## sys.path.insert(0, mod_path)
## print sys.path

from subprocess import Popen, PIPE

import re




from pydflatex import Runner, Cleaner, LaTeXError, LogProcessor, Typesetter
from pydflatex.latex_logger import LaTeXLoggerColour

import termstyle
termstyle.enable()
colours = dict([(style, LaTeXLoggerColour.styled('', style)[:-8]) for style in LaTeXLoggerColour.colours])

class Harness(unittest.TestCase):


	def setup_logger(self):
		import logging
		self.logfile = tempfile.NamedTemporaryFile()
		self.handler = logging.FileHandler(self.logfile.name)
		self.t.logger = self.t.setup_logger([self.handler])

	def typeset(self, file_name, with_binary=False, ):
		tex_path = os.path.join(latex_dir, file_name)
		if with_binary:
			self.output = Popen([bin_path, tex_path], stderr=PIPE).communicate()[1]
		else:
			try:
				self.t.tex_path = tex_path
				self.t.run()
			finally:
				self.output = self.logfile.read()

	def assert_contains(self, match, line=None, regexp=False):
		all_out = self.output
		if line is None:
			out = all_out
		else:
			out = all_out.splitlines()[line]
		if not regexp:
			does_match = match in out
		else:
			does_match = re.search(match, self.output, re.MULTILINE)
		if not does_match:
			raise AssertionError("'%s' not in\n%s" % (match, out))

	def assert_success(self):
		self.assert_contains('Typesetting of')
		self.assert_contains('completed')
		self.assert_contains(colours['success'])

class TestLogParse(Harness):
	def setUp(self):
		self.t = LogProcessor(options={'colour':True, 'debug':False})
		self.setup_logger()

	def process_log(self, name):
		path = os.path.join(test_dir, 'latex', name)
		try:
			self.t.process_log(path + os.path.extsep + 'testlog')
		finally:
			self.output = self.logfile.read().decode('utf8')

	def test_error(self):
		self.process_log('error')
{'fraction_non_alphanumeric': 0.13638433515482695, 'fraction_numerical': 0.0020491803278688526, 'mean_word_length': 4.7126137841352405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '71442', 'n_tokens_mistral': 1740, 'n_tokens_neox': 1588, 'n_words': 332}