<commit_before># -*- coding: UTF-8 -*-

"""
Tools for working with files and file systems
"""

from __future__ import division, unicode_literals, absolute_import

import os
import re
import itertools
import functools
import calendar
import contextlib
import logging
import datetime
import glob
import tempfile

log = logging.getLogger(__name__)

def get_unique_pathname(path, root = ''):
	"""Return a pathname possibly with a number appended to it so that it is
	unique in the directory."""
	path = os.path.join(root, path)
	# consider the path supplied, then the paths with numbers appended
	potentialPaths = itertools.chain((path,), __get_numbered_paths(path))
	potentialPaths = itertools.ifilterfalse(os.path.exists, potentialPaths)
	return potentialPaths.next()

def __get_numbered_paths(filepath):
	"""Append numbers in sequential order to the filename or folder name
	Numbers should be appended before the extension on a filename."""
	format = '%s (%%d)%s' % splitext_files_only(filepath)
	return itertools.imap(lambda n: format % n, itertools.count(1))

def splitext_files_only(filepath):
	"Custom version of splitext that doesn't perform splitext on directories"
	return (filepath, '') if os.path.isdir(filepath) else os.path.splitext()

def set_time(filename, mod_time):
	"""
	Set the modified time of a file
	"""
	log.debug('Setting modified time to %s', mod_time)
	mtime = calendar.timegm(mod_time.utctimetuple())
	# utctimetuple discards microseconds, so restore it (for consistency)
	mtime += mod_time.microsecond / 1000000
	atime = os.stat(filename).st_atime
	os.utime(filename, (atime, mtime))

def get_time(filename):
	"""
	Get the modified time for a file as a datetime instance
	"""
	ts = os.stat(filename).st_mtime
	return datetime.datetime.utcfromtimestamp(ts)

def insert_before_extension(filename, content):
	"""
	Given a filename and some content, insert the content just before
	the extension.

	>>> insert_before_extension('pages.pdf', '-old')
	'pages-old.pdf'
	"""
	parts = list(os.path.splitext(filename))
	parts[1:1] = [content]
	return ''.join(parts)

class DirectoryStack(list):
	r"""
	...

	DirectoryStack includes a context manager function that can be used
	to easily perform an operation in a separate directory.

	>>> orig_dir = os.getcwd()
	>>> stack = DirectoryStack()
	>>> with stack.context('/'): context_dir = os.getcwd()
	>>> orig_dir == os.getcwd()
	True
	>>> orig_dir == context_dir
	False
	>>> len(stack)
	0
	>>> stack.pushd('/')
	>>> len(stack)
	1
	>>> os.getcwd() == os.path.abspath('/')
	True
	>>> last_dir = stack.popd()
	>>> last_dir == context_dir
	True
	>>> os.getcwd() == orig_dir
	True
	"""
	def pushd(self, new_dir):
		self.append(os.getcwd())
		os.chdir(new_dir)

	def popd(self):
		res = os.getcwd()
		os.chdir(self.pop())
		return res

	@contextlib.contextmanager
	def context(self, new_dir):
		self.pushd(new_dir)
		try:
			yield
		finally:
			self.popd()

def recursive_glob(root, spec):
	"""
	Like iglob, but recurse directories

	>>> any('filesystem.py' in result for result in recursive_glob('.', '*.py'))
	True

	>>> all(result.startswith('.') for result in recursive_glob('.', '*.py'))
	True<commit_msg>Update test to match new location.<commit_after># -*- coding: UTF-8 -*-

"""
Tools for working with files and file systems
"""

from __future__ import division, unicode_literals, absolute_import

import os
import re
import itertools
import functools
import calendar
import contextlib
import logging
import datetime
import glob
import tempfile

log = logging.getLogger(__name__)

def get_unique_pathname(path, root = ''):
	"""Return a pathname possibly with a number appended to it so that it is
	unique in the directory."""
	path = os.path.join(root, path)
	# consider the path supplied, then the paths with numbers appended
	potentialPaths = itertools.chain((path,), __get_numbered_paths(path))
	potentialPaths = itertools.ifilterfalse(os.path.exists, potentialPaths)
	return potentialPaths.next()

def __get_numbered_paths(filepath):
	"""Append numbers in sequential order to the filename or folder name
	Numbers should be appended before the extension on a filename."""
	format = '%s (%%d)%s' % splitext_files_only(filepath)
	return itertools.imap(lambda n: format % n, itertools.count(1))

def splitext_files_only(filepath):
	"Custom version of splitext that doesn't perform splitext on directories"
	return (filepath, '') if os.path.isdir(filepath) else os.path.splitext()

def set_time(filename, mod_time):
	"""
	Set the modified time of a file
	"""
	log.debug('Setting modified time to %s', mod_time)
	mtime = calendar.timegm(mod_time.utctimetuple())
	# utctimetuple discards microseconds, so restore it (for consistency)
	mtime += mod_time.microsecond / 1000000
	atime = os.stat(filename).st_atime
	os.utime(filename, (atime, mtime))

def get_time(filename):
	"""
	Get the modified time for a file as a datetime instance
	"""
	ts = os.stat(filename).st_mtime
	return datetime.datetime.utcfromtimestamp(ts)

def insert_before_extension(filename, content):
	"""
	Given a filename and some content, insert the content just before
	the extension.

	>>> insert_before_extension('pages.pdf', '-old')
	'pages-old.pdf'
	"""
	parts = list(os.path.splitext(filename))
	parts[1:1] = [content]
	return ''.join(parts)

class DirectoryStack(list):
	r"""
	...

	DirectoryStack includes a context manager function that can be used
	to easily perform an operation in a separate directory.

	>>> orig_dir = os.getcwd()
	>>> stack = DirectoryStack()
	>>> with stack.context('/'): context_dir = os.getcwd()
	>>> orig_dir == os.getcwd()
	True
	>>> orig_dir == context_dir
	False
	>>> len(stack)
	0
	>>> stack.pushd('/')
	>>> len(stack)
	1
	>>> os.getcwd() == os.path.abspath('/')
	True
	>>> last_dir = stack.popd()
	>>> last_dir == context_dir
	True
	>>> os.getcwd() == orig_dir
	True
	"""
	def pushd(self, new_dir):
		self.append(os.getcwd())
		os.chdir(new_dir)

	def popd(self):
		res = os.getcwd()
		os.chdir(self.pop())
		return res

	@contextlib.contextmanager
	def context(self, new_dir):
		self.pushd(new_dir)
		try:
			yield
		finally:
			self.popd()

def recursive_glob(root, spec):
	"""
	Like iglob, but recurse directories

	>>> any('path.py' in result for result in recursive_glob('.', '*.py'))
	True

	>>> all(result.startswith('.') for result in recursive_glob('.', '*.py'))
	True
{'fraction_non_alphanumeric': 0.13183178453299732, 'fraction_numerical': 0.004095133091825485, 'mean_word_length': 4.7779799818016375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1301122', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2110, 'n_words': 614}