<commit_before># fileutils - helper methods related to the file system
#
# Copyright (c) 2013 - 2018 Software AG, Darmstadt, Germany and/or its licensors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# $Id: fileutils.py 301527 2017-02-06 15:31:43Z matj $
#

import shutil, os, os.path, time, platform, threading
import stat, sys
from utils.flatten import getStringList
import subprocess, errno

import logging
log = logging.getLogger('fileutils')

__isWindows = platform.system()=='Windows'

if __isWindows: # ugly ugly hacks due to stupid windows filesystem semantics. See http://bugs.python.org/issue4944
	try:
		import win32file
		class openForWrite:
			def __init__(self, dest, mode):
				assert mode == 'wb' # the Win32 API only has binary mode, not text mode, so only accept opening in this mode. This also means you must deal with any line ending issues yourself when using this.
				self.dest = dest
			def __enter__(self):
				self.Fd = win32file.CreateFile(self.dest, win32file.GENERIC_WRITE, 
					win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE  | win32file.FILE_SHARE_DELETE, 
					None, win32file.CREATE_ALWAYS, win32file.FILE_ATTRIBUTE_NORMAL, None)
				return self
			def write(self, string):
				win32file.WriteFile(self.Fd, string)
			def writelines(self, lines):
				for l in lines:
					self.write(l)
			def close(self):
				win32file.CloseHandle(self.Fd)
			def __exit__(self, ex_type, ex_val, tb):
				self.close()
	except:
		openForWrite = open
else:
	openForWrite = open


def mkdir(newdir):
	""" Recursively create the specified directory if it doesn't already exist. 
	
	If it does, exit without error. 

	@param newdir: The path to create.
	@return: newdir, to allow fluent use of this method. 
	"""
	origdir = newdir<commit_msg>add assertion to fileutils.openForWrite to stop people passing unicode strings in - doesn't make sense when writing as binary, and seems to result in implicit conversion to UTF-16 which is almost certainly not what user wants<commit_after># fileutils - helper methods related to the file system
#
# Copyright (c) 2013 - 2018 Software AG, Darmstadt, Germany and/or its licensors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# $Id: fileutils.py 301527 2017-02-06 15:31:43Z matj $
#

import shutil, os, os.path, time, platform, threading
import stat, sys
from utils.flatten import getStringList
import subprocess, errno

import logging
log = logging.getLogger('fileutils')

__isWindows = platform.system()=='Windows'

if __isWindows: # ugly ugly hacks due to stupid windows filesystem semantics. See http://bugs.python.org/issue4944
	try:
		import win32file
		class openForWrite:
			def __init__(self, dest, mode):
				assert mode == 'wb' # the Win32 API only has binary mode, not text mode, so only accept opening in this mode. This also means you must deal with any line ending issues yourself when using this.
				self.dest = dest
			def __enter__(self):
				self.Fd = win32file.CreateFile(self.dest, win32file.GENERIC_WRITE, 
					win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE  | win32file.FILE_SHARE_DELETE, 
					None, win32file.CREATE_ALWAYS, win32file.FILE_ATTRIBUTE_NORMAL, None)
				return self
			def write(self, string):
				assert isinstance(string, str), repr(string) # must pass byte strings not unicode objects when writing in binary mode
				win32file.WriteFile(self.Fd, string)
			def writelines(self, lines):
				for l in lines:
					self.write(l)
			def close(self):
				win32file.CloseHandle(self.Fd)
			def __exit__(self, ex_type, ex_val, tb):
				self.close()
	except:
		openForWrite = open
else:
	openForWrite = open


def mkdir(newdir):
	""" Recursively create the specified directory if it doesn't already exist. 
	
	If it does, exit without error. 

	@param newdir: The path to create.
	@return: newdir, to allow fluent use of this method. 
	"""
	origdir = newdir
{'fraction_non_alphanumeric': 0.08040816326530613, 'fraction_numerical': 0.024081632653061225, 'mean_word_length': 3.9705882352941178, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2597321', 'n_tokens_mistral': 1659, 'n_tokens_neox': 1458, 'n_words': 575}