<commit_before>#!/usr/bin/env python

import os
import os.path
import sys
import shutil
import logging
import fnmatch
from src import __version__
from setuptools import setup, find_packages

def convert_source_for_version(source_package, destination_package, version):
	tag_line = r'#--IMPORT_ALL_FROM_FUTURE--#'
	
	shutil.rmtree(destination_package,  ignore_errors=True)
	shutil.copytree(source_package, destination_package)
	
	if version != 2:
		# We wrote program implicated by version 3, if python version large than 2,
		# we need not change the sources.
		return
		
	for root, dirs, files in os.walk(destination_package):
		for afile in files:
			if fnmatch.fnmatch(afile, '*.py') or fnmatch.fnmatch(afile, '*.pyw'):
				file_path = os.path.join(root, afile)
				source_file = open(file_path, 'rb+')
				content = source_file.read()
				founded_index = content.find(tag_line)
				if founded_index >= 0:
					source_file.seek(0) # Go to beginning of file ...
					source_file.write(content[0:founded_index]) # All things before tag line
					# Import all future stuffs while we are using python 2.7.x
					source_file.write('from __future__ import nested_scopes\n')
					source_file.write('from __future__ import generators\n')
					source_file.write('from __future__ import division\n')
					source_file.write('from __future__ import absolute_import\n')
					source_file.write('from __future__ import with_statement\n')
					source_file.write('from __future__ import print_function\n')
					source_file.write('from __future__ import unicode_literals\n')
					source_file.write(content[founded_index+len(tag_line):]) # Rest after tag line
				source_file.close()

from_package = 'src'
to_package = 'rabird'

logging.basicConfig(level=logging.INFO)

source_version_file_path = 'source_version.txt'
source_version_file = None
if os.path.exists(source_version_file_path):
	source_version_file = open(source_version_file_path, 'rb+')
	version = int(source_version_file.read())
	if version != sys.version_info.major:
		source_version_file.seek(0)
		source_version_file.write(str(sys.version_info.major).encode('utf-8'))
		convert_source_for_version(from_package, to_package, sys.version_info.major)
	else:
		# If there have any file in 'from_package' newer than source_version_file's 
		# modify time, we do the complete convertion. 
		status = os.fstat(source_version_file.fileno())
		
		is_need_convert = False
		for root, dirs, files in os.walk(from_package):
			for afile in files:
				file_path = os.path.join(root, afile)
				if os.stat(file_path).st_mtime > status.st_mtime:
					is_need_convert = True
					break
			
			if is_need_convert:
				break
		
		if is_need_convert:
			convert_source_for_version(from_package, to_package, sys.version_info.major)
else:
	source_version_file = open(source_version_file_path, 'wb')
	source_version_file.write(str(sys.version_info.major).encode('utf-8'))
	convert_source_for_version(from_package, to_package, sys.version_info.major)
source_version_file.close()

# Exclude the original source package, only accept the preprocessed package!
our_packages = find_packages(exclude=[from_package]) 

our_requires = [
	'six>=1.3.0'
	]

if sys.platform == "win32":
	our_requires.append('pywin32>=218')
<commit_msg>Renamed 'convert_source_for_version()' to 'convert_source()'. Removed 'version' argument of convert_source(), it's unused, we could get the version from sys.version_info.<commit_after>#!/usr/bin/env python

import os
import os.path
import sys
import shutil
import logging
import fnmatch
from src import __version__
from setuptools import setup, find_packages

def convert_source(source_path, destination_path):
	tag_line = r'#--IMPORT_ALL_FROM_FUTURE--#'
	
	shutil.rmtree(destination_path,  ignore_errors=True)
	shutil.copytree(source_path, destination_path)
	
	if sys.version_info.major != 2:
		# We wrote program implicated by version 3, if python version large than 2,
		# we need not change the sources.
		return
		
	for root, dirs, files in os.walk(destination_path):
		for afile in files:
			if fnmatch.fnmatch(afile, '*.py') or fnmatch.fnmatch(afile, '*.pyw'):
				file_path = os.path.join(root, afile)
				source_file = open(file_path, 'rb+')
				content = source_file.read()
				founded_index = content.find(tag_line)
				if founded_index >= 0:
					source_file.seek(0) # Go to beginning of file ...
					source_file.write(content[0:founded_index]) # All things before tag line
					# Import all future stuffs while we are using python 2.7.x
					source_file.write('from __future__ import nested_scopes\n')
					source_file.write('from __future__ import generators\n')
					source_file.write('from __future__ import division\n')
					source_file.write('from __future__ import absolute_import\n')
					source_file.write('from __future__ import with_statement\n')
					source_file.write('from __future__ import print_function\n')
					source_file.write('from __future__ import unicode_literals\n')
					source_file.write(content[founded_index+len(tag_line):]) # Rest after tag line
				source_file.close()

from_package = 'src'
to_package = 'rabird'

logging.basicConfig(level=logging.INFO)

source_version_file_path = 'source_version.txt'
source_version_file = None
if os.path.exists(source_version_file_path):
	source_version_file = open(source_version_file_path, 'rb+')
	version = int(source_version_file.read())
	if version != sys.version_info.major:
		source_version_file.seek(0)
		source_version_file.write(str(sys.version_info.major).encode('utf-8'))
		convert_source(from_package, to_package)
	else:
		# If there have any file in 'from_package' newer than source_version_file's 
		# modify time, we do the complete convertion. 
		status = os.fstat(source_version_file.fileno())
		
		is_need_convert = False
		for root, dirs, files in os.walk(from_package):
			for afile in files:
				file_path = os.path.join(root, afile)
				if os.stat(file_path).st_mtime > status.st_mtime:
					is_need_convert = True
					break
			
			if is_need_convert:
				break
		
		if is_need_convert:
			convert_source(from_package, to_package)
else:
	source_version_file = open(source_version_file_path, 'wb')
	source_version_file.write(str(sys.version_info.major).encode('utf-8'))
	convert_source(from_package, to_package)
source_version_file.close()

# Exclude the original source package, only accept the preprocessed package!
our_packages = find_packages(exclude=[from_package]) 

our_requires = [
	'six>=1.3.0'
	]

if sys.platform == "win32":
	our_requires.append('pywin32>=218')

{'fraction_non_alphanumeric': 0.10170010721396845, 'fraction_numerical': 0.006432838106907643, 'mean_word_length': 4.963470319634703, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6751900', 'n_tokens_mistral': 2409, 'n_tokens_neox': 2130, 'n_words': 495}