<commit_before>#!/usr/bin/python
import os
import sys
import fnmatch
import glob
import shutil

def glob_recursive(base_path, pattern):
	matches = []

	for root, dirnames, filenames in os.walk(base_path):
		for filename in fnmatch.filter(filenames, pattern):
			matches.append(os.path.join(root, filename))
	
	return matches

build_dir = './build'

def compress(in_name, out_name):
	os.system('yui-compressor --type js --preserve-semi -o ' + out_name + ' ' + in_name)

print 'Rebuilding...'
shutil.rmtree(build_dir)
os.mkdir(build_dir)

print 'Compressing scripts...'
scripts_path = 'scripts/'
os.mkdir(build_dir + '/' + scripts_path)
scripts = map(lambda x: x[len(scripts_path):], glob.glob(scripts_path + '*.js'))

for script in scripts:
	print '\tCompressing ' + script
	compress('./' + scripts_path + script, build_dir + '/' + scripts_path + script)
	
print 'Compressing plugins...'
plugins_path = 'plugins/'
os.mkdir(build_dir + '/' + plugins_path)
plugins = map(lambda x: x[len(plugins_path):], glob.glob(plugins_path + '*.js'))

for plugin in plugins:
	print '\tCompressing ' + plugin
	compress('./' + plugins_path + plugin, build_dir + '/' + plugins_path + plugin)

print 'Concatenating plugins...'

plugin_data = []

for plugin in plugins:
	'\tMunching ' + plugin
	
	for line in open(plugins_path + plugin, 'r'):
		plugin_data.append(line)

plugs_concat_filename = build_dir + '/' + plugins_path + 'all.plugins'<commit_msg>Updated the build script to batch-compress javascript files now that yui-compressor supports wildcards, which massively reduces build turn around times.<commit_after>#!/usr/bin/python
import os
import sys
import fnmatch
import glob
import shutil

def glob_recursive(base_path, pattern):
	matches = []

	for root, dirnames, filenames in os.walk(base_path):
		for filename in fnmatch.filter(filenames, pattern):
			matches.append(os.path.join(root, filename))
	
	return matches

build_dir = './build'

def compress(in_name, out_name):
	os.system('yui-compressor --type js --preserve-semi -o ' + out_name + ' ' + in_name)

print 'Rebuilding...'
shutil.rmtree(build_dir)
os.mkdir(build_dir)

print 'Compressing scripts...'
scripts_path = 'scripts/'
os.mkdir(build_dir + '/' + scripts_path)

scripts = map(lambda x: x[len(scripts_path):], glob.glob(scripts_path + '*.js'))

for script in scripts:
	print './' + scripts_path + script + ' -> ' + build_dir + '/' + scripts_path + script
	shutil.copy('./' + scripts_path + script, build_dir + '/' + scripts_path + script)

compress(build_dir + '/' + scripts_path + '*.js', '.js:.js')
	
print 'Compressing plugins...'
plugins_path = 'plugins/'
os.mkdir(build_dir + '/' + plugins_path)
plugins = map(lambda x: x[len(plugins_path):], glob.glob(plugins_path + '*.js'))

for plugin in plugins:
	shutil.copy('./' + plugins_path + plugin, build_dir + '/' + plugins_path + plugin)

compress(build_dir + '/' + plugins_path + '*.js', '.js:.js')

print 'Concatenating plugins...'

plugin_data = []

for plugin in plugins:
	'\tMunching ' + plugin
	
	for line in open(plugins_path + plugin, 'r'):
		plugin_data.append(line)

plugs_concat_filename = build_dir + '/' + plugins_path + 'all.plugins'
{'fraction_non_alphanumeric': 0.15213946117274169, 'fraction_numerical': 0.0, 'mean_word_length': 4.546572934973638, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6007311', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1052, 'n_words': 264}