<commit_before>import os
import sys
import shutil
import zipfile
import json

from lib import tools_lib as Tools

def PrintInfo (message):
	print ('INFO: ' + message)

def PrintError (message):
	print ('ERROR: ' + message)

def GetVersion (rootDir):
	packageJson = None
	with open (os.path.join (rootDir, 'package.json')) as packageJsonFile:
		packageJson = json.load (packageJsonFile)
	return packageJson['version']

def ESLintFolder (folder):
	result = Tools.RunCommand ('eslint', [folder])
	if result != 0:
		return False
	return True

def CompressCssFiles (inputFiles, outputFile):
	parameters = ['-o', outputFile]
	for inputFile in inputFiles:
		extension = os.path.splitext (inputFile)[1]
		if extension == '.css':
			parameters.append (inputFile)
	result = Tools.RunCommand ('cleancss', parameters)
	if result != 0:
		return False
	return True

def CompressJavascriptFiles (inputFiles, outputFile):
	parameters = []
	for inputFile in inputFiles:
		extension = os.path.splitext (inputFile)[1]
		if extension == '.js':
			parameters.append ('--js=' + inputFile)
	parameters.append ('--js_output_file=' + outputFile)
	result = Tools.RunCommand ('google-closure-compiler', parameters)
	if result != 0:
		return False
	return True

def CreateDestinationDir (config, rootDir, websiteDir, version, testBuild):
	if not os.path.exists (websiteDir):
		os.makedirs (websiteDir)

	webSourcesDir = os.path.join (websiteDir, 'o3dv')
	if not os.path.exists (webSourcesDir):
		os.makedirs (webSourcesDir)

	shutil.copy2 (os.path.join (rootDir, 'website', 'index.html'), websiteDir)
	shutil.copy2 (os.path.join (rootDir, 'website', 'embed.html'), websiteDir)
	shutil.copy2 (os.path.join (rootDir, 'website', 'robots.txt'), websiteDir)
	shutil.copytree (os.path.join (rootDir, 'libs'), os.path.join (websiteDir, 'libs'))
	shutil.copytree (os.path.join (rootDir, 'website', 'assets'), os.path.join (websiteDir, 'assets'))
	shutil.copytree (os.path.join (rootDir, 'website', 'o3dv', 'css', 'Quicksand'), os.path.join (websiteDir, 'o3dv', 'Quicksand'))
	shutil.copytree (os.path.join (rootDir, 'website', 'info'), os.path.join (websiteDir, 'info'))
	
	libFiles = config['lib_files']
	importerFiles = ['o3dv/o3dv.min.js']
	websiteFiles = [
		'o3dv/o3dv.website.min.css',
		'o3dv/o3dv.website.min.js'
	]
	
	htmlFileNames = [
		'index.html',
		'embed.html',
		os.path.join ('info', 'index.html'),
		os.path.join ('info', 'cookies.html')
	]
	for htmlFileName in htmlFileNames:
		htmlFilePath = os.path.join (websiteDir, htmlFileName)
		replacer = Tools.TokenReplacer (htmlFilePath, False)
		replacer.ReplaceTokenFileLinks ('<!-- libs start -->', '<!-- libs end -->', libFiles, None)
		replacer.ReplaceTokenFileLinks ('<!-- engine start -->', '<!-- engine end -->', importerFiles, version)
		replacer.ReplaceTokenFileLinks ('<!-- website start -->', '<!-- website end -->', websiteFiles, version)
		externalScriptContent = ''
		externalScriptContent += '<script type="text/javascript">' + replacer.eolChar
		externalScriptContent += '    OV.ExternalLibLocation = \'libs\';' + replacer.eolChar
		externalScriptContent += '</script>'
		replacer.ReplaceTokenContent ('<!-- externals start -->', '<!-- externals end -->', externalScriptContent)
		metaFile = os.path.join (rootDir, 'tools', 'website_meta_data.txt')
		if os.path.exists (metaFile):
			metaContent = Tools.GetFileContent (metaFile)
			replacer.ReplaceTokenContent ('<!-- meta start -->', '<!-- meta end -->', metaContent)
		analyticsFile = os.path.join (rootDir, 'tools', 'website_analytics_data.txt')
		if os.path.exists (analyticsFile) and not testBuild:
			analyticsContent = Tools.GetFileContent (analyticsFile)
			replacer.ReplaceTokenContent ('<!-- analytics start -->', '<!-- analytics end -->', analyticsContent)
		scriptFile = os.path.join (rootDir, 'tools', 'website_script_data.txt')
		if os.path.exists (scriptFile):
			scriptContent = Tools.GetFileContent (scriptFile)
			replacer.ReplaceTokenContent ('<!-- script start -->', '<!-- script end -->', scriptContent)
		replacer.WriteToFile (htmlFilePath)

def CreatePackage (rootDir, websiteDir, packageDir, version):
	if not os.path.exists (packageDir):
		os.makedirs (packageDir)

	libs = [
		'three.min-129.js',
		'three.license.md'
	]
	externalLibs = [
		'draco_decoder.js',
		'draco_decoder.wasm',
		'draco.license.md',
		'rhino3dm.min.js',
		'rhino3dm.wasm',
		'rhino3dm.license.md'
	]
	zipPath = os.path.join (packageDir, 'o3dv_' + version + '.zip')
	zip = zipfile.ZipFile (zipPath, mode = 'w', compression = zipfile.ZIP_DEFLATED)
	for lib in libs:
		zip.write (os.path.join (websiteDir, 'libs', lib), lib)
	for lib in externalLibs:
		zip.write (os.path.join (websiteDir, 'libs', lib), 'libs/' + lib)<commit_msg>Add loaders to the package.<commit_after>import os
import sys
import shutil
import zipfile
import json

from lib import tools_lib as Tools

def PrintInfo (message):
	print ('INFO: ' + message)

def PrintError (message):
	print ('ERROR: ' + message)

def GetVersion (rootDir):
	packageJson = None
	with open (os.path.join (rootDir, 'package.json')) as packageJsonFile:
		packageJson = json.load (packageJsonFile)
	return packageJson['version']

def ESLintFolder (folder):
	result = Tools.RunCommand ('eslint', [folder])
	if result != 0:
		return False
	return True

def CompressCssFiles (inputFiles, outputFile):
	parameters = ['-o', outputFile]
	for inputFile in inputFiles:
		extension = os.path.splitext (inputFile)[1]
		if extension == '.css':
			parameters.append (inputFile)
	result = Tools.RunCommand ('cleancss', parameters)
	if result != 0:
		return False
	return True

def CompressJavascriptFiles (inputFiles, outputFile):
	parameters = []
	for inputFile in inputFiles:
		extension = os.path.splitext (inputFile)[1]
		if extension == '.js':
			parameters.append ('--js=' + inputFile)
	parameters.append ('--js_output_file=' + outputFile)
	result = Tools.RunCommand ('google-closure-compiler', parameters)
	if result != 0:
		return False
	return True

def CreateDestinationDir (config, rootDir, websiteDir, version, testBuild):
	if not os.path.exists (websiteDir):
		os.makedirs (websiteDir)

	webSourcesDir = os.path.join (websiteDir, 'o3dv')
	if not os.path.exists (webSourcesDir):
		os.makedirs (webSourcesDir)

	shutil.copy2 (os.path.join (rootDir, 'website', 'index.html'), websiteDir)
	shutil.copy2 (os.path.join (rootDir, 'website', 'embed.html'), websiteDir)
	shutil.copy2 (os.path.join (rootDir, 'website', 'robots.txt'), websiteDir)
	shutil.copytree (os.path.join (rootDir, 'libs'), os.path.join (websiteDir, 'libs'))
	shutil.copytree (os.path.join (rootDir, 'website', 'assets'), os.path.join (websiteDir, 'assets'))
	shutil.copytree (os.path.join (rootDir, 'website', 'o3dv', 'css', 'Quicksand'), os.path.join (websiteDir, 'o3dv', 'Quicksand'))
	shutil.copytree (os.path.join (rootDir, 'website', 'info'), os.path.join (websiteDir, 'info'))
	
	libFiles = config['lib_files']
	importerFiles = ['o3dv/o3dv.min.js']
	websiteFiles = [
		'o3dv/o3dv.website.min.css',
		'o3dv/o3dv.website.min.js'
	]
	
	htmlFileNames = [
		'index.html',
		'embed.html',
		os.path.join ('info', 'index.html'),
		os.path.join ('info', 'cookies.html')
	]
	for htmlFileName in htmlFileNames:
		htmlFilePath = os.path.join (websiteDir, htmlFileName)
		replacer = Tools.TokenReplacer (htmlFilePath, False)
		replacer.ReplaceTokenFileLinks ('<!-- libs start -->', '<!-- libs end -->', libFiles, None)
		replacer.ReplaceTokenFileLinks ('<!-- engine start -->', '<!-- engine end -->', importerFiles, version)
		replacer.ReplaceTokenFileLinks ('<!-- website start -->', '<!-- website end -->', websiteFiles, version)
		externalScriptContent = ''
		externalScriptContent += '<script type="text/javascript">' + replacer.eolChar
		externalScriptContent += '    OV.ExternalLibLocation = \'libs\';' + replacer.eolChar
		externalScriptContent += '</script>'
		replacer.ReplaceTokenContent ('<!-- externals start -->', '<!-- externals end -->', externalScriptContent)
		metaFile = os.path.join (rootDir, 'tools', 'website_meta_data.txt')
		if os.path.exists (metaFile):
			metaContent = Tools.GetFileContent (metaFile)
			replacer.ReplaceTokenContent ('<!-- meta start -->', '<!-- meta end -->', metaContent)
		analyticsFile = os.path.join (rootDir, 'tools', 'website_analytics_data.txt')
		if os.path.exists (analyticsFile) and not testBuild:
			analyticsContent = Tools.GetFileContent (analyticsFile)
			replacer.ReplaceTokenContent ('<!-- analytics start -->', '<!-- analytics end -->', analyticsContent)
		scriptFile = os.path.join (rootDir, 'tools', 'website_script_data.txt')
		if os.path.exists (scriptFile):
			scriptContent = Tools.GetFileContent (scriptFile)
			replacer.ReplaceTokenContent ('<!-- script start -->', '<!-- script end -->', scriptContent)
		replacer.WriteToFile (htmlFilePath)

def CreatePackage (rootDir, websiteDir, packageDir, version):
	if not os.path.exists (packageDir):
		os.makedirs (packageDir)

	libs = [
		'three.min-129.js',
		'three.license.md'
	]
	externalLibs = [
		'draco_decoder.js',
		'draco_decoder.wasm',
		'draco.license.md',
		'rhino3dm.min.js',
		'rhino3dm.wasm',
		'rhino3dm.license.md'
	]
	zipPath = os.path.join (packageDir, 'o3dv_' + version + '.zip')
	zip = zipfile.ZipFile (zipPath, mode = 'w', compression = zipfile.ZIP_DEFLATED)
	for lib in libs:
		zip.write (os.path.join (websiteDir, 'libs', lib), lib)
	for lib in externalLibs:
		zip.write (os.path.join (websiteDir, 'libs', lib), 'libs/' + lib)
	for lib in os.listdir (os.path.join (websiteDir, 'libs', 'three_loaders')):
		zip.write (os.path.join (websiteDir, 'libs', 'three_loaders', lib), 'libs/three_loaders/' + lib)
{'fraction_non_alphanumeric': 0.1618563122923588, 'fraction_numerical': 0.0049833887043189366, 'mean_word_length': 4.620186697782964, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5663111', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3242, 'n_words': 767}