<commit_before>#!/usr/bin/env python
__version__ = '0.9.9'
import os, sys, subprocess, hashlib
#import pythonjs
#import pythonjs.pythonjs
#import pythonjs.python_to_pythonjs
#import pythonjs.pythonjs_to_cpp
#import pythonjs.pythonjs_to_verilog
#import pythonjs.typedpython as typedpython
import tempfile


def compile_js( script, module_path, main_name='main', directjs=False, directloops=False ):
	'''
	directjs = False     ## compatible with pythonjs-minimal.js
	directloops = False  ## allows for looping over strings, arrays, hmtlElements, etc. if true outputs cleaner code.
	'''
	fastjs = True  ## this is now the default, and complete python mode is deprecated
	result = {}

	pyjs = python_to_pythonjs(
		script,
		module_path=module_path,
		fast_javascript = fastjs,
		pure_javascript = directjs
	)

	if isinstance(pyjs, dict):  ## split apart by webworkers
		for jsfile in a:
			result[ jsfile ] = translate_to_javascript(
				a[jsfile],
				webworker=jsfile != 'main',
				requirejs=False,
				insert_runtime=False,
				fast_javascript = fastjs,
				fast_loops      = directloops
			)

	else:

		code = translate_to_javascript(
			pyjs,
			requirejs=False,
			insert_runtime=False,
			fast_javascript = fastjs,
			fast_loops      = directloops
		)
		if isinstance(code, dict):
			result.update( code )
		else:
			result['main'] = code

	if main_name != 'main':
		#assert main_name.endswith('.js')  ## allow tag names
		result[main_name] = result.pop('main')

	return result

def compile_java( javafiles ):
	assert 'JAVA_HOME' in os.environ
	tmpdir  = tempfile.gettempdir()
	cmd = ['javac']
	cmd.extend( javafiles )
	print ' '.join(cmd)
	subprocess.check_call(cmd, cwd=tmpdir)
	classfiles = [jfile.replace('.java', '.class') for jfile in javafiles]
	cmd = ['jar', 'cvf', 'mybuild.jar']
	cmd.extend( classfiles )
	print ' '.join(cmd)
	subprocess.check_call(cmd, cwd=tmpdir)
	jarfile = os.path.join(tmpdir,'mybuild.jar')
	assert os.path.isfile(jarfile)
	return {'class-files':classfiles, 'jar':jarfile}


def compile_giws_bindings( xml ):
	tmpdir  = tempfile.gettempdir()
	tmpfile = os.path.join(tmpdir, 'rusthon_giws.xml')
	open(tmpfile, 'wb').write(xml)
	cmd = [
		'giws',
		'--description-file='+tmpfile,
		'--output-dir='+tmpdir,
		#'--per-package',
		'--disable-return-size-array',
		#'--throws-exception-on-error', # requires GiwsException.hxx and GiwsException.cpp
	]
	#subprocess.check_call(cmd)
	proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
	proc.wait()
	if proc.returncode:
		raise RuntimeError(proc.stderr.read())
	else:
		headers = []
		impls = []
		for line in proc.stdout.read().splitlines():
			if line.endswith(' generated ...'):  ## TODO something better
				name = line.split()[0]
				if name.endswith('.hxx'):
					headers.append( name )
				elif name.endswith('.cpp'):
					impls.append( name )

		code = []
		for header in headers:
			data = open(os.path.join(tmpdir,header), 'rb').read()
			code.append( data )

		for impl in impls:
			data = open(os.path.join(tmpdir,impl), 'rb').read()
			lines = ['/* %s */' %impl]
			includes = []  ## ignore these
			for line in data.splitlines():
				if line.startswith('#include'):
					includes.append(line)
				else:
					lines.append(line)
			code.append( '\n'.join(lines) )

		return '\n'.join(code)

def java_to_rusthon( input ):
	j2pybin = 'j2py'
	if os.path.isfile(os.path.expanduser('~/java2python/bin/j2py')):
		j2pybin = os.path.expanduser('~/java2python/bin/j2py')
	print('======== %s : translate to rusthon' %j2pybin)
	j2py = subprocess.Popen([j2pybin, '--rusthon'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
	stdout,stderr  = j2py.communicate( input )
	if stderr: raise RuntimeError(stderr)
	if j2py.returncode: raise RuntimeError('j2py error!')
	print(stdout)
	print('---------------------------------')
	rcode = typedpython.transform_source(stdout.replace('    ', '\t'))
	print(rcode)
	print('---------------------------------')
	return rcode


def convert_to_markdown_project(path, rust=False, python=False, asm=False, c=False, cpp=False, java=False, java2rusthon=False, giws=False, file_metadata=False):
	files = []
	exts = []
	if rust: exts.append('.rs')
	if python: exts.append('.py')
	if asm: exts.append('.s')
	if c:   exts.append('.c')
	if cpp: exts.append('.cpp')
	if java: exts.append('.java')
	exts = tuple(exts)

	if os.path.isfile(path):
		files.append( path )

	elif os.path.isdir(path):
		for name in os.listdir(path):
			if name.endswith( exts ):
				files.append(os.path.join(path,name))

	project = []
	for file in files:
		print 'reading: ', file

		data = open(file, 'rb').read()
		header = []
		if file_metadata:
			header.extend([
			'#' + os.path.split(file)[-1],
			'* file: ' + file,
			'* md5sum   : %s' %hashlib.md5(data).hexdigest(),
			'* changed  : %s' %os.stat(file).st_mtime,
			''
			])
		lang = ''
		if file.endswith('.rs'):
			lang = 'rust'
		elif file.endswith('.py'):
			lang = 'python'
		elif file.endswith('.s'):
			lang = 'asm'
		elif file.endswith('.c'):
			lang = 'c'
		elif file.endswith('.cpp'):
			lang = 'c++'
		elif file.endswith('.java'):
			if java2rusthon:
				## pre converts into rusthon in markdown, for hand editing ##
				lang = 'rusthon'
				data = java_to_rusthon(data)
			else:
				## this still trigers java2rusthon at final compile time ##
				lang = 'java'<commit_msg>Fixed use python 3 style prints<commit_after>#!/usr/bin/env python
__version__ = '0.9.9'
import os, sys, subprocess, hashlib
#import pythonjs
#import pythonjs.pythonjs
#import pythonjs.python_to_pythonjs
#import pythonjs.pythonjs_to_cpp
#import pythonjs.pythonjs_to_verilog
#import pythonjs.typedpython as typedpython
import tempfile


def compile_js( script, module_path, main_name='main', directjs=False, directloops=False ):
	'''
	directjs = False     ## compatible with pythonjs-minimal.js
	directloops = False  ## allows for looping over strings, arrays, hmtlElements, etc. if true outputs cleaner code.
	'''
	fastjs = True  ## this is now the default, and complete python mode is deprecated
	result = {}

	pyjs = python_to_pythonjs(
		script,
		module_path=module_path,
		fast_javascript = fastjs,
		pure_javascript = directjs
	)

	if isinstance(pyjs, dict):  ## split apart by webworkers
		for jsfile in a:
			result[ jsfile ] = translate_to_javascript(
				a[jsfile],
				webworker=jsfile != 'main',
				requirejs=False,
				insert_runtime=False,
				fast_javascript = fastjs,
				fast_loops      = directloops
			)

	else:

		code = translate_to_javascript(
			pyjs,
			requirejs=False,
			insert_runtime=False,
			fast_javascript = fastjs,
			fast_loops      = directloops
		)
		if isinstance(code, dict):
			result.update( code )
		else:
			result['main'] = code

	if main_name != 'main':
		#assert main_name.endswith('.js')  ## allow tag names
		result[main_name] = result.pop('main')

	return result

def compile_java( javafiles ):
	assert 'JAVA_HOME' in os.environ
	tmpdir  = tempfile.gettempdir()
	cmd = ['javac']
	cmd.extend( javafiles )
	print(' '.join(cmd))
	subprocess.check_call(cmd, cwd=tmpdir)
	classfiles = [jfile.replace('.java', '.class') for jfile in javafiles]
	cmd = ['jar', 'cvf', 'mybuild.jar']
	cmd.extend( classfiles )
	print(' '.join(cmd))
	subprocess.check_call(cmd, cwd=tmpdir)
	jarfile = os.path.join(tmpdir,'mybuild.jar')
	assert os.path.isfile(jarfile)
	return {'class-files':classfiles, 'jar':jarfile}


def compile_giws_bindings( xml ):
	tmpdir  = tempfile.gettempdir()
	tmpfile = os.path.join(tmpdir, 'rusthon_giws.xml')
	open(tmpfile, 'wb').write(xml)
	cmd = [
		'giws',
		'--description-file='+tmpfile,
		'--output-dir='+tmpdir,
		#'--per-package',
		'--disable-return-size-array',
		#'--throws-exception-on-error', # requires GiwsException.hxx and GiwsException.cpp
	]
	#subprocess.check_call(cmd)
	proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
	proc.wait()
	if proc.returncode:
		raise RuntimeError(proc.stderr.read())
	else:
		headers = []
		impls = []
		for line in proc.stdout.read().splitlines():
			if line.endswith(' generated ...'):  ## TODO something better
				name = line.split()[0]
				if name.endswith('.hxx'):
					headers.append( name )
				elif name.endswith('.cpp'):
					impls.append( name )

		code = []
		for header in headers:
			data = open(os.path.join(tmpdir,header), 'rb').read()
			code.append( data )

		for impl in impls:
			data = open(os.path.join(tmpdir,impl), 'rb').read()
			lines = ['/* %s */' %impl]
			includes = []  ## ignore these
			for line in data.splitlines():
				if line.startswith('#include'):
					includes.append(line)
				else:
					lines.append(line)
			code.append( '\n'.join(lines) )

		return '\n'.join(code)

def java_to_rusthon( input ):
	j2pybin = 'j2py'
	if os.path.isfile(os.path.expanduser('~/java2python/bin/j2py')):
		j2pybin = os.path.expanduser('~/java2python/bin/j2py')
	print('======== %s : translate to rusthon' %j2pybin)
	j2py = subprocess.Popen([j2pybin, '--rusthon'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
	stdout,stderr  = j2py.communicate( input )
	if stderr: raise RuntimeError(stderr)
	if j2py.returncode: raise RuntimeError('j2py error!')
	print(stdout)
	print('---------------------------------')
	rcode = typedpython.transform_source(stdout.replace('    ', '\t'))
	print(rcode)
	print('---------------------------------')
	return rcode


def convert_to_markdown_project(path, rust=False, python=False, asm=False, c=False, cpp=False, java=False, java2rusthon=False, giws=False, file_metadata=False):
	files = []
	exts = []
	if rust: exts.append('.rs')
	if python: exts.append('.py')
	if asm: exts.append('.s')
	if c:   exts.append('.c')
	if cpp: exts.append('.cpp')
	if java: exts.append('.java')
	exts = tuple(exts)

	if os.path.isfile(path):
		files.append( path )

	elif os.path.isdir(path):
		for name in os.listdir(path):
			if name.endswith( exts ):
				files.append(os.path.join(path,name))

	project = []
	for file in files:
		print('reading: ', file)

		data = open(file, 'rb').read()
		header = []
		if file_metadata:
			header.extend([
			'#' + os.path.split(file)[-1],
			'* file: ' + file,
			'* md5sum   : %s' %hashlib.md5(data).hexdigest(),
			'* changed  : %s' %os.stat(file).st_mtime,
			''
			])
		lang = ''
		if file.endswith('.rs'):
			lang = 'rust'
		elif file.endswith('.py'):
			lang = 'python'
		elif file.endswith('.s'):
			lang = 'asm'
		elif file.endswith('.c'):
			lang = 'c'
		elif file.endswith('.cpp'):
			lang = 'c++'
		elif file.endswith('.java'):
			if java2rusthon:
				## pre converts into rusthon in markdown, for hand editing ##
				lang = 'rusthon'
				data = java_to_rusthon(data)
			else:
				## this still trigers java2rusthon at final compile time ##
				lang = 'java'
{'fraction_non_alphanumeric': 0.16777083138938137, 'fraction_numerical': 0.004385555659232994, 'mean_word_length': 3.6742259049280417, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1937541', 'n_tokens_mistral': 4268, 'n_tokens_neox': 3854, 'n_words': 841}