<commit_before>#!/usr/bin/env python
"""
Run the Melisma harmonic analyzer on a series of midi files 

Effectively runs:
$ mftext/mftext MIDIFILE | meter/meter | harmony/harmony > OUTFILE

Usage:
./melisma-meter_harmony.py MIDI_DIRECTORY PROCESS_POOL_SIZE
"""

import sys,os,fnmatch
from multiprocessing import Pool
from subprocess import Popen,PIPE

MELISMA_ROOT = "../lib/melisma2003/"

def run(filename):
	fp,ext = os.path.splitext(filename)
	kfilepath = fp + '.k'
	kfileerr = fp + '.err'

	kfile = open(kfilepath,'w')
	kerr = open(kfileerr,'w')

	mftext = Popen([MELISMA_ROOT + 'mftext/mftext', filename],stdout=PIPE,stderr=kerr)
	meter = Popen([MELISMA_ROOT + 'meter/meter'],stdin=mftext.stdout,stdout=PIPE,stderr=kerr)
	harmony = Popen([MELISMA_ROOT + 'harmony/harmony'],stdin=meter.stdout,stdout=kfile,stderr=kerr)
	harmony.wait()

	kfile.close()
	kerr.close()
	
	print "finishing", kfilepath

def main():
	if len(sys.argv) >= 3:
		midiDirectory = sys.argv[1]
		poolSize = int(sys.argv[2])
	else:
		print __doc__
		sys.exit(1)

	midi_files = []

	# all midi files
	for (root, dirnames, filenames) in os.walk(midiDirectory):
		for filename in fnmatch.filter(filenames,'*.mid'):
			midi_files.append(os.path.abspath(os.path.join(root, filename)))

	# create and run the process pool
	p = Pool(poolSize)
	p.map(run,midi_files)


if __name__ == "__main__":
	main()
	pass<commit_msg>adding checks to melisma runner<commit_after>#!/usr/bin/env python
"""
Run the Melisma harmonic analyzer on a series of midi files 

Effectively runs:
$ mftext/mftext MIDIFILE | meter/meter | harmony/harmony > OUTFILE

Usage:
./melisma-meter_harmony.py MIDI_DIRECTORY PROCESS_POOL_SIZE
"""

import sys,os,fnmatch
from multiprocessing import Pool
from subprocess import Popen,PIPE

MELISMA_ROOT = "../lib/melisma2003/"

def cat_err(filename):
	with open(filename) as f:

		# print out the last five lines
		for line in f.readlines()[-5:]:
			sys.stderr.write(line)
		sys.stderr.write("\n")

def run(filename):
	fp,ext = os.path.splitext(filename)
	kfilepath = fp + '.k'
	kfileerr = fp + '.err'

	kfile = open(kfilepath,'w')
	kerr = open(kfileerr,'w')

	mftext = Popen([MELISMA_ROOT + 'mftext/mftext', filename],stdout=PIPE,stderr=kerr)
	meter = Popen([MELISMA_ROOT + 'meter/meter'],stdin=mftext.stdout,stdout=PIPE,stderr=kerr)
	harmony = Popen([MELISMA_ROOT + 'harmony/harmony'],stdin=meter.stdout,stdout=kfile,stderr=kerr)
	harmony.wait()

	# a VERY rough measure of success / error
	if kfile.tell() > 0:
		sys.stdout.write("success " + kfilepath + "\n")
	else:
		sys.stderr.write("error " + kfilepath + "\n")
		cat_err(kfileerr)
	
	kfile.close()
	kerr.close()

def main():
	if len(sys.argv) >= 3:
		midiDirectory = sys.argv[1]
		poolSize = int(sys.argv[2])
	else:
		print __doc__
		sys.exit(1)

	midi_files = []

	# all midi files
	for (root, dirnames, filenames) in os.walk(midiDirectory):
		for filename in fnmatch.filter(filenames,'*.mid'):
			midi_files.append(os.path.abspath(os.path.join(root, filename)))

	# create and run the process pool
	p = Pool(poolSize)
	p.map(run,midi_files)


if __name__ == "__main__":
	main()
	pass
{'fraction_non_alphanumeric': 0.14459762744469382, 'fraction_numerical': 0.005771080474511061, 'mean_word_length': 4.767097966728281, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '355485', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1201, 'n_words': 254}