<commit_before>#!/usr/bin/python

# ABOUT:
#	A python script I have been working on to make the Makefile for a generic  
#	TeX document. It is probably similar to latexmk [1], but there is something
#	to be said for writing some things yourself!
#
# PUBLIC REPOSITORY:
#	https://github.com/pavdpr/latexmake
#
# SUGGESTED USE:
#	- Put the git repository on your machine (the location is /path/to/repo)	
#	- In your path (type "echo $PATH" w/o quotes in terminal to see the locatons
#	of path. A further suggestion: add a folder to PATH that does not require root
#	access, e.g, ~/bin), make a link to this file:
#		cd /someplace/in/path
#		ln -s /path/to/repo/latexmake.py latexmake
#
# HISTORY:
#	2014-01-30: <NAME>
#	- Initial version (0.0.1)
#
#	2014-05-28: <NAME>
#	- 
#
# TODO:
#	- Fix ability to read in multiline package commands
#		+ Strip all newlines?
#	- Add in latexdiff support
#		+ use gitlatexdiff https://github.com/daverted/gitlatexdiff as an example
#	- Add in git support
#
# LICENSE
# 	The MIT License (MIT)
#
#	Copyright 2014 <NAME>
#
# 	Permission is hereby granted, free of charge, to any person obtaining a copy
# 	of this software and associated documentation files (the "Software"), to 
#	deal in the Software without restriction, including without limitation the 
#	rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 
#	sell copies of the Software, and to permit persons to whom the Software is 
#	furnished to do so, subject to the following conditions:
#
#	The above copyright notice and this permission notice shall be included in 
#	all copies or substantial portions of the Software.
#
#	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
#	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
#	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
#	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
#	FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 
#	IN THE SOFTWARE.
#
# REFERENCES:
#	[1] http://users.phys.psu.edu/~collins/software/latexmk-jcc/ [2014-01-08]
#	[*] http://tex.stackexchange.com/questions/7770/file-extensions-of-latex-related-files
#	
#


# import other packages
import os;          # for interacting with files and directories
import re;          # for parsing strings
import sys;         # for getting command-line input
import datetime;    # for adding date stamps
import platform;    # for determining if a mac to use open


# set the version number
latexmake_version_major = 0;
latexmake_version_minor = 0;
latexmake_version_revision = 2;


#================================================================================
#
#		Exception Classes
#
#================================================================================


#-------------------------------------------------------------------------------
class latexmake_invalidInput( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidInput( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_noInput( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_noInput( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_invalidBasename( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidBasename( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_nonexistantFile( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidBasename( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_invalidArgument( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidArgument( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_invalidBracketOrder( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidArgument( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_makeDoesNotExist( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_makeDoesNotExist( RuntimeError )
#-------------------------------------------------------------------------------


#================================================================================
#
#		Executable testing code
#
#================================================================================


#-------------------------------------------------------------------------------
def is_exe( fpath ):
	# used to see if fpath is an executable
	# http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python
	return os.path.isfile( fpath ) and os.access( fpath, os.X_OK );
# fed is_exe( fpath )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def functionExists( program ):

	fpath, fname = os.path.split( program );
	if fpath:
		return is_exe( program );
	else:
		for path in os.environ[ "PATH" ].split( os.pathsep ):
			path = path.strip( '"' ); # TODO: see if I can use "\""
			exe_file = os.path.join( path, program );
			if is_exe( exe_file ):
				return True;

		return False;
# fed functionExists( program )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def which( program ):
	# a unix-like which
	# http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python

	fpath, fname = os.path.split( program );
	if fpath:
		if is_exe( program ):
			return program;
	else:
		for path in os.environ[ "PATH" ].split( os.pathsep ):
			path = path.strip( '"' );
			exe_file = os.path.join( path, program );
			if is_exe( exe_file ):
				return exe_file;

	# program does not exist, return None
	return None;
# fed which( program ) 
#-------------------------------------------------------------------------------


#================================================================================
#
#		Output Messages
#
#================================================================================


#-------------------------------------------------------------------------------
def latexmake_usage():
	output = "latexmake [options] basefilename\n";
	output += "\t--tex=/path/to/tex compiler\n";
	output += "\t--bib=/path/to/bib compiler\n";
	#output += "\t--nooverwrite\t\t\tWill not overwrite a Makefile\n";
	return output
# fed latexmake_usage()
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def latexmake_copyright():
	output = "latexmake\n";
	output += "\n\tCopyright 2014 <NAME>\n";
	return output
# fed latexmake_copyright()
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def latexmake_version():
	output = \
	str( latexmake_version_major ) + "." + \
	str( latexmake_version_minor ) + "." + \
	str( latexmake_version_revision );
	return output;
# fed latexmake_version()
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def latexmake_header():
	output  = "# Generated by latexmake v" + latexmake_version() + "\n";
	output += "# Created on " + str( datetime.datetime.utcnow() ) + " UTC\n";
	output += "# latexmake repository: https://github.com/pavdpr/latexmake\n";
	return output;
# fed latexmake_header()
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def warning( message ):
	print message;
# fed warning( message );
#-------------------------------------------------------------------------------


#================================================================================
#
#		helper functions
#
#================================================================================

#-------------------------------------------------------------------------------
def unique( data ):
	# I do not think this can be solved with list comprehension
	output = [];
	for item in data:
		if item not in output:
			output.append( item );
	return output;

# fed unique( data )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def compliment( data, exclude ):
	return [ item for item in data if item not in exclude ];

# fed compliment( data, exclude )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseLongLines( line, lineLength, tabLength, numTabs, extra ):
	if numTabs < 0:
		numTabs = 0;

	maxLength = lineLength - numTabs * tabLength - 2;
	output = [];
	tmp = "";
	l = line.split();
	ct = 0;
	while len( l ) > 0:
		run = True;
		while ( run ):
			if len( tmp ) == 0:
				tmp = l[ 0 ];
				l = l[ 1: ];
				if len( l ) == 0:
					run = False;
			elif len( tmp + " " + l[ 0 ] ) > maxLength:
				run = False;
			else:
				tmp += ( " " + l[ 0 ] );
				l = l[ 1: ];
				if len( l ) == 0:
					run = False;

		output.append( tmp );
		tmp = "";
		if extra:
			# add room for double tab on second or higher lines
			ct = ct + 1;
			if ( ct == 1 ) and ( lineLength - 2 - ( numTabs + 1 ) * tabLength > 0 ):
				maxLength -= tabLength;

	return output;
# fed parseLongLines( line, maxLength )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def writeLongLines( fid, line, lineLength, tabLength, numTabs, extra ):
	lines = parseLongLines( line, lineLength, tabLength, numTabs, extra );
	if numTabs < 0:
		numTabs = 0;

	tabs = "";
	for i in range( 0, numTabs ):
		tabs += "\t";

	if not extra:
		for l in lines[ :-1 ]:
			fid.write( tabs + l + " \\\n" );
		fid.write( tabs + lines[ -1 ] + "\n" );
	else:
		fid.write( tabs + lines[0] + " \\\n" );
		tabs += "\t";
		for l in lines[ 1:-1 ]:
			fid.write( tabs + l + " \\\n" );
		fid.write( tabs + lines[ -1 ] + "\n" );
	return;
# fed parseLongLines( line, maxLength )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseEquals( s ):
	try:
		idx = s.find( "=" );
		if idx < 0:
			raise latexmake_invalidArgument( s );
		left = s[ :idx ];
		right = s[ idx+1: ];
	except latexmake_invalidArgument, e:
		raise latexmake_invalidArgument( e.args );
	else:
		return ( left, right );
# fed parseEquals( s )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findUnescaped( line, char ):
	idx = line.find( char );
	if idx > 0:
		if line[ idx - 1 ] == "\\":
			if idx < len( line ) - len( char ):
				# recursively look for the next char
				tmp = findUnescaped( line[ idx + len( char ): ], char );
				if tmp < 0:
					return tmp;
				else:
					return tmp + idx + len( char );
			else:
				return -1;
	# if idx == 0: cannot be escaped => return idx (=0)
	# if idx < 0: return idx (=-1)

	return idx;
# fed findNext( line, char )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def purifyListOfStrings( l, key ):
	return [ item for item in l if not re.search( key, item ) ];
# fed def purifyListOfStrings( l, key )	
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseDataInSquiglyBraces( line ):
	output = [];
	lstack = [];

	lidx = findUnescaped( line, "{" );
	ridx = findUnescaped( line, "}" );

	# no braces
	if lidx < 0:
		return [];

	run = True;
	while run:

		if len( lstack ) == 0:
			if lidx < 0:
				if ridx < 0:
					run = False;
				else:
					message = "In string: '" + line + "':\n";
					message += "\tunbalanced {}";
					raise latexmake_invalidBracketOrder( message );
			else:
				lstack.append( lidx );
				tmp = findUnescaped( line[ lidx+1:], "{" );
				if tmp < 0:
					lidx = -1;
				else:
					lidx += tmp + 1;
		elif lidx < ridx and lidx > 0:
			lstack.append( lidx );
			tmp = findUnescaped( line[ lidx+1:], "{" );
			if tmp < 0:
				lidx = -1;
			else:
				lidx += tmp + 1;
		elif ridx >= 0:
			if ( len( lstack  ) == 0 ):
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced {}";
				raise latexmake_invalidBracketOrder( message );
			elif lstack[-1] >= ridx:
				message = "In string: '" + line + "':\n";
				message += "\t'}' appears before '{'";
				raise latexmake_invalidBracketOrder( message );

			output.append( line[ lstack.pop() + 1:ridx ] );
			tmp = findUnescaped( line[ ridx+1:], "}" );
			if tmp < 0:
				ridx = -1;
			else:
				ridx += tmp + 1;
		else:
			if findUnescaped( line[ ridx+1:], "}" ) < 0:
				run = False;
			else:
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced {}";
				raise latexmake_invalidBracketOrder( message );

	return output;
# fed parseDataInSquiglyBraces( line )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseDataInSquareBraces( line ):
	output = [];
	lstack = [];

	lidx = findUnescaped( line, "[" );
	ridx = findUnescaped( line, "]" );

	# no braces
	if lidx < 0:
		return [];

	run = True;
	while run:

		if len( lstack ) == 0:
			if lidx < 0:
				if ridx < 0:
					run = False;
				else:
					message = "In string: '" + line + "':\n";
					message += "\tunbalanced []";
					raise latexmake_invalidBracketOrder( message );
			else:
				lstack.append( lidx );
				tmp = findUnescaped( line[ lidx+1:], "[" );
				if tmp < 0:
					lidx = -1;
				else:
					lidx += tmp + 1;
		elif lidx < ridx and lidx > 0:
			lstack.append( lidx );
			tmp = findUnescaped( line[ lidx+1:], "[" );
			if tmp < 0:
				lidx = -1;
			else:
				lidx += tmp + 1;
		elif ridx >= 0:
			if ( len( lstack  ) == 0 ):
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced []";
				raise latexmake_invalidBracketOrder( message );
			elif lstack[-1] >= ridx:
				message = "In string: '" + line + "':\n";
				message += "\t']' appears before '['";
				raise latexmake_invalidBracketOrder( message );

			output.append( line[ lstack.pop() + 1:ridx ] );
			tmp = findUnescaped( line[ ridx+1:], "]" );
			if tmp < 0:
				ridx = -1;
			else:
				ridx += tmp + 1;
		else:
			if findUnescaped( line[ ridx+1:], "]" ) < 0:
				run = False;
			else:
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced []";
				raise latexmake_invalidBracketOrder( message );

	return output;
# fed parseDataInSquareBraces( line )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseDataInParentheses( line ):
	output = [];
	lstack = [];

	lidx = findUnescaped( line, "(" );
	ridx = findUnescaped( line, ")" );

	# no braces
	if lidx < 0:
		return [];

	run = True;
	while run:

		if len( lstack ) == 0:
			if lidx < 0:
				if ridx < 0:
					run = False;
				else:
					message = "In string: '" + line + "':\n";
					message += "\tunbalanced ()";
					raise latexmake_invalidBracketOrder( message );
			else:
				lstack.append( lidx );
				tmp = findUnescaped( line[ lidx+1:], "(" );
				if tmp < 0:
					lidx = -1;
				else:
					lidx += tmp + 1;
		elif lidx < ridx and lidx > 0:
			lstack.append( lidx );
			tmp = findUnescaped( line[ lidx+1:], "(" );
			if tmp < 0:
				lidx = -1;
			else:
				lidx += tmp + 1;
		elif ridx >= 0:
			if ( len( lstack  ) == 0 ):
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced ()";
				raise latexmake_invalidBracketOrder( message );
			elif lstack[-1] >= ridx:
				message = "In string: '" + line + "':\n";
				message += "\t')' appears before '('";
				raise latexmake_invalidBracketOrder( message );

			output.append( line[ lstack.pop() + 1:ridx ] );
			tmp = findUnescaped( line[ ridx+1:], ")" );
			if tmp < 0:
				ridx = -1;
			else:
				ridx += tmp + 1;
		else:
			if findUnescaped( line[ ridx+1:], ")" ) < 0:
				run = False;
			else:
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced ()";
				raise latexmake_invalidBracketOrder( message );

	return output;
# fed parseDataInParenthesies( line )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseDataInAngleBraces( line ):
	lstack = [];

	lidx = findUnescaped( line, "<" );
	ridx = findUnescaped( line, ">" );

	# no braces
	if lidx < 0:
		return [];

	run = True;
	while run:

		if len( lstack ) == 0:
			if lidx < 0:
				if ridx < 0:
					run = False;
				else:
					message = "In string: '" + line + "':\n";
					message += "\tunbalanced <>";
					raise latexmake_invalidBracketOrder( message );
			else:
				lstack.append( lidx );
				tmp = findUnescaped( line[ lidx+1:], "<" );
				if tmp < 0:
					lidx = -1;
				else:
					lidx += tmp + 1;
		elif lidx < ridx and lidx > 0:
			lstack.append( lidx );
			tmp = findUnescaped( line[ lidx+1:], "<" );
			if tmp < 0:
				lidx = -1;
			else:
				lidx += tmp + 1;
		elif ridx >= 0:
			if ( len( lstack  ) == 0 ):
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced <>";
				raise latexmake_invalidBracketOrder( message );
			elif lstack[-1] >= ridx:
				message = "In string: '" + line + "':\n";
				message += "\t'>' appears before '<'";
				raise latexmake_invalidBracketOrder( message );

			output.append( line[ lstack.pop() + 1:ridx ] );
			tmp = findUnescaped( line[ ridx+1:], ">" );
			if tmp < 0:
				ridx = -1;
			else:
				ridx += tmp + 1;
		else:
			if findUnescaped( line[ ridx+1:], ">" ) < 0:
				run = False;
			else:
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced <>";
				raise latexmake_invalidBracketOrder( message );

	return output;
# fed parseDataInAngleBraces( line )
#-------------------------------------------------------------------------------



#================================================================================
#
#		TeX file parsing
#
#================================================================================


#-------------------------------------------------------------------------------
def parseCommaSeparatedData( line ):
	tmp = line.split( "," );
	# trim whitespace
	return [ item.strip() for item in tmp ];
# fed parseCommaSeparatedData( line )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def removeTeXcomments( texFile, thisFileName ):
	texFileList = texFile.splitlines();
	output = "";
	for line in texFileList:

		# remove empty lines as well
		if len( line ) > 0:
			loc = line.find( "%" );
			if loc < 0:
				output += ( line + "\n" );
			elif loc > 0:
				run = True;
				tmp = "";
				while run:
					if len( line ) == 0:
						run = False;
					elif loc < 0:
						tmp += line;
						run = False;
					elif loc > 0:
						if line[loc-1] != "\\":
							# We have a real comment and not an escaped one
							tmp += line[:loc];
							run = False;
						else:
							# we have an escaped comment, this is valid
							tmp += line[:loc+1];
							line = line[loc+1:];
							loc = line.find( "%" )
							# run = True;
					else:
						# loc is 0
						run = False;
				if len( tmp ) > 0:
					output += ( tmp + "\n" );	
			else:
				# remove lines that are all comment
				# do nothing here
				pass;
	return output
# fed removeTeXcomments( texFile )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findPackages( texFile, params, thisFileName ):
	# TODO: This does not work in cases like
	# \usepackage[sorting=ydnt,		% reverse sort by date
	# bibstyle=authoryear,			% use author year style (not sure how much this does w/o /cite{})
	# defernumbers=true,			%
	# maxnames=99,			 		% show up to 99 author names in the reference
	# firstinits=true, 				% use first initials
	# terseinits=false,				% period after initials
	# uniquename=init,				%
	# dashed=false,					%
	# uniquename=false,				%
	# doi=true,						% include doi where present
	# isbn=false,					% include isbn where present
	# useprefix=true,				%
	# natbib=true,					%
	# backend=biber]				% use biber on the backend 
	# {biblatex}
	#
	# I probably need to abandon regular expressions
	# 
	# TODO: also have this work with \requirepackage

	key = r"\\usepackage(\[.*\])?(\{.*\})";
	locs = re.findall( key, texFile );
	packages = [];
	for tmp in locs:
		line = tmp[ -1 ];
		for part in purifyListOfStrings( parseDataInSquiglyBraces( line ), \
				r"[\{\}]" ):
			p = parseCommaSeparatedData( part );
			packages += p;
			for package in p:
				# check to see if it is local
				params = findLocalStyFiles( package, params, thisFileName );
				if package == "biblatex":
					# TODO finish
					m = re.search( r"\\usepackage(\[.*\]?\{\s*biblatex\s*\})", \
						texFile );
					optionStrings = parseDataInSquareBraces( m.group() );
					for optionString in optionStrings:
						options = parseCommaSeparatedData( optionString );
						opts = [ parseEquals( option ) for option in options ];
						backend = "bibtex"; # default
						for opt in opts:
							if opt[ 0 ] == "backend":
								backend = opt[ 1 ];
						if functionExists( backend ):
							params[ "bib_engine" ] = backend.upper();
						else:
							warning( "The bibliography backend \"" + backend + \
								"\" cannot be found" );

				elif package == "epstopdf":
					if params[ "tex_engine" ] == "PDFLATEX":
						params[ "fig_extensions" ] += ".eps";

				elif package == "makeidx":
					params[ "make_index_in_default" ] = True;

				# elif package == "glossaries":
				# 	params[ "make_glossary_in_default" ] = True;

	# update the packages list in params				
	params[ "packages" ] += packages;
	return params;
# fed findPackages( texFile, params )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findGraphicsPaths( texFile, params, thisFileName ):
	locs = re.findall( r"\\graphicspath\{(.*)\}", texFile );
	for line in locs:
		for part in purifyListOfStrings( parseDataInSquiglyBraces( line ), \
			r"[\{\}]" ):
			if os.path.isdir( part ) and os.path.exists( part ):
				# we are a valid path
				params[ "graphics_paths" ].append( os.path.abspath( part ) );
				if os.path.abspath( part ) not in params[ "sub_paths" ]:
					params[ "sub_paths" ].append( os.path.abspath( part ) );
			else:
				#TODO?: raise exception
				warning( "In \"" + thisFileName + "\": \"" + part + \
					"\" is not a valid graphicspath" );
	return params;
# fed findGraphicsPaths( texFile, params )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findGraphicsExtensions( texFile, params, thisFileName ):
	locs = re.findall( r"\\DeclareGraphicsExtensions\{(.*)\}", texFile );
	if not locs:
		return params;
	for item in locs:
		if parseDataInSquiglyBraces( item ):
			# residual squiglies
			item = parseDataInSquiglyBraces( item );
			params[ "fig_extensions" ] = parseCommaSeparatedData( item );
			#TODO: check to see if I can have a .jpg if the g.e. is just .eps
	return params;
# fed findGraphicsExtensions( texFile, params )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findFigures( texFile, params, thisFileName ):
	# TODO: also get pgf figures
	m = re.findall( r"\\includegraphics(\[.*\])?\s*(\{\w*\})", texFile );
	if not m:<commit_msg>Added ability for spaces or slashes in filenames<commit_after>#!/usr/bin/python

# ABOUT:
#	A python script I have been working on to make the Makefile for a generic  
#	TeX document. It is probably similar to latexmk [1], but there is something
#	to be said for writing some things yourself!
#
# PUBLIC REPOSITORY:
#	https://github.com/pavdpr/latexmake
#
# SUGGESTED USE:
#	- Put the git repository on your machine (the location is /path/to/repo)	
#	- In your path (type "echo $PATH" w/o quotes in terminal to see the locatons
#	of path. A further suggestion: add a folder to PATH that does not require root
#	access, e.g, ~/bin), make a link to this file:
#		cd /someplace/in/path
#		ln -s /path/to/repo/latexmake.py latexmake
#
# HISTORY:
#	2014-01-30: <NAME>
#	- Initial version (0.0.1)
#
#	2014-05-28: <NAME>
#	- 
#
# TODO:
#	- Fix ability to read in multiline package commands
#		+ Strip all newlines?
#	- Add in latexdiff support
#		+ use gitlatexdiff https://github.com/daverted/gitlatexdiff as an example
#	- Add in git support
#
# LICENSE
# 	The MIT License (MIT)
#
#	Copyright 2014 <NAME>
#
# 	Permission is hereby granted, free of charge, to any person obtaining a copy
# 	of this software and associated documentation files (the "Software"), to 
#	deal in the Software without restriction, including without limitation the 
#	rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 
#	sell copies of the Software, and to permit persons to whom the Software is 
#	furnished to do so, subject to the following conditions:
#
#	The above copyright notice and this permission notice shall be included in 
#	all copies or substantial portions of the Software.
#
#	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
#	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
#	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
#	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
#	FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 
#	IN THE SOFTWARE.
#
# REFERENCES:
#	[1] http://users.phys.psu.edu/~collins/software/latexmk-jcc/ [2014-01-08]
#	[*] http://tex.stackexchange.com/questions/7770/file-extensions-of-latex-related-files
#	
#


# import other packages
import os;          # for interacting with files and directories
import re;          # for parsing strings
import sys;         # for getting command-line input
import datetime;    # for adding date stamps
import platform;    # for determining if a mac to use open


# set the version number
latexmake_version_major = 0;
latexmake_version_minor = 0;
latexmake_version_revision = 2;


#================================================================================
#
#		Exception Classes
#
#================================================================================


#-------------------------------------------------------------------------------
class latexmake_invalidInput( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidInput( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_noInput( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_noInput( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_invalidBasename( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidBasename( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_nonexistantFile( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidBasename( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_invalidArgument( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidArgument( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_invalidBracketOrder( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_invalidArgument( RuntimeError )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
class latexmake_makeDoesNotExist( RuntimeError ):
   def __init__( self, arg ):
      self.args = arg;
# class latexmake_makeDoesNotExist( RuntimeError )
#-------------------------------------------------------------------------------


#================================================================================
#
#		Executable testing code
#
#================================================================================


#-------------------------------------------------------------------------------
def is_exe( fpath ):
	# used to see if fpath is an executable
	# http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python
	return os.path.isfile( fpath ) and os.access( fpath, os.X_OK );
# fed is_exe( fpath )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def functionExists( program ):

	fpath, fname = os.path.split( program );
	if fpath:
		return is_exe( program );
	else:
		for path in os.environ[ "PATH" ].split( os.pathsep ):
			path = path.strip( '"' ); # TODO: see if I can use "\""
			exe_file = os.path.join( path, program );
			if is_exe( exe_file ):
				return True;

		return False;
# fed functionExists( program )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def which( program ):
	# a unix-like which
	# http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python

	fpath, fname = os.path.split( program );
	if fpath:
		if is_exe( program ):
			return program;
	else:
		for path in os.environ[ "PATH" ].split( os.pathsep ):
			path = path.strip( '"' );
			exe_file = os.path.join( path, program );
			if is_exe( exe_file ):
				return exe_file;

	# program does not exist, return None
	return None;
# fed which( program ) 
#-------------------------------------------------------------------------------


#================================================================================
#
#		Output Messages
#
#================================================================================


#-------------------------------------------------------------------------------
def latexmake_usage():
	output = "latexmake [options] basefilename\n";
	output += "\t--tex=/path/to/tex compiler\n";
	output += "\t--bib=/path/to/bib compiler\n";
	#output += "\t--nooverwrite\t\t\tWill not overwrite a Makefile\n";
	return output
# fed latexmake_usage()
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def latexmake_copyright():
	output = "latexmake\n";
	output += "\n\tCopyright 2014 <NAME>\n";
	return output
# fed latexmake_copyright()
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def latexmake_version():
	output = \
	str( latexmake_version_major ) + "." + \
	str( latexmake_version_minor ) + "." + \
	str( latexmake_version_revision );
	return output;
# fed latexmake_version()
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def latexmake_header():
	output  = "# Generated by latexmake v" + latexmake_version() + "\n";
	output += "# Created on " + str( datetime.datetime.utcnow() ) + " UTC\n";
	output += "# latexmake repository: https://github.com/pavdpr/latexmake\n";
	return output;
# fed latexmake_header()
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def warning( message ):
	print message;
# fed warning( message );
#-------------------------------------------------------------------------------


#================================================================================
#
#		helper functions
#
#================================================================================

#-------------------------------------------------------------------------------
def unique( data ):
	# I do not think this can be solved with list comprehension
	output = [];
	for item in data:
		if item not in output:
			output.append( item );
	return output;

# fed unique( data )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def compliment( data, exclude ):
	return [ item for item in data if item not in exclude ];

# fed compliment( data, exclude )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseLongLines( line, lineLength, tabLength, numTabs, extra ):
	if numTabs < 0:
		numTabs = 0;

	maxLength = lineLength - numTabs * tabLength - 2;
	output = [];
	tmp = "";
	l = line.split();
	ct = 0;
	while len( l ) > 0:
		run = True;
		while ( run ):
			if len( tmp ) == 0:
				tmp = l[ 0 ];
				l = l[ 1: ];
				if len( l ) == 0:
					run = False;
			elif len( tmp + " " + l[ 0 ] ) > maxLength:
				run = False;
			else:
				tmp += ( " " + l[ 0 ] );
				l = l[ 1: ];
				if len( l ) == 0:
					run = False;

		output.append( tmp );
		tmp = "";
		if extra:
			# add room for double tab on second or higher lines
			ct = ct + 1;
			if ( ct == 1 ) and ( lineLength - 2 - ( numTabs + 1 ) * tabLength > 0 ):
				maxLength -= tabLength;

	return output;
# fed parseLongLines( line, maxLength )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def writeLongLines( fid, line, lineLength, tabLength, numTabs, extra ):
	lines = parseLongLines( line, lineLength, tabLength, numTabs, extra );
	if numTabs < 0:
		numTabs = 0;

	tabs = "";
	for i in range( 0, numTabs ):
		tabs += "\t";

	if not extra:
		for l in lines[ :-1 ]:
			fid.write( tabs + l + " \\\n" );
		fid.write( tabs + lines[ -1 ] + "\n" );
	else:
		fid.write( tabs + lines[0] + " \\\n" );
		tabs += "\t";
		for l in lines[ 1:-1 ]:
			fid.write( tabs + l + " \\\n" );
		fid.write( tabs + lines[ -1 ] + "\n" );
	return;
# fed parseLongLines( line, maxLength )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseEquals( s ):
	try:
		idx = s.find( "=" );
		if idx < 0:
			raise latexmake_invalidArgument( s );
		left = s[ :idx ];
		right = s[ idx+1: ];
	except latexmake_invalidArgument, e:
		raise latexmake_invalidArgument( e.args );
	else:
		return ( left, right );
# fed parseEquals( s )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findUnescaped( line, char ):
	idx = line.find( char );
	if idx > 0:
		if line[ idx - 1 ] == "\\":
			if idx < len( line ) - len( char ):
				# recursively look for the next char
				tmp = findUnescaped( line[ idx + len( char ): ], char );
				if tmp < 0:
					return tmp;
				else:
					return tmp + idx + len( char );
			else:
				return -1;
	# if idx == 0: cannot be escaped => return idx (=0)
	# if idx < 0: return idx (=-1)

	return idx;
# fed findNext( line, char )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def purifyListOfStrings( l, key ):
	return [ item for item in l if not re.search( key, item ) ];
# fed def purifyListOfStrings( l, key )	
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseDataInSquiglyBraces( line ):
	output = [];
	lstack = [];

	lidx = findUnescaped( line, "{" );
	ridx = findUnescaped( line, "}" );

	# no braces
	if lidx < 0:
		return [];

	run = True;
	while run:

		if len( lstack ) == 0:
			if lidx < 0:
				if ridx < 0:
					run = False;
				else:
					message = "In string: '" + line + "':\n";
					message += "\tunbalanced {}";
					raise latexmake_invalidBracketOrder( message );
			else:
				lstack.append( lidx );
				tmp = findUnescaped( line[ lidx+1:], "{" );
				if tmp < 0:
					lidx = -1;
				else:
					lidx += tmp + 1;
		elif lidx < ridx and lidx > 0:
			lstack.append( lidx );
			tmp = findUnescaped( line[ lidx+1:], "{" );
			if tmp < 0:
				lidx = -1;
			else:
				lidx += tmp + 1;
		elif ridx >= 0:
			if ( len( lstack  ) == 0 ):
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced {}";
				raise latexmake_invalidBracketOrder( message );
			elif lstack[-1] >= ridx:
				message = "In string: '" + line + "':\n";
				message += "\t'}' appears before '{'";
				raise latexmake_invalidBracketOrder( message );

			output.append( line[ lstack.pop() + 1:ridx ] );
			tmp = findUnescaped( line[ ridx+1:], "}" );
			if tmp < 0:
				ridx = -1;
			else:
				ridx += tmp + 1;
		else:
			if findUnescaped( line[ ridx+1:], "}" ) < 0:
				run = False;
			else:
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced {}";
				raise latexmake_invalidBracketOrder( message );

	return output;
# fed parseDataInSquiglyBraces( line )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseDataInSquareBraces( line ):
	output = [];
	lstack = [];

	lidx = findUnescaped( line, "[" );
	ridx = findUnescaped( line, "]" );

	# no braces
	if lidx < 0:
		return [];

	run = True;
	while run:

		if len( lstack ) == 0:
			if lidx < 0:
				if ridx < 0:
					run = False;
				else:
					message = "In string: '" + line + "':\n";
					message += "\tunbalanced []";
					raise latexmake_invalidBracketOrder( message );
			else:
				lstack.append( lidx );
				tmp = findUnescaped( line[ lidx+1:], "[" );
				if tmp < 0:
					lidx = -1;
				else:
					lidx += tmp + 1;
		elif lidx < ridx and lidx > 0:
			lstack.append( lidx );
			tmp = findUnescaped( line[ lidx+1:], "[" );
			if tmp < 0:
				lidx = -1;
			else:
				lidx += tmp + 1;
		elif ridx >= 0:
			if ( len( lstack  ) == 0 ):
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced []";
				raise latexmake_invalidBracketOrder( message );
			elif lstack[-1] >= ridx:
				message = "In string: '" + line + "':\n";
				message += "\t']' appears before '['";
				raise latexmake_invalidBracketOrder( message );

			output.append( line[ lstack.pop() + 1:ridx ] );
			tmp = findUnescaped( line[ ridx+1:], "]" );
			if tmp < 0:
				ridx = -1;
			else:
				ridx += tmp + 1;
		else:
			if findUnescaped( line[ ridx+1:], "]" ) < 0:
				run = False;
			else:
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced []";
				raise latexmake_invalidBracketOrder( message );

	return output;
# fed parseDataInSquareBraces( line )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseDataInParentheses( line ):
	output = [];
	lstack = [];

	lidx = findUnescaped( line, "(" );
	ridx = findUnescaped( line, ")" );

	# no braces
	if lidx < 0:
		return [];

	run = True;
	while run:

		if len( lstack ) == 0:
			if lidx < 0:
				if ridx < 0:
					run = False;
				else:
					message = "In string: '" + line + "':\n";
					message += "\tunbalanced ()";
					raise latexmake_invalidBracketOrder( message );
			else:
				lstack.append( lidx );
				tmp = findUnescaped( line[ lidx+1:], "(" );
				if tmp < 0:
					lidx = -1;
				else:
					lidx += tmp + 1;
		elif lidx < ridx and lidx > 0:
			lstack.append( lidx );
			tmp = findUnescaped( line[ lidx+1:], "(" );
			if tmp < 0:
				lidx = -1;
			else:
				lidx += tmp + 1;
		elif ridx >= 0:
			if ( len( lstack  ) == 0 ):
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced ()";
				raise latexmake_invalidBracketOrder( message );
			elif lstack[-1] >= ridx:
				message = "In string: '" + line + "':\n";
				message += "\t')' appears before '('";
				raise latexmake_invalidBracketOrder( message );

			output.append( line[ lstack.pop() + 1:ridx ] );
			tmp = findUnescaped( line[ ridx+1:], ")" );
			if tmp < 0:
				ridx = -1;
			else:
				ridx += tmp + 1;
		else:
			if findUnescaped( line[ ridx+1:], ")" ) < 0:
				run = False;
			else:
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced ()";
				raise latexmake_invalidBracketOrder( message );

	return output;
# fed parseDataInParenthesies( line )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def parseDataInAngleBraces( line ):
	lstack = [];

	lidx = findUnescaped( line, "<" );
	ridx = findUnescaped( line, ">" );

	# no braces
	if lidx < 0:
		return [];

	run = True;
	while run:

		if len( lstack ) == 0:
			if lidx < 0:
				if ridx < 0:
					run = False;
				else:
					message = "In string: '" + line + "':\n";
					message += "\tunbalanced <>";
					raise latexmake_invalidBracketOrder( message );
			else:
				lstack.append( lidx );
				tmp = findUnescaped( line[ lidx+1:], "<" );
				if tmp < 0:
					lidx = -1;
				else:
					lidx += tmp + 1;
		elif lidx < ridx and lidx > 0:
			lstack.append( lidx );
			tmp = findUnescaped( line[ lidx+1:], "<" );
			if tmp < 0:
				lidx = -1;
			else:
				lidx += tmp + 1;
		elif ridx >= 0:
			if ( len( lstack  ) == 0 ):
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced <>";
				raise latexmake_invalidBracketOrder( message );
			elif lstack[-1] >= ridx:
				message = "In string: '" + line + "':\n";
				message += "\t'>' appears before '<'";
				raise latexmake_invalidBracketOrder( message );

			output.append( line[ lstack.pop() + 1:ridx ] );
			tmp = findUnescaped( line[ ridx+1:], ">" );
			if tmp < 0:
				ridx = -1;
			else:
				ridx += tmp + 1;
		else:
			if findUnescaped( line[ ridx+1:], ">" ) < 0:
				run = False;
			else:
				message = "In string: '" + line + "':\n";
				message += "\tunbalanced <>";
				raise latexmake_invalidBracketOrder( message );

	return output;
# fed parseDataInAngleBraces( line )
#-------------------------------------------------------------------------------



#================================================================================
#
#		TeX file parsing
#
#================================================================================


#-------------------------------------------------------------------------------
def parseCommaSeparatedData( line ):
	tmp = line.split( "," );
	# trim whitespace
	return [ item.strip() for item in tmp ];
# fed parseCommaSeparatedData( line )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def removeTeXcomments( texFile, thisFileName ):
	texFileList = texFile.splitlines();
	output = "";
	for line in texFileList:

		# remove empty lines as well
		if len( line ) > 0:
			loc = line.find( "%" );
			if loc < 0:
				output += ( line + "\n" );
			elif loc > 0:
				run = True;
				tmp = "";
				while run:
					if len( line ) == 0:
						run = False;
					elif loc < 0:
						tmp += line;
						run = False;
					elif loc > 0:
						if line[loc-1] != "\\":
							# We have a real comment and not an escaped one
							tmp += line[:loc];
							run = False;
						else:
							# we have an escaped comment, this is valid
							tmp += line[:loc+1];
							line = line[loc+1:];
							loc = line.find( "%" )
							# run = True;
					else:
						# loc is 0
						run = False;
				if len( tmp ) > 0:
					output += ( tmp + "\n" );	
			else:
				# remove lines that are all comment
				# do nothing here
				pass;
	return output
# fed removeTeXcomments( texFile )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findPackages( texFile, params, thisFileName ):
	# TODO: This does not work in cases like
	# \usepackage[sorting=ydnt,		% reverse sort by date
	# bibstyle=authoryear,			% use author year style (not sure how much this does w/o /cite{})
	# defernumbers=true,			%
	# maxnames=99,			 		% show up to 99 author names in the reference
	# firstinits=true, 				% use first initials
	# terseinits=false,				% period after initials
	# uniquename=init,				%
	# dashed=false,					%
	# uniquename=false,				%
	# doi=true,						% include doi where present
	# isbn=false,					% include isbn where present
	# useprefix=true,				%
	# natbib=true,					%
	# backend=biber]				% use biber on the backend 
	# {biblatex}
	#
	# I probably need to abandon regular expressions
	# 
	# TODO: also have this work with \requirepackage

	key = r"\\usepackage(\[.*\])?(\{.*\})";
	locs = re.findall( key, texFile );
	packages = [];
	for tmp in locs:
		line = tmp[ -1 ];
		for part in purifyListOfStrings( parseDataInSquiglyBraces( line ), \
				r"[\{\}]" ):
			p = parseCommaSeparatedData( part );
			packages += p;
			for package in p:
				# check to see if it is local
				params = findLocalStyFiles( package, params, thisFileName );
				if package == "biblatex":
					# TODO finish
					m = re.search( r"\\usepackage(\[.*\]?\{\s*biblatex\s*\})", \
						texFile );
					optionStrings = parseDataInSquareBraces( m.group() );
					for optionString in optionStrings:
						options = parseCommaSeparatedData( optionString );
						opts = [ parseEquals( option ) for option in options ];
						backend = "bibtex"; # default
						for opt in opts:
							if opt[ 0 ] == "backend":
								backend = opt[ 1 ];
						if functionExists( backend ):
							params[ "bib_engine" ] = backend.upper();
						else:
							warning( "The bibliography backend \"" + backend + \
								"\" cannot be found" );

				elif package == "epstopdf":
					if params[ "tex_engine" ] == "PDFLATEX":
						params[ "fig_extensions" ] += ".eps";

				elif package == "makeidx":
					params[ "make_index_in_default" ] = True;

				# elif package == "glossaries":
				# 	params[ "make_glossary_in_default" ] = True;

	# update the packages list in params				
	params[ "packages" ] += packages;
	return params;
# fed findPackages( texFile, params )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findGraphicsPaths( texFile, params, thisFileName ):
	locs = re.findall( r"\\graphicspath\{(.*)\}", texFile );
	for line in locs:
		for part in purifyListOfStrings( parseDataInSquiglyBraces( line ), \
			r"[\{\}]" ):
			if os.path.isdir( part ) and os.path.exists( part ):
				# we are a valid path
				params[ "graphics_paths" ].append( os.path.abspath( part ) );
				if os.path.abspath( part ) not in params[ "sub_paths" ]:
					params[ "sub_paths" ].append( os.path.abspath( part ) );
			else:
				#TODO?: raise exception
				warning( "In \"" + thisFileName + "\": \"" + part + \
					"\" is not a valid graphicspath" );
	return params;
# fed findGraphicsPaths( texFile, params )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findGraphicsExtensions( texFile, params, thisFileName ):
	locs = re.findall( r"\\DeclareGraphicsExtensions\{(.*)\}", texFile );
	if not locs:
		return params;
	for item in locs:
		if parseDataInSquiglyBraces( item ):
			# residual squiglies
			item = parseDataInSquiglyBraces( item );
			params[ "fig_extensions" ] = parseCommaSeparatedData( item );
			#TODO: check to see if I can have a .jpg if the g.e. is just .eps
	return params;
# fed findGraphicsExtensions( texFile, params )
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def findFigures( texFile, params, thisFileName ):
	# TODO: also get pgf figures
	m = re.findall( r"\\includegraphics(\[.*\])?\s*(\{[\w\/ ]\})", texFile );
	if not m:
{'fraction_non_alphanumeric': 0.3415004635931428, 'fraction_numerical': 0.008009311317590894, 'mean_word_length': 1.5191074889430005, 'pattern_counts': {'":': 18, '<': 105, '<?xml version=': 0, '>': 67, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5163156', 'n_tokens_mistral': 16658, 'n_tokens_neox': 13963, 'n_words': 4184}