<commit_before>#! /usr/bin/env python
"""
Script to produce a list of reference keys from a .bbl file created by bibtex.

The output is useful in combination with bibsearch.py.
(Pass the output to bibsearch.py to create a
custom database for a particular latex document. This avoids the
necessity of sending a huge bibtex database along with a manuscript
when submitting to a journal.)

:author: <NAME>
:contact: http://www.schwilk.org
:author: <NAME> (small changes)
:copyright: 2006 by <NAME>
:license: MIT (see `license.txt`_)
:date: $Date: 2007-09-03 $

.. _license.txt: ./license.txt
"""
__docformat__ = "restructuredtext en"
__authors__  =    ["<NAME>", "<NAME>"]
__version__ = "1.5.3"
__needs__ = '2.4'


###################  IMPORTS  ##################################################
#import from standard library
import sys
import logging

#configure logger
logging.basicConfig(format='\n%(levelname)s:\n%(message)s\n')
reflist_logger = logging.getLogger('bibstuff_logger')
################################################################################


def main():
	"""Command-line tool"""
        
	from optparse import OptionParser
	
	usage = """usage: %prog FILE
	
	For example, to create a stripped down database
	for a particular latex document:
	python %prog FILE.bbl | python bibsearch.py DB.bib -l > NEW_DB.bib
	"""

	parser = OptionParser(usage=usage, version ="%prog " + __version__)
	parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
					  help="Print INFO messages to stdout, default=%default")
	parser.add_option("-V", "--very_verbose", action="store_true", dest="very_verbose", default=False,
					  help="Print DEBUG messages to stdout, default=%default")
	   
	(options, args) = parser.parse_args()
	if options.verbose:
		reflist_logger.setLevel(logging.INFO)
	if options.very_verbose:
		reflist_logger.setLevel(logging.DEBUG)
	reflist_logger.info("Script running.\nargs=%s"%(args))

	try :
		src = open(args[0]).read()
	except :
		src = sys.stdin.read()

	items = src.split('\n\n')
	for i in items :
		i = i.strip()
		if (i[:8] == '\\bibitem') :
			s = i.find(']')
			e = i.find('}', s)
			print i[s+2:e]


if __name__ == '__main__':
        main()
<commit_msg>handle nested braces; handle harvard style; add function 'get_braces_contents'<commit_after>#! /usr/bin/env python
"""
Script to produce a list of reference keys from a .bbl file created by bibtex.

The output is useful in combination with bibsearch.py.
(Pass the output to bibsearch.py to create a
custom database for a particular latex document. This avoids the
necessity of sending a huge bibtex database along with a manuscript
when submitting to a journal.)

:author: <NAME>
:contact: http://www.schwilk.org
:author: <NAME> (small changes)
:copyright: 2006 by <NAME>
:license: MIT (see `license.txt`_)
:date: $Date: 2007-09-03 $

.. _license.txt: ./license.txt
"""
__docformat__ = "restructuredtext en"
__authors__  =    ["<NAME>", "<NAME>"]
__version__ = "1.5.3"
__needs__ = '2.4'


###################  IMPORTS  ##################################################
#import from standard library
import sys
import logging

#configure logger
logging.basicConfig(format='\n%(levelname)s:\n%(message)s\n')
reflist_logger = logging.getLogger('bibstuff_logger')
################################################################################

def get_braces_contents(s):
	result = list()
	word = list()
	lvl = 0
	for c in s:
		if c == '}': lvl -= 1
		if lvl > 0:
			word.append(c)
		if c == '{': lvl += 1
		if lvl==0 and word:
			result.append(''.join(word))
			word = list()
	return result

def main():
	"""Command-line tool"""
        
	from optparse import OptionParser
	
	usage = """usage: %prog FILE
	
	For example, to create a stripped down database
	for a particular latex document:
	python %prog FILE.bbl | python bibsearch.py DB.bib -l > NEW_DB.bib
	"""

	parser = OptionParser(usage=usage, version ="%prog " + __version__)
	parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
					  help="Print INFO messages to stdout, default=%default")
	parser.add_option("-V", "--very_verbose", action="store_true", dest="very_verbose", default=False,
					  help="Print DEBUG messages to stdout, default=%default")
	   
	(options, args) = parser.parse_args()
	if options.verbose:
		reflist_logger.setLevel(logging.INFO)
	if options.very_verbose:
		reflist_logger.setLevel(logging.DEBUG)
	reflist_logger.info("Script running.\nargs=%s"%(args))

	try :
		src = open(args[0]).read()
		reflist_logger.debug('successfully read %s'%args[0])
	except :
		src = sys.stdin.read()

	items = src.split('\n\n')
	for i in items :
		i = i.strip()
		if (i.startswith('\\bibitem')) :
			i = i[8:].strip()
			if i[0] == '[':
				i = i[i.find(']') + 1:]
			print(get_braces_contents(i)[0])
		elif (i.startswith('\\harvarditem')) :
			print(get_braces_contents(i)[2])


if __name__ == '__main__':
        main()

{'fraction_non_alphanumeric': 0.20044363783020772, 'fraction_numerical': 0.009881024400080662, 'mean_word_length': 4.486725663716814, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1584916', 'n_tokens_mistral': 1748, 'n_tokens_neox': 1596, 'n_words': 455}