<commit_before># Copyright (C) 2007, 2008  <NAME>
#
# This file is part of pybtex.
#
# pybtex is free software; you can redistribute it and/or modify
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# pybtex is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pybtex; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
# USA

from pyparsing import (Suppress, QuotedString, Word,
        alphas, printables, Regex, lineno, line, col,
        Forward, ZeroOrMore, OneOrMore, Group,
        restOfLine, StringEnd, ParserElement, downcaseTokens)
from pybtex.bibtex.interpreter import (Integer, String, QuotedVar,
        Identifier, FunctionLiteral, BibTeXError)

#ParserElement.enablePackrat()

def process_int_literal(s, loc, toks):
    try:
        return Integer(int(toks[0][1:]))
    except ValueError:
        raise BibTeXError('%i:%i invalid integer literal\n%s' %
                (lineno(loc, s), col(loc, s), line(loc, s)))

def process_string_literal(toks):
    return String(toks[0])

def process_identifier(toks):
    name = toks[0].lower()
    if name[0] == "'":
        return QuotedVar(name[1:])
    else:
        return Identifier(name)

def process_function(toks):
    return FunctionLiteral(toks[0])

comment = '%' + restOfLine
lbrace = Suppress('{')
rbrace = Suppress('}')
intLiteral = Regex(r'#-?\d+').setParseAction(process_int_literal)
stringLiteral = QuotedString('"').setParseAction(process_string_literal)
restrictedPrintables = ''.join(c for c in printables if not c in '#%^&{}_~\\')
nonnums = ''.join(c for c in restrictedPrintables if not c.isdigit())
identifier = Word(nonnums, restrictedPrintables).setParseAction(process_identifier)
token = stringLiteral | intLiteral | identifier
tokenList = Forward()
tokenList.setParseAction(process_function)
tokenList << Group(lbrace + ZeroOrMore(token | tokenList) + rbrace)
commandName = Word(alphas).setParseAction(downcaseTokens)
arg = Group(lbrace + ZeroOrMore(token | tokenList) + rbrace)
command = commandName + ZeroOrMore(arg)
bstGrammar = OneOrMore(command) + StringEnd()

# sloooooow
# bstGrammar.ignore(comment)

# somewhat faster
def strip_comment(s):
    quotes = 0
    pos = 0
    for char in s:
        if char == '"':
            quotes += 1
        elif char == '%':
            if quotes % 2 == 0:
                break 
        pos += 1
    return s[:pos]

def parse_file(filename):
    l = []
    f = file(filename)
    s = ''.join(strip_comment(line) for line in f)
    return bstGrammar.parseString(s)


if __name__ == '__main__':
    import sys
    print parse_file(sys.argv[1])<commit_msg>changed some variable names to be more meaningful<commit_after># Copyright (C) 2007, 2008  <NAME>
#
# This file is part of pybtex.
#
# pybtex is free software; you can redistribute it and/or modify
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# pybtex is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pybtex; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
# USA

from pyparsing import (Suppress, QuotedString, Word,
        alphas, printables, Regex, lineno, line, col,
        Forward, ZeroOrMore, OneOrMore, Group,
        restOfLine, StringEnd, ParserElement, downcaseTokens)
from pybtex.bibtex.interpreter import (Integer, String, QuotedVar,
        Identifier, FunctionLiteral, BibTeXError)

#ParserElement.enablePackrat()

def process_int_literal(s, loc, toks):
    try:
        return Integer(int(toks[0][1:]))
    except ValueError:
        raise BibTeXError('%i:%i invalid integer literal\n%s' %
                (lineno(loc, s), col(loc, s), line(loc, s)))

def process_string_literal(toks):
    return String(toks[0])

def process_identifier(toks):
    name = toks[0].lower()
    if name[0] == "'":
        return QuotedVar(name[1:])
    else:
        return Identifier(name)

def process_function(toks):
    return FunctionLiteral(toks[0])

comment = '%' + restOfLine
lbrace = Suppress('{')
rbrace = Suppress('}')
intLiteral = Regex(r'#-?\d+').setParseAction(process_int_literal)
stringLiteral = QuotedString('"').setParseAction(process_string_literal)
restrictedPrintables = ''.join(c for c in printables if not c in '#%^&{}_~\\')
nonnums = ''.join(c for c in restrictedPrintables if not c.isdigit())
identifier = Word(nonnums, restrictedPrintables).setParseAction(process_identifier)
token = stringLiteral | intLiteral | identifier
tokenList = Forward()
tokenList.setParseAction(process_function)
tokenList << Group(lbrace + ZeroOrMore(token | tokenList) + rbrace)
commandName = Word(alphas).setParseAction(downcaseTokens)
arg = Group(lbrace + ZeroOrMore(token | tokenList) + rbrace)
command = commandName + ZeroOrMore(arg)
bstGrammar = OneOrMore(command) + StringEnd()

# sloooooow
# bstGrammar.ignore(comment)

# somewhat faster
def strip_comment(line):
    quotes = 0
    pos = 0
    for char in line:
        if char == '"':
            quotes += 1
        elif char == '%':
            if quotes % 2 == 0:
                break 
        pos += 1
    return line[:pos]

def parse_file(filename):
    bst_file = open(filename)
    bst = ''.join(strip_comment(line) for line in bst_file)
    return bstGrammar.parseString(bst)


if __name__ == '__main__':
    import sys
    print parse_file(sys.argv[1])
{'fraction_non_alphanumeric': 0.10952380952380952, 'fraction_numerical': 0.01116584564860427, 'mean_word_length': 3.9240097008892483, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3584095', 'n_tokens_mistral': 1944, 'n_tokens_neox': 1871, 'n_words': 597}