<commit_before>import os
import subprocess
import tokenize

__author__ = 'vedran'

from pyparsing import Word, Literal, alphanums, OneOrMore, Optional, restOfLine, Group, NotAny


class ServerParser(object):
    def __init__(self):
        super(ServerParser, self).__init__()

        word = Word(alphanums + '-' + '_' + '.' + '/' + '$' + ':')
        server = Literal('server').suppress()
        location = Literal('location')
        lbrace = Literal('{').suppress()
        rbrace = Literal('}').suppress()

        config_line = NotAny(rbrace) + word + Group(OneOrMore(word)) + Literal(';').suppress()
        location_def = location + word + lbrace + Group(OneOrMore(Group(config_line))) + rbrace
        self.server_def = server + lbrace + OneOrMore(Group(location_def) | Group(config_line)) + rbrace

        comment = Literal('#') + Optional(restOfLine)
        self.server_def.ignore(comment)

    def parse(self, input_):
        parsed = self.server_def.parseString(input_)
        server_ = {}
        locations = []
        for part in parsed:
            k = part[0]<commit_msg>Minor code formatting changes<commit_after>import os
import subprocess
from pyparsing import Word, Literal, alphanums, OneOrMore, Optional, restOfLine, Group, NotAny

__author__ = 'vedran'


class ServerParser(object):
    def __init__(self):
        super(ServerParser, self).__init__()

        word = Word(alphanums + '-' + '_' + '.' + '/' + '$' + ':')
        server = Literal('server').suppress()
        location = Literal('location')
        lbrace = Literal('{').suppress()
        rbrace = Literal('}').suppress()

        config_line = NotAny(rbrace) + word + Group(OneOrMore(word)) + Literal(';').suppress()
        location_def = location + word + lbrace + Group(OneOrMore(Group(config_line))) + rbrace
        self.server_def = server + lbrace + OneOrMore(Group(location_def) | Group(config_line)) + rbrace

        comment = Literal('#') + Optional(restOfLine)
        self.server_def.ignore(comment)

    def parse(self, input_):
        parsed = self.server_def.parseString(input_)
        server_ = {}
        locations = []
        for part in parsed:
            k = part[0]
{'fraction_non_alphanumeric': 0.1417466849565615, 'fraction_numerical': 0.0009144947416552355, 'mean_word_length': 3.0593692022263452, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '758178', 'n_tokens_mistral': 678, 'n_tokens_neox': 660, 'n_words': 138}