<commit_before>import ply.yacc as yacc
from lexer import tokens, lexer
from collections import namedtuple
import logging


logging.basicConfig()
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

Mailbox = namedtuple('Mailbox', ['display_name', 'local_part', 'domain'])
Url     = namedtuple('Url',     ['address'])


# Parsing rules

start = 'mailbox_or_url_list'

def p_expression_mailbox_or_url_list(p):
    '''mailbox_or_url_list : mailbox_or_url_list delim mailbox_or_url
                           | mailbox_or_url_list delim
                           | mailbox_or_url'''
    if len(p) == 4:
        p[0] = p[1] + [p[3]]
    elif len(p) == 3:
        p[0] = p[1]
    elif len(p) == 2:
        p[0] = [p[1]]

def p_expression_delim(p):
    '''delim : delim delim
             | ofwsp COMMA ofwsp
             | ofwsp SEMICOLON ofwsp'''

def p_expression_mailbox_or_url(p):
    '''mailbox_or_url : mailbox
                      | url'''
    p[0] = p[1]

def p_expression_url(p):
    'url : URL'
    p[0] = Url(p[1])

def p_expression_mailbox(p):
    '''mailbox : addr_spec
               | name_addr'''
    p[0] = p[1]

def p_expression_name_addr(p):
    'name_addr : phrase angle_addr'
    p[0] = Mailbox(' '.join(p[1].split()), p[2].local_part, p[2].domain)

def p_expression_angle_addr(p):
    'angle_addr : LANGLE ofwsp addr_spec ofwsp RANGLE'
    p[0] = Mailbox('', p[3].local_part, p[3].domain)

def p_expression_addr_spec(p):
    '''addr_spec : DOT_ATOM      AT DOT_ATOM
                 | DOT_ATOM      AT ATOM
                 | DOT_ATOM      AT domain_literal
                 | ATOM          AT DOT_ATOM
                 | ATOM          AT ATOM
                 | ATOM          AT domain_literal
                 | quoted_string AT DOT_ATOM
                 | quoted_string AT ATOM
                 | quoted_string AT domain_literal
    '''
    p[0] = Mailbox('', p[1], p[3])

def p_expression_quoted_string(p):
    '''quoted_string : DQUOTE quoted_string_text DQUOTE'''
    p[0] = '"{}"'.format(p[2])

def p_expression_quoted_string_text(p):
    '''quoted_string_text : quoted_string_text QTEXT
                          | quoted_string_text QPAIR
                          | quoted_string_text fwsp
                          | QTEXT
                          | QPAIR
                          | fwsp
                          |''' # NOTE: `empty` is invalid but has been added for backwards compatability
    if len(p) == 3:
        p[0] = '{}{}'.format(p[1], p[2])
    elif len(p) == 2:
        p[0] = p[1]
    elif len(p) == 1:
        p[0] = ''

def p_expression_domain_literal(p):
    '''domain_literal : LBRACKET domain_literal_text RBRACKET'''
    p[0] = '[{}]'.format(p[2])

def p_expression_domain_literal_text(p):
    '''domain_literal_text : domain_literal_text DTEXT
                           | domain_literal_text fwsp
                           | DTEXT
                           | fwsp'''
    if len(p) == 3:
        p[0] = '{}{}'.format(p[1], p[2])
    elif len(p) == 2:
        p[0] = p[1]

def p_expression_phrase(p):
    '''phrase : phrase ATOM
              | phrase DOT_ATOM
              | phrase DOT
              | phrase quoted_string
              | phrase fwsp
              | ATOM
              | DOT_ATOM
              | DOT
              | quoted_string'''
    if len(p) == 3:
        p[0] = '{}{}'.format(p[1], p[2])
    elif len(p) == 2:
        p[0] = p[1]

def p_expression_ofwsp(p):<commit_msg>Preserve quoted whitespace in display names<commit_after>import ply.yacc as yacc
from lexer import tokens, lexer
from collections import namedtuple
import logging


logging.basicConfig()
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

Mailbox = namedtuple('Mailbox', ['display_name', 'local_part', 'domain'])
Url     = namedtuple('Url',     ['address'])


# Parsing rules

start = 'mailbox_or_url_list'

def p_expression_mailbox_or_url_list(p):
    '''mailbox_or_url_list : mailbox_or_url_list mailbox_or_url
                           | mailbox_or_url_list COMMA
                           | mailbox_or_url_list SEMICOLON
                           | mailbox_or_url_list fwsp
                           | mailbox_or_url'''
    if len(p) == 3 and isinstance(p[2], basestring):
        p[0] = p[1]
    elif len(p) == 3:
        p[0] = p[1] + [p[2]]
    elif len(p) == 2:
        p[0] = [p[1]]

def p_expression_mailbox_or_url(p):
    '''mailbox_or_url : mailbox
                      | url'''
    p[0] = p[1]

def p_expression_url(p):
    'url : URL'
    p[0] = Url(p[1])

def p_expression_mailbox(p):
    '''mailbox : addr_spec
               | name_addr'''
    p[0] = p[1]

def p_expression_name_addr(p):
    'name_addr : phrase ofwsp angle_addr'
    p[0] = Mailbox(p[1], p[3].local_part, p[3].domain)

def p_expression_angle_addr(p):
    'angle_addr : LANGLE ofwsp addr_spec ofwsp RANGLE'
    p[0] = Mailbox('', p[3].local_part, p[3].domain)

def p_expression_addr_spec(p):
    'addr_spec : local_part AT domain'
    p[0] = Mailbox('', p[1], p[3])

def p_expression_local_part(p):
    '''local_part : DOT_ATOM
                  | ATOM
                  | quoted_string'''
    p[0] = p[1]

def p_expression_domain(p):
    '''domain : DOT_ATOM
              | ATOM
              | domain_literal'''
    p[0] = p[1]

def p_expression_quoted_string(p):
    '''quoted_string : DQUOTE quoted_string_text DQUOTE'''
    p[0] = '"{}"'.format(p[2])

def p_expression_quoted_string_text(p):
    '''quoted_string_text : quoted_string_text QTEXT
                          | quoted_string_text QPAIR
                          | quoted_string_text fwsp
                          | QTEXT
                          | QPAIR
                          | fwsp
                          |''' # NOTE: `empty` is invalid but has been added for backwards compatability
    if len(p) == 3:
        p[0] = '{}{}'.format(p[1], p[2])
    elif len(p) == 2:
        p[0] = p[1]
    elif len(p) == 1:
        p[0] = ''

def p_expression_domain_literal(p):
    '''domain_literal : LBRACKET domain_literal_text RBRACKET'''
    p[0] = '[{}]'.format(p[2])

def p_expression_domain_literal_text(p):
    '''domain_literal_text : domain_literal_text DTEXT
                           | domain_literal_text fwsp
                           | DTEXT
                           | fwsp'''
    if len(p) == 3:
        p[0] = '{}{}'.format(p[1], p[2])
    elif len(p) == 2:
        p[0] = p[1]

def p_expression_phrase(p):
    '''phrase : phrase fwsp ATOM
              | phrase fwsp DOT_ATOM
              | phrase fwsp DOT
              | phrase fwsp quoted_string
              | phrase ATOM
              | phrase DOT_ATOM
              | phrase DOT
              | phrase quoted_string
              | ATOM
              | DOT_ATOM
              | DOT
              | quoted_string'''
    if len(p) == 4:
        p[0] = '{} {}'.format(p[1], p[3])
    if len(p) == 3:
        p[0] = '{}{}'.format(p[1], p[2])
    elif len(p) == 2:
        p[0] = p[1]

def p_expression_ofwsp(p):
{'fraction_non_alphanumeric': 0.13072641644070215, 'fraction_numerical': 0.016412159269302125, 'mean_word_length': 1.8780287474332649, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6316754', 'n_tokens_mistral': 2495, 'n_tokens_neox': 2425, 'n_words': 501}