<commit_before>"""This is a simple _example_ of using picoparse to parse XML - not for real use. 

This example looks at XML as it is a textual stucture that the majority are familiar with. 
Comments throughout the file explain what is going on
"""
# Copyright (c) 2008, <NAME>, <NAME>
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# * Redistributions of source code must retain the above copyright notice, 
#   this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice, 
#   this list of conditions and the following disclaimer in the documentation  
#   and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
# POSSIBILITY OF SUCH DAMAGE.

from picoparse import one_of, many, many1, not_one_of, run_parser, tri, commit, optional, fail
from picoparse import choice, string, peek, cut, string, eof, many_until, any_token
from functools import partial

# some general functions to help build parsers; build string exists so we can compose it 
# with parser that return a series of characters or strings
def compose(f, g):
    return lambda *args, **kwargs: f(g(*args, **kwargs))

def build_string(iterable):
    return u''.join(iterable)

# We define common primative parsers by partial application. This is similar to the lexical 
# analysis stage of a more traditional parser tool
open_angle = partial(one_of, '<')
close_angle = partial(one_of, '>')
slash = partial(one_of, '/')
question = partial(one_of, '?')
equals = partial(one_of, '=')
quote = partial(one_of, "\"'")
whitespace_char = partial(one_of, ' \t\n\r')
whitespace = partial(many, whitespace_char)
whitespace1 = partial(many1, whitespace_char)
element_text = compose(build_string, partial(many1, partial(not_one_of, ' \t\r\n<>/=!')))
decimal_digit = partial(one_of, '0123456789')
hex_decimal_digit = partial(one_of, '0123456789AaBbCcDdEeFf')

# next we define the processing of an entity and then an xml_char, which are used later in node
# definitions. A more sophisticated parser would let additional entities be regestered with 
# directives
named_entities = {
    'amp':'&',
    'quot':'"',
    'apos':"'",
    'lt':'<',
    'gt':'>',
}

# entity is the toplevel parser for &...; notation, it chooses which sub parser to use
def entity():
    one_of('&')
    ent = choice(named_entity, numeric_entity)
    one_of(';')
    return ent

def named_entity():
    name = build_string(many1(partial(not_one_of,';#')))
    if name not in named_entities: fail()
    return named_entities[name]

def hex_entity():
    one_of('x')
    return chr(int(build_string(many1(hex_decimal_digit)), 16))
    
def dec_entity():
    return chr(int(build_string(many1(decimal_digit)), 10))

def numeric_entity():
    one_of('#')
    return choice(hex_entity, dec_entity)

# finally, xml_char is created by partially apply choice to either, normal characters or entities
xml_char = partial(choice, partial(not_one_of, '<>&'), entity)

# we want to be able to parse quoted strings, sometimes caring what the form of the content is
def quoted_parser(parser):
    quote_char = quote()
    value, _ = many_until(any_token, tri(partial(one_of, quote_char)))
    return build_string(value)
    
# now we are ready to start implementing the main document parsers.
# xml is our entrypoint parser. The XML spec requires a specific (but optional) prolog
# before the root element node, of which there may only be one
def xml():
    prolog()
    whitespace()
    n = element()
    whitespace()
    eof()
    return n

def prolog():
    whitespace()
    optional(tri(partial(processing, xmldecl)), None)
    many(partial(choice, processing, comment, whitespace1))
    optional(doctype, None)    
    many(partial(choice, processing, comment, whitespace1))

# The xml declaration could be parsed with a specialised processing directive, or it can be
# handled as a special case. We are going to handle it as a specialisation of the processing.
# our processing directive can optionally be given a parser to provide more detailed handling<commit_msg>tri is not needed here because one_of will fail without consuming so there is no need to backtrack<commit_after>"""This is a simple _example_ of using picoparse to parse XML - not for real use. 

This example looks at XML as it is a textual stucture that the majority are familiar with. 
Comments throughout the file explain what is going on
"""
# Copyright (c) 2008, <NAME>, <NAME>
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# * Redistributions of source code must retain the above copyright notice, 
#   this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice, 
#   this list of conditions and the following disclaimer in the documentation  
#   and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
# POSSIBILITY OF SUCH DAMAGE.

from picoparse import one_of, many, many1, not_one_of, run_parser, tri, commit, optional, fail
from picoparse import choice, string, peek, cut, string, eof, many_until, any_token
from functools import partial

# some general functions to help build parsers; build string exists so we can compose it 
# with parser that return a series of characters or strings
def compose(f, g):
    return lambda *args, **kwargs: f(g(*args, **kwargs))

def build_string(iterable):
    return u''.join(iterable)

# We define common primative parsers by partial application. This is similar to the lexical 
# analysis stage of a more traditional parser tool
open_angle = partial(one_of, '<')
close_angle = partial(one_of, '>')
slash = partial(one_of, '/')
question = partial(one_of, '?')
equals = partial(one_of, '=')
quote = partial(one_of, "\"'")
whitespace_char = partial(one_of, ' \t\n\r')
whitespace = partial(many, whitespace_char)
whitespace1 = partial(many1, whitespace_char)
element_text = compose(build_string, partial(many1, partial(not_one_of, ' \t\r\n<>/=!')))
decimal_digit = partial(one_of, '0123456789')
hex_decimal_digit = partial(one_of, '0123456789AaBbCcDdEeFf')

# next we define the processing of an entity and then an xml_char, which are used later in node
# definitions. A more sophisticated parser would let additional entities be regestered with 
# directives
named_entities = {
    'amp':'&',
    'quot':'"',
    'apos':"'",
    'lt':'<',
    'gt':'>',
}

# entity is the toplevel parser for &...; notation, it chooses which sub parser to use
def entity():
    one_of('&')
    ent = choice(named_entity, numeric_entity)
    one_of(';')
    return ent

def named_entity():
    name = build_string(many1(partial(not_one_of,';#')))
    if name not in named_entities: fail()
    return named_entities[name]

def hex_entity():
    one_of('x')
    return chr(int(build_string(many1(hex_decimal_digit)), 16))
    
def dec_entity():
    return chr(int(build_string(many1(decimal_digit)), 10))

def numeric_entity():
    one_of('#')
    return choice(hex_entity, dec_entity)

# finally, xml_char is created by partially apply choice to either, normal characters or entities
xml_char = partial(choice, partial(not_one_of, '<>&'), entity)

# we want to be able to parse quoted strings, sometimes caring what the form of the content is
def quoted_parser(parser):
    quote_char = quote()
    value, _ = many_until(any_token, partial(one_of, quote_char))
    return build_string(value)
    
# now we are ready to start implementing the main document parsers.
# xml is our entrypoint parser. The XML spec requires a specific (but optional) prolog
# before the root element node, of which there may only be one
def xml():
    prolog()
    whitespace()
    n = element()
    whitespace()
    eof()
    return n

def prolog():
    whitespace()
    optional(tri(partial(processing, xmldecl)), None)
    many(partial(choice, processing, comment, whitespace1))
    optional(doctype, None)    
    many(partial(choice, processing, comment, whitespace1))

# The xml declaration could be parsed with a specialised processing directive, or it can be
# handled as a special case. We are going to handle it as a specialisation of the processing.
# our processing directive can optionally be given a parser to provide more detailed handling
{'fraction_non_alphanumeric': 0.08798714343109683, 'fraction_numerical': 0.007432703897147449, 'mean_word_length': 4.600112485939258, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1582942', 'n_tokens_mistral': 2990, 'n_tokens_neox': 2808, 'n_words': 1206}