<commit_before>""" The pySUMO parsing module. It contains functions to parse and serialize
kif-files. It also handles the Ontology's data structure and parses the mapping
of SUMO terms to WordNet.


This module contains:

- AbstractSyntaxTree: The in-memory representation of an Ontology.
- Ontology: Contains basic information about an Ontology.

"""

import re

from .logger import actionlog
from enum import Enum
from pickle import dumps

def tokenize_docstring(chars, f):
    n = 0
    ret = []

    while  chars.count('"')%2 != 0 :
        n += 1
        line = f.readline()
        line = line.strip()
        chars = "".join([chars, line])
    chars = chars.split('"')
    while len(chars) > 1:
        c = tokenize(chars.pop(0))
        ret.extend(c)
        ret.append("".join(['"',chars.pop(0),'"']))
    ret.extend(tokenize(chars.pop(0)))
    return (ret, n)


def tokenize(chars):
    return chars.replace('(', ' ( ').replace(')', ' ) ').split()

def cleanup(chars):
    if '"' in chars and ';' in chars and chars.find(";") > chars.find('"'):
        return chars
    chars = chars.split(";")
    chars = chars[0]
    chars = chars.strip()
    return chars

def atom(token):
    "Numbers become numbers; every other token is a symbol."
    try: return int(token)
    except ValueError:
        try: return float(token)
        except ValueError:
            return Symbol(token)

def kifparse(infile, ontology, ast=None):
    """ Parse an ontology and return an AbstractSyntaxTree.

    Args:

    - ontology: the ontology to parse
    - graph: a modified graph of this ontology
    - ast: the AST of ontologies which are needed from this ontology
    -infile: the file object to parse

    Returns:

    - AbstractSyntaxTree

    """
    root = AbstractSyntaxTree(ontology)
    oldline = None
    linenumber = -1
    for i, line in enumerate(infile):
        line = cleanup(line)
        if line == "":
            continue
        if '"' in line:
            line, n = tokenize_docstring(line, infile)
            i += n
        else:
            line = tokenize(line)
        if oldline != None:
            line = oldline + line
            oldline = None
        if line[0] != '(':
            raise ParseError(" ".join(line),  i+1)
        if line.count('(') != line.count(')'):
            if linenumber == -1:
                linenumber = i
            oldline = line
            continue
        if linenumber == -1:
            linenumber = i
        while line != [] and line.count('(') == line.count(')'):
            node = AbstractSyntaxTree(ontology, line=linenumber)
            parsed = node.parse(line)<commit_msg>make some helper methodes private<commit_after>""" The pySUMO parsing module. It contains functions to parse and serialize
kif-files. It also handles the Ontology's data structure and parses the mapping
of SUMO terms to WordNet.


This module contains:

- AbstractSyntaxTree: The in-memory representation of an Ontology.
- Ontology: Contains basic information about an Ontology.

"""

import re

from .logger import actionlog
from enum import Enum
from pickle import dumps

def _tokenize_docstring(chars, f):
    n = 0
    ret = []

    while  chars.count('"')%2 != 0 :
        n += 1
        line = f.readline()
        line = line.strip()
        chars = "".join([chars, line])
    chars = chars.split('"')
    while len(chars) > 1:
        c = _tokenize(chars.pop(0))
        ret.extend(c)
        ret.append("".join(['"',chars.pop(0),'"']))
    ret.extend(_tokenize(chars.pop(0)))
    return (ret, n)


def _tokenize(chars):
    return chars.replace('(', ' ( ').replace(')', ' ) ').split()

def _cleanup(chars):
    if '"' in chars and ';' in chars and chars.find(";") > chars.find('"'):
        return chars
    chars = chars.split(";")
    chars = chars[0]
    chars = chars.strip()
    return chars

def kifparse(infile, ontology, ast=None):
    """ Parse an ontology and return an AbstractSyntaxTree.

    Args:

    - ontology: the ontology to parse
    - graph: a modified graph of this ontology
    - ast: the AST of ontologies which are needed from this ontology
    -infile: the file object to parse

    Returns:

    - AbstractSyntaxTree

    """
    root = AbstractSyntaxTree(ontology)
    oldline = None
    linenumber = -1
    for i, line in enumerate(infile):
        line = _cleanup(line)
        if line == "":
            continue
        if '"' in line:
            line, n = _tokenize_docstring(line, infile)
            i += n
        else:
            line = _tokenize(line)
        if oldline != None:
            line = oldline + line
            oldline = None
        if line[0] != '(':
            raise ParseError(" ".join(line),  i+1)
        if line.count('(') != line.count(')'):
            if linenumber == -1:
                linenumber = i
            oldline = line
            continue
        if linenumber == -1:
            linenumber = i
        while line != [] and line.count('(') == line.count(')'):
            node = AbstractSyntaxTree(ontology, line=linenumber)
            parsed = node.parse(line)
{'fraction_non_alphanumeric': 0.12465645857872007, 'fraction_numerical': 0.005496662740478995, 'mean_word_length': 2.2003768844221105, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1417928', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1499, 'n_words': 484}