<commit_before># -*- coding: utf-8 -*-
#
# Poio Tools for Linguists
#
# Copyright (C) 2009-2013 Poio Project
# Author: <NAME> <<EMAIL>>
# URL: <http://media.cidles.eu/poio/>
# For license information, see LICENSE.TXT

from __future__ import unicode_literals

import sys
import unicodedata
import re
import codecs
import collections

import poioapi.io.graf
import poioapi.data

# Compile necessary regular expression
re_tier_marker = re.compile("^" + r"\\(\S+)(?=($|\s+))")
re_line_break = re.compile(r"(\r\n|\n|\r)+$")
re_word = re.compile(r"(?<=\s)(\S+)(?:\s|$)", re.UNICODE)

# Set the type of string
if sys.version_info[:2] >= (3, 0):
    string_type = str
else:
    string_type = basestring

class Parser(poioapi.io.graf.BaseParser):

    def __init__(self, input_stream, record_marker = 'ref',
        record_level_markers = ['ref', 'ft', 'nt', 'rf', 'rt', 'id', 'dt'],
        word_level_markers = ['tx'],
        morpheme_level_markers = ['mb'],
        tag_level_markers = ['ge', 'ps']):
        """Class's constructor.

        Parameters
        ----------
        input_stream : str or IO stream
            Path of the Toolbox TXT file.
        record_marker : str
            The marker that marks the start of a Toolbox record in the input
            file.

        """

        self._input_stream = None
        self.tier_hierarchy = None

        self.input_stream = input_stream
        self.record_marker = record_marker

        self.record_level_markers = record_level_markers
        self.word_level_markers = word_level_markers
        self.morpheme_level_markers = morpheme_level_markers
        self.tag_level_markers = tag_level_markers

        self.parse()

    def input_stream():
        doc = "The input_stream property."
        def fget(self):
            return self._input_stream
        def fset(self, value):
            if not hasattr(value, 'read'):
                self._input_stream = open(value, "rb")
            else:
                self._input_stream = value
        def fdel(self):
            del self._input_stream
        return locals()
    input_stream = property(**input_stream())

    def parse(self):
        """This method will parse the input file.

        """
        self._tiers = list()
        self._annotations_for_parent = collections.defaultdict(list)
        self._get_tiers()

        # create tier hierarchy
        if self.tier_hierarchy == None:
            new_tier_hierarchy = [ self.record_marker ]
            
            word_tiers = [
                t for t in self.word_level_markers if t in self._tiers]
            morpheme_tiers = [
                t for t in self.morpheme_level_markers if t in self._tiers]
            tag_tiers = [
                t for t in self.tag_level_markers if t in self._tiers]
            morpheme_tiers.append(tag_tiers)
            word_tiers.append(morpheme_tiers)
            new_tier_hierarchy.append(word_tiers)

            record_tiers = [t for t in self.record_level_markers \
                if t in self._tiers and t != self.record_marker]
            new_tier_hierarchy.append(record_tiers)

            self.tier_hierarchy = poioapi.data.DataStructureType(
                new_tier_hierarchy)
        
        self._build_annotations()

    def _get_tiers(self):
        # Go through lines in the input file
        for line in self.input_stream:
            # Hack: sometimes there are characters that we cannot decode
            line = line.decode("utf-8", 'ignore')
            match_tier_marker = re_tier_marker.search(line)
            if match_tier_marker:
                tier_marker = match_tier_marker.group(1)
            if tier_marker not in self._tiers:
                self._tiers.append(tier_marker)
        self.input_stream.seek(0)

    def _build_annotations(self):
        """
        Helper method to parse the input file and store intermediate information
        in attributes.

        """
 
        elements = dict()
        ids = dict()
        current_id = 0

        first_marker_found = False
       
        # Go through lines in the input file
        for line in self.input_stream:
            # Hack: sometimes there are characters that we cannot decode
            line = line.decode("utf-8", 'ignore')
            line = line.strip()
            if line == "":
                continue

            tier_marker = None
            match_tier_marker = re_tier_marker.search(line)
            if match_tier_marker:
                tier_marker = match_tier_marker.group(1)
            else:
                continue

            if not first_marker_found:
                if tier_marker != self.record_marker:
                    continue
                else:
                    first_marker_found = True

            if tier_marker == self.record_marker:
                if len(elements) > 0:
                    self._process_record(elements, ids)
                    elements = dict()
                    ids = dict()

            if tier_marker not in elements:
                elements[tier_marker] = dict()
            if tier_marker not in ids:
                ids[tier_marker] = dict()

            if tier_marker in self.word_level_markers or \
                    tier_marker in self.morpheme_level_markers or \
                    tier_marker in self.tag_level_markers:
                for j, match in enumerate(re_word.finditer(line)):
                    elements[tier_marker][match.start(1)] = match.group(1)
                    ids[tier_marker][match.start(1)] = "a{0}".format(current_id)
                    current_id += 1
            else:
                content = re_tier_marker.sub("", line)
                elements[tier_marker][len(match_tier_marker.group(0))] = content
                ids[tier_marker][len(match_tier_marker.group(0))] = \
                    "a{0}".format(current_id)
                current_id += 1

<commit_msg>Corrected alignment in Toolbox parser.<commit_after># -*- coding: utf-8 -*-
#
# Poio Tools for Linguists
#
# Copyright (C) 2009-2013 Poio Project
# Author: <NAME> <<EMAIL>>
# URL: <http://media.cidles.eu/poio/>
# For license information, see LICENSE.TXT

from __future__ import unicode_literals

import sys
import unicodedata
import re
import codecs
import collections

import poioapi.io.graf
import poioapi.data

# Compile necessary regular expression
re_tier_marker = re.compile("^" + r"\\(\S+)(?=($|\s+))")
re_line_break = re.compile(r"(\r\n|\n|\r)+$")
re_word = re.compile(r"(?<=\s)(\S+)(?:\s|$)", re.UNICODE)

# Set the type of string
if sys.version_info[:2] >= (3, 0):
    string_type = str
else:
    string_type = basestring

def char_len(string):
    """
    Method to calculate string length for Toolbox alignment.
    Based on Taras Zakharko's method.

    """
    length = 0
    
    # Go through Unicode characters in the string
    for character in string:
        # Get the length of the current character
        num_bytes = len(character.encode("utf-8"))
        
        # Get the unicode property for the current character
        category = unicodedata.category(character)
        
        # Do we want to skip invisibles?
        if category not in [ "Mn", "Mc","Me" ]:
            length += num_bytes

    return length

class Parser(poioapi.io.graf.BaseParser):

    def __init__(self, input_stream, record_marker = 'ref',
        record_level_markers = ['ref', 'ft', 'nt', 'rf', 'rt', 'id', 'dt'],
        word_level_markers = ['tx'],
        morpheme_level_markers = ['mb'],
        tag_level_markers = ['ge', 'ps']):
        """Class's constructor.

        Parameters
        ----------
        input_stream : str or IO stream
            Path of the Toolbox TXT file.
        record_marker : str
            The marker that marks the start of a Toolbox record in the input
            file.

        """

        self._input_stream = None
        self.tier_hierarchy = None

        self.input_stream = input_stream
        self.record_marker = record_marker

        self.record_level_markers = record_level_markers
        self.word_level_markers = word_level_markers
        self.morpheme_level_markers = morpheme_level_markers
        self.tag_level_markers = tag_level_markers

        self.parse()

    def input_stream():
        doc = "The input_stream property."
        def fget(self):
            return self._input_stream
        def fset(self, value):
            if not hasattr(value, 'read'):
                self._input_stream = open(value, "rb")
            else:
                self._input_stream = value
        def fdel(self):
            del self._input_stream
        return locals()
    input_stream = property(**input_stream())

    def parse(self):
        """This method will parse the input file.

        """
        self._tiers = list()
        self._annotations_for_parent = collections.defaultdict(list)
        self._get_tiers()

        # create tier hierarchy
        if self.tier_hierarchy == None:
            new_tier_hierarchy = [ self.record_marker ]
            
            word_tiers = [
                t for t in self.word_level_markers if t in self._tiers]
            morpheme_tiers = [
                t for t in self.morpheme_level_markers if t in self._tiers]
            tag_tiers = [
                t for t in self.tag_level_markers if t in self._tiers]
            morpheme_tiers.append(tag_tiers)
            word_tiers.append(morpheme_tiers)
            new_tier_hierarchy.append(word_tiers)

            record_tiers = [t for t in self.record_level_markers \
                if t in self._tiers and t != self.record_marker]
            new_tier_hierarchy.append(record_tiers)

            self.tier_hierarchy = poioapi.data.DataStructureType(
                new_tier_hierarchy)
        
        self._build_annotations()

    def _get_tiers(self):
        # Go through lines in the input file
        for line in self.input_stream:
            # Hack: sometimes there are characters that we cannot decode
            line = line.decode("utf-8", 'ignore')
            match_tier_marker = re_tier_marker.search(line)
            if match_tier_marker:
                tier_marker = match_tier_marker.group(1)
            if tier_marker not in self._tiers:
                self._tiers.append(tier_marker)
        self.input_stream.seek(0)

    def _build_annotations(self):
        """
        Helper method to parse the input file and store intermediate information
        in attributes.

        """
 
        elements = dict()
        ids = dict()
        current_id = 0

        first_marker_found = False
       
        # Go through lines in the input file
        for line in self.input_stream:
            # Hack: sometimes there are characters that we cannot decode
            line = line.decode("utf-8", 'ignore')
            line = line.strip()
            if line == "":
                continue

            tier_marker = None
            match_tier_marker = re_tier_marker.search(line)
            if match_tier_marker:
                tier_marker = match_tier_marker.group(1)
            else:
                continue

            if not first_marker_found:
                if tier_marker != self.record_marker:
                    continue
                else:
                    first_marker_found = True

            if tier_marker == self.record_marker:
                if len(elements) > 0:
                    self._process_record(elements, ids)
                    elements = dict()
                    ids = dict()

            if tier_marker not in elements:
                elements[tier_marker] = dict()
            if tier_marker not in ids:
                ids[tier_marker] = dict()

            if tier_marker in self.word_level_markers or \
                    tier_marker in self.morpheme_level_markers or \
                    tier_marker in self.tag_level_markers:
                for j, match in enumerate(re_word.finditer(line)):
                    pos = char_len(line[:match.start(1)])
                    elements[tier_marker][pos] = match.group(1)
                    ids[tier_marker][pos] = "a{0}".format(current_id)
                    current_id += 1
            else:
                content = re_tier_marker.sub("", line)
                elements[tier_marker][len(match_tier_marker.group(0))] = content
                ids[tier_marker][len(match_tier_marker.group(0))] = \
                    "a{0}".format(current_id)
                current_id += 1


{'fraction_non_alphanumeric': 0.0922842964623019, 'fraction_numerical': 0.004562189851128542, 'mean_word_length': 1.847538742023701, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1749237', 'n_tokens_mistral': 3759, 'n_tokens_neox': 3524, 'n_words': 887}