<commit_before># -*- coding: utf-8 -*-
# Poio Tools for Linguists
#
# Copyright (C) 2009-2013 Poio Project
# Author: <NAME> <<EMAIL>>
# URL: <http://media.cidles.eu/poio/>
# For license information, see LICENSE.TXT

import sys, getopt

from poioapi import annotationtree, data
from poioapi.io.pickle import Parser

def main(argv):

    inputfile = ''
    outputfile = ''

    try:
        opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile="])
    except getopt.GetoptError:
        print('graf2pickle.py -i <inputfile> -o <outputfile>')
        sys.exit(2)

    for opt, arg in opts:
        if opt == '-h':
            print('graf2pickle.py -i <inputfile> -o <outputfile>')
            sys.exit()
        elif opt in ('-i', '--ifile'):
            inputfile = arg
        elif opt in ('-o', '--ofile'):
            outputfile = arg

    if inputfile == "" or outputfile == "":
        print('graf2pickle.py -i <inputfile> -o <outputfile>')
        sys.exit()

    # Define data structure hierarchy
    data_hierarchy = data.GRAID

    # Create the annotation tree with the created data structure
    annotation_tree = annotationtree.AnnotationTree(data_hierarchy)

    parser = Parser(annotation_tree, inputfile)

    annotation_tree = parser.load_as_tree()

    annotation_tree.save_tree_as_pickle(outputfile)
<commit_msg>Data hierarchy changed to class type.<commit_after># -*- coding: utf-8 -*-
# Poio Tools for Linguists
#
# Copyright (C) 2009-2013 Poio Project
# Author: <NAME> <<EMAIL>>
# URL: <http://media.cidles.eu/poio/>
# For license information, see LICENSE.TXT

import sys, getopt

from poioapi import annotationtree, data
from poioapi.io.pickle import Parser

def main(argv):

    inputfile = ''
    outputfile = ''

    try:
        opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile="])
    except getopt.GetoptError:
        print('graf2pickle.py -i <inputfile> -o <outputfile>')
        sys.exit(2)

    for opt, arg in opts:
        if opt == '-h':
            print('graf2pickle.py -i <inputfile> -o <outputfile>')
            sys.exit()
        elif opt in ('-i', '--ifile'):
            inputfile = arg
        elif opt in ('-o', '--ofile'):
            outputfile = arg

    if inputfile == "" or outputfile == "":
        print('graf2pickle.py -i <inputfile> -o <outputfile>')
        sys.exit()

    # Define data structure hierarchy
    data_hierarchy = data.DataStructureTypeGraid()

    # Create the annotation tree with the created data structure
    annotation_tree = annotationtree.AnnotationTree(data_hierarchy)

    parser = Parser(annotation_tree, inputfile)

    annotation_tree = parser.load_as_tree()

    annotation_tree.save_tree_as_pickle(outputfile)

{'fraction_non_alphanumeric': 0.13271719038817006, 'fraction_numerical': 0.009611829944547136, 'mean_word_length': 2.7952314165497896, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2323431', 'n_tokens_mistral': 909, 'n_tokens_neox': 866, 'n_words': 227}