<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Segmentation is the task of splitting a data sequence into chuncks which can
then be classified chunk by chunk.

In the case of handwritten formula recognition it might be ok to segment by
strokes. However, one should note that cursive handwriting might make it
necessary to split on the point level.

Segmenting in the order in which strokes were written is also problematic as
delayed strokes (e.g. for extending a fraction stroke which was too short)
might occur.

This module contains algorithms for segmentation.
"""

import logging
import json

import itertools
import numpy
import os

import pickle
import pkg_resources

import time

import scipy.sparse.csgraph

# hwrt modules
from . import utils
from .HandwrittenData import HandwrittenData
from . import features
from . import geometry
from . import partitions
from . import less_than


def main():
    logging.info("Get single classifier")
    single_clf = single_classifier()

    logging.info("Get stroke_segmented_classifier "
                 "(decided if two strokes are in one symbol)")
    logging.info("Start creation of training set")
    X, y, recordings = get_dataset()
    logging.info("Start training")
    nn = get_nn_classifier(X, y)
    stroke_segmented_classifier = lambda X: nn(X)[0][1]
    y_predicted = numpy.argmax(nn(X), axis=1)
    classification = [yi == yip for yi, yip in zip(y, y_predicted)]
    err = float(sum([not i for i in classification])) / len(classification)
    logging.info("Error: %0.2f (for %i training examples)", err, len(y))

    logging.info("Get single stroke classifier")
    single_stroke_clf = None  # single_symbol_stroke_classifier()

    logging.info("Start testing")<commit_msg>hwrt/segmentation.py: Fixed relative import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Segmentation is the task of splitting a data sequence into chuncks which can
then be classified chunk by chunk.

In the case of handwritten formula recognition it might be ok to segment by
strokes. However, one should note that cursive handwriting might make it
necessary to split on the point level.

Segmenting in the order in which strokes were written is also problematic as
delayed strokes (e.g. for extending a fraction stroke which was too short)
might occur.

This module contains algorithms for segmentation.
"""

import logging
import json

import itertools
import numpy
import os

import pickle
import pkg_resources

import time

import scipy.sparse.csgraph

# hwrt modules
from . import utils
from .HandwrittenData import HandwrittenData
from . import features
from . import geometry
from . import partitions
from .utils import less_than


def main():
    logging.info("Get single classifier")
    single_clf = single_classifier()

    logging.info("Get stroke_segmented_classifier "
                 "(decided if two strokes are in one symbol)")
    logging.info("Start creation of training set")
    X, y, recordings = get_dataset()
    logging.info("Start training")
    nn = get_nn_classifier(X, y)
    stroke_segmented_classifier = lambda X: nn(X)[0][1]
    y_predicted = numpy.argmax(nn(X), axis=1)
    classification = [yi == yip for yi, yip in zip(y, y_predicted)]
    err = float(sum([not i for i in classification])) / len(classification)
    logging.info("Error: %0.2f (for %i training examples)", err, len(y))

    logging.info("Get single stroke classifier")
    single_stroke_clf = None  # single_symbol_stroke_classifier()

    logging.info("Start testing")
{'fraction_non_alphanumeric': 0.08255846717385179, 'fraction_numerical': 0.0033812341504649195, 'mean_word_length': 4.071428571428571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5069825', 'n_tokens_mistral': 1033, 'n_tokens_neox': 991, 'n_words': 401}