<commit_before>"""Build a sentence tokenizer for a language. Latin below.
Some guidance available here: http://wiki.apertium.org/wiki/Sentence_segmenting
"""

__author__ = '<NAME> <<EMAIL>>'
__license__ = 'MIT License. See LICENSE.'

import pickle
from nltk.tokenize.punkt import PunktLanguageVars
from nltk.tokenize.punkt import PunktSentenceTokenizer
import os

CLTK_DATA_DIR_REL = '~/cltk_data'
CLTK_DATA_DIR_ABS = os.path.expanduser(CLTK_DATA_DIR_REL)
GREEK_SENTENCE_TOKENIZER_DIR = os.path.join(CLTK_DATA_DIR_ABS,
                                            'compiled/sentence_tokens_greek')
FILE_NAME = 'greek.pickle'
PICKLE_PATH = os.path.join(GREEK_SENTENCE_TOKENIZER_DIR, FILE_NAME)


def tokenize_sentences(input_string):
    """Tokenize incoming Greek strings and output a list.
    :param input_string: str
    :rtype : list
    """
    with open(PICKLE_PATH, 'rb') as open_pickle:
        training_set = pickle.load(open_pickle)

    language_punkt_vars = PunktLanguageVars
    language_punkt_vars.sent_end_chars = ('.', ';',)
    language_punkt_vars.internal_punctuation = (',', '·')
    training_set.INCLUDE_ALL_COLLOCS = True<commit_msg>re-add greek.pickle in ~/cltk_data<commit_after>"""Build a sentence tokenizer for a language. Latin below.
Some guidance available here: http://wiki.apertium.org/wiki/Sentence_segmenting
"""

__author__ = '<NAME> <<EMAIL>>'
__license__ = 'MIT License. See LICENSE.'

import pickle
from nltk.tokenize.punkt import PunktLanguageVars
from nltk.tokenize.punkt import PunktSentenceTokenizer
import os


CLTK_DATA_DIR_REL = '~/cltk_data'
CLTK_DATA_DIR_ABS = os.path.expanduser(CLTK_DATA_DIR_REL)
GREEK_SENTENCE_TOKENIZER_DIR = os.path.join(CLTK_DATA_DIR_ABS,
                                            'compiled/sentence_tokens_greek')


def tokenize_sentences(input_string):
    """Tokenize incoming Greek strings and output a list.
    :param input_string: str
    :rtype : list
    """
    with open('greek.pickle', 'rb') as open_pickle:
        training_set = pickle.load(open_pickle)

    language_punkt_vars = PunktLanguageVars
    language_punkt_vars.sent_end_chars = ('.', ';',)
    language_punkt_vars.internal_punctuation = (',', '·')
    training_set.INCLUDE_ALL_COLLOCS = True
{'fraction_non_alphanumeric': 0.10241502683363149, 'fraction_numerical': 0.0, 'mean_word_length': 4.214452214452215, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4614065', 'n_tokens_mistral': 787, 'n_tokens_neox': 768, 'n_words': 147}