<commit_before>#!/usr/bin/env python
# encoding: utf-8
import os
import six
import numbers
import numpy as np
import pandas as pd
import re
import scipy.sparse as sp
from itertools import chain

from .model import get_convo_nn2
from .utils import create_n_gram_df, create_char_dict, pad_dict, CHARS_MAP, CHAR_TYPES_MAP
from .stop_words import THAI_STOP_WORDS

module_path = os.path.dirname(__file__)
weight_path = os.path.join(module_path, 'weight', 'cnn_without_ne_ab.h5')

# load model when importing library
model = get_convo_nn2()
model.load_weights(weight_path)

def tokenize(text):
    """
    Tokenize given Thai text string

    Input
    =====
    text: str, Thai text string

    Output
    ======
    tokens: list, list of tokenized words

    Example
    =======
    >> deepcut.tokenize('ตัดคำได้ดีมาก')
    >> ['ตัดคำ','ได้','ดี','มาก']

    """
    n_pad = 21
    n_pad_2 = int((n_pad - 1)/2)

    if len(text) == 0:
        return [''] # case of empty string

    if isinstance(text, str):
        text = text.decode('utf-8')

    char_dict = create_char_dict(text)
    char_dict_pad = pad_dict(char_dict, n_pad=n_pad)
    char_df = pd.DataFrame(char_dict_pad)
    char_df['char'] = char_df['char'].map(lambda x: CHARS_MAP.get(x, 80))
    char_df['type'] = char_df['type'].map(lambda x: CHAR_TYPES_MAP.get(x, 4))
    char_df_ngram = create_n_gram_df(char_df, n_pad=n_pad)

    char_row = ['char' + str(i + 1) for i in range(n_pad_2)] + \
               ['char-' + str(i + 1) for i in range(n_pad_2)] + ['char']
    type_row = ['type' + str(i + 1) for i in range(n_pad_2)] + \
               ['type-' + str(i + 1) for i in range(n_pad_2)] + ['type']

    x_char = char_df_ngram[char_row].as_matrix()
    x_type = char_df_ngram[type_row].as_matrix()

    y_predict = model.predict([x_char, x_type])
    y_predict = (y_predict.ravel() > 0.5).astype(int)
    word_end = list(y_predict[1:]) + [1]

    try:
        with open('custom_dict.txt') as f:
            word_list = f.readlines()
        for word in word_list:
            if isinstance(word, str):
                word = word.decode('utf-8')
            word = word.strip('\n')
            word_end = _custom_dict(word, text, word_end)
    except:
        pass

    tokens = []
    word = ''
    for char, w_e in zip(text, word_end):
        word += char
        if w_e:
            tokens.append(word)
            word = ''
    return tokens

def _custom_dict(word, text, word_end):
    word_length = len(word)
    initial_loc = 0

    while True:
        try:
            start_char = re.search(word, text).start()
            first_char = start_char+initial_loc
            last_char = first_char+word_length-1

            initial_loc += start_char+word_length
            text = text[start_char+word_length:]
            word_end[first_char:last_char] = (word_length-1)*[0]
            word_end[last_char] = 1
        except:
            break
    return word_end


def _document_frequency(X):
    """Count the number of non-zero values for each feature in sparse X."""
    if sp.isspmatrix_csr(X):
        return np.bincount(X.indices, minlength=X.shape[1])<commit_msg>Only decode to utf-8 in Python 2<commit_after>#!/usr/bin/env python
# encoding: utf-8
import os
import sys
import six
import numbers
import numpy as np
import pandas as pd
import re
import scipy.sparse as sp
from itertools import chain

from .model import get_convo_nn2
from .utils import create_n_gram_df, create_char_dict, pad_dict, CHARS_MAP, CHAR_TYPES_MAP
from .stop_words import THAI_STOP_WORDS

module_path = os.path.dirname(__file__)
weight_path = os.path.join(module_path, 'weight', 'cnn_without_ne_ab.h5')

# load model when importing library
model = get_convo_nn2()
model.load_weights(weight_path)

def tokenize(text):
    """
    Tokenize given Thai text string

    Input
    =====
    text: str, Thai text string

    Output
    ======
    tokens: list, list of tokenized words

    Example
    =======
    >> deepcut.tokenize('ตัดคำได้ดีมาก')
    >> ['ตัดคำ','ได้','ดี','มาก']

    """
    n_pad = 21
    n_pad_2 = int((n_pad - 1)/2)

    if len(text) == 0:
        return [''] # case of empty string

    if isinstance(text, str) and sys.version_info.major == 2:
        text = text.decode('utf-8')

    char_dict = create_char_dict(text)
    char_dict_pad = pad_dict(char_dict, n_pad=n_pad)
    char_df = pd.DataFrame(char_dict_pad)
    char_df['char'] = char_df['char'].map(lambda x: CHARS_MAP.get(x, 80))
    char_df['type'] = char_df['type'].map(lambda x: CHAR_TYPES_MAP.get(x, 4))
    char_df_ngram = create_n_gram_df(char_df, n_pad=n_pad)

    char_row = ['char' + str(i + 1) for i in range(n_pad_2)] + \
               ['char-' + str(i + 1) for i in range(n_pad_2)] + ['char']
    type_row = ['type' + str(i + 1) for i in range(n_pad_2)] + \
               ['type-' + str(i + 1) for i in range(n_pad_2)] + ['type']

    x_char = char_df_ngram[char_row].as_matrix()
    x_type = char_df_ngram[type_row].as_matrix()

    y_predict = model.predict([x_char, x_type])
    y_predict = (y_predict.ravel() > 0.5).astype(int)
    word_end = list(y_predict[1:]) + [1]

    try:
        with open('custom_dict.txt') as f:
            word_list = f.readlines()
        for word in word_list:
            if isinstance(word, str) and sys.version_info.major == 2:
                word = word.decode('utf-8')
            word = word.strip('\n')
            word_end = _custom_dict(word, text, word_end)
    except:
        pass

    tokens = []
    word = ''
    for char, w_e in zip(text, word_end):
        word += char
        if w_e:
            tokens.append(word)
            word = ''
    return tokens


def _custom_dict(word, text, word_end):
    word_length = len(word)
    initial_loc = 0

    while True:
        try:
            start_char = re.search(word, text).start()
            first_char = start_char+initial_loc
            last_char = first_char+word_length-1

            initial_loc += start_char+word_length
            text = text[start_char+word_length:]
            word_end[first_char:last_char] = (word_length-1)*[0]
            word_end[last_char] = 1
        except:
            break
    return word_end


def _document_frequency(X):
    """Count the number of non-zero values for each feature in sparse X."""
    if sp.isspmatrix_csr(X):
        return np.bincount(X.indices, minlength=X.shape[1])
{'fraction_non_alphanumeric': 0.12830957230142567, 'fraction_numerical': 0.010966630111232962, 'mean_word_length': 2.6880415944540728, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1258457', 'n_tokens_mistral': 2348, 'n_tokens_neox': 2257, 'n_words': 519}