<commit_before># ----------------------------------------------------------------------------
# Copyright 2014-2016 Nervana Systems Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ----------------------------------------------------------------------------
"""
Defines text datatset preprocessing routines
"""
from future import standard_library
standard_library.install_aliases()  # triggers E402, hence noqa below
from builtins import map  # noqa
import numpy as np  # noqa
import re  # noqa
from neon.util.compat import pickle  # noqa


def clean_string(string):
    """
    Tokenization/string cleaning.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()


def pad_sentences(sentences, sentence_length=None, dtype=np.int32, pad_val=0.):
    lengths = [len(sent) for sent in sentences]

    nsamples = len(sentences)
    if sentence_length is None:
        sentence_length = np.max(lengths)

    X = (np.ones((nsamples, sentence_length)) * pad_val).astype(dtype=np.int32)
    for i, sent in enumerate(sentences):
        trunc = sent[-sentence_length:]
        X[i, -len(trunc):] = trunc
    return X


def pad_data(path, vocab_size=20000, sentence_length=100, oov=2,
             start=1, index_from=3, seed=113, test_split=0.2):
    f = open(path, 'rb')
    X, y = pickle.load(f)
    f.close()

    np.random.seed(seed)
    np.random.shuffle(X)
    np.random.seed(seed)
    np.random.shuffle(y)

    if start is not None:<commit_msg>shuffle lines no-op to avoid spurious protex match (#879)<commit_after># ----------------------------------------------------------------------------
# Copyright 2014-2016 Nervana Systems Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ----------------------------------------------------------------------------
"""
Defines text datatset preprocessing routines
"""
from future import standard_library
standard_library.install_aliases()  # triggers E402, hence noqa below
from builtins import map  # noqa
import numpy as np  # noqa
import re  # noqa
from neon.util.compat import pickle  # noqa


def clean_string(base):
    """
    Tokenization/string cleaning.
    Original from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    base = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", base)
    base = re.sub(r"\'re", " \'re", base)
    base = re.sub(r"\'d", " \'d", base)
    base = re.sub(r"\'ll", " \'ll", base)
    base = re.sub(r"\'s", " \'s", base)
    base = re.sub(r"\'ve", " \'ve", base)
    base = re.sub(r"n\'t", " n\'t", base)
    base = re.sub(r"!", " ! ", base)
    base = re.sub(r",", " , ", base)
    base = re.sub(r"\)", " \) ", base)
    base = re.sub(r"\(", " \( ", base)
    base = re.sub(r"\?", " \? ", base)
    base = re.sub(r"\s{2,}", " ", base)
    return base.strip().lower()


def pad_sentences(sentences, sentence_length=None, dtype=np.int32, pad_val=0.):
    lengths = [len(sent) for sent in sentences]

    nsamples = len(sentences)
    if sentence_length is None:
        sentence_length = np.max(lengths)

    X = (np.ones((nsamples, sentence_length)) * pad_val).astype(dtype=np.int32)
    for i, sent in enumerate(sentences):
        trunc = sent[-sentence_length:]
        X[i, -len(trunc):] = trunc
    return X


def pad_data(path, vocab_size=20000, sentence_length=100, oov=2,
             start=1, index_from=3, seed=113, test_split=0.2):
    f = open(path, 'rb')
    X, y = pickle.load(f)
    f.close()

    np.random.seed(seed)
    np.random.shuffle(X)
    np.random.seed(seed)
    np.random.shuffle(y)

    if start is not None:
{'fraction_non_alphanumeric': 0.20561106840891621, 'fraction_numerical': 0.015564950038431976, 'mean_word_length': 2.567511994516792, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6321442', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1666, 'n_words': 470}