<commit_before># illustrate NLP preprocessing
# convert str into vector number of fixed length for input
# then can further be embedded into a real vector
import pandas
import tflearn
import tensorflow
from tensorflow.contrib import learn

MAX_TOKEN_LEN = 200
MIN_FREQ = 0  # cum freq to be added to vocab
VOCAB_PATH = 'models/dbpedia.tfl.vocab'

dbpedia = learn.datasets.load_dataset('dbpedia')
# X = ['Abbott of Farnham E D Abbott Limited was a British coachbuilding business based in Farnham Surrey trading under that name from 1929. A major part of their output was under sub-contract to motor vehicle manufacturers. Their business closed in 1972.', ...]
# Y = [int labels]

# limiting size first
str_trainX = pandas.DataFrame(dbpedia.train.data)[1][:5600]
trainY = pandas.Series(dbpedia.train.target)[:5600]
str_testX = pandas.DataFrame(dbpedia.test.data)[1][:700]
testY = pandas.Series(dbpedia.test.target)[:700]


# --------------------------------------
# TFLearn vocab processor
encoder = tflearn.data_utils.VocabularyProcessor(
    MAX_TOKEN_LEN, min_frequency=MIN_FREQ)

trainX = encoder.fit_transform(str_trainX)
testX = encoder.fit_transform(str_testX)
encoder.save(VOCAB_PATH)

# encoder.restore(VOCAB_PATH)
# trainX = encoder.transform(str_trainX)
# testX = encoder.transform(str_testX)

org = encoder.reverse(list(trainX)[:1])
print(list(org))


# --------------------------------------
# sklearn
import sklearn
from sklearn import feature_extraction

encoder = feature_extraction.text.TfidfVectorizer()
trainX = encoder.fit_transform(str_trainX).todense()
testX = encoder.fit_transform(str_testX).todense()
# not really that vital to invert X
org = encoder.inverse_transform(testX[:1])
print(list(org))
<commit_msg>add dict retrieval eg in nlp preprocessing encoder<commit_after># illustrate NLP preprocessing
# convert str into vector number of fixed length for input
# then can further be embedded into a real vector
import pandas
import tflearn
import tensorflow
from tensorflow.contrib import learn

MAX_TOKEN_LEN = 200
MIN_FREQ = 0  # cum freq to be added to vocab
VOCAB_PATH = 'models/dbpedia.tfl.vocab'

dbpedia = learn.datasets.load_dataset('dbpedia')
# X = ['Abbott of Farnham E D Abbott Limited was a British coachbuilding business based in Farnham Surrey trading under that name from 1929. A major part of their output was under sub-contract to motor vehicle manufacturers. Their business closed in 1972.', ...]
# Y = [int labels]

# limiting size first
str_trainX = pandas.DataFrame(dbpedia.train.data)[1][:5600]
trainY = pandas.Series(dbpedia.train.target)[:5600]
str_testX = pandas.DataFrame(dbpedia.test.data)[1][:700]
testY = pandas.Series(dbpedia.test.target)[:700]


# --------------------------------------
# TFLearn vocab processor
encoder = tflearn.data_utils.VocabularyProcessor(
    MAX_TOKEN_LEN, min_frequency=MIN_FREQ)

trainX = encoder.fit_transform(str_trainX)
testX = encoder.fit_transform(str_testX)
encoder.save(VOCAB_PATH)
# the dictionary
print(encoder.vocabulary_._mapping)
# encoder.restore(VOCAB_PATH)
# trainX = encoder.transform(str_trainX)
# testX = encoder.transform(str_testX)

org = encoder.reverse(list(trainX)[:1])
print(list(org))


# --------------------------------------
# sklearn
import sklearn
from sklearn import feature_extraction

encoder = feature_extraction.text.TfidfVectorizer()
trainX = encoder.fit_transform(str_trainX).todense()
testX = encoder.fit_transform(str_testX).todense()
# not really that vital to invert X
org = encoder.inverse_transform(testX[:1])
print(list(org))

{'fraction_non_alphanumeric': 0.13537855333520968, 'fraction_numerical': 0.016887137630171686, 'mean_word_length': 4.510077519379845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2190834', 'n_tokens_mistral': 1195, 'n_tokens_neox': 1116, 'n_words': 302}