<commit_before>'''Example 


'''

from __future__ import print_function
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras.layers import LSTM
from keras.optimizers import Nadam # SGD #Adam #RMSprop
from keras.utils.data_utils import get_file
import numpy as np
import random
import sys
import os
import time

framelen=16
epoch_time = int(time.time())

fname="test/LDC97S44-8k.c2cb"
odir="out/LDC97S44/"+str(epoch_time)+"/"
os.makedirs(odir)
ofname=odir+"out-c2cb-"
modelf=odir+"model-"
#text = open(fname).read()
text = np.fromfile(fname, dtype=np.uint8)
print('corpus length:', len(text))


len_text = len(text)
#len_text = 16000

'''
bytes=[]
for i,t in enumerate(text):
    bytes.append(ord(t))
text = bytes
'''

num_frames = int(len_text / framelen)
#chars = sorted(list(set(text)))
#print('total chars:', len(chars))
#char_indices = dict((c, i) for i, c in enumerate(chars))
#indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 100
step = 3
sentences = []
next_chars = []
print("scanning byte sequence")

frames = []

for j in range(0, num_frames):
    i = j*framelen
    frames.append(text[i: i+framelen])
print('nb frames:', len(frames), len_text/framelen)
print("scanning frames of length ", len(frames[0]))
for i in range(0, num_frames-maxlen, step):
    # print("getting set from:", i)
    sentences.append(frames[i: i + maxlen])
    next_chars.append(frames[i + maxlen])
print("got sets")
print('nb sequences in bytes:', len(sentences))


print('Vectorization...')
X = np.zeros((len(sentences), maxlen, framelen), dtype=np.uint8)
y = np.zeros((len(sentences), framelen), dtype=np.uint8)
for i, sentence in enumerate(sentences):
    for t, frame in enumerate(sentence):
        #X[i, t, char_indices[char]] = 1
        #for p, prop in enumerate(frame):
         #   X[i, t, p] = (prop)
        X[i,t] = frame
    #y[i, char_indices[next_chars[i]]] = 1
    #for n in range(0, framelen):
    #  y[i, n] = (next_chars[i][n])
    y[i] = next_chars[i]


# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, framelen), return_sequences=True))
model.add(LSTM(128, return_sequences=True))
model.add(LSTM(128))
model.add(Dense(framelen))
model.add(Dense(framelen))
model.add(Dense(framelen))
model.add(Dense(framelen))
#model.add(Dropout(0.2))
#model.add(Dense(framelen))
#model.add(Activation('softmax'))

optimizer = Nadam() #SGD() #Adam() #RMSprop(lr=0.01)
model.compile(loss='mean_absolute_error', optimizer=optimizer)<commit_msg>1 large lstm<commit_after>'''Example 


'''

from __future__ import print_function
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras.layers import LSTM
from keras.optimizers import Nadam # SGD #Adam #RMSprop
from keras.utils.data_utils import get_file
import numpy as np
import random
import sys
import os
import time

framelen=16
epoch_time = int(time.time())

fname="test/LDC97S44-8k.c2cb"
odir="out/LDC97S44/"+str(epoch_time)+"/"
os.makedirs(odir)
ofname=odir+"out-c2cb-"
modelf=odir+"model-"
#text = open(fname).read()
text = np.fromfile(fname, dtype=np.uint8)
print('corpus length:', len(text))


len_text = len(text)
#len_text = 16000

'''
bytes=[]
for i,t in enumerate(text):
    bytes.append(ord(t))
text = bytes
'''

num_frames = int(len_text / framelen)
#chars = sorted(list(set(text)))
#print('total chars:', len(chars))
#char_indices = dict((c, i) for i, c in enumerate(chars))
#indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 100
step = 3
sentences = []
next_chars = []
print("scanning byte sequence")

frames = []

for j in range(0, num_frames):
    i = j*framelen
    frames.append(text[i: i+framelen])
print('nb frames:', len(frames), len_text/framelen)
print("scanning frames of length ", len(frames[0]))
for i in range(0, num_frames-maxlen, step):
    # print("getting set from:", i)
    sentences.append(frames[i: i + maxlen])
    next_chars.append(frames[i + maxlen])
print("got sets")
print('nb sequences in bytes:', len(sentences))


print('Vectorization...')
X = np.zeros((len(sentences), maxlen, framelen), dtype=np.uint8)
y = np.zeros((len(sentences), framelen), dtype=np.uint8)
for i, sentence in enumerate(sentences):
    for t, frame in enumerate(sentence):
        #X[i, t, char_indices[char]] = 1
        #for p, prop in enumerate(frame):
         #   X[i, t, p] = (prop)
        X[i,t] = frame
    #y[i, char_indices[next_chars[i]]] = 1
    #for n in range(0, framelen):
    #  y[i, n] = (next_chars[i][n])
    y[i] = next_chars[i]


# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(384, input_shape=(maxlen, framelen))) #, return_sequences=True))
#model.add(LSTM(128, return_sequences=True))
#model.add(LSTM(128))
model.add(Dense(framelen))
model.add(Dense(framelen))
model.add(Dense(framelen))
model.add(Dense(framelen))
#model.add(Dropout(0.2))
#model.add(Dense(framelen))
#model.add(Activation('softmax'))

optimizer = Nadam() #SGD() #Adam() #RMSprop(lr=0.01)
model.compile(loss='mean_absolute_error', optimizer=optimizer)
{'fraction_non_alphanumeric': 0.1632769435468542, 'fraction_numerical': 0.01729709180764113, 'mean_word_length': 5.205188679245283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1419791', 'n_tokens_mistral': 2047, 'n_tokens_neox': 1975, 'n_words': 465}