<commit_before>"""
Watstock's market prediction neural network.
"""

from __future__ import print_function
import time
import datetime
from dateutil.relativedelta import relativedelta
import numpy as np
import math

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import GRU
from keras.layers import Dropout
from keras.layers import Activation
from keras.callbacks import EarlyStopping

from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import MinMaxScaler

# fixing random generator
np.random.seed(1234)


def create_dataset(df, days_ahead=1, timesteps=1):
  """Converts a dataframe of values into a matrix"""
  dataX, dataY = [], []
  for i in xrange(len(df) - timesteps - days_ahead + 1):
    dataX.append(df[i:(i + timesteps), :])
    dataY.append(df[i + timesteps + days_ahead - 1, :])

  return np.array(dataX), np.array(dataY)

def split_dataset(dataframe, days_ahead=1, timesteps=30, testset=30):
  # split into train and test datasets
  testset_ratio = 1.0 * (testset + timesteps + days_ahead - 1)/len(dataframe)
  train_size = int(len(dataframe) * (1 - testset_ratio))
  train_df, test_df = dataframe[0:train_size,:], dataframe[train_size:,:]

  # Reshape into X=...,t-3,t-2,t-1,t and Y=t+days_ahead
  X_train, Y_train = create_dataset(train_df, days_ahead=days_ahead, timesteps=timesteps)
  X_test, Y_test = create_dataset(test_df, days_ahead=days_ahead, timesteps=timesteps)

  # Reshape inputs from [samples, features] to [samples, timesteps, features] format
  X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], X_train.shape[2]))
  X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], X_test.shape[2]))

  # Leave only columns for prediction
  Y_train = Y_train[:,-1:]
  Y_test = Y_test[:,-1:]

  return X_train, Y_train, X_test, Y_test

def build_model(layers, sequence_length, dropout=None):
  model = Sequential()

  for i in xrange(len(layers) - 2):
    return_sequences = i < len(layers) - 3
    layer = None
    if i == 0:
      layer = GRU(layers[i+1],
                  input_shape=(sequence_length, layers[i]),
                  return_sequences=return_sequences)
    else:
      layer = GRU(layers[i+1],
                  return_sequences=return_sequences)

    model.add(layer)

  # adding dropout
  if dropout != None:
    model.add(Dropout(dropout))

  model.add(Dense(layers[-1]))
  model.add(Activation('linear')) # Since we are doing a regression, its activation is linear

  model.compile(loss='mse', optimizer='rmsprop')

  return model


def train_model(model, data, batch_size=1, epochs=100, valset=30, patience=5, verbose=0):
  X_train, Y_train = data

  early_stopping = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose)
  val_ratio = 1.0 * valset / len(X_train)

  start_time = time.time()
  model.fit(X_train, 
            Y_train, 
            batch_size=batch_size, 
            nb_epoch=epochs,
            verbose=verbose,
            validation_split=val_ratio, 
            callbacks=[early_stopping])

  return (time.time() - start_time)

def normalize_dataframe(df):

  scaler = MinMaxScaler(feature_range=(0, 1))

  dataset = df.copy().values
  dataset = dataset.astype('float32')  
  dataset = scaler.fit_transform(dataset)

  return dataset, scaler

def calculate_model(symbol, dataset, scaler, architecture, days_ahead=1, timesteps=15, testset=30, valset=30, dropout=None, batch_size=10, epochs=500, early_stopping_patience=5, verbose=1):

  if verbose == 1:
    print('\n%s-day prediction model for %s' % (days_ahead, symbol))

  # Split into train and test sets
  X_train, Y_train, X_test, Y_test = split_dataset(dataset, timesteps=timesteps, testset=testset, days_ahead=days_ahead)
  if verbose == 1:<commit_msg>Do not show epochs progress<commit_after>"""
Watstock's market prediction neural network.
"""

from __future__ import print_function
import time
import datetime
from dateutil.relativedelta import relativedelta
import numpy as np
import math

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import GRU
from keras.layers import Dropout
from keras.layers import Activation
from keras.callbacks import EarlyStopping

from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import MinMaxScaler

# fixing random generator
np.random.seed(1234)


def create_dataset(df, days_ahead=1, timesteps=1):
  """Converts a dataframe of values into a matrix"""
  dataX, dataY = [], []
  for i in xrange(len(df) - timesteps - days_ahead + 1):
    dataX.append(df[i:(i + timesteps), :])
    dataY.append(df[i + timesteps + days_ahead - 1, :])

  return np.array(dataX), np.array(dataY)

def split_dataset(dataframe, days_ahead=1, timesteps=30, testset=30):
  # split into train and test datasets
  testset_ratio = 1.0 * (testset + timesteps + days_ahead - 1)/len(dataframe)
  train_size = int(len(dataframe) * (1 - testset_ratio))
  train_df, test_df = dataframe[0:train_size,:], dataframe[train_size:,:]

  # Reshape into X=...,t-3,t-2,t-1,t and Y=t+days_ahead
  X_train, Y_train = create_dataset(train_df, days_ahead=days_ahead, timesteps=timesteps)
  X_test, Y_test = create_dataset(test_df, days_ahead=days_ahead, timesteps=timesteps)

  # Reshape inputs from [samples, features] to [samples, timesteps, features] format
  X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], X_train.shape[2]))
  X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], X_test.shape[2]))

  # Leave only columns for prediction
  Y_train = Y_train[:,-1:]
  Y_test = Y_test[:,-1:]

  return X_train, Y_train, X_test, Y_test

def build_model(layers, sequence_length, dropout=None):
  model = Sequential()

  for i in xrange(len(layers) - 2):
    return_sequences = i < len(layers) - 3
    layer = None
    if i == 0:
      layer = GRU(layers[i+1],
                  input_shape=(sequence_length, layers[i]),
                  return_sequences=return_sequences)
    else:
      layer = GRU(layers[i+1],
                  return_sequences=return_sequences)

    model.add(layer)

  # adding dropout
  if dropout != None:
    model.add(Dropout(dropout))

  model.add(Dense(layers[-1]))
  model.add(Activation('linear')) # Since we are doing a regression, its activation is linear

  model.compile(loss='mse', optimizer='rmsprop')

  return model


def train_model(model, data, batch_size=1, epochs=100, valset=30, patience=5, verbose=0):
  X_train, Y_train = data

  early_stopping = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose)
  val_ratio = 1.0 * valset / len(X_train)

  start_time = time.time()
  model.fit(X_train, 
            Y_train, 
            batch_size=batch_size, 
            nb_epoch=epochs,
            verbose=0,
            validation_split=val_ratio, 
            callbacks=[early_stopping])

  return (time.time() - start_time)

def normalize_dataframe(df):

  scaler = MinMaxScaler(feature_range=(0, 1))

  dataset = df.copy().values
  dataset = dataset.astype('float32')  
  dataset = scaler.fit_transform(dataset)

  return dataset, scaler

def calculate_model(symbol, dataset, scaler, architecture, days_ahead=1, timesteps=15, testset=30, valset=30, dropout=None, batch_size=10, epochs=500, early_stopping_patience=5, verbose=1):

  if verbose == 1:
    print('\n%s-day prediction model for %s' % (days_ahead, symbol))

  # Split into train and test sets
  X_train, Y_train, X_test, Y_test = split_dataset(dataset, timesteps=timesteps, testset=testset, days_ahead=days_ahead)
  if verbose == 1:
{'fraction_non_alphanumeric': 0.1163645915418965, 'fraction_numerical': 0.01720514841082217, 'mean_word_length': 4.2772002772002775, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3029774', 'n_tokens_mistral': 2772, 'n_tokens_neox': 2586, 'n_words': 615}