<jupyter_start><jupyter_text>Import and Functions<jupyter_code>import pandas as pd 
import numpy as np
import seaborn as sns
import requests
from bs4 import BeautifulSoup
import pandas_profiling
import sklearn
import boto3
from boto3 import session

from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
#from sklearn.cross_validation import cross_val_score, cross_val_predict, train_test_split, KFold, StratifiedKFold
from sklearn import datasets, linear_model, metrics, preprocessing, pipeline
from sklearn.pipeline import make_union, make_pipeline, Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import mean_squared_error, classification_report, confusion_matrix, accuracy_score, r2_score
from sklearn.neighbors import KNeighborsClassifier

import scipy.stats as stats
import statsmodels.api as sm
import statsmodels.formula.api as smf
import patsy 

import matplotlib.pyplot as plt 
%matplotlib inline
import seaborn as sns
import plotly

import re
import sys
import datetime as dt
import time
from time import sleep


!pip install fancyimpute
import fancyimpute
# documentation: https://pypi.org/project/fancyimpute/
#Exploratory Data Analysis

def get_variable_category(series):
    unique_count = series.nunique(dropna=False)
    total_count = len(series)
    if pd.api.types.is_numeric_dtype(series):
        return 'Numerical'
    elif pd.api.types.is_datetime64_dtype(series):
        return 'Date'
    elif unique_count==total_count:
        return 'Text (Unique)'
    else:
        return 'Categorical'

def print_variable_categories(data):
    for column_name in data.columns:
        print(column_name, ': ', get_variable_category(data[column_name]))
        
def datainspect(dataframe):
    '''Inspect data'''
    print('ROWS AND COLUMNS: \n', dataframe.shape,'\n')
    print('MISSING VALUES: \n', dataframe.isnull().sum(),'\n')
    print('DUPLICATE ROWS \n', dataframe.duplicated().sum(),'\n') 
    print('DATA TYPES: \n', dataframe.dtypes,"\n")
    print('DATAFRAME DESCRIBE: \n \n', dataframe.describe(include='all'),'\n')
    print('UNIQUE VALUES:')
    for item in dataframe:
        print(item, dataframe[item].nunique())
    print('\n')
    print('VARIABLE CATEGORIES:', '\n' )
    print(print_variable_categories(dataframe))

# Code to read csv file into Colaboratory:
!pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
# Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)<jupyter_output>[?25l
[K    1% |▎                               | 10kB 16.2MB/s eta 0:00:01
[K    2% |▋                               | 20kB 1.9MB/s eta 0:00:01
[K    3% |█                               | 30kB 2.7MB/s eta 0:00:01
[K    4% |█▎                              | 40kB 1.8MB/s eta 0:00:01
[K    5% |█▋                              | 51kB 2.2MB/s eta 0:00:01
[K    6% |██                              | 61kB 2.6MB/s eta 0:00:01
[K    7% |██▎                             | 71kB 3.0MB/s eta 0:00:01
[K    8% |██▋                             | 81kB 3.4MB/s eta 0:00:01
[K    9% |███                             | 92kB 3.8MB/s eta 0:00:01
[K    10% |███▎                            | 102kB 2.9MB/s eta 0:00:01
[K    11% |███▋                            | 112kB 2.9MB/s eta 0:00:01
[K    12% |████                            | 122kB 4.1MB/s eta 0:00:01
[K    13% |████▎                           | 133kB 4.1MB/s eta 0:00:01
[K    14% |████▋                           | 143kB 7.8MB/s eta 0:00:01
[[...]<jupyter_text>**Datasets**Historical Consumption: A selected time series of consumption data for over 200 buildings.List of Variables that are included in the multiple datasets:**Training Dataset Variables*** **obs_id** - An arbitrary ID for the observations* **SiteId** - An arbitrary ID number for the building, matches across datasets* **ForecastId** - An ID for a timeseries that is part of a forecast (can be matched with the submission file)* **Timestamp** - The time of the measurement* **Value** - A measure of consumption for that building**Building Metadata Dataset**: Additional information about the included buildings.* **SiteId** - An arbitrary ID number for the building, matches across datasets* **Surface** - The surface area of the building* **Sampling** - The number of minutes between each observation for this site. The timestep size for each ForecastId can be found in the separate "Submission Forecast Period" file on the data download page.* **BaseTemperature** - The base temperature for the building* **[DAY_OF_WEEK]IsDayOff** - True if DAY_OF_WEEK is not a work dayHistorical Weather DataThis dataset contains temperature data from several stations near each site. For each site several temperature measurements were retrieved from stations in a radius of 30 km if available.Note: Not all sites will have available weather data.Note: Weather data is available for test periods under the assumption that reasonably accurate forecasts will be available to algorithms that the time that we are attempting to make predictions about the future.**Weather Dataset Variables*** **SiteId** - An arbitrary ID number for the building, matches across datasets* **Timestamp** - The time of the measurement* **Temperature** - The temperature as measured at the weather station* **Distance** - The distance in km from the weather station to the building in kmPublic HolidaysPublic holidays at the sites included in the dataset, which may be helpful for identifying days where consumption may be lower than expected.Note: Not all sites will have available public holiday data.**Holidays Dataset*** **SiteId** - An arbitrary ID number for the building, matches across datasets* **Date** - The date of the holiday* **Holiday** - The name of the holiday  Submission Forecast Period (not using) **There are no days off for buildings Monday - Thursday****Minimal days off for friday, saturday, sunday<jupyter_code>submission_forecast_period = 'https://drive.google.com/open?id=1S59Hrh4yduw3Ee_X5yHzHDlSbg-1PkUK'
fluff, id = submission_forecast_period.split('=')
print (id)
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('power-laws-forecasting-energy-consumption-submission-forecast-period.csv')  
submission_forecast_period = pd.read_csv('power-laws-forecasting-energy-consumption-submission-forecast-period.csv', sep = ';')
submission_forecast_period.head()
datainspect(submission_forecast_period)
submission_forecast_period.plot(kind='scatter', x='ForecastId', y='ForecastPeriodNS');
correlation = submission_forecast_period.corr()                            # correlation between ALL variables 
sns.heatmap(correlation, cmap='bwr')<jupyter_output><empty_output><jupyter_text>Submission Format (not using)<jupyter_code>submission_format = 'https://drive.google.com/open?id=1B196PxseHWY4DTGgy1dUMqwtm7MFEw_r'
fluff, id = submission_format.split('=')
print (id)
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('power-laws-forecasting-energy-consumption-submission-format.csv')  
submission_format = pd.read_csv('power-laws-forecasting-energy-consumption-submission-format.csv', sep = ';')
submission_format.head()
datainspect(submission_format)<jupyter_output>ROWS AND COLUMNS: 
 (1309176, 5) 

MISSING VALUES: 
 obs_id        0
SiteId        0
Timestamp     0
ForecastId    0
Value         0
dtype: int64 

DUPLICATE ROWS 
 0 

DATA TYPES: 
 obs_id          int64
SiteId          int64
Timestamp      object
ForecastId      int64
Value         float64
dtype: object 

DATAFRAME DESCRIBE: 
 
               obs_id        SiteId   Timestamp    ForecastId      Value
count   1.309176e+06  1.309176e+06     1309176  1.309176e+06  1309176.0
unique           NaN           NaN        2269           NaN        NaN
top              NaN           NaN  2016-07-26           NaN        NaN
freq             NaN           NaN        2597           NaN        NaN
mean    3.931478e+06  1.621134e+02         NaN  3.482680e+03        0.0
std     2.271962e+06  9.393814e+01         NaN  2.009967e+03        0.0
min     2.000000e+00  1.000000e+00         NaN  1.000000e+00        0.0
25%     1.963225e+06  8.700000e+01         NaN  1.744000e+03        0.0
50%     3.930562e+0[...]<jupyter_text>Holidays<jupyter_code>holidays = 'https://drive.google.com/open?id=1H0TqoTnei1_8DP-ttDsOsJyVkmytQZLG'
fluff, id = holidays.split('=')
print (id)
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('power-laws-forecasting-energy-consumption-holidays.csv')  
holidays = pd.read_csv('power-laws-forecasting-energy-consumption-holidays.csv', sep = ';')
holidays.head()<jupyter_output><empty_output><jupyter_text>**Holidays Dataset**SiteId - An arbitrary ID number for the building, matches across datasetsDate - The date of the holidayHoliday - The name of the holidayI'm thinking this dataset does not tell us much since we already have the days that the building is on and off so this may be redundant data.<jupyter_code>datainspect(holidays)
holidays['Holiday'].value_counts().head()<jupyter_output><empty_output><jupyter_text>Metadata<jupyter_code>metadata = 'https://drive.google.com/open?id=1dpwWIs3EyLkaaOSN8ws8gMHgCFsRTYK3'
fluff, id = metadata.split('=')
print (id)
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('power-laws-forecasting-energy-consumption-metadata.csv')  
metadata = pd.read_csv('power-laws-forecasting-energy-consumption-metadata.csv', sep = ';')
metadata.head()
datainspect(metadata)
# sampling and BaseTemperature do not tell us much about the energy consumption
metadata['MondayIsDayOff'].value_counts()

def dayoff(dataframe):
    print('MONDAY IS A DAY OFF: \n', dataframe.MondayIsDayOff.value_counts(),'\n')
    print('TUESDAY IS A DAY OFF: \n', dataframe.TuesdayIsDayOff.value_counts(),'\n')
    print('WEDNESDAY IS A DAY OFF: \n', dataframe.WednesdayIsDayOff.value_counts(),'\n')
    print('THURSDAY IS A DAY OFF: \n', dataframe.ThursdayIsDayOff.value_counts(),'\n')
    print('FRIDAY IS A DAY OFF: \n', dataframe.FridayIsDayOff.value_counts(),'\n')
    print('SATURDAY IS A DAY OFF: \n', dataframe.SaturdayIsDayOff.value_counts(),'\n')
    print('SUNDAY IS A DAY OFF: \n', dataframe.SundayIsDayOff.value_counts(),'\n')
dayoff(metadata)
metadata.plot(kind='scatter', x='Surface', y='BaseTemperature');<jupyter_output>'c' argument looks like a single numeric RGB or RGBA sequence, which should be avoided as value-mapping will have precedence in case its length matches with 'x' & 'y'.  Please use a 2-D array with a single row if you really want to specify the same RGB or RGBA value for all points.<jupyter_text>Training Data<jupyter_code>training_data = 'https://drive.google.com/open?id=1Ri9t0yrf2A99S8Lqy3DwvydP-jmsokeL'
fluff, id = training_data.split('=')
print (id)
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('power-laws-forecasting-energy-consumption-training-data.csv')  
training_data = pd.read_csv('power-laws-forecasting-energy-consumption-training-data.csv', sep = ';')
training_data.head()
datainspect(training_data)
training_data.plot(kind='scatter', x='ForecastId', y='Value');
corr = training_data.corr()                            # correlation between ALL variables 
sns.heatmap(corr, cmap='bwr')<jupyter_output><empty_output><jupyter_text>**Correlation Pot***   forecastId an SiteId are very correlated **Inspecting the Training Data:***   There are no missing values*   Multiple time-stamps for same Site-id  Weather<jupyter_code>weather = 'https://drive.google.com/open?id=1QZtFjz61NS0Ebbma42LrGtYZFRSsF3Ij'
fluff, id = weather.split('=')
print (id)
downloaded = drive.CreateFile({'id':id}) 
downloaded.GetContentFile('power-laws-forecasting-energy-consumption-weather.csv')  
weather = pd.read_csv('power-laws-forecasting-energy-consumption-weather.csv', sep = ';')
weather.head()
datainspect(weather)
corr2 = weather.corr()                            # correlation between ALL variables 
sns.heatmap(corr2, cmap='bwr')
# Variables
weather_distance = weather[['Distance']]
weather_temp = weather[['Temperature']]
training_data_value = training_data[['Value']]
training_data_ts = training_data[['Timestamp']]
metadata_basetemp = metadata[['BaseTemperature']]
metadata_surface = metadata[['Surface']]
plt.plot(metadata_basetemp)
plt.plot(metadata_surface)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10736842105263159, 'fraction_numerical': 0.03243664717348928, 'mean_word_length': 3.925499231950845, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '506373', 'license': 'MIT', 'n_tokens_mistral': 4427, 'n_tokens_neox': 4012, 'n_words': 1090, 'path': 'team_high_energy_(previous).ipynb', 'repo_name': 'sianlewis/energy_consumption', 'size': '12825'}