<jupyter_start><jupyter_text>Hackaton IESB 2021.2  recuperando os dados do repositório<jupyter_code>!git clone https://github.com/mendelson/polar-sample-database.git
!ls<jupyter_output>polar-sample-database  sample_data<jupyter_text>Imports necessários<jupyter_code>from sklearn import linear_model
from sklearn.model_selection import train_test_split
import json
import numpy as np
from datetime import datetime
import pandas as pd
import plotly.express as px<jupyter_output><empty_output><jupyter_text>Funções auxiliares<jupyter_code>def readFile(url):
  with open(url, 'r') as myfile:
      data=myfile.read()
  return data

def formatExcludingUTC(str):
  return str[:str.rindex('-')]

def getDateFromStr(str, pattern):
  return datetime.strptime(str, pattern).replace(hour=0, minute=0, second=0, microsecond=0)

def collect(array, field, start, end):
  l = []
  for e in array:
    if (getDateFromStr(formatExcludingUTC(e['datetimeStr']), '%Y-%m-%dT%H:%M:%S') >= getDateFromStr(start, '%Y-%m-%dT%H:%M:%S.%f') and getDateFromStr(formatExcludingUTC(e['datetimeStr']), '%Y-%m-%dT%H:%M:%S') <= getDateFromStr(end, '%Y-%m-%dT%H:%M:%S.%f')) :
      l.append(e[field] if e[field] else 0)
  return l<jupyter_output><empty_output><jupyter_text>Histograma sobre os dados<jupyter_code>#dados gerais

filesList = !ls polar-sample-database/dataset/

stats = []

filtered = []

for f in filesList:
  dataExercises = json.loads(readFile('polar-sample-database/dataset/'+f))
  filter = True
  stats.append('TOTAL')
  if 'physicalInformationSnapshot' in dataExercises:
    stats.append('PHYSICAL_INFO')
  else:
    filter = False

  if 'kiloCalories' in dataExercises:
    stats.append('CALORIES_INFO')
  else:
    filter = False


  weatherFiles = !ls polar-sample-database/weather\ dataset | grep {dataExercises['startTime'][:dataExercises['startTime'].index('T')]} | grep .json
  if len(weatherFiles) > 0:
    stats.append("WEATHER_INFO")
  else:
    filter = False


  if filter:
    stats.append("FILTERED")
    filtered.append(dataExercises['exercises'][0]['sport'])

df = pd.DataFrame(stats)

fig = px.histogram(df, x=0, title='Visão geral dos dados', labels={'Visão geral dos dados'})
fig.show()
df = pd.DataFrame(filtered)

fig = px.histogram(df, x=0, title='Visão da barra de filtrados')
fig.show()<jupyter_output><empty_output><jupyter_text>Tentando prever o tipo de esporte com base em informações climáticas<jupyter_code>filesList = !ls polar-sample-database/dataset/

print(len(filesList))

records = []

for f in filesList:
  dataExercises = json.loads(readFile('polar-sample-database/dataset/'+f))
  for ex in dataExercises['exercises']:
    weatherFiles = !ls polar-sample-database/weather\ dataset | grep {ex['startTime'][:ex['startTime'].index('T')]} | grep .json
    if len(weatherFiles) > 0:
      dataWeather = json.loads(readFile('polar-sample-database/weather dataset/'+weatherFiles[0]))
      values = list(dataWeather['locations'].values())[0]['values']
      records.append((ex['sport'], ex['startTime'], ex['stopTime'], np.nanmean(collect(values, 'temp', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'wspd', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'humidity', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'cloudcover', ex['startTime'], ex['stopTime']))))


print(records)

print(len(records))
#Creating the X and Y variables
X = []
y = []
for record in records:
  y.append(record[0])
  X.append(list(record[3:]))

#Splitting the recors into training and test datasets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

#Regression with scktlearn
clf = linear_model.LogisticRegression(max_iter=10000).fit(X_train, y_train)
clf.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Tentando prever o gasto calórico com base em (duração, tipo de esporte, informções do clima)<jupyter_code>filesList = !ls polar-sample-database/dataset/

print(len(filesList))

records = []

token = {}

for f in filesList:
  dataExercises = json.loads(readFile('polar-sample-database/dataset/'+f))
  if('kiloCalories' in dataExercises):
    for ex in dataExercises['exercises']:
      weatherFiles = !ls polar-sample-database/weather\ dataset | grep {ex['startTime'][:ex['startTime'].index('T')]} | grep .json
      if len(weatherFiles) > 0:
        dataWeather = json.loads(readFile('polar-sample-database/weather dataset/'+weatherFiles[0]))
        values = list(dataWeather['locations'].values())[0]['values']
        token[ex['sport']] = token[ex['sport']] if ex['sport'] in token else len(token)
        records.append((dataExercises['kiloCalories'], token[ex['sport']], (datetime.strptime(ex['stopTime'], '%Y-%m-%dT%H:%M:%S.%f') - datetime.strptime(ex['startTime'], '%Y-%m-%dT%H:%M:%S.%f')).total_seconds() , np.nanmean(collect(values, 'temp', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'wspd', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'humidity', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'cloudcover', ex['startTime'], ex['stopTime']))))


print(records)

print(len(records))

print(token)
#Creating the X and Y variables
X = []
y = []
for record in records:
  y.append(record[0])
  X.append(list(record[1:]))

#Splitting the recors into training and test datasets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

#Regression with scktlearn
clf = linear_model.LinearRegression(normalize=True).fit(X_train, y_train)
print(clf.score(X_test, y_test))
print(clf.predict([[4, 1480.125, 30.200000000000003, 3.6, 76.24, 55.0]]))<jupyter_output>0.7145503163556874
[87.29107809]<jupyter_text>Testando se os atributos do clima influenciam ou não, i.e. removendo eles da entrada<jupyter_code>filesList = !ls polar-sample-database/dataset/

print(len(filesList))

records = []

token = {}

for f in filesList:
  dataExercises = json.loads(readFile('polar-sample-database/dataset/'+f))
  if('kiloCalories' in dataExercises):
    for ex in dataExercises['exercises']:
      weatherFiles = !ls polar-sample-database/weather\ dataset | grep {ex['startTime'][:ex['startTime'].index('T')]} | grep .json
      if len(weatherFiles) > 0:
        dataWeather = json.loads(readFile('polar-sample-database/weather dataset/'+weatherFiles[0]))
        values = list(dataWeather['locations'].values())[0]['values']
        token[ex['sport']] = token[ex['sport']] if ex['sport'] in token else len(token)
        records.append((dataExercises['kiloCalories'], token[ex['sport']], (datetime.strptime(ex['stopTime'], '%Y-%m-%dT%H:%M:%S.%f') - datetime.strptime(ex['startTime'], '%Y-%m-%dT%H:%M:%S.%f')).total_seconds()))


print(records)

print(len(records))

print(token)
#Creating the X and Y variables
X = []
y = []
for record in records:
  y.append(record[0])
  X.append(list(record[1:]))

#Splitting the recors into training and test datasets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

#Regression with scktlearn
clf = linear_model.LinearRegression(normalize=True).fit(X_train, y_train)
print(clf.score(X_test, y_test))
print(clf.predict([[0, 2960.25]]))<jupyter_output>0.5950733127996843
[486.4604288]<jupyter_text>Invertendo, se é possível determinar a duração pelo clima e a quantidade de calorias alvo!<jupyter_code>filesList = !ls polar-sample-database/dataset/

print(len(filesList))

records = []

token = {}

for f in filesList:
  dataExercises = json.loads(readFile('polar-sample-database/dataset/'+f))
  if('kiloCalories' in dataExercises):
    for ex in dataExercises['exercises']:
      weatherFiles = !ls polar-sample-database/weather\ dataset | grep {ex['startTime'][:ex['startTime'].index('T')]} | grep .json
      if len(weatherFiles) > 0:
        dataWeather = json.loads(readFile('polar-sample-database/weather dataset/'+weatherFiles[0]))
        values = list(dataWeather['locations'].values())[0]['values']
        token[ex['sport']] = token[ex['sport']] if ex['sport'] in token else len(token)
        records.append(((datetime.strptime(ex['stopTime'], '%Y-%m-%dT%H:%M:%S.%f') - datetime.strptime(ex['startTime'], '%Y-%m-%dT%H:%M:%S.%f')).total_seconds(), dataExercises['kiloCalories'], token[ex['sport']], np.nanmean(collect(values, 'temp', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'wspd', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'humidity', ex['startTime'], ex['stopTime'])), np.mean(collect(values, 'cloudcover', ex['startTime'], ex['stopTime']))))


print(records)

print(len(records))

print(token)
#Creating the X and Y variables
X = []
y = []
for record in records:
  y.append(record[0])
  X.append(list(record[1:]))

#Splitting the recors into training and test datasets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

#Regression with scktlearn
clf = linear_model.LinearRegression(normalize=True).fit(X_train, y_train)
print(clf.score(X_test, y_test))
print(clf.predict([[301, 0, 21.452, 6.68, 46.407999999999994, 13.54]]))<jupyter_output>0.8862809240780773
[1621.84528616]<jupyter_text>Testando com "data augmentation"<jupyter_code>#Creating the X and Y variables
X = []
y = []
for record in records:
  y.append(record[0])
  X.append(list(record[1:]))

X.extend(X)
X.extend(X)
X.extend(X)
X.extend(X)
y.extend(y)
y.extend(y)
y.extend(y)
y.extend(y)

print(len(X))

#Splitting the recors into training and test datasets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

#Regression with scktlearn
clf = linear_model.LinearRegression(normalize=True).fit(X_train, y_train)
print(clf.score(X_test, y_test))
print(clf.predict([[301, 0, 21.452, 6.68, 46.407999999999994, 13.54]]))<jupyter_output>576
0.7564886473960362
[1662.92613282]
{'fraction_non_alphanumeric': 0.17127974670615873, 'fraction_numerical': 0.02839342253089572, 'mean_word_length': 6.064935064935065, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '180846', 'license': 'MIT', 'n_tokens_mistral': 3798, 'n_tokens_neox': 3572, 'n_words': 679, 'path': 'solution/HackatonIESB2021_2.ipynb', 'repo_name': 'olaioo/polar-sample-database', 'size': '9791'}