<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os, json
from stravalib import client, unithelper
import pandas as pd
from datetime import datetime, timedelta
from sqlalchemy import create_engine
from unidecode import unidecode

def coerce_uni(string):
    ''' coerce ascii2 unicode for a string'''
    if string:
        string = unicode(unidecode(string))
    return string

def GetActivities(client, startDate, endDate, limit):
    # Returns a list of Strava activity objects, up to the number specified by
    # limit
    startDate = datetime.strptime(startDate, '%Y-%m-%d')
    endDate = datetime.strptime(endDate, '%Y-%m-%d')
    print "limit is: " + str(limit)
    activities = client.get_activities(before=endDate,
                                       after=startDate,
                                       limit=limit)
    return activities


def GetStreams(client, activity, types, resolution):
    # Returns a Strava 'stream', which is timeseries data from an activity
    streams = client.get_activity_streams(activity,
                                          types=types,
                                          series_type='distance',
                                          resolution=resolution)
    return streams


def DataFrame(dict, types):
    # Converts a Stream into a dataframe, and returns the dataframe
    df = pd.DataFrame()
    for item in types:
        if item in dict.keys():
            df.append(item.data)
    df.fillna('', inplace=True)
    return df


def ParseActivity(client, act, types, resolution):
    # Parses an activity from an athlete into a dataframe.  Returns the
    # dataframe
    act_id = act.id
    name = coerce_uni(act.name)
    streams = GetStreams(client, act_id, types, resolution)
    df = pd.DataFrame()
    # Write each row to a dataframe
    for item in types:
        if item in streams.keys():
            df[item] = pd.Series(streams[item].data, index=None)
        df['act_id'] = act.id
        df['act_startDate'] = pd.to_datetime(act.start_date_local)
    return df


def get_acts_in_db(engine, table_name):
    # Return a list of already cached activities in the database
    already_dl_act_id_list = []
    try:
        args = 'SELECT Distinct act_id from "%s"' % (table_name)
        df = pd.read_sql(args, engine)
        already_dl_act_id_list = df['act_id'].tolist()
    except:
        print "no activities in database!  downloading all activities in range..."

    return already_dl_act_id_list


def check_if_new_act(act_list, already_dl_act_id_list):
    # Returns new activities not yet in the database
    for act in list(act_list):
        if act.id in already_dl_act_id_list:
            act_list.remove(act)

    return act_list


def loop_activities(client, activities, already_dl_act_id_list, types):
    # Loops through a list of activities, creating a dict of dataframes, one
    # entry for each activity
    df_lst = {}
    for act in activities:
        if act.id not in already_dl_act_id_list:
            df_lst[act.start_date] = ParseActivity(client, act, types)
    return df_lst


def calctime(time_sec, startdate):
    # calculates timestamp from the elapsed time.  Returns a datetime value.
    timestamp = startdate + timedelta(seconds=float(time_sec))

    return timestamp


def split_lat(series):<commit_msg>fixed trainer and manual omission<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os, json
from stravalib import client, unithelper
import pandas as pd
from datetime import datetime, timedelta
from sqlalchemy import create_engine
from unidecode import unidecode

def coerce_uni(string):
    ''' coerce ascii2 unicode for a string'''
    if string:
        string = unicode(unidecode(string))
    return string

def GetActivities(client, startDate, endDate, limit):
    # Returns a list of Strava activity objects, up to the number specified by
    # limit
    startDate = datetime.strptime(startDate, '%Y-%m-%d')
    endDate = datetime.strptime(endDate, '%Y-%m-%d')
    print "limit is: " + str(limit)
    activities = client.get_activities(before=endDate,
                                       after=startDate,
                                       limit=limit)
    return activities


def GetStreams(client, activity, types, resolution):
    # Returns a Strava 'stream', which is timeseries data from an activity
    streams = client.get_activity_streams(activity,
                                          types=types,
                                          series_type='distance',
                                          resolution=resolution)
    return streams


def DataFrame(dict, types):
    # Converts a Stream into a dataframe, and returns the dataframe
    df = pd.DataFrame()
    for item in types:
        if item in dict.keys():
            df.append(item.data)
    df.fillna('', inplace=True)
    return df


def ParseActivity(client, act, types, resolution):
    # Parses an activity from an athlete into a dataframe.  Returns the
    # dataframe
    act_id = act.id
    name = coerce_uni(act.name)
    streams = GetStreams(client, act_id, types, resolution)
    df = pd.DataFrame()
    # Write each row to a dataframe
    for item in types:
        if item in streams.keys():
            df[item] = pd.Series(streams[item].data, index=None)
        df['act_id'] = act.id
        df['act_startDate'] = pd.to_datetime(act.start_date_local)
    return df


def get_acts_in_db(engine, table_name):
    # Return a list of already cached activities in the database
    already_dl_act_id_list = []
    try:
        args = 'SELECT Distinct act_id from "%s"' % (table_name)
        df = pd.read_sql(args, engine)
        already_dl_act_id_list = df['act_id'].tolist()
    except:
        print "no activities in database!  downloading all activities in range..."

    return already_dl_act_id_list


def check_if_new_act(act_list, already_dl_act_id_list):
    # Returns new activities not yet in the database
    for act in list(act_list):
        if act.id in already_dl_act_id_list:
            act_list.remove(act)

    return act_list


def loop_activities(client, activities, already_dl_act_id_list, types):
    # Loops through a list of activities, creating a dict of dataframes, one
    # entry for each activity
    df_lst = {}
    for act in activities:
        if act.id not in already_dl_act_id_list and act.trainer==False and act.manual==False:
            df_lst[act.start_date] = ParseActivity(client, act, types)
    return df_lst


def calctime(time_sec, startdate):
    # calculates timestamp from the elapsed time.  Returns a datetime value.
    timestamp = startdate + timedelta(seconds=float(time_sec))

    return timestamp


def split_lat(series):
{'fraction_non_alphanumeric': 0.08406356750334175, 'fraction_numerical': 0.0005940888162780336, 'mean_word_length': 2.6498644986449866, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '612142', 'n_tokens_mistral': 1982, 'n_tokens_neox': 1929, 'n_words': 603}