<jupyter_start><jupyter_text>Load and Merge Data- want to see cycle data<jupyter_code>fp = os.path.join('..', 'data_out', 'Symptoms_clean.csv')
symptoms =pd. read_csv(fp)
#set zeroes as nans
symptoms.replace(0, np.NaN, inplace=True)
symptoms.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 13512 entries, 0 to 13511
Data columns (total 14 columns):
symptom_id    13512 non-null int64
user_id       13512 non-null int64
acne          4230 non-null float64
backache      5027 non-null float64
bloating      5278 non-null float64
cramp         6943 non-null float64
diarrhea      2233 non-null float64
dizzy         2170 non-null float64
headache      4402 non-null float64
mood          5177 non-null float64
nausea        2194 non-null float64
sore          4090 non-null float64
date          13512 non-null object
date_fix      13511 non-null object
dtypes: float64(10), int64(2), object(2)
memory usage: 1.4+ MB<jupyter_text>Agg Symptoms<jupyter_code>categories = ['acne', 'backache', 'bloating', 'cramp',
       'diarrhea', 'dizzy', 'headache', 'mood', 'nausea', 'sore']
agg_symptoms = symptoms[categories].agg(['mean', 'count']).T
agg_symptoms.sort_values('count', ascending=False)
agg_symptoms.reset_index().plot.bar(x='index', y='count', title='Top Reported Symptoms')<jupyter_output><empty_output><jupyter_text>Agg to the User Level<jupyter_code>users_agg_symptoms = symptoms.groupby(['user_id'], as_index=False)[categories].mean()
users_agg_symptoms.sample(5)<jupyter_output><empty_output><jupyter_text>Cycle Data<jupyter_code>fp = os.path.join('..', 'data_out', 'Clean_periods_clean.csv')
periods = pd.read_csv(fp, parse_dates=['start_date_fix', 'end_date_fix'])
periods.length = pd.to_timedelta(periods.length, unit='days')
periods.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 31964 entries, 0 to 31963
Data columns (total 7 columns):
period_id         31964 non-null int64
start_date        31964 non-null object
end_date          31964 non-null object
user_id           31964 non-null int64
start_date_fix    31964 non-null datetime64[ns]
end_date_fix      31964 non-null datetime64[ns]
length            31964 non-null timedelta64[ns]
dtypes: datetime64[ns](2), int64(2), object(2), timedelta64[ns](1)
memory usage: 1.7+ MB<jupyter_text>create cycle lengths- drop any cycle with length over 45 days<jupyter_code>print('predrop:', periods.shape[0])
periods_multi_respos = periods.groupby('user_id').filter(lambda x: x.shape[0] > 1).copy()
print('postdrop:', periods_multi_respos.shape[0])
#sort
periods_multi_respos.sort_values(['user_id', 'start_date_fix'], inplace=True)
#roll column
periods_multi_respos['previous_start_date'] = periods_multi_respos.start_date_fix.shift(1)
#calc difference
periods_multi_respos['cycle_length'] = periods_multi_respos['start_date_fix'] - periods_multi_respos['previous_start_date']
#set non repeated user it 
cumcount = periods_multi_respos.groupby('user_id').cumcount()
periods_multi_respos.loc[cumcount==0, 'cycle_length'] = np.NaN
#drop missing data
periods_multi_respos.dropna(subset=['cycle_length'], inplace=True)
#check cycle_length stats
print('stats pre drop:', periods_multi_respos.cycle_length.describe(), sep='\n')
#drop any cycle ove 45 days
periods_multi_respos = periods_multi_respos[periods_multi_respos.cycle_length < pd.to_timedelta(45, unit='days')]
print('stats post drop:', periods_multi_respos.cycle_length.describe(), sep='\n')
#create cycle df
select = ['user_id', 'previous_start_date', 'cycle_length']
cycles = periods_multi_respos[select].copy()
cycles.rename(columns={'previous_start_date': 'cycle_start_date'}, inplace=True)
fp = os.path.join('..', 'data_out', 'cycles.csv')
cycles.to_csv(fp, index=False)
cycles.sample(5)<jupyter_output>predrop: 31964
postdrop: 28549
stats pre drop:
count                      25377
mean     36 days 18:27:59.347440
std      34 days 18:54:53.149531
min              0 days 00:00:00
25%             27 days 00:00:00
50%             29 days 00:00:00
75%             34 days 00:00:00
max            947 days 00:00:00
Name: cycle_length, dtype: object
stats post drop:
count                      21962
mean     28 days 18:00:25.571441
std       5 days 07:30:12.911247
min              0 days 00:00:00
25%             26 days 00:00:00
50%             28 days 00:00:00
75%             31 days 00:00:00
max             44 days 00:00:00
Name: cycle_length, dtype: object<jupyter_text>Get data to add to data set<jupyter_code>fp = os.path.join('..', 'data_out', 'User_clean.csv')
users = pd.read_csv(fp)
users = users[['user_id', 'cycle_length_initial', 'period_length_initial']]
users.columns = ['user_id', 'expected_cycle_length', 'expected_period_length']
#expected dates as timedeltas
users.expected_cycle_length = pd.to_timedelta(users.expected_cycle_length, unit='days')
users.expected_period_length = pd.to_timedelta(users.expected_period_length, unit='days')
users.sample(5)<jupyter_output><empty_output><jupyter_text>get average period length<jupyter_code>mean_period_days = pd.to_timedelta(periods.groupby('user_id').apply(lambda x: x.length.dt.days.mean()), unit='days').reset_index()
mean_period_days.columns = ['user_id', 'mean_period_length']
mean_period_days.sample(5)<jupyter_output><empty_output><jupyter_text>get average cycle<jupyter_code>mean_cycle_days = pd.to_timedelta(cycles.groupby('user_id')['cycle_length'].apply(lambda x: x.dt.days.mean()), unit='days').reset_index()
mean_cycle_days.columns = ['user_id', 'mean_cycle_length']
mean_cycle_days.sample(5)<jupyter_output><empty_output><jupyter_text>format data for modeling- data adds, symptoms, expected values<jupyter_code>print('premerge:', cycles.shape)
model_sample = cycles.merge(users, on='user_id', how='left')
model_sample = model_sample.merge(mean_period_days, on='user_id', how='left')
model_sample = model_sample.merge(mean_cycle_days, on='user_id', how='left')
model_sample = model_sample.merge(users_agg_symptoms, on='user_id', how='left')
print('postmerge:', model_sample.shape)
fp = os.path.join('..', 'data_out', 'model_sampl.csv')
model_sample.to_csv(fp, index=False)
model_sample.sample(5)<jupyter_output>premerge: (21962, 3)
postmerge: (21962, 17)<jupyter_text>Erroraverage estimated cycle length error is 3 days 11 hours<jupyter_code>np.abs(model_sample.cycle_length - model_sample.expected_cycle_length).describe()<jupyter_output><empty_output><jupyter_text>Look at good gap in data<jupyter_code>days = cycles.cycle_length.dt.days
days = days[(days > 0 ) & (days < 100)]
days.plot.hist()<jupyter_output><empty_output><jupyter_text>Review cycles data<jupyter_code>cycles['cycle_length_days'] = cycles.cycle_length.dt.days
user_cycles = cycles.groupby('user_id')['cycle_length_days'].agg(['std', 'mean', 'count'])
user_cycles = user_cycles.dropna()
#print('mean std for cycle: {:,.4f}'.format(user_cycles['std'].mean()))
user_cycles.mean()
user_cycles['std'].plot.hist(title='User Cycle Length Standard Deviations')<jupyter_output><empty_output><jupyter_text>add columns for modeling<jupyter_code>model_sample['expected_cycle_length_days'] = model_sample['expected_cycle_length'].dt.days
model_sample['expected_period_length_days'] = model_sample['expected_period_length'].dt.days
model_sample['mean_period_length_days'] = model_sample['mean_period_length'].dt.days
model_sample['mean_cycle_length_days'] = model_sample['mean_cycle_length'].dt.days
model_sample['cycle_length_days'] = model_sample['cycle_length'].dt.days
just_symptoms = ['acne', 'backache', 'bloating', 'cramp', 'diarrhea', 'dizzy',
       'headache', 'mood', 'nausea', 'sore']
for col in just_symptoms:
    model_sample[col + '_0'] = model_sample[col].fillna(0)
    def to_bool(score):
        if np.isnan(score):
            return 0
        return 1
    model_sample[col + '_bool'] = model_sample[col].apply(to_bool)<jupyter_output><empty_output><jupyter_text>create a test and train data set- numpy expects a array of shape (sample, features)<jupyter_code>from sklearn.model_selection import train_test_split

features = ['expected_cycle_length_days', 'expected_period_length_days', 'mean_period_length_days', 'mean_cycle_length_days']

feature_data = model_sample[features].values
target_data = model_sample[['cycle_length_days']].values

x_train, x_test, y_train, y_test = train_test_split(feature_data, target_data, test_size=0.1, random_state=42)

print('train shape: {}\ntest shape: {}'.format(x_train.shape, x_test.shape))
def build_test(df, feature_cols, target_col):
    feature_data = model_sample[feature_cols].values
    target_data = model_sample[[target_col]].values
    return train_test_split(feature_data, target_data, test_size=0.1, random_state=42)

def many_lr(df, features_list, target_col):
    for index, feature_cols in enumerate(features_list):
        print('Run {}'.format(index + 1))
        x_train, x_test, y_train, y_test = build_test(df, feature_cols=feature_cols, target_col=target_col)
        lin_model = linear_model.LinearRegression()
        lin_model.fit(x_train, y_train)
        lin_predictions = lin_model.predict(x_test)
        for feature, coef in zip(feature_cols, lin_model.coef_[0]):
            print('{1}: {0:,.4f} '.format(coef, feature))
        # The mean squared error
        print("Mean squared error: %.2f"
              % mean_squared_error(y_test, lin_predictions))
        # Explained variance score: 1 is perfect prediction
        print('Variance score: %.2f\n' % r2_score(y_test, lin_predictions))<jupyter_output><empty_output><jupyter_text>Try linear regression<jupyter_code>from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
#get model and train
lin_model = linear_model.LinearRegression()
lin_model.fit(x_train, y_train)
lin_predictions = lin_model.predict(x_test)
# The coefficients
for feature, coef in zip(features, lin_model.coef_[0]):
    print('{0:,.4f} {1}'.format(coef, feature))
# The mean squared error
print("Mean squared error: %.2f"
      % mean_squared_error(y_test, lin_predictions))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.2f' % r2_score(y_test, lin_predictions))
just_symptoms = ['acne_bool', 'backache_bool', 'bloating_bool', 'cramp_bool', 'diarrhea_bool', 'dizzy_bool',
       'headache_bool', 'mood_bool', 'nausea_bool', 'sore_bool']<jupyter_output><empty_output><jupyter_text>Build and run many times- adding the symptoms has almost no effect on the results<jupyter_code>features = ['expected_cycle_length_days', 'expected_period_length_days', 'mean_period_length_days', 'mean_cycle_length_days']
just_expected = ['expected_cycle_length_days', 'expected_period_length_days']
just_symptoms_0 = ['acne_0', 'backache_0', 'bloating_0', 'cramp_0', 'diarrhea_0', 'dizzy_0',
       'headache_0', 'mood_0', 'nausea_0', 'sore_0']
just_symptoms_bool = ['acne_bool', 'backache_bool', 'bloating_bool', 'cramp_bool', 'diarrhea_bool', 'dizzy_bool',
       'headache_bool', 'mood_bool', 'nausea_bool', 'sore_bool']
many = features + just_symptoms_bool
many_lr(model_sample, features_list=[features, just_expected, just_symptoms_0, just_symptoms_bool, many], target_col='cycle_length_days')<jupyter_output>Run 1
expected_cycle_length_days: 0.0079 
expected_period_length_days: 0.0029 
mean_period_length_days: -0.0080 
mean_cycle_length_days: 0.9887 
Mean squared error: 15.60
Variance score: 0.45

Run 2
expected_cycle_length_days: 0.5495 
expected_period_length_days: 0.1073 
Mean squared error: 26.85
Variance score: 0.06

Run 3
acne_0: 0.0059 
backache_0: 0.0052 
bloating_0: -0.0070 
cramp_0: 0.0055 
diarrhea_0: 0.0032 
dizzy_0: 0.0045 
headache_0: -0.0056 
mood_0: -0.0048 
nausea_0: -0.0004 
sore_0: -0.0072 
Mean squared error: 28.48
Variance score: 0.00

Run 4
acne_bool: 0.4689 
backache_bool: 0.2229 
bloating_bool: -0.2869 
cramp_bool: 0.3714 
diarrhea_bool: 0.2805 
dizzy_bool: -0.0201 
headache_bool: -0.0577 
mood_bool: -0.0924 
nausea_bool: -0.1486 
sore_bool: -0.5823 
Mean squared error: 28.54
Variance score: 0.00

Run 5
expected_cycle_length_days: 0.0076 
expected_period_length_days: 0.0029 
mean_period_length_days: -0.0085 
mean_cycle_length_days: 0.9888 
acne_bool: -0.0211 
backac[...]<jupyter_text>Run model of just data with feature info<jupyter_code>all_symptoms_sample = model_sample[model_sample[just_symptoms].sum(axis=1) == 0]
many_lr(all_symptoms_sample, features_list=[features, just_expected, just_symptoms_0, just_symptoms_bool, many], target_col='cycle_length_days')<jupyter_output>Run 1
expected_cycle_length_days: 0.0079 
expected_period_length_days: 0.0029 
mean_period_length_days: -0.0080 
mean_cycle_length_days: 0.9887 
Mean squared error: 15.60
Variance score: 0.45

Run 2
expected_cycle_length_days: 0.5495 
expected_period_length_days: 0.1073 
Mean squared error: 26.85
Variance score: 0.06

Run 3
acne_0: 0.0059 
backache_0: 0.0052 
bloating_0: -0.0070 
cramp_0: 0.0055 
diarrhea_0: 0.0032 
dizzy_0: 0.0045 
headache_0: -0.0056 
mood_0: -0.0048 
nausea_0: -0.0004 
sore_0: -0.0072 
Mean squared error: 28.48
Variance score: 0.00

Run 4
acne_bool: 0.4689 
backache_bool: 0.2229 
bloating_bool: -0.2869 
cramp_bool: 0.3714 
diarrhea_bool: 0.2805 
dizzy_bool: -0.0201 
headache_bool: -0.0577 
mood_bool: -0.0924 
nausea_bool: -0.1486 
sore_bool: -0.5823 
Mean squared error: 28.54
Variance score: 0.00

Run 5
expected_cycle_length_days: 0.0076 
expected_period_length_days: 0.0029 
mean_period_length_days: -0.0085 
mean_cycle_length_days: 0.9888 
acne_bool: -0.0211 
backac[...]<jupyter_text>Try SVM- im surprised these aren't as good<jupyter_code>from sklearn import svm
x_train, x_test, y_train, y_test = build_test(model_sample, features, 'cycle_length_days')
svm_model = svm.SVC(kernel='linear')
svm_model.fit(x_train, np.ravel(y_train))
svm_results = svm_model.predict(x_test)
# The mean squared error
print("Mean squared error: %.2f"
      % mean_squared_error(y_test.tolist(), svm_results.tolist()))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.4f' % r2_score(y_test.tolist(), svm_results.tolist()))
x_train, x_test, y_train, y_test = build_test(model_sample, features, 'cycle_length_days')
svm_model = svm.SVC(kernel='rbf')
svm_model.fit(x_train, np.ravel(y_train))
svm_results = svm_model.predict(x_test)
# The mean squared error
print("Mean squared error: %.2f"
      % mean_squared_error(y_test.tolist(), svm_results.tolist()))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.4f' % r2_score(y_test.tolist(), svm_results.tolist()))
from scipy import stats
stats.describe(svm_results[:, np.newaxis] - y_test)<jupyter_output><empty_output><jupyter_text>Do naive test- mean suared error has significant improvement- r2_score is just 0... as expected<jupyter_code>mean_cycle = model_sample.cycle_length_days.mean()
naive_error = mean_squared_error(y_true=model_sample['cycle_length_days'].to_list(), 
                   y_pred=np.full(shape=(model_sample.shape[0],), fill_value=mean_cycle).tolist())
r_val = r2_score(y_true=model_sample['cycle_length_days'].to_list(), 
                   y_pred=np.full(shape=(model_sample.shape[0],), fill_value=mean_cycle).tolist())
print('Naive Error: {}\nr2_score: {}'.format(naive_error, r_val))
(naive_error - 15.60) / naive_error<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1254332613955922, 'fraction_numerical': 0.05499967301026748, 'mean_word_length': 5.8851868527690225, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '357154', 'license': 'MIT', 'n_tokens_mistral': 6154, 'n_tokens_neox': 5577, 'n_words': 1088, 'path': 'code/Cycle Modeling.ipynb', 'repo_name': 'zombieprojectako/PhillyAI_Hackathon_SingleSyllable', 'size': '15291'}