<jupyter_start><jupyter_text>Pathology data information  show data<jupyter_code>import pandas as pd
import numpy as np
import xlrd
df = pd.read_excel('raw_score20170306 recheck.xlsx', header = 0)
df.head()
df.columns.values
df.loc[:, ['matnumber', 'DateARFI', 'Birth', 'Sex', 'fatty']]<jupyter_output><empty_output><jupyter_text>data cleaning<jupyter_code># values standardization
from dateutil import relativedelta
for i in df.index:
    df.loc[i, 'Age'] = relativedelta.relativedelta(df['DateARFI'][i], df['Birth'][i]).years
df['Sex'].replace({'F':0, 'M':1}, inplace = True)
df['fatty'].replace({'1(2011)':1, '<5':5, '10 (<5%)':5, '20 (<5%)':5, 'no fatty':0}, inplace = True)
df.loc[:, ['matnumber', 'Age', 'Sex', 'fatty']]<jupyter_output><empty_output><jupyter_text>group in four stages<jupyter_code>df0 = pd.DataFrame(columns = ['Age', 'Sex', 'fatty'])
df1 = pd.DataFrame(columns = ['Age', 'Sex', 'fatty'])
df2 = pd.DataFrame(columns = ['Age', 'Sex', 'fatty'])
df3 = pd.DataFrame(columns = ['Age', 'Sex', 'fatty'])
for i in range(len(df.index)):
    if int(df['fatty'][i]) < 5:
        df0 = df0.append({'Age': df['Age'][i], 'Sex': df['Sex'][i], 'fatty': df['fatty'][i]}, ignore_index=True)
    elif int(df['fatty'][i]) <= 33:
        df1 = df1.append({'Age': df['Age'][i], 'Sex': df['Sex'][i], 'fatty': df['fatty'][i]}, ignore_index=True)
    elif int(df['fatty'][i]) < 66:
        df2 = df2.append({'Age': df['Age'][i], 'Sex': df['Sex'][i], 'fatty': df['fatty'][i]}, ignore_index=True)
    elif int(df['fatty'][i]) <= 100:
        df3 = df3.append({'Age': df['Age'][i], 'Sex': df['Sex'][i], 'fatty': df['fatty'][i]}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>calculate patient information<jupyter_code>dfx= [df0, df1, df2, df3, df]
for i in range(5):
    print('df{0}'.format(i))
    print('Sample size : {0}'.format(len(dfx[i])))
    print('Age : {0:.2f} ± {1:.2f} ({2:.0f} - {3:.0f})'.format(dfx[i]['Age'].mean(), dfx[i]['Age'].std(), dfx[i]['Age'].min(), dfx[i]['Age'].max()))
    print('Sex : {0} woman & {1} man'.format(len(dfx[i]['Sex']) - dfx[i]['Sex'].sum(), dfx[i]['Sex'].sum()))<jupyter_output>df0
Sample size : 46
Age : 52.04 ± 11.80 (26 - 75)
Sex : 19.0 woman & 27.0 man
df1
Sample size : 26
Age : 51.19 ± 10.45 (30 - 68)
Sex : 8.0 woman & 18.0 man
df2
Sample size : 25
Age : 54.32 ± 10.31 (39 - 74)
Sex : 4 woman & 21 man
df3
Sample size : 14
Age : 46.71 ± 10.13 (35 - 70)
Sex : 4 woman & 10 man
df4
Sample size : 111
Age : 51.68 ± 11.04 (26 - 75)
Sex : 35 woman & 76 man<jupyter_text>New Pathology data information  show data<jupyter_code>import pandas as pd
import numpy as np
import xlrd
df = pd.read_excel('CGMH_gastro.xlsx', header = 1)
df.head()
df.columns.values
df[['收案編號', 'Age', 'Sex', 'fatty%', 'BMI']]<jupyter_output><empty_output><jupyter_text>data cleaning<jupyter_code># delete data without pathology score
df.dropna(axis = 0, subset = ['fatty%'], inplace = True)
# values standardization
df['Sex'].replace({'F':0, 'M':1}, inplace = True)
df['fatty%'].replace({'< 5':3, '>95':96, 'no fatty chcange':0}, inplace = True)
df = df.reset_index(drop=True)
df[['收案編號', 'Age', 'Sex', 'fatty%', 'BMI']]<jupyter_output><empty_output><jupyter_text>group in four stages<jupyter_code>df0 = pd.DataFrame(columns = ['Age', 'Sex', 'fatty', 'BMI'])
df1 = pd.DataFrame(columns = ['Age', 'Sex', 'fatty', 'BMI'])
df2 = pd.DataFrame(columns = ['Age', 'Sex', 'fatty', 'BMI'])
df3 = pd.DataFrame(columns = ['Age', 'Sex', 'fatty', 'BMI'])
for i in range(len(df.index)):
    if int(df['fatty%'][i]) < 5:
        df0 = df0.append({'Age': df['Age'][i], 'Sex': df['Sex'][i], 'fatty': df['fatty%'][i], 'BMI': df['BMI'][i]}, ignore_index=True)
    elif int(df['fatty%'][i]) <= 33:
        df1 = df1.append({'Age': df['Age'][i], 'Sex': df['Sex'][i], 'fatty': df['fatty%'][i], 'BMI': df['BMI'][i]}, ignore_index=True)
    elif int(df['fatty%'][i]) < 66:
        df2 = df2.append({'Age': df['Age'][i], 'Sex': df['Sex'][i], 'fatty': df['fatty%'][i], 'BMI': df['BMI'][i]}, ignore_index=True)
    elif int(df['fatty%'][i]) <= 100:
        df3 = df3.append({'Age': df['Age'][i], 'Sex': df['Sex'][i], 'fatty': df['fatty%'][i], 'BMI': df['BMI'][i]}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>calculate patient information<jupyter_code>dfx= [df0, df1, df2, df3, df]
for i in range(5):
    print('df{0}'.format(i))
    print('Sample size : {0}'.format(len(dfx[i])))
    print('Age : {0:.2f} ± {1:.2f} ({2:.0f} - {3:.0f})'.format(dfx[i]['Age'].mean(), dfx[i]['Age'].std(), dfx[i]['Age'].min(), dfx[i]['Age'].max()))
    print('Sex : {0} woman & {1} man'.format(len(dfx[i]['Sex']) - dfx[i]['Sex'].sum(), dfx[i]['Sex'].sum()))
    print('BMI : {0:.2f} ± {1:.2f} ({2:.0f} - {3:.0f})'.format(dfx[i]['BMI'].mean(), dfx[i]['BMI'].std(), dfx[i]['BMI'].min(), dfx[i]['BMI'].max()))<jupyter_output>df0
Sample size : 14
Age : 45.00 ± 11.07 (24 - 67)
Sex : 9.0 woman & 5.0 man
BMI : 23.79 ± 3.88 (19 - 32)
df1
Sample size : 20
Age : 52.95 ± 10.95 (28 - 69)
Sex : 9.0 woman & 11.0 man
BMI : 26.60 ± 3.14 (21 - 32)
df2
Sample size : 14
Age : 54.07 ± 11.18 (30 - 69)
Sex : 7.0 woman & 7.0 man
BMI : 28.43 ± 4.77 (23 - 41)
df3
Sample size : 26
Age : 43.38 ± 11.97 (19 - 63)
Sex : 9.0 woman & 17.0 man
BMI : 27.63 ± 2.81 (20 - 32)
df4
Sample size : 74
Age : 48.30 ± 12.14 (19 - 69)
Sex : 34 woman & 40 man
BMI : 26.78 ± 3.81 (19 - 41)
{'fraction_non_alphanumeric': 0.2586915887850467, 'fraction_numerical': 0.0691588785046729, 'mean_word_length': 4.835332606324973, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '78490', 'license': 'MIT', 'n_tokens_mistral': 2636, 'n_tokens_neox': 2288, 'n_words': 520, 'path': 'Cases recorded.ipynb', 'repo_name': 'Lance0218/Ultrasound-Machine-Learning', 'size': '5350'}