<jupyter_start><jupyter_text>手寫csv<jupyter_code>files=os.listdir()
files
csvFilesToRead=[] #back1.csv
for file in files:
    if 'Demo.csv' in file or 'Phase2.csv' in file or 'Phase3.csv' in file or 'Home.csv' in file:
        csvFilesToRead.append(file)

csvFileAmount=len(csvFilesToRead)
print('csvFilesToRead',csvFilesToRead)
#人口學
demoDf=pd.read_csv('Demo.csv',header=0)
demoDf=demoDf.dropna(axis=0, how='all')
demoDf=demoDf.dropna(axis=1, how='all')
examDate=demoDf.loc[demoDf['id']==id].loc[0].受試日期

# demoDf=demoDf.sort_values(['id','order'])
# demoDf=demoDf.drop_duplicates(subset='id')
# # demoDf
# examDate=demoDf['受試日期']
# # type(examDate)
# examDate=examDate.values
#Phase2 3
phase2Df=pd.read_csv('Phase2.csv',header=0)
phase3Df=pd.read_csv('Phase3.csv',header=0)
phase2Df=phase2Df.dropna(axis=0, how='all')
phase2Df=phase2Df.dropna(axis=1, how='all')
phase3Df=phase3Df.dropna(axis=0, how='all')
phase3Df=phase3Df.dropna(axis=1, how='all')
phase3Df=phase3Df.drop(phase3Df.columns[9], axis=1)
phase3Df.insert(8,'activity',phase3Df.pop(phase3Df.columns[4]))
phase23Df=phase2Df.append(phase3Df,sort =False, ignore_index =True)
phase23Df=phase23Df.sort_values(['id','time1'])
# maxID=int(phase23Df.tail(1).iat[0,0])
phase23Df=phase23Df.loc[phase23Df['id']==id]
phase23Df=phase23Df.reset_index()
phase23Df=phase23Df.drop(columns=['index'])

phase23Output=pd.DataFrame(columns=['id','roomt','roomw','time1','infra','ear','subjective','hr','activity'])
# insert sec
for row in phase23Df.itertuples():
    temp=[row]*60
    temp=pd.DataFrame(temp)
    temp.insert(len(phase23Output.columns)-1,'sec',range(0,60))
    phase23Output=phase23Output.append(temp,sort =False, ignore_index =True)
phase23Output=phase23Output.reset_index()
for boo in phase23Output.duplicated():
    if boo == True:
        print('有重複')

phase23Output.insert(len(phase23Output.columns)-1,'date',examDate)
print(phase23Output.shape)
phase23Output.tail()<jupyter_output>(2400, 13)<jupyter_text>Home.csv<jupyter_code>homeDf=pd.read_csv('Home.csv',header=0)
homeDf=homeDf.dropna(axis=0, how='all')
homeDf=homeDf.dropna(axis=1, how='all')
homeDf=homeDf.sort_values(['id'])
homeDf=homeDf.dropna(subset=['id'])
homeDf.insert(1,'roomt',homeDf.pop(homeDf.columns[3]))
homeDf.insert(2,'roomw','NaN')
# homeDf.isnull().sum()
homeDf.insert(8,'date',homeDf.pop(homeDf.columns[3]))
homeDf.insert(4,'infra','NaN')
homeDf.insert(8,'activity','NaN')
homeDf.insert(10,'特殊事件',homeDf.pop(homeDf.columns[9]))

homeDf=homeDf.loc[homeDf['id']==id]
homeDf=homeDf.reset_index()
homeDf=homeDf.drop(columns=['index'])

homeOutput=pd.DataFrame(columns=['id','roomt','roomw','time1','infra','ear','subjective','hr','activity','date','特殊事件','sec'])
# insert sec
for row in homeDf.itertuples():
    temp=[row]*60
    temp=pd.DataFrame(temp)
    temp.insert(len(homeOutput.columns)-1,'sec',range(0,60))
    homeOutput=homeOutput.append(temp,sort =False, ignore_index =True)
homeOutput=homeOutput.reset_index()
for boo in homeOutput.duplicated():
    if boo == True:
        print('有重複')
        
print(homeOutput.shape)
homeOutput.tail()<jupyter_output>(3300, 14)<jupyter_text>Combine phase & home<jupyter_code>outputDf=phase23Output.append(homeOutput,sort=False)
outputDf=outputDf.sort_values(['id','date','time1'])
outputDf=outputDf.drop_duplicates()
outputDf=outputDf.reset_index()
outputDf=outputDf.drop(columns=['level_0','index'])
print(outputDf.shape)
outputDf.tail()<jupyter_output>(5700, 13)<jupyter_text>Export<jupyter_code>fileName='phase23HomeOutput_'+(str)(id)+'.csv'
fileName
outputDf.to_csv(fileName, sep='\t', encoding='utf-8') #輸出tab分隔, utf-8編碼.csv<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.190436933223413, 'fraction_numerical': 0.039296510030228084, 'mean_word_length': 9.55072463768116, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '562984', 'license': 'Apache-2.0', 'n_tokens_mistral': 1576, 'n_tokens_neox': 1534, 'n_words': 148, 'path': 'Main2.ipynb', 'repo_name': 'hank1992/SmartBandDecode', 'size': '3639'}