<commit_before># Author: <NAME>
# Data Frame exploration.
# Extract data from Documents and summaries into rewsum1 to table in DB


# SUBSET CREATION

# Create Subset and merge all columns into one column called "Documents". Column "Summaries" = summary of document
dfData = with(df, paste0(Document1, Document2, Document3, Document4, Document5, Document6, Document7, Document8, Document9, Document10, Document11, Document12, Document13, Document14, Document15, Document16, Document17, Document18, Document19, Document20, Document21, Document21, Document22, Document23, Document24, Document25))
#
#
dfEdited = data.frame(dfData)
# Create new column named "Documents"
colnames(dfEdited) ="Documents"
dfEdited$Summaries = df$Sum1

# Status echo
cat('Extracting completed.')
<commit_msg>Update extracterDB.r<commit_after># Data Frame exploration.
# Extract data from Documents and document summaries into new subset and prepare them for extarction to table in DB.

# Extract Documents from df > dfData
counter = 1
while(counter <= 25){
  for (m in doc) {
    if ((exists("dfData"))==TRUE) {
      dfEdit=data.frame(with(df, paste0(df[[m]])), stringsAsFactors=FALSE)
      colnames(dfEdit)="Documents"
      dfData=rbind(dfData,dfEdit)
    } else {
      dfData=data.frame(with(df, paste0(df[[m]])),stringsAsFactors=FALSE)
      colnames(dfData)="Documents"
    }
    counter = counter + 1
    
  }
}


# Extract Document names from df > dfDataNames

docNames = c('DocumentName1', 'DocumentName2', 'DocumentName3', 'DocumentName4', 'DocumentName5', 'DocumentName6', 'DocumentName7', 'DocumentName8', 'DocumentName9', 'DocumentName10', 'DocumentName11', 'DocumentName12', 'DocumentName13', 'Document14', 'DocumentName15', 'DocumentName16', 'DocumentName17', 'DocumentName18', 'DocumentName19', 'DocumentName20', 'DocumentName21', 'DocumentName22', 'DocumentName23', 'DocumentName24', 'DocumentName25')
counter = 1
while(counter <= 25){
  for (w in docNames) {
    if ((exists("dfDataNames"))==TRUE) {
      dfEditNames=data.frame(with(df, paste0(df[[w]])), stringsAsFactors=FALSE)
      colnames(dfEditNames)="DocSum"
      dfDataNames=rbind(dfDataNames,dfEditNames)
    } else {
      dfDataNames=data.frame(with(df, paste0(df[[w]])),stringsAsFactors=FALSE)
      colnames(dfDataNames)="DocSum"
    }
    counter = counter + 1
    
  }
}

#Merge data frames together
dfData$DocSum = dfDataNames$DocSum

# Status echo
cat('Extracting completed. Moving to another stage...')

{'fraction_non_alphanumeric': 0.1279870392871608, 'fraction_numerical': 0.040097205346294046, 'mean_word_length': 5.569148936170213, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2865729', 'n_tokens_mistral': 822, 'n_tokens_neox': 769, 'n_words': 198}