<commit_before>#For use with python 3.x.
# Program only checks xml files in directory that contains this python file.
# Delete all csv files before running script. Script appends to csv files of 
# the same name if they already exist.

# libraries
from bs4 import BeautifulSoup            # select XML tags and parse text
from os import walk, getcwd, listdir     # used to grab all files in directory of script (c.f. line 29)
import os                                # used to split off filename root from filename extension (c.f. line 31)
import csv                               # interact with csv files
#import re                               # use regular expressions to standardize authors (in future versions? This is presently done by hand.)
#import sys                              # take input from command line (in future versions?)

## GLOBAL VARIABLES & FUNCTIONS
totalEpigraphCount = 0                   #number of epigraphs in xml files in corpus
epigraphlessFileCount = 0                #number of xml files in corpus that do not have epigraphs

def remove_characters(listofstrings, characters_to_be_removed):
    for string in range(0,len(listofstrings)):
        cleaned_text = ""
        for character in listofstrings[string]:
            if character not in characters_to_be_removed:
                cleaned_text += character 
        listofstrings[string] = cleaned_text 
    return listofstrings 


## COLLECTING INFORMATION FROM CORPUS 
allFilesInDirectory = [ filename for filename in listdir(getcwd()) if filename.endswith('.xml')] #get filenames in current directory ending in ".xml"
for document in range(0, len(allFilesInDirectory)):                   # Loop through all files in directory
    root, ext = os.path.splitext(allFilesInDirectory[document])       # Select file extension for particular file "x" in the list "allFilesInDirectory"
    if (ext == '.xml'):                                               # If file ends in ".xml", read file. Skip file otherwise. 
    # open file to be read
        readfile = open(str(allFilesInDirectory[document]))	          # Specify file to be read & open file
        soup = BeautifulSoup(readfile)                                # Make "soup" object of file to search 
    
    # collect novel author, title of novel, pub date, epigraph, epigraph attrib, pub location, publisher, & encoding company from individual file
        author_list = [author.text for author in soup('author')]   # (1) collect text "author" entries
        title_list = [title.text for title in soup('title')]       # (2) collect text "title" entries
        
        publication_date = [date.text for date in soup('date')]    # (3) collect text pub year entries
        if "eaf" in root:                                          ### select correct year depending on EAF or Wright corpus. Throw warning if not one of these two corpora.
                pub_year = str(publication_date[1])                ### pick 2nd date tag for EAF corpus
        else:
            if "VAC" in root:
                pub_year = str(publication_date[0])                ### pick 1st date tag for Wright corpus
            else: 
                pub_year = 'Unknown Corpus, see terminal warning'  ### WARNING: user must check pub year entry
                print('WARNING: Check publication year for file ' + root +'.'+ ext + '\n') 
                print('List of publication dates in file: \n')
                print(publication_date)
        
        publication_place = [pubplace.text for pubplace in soup('pubplace')]  #(4) collect text pub location
        if len(soup('epigraph')) > 0:                                         #(5) collect entries tagged "epigraph"
            epigraph_list = [epigraph.text for epigraph in soup('epigraph')]  
            epigraph_attribution = ["No Attribution" if soup('epigraph')[epigraphs].bibl == None \
                                                 else  soup('epigraph')[epigraphs].bibl.text \
                                                 for epigraphs in range(0,len(soup('epigraph')))] #(6) collect epigraph attributions
            #see how many quote tags are nested in epigraph tags (for error checking; c.f. line 102)
            quote_tags_in_epigraph = [0 if soup('epigraph')[epigraphs].quote == None \
                                                 else 1 for epigraphs in range(0,len(soup('epigraph')))] # how often is quote tag appearing in epigraph tag? (used to help hunt for untagged epigraphs in corpus)
        else: 
            epigraph_list = ['No Epigraphs']
            epigraph_attribution = ['No Epigraphs']
        if len(soup('publisher')) > 0:         
            publishers = [publisher.text for publisher in soup('publisher')]
        else: 
            publishers = ['Unknown Publisher', 'Unknown Publisher','Unknown Publisher']

    # (7) identify company/individuals that produced each xml file (for exploring provenance of corpus)
        encoders = []
        encoding_counter = 0

        ## for Early American Fiction corpus ...
        if root[:3] == 'eaf': #Not ideal way to handle this, but encoder always 1st 'name' tag in EAF files
            encoders.append(soup('name')[0].text)
            encoding_counter = 1
        
        ## for Wright American Fiction corpus ...
        if root[:3] == 'VAC':            # Wright American Fiction corpus files begin with "VAC" 
            encoders = [soup('change')[encoder].get('who') for encoder in range(0,len(soup('change')))] #get encoders from 'who' attrs in 'change' tags
            encoding_counter = 1

            ### remove duplicate encoders entries, if present
            duplicate_list = []
            for x in range(len(encoders)): #find duplicates and place in 'duplicate_list'
                    if x != 0:
                        if (encoders[0] == encoders[x]):
                            duplicate_list.append(x)
            for deletions in range(len(duplicate_list)): # mark duplicates by replacing element with 'To Erase'
                encoders[duplicate_list[deletions]] = "To Erase"
            for deletions in range(len(encoders)-1,0,-1): # delete duplicate entries
                if encoders[deletions] == "To Erase":
                    del encoders[deletions]

        if encoding_counter == 0:
            print('WARNING: No case selected for encoder attribution for ' + root + '. Check file.')

        if len(encoders) == 0:
            print('WARNING: no encoder info found for ' + root + '. Check file.')  

    # (8) identify epigraphs with 'quote' tag & tracking of who did encoding (see also lines 47-50)  
        total_epigraph_tags = str(len(soup('epigraph')))        # number of tagged "epigraph"s in file
        total_quote_tags = str(len(soup('quote')))              # number of tagged "quote"s in file
        quotes_in_epigraphs =   # number of "quote"s in "epigraph"s      

    ## CLEANING INFORMATION COLLECTED FROM CORPUS
    # remove "/n" characters
        epigraph_attribution = remove_characters(epigraph_attribution, '-\n')
        author_list = remove_characters(author_list, '\n')
        title_list = remove_characters(title_list, '\n')
        publication_place = remove_characters(publication_place, '\n')
        publishers = remove_characters(publishers, '\n')
        pub_year = remove_characters([pub_year], '\n')[0]
        encoders = str(remove_characters(encoders, '\n'))                                  
    
    # standardize names in author list (Will do this manually for corpus...)
    # generate a dict for first and last names based on corpus entries for XML texts (Will do this manually for corpus...)

        readfile.close() #close file "x"

# Error Checking Print-To-Terminal: print all information collected
        if (len(soup('epigraph')) == 0):                         #check if file has epigraphs                
            if (len(soup('author')) > 0):
                epigraphlessFileCount += 1     
            else:
                if (len(soup('author')) == 0):
                    author_list = ['NO AUTHOR TAG IN FILE. CHECK XML FILE!']
                    epigraphlessFileCount += 1 
        else:
            for i in range(0, len(soup.findAll('epigraph'))):          
                if (len(soup.findAll('author')) == 0):
                   totalEpigraphCount += 1
                else:    
                    totalEpigraphCount += 1
<commit_msg>finished quote-to-epigraph comparison, but it's not yet providing the correct quote & epigraph tag counts<commit_after>#For use with python 3.x.
# Program only checks xml files in directory that contains this python file.
# Delete all csv files before running script. Script appends to csv files of 
# the same name if they already exist.

# libraries
from bs4 import BeautifulSoup            # select XML tags and parse text
from os import walk, getcwd, listdir     # used to grab all files in directory of script (c.f. line 29)
import os                                # used to split off filename root from filename extension (c.f. line 31)
import csv                               # interact with csv files
#import re                               # use regular expressions to standardize authors (in future versions? This is presently done by hand.)
#import sys                              # take input from command line (in future versions?)

## GLOBAL VARIABLES & FUNCTIONS
totalEpigraphCount = 0                   #number of epigraphs in xml files in corpus
epigraphlessFileCount = 0                #number of xml files in corpus that do not have epigraphs

def remove_characters(listofstrings, characters_to_be_removed):
    for string in range(0,len(listofstrings)):
        cleaned_text = ""
        for character in listofstrings[string]:
            if character not in characters_to_be_removed:
                cleaned_text += character 
        listofstrings[string] = cleaned_text 
    return listofstrings 


## COLLECTING INFORMATION FROM CORPUS 
allFilesInDirectory = [ filename for filename in listdir(getcwd()) if filename.endswith('.xml')] #get filenames in current directory ending in ".xml"
for document in range(0, len(allFilesInDirectory)):                   # Loop through all files in directory
    root, ext = os.path.splitext(allFilesInDirectory[document])       # Select file extension for particular file "x" in the list "allFilesInDirectory"
    if (ext == '.xml'):                                               # If file ends in ".xml", read file. Skip file otherwise. 
    # open file to be read
        readfile = open(str(allFilesInDirectory[document]))	          # Specify file to be read & open file
        soup = BeautifulSoup(readfile)                                # Make "soup" object of file to search 
    
    # collect novel author, title of novel, pub date, epigraph, epigraph attrib, pub location, publisher, & encoding company from individual file
        author_list = [author.text for author in soup('author')]   # (1) collect text "author" entries
        title_list = [title.text for title in soup('title')]       # (2) collect text "title" entries
        
        publication_date = [date.text for date in soup('date')]    # (3) collect text pub year entries
        if "eaf" in root:                                          ### select correct year depending on EAF or Wright corpus. Throw warning if not one of these two corpora.
                pub_year = str(publication_date[1])                ### pick 2nd date tag for EAF corpus
        else:
            if "VAC" in root:
                pub_year = str(publication_date[0])                ### pick 1st date tag for Wright corpus
            else: 
                pub_year = 'Unknown Corpus, see terminal warning'  ### WARNING: user must check pub year entry
                print('WARNING: Check publication year for file ' + root +'.'+ ext + '\n') 
                print('List of publication dates in file: \n')
                print(publication_date)
        
        publication_place = [pubplace.text for pubplace in soup('pubplace')]  #(4) collect text pub location
        if len(soup('epigraph')) > 0:                                         #(5) collect entries tagged "epigraph"
            epigraph_list = [epigraph.text for epigraph in soup('epigraph')]  
            epigraph_attribution = ["No Attribution" if soup('epigraph')[epigraphs].bibl == None \
                                                 else  soup('epigraph')[epigraphs].bibl.text \
                                                 for epigraphs in range(0,len(soup('epigraph')))] #(6) collect epigraph attributions
            
            ##see how many quote tags are nested in epigraph tags (for error checking; c.f. line 102)
            #print('bool check for ' + root)
            #print("epigraph: " + str(bool(soup('epigraph'))))
            #print("quote: " + str(bool(soup('quote'))))

            if bool(soup('epigraph')) & bool(soup('quote')) == True :  # don't check if there are zero "epipgraph" and/or "quote" tags
                quote_tags_in_epigraph = [0 if soup('epigraph')[epigraphs].quote == None \
                                         else 1 for epigraphs in range(0,len(soup('epigraph')))] # how often is quote tag appearing in epigraph tag? (used to help hunt for untagged epigraphs in corpus)
            else:
                quote_tags_in_epigraph = [0] # either no "quote" or "epigraph" tags or neither, so no quote-in-epigraph tags                             
        else: 
            epigraph_list = ['No Epigraphs']
            epigraph_attribution = ['No Epigraphs']

        if len(soup('publisher')) > 0:         
            publishers = [publisher.text for publisher in soup('publisher')]
        else: 
            publishers = ['Unknown Publisher', 'Unknown Publisher','Unknown Publisher']

    # (7) identify company/individuals that produced each xml file (for exploring provenance of corpus)
        encoders = []
        encoding_counter = 0

        ## for Early American Fiction corpus ...
        if root[:3] == 'eaf': #Not ideal way to handle this, but encoder always 1st 'name' tag in EAF files
            encoders.append(soup('name')[0].text)
            encoding_counter = 1
        
        ## for Wright American Fiction corpus ...
        if root[:3] == 'VAC':            # Wright American Fiction corpus files begin with "VAC" 
            encoders = [soup('change')[encoder].get('who') for encoder in range(0,len(soup('change')))] #get encoders from 'who' attrs in 'change' tags
            encoding_counter = 1

            ### remove duplicate encoders entries, if present
            duplicate_list = []
            for x in range(len(encoders)): #find duplicates and place in 'duplicate_list'
                    if x != 0:
                        if (encoders[0] == encoders[x]):
                            duplicate_list.append(x)
            for deletions in range(len(duplicate_list)): # mark duplicates by replacing element with 'To Erase'
                encoders[duplicate_list[deletions]] = "To Erase"
            for deletions in range(len(encoders)-1,0,-1): # delete duplicate entries
                if encoders[deletions] == "To Erase":
                    del encoders[deletions]

        if encoding_counter == 0:
            print('WARNING: No case selected for encoder attribution for ' + root + '. Check file.')

        if len(encoders) == 0:
            print('WARNING: no encoder info found for ' + root + '. Check file.')  

    # (8) identify epigraphs with 'quote' tag & tracking of who did encoding (see also lines 47-50)  
        total_epigraph_tags = str(len(soup('epigraph')))        # number of tagged "epigraph"s in file
        total_quote_tags = str(len(soup('quote')))              # number of tagged "quote"s in file
        #print(root)
        #print("epigraph: " + str(len(soup('epigraph'))) + "  " + str(bool(soup('epigraph'))))
        if bool(soup('epigraph')) == True:
        #print(quote_tags_in_epigraph)
            quotes_in_epigraphs = str(sum(quote_tags_in_epigraph))  # number of "quote"s in "epigraph"s  
        else:
            quotes_in_epigraphs = "0"    

    ## CLEANING INFORMATION COLLECTED FROM CORPUS
    # remove "/n" characters
        epigraph_attribution = remove_characters(epigraph_attribution, '-\n')
        author_list = remove_characters(author_list, '\n')
        title_list = remove_characters(title_list, '\n')
        publication_place = remove_characters(publication_place, '\n')
        publishers = remove_characters(publishers, '\n')
        pub_year = remove_characters([pub_year], '\n')[0]
        encoders = str(remove_characters(encoders, '\n'))                                  
    
    # standardize names in author list (Will do this manually for corpus...)
    # generate a dict for first and last names based on corpus entries for XML texts (Will do this manually for corpus...)

        readfile.close() #close file "x"

# Error Checking Print-To-Terminal: print all information collected
        if (len(soup('epigraph')) == 0):                         #check if file has epigraphs                
            if (len(soup('author')) > 0):
                epigraphlessFileCount += 1     
            else:
                if (len(soup('author')) == 0):
                    author_list = ['NO AUTHOR TAG IN FILE. CHECK XML FILE!']
                    epigraphlessFileCount += 1 
        else:
            for i in range(0, len(soup.findAll('epigraph'))):          
                if (len(soup.findAll('author')) == 0):
                   totalEpigraphCount += 1
                else:    
                    totalEpigraphCount += 1

{'fraction_non_alphanumeric': 0.09251524046060058, 'fraction_numerical': 0.006886430345450441, 'mean_word_length': 2.0557088651259057, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5157968', 'n_tokens_mistral': 4908, 'n_tokens_neox': 4545, 'n_words': 1630}