<jupyter_start><jupyter_text>example journal-article-10.2307_4214358.xml```xml<article xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"         xmlns:xlink="http://www.w3.org/1999/xlink"         xmlns:mml="http://www.w3.org/1998/Math/MathML"         dtd-version="1.0"         article-type="research-article">                  slaveasteurorev2         j101339                     The Slavonic and East European Review                              Maney Publishing                  00376795         22224327                              4214358                                    Language                                          Russian and Soviet Loanwords and Calques in the Czech Lexicon since the Beginning of the Twentieth Century                  ``` example journal-article-10.2307_4214358-ngram1.txt```textRussian	207czech	163soviet	111from	108were	98which	97s	86have	79lexical	72terms	67more	65other	60pp	59```<jupyter_code>hatebase = pd.read_csv('slavic_review_data/hatebase.csv')

hatebase
# Full list (but not very helpful)
hatebase = hatebase.term.str.lower().tolist() #all JSTOR terms are lowercase, so converting hatebase to lower
# A shorted list of term that are present in the corpus and suggest need for investigation
short_hatebase = [
"negro",
"nigger",
"retarded",
"yid",
"whore",
"retard",
"whores",
"kike",
"bitches",
"khokhol",
"redskin",
"faggot",
"khokhols",
"dykes",
"faggots",
"half breeds",
"darkie",
"sluts",
"tar baby",
"palefaces",
"yellows",
"zhidovka",
"homos",
"coons",
"fags",
"gey",
"suka",
"chukchi",
]
@dataclass
class Item:
    id: int = None
    author: str = None
    journal_title: str = None
    book_title: str = None
    book_chapter: str = None
    article_title: str = None
    year: str =None
    file: str = None
    collection: str = None
    jstor_url: str = None
    ngrams: List[tuple] = field(default_factory=list)
    
    #list of all tokens 
    def tokens(self):
        tokens = [a[0] for a in self.ngrams]
        return tokens
        
    #frequency for a specific token
    def token_freq(self, token:str):
        a = [a for a in self.ngrams if a[0] == token]
        return int(a[0][1])

    #frequencies for a list of token
    def token_list(self, tokens:list):
        return [a for a in self.ngrams if a[0] in tokens]
           
    def hatebase_terms(self):
        intersection = set([a[0].lower() for a in self.ngrams]).intersection(set(hatebase))
        return [a for a in self.ngrams if a[0].lower() in intersection] 
        
     
def make_items(path):
    current_dir = Path(path)
    return_items = []
    if (current_dir / 'metadata').exists():
        count = 0 
        for file in tqdm((current_dir / 'metadata').iterdir(), 
                         total=len(list((current_dir / 'metadata').iterdir()))):
            item = Item()
            item.id = count
            count += 1
            item.file = str(file.name)
            item.collection = str(current_dir).split('/')[-1]
            
            
            root = etree.fromstring(file.read_bytes())
            try:
                item.jstor_url = root.xpath('//self-uri')[0].attrib['{http://www.w3.org/1999/xlink}href']
            except IndexError:
                continue
            if 'article' in file.name:
                try:
                    item.journal_title = root.xpath('//journal-title')[0].text
                except IndexError:
                    continue
                if root.xpath('//copyright-year'):
                    item.year = root.xpath('//copyright-year')[0].text
                if root.xpath('//string-name'):
                    item.author = root.xpath('//string-name')[0].text #only first author
                if root.xpath('//article-title'): #TODO not working, no results
                    article_title = root.xpath('//article-title')[0].text 
            if 'book' in file.name:
                if root.xpath('//book-title'):
                    item.book_title = root.xpath('//book-title')[0].text
                book_part_id = file.stem.split('_')[-1]
                i = int(book_part_id.split('.')[-1])
                try:
                    item.book_chapter = root.xpath('//title')[i].text   
                except IndexError:
                    pass
            
            sub_dirs = ['ngram1','ngram2','ngram3']
            for sub_dir in sub_dirs:
                #book-chapter-10.2307_j.ctt1pc5dgp.4.xml maps to 
                #book-chapter-10.2307_j.ctt1pc5dgp.4-ngram1.txt
                txt_file = str(file.stem)+'-'+sub_dir+'.txt'
                ngram_file = (current_dir / sub_dir / txt_file )
                if ngram_file.exists():
                    text = ngram_file.read_text()
                    for line in text.split('\n'):
                        word=line.split('\t')[0]
                        freq=line.split('\t')[-1]
                        item.ngrams.append((word,freq))
            return_items.append(item)
            
        return return_items
                
                
    
    else:
        print('Missing metadata directory')
        raise SystemExit(0)
        
    
directories = ['/home/ajanco/projects/slavic_review/slavic_review_data/SlavicStudiesCluster1991to2020'] #,'/home/ajanco/projects/slavic_review/slavic_review_data/AfricanAmericanStudiesCluster1985to2020']
main_items = []
for directory in directories:
    main_items.extend(make_items(directory))

    
import pickle 
pickle.dump( main_items, open( "slav_stud_items.p", "wb" ) )


import pickle
main_items = pickle.load( open( "slav_stud_items.p", "rb" ) )
import gender_guesser.detector as gender
d = gender.Detector()
d.get_gender("Igal")

gender = [d.get_gender(a.author.split(' ')[0]) for a in main_items if a.author]

Counter(gender)
years = [a.year for a in main_items if a.year]
Counter(years)
authors = [a.author for a in main_items if a.author]
Counter(authors).most_common(50)
article_title = [a.article_title for a in main_items if a.article_title]
Counter(article_title).most_common()
data = []
for text in tqdm(main_items): 
    row = {}
    row['url'] = text.jstor_url
    row['file'] = text.file
    row['hate_terms'] = text.token_list(short_hatebase) #text.hatebase_terms()
    row['hate_tokens'] = [a[0] for a in row['hate_terms']]
    row['number_of_terms'] = len(row['hate_terms'])

    data.append(row)
df = pd.DataFrame(data)
pd.set_option('display.max_colwidth', -1)

def make_clickable(val):
    return '<a target="_blank" href="{}">{}</a>'.format(val,val)

df2= df[['url','file','hate_tokens','number_of_terms']].sort_values('number_of_terms', ascending=False).head(50)
df2.style.format({'url': make_clickable})
directories = ['/home/ajanco/projects/slavic_review/slavic_review_data/SlavicStudiesCluster1991to2020','/home/ajanco/projects/slavic_review/slavic_review_data/AfricanAmericanStudiesCluster1985to2020']
    
data = []
for directory in directories:
    current_dir = Path(directory)
    for file in tqdm((current_dir / 'metadata').iterdir(), total=len(list((current_dir / 'metadata').iterdir()))):
        row= {}
        row['category'] = str(current_dir).split('/')[-1]
        row['text'] = ''
        sub_dirs = ['ngram1','ngram2','ngram3']
        for sub_dir in sub_dirs:
            txt_file = str(file.stem)+'-'+sub_dir+'.txt'
            ngram_file = (current_dir / sub_dir / txt_file )
            if ngram_file.exists():
                text = ngram_file.read_text()
                for line in text.split('\n'):
                    word=line.split('\t')[0]
                    row['text'] += word + ' '

        data.append(row)
df = pd.DataFrame(data)
df.head()
import scattertext as st
import spacy 

# Turn it into a Scattertext Corpus 
nlp = spacy.load('en_core_web_md')
nlp.max_length = 2325753
corpus = st.CorpusFromPandas(df,
                             category_col='category', 
                             text_col='text',
                             nlp=nlp).build()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1315558893478533, 'fraction_numerical': 0.025660282888972336, 'mean_word_length': 2.2024048096192383, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '177133', 'license': 'MIT', 'n_tokens_mistral': 2735, 'n_tokens_neox': 2532, 'n_words': 517, 'path': '0_corpus_and_data/1_join_JSTOR_metadata_with_project_data.ipynb', 'repo_name': 'Russian-NLP/Reading-Racial-Discrimination-in-Slavic-Studies-Scholarship', 'size': '7989'}