<jupyter_start><jupyter_text>AboutThis script uses stanford named entity recognizer to find the proper names in my women article corpus and exports a new csv column with the article text without proper nouns. At the end, it also tries to do a little analysis with those named entites.  NER<jupyter_code>import ner
import os
import re
import csv
from urllib import urlopen
import sys

csv.field_size_limit(sys.maxsize)
 
tagger = ner.SocketNER(host='localhost', port=8080)
# test
entities = tagger.get_entities("The dictatorship of President <NAME> caused extreme hardships \
                                for all but a few hundred thousand of Rumania's 23 million citizens. \
                                But in the case of mothers and babies, his rule apparently had the most \
                                tragic consequences. Mr. Ceausescu, who was ousted in a popular uprising a \
                                month ago, decreed in 1967, two years after he came to power, that Rumania's \
                                population, then about 22 million, should increase to 30 million. The reason he \
                                gave was simply that he wanted a bigger Rumania - an assertion widely interpreted \
                                now as an early indication of his megalomania. And to achieve his goal he banned \
                                abortions, made contraception illegal and ordered that Rumanian women of \
                                child-bearing age have five children each. No Precise Accounting Harsh fines were \
                                ordered for women caught having abortions, and doctors or medical technicians who \
                                a consultant for M<NAME> International, a British charity that assi")
entities
for key in entities:
    entities[key] = set(entities[key])
entities
#read csv and read into a list of dictionaries
women = []
with open('Data/Corpora/women-foreign.csv') as csvfile:
    reader = csv.DictReader(csvfile)
    for row in reader:
        women.append(row)
women[1]
# remove named entities
for article in women:
    entities = tagger.get_entities(article['TEXT'])
    for key in entities:
        entities[key] = set(entities[key])
    article['entities'] = entities
    entities = [item for sublist in entities.values() for item in sublist]
    text = article['TEXT']
    for noun in entities:
        text = text.replace(noun.encode('utf-8'),'')
    article['TEXT-NO-NOUN'] = text<jupyter_output><empty_output><jupyter_text>Write File<jupyter_code>keys = women[1].keys()
keys
with open('Data/Corpora/women-processed.csv', 'wb') as output_file:
    dict_writer = csv.DictWriter(output_file, keys)
    dict_writer.writeheader()
    dict_writer.writerows(women)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.06818993216708319, 'fraction_numerical': 0.006069260978222063, 'mean_word_length': 2.515683814303639, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '606893', 'license': 'MIT', 'n_tokens_mistral': 761, 'n_tokens_neox': 721, 'n_words': 290, 'path': '02_NER.ipynb', 'repo_name': 'rochelleterman/worlds-women', 'size': '2818'}