<jupyter_start><jupyter_text>Picking adjectives<jupyter_code>words=[]
for item in vocab_sorted.items():
    if re.match('[a-z]{2,}$',item[0]):
        words.append(item[0])
len(words)
nouns = []
adjs = []
for ix,word in enumerate(words):
    if nlp(word)[0].pos_ == 'NOUN' and len(nouns) < 1000:
        nouns.append(nlp(word)[0].text)
    elif nlp(word)[0].pos_ == 'ADJ' and len(adjs) < 2000:
        adjs.append(nlp(word)[0].text)<jupyter_output><empty_output><jupyter_text>Finding gradable adjectives<jupyter_code>from collections import defaultdict
import textacy
import textacy.datasets
cw = textacy.datasets.CapitolWords()
cw.download()
adjectives_encountered = []
unique_adjectives_encountered = set()
for text,record in cw.records():
    processed = nlp(text)
    
    adjectives_encountered += [token for token in processed if token.text in adjs]
    
    for token in processed:
        if token.text in adjs:
            unique_adjectives_encountered |= set([token.text])
len(adjectives_encountered),len(unique_adjectives_encountered)
gradable = defaultdict(int)
non_gradable = defaultdict(int)
modifiers = ['somewhat','very','really','extremely','rather']
for adj in adjectives_encountered:
    if len([x for x in adj.children if x.text in modifiers])>0:
        gradable[adj.text] += 1
    else:
        non_gradable[adj.text]+=1
combined = defaultdict(list)
for adj in unique_adjectives_encountered:
    toAdd = []

    toAdd.append(gradable[adj])
    toAdd.append(non_gradable[adj])
    combined[adj] = toAdd
adjs = defaultdict(list)
for adj in combined:
    occurences = sum(combined[adj])
    gradability_score = round(float((combined[adj][0])/occurences) * 100, 3)
    if occurences > 100 and gradability_score > 0.5:
        adjs[adj] = gradability_score
adjs = [k for k, v in sorted(adjs.items(), key=lambda item: item[1],reverse=True)][:200]
with open('gradable_adjectives.txt', 'w') as f:
    for item in adjs:
        f.write("%s\n" % item)<jupyter_output><empty_output><jupyter_text>Generating sentences<jupyter_code>sentences = []
for noun in nouns:
    for adj in adjs:
        sentences.append('The '+noun+' is '+adj+'.')
        sentences.append('The '+noun+' are '+adj+'.')
len(sentences)
sentences[:10]<jupyter_output><empty_output><jupyter_text>Filtering by GPT perplexity<jupyter_code>from pytorch_pretrained_bert import GPT2LMHeadModel, GPT2Tokenizer
device = torch.device('cuda:0')
model_id = 'gpt2'
model_gpt = GPT2LMHeadModel.from_pretrained(model_id).to(device)
tokenizer_gpt = GPT2Tokenizer.from_pretrained(model_id)
def process_gpt(sentence):
    tokens = ["[CLS]"] + tokenizer_gpt.tokenize(sentence)
    tokens_ids = tokenizer_gpt.convert_tokens_to_ids(tokens)
    tokens_ids = torch.tensor([tokens_ids,], dtype=torch.long).to(device)
    with torch.no_grad():
        outputs = model_gpt(tokens_ids, lm_labels=tokens_ids)
        log_likelihood = outputs.item()
    return np.exp(log_likelihood) 
pairs = {}
for sentence in sentences:
    pairs[sentence] = process_gpt(sentence)
df = pd.DataFrame.from_dict(pairs, orient='index').reset_index()
df = df.rename(columns={"index": "sentence", 0: "perplexity"})
df.sort_values(by='perplexity', ascending=True)
df.sort_values(by='perplexity', ascending=True).to_csv('/home/lisa/hobbies/modifiers_all.csv')
df.sort_values(by='perplexity', ascending=True).head(10000).to_csv('/home/lisa/hobbies/modifiers_top10k.csv')<jupyter_output><empty_output><jupyter_text>Producing negative vs positive sentence pairs<jupyter_code>ten_k = df.sort_values(by='perplexity', ascending=True).head(10000)
pos_10k = []
neg_10k = []
for sentence in ten_k['sentence'].values:
    words = sentence.split(' ')
    aff = ' '.join(words[:3]+['[MASK]']+words[3:])    
    if words[2] == 'is':
        neg = ' '.join(words[:2]+["isn't [MASK]"]+words[3:])
    else:
        neg = ' '.join(words[:2]+["aren't [MASK]"]+words[3:])
    pos_10k.append(aff)
    neg_10k.append(neg)
with open('10k_aff.txt', 'a') as f:
    for sentence in pos_10k:
        f.write(sentence+'\n')
        
with open('10k_neg.txt', 'a') as f:
    for sentence in neg_10k:
        f.write(sentence+'\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15088899567515618, 'fraction_numerical': 0.01826045170591062, 'mean_word_length': 4.9133522727272725, 'pattern_counts': {'":': 1, '<': 23, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '33496', 'license': 'MIT', 'n_tokens_mistral': 1594, 'n_tokens_neox': 1531, 'n_words': 271, 'path': 'code_and_data/making_10k_sentences.ipynb', 'repo_name': 'altsoph/artificial_degree_modifiers', 'size': '4162'}