<jupyter_start><jupyter_text>arxiv<jupyter_code>import pandas as pd
import json
data=[]
with open('../csci-544-project/data/longformer_arxiv.txt') as f:
    text = f.readline()
    while text:
        json_data = json.loads(text)
        data.append(json_data)
        text = f.readline()
    final_df = pd.DataFrame(data)
final_df 
from torch.utils.data import Dataset, DataLoader
from datasets import load_metric
metric = load_metric("rouge")
# using GPT-3 model with pred of longformer
import os
import openai
from tqdm import tqdm
openai.organization = "org-EisHcNn58Kak28bngV5bsOuz"
openai.api_key = "<KEY>"
engine_list = openai.Engine.list() 
#openai.api_key = os.getenv("OPENAI_API_KEY")
output = []
#final_output = []
for i in tqdm(range(len(final_df))):
    try:
        beginning_tag = "Original:\n"
        ending_tag = "\n````\nPolished Sentence:"
        text =  beginning_tag + str(final_df["predicted_abstract"].iloc[i])+ ending_tag
        response = openai.Completion.create(
          engine="curie",
          prompt=text,
          temperature=0,
          max_tokens=500,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0,
          stop=["````"]
        )
        output.append(response["choices"][0]['text'])
    except:# if the token exceed the maximum tokens number in GPT-3 model,then just use first half text
        beginning_tag = "Original:\n"
        ending_tag = "\n````\nPolished Sentence:"
        a = final_df["predicted_abstract"].iloc[i][:int(len(final_df["predicted_abstract"].iloc[i])/2)]
        text = beginning_tag + str(a)+ ending_tag
        response = openai.Completion.create(
          engine="curie",
          prompt=text,
          temperature=0,
          max_tokens=500,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0,
          stop=["````"]
        )
        output.append(response["choices"][0]['text'])
#final_output.append(output)
pred_df = pd.DataFrame()
pred_df["summary"] = output 
pred_df.to_csv("arxiv_after_longformer_summary_100.csv",sep = ' ',index = False,header = False)
from torch.utils.data import Dataset, DataLoader
from datasets import load_metric
metric = load_metric("rouge")
fake_preds = output
fake_labels = final_df['abstract']
rouge_raw_after_longformer = metric.compute(predictions=fake_preds, references=fake_labels)
rouge_raw_after_longformer
# using GPT-3 model with pubmed bigbird pred text with summary method
import os
import openai
from tqdm import tqdm
openai.organization = "org-EisHcNn58Kak28bngV5bsOuz"
openai.api_key = "<KEY>"
engine_list = openai.Engine.list() 
#openai.api_key = os.getenv("OPENAI_API_KEY")
output = []
#final_output = []
for i in tqdm(range(len(final_df))):
    try:
        beginning_tag = "My second grader asked me what this passage means:\n\"\"\"\n"
        ending_tag = "\n\"\"\"\nI rephrased it for him, in plain language a second grader can understand:\n\"\"\"\n"
        text =  beginning_tag + str(final_df["predicted_abstract"].iloc[i])+ ending_tag
        response = openai.Completion.create(
          engine="curie",
          prompt=text,
          temperature=0,
          max_tokens=500,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0,
          stop=["\"\"\""]
        )
        output.append(response["choices"][0]['text'])
    except:# if the token exceed the maximum tokens number in GPT-3 model,then just use first half text
        beginning_tag = "My second grader asked me what this passage means:\n\"\"\"\n"
        ending_tag = "\n\"\"\"\nI rephrased it for him, in plain language a second grader can understand:\n\"\"\"\n"
        a = final_df["predicted_abstract"].iloc[i][:int(len(final_df["predicted_abstract"].iloc[i])/2)]
        text = beginning_tag + str(a)+ ending_tag
        response = openai.Completion.create(
          engine="curie",
          prompt=text,
          temperature=0,
          max_tokens=500,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0,
          stop=["\"\"\""]
        )
        output.append(response["choices"][0]['text'])
#final_output.append(output)
pred_df = pd.DataFrame()
pred_df["summary"] = output 
pred_df.to_csv("pubmed_after_bigbird_summary_100.csv",sep = ' ',index = False,header = False)
fake_preds = output
fake_labels = final_df['abstract']
rouge_pubmed_raw_after_longformer = metric.compute(predictions=fake_preds, references=fake_labels)
rouge_pubmed_raw_after_longformer<jupyter_output><empty_output><jupyter_text>pubmed<jupyter_code>data=[]
with open('../csci-544-project/data/longformer_pubmed.txt') as f:
    text = f.readline()
    while text:
        json_data = json.loads(text)
        data.append(json_data)
        text = f.readline()
    final_df = pd.DataFrame(data)
final_df 
# using GPT-3 model with pubmed pred of longformer
import os
import openai
from tqdm import tqdm
openai.organization = "org-EisHcNn58Kak28bngV5bsOuz"
openai.api_key = "<KEY>"
engine_list = openai.Engine.list() 
#openai.api_key = os.getenv("OPENAI_API_KEY")
output = []
#final_output = []
for i in tqdm(range(len(final_df))):
    try:
        beginning_tag = "Original:\n"
        ending_tag = "\n````\nPolished Sentence:"
        text =  beginning_tag + str(final_df["predicted_abstract"].iloc[i])+ ending_tag
        response = openai.Completion.create(
          engine="curie",
          prompt=text,
          temperature=0,
          max_tokens=500,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0,
          stop=["````"]
        )
        output.append(response["choices"][0]['text'])
    except:# if the token exceed the maximum tokens number in GPT-3 model,then just use first half text
        beginning_tag = "Original:\n"
        ending_tag = "\n````\nPolished Sentence:"
        a = final_df["predicted_abstract"].iloc[i][:int(len(final_df["predicted_abstract"].iloc[i])/2)]
        text = beginning_tag + str(a)+ ending_tag
        response = openai.Completion.create(
          engine="curie",
          prompt=text,
          temperature=0,
          max_tokens=500,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0,
          stop=["````"]
        )
        output.append(response["choices"][0]['text'])
#final_output.append(output)
pred_df = pd.DataFrame()
pred_df["summary"] = output 
pred_df.to_csv("pubmed_after_longformer_summary_100.csv",sep = ' ',index = False,header = False)
fake_preds = output
fake_labels = final_df['abstract']
rouge_pubmed_raw_after_longformer = metric.compute(predictions=fake_preds, references=fake_labels)
rouge_pubmed_raw_after_longformer
# using GPT-3 model with pubmed bigbird pred text with summary method
import os
import openai
from tqdm import tqdm
openai.organization = "org-EisHcNn58Kak28bngV5bsOuz"
openai.api_key = "<KEY>"
engine_list = openai.Engine.list() 
#openai.api_key = os.getenv("OPENAI_API_KEY")
output = []
#final_output = []
for i in tqdm(range(len(final_df))):
    try:
        beginning_tag = "My second grader asked me what this passage means:\n\"\"\"\n"
        ending_tag = "\n\"\"\"\nI rephrased it for him, in plain language a second grader can understand:\n\"\"\"\n"
        text =  beginning_tag + str(final_df["predicted_abstract"].iloc[i])+ ending_tag
        response = openai.Completion.create(
          engine="curie",
          prompt=text,
          temperature=0,
          max_tokens=500,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0,
          stop=["\"\"\""]
        )
        output.append(response["choices"][0]['text'])
    except:# if the token exceed the maximum tokens number in GPT-3 model,then just use first half text
        beginning_tag = "My second grader asked me what this passage means:\n\"\"\"\n"
        ending_tag = "\n\"\"\"\nI rephrased it for him, in plain language a second grader can understand:\n\"\"\"\n"
        a = final_df["predicted_abstract"].iloc[i][:int(len(final_df["predicted_abstract"].iloc[i])/2)]
        text = beginning_tag + str(a)+ ending_tag
        response = openai.Completion.create(
          engine="curie",
          prompt=text,
          temperature=0,
          max_tokens=500,
          top_p=1,
          frequency_penalty=0,
          presence_penalty=0,
          stop=["\"\"\""]
        )
        output.append(response["choices"][0]['text'])
#final_output.append(output)
pred_df = pd.DataFrame()
pred_df["summary"] = output 
pred_df.to_csv("pubmed_after_bigbird_summary_100.csv",sep = ' ',index = False,header = False)
fake_preds = output
fake_labels = final_df['abstract']
rouge_pubmed_raw_after_longformer = metric.compute(predictions=fake_preds, references=fake_labels)
rouge_pubmed_raw_after_longformer
report_df = pd.DataFrame(columns = ['data_resource', 'longformer_rough_1','longformer_rough_2','longformer_rough_L'])
arxiv = {'data_resource':'arxiv','longformer_rough_1':34.69,'longformer_rough_2':15.24,'longformer_rough_L':21.52}
pubmed = {'data_resource':'pubmed','longformer_rough_1':35.73,'longformer_rough_2':14.67,'longformer_rough_L':22.44}

report_df = report_df.append(arxiv, ignore_index = True)
report_df = report_df.append(pubmed, ignore_index = True)
report_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1323831119749487, 'fraction_numerical': 0.015549076773566569, 'mean_word_length': 3.3626943005181347, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '508068', 'license': 'MIT', 'n_tokens_mistral': 3164, 'n_tokens_neox': 3088, 'n_words': 587, 'path': 'notebooks/GPT-3_after_longformer.ipynb', 'repo_name': 'Anthonyive/csci-544-project', 'size': '9445'}