<jupyter_start><jupyter_text>Tips on running the prune    1. Define the experiement in the block below    2. Define your experiment in the exp block (search "[DEFINE YOUR EXP HERE]")    3. Generate the sudo dataset and run by the "[RUN YOUR EXP HERE]" block    4. Make the exp results readable by running the "[MAKE THE EXP READABLE]" block<jupyter_code># Select the experiment type
exp = "wsc" # dev<jupyter_output><empty_output><jupyter_text>Helper Functions<jupyter_code>def load_dataset(filename):
    # read a winogrande style jsonl file
    # qID, sentence, option1, option2, answer
    this_set = []
    
    
    with open(filename,"r") as json_file:
        json_list = list(json_file)

        for json_str in json_list:
            result = json.loads(json_str)
            this_set.append(result)
    print("Loaded "+ filename + " with "+ str(len(this_set)) + " items.")
    return this_set
# prune a type of dep token

def prune_pos(text, target):
    doc = nlp(text)
    temp = []
    count = 0
    for token in doc:
        if token.text == "_":
            temp.append(token.text)
            
        elif token.dep_  in target:
            count = 1
            continue
        else:
            # print(token.dep_)
            temp.append(token.text)
            
    return " ".join(temp[:-1]) + ".", count

def generate_pos_set(all_sents, target):
    all_target = []
    
    for sent in all_sents:
        # print(sent)

        doc = nlp(sent)
        tars = [token.text for token in doc if token.pos_ == target]
        
#         for token in doc:
#             print(token.text,": ",token.pos_)
            
        try:
            tars.remove("_")
        except:
            a = "nothing"
            
        all_target.extend(tars)
        # print("_________________________________")
    
    return all_target
# Pruning function

def pruning_exp(wsc, prune_type, pruned_tokens, all_sents):
    # select type from normal, cand-names, cand-nonenames pos, dep
    # for cand
    pruned_wsc = []
    num_case = []
    
    if "cand" in prune_type:
        new_ADP = pruned_tokens # names

    for piece in wsc:
        # print(piece)
        new_piece = {}
        for key, value in piece.items():
            if key != "sentence":
                new_piece[key] = value
            else:
                temp = value
                if value[-1] != ".":
                    value += "."

                temp_case = 0
                
                if "cand" in prune_type:
                    # Prune the cands
                    pruned_tokens=[]
                    cand1 = piece["option1"]
                    cand2 = piece["option2"]
                    pruned_tokens.extend(tokenizer.tokenize(cand1))
                    pruned_tokens.extend(tokenizer.tokenize(cand2))
                    
                    names = []
                    for name in new_ADP:
                        try:
                            pruned_tokens.remove(name)
                            names.append(name)
                        except:
                            a = "nothing"
                    if "names" in prune_type:
                        pruned_tokens = names


                    if len(pruned_tokens) > 0:
                        temp_case = 1
                    
                if "cand" in prune_type or "pos" in prune_type or "normal" in prune_type:
                    # print("Pruning cand/pos/tokens")
                    temp = temp.replace(".", " [PERIOD]")
                    temp = temp.replace(",", " [COMMA]")
                    temp = temp.replace("?", " [QUESTION]")
                    temp = temp.replace("n't", " n't")
                    temp = temp.replace("n’t", " n't")
                    
                    
                    sent_tokens = temp.split(" ")
                    output_tokens = []
                    
                    for st in sent_tokens:
                        if st in pruned_tokens or st.lower() in pruned_tokens and "cand" not in prune_type:
                            temp_case = 1
                        else:
                            output_tokens.append(st)
                    
                    out_sent = " ".join(output_tokens)
                    new_piece[key] = out_sent.replace(" [PERIOD]", ".").replace(" [COMMA]", ",").replace(" [QUESTION]", "?").replace(" n't", "n't")

                    num_case.append(temp_case)
                
                if "dep" in prune_type:
                    # print("Pruning by dependency tags")
                    # special remove all tokens    
                    new_piece[key], temp_count=prune_pos(value.lower(), pruned_tokens)
                    num_case.append(temp_count)

        pruned_wsc.append(new_piece)

    return pruned_wsc, num_case
def generate_sudo_set(cover_set, output_folder):
    # generate the seleted/unselected set
    with open(output_folder+"/dev.jsonl","w") as f1:
        with open(output_folder+"/wsc.jsonl", "w") as f2:
            for i,piece in enumerate(cover_set):
                # if i in selected_ids:
                f1.write(json.dumps(piece))
                f1.write("\n")
                f2.write(json.dumps(piece))
                f2.write("\n")
               
            
def acquire_prediction(ori_pred_path, prune_pred_path, wsc, pruned_wsc, case_map):
    
    new_wsc = []
    
    with open(ori_pred_path) as f:
        ori_ans = json.load(f)
        
    with open(prune_pred_path) as f:
        prune_ans = json.load(f)
        
    for i, case in enumerate(case_map):
        temp = {}
        if case == 1:
            temp["qID"] = wsc[i]["qID"]
            temp["sentence"] = wsc[i]["sentence"]
            temp["pruned_sentence"] = pruned_wsc[i]["sentence"]
            temp["option1"] = wsc[i]["option1"]
            temp["option2"] = wsc[i]["option2"]
            temp["answer"] = wsc[i]["answer"]
            temp["ori_ans"] = ori_ans[i]
            temp["prune_ans"] = prune_ans[i]
            new_wsc.append(temp)
            
    return new_wsc


def write_readable_wsc(target, info, name):
    with open(name+".txt","w") as f:
        f.write("======================================================"+"\n")
        f.write("This is the output answer for the experiment:"+ info + "\n")
        f.write("======================================================"+ "\n" +"\n"+ "\n")
        
        for case in target:
            f.write("--------------------"+"qID: "+ case["qID"]+ "--------------------"+"\n")
            f.write("label: "+ str(case["answer"])+"\n")
            f.write("original_correctness: "+ str(case["ori_ans"])+"\n")
            f.write("pruned_correctness: "+ str(case["prune_ans"])+"\n")
            f.write("option1: "+ str(case["option1"])+"\n")
            f.write("option2: "+ str(case["option2"])+"\n"+"\n")
            
            f.write("ori sentence: "+"\n")
            f.write(case["sentence"]+"\n"+"\n")
            f.write("pruned sentence: "+"\n")
            f.write(case["pruned_sentence"]+"\n"+"\n")
            f.write("----------------------------------------------------------------"+"\n"+"\n")
            
    with open(name+".json","w") as f:
        json.dump(target, f)
        
    return True<jupyter_output><empty_output><jupyter_text>Running the experiments<jupyter_code>if exp == "wsc":
    wsc = load_dataset("./data/wsc.jsonl")
elif exp == "dev":
    wsc = load_dataset("./data/dev.jsonl")
# xs = load_dataset("train_xs.jsonl")
# s = load_dataset("train_s.jsonl")
# m = load_dataset("train_m.jsonl")
# l = load_dataset("train_l.jsonl")
# xl = load_dataset("train_xl.jsonl")
all_sents = []
for i in range(len(wsc)):
    print(wsc[i])
    all_sents.append(wsc[i]["sentence"])
# find the most frequent tokens
from collections import Counter

words = []

for sent in all_sents:
    temp_words = tokenizer.tokenize(sent)
    for temp_word in temp_words:
        
        if temp_word not in sw:
            words.append(temp_word)
        #words.append(temp_word)

counter = Counter()
counter.update(words)
most_common = counter.most_common(100)
print(most_common)
# Generate the pos tokens to be pruned
pos_det = ["his","her","my", "our","your","their","its"]
pos_det += ["His","Her","My", "Our","Your","Their","Its"]

# all_ADP = generate_pos_set(all_sents, "PUNCT")
# all_ADP = generate_pos_set(all_sents, "VERB")
# all_ADP = generate_pos_set(all_sents, "PRON")

all_ADP = generate_pos_set(all_sents, "ADJ")
all_ADP = list(filter(lambda a: a != "that", all_ADP))
for possessive_determiner in pos_det:
    all_ADP = list(filter(lambda a: a != possessive_determiner, all_ADP))

# all_ADP = generate_pos_set(all_sents, "ADV")
# all_ADP = list(filter(lambda a: a != "n't", all_ADP))
# all_ADP = list(filter(lambda a: a != "not", all_ADP))
# all_ADP = list(filter(lambda a: a != "n’t", all_ADP))

# all_ADP = generate_pos_set(all_sents, "ADP")
# all_ADP = generate_pos_set(all_sents, "NOUN")

# all_ADP = generate_pos_set(all_sents, "PROPN")
# all_ADP.append('Kamchatka')

pruned_tokens = all_ADP
print(pruned_tokens)
# pruned_tokens = []

# pruned_tokens = [","]
# pruned_tokens = ["."]
# pruned_tokens = ["the", "The"]
# pruned_tokens = ["a", "A"]
# pruned_tokens = ["this", "that", "these", "those", "This", "That", "These", "Those"]
# pruned_tokens = ["has", "have", "had", "Has", "Have", "Had"]
# pruned_tokens = ["Because", "because"]
# # pruned_tokens = ["go", "going", "went", "gone", "goes"]
# pruned_tokens = ["but", "But"]
# pruned_tokens = ["though", "although", "Though", "Although","since", "Since"] 
# pruned_tokens = ["but", "But", "and", "And"]
# pruned_tokens = ["very", "Very"]
# pruned_tokens = ["so", "So"]
# pruned_tokens = ["in", "In"]
# pruned_tokens = ["of", "Of"]
# pruned_tokens = ["on", "On"]
# pruned_tokens = ["with", "With"]

# pruned_tokens = ["wanted", "Wanted"]
# pruned_tokens = ["new", "New"]
# pruned_tokens = ["instead", "Instead"]
# pruned_tokens = ["get", "Get"]
# pruned_tokens = ["could", "Could"]
# pruned_tokens = ["always", "Always"]
# pruned_tokens = ["used", "Used"]


# pruned_tokens = ["and", "or", "but", "as", "because", "for", "just as", "neither", "nor", "not only", "so", "whether", "yet"]
# pruned_tokens.extend(["And", "Or", "But", "As", "Because", "For", "Just as", "Neither", "Nor", "Not only", "So", "Whether", "Yet"])
# pruned_tokens = ["dobj", "pobj"]
# pruned_tokens = ["nsubj"]
# pruned_tokens = ["ROOT"]
# pruned_tokens = ["det"]
# pruned_tokens = ["amod", "advmod", "npadvmod"]
# pruned_tokens = ["neg"]
# select type from normal, cand-names, cand-other pos, dep
# when the type is dep, the pruned tokens is the dep tokens that you want to prune

########################## [DEFINE YOUR EXP HERE] ###############################

pruned_wsc, case_map = pruning_exp(wsc, "normal", pruned_tokens, all_sents)
print(len(wsc))
print(len(pruned_wsc))
print(sum(case_map))
# Find the acc for the covered subset from original data evaluation
if exp == "dev":
    with open("./data/grande_dev.json") as f:
        roberta_ans = json.load(f)

if exp == "wsc":
    with open("./data/grande_wsc.json") as f:
        roberta_ans = json.load(f)
    
    
partial_roberta = []

gcovered = 0
rcovered = 0


    
for i, case in enumerate(case_map):
    if case == 0:
        continue

    partial_roberta.append(roberta_ans[i])
    rcovered += 1

print(np.sum(partial_roberta)/rcovered)
# generate the pruned dataset for further prediction
generate_sudo_set(pruned_wsc, "./data/prune")
#################### [RUN YOUR EXP HERE] ##############################

!CUDA_VISIBLE_DEVICES=1 python ./scripts/run_experiment.py \
 --model_type roberta_mc \
 --model_name_or_path ./output/xl \
 --task_name winogrande \
 --do_eval \
 --do_lower_case \
 --data_dir ./data/prune \
 --max_seq_length 80 \
 --per_gpu_eval_batch_size 4 \
 --per_gpu_train_batch_size 4 \
 --learning_rate 0 \
 --output_dir ./output/prune/ \
 --logging_steps 4752 \
 --save_steps 4750 \
 --seed 42 \
 --data_cache_dir ./output/prune \
 --warmup_pct 0.1 \
 --evaluate_during_training
if exp == "dev":
    ori_pred_path = "./data/grande_dev.json"
if exp == "wsc":
    ori_pred_path = "./data/grande_wsc.json"
    
prune_pred_path = "./output/prune/correctness_dev.json"
# prune_pred_path = "./output/truncate/correctness_dev.json"

# pruned_wsc = load_dataset("./data/truncate/dev.jsonl")
# print(len(pruned_wsc))

output_wsc = acquire_prediction(ori_pred_path, prune_pred_path, wsc, pruned_wsc,case_map)

print(len(output_wsc))
###################### [MAKE THE EXP READABLE] ##############################

info = "Prune the main sentence"
# name = "./readable_pruned/truncate"
name = "./readable_pruned_dev/truncate"
write_readable_wsc(output_wsc, info, name)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17414103467167566, 'fraction_numerical': 0.0046959380136182205, 'mean_word_length': 2.294997421351212, 'pattern_counts': {'":': 10, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '284550', 'license': 'MIT', 'n_tokens_mistral': 4399, 'n_tokens_neox': 4149, 'n_words': 908, 'path': 'hard_PCR (WSC)/winogrande/wsc_pruning_exp.ipynb', 'repo_name': 'HKUST-KnowComp/PCR', 'size': '12777'}