<jupyter_start><jupyter_text>Making the train , test , Validation_test_alike<jupyter_code>'''
with open('train_data.json', encoding='utf-8') as f:
                foo = json.load(f)

                for key in foo.keys():
                    for info in foo[key]['info']:
                        sentence=foo[key]['sentence']
                        #print(info['target']+ " ## "+ sentence)
                        #print(sentence)
                        sentence = [clean_sentence(x) for x in sentence.split(" ")]
                        #print(sentence)
                        
                        sentence=' '.join(sentence)
                        #print(sentence)
                        
                        sentence=' '.join(sentence.split())
                        
                        target= info['target']
                        #print("Target = "+ target)
                        
                         
                        print(info['target'].lower()+ " ## "+ sentence)
                        text_left, _, text_right = [s.lower().strip() for s in sentence.partition(target)]
                        #print( text_left + text_right)
                        text_raw_indices =tokenizer.text_to_sequence(text_left + " " + target + " " + text_right)
                        #print(text_raw_indices)
                        
                        text_raw_without_target_indices = tokenizer.text_to_sequence(text_left + " " + text_right)
                        #print(text_raw_without_target_indices)
                        text_left_indices = tokenizer.text_to_sequence(text_left)
                        #print(text_left_indices)
                        text_left_with_target_indices = tokenizer.text_to_sequence(text_left + " " + target)
                        #print(text_left_with_target_indices)
                        text_right_indices = tokenizer.text_to_sequence(text_right, reverse=True)
                        #print( text_right_indices)
                        text_right_with_target_indices = tokenizer.text_to_sequence(" " + target + " " + text_right, reverse=True)
                        #print(text_right_with_target_indices)
                        target_indices = tokenizer.text_to_sequence(target)
                        #print(target_indices)
                        
                        #if data=='train' :
                        sentiment_score = info['sentiment_score']
                            #print(sentiment_score)
                        aspect= info['aspects']
                            #print("Aspect "+ aspect)
                            #sentiment_score = rescale(sentiment_score,[-1,1],[0,1])
                            #print(sentiment_score)



                        data = {
                                    'text_raw_indices': text_raw_indices,
                                    'text_raw_without_target_indices': text_raw_without_target_indices,
                                    'text_left_indices': text_left_indices,
                                    'text_left_with_target_indices': text_left_with_target_indices,
                                    'text_right_indices': text_right_indices,
                                    'text_right_with_target_indices': text_right_with_target_indices,
                                    'target_indices': target_indices,
                                    
                        }
                        
                        #if data=='train': 
                        data['polarity']= sentiment_score
                        
                       
 '''                       
'''
Combining test_headline and test_post

import json
def load_test_data():
    with open(r'D:\Sentiment-Analysis\test\task1_post_ABSA_test.json', encoding='utf-8') as f1:
        foo1 = json.load(f1)
        with open(r'D:\Sentiment-Analysis\test\task1_headline_ABSA_test.json', encoding='utf-8') as f2:
                foo2= json.load(f2)
                test = {**foo1, **foo2}
                data = json.dumps(test)
                with open(r"D:\Sentiment-Analysis\test.json","w") as f:
                    f.write(data)
                
#load_test_data()
'''


'''
import pandas as pd
import re
import json
def validation_headline():
    df = pd.read_csv(r'D:\Datasets\FinanceHeadlineDataset\gold_standard\test_headlines_samples - Sheet1.tsv', sep='\t')
    df.head()
    d={}

    for index, row in df.iterrows():

        target = re.sub('\d+',"", row["id"])
        target=target.lstrip("_")
        inner_d={}
        inner_d_1={}
        inner_d['info']=[]

        inner_d['sentence']=row['sentence']

        inner_d_1["target"]=target
        inner_d_1["sentiment_score"]=row["sentiment_scores"]
        aspect_h=row["aspect"].split("/")
        a=aspect_h[0]+"/"+aspect_h[1]
        aspect=[]
        aspect.append(a)
        inner_d_1["aspects"]=aspect
        inner_d['info'].append(inner_d_1)

        d[row["id"]]=inner_d
    return d

def validation_posts():
    df = pd.read_csv(r'D:\Datasets\FinanceHeadlineDataset\gold_standard\test_set_post - Sheet1.tsv', sep='\t')
    df.head()
    d={}

    for index, row in df.iterrows():

        target = re.sub('\d+',"", row["id"])
        target=target.lstrip("$")
        inner_d={}
        inner_d_1={}
        inner_d['info']=[]

        inner_d['sentence']=row['sentence']

        inner_d_1["target"]=target
        inner_d_1["sentiment_score"]=row["sentiment_score"]
        a=row["aspect_category_1"]+"/"+row["aspect_category_2"]
        aspect=[]
        aspect.append(a)
        inner_d_1["aspects"]=aspect
        inner_d['info'].append(inner_d_1)

        d[row["id"]]=inner_d
    return d
foo1=validation_headline()
foo2=validation_posts()
validation_test = {**foo1, **foo2}    
data = json.dumps(validation_test)
with open(r"D:\Sentiment-Analysis\validation_test.json","w") as f:
                    f.write(data)
                    
'''<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11487727500417431, 'fraction_numerical': 0.006177992987143096, 'mean_word_length': 1.521043771043771, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '119887', 'license': 'MIT', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1646, 'n_words': 230, 'path': 'Code/Data_utils.ipynb', 'repo_name': 'simra-shahid/Aspect-Based-Sentiment-Analysis-on-Financial-News-and-Posts', 'size': '5989'}