<jupyter_start><jupyter_text>Merge data from review back to original stanza data.<jupyter_code>#Load data used for dataset creation
df_shuffled = pd.read_pickle('intermediate_store/preprocessed_ads_V1.0.pickle')

#Load reviewed annotation data Appen
with open('reviewed_data/initial_1300.json') as json_file:
    rewiewed_json = json.load(json_file)
    
#Quick check if the token length is still the same
print('Example 0 how many tokens are in this sentence initially after stanza ', sum(len(row) for row in df_shuffled.loc[9119].doc.to_dict()))
print('Example 0 how many tokens are in this sentence after review', len(rewiewed_json[0]['data']['clean_description'].split(' ')))

#Detailed comparison


df_shuffled.loc[9119].doc.to_dict()
rewiewed_json[0]['data']['clean_description'].split(' ')
rewiewed_json[0]['data']
# In detail comparison between the spans --> Do this for all of them
flat_list_orig = [item for sublist in df_shuffled.loc[9119].doc.to_dict() for item in sublist]
flat_list_rev = rewiewed_json[0]['data']['clean_description'].split(' ')

for i, rev_span in enumerate(flat_list_rev):
    print ("Orig: {} Review: {} --> is the same: {} ".format(flat_list_orig[i]['text'], rev_span, flat_list_orig[i]['text']==rev_span))
rewiewed_json[0]<jupyter_output><empty_output><jupyter_text>Evaluate how many rows cause a problem and subsequently merge all rows without conflicts in a new df.<jupyter_code>def split_with_indexes(string):
    token_list = string.split(' ')
    tuple_list = []
    curr_start = 0
    for token in token_list:
        tuple_list.append( (token, curr_start, curr_start+len(token)-1) )#real end char, not start of next one
        curr_start += len(token) + 1
    return tuple_list
def annotate_doc(review_json, doc):
    
    #create splitted review token list
    #create a tag list with same length
    reviewed_tuples = split_with_indexes(review_json['data']['clean_description'])
    tag_list_rev = ['O' for i in range(len(reviewed_tuples))]
    
    #For each annotation check which tokens shall be labelled
    for tag_dict in review_json['completions'][0]['result']:
        first=True
        
        for j, (token, r_start, r_end) in enumerate(reviewed_tuples):
            tag_start = tag_dict['value']['start']
            tag_end = tag_dict['value']['end'] - 1 #real end char, not start of next one
            #print('r_start {} r_end {} tag_start {} tag_end {} token: {}'.format(r_start, r_end, tag_start, tag_end, token))
            if tag_end >= r_start and tag_start <= r_end and tag_dict['value']['labels'] == ['Drug']:
                if first:
                    tag_list_rev[j] = 'B-Drug'
                    first=False
                else:
                    tag_list_rev[j] = 'I-Drug'
    
    #loop over doc and annotate the tokens in misc.
    token_id=0
    for sentence in doc:
        for token_dict in sentence:
            token_dict['misc'] = token_dict['misc'] + '|label=' + tag_list_rev[token_id]
            token_id+=1
    return doc
    
#split_with_indexes(rewiewed_json[300]['data']['clean_description'])   
#annotate_doc(rewiewed_json[115], df_shuffled.loc[631].doc.to_dict())
annotate_doc(rewiewed_json[79], df_shuffled.loc[115].doc.to_dict())
#Check each review item if the annotations stayed the same

amt_same_len = 0
amt_not_same_len = 0
amt_exactly_same = 0
amt_not_exactly_same = 0

for i, review in enumerate(rewiewed_json):
    p_id = review['data']['p_id']
    flat_list_orig = [item for sublist in df_shuffled.loc[p_id].doc.to_dict() for item in sublist]
    flat_list_rev =review['data']['clean_description'].split(' ')
    length_problem=False
    
    #Check if length matches
    if len(flat_list_orig) == len(flat_list_rev):
        amt_same_len+=1
    else:
        amt_not_same_len +=1
        print('Length Problem at pandas_id: {} and json item nr: {}'.format(p_id, i))
        length_problem=True
    
    #Check each token if they are the equal:
    same = True
    for j, rev_span in enumerate(flat_list_rev):
        if not flat_list_orig[j]['text']==rev_span:
            if not ( flat_list_orig[j]['text'] == rev_span[:-1] and rev_span[-1:] == '.' ):
                same = False
                print('Detail Problem at pandas_id: {} and json item nr: {}'.format(p_id, i))
                break
    if same:
        amt_exactly_same+=1
        #If length and exact tokens are ok --> add label!
        if not length_problem:
            new_doc = annotate_doc(review, df_shuffled.loc[p_id].doc.to_dict())
            new_doc = Document(new_doc)
            df_shuffled.at[p_id, 'doc'] = new_doc
    else:
        amt_not_exactly_same+=1
    
print('{} rows had still the same amount of tokens. {} not. this is {} %'.format(amt_same_len, amt_not_same_len, amt_not_same_len/amt_same_len ))
print('{} rows had exactly the same tokens. {} not. this is {} %'.format(amt_exactly_same, amt_not_exactly_same, amt_not_exactly_same/amt_exactly_same))
#Investigate on errors
#rewiewed_json[344]['data']['clean_description'].split(' ')
#df_shuffled.loc[742].doc.to_dict()

# Length Problem at pandas_id: 742 and json item nr: 344 --> I have no idea what happened here. in the "shortTextDrugsV1.0.tsv" 
#which I uploaded to appen it was still correct. In the Appen Export it was only a single " in clean description --> guess that stuff
# was lost in the cloud.
# Length Problem at pandas_id: 802 and json item nr: 1105 
#--> That's a simple problem (a additional space in the end which got removed at trim()) 

test_dict = df_shuffled.loc[115].doc.to_dict()
test_dict
#conll = CoNLL.convert_dict(test_dict)
#conll
rewiewed_json[79]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1181770005299417, 'fraction_numerical': 0.01448507330860272, 'mean_word_length': 3.6716171617161715, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '176938', 'license': 'MIT', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1855, 'n_words': 504, 'path': 'various_scripts/merge_annotations_with_raw_data.ipynb', 'repo_name': 'jbogensperger/DRUG_CROSSNER', 'size': '5661'}