<jupyter_start><jupyter_text>OCR evaluation<jupyter_code>import pandas as pd
p ="/home/naresh/Tarento/anuvaad/anuvaad-etl/anuvaad-extractor/block-merger/src/notebooks/ocrevaluation/37863_2018_5_1502_16064_Judgement_19-Aug-2019_HIN/37863_2018_5_1502_16064_Judgement_19-Aug-2019_HIN.csv"
d = pd.read_csv(p)
from difflib import SequenceMatcher

def similar(a, b):
    return SequenceMatcher(None, a, b).ratio()



import ast
def post_process(d,image_path):
    word_lis = []
    text_lis = []
    updated_text_lis = []
    image   = Image.open(image_path)
    image   = image.resize((page_width,page_height))
    draw    = ImageDraw.Draw(image)
    for word_coord in d['word_coords']:
        #res = ast.literal_eval(word_coord)
        res=word_coord
        for word in range(len(res)):
            if res[word]['conf']<60 and len(str(res[word]['text']))>1:
                word_lis.append(res[word]); text_lis.append(res[word]['text'])
                left   = res[word]['text_left']; top = res[word]['text_top']
                right  = (res[word]['text_left'] + res[word]['text_width']); bottom = (res[word]['text_top'] + res[word]['text_height'])
                #draw.rectangle(((left, top), (right,bottom)), outline='red')
                crop_image = image.crop((left-5, top-5, right+5, bottom+5)) 
                crop_image.save("/home/naresh/check/"+str(uuid.uuid4())+'.jpg')
                temp_df_eng = pytesseract.image_to_data(crop_image,config='--psm 6', lang= "eng",output_type=Output.DATAFRAME)
                temp_df_hin = pytesseract.image_to_data(crop_image, config='--psm 6',lang= "Devanagari",output_type=Output.DATAFRAME)
                temp_df_eng = temp_df_eng[temp_df_eng['conf']>0]
                temp_df_hin = temp_df_hin[temp_df_hin['conf']>=0]
                org_text  = str(res[word]['text']); eng_text = ""; hin_text =""
                eng_conf =0; hin_conf = 0
                for index,row in temp_df_eng.iterrows():
                    eng_text=eng_text+" "+str(row['text'])
                    eng_conf= eng_conf+row['conf']
                for index,row in temp_df_hin.iterrows():
                    hin_text=hin_text+" "+str(row['text'])
                    hin_conf= hin_conf+row['conf']
                eng_sim = similar(org_text,eng_text)
                hin_sim = similar(org_text,hin_text)
                eng_df_len = len(temp_df_eng); hin_df_len = len(temp_df_hin);
                if eng_df_len==0:
                    eng_df_len=1
                if hin_df_len==0:
                    hin_df_len=1
                 
                if eng_sim>hin_sim:
                    conf = eng_conf/eng_df_len
                    text = eng_text
                elif eng_sim<hin_sim:
                    conf = hin_conf/hin_df_len
                    text = hin_text
                else:
                    if eng_conf>hin_conf:
                        text = eng_text
                        conf = eng_conf/eng_df_len
                    elif eng_conf<hin_conf:
                        conf = hin_conf/hin_df_len
                        text = hin_text
                    else:
                        conf = hin_conf/hin_df_len
                        text = hin_text
                if len(eng_text)==0 and len(hin_text)==0:
                    text  = org_text
                    conf  = res[word]['conf']
                print("original text:",res[word]['text'])
                print("english text", eng_text)
                print("hindi text", hin_text)
                print("final text", text)
            else:
                left   = res[word]['text_left']; top = res[word]['text_top']
                right  = (res[word]['text_left'] + res[word]['text_width']); bottom = (res[word]['text_top'] + res[word]['text_height'])
                #draw.rectangle(((left, top), (right,bottom)), outline='red')
    #image.show()

                



            
post_process(h_df,img_filepath)
from difflib import SequenceMatcher

def similar(a, b):
    return SequenceMatcher(None, a, b).ratio()
similar("पी0डब्लूण","पी0०डब्लू0")
import pandas as pd
p ="/home/naresh/Tarento/anuvaad/anuvaad-etl/anuvaad-extractor/block-merger/src/notebooks/char_data.csv"
d = pd.read_csv(p)
d.sort_values('contribution_to_detection_error')
d['contribution_to_detection_error'][-10:].sum()
#updated
import pandas as pd
p ="/home/naresh/Tarento/anuvaad/anuvaad-etl/anuvaad-extractor/block-merger/src/notebooks/char_data.csv"
d = pd.read_csv(p)
d.sort_values('contribution_to_detection_error')
d['contribution_to_detection_error'][-10:].sum()
#updated
import pandas as pd
p ="/home/naresh/Tarento/anuvaad/anuvaad-etl/anuvaad-extractor/block-merger/src/notebooks/char_data.csv"
d = pd.read_csv(p)
d.sort_values('contribution_to_detection_error')
d['contribution_to_detection_error'][-10:].sum()
(((d['contribution_to_overall_error']*d['total_detected_chars']).sum())/(d['total_detected_chars'].sum()))*len(d)
(((d['contribution_to_detection_error']*d['total_detected_chars']).sum())/(d['total_detected_chars'].sum()))*len(d)
d['contribution_to_overall_error'].hist(
k=[]
import shutil
for i in glob.glob(base_dir+'/ocrevaluation/*/*.html'):
    k.append(i.split('/')[-1].split('.html')[0])
    #
for i in glob.glob(input_dir+'/*.pdf'):
    if i.split('/')[-1].split('.pdf')[0] in k:
        os.remove(i)
        
import shutil
for i in glob.glob(base_dir+'/ocrevaluation/*/*.html'):
    shutil.copy(i,base_dir+'/html_file/')
import pandas as pd
from PIL import Image, ImageDraw
import copy
from google.oauth2 import service_account
from google.cloud import vision
from services.xml_document_info import (get_xml_info, get_xml_image_info)
from services.get_xml import  create_pdf_processing_paths, extract_pdf_metadata, process_input_pdf
from anuvaad_auditor.loghandler import log_info
from anuvaad_auditor.loghandler import log_error
from src.services import main
from src.services.get_underline import get_underline
from services import get_xml
from src.services.child_text_unify_to_parent import ChildTextUnify
from services.preprocess import prepocess_pdf_regions
from services.get_tables import page_num_correction , get_text_table_line_df
from src.services.ocr_text_utilities import  tesseract_ocr
import src.utilities.app_context as app_context
import config
def draw_box(filepath, desired_width, desired_height, df, color="green", save=False):
    image  = Image.open(filepath)
    image  = image.resize((desired_width, desired_height))
    draw   = ImageDraw.Draw(image)
    
    for index, row in df.iterrows():
        left   = int(row['text_left'])
        right  = int(row['text_width'] + left)
        top    = int(row['text_top'])
        bottom = int(row["text_height"] + top)
        
        draw.rectangle(((left, top), (right,bottom)), outline=color)
    save_filepath = os.path.join(os.path.dirname(filepath), 'processed_' + os.path.basename(filepath))
    if save:
        image.save(save_filepath)
    
    return image

def show_df(df):
    return df.head(df.shape[0])
app_context.application_context= dict({'input': {'files': [{'locale': 'hi', 'path': 'RTIOrder_hi.pdf', 'type': 'pdf'}]}, 'jobID': 'BM-15913540488115873', 'state': 'INITIATED', 'status': 'STARTED', 'stepOrder': 0, 'workflowCode': 'abc', 'tool': 'BM', 'metadata': {'module': 'WORKFLOW-MANAGER', 'receivedAt': 15993163946431696, 'sessionID': '4M1qOZj53tIZsCoLNzP0oP', 'userID': 'd4e0b570-b72a-44e5-9110-5fdd54370a9d'}, 'taskID': 'BM-16000776881221366'})<jupyter_output><empty_output><jupyter_text>start of the program<jupyter_code>'''
  folder structure of test data goes like this
  - notebooks
      - sample-data
          - input
          - output
          
  the pdfs are present in "input" directory and they are *.pdf is added into .gitignore
  just to save repo size.
'''

base_dir   = os.getcwd()
input_dir  = os.path.join(base_dir, 'sample-data','input')
save_dir   = os.path.join(base_dir, 'sample-data', 'bbox_output')
output_dir = os.path.join(base_dir, 'sample-data', 'output')



filename = 'ACFrOgDVXO3WTb1f2eL55weMN3KlNnnJ<KEY>'
#filename   = 'SC_2011_CA1025_ta.pdf'
#filename = 'SC_2009_CA2062_ta.pdf'
#filename = 'SC_2017_CA2502_ta.pdf'
#special error filename = 'ma_hc_2000_SA2034_ta.pdf'

#filename = 'ma_hc_2001_SA725_ta.pdf'
#error filename = 'ma_hc_2007_CMA1134_ta.pdf'
# error filename = 'SC_2019_CA117_ta.pdf'
lang ='hi'
index = 0

#correct output = ['SC_2011_CA1025_ta.pdf','SC_2009_CA2062_ta.pdf','SC_2017_CA2502_ta.pdf','ma_hc_2001_SA725_ta.pdf']
#wrong output = ['ma_hc_2017_WA1248_alt_ta.pdf','ma_hc_2000_SA2034_ta.pdf','ma_hc_2007_CMA1134_ta.pdf','SC_2019_CA117_ta.pdf']<jupyter_output><empty_output><jupyter_text>extract the pdf into xml & background images for downstream processing<jupyter_code>pdf_filepath      = os.path.join(input_dir, filename)

img_dfs,xml_dfs, page_width, page_height,working_dir, pdf_bg_img_filepaths, pdf_image_paths  = get_xml.process_input_pdf(filename, input_dir, lang)
#img_dfs, xml_dfs, page_width, page_height, working_dir, pdf_bg_img_filepaths = get_xml.process_input_pdf(pdf_filepath, output_dir, 'JOBID_1000')
#header_region, footer_region = prepocess_pdf_regions(xml_dfs, page_height)<jupyter_output>[2020-10-01 16:53:34,018] {loghandler.py:20} MainThread INFO in loghandler: created processing directories successfully /home/naresh/Tarento/anuvaad/anuvaad-etl/anuvaad-extractor/block-merger/src/notebooks/sample-data/input/ACFrOgDVXO3WTb1f2eL55weMN3KlNnnJAN2gXeUIo9M7Ap8vmX-6yQ41YK8s4aIrbR_Zaoc3XZQC_-toLeVE3UFysQySvhA3sfotiA16czTqcb48eVGa8wU2gerc47k8aRsCB6CUvGPGwPXbulGR_8ab8cd3a-03d8-11eb-ac9d-38baf82f7425
[2020-10-01 16:53:34,030] {loghandler.py:20} MainThread INFO in loghandler: filepath /home/naresh/Tarento/anuvaad/anuvaad-etl/anuvaad-extractor/block-merger/src/notebooks/sample-data/input/<KEY>GR.pdf, working_dir /home/naresh/Tarento/anuvaad/anuvaad-etl/anuvaad-extractor/block-merger/src/notebooks/sample-data/input/ACFrOgDVXO3WTb1f2eL55weMN3KlNnnJAN2gXeUIo9M7Ap8vmX-6yQ41YK8s4aIrbR_Zaoc3XZQC_-toLeVE3UFysQySvhA3sfotiA16czTqcb48eVGa8wU2gerc47k8aRs[...]<jupyter_text>Draw Bounding box using dfs<jupyter_code>def doc_structure_analysis(xml_dfs,img_dfs,working_dir ,lang, page_width, page_height, pdf_bg_img_filepaths,pdf_image_paths):
    
    '''
        Document structure analysis to get:
            - in_dfs
            - table_dfs
            - line_dfs
            - h_dfs
            - v_dfs
            - p_dfs
            - text_block_dfs

    '''
    log_info("document structure analysis started  ===>", app_context.application_context )

    header_region, footer_region = prepocess_pdf_regions(xml_dfs, page_height)
    text_merger = ChildTextUnify()
    
    in_dfs, table_dfs, line_dfs,bg_dfs = get_text_table_line_df(xml_dfs, img_dfs, pdf_bg_img_filepaths,check=True)
    #in_dfs[0]['font_family']="times"
    #in_dfs[0]['font_family_updated']="times"
    h_dfs                              = get_xml.get_hdfs(in_dfs,header_region,footer_region)
    
    if lang != 'en':
        h_dfs = tesseract_ocr(pdf_image_paths, page_width, page_height, h_dfs, lang)
        for h_df in h_dfs:
            h_df['children'] = None
    v_dfs                              = get_xml.get_vdfs(h_dfs)
    p_dfs                              = get_xml.get_pdfs(v_dfs, lang)
    p_dfs , line_dfs                   = get_underline(p_dfs,line_dfs,app_context.application_context)
    
    
    if lang=='en':
        p_dfs  = text_merger.unify_child_text_blocks(p_dfs)
    
    
    log_info( "document structure analysis successfully completed", app_context.application_context )
    return in_dfs,h_dfs, v_dfs, p_dfs
in_dfs,h_dfs, v_dfs, p_dfs = doc_structure_analysis([xml_dfs[index]],[img_dfs[index]],working_dir, lang, page_width, page_height, [pdf_bg_img_filepaths[index]],[pdf_image_paths[index]])
page_index_path      = 1
page_index = 0
img_filepath    = pdf_image_paths[page_index_path]

in_df   = in_dfs[page_index]
h_df    = h_dfs[page_index]
v_df    = v_dfs[page_index]
p_df    = p_dfs[page_index]
#ocr_df  = ocr_dfs[page_index]
draw_box(pdf_image_paths[0], page_width, page_height,p_df, color='red')<jupyter_output><empty_output><jupyter_text>Draw bounding box for json response<jupyter_code>from src.services import main
pdf_path = input_dir+"/"+filename
save_path  = save_dir+"/"+str(filename.split('.pdf')[0])
if not os.path.exists(save_path):
    os.system('mkdir -p {0}'.format(save_path))
data = main.DocumentStructure(app_context, filename, lang='hi',base_dir=input_dir)
response = data['result']
def draw_bbox_image(draw,page_data):
    
    if page_data['images']:
        try:
            for image_block in page_data['images']:
                    top     = image_block["text_top"];         left   = image_block["text_left"];  
                    bottom  = top+image_block["text_height"];  right  = left+image_block["text_width"]
                    draw.rectangle(((left, top), (right,bottom)), outline='green')
            return draw
        except:
            pass
def draw_bbox_table(draw,page_data):
    
    if page_data['tables']:
        try:
            for table_block in page_data['tables']:
                    top     = table_block["text_top"];         left   = table_block["text_left"];  
                    bottom  = top+table_block["text_height"];  right  = left+table_block["text_width"]
                    draw.rectangle(((left, top), (right,bottom)), outline='blue')
            return draw
        except:
            pass
def draw_bbox_text(draw,page_data):
    
    if page_data['text_blocks']:
        try:
            for text_block in page_data['text_blocks']:
                top     = text_block["text_top"];         left   = text_block["text_left"];  
                bottom  = top+text_block["text_height"];  right  = left+text_block["text_width"]
                draw.rectangle(((left, top), (right,bottom)), outline='red')
            return draw
        except:
            pass
def draw_bbox_pdf(data,image_files,save_path):
    for page_no in range(len(data)):
        image_path = sorted(image_files)[page_no]
        page_data = data[page_no]
        page_width = page_data['page_width']; page_height = page_data['page_height']
        image  = Image.open(image_path)
        image  = image.resize((page_width, page_height))
        draw   = ImageDraw.Draw(image)
        draw  = draw_bbox_text(draw,page_data)
        draw  = draw_bbox_image(draw,page_data)
        draw  = draw_bbox_table(draw,page_data)
        save_filepath = os.path.join(save_path,image_path.split('images/')[1])
        image.save(save_filepath)
            
draw_bbox_pdf(response,pdf_image_paths,save_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12470187393526405, 'fraction_numerical': 0.02787052810902896, 'mean_word_length': 3.273733255678509, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '558802', 'license': 'MIT', 'n_tokens_mistral': 5570, 'n_tokens_neox': 5204, 'n_words': 780, 'path': 'anuvaad-etl/anuvaad-extractor/block-merger/src/notebooks/ds_driver_v3.ipynb', 'repo_name': 'srihari-nagaraj/anuvaad', 'size': '14931'}