Create utility/utils.py

utility/utils.py

@@ -0,0 +1,352 @@
+# libraries
+import os
+from huggingface_hub import InferenceClient
+from dotenv import load_dotenv
+import json
+import re
+import easyocr
+import spacy
+from PIL import Image, ImageEnhance, ImageDraw
+import cv2
+import numpy as np
+from paddleocr import PaddleOCR
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Authenticate with Hugging Face
+HFT = os.getenv('HF_TOKEN')
+
+# Initialize the InferenceClient
+client = InferenceClient(model="mistralai/Mistral-7B-Instruct-v0.3", token=HFT)
+
+# Initialize EasyOCR reader for extracting text
+reader = easyocr.Reader(['en'])
+
+# Initialize spaCy's English model
+nlp = spacy.load("en_core_web_sm")
+
+def draw_boxes(image, bounds, color='red', width=2):
+    draw = ImageDraw.Draw(image)
+    for bound in bounds:
+        p0, p1, p2, p3 = bound[0]
+        draw.line([*p0, *p1, *p2, *p3, *p0], fill=color, width=width)
+    return image
+
+#Image Quality upscaling 
+# Load image using OpenCV
+def load_image(image_path):
+    return cv2.imread(image_path)
+
+# Function for upscaling image using OpenCV's INTER_CUBIC or ESRGAN (if available)
+def upscale_image(image, scale=2):
+    height, width = image.shape[:2]
+    # Simple upscaling using cubic interpolation
+    upscaled_image = cv2.resize(image, (width * scale, height * scale), interpolation=cv2.INTER_CUBIC)
+    return upscaled_image
+
+# Function to denoise the image (reduce noise)
+def reduce_noise(image):
+    return cv2.fastNlMeansDenoisingColored(image, None, 10, 10, 7, 21)
+
+# Function to sharpen the image
+def sharpen_image(image):
+    kernel = np.array([[0, -1, 0],
+                       [-1, 5, -1],
+                       [0, -1, 0]])
+    sharpened_image = cv2.filter2D(image, -1, kernel)
+    return sharpened_image
+
+# Function to increase contrast and enhance details without changing color
+def enhance_image(image):
+    # Convert from BGR to RGB for PIL processing, then back to BGR
+    pil_img = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+    enhancer = ImageEnhance.Contrast(pil_img)
+    enhanced_image = enhancer.enhance(1.5)
+    # Convert back to BGR
+    enhanced_image_bgr = cv2.cvtColor(np.array(enhanced_image), cv2.COLOR_RGB2BGR)
+    return enhanced_image_bgr
+
+# Complete function to process image
+def process_image(image_path, scale=2):
+    # Load the image
+    image = load_image(image_path)
+    
+    # Upscale the image
+    upscaled_image = upscale_image(image, scale)
+    
+    # Reduce noise
+    denoised_image = reduce_noise(upscaled_image)
+    
+    # Sharpen the image
+    sharpened_image = sharpen_image(denoised_image)
+    
+    # Enhance the image contrast and details without changing color
+    final_image = enhance_image(sharpened_image)
+    
+    return final_image
+
+
+def ocr_with_paddle(img):
+    finaltext = ''
+    ocr = PaddleOCR(lang='en', use_angle_cls=True)
+    # img_path = 'exp.jpeg'
+    result = ocr.ocr(img)
+    
+    for i in range(len(result[0])):
+        text = result[0][i][1][0]
+        finaltext += ' '+ text
+    return finaltext
+
+
+def extract_text_from_images(image_paths):
+    all_extracted_texts = {}
+    all_extracted_imgs={}
+    for image_path in image_paths:
+        # Enhance the image before OCR
+        enhanced_image = process_image(image_path, scale=2)
+        bounds = reader.readtext(enhanced_image)
+            # Draw boxes on the processed image
+        img_result = Image.fromarray(enhanced_image)
+        draw_boxes(img_result, bounds)
+
+        result_image_path = os.path.join(RESULT_FOLDER, f'result_{os.path.basename(image_path)}')
+        img_result.save(result_image_path)  # Save the processed image
+
+        # Perform OCR on the enhanced image
+        result=ocr_with_paddle(enhanced_image)
+        
+        all_extracted_texts[image_path] =result
+        all_extracted_imgs[image_path] = result_image_path
+     # Convert to JSON-compatible structure
+    all_extracted_imgs_json = {str(k): str(v) for k, v in all_extracted_imgs.items()}
+    return all_extracted_texts,all_extracted_imgs_json
+
+# Function to call the Gemma model and process the output as Json 
+def Data_Extractor(data, client):
+    text = f'''Act as a  Text extractor for the following text given in text: {data} 
+    extract text in the following output JSON string:
+    {{
+    "Name": ["Identify and Extract All the person's name from the text."],
+    "Designation": ["Extract All the designation or job title mentioned in the text."],
+    "Company": ["Extract All the company or organization name if mentioned."],
+    "Contact": ["Extract All phone number, including country codes if present."],
+    "Address": ["Extract All the full postal address or location mentioned in the text."],
+    "Email": ["Identify and Extract All valid email addresses mentioned in the text else 'Not found'."],
+    "Link": ["Identify and Extract any website URLs or social media links present in the text."]
+    }}    
+    Output:
+    '''
+    # Call the API for inference
+    response = client.text_generation(text, max_new_tokens=600)
+    
+    print("parse in text ---:",response)
+
+    # Convert the response text to JSON
+    try:
+        json_data = json.loads(response)
+        return json_data
+    except json.JSONDecodeError as e:
+        return {"error": f"Error decoding JSON: {e}"}   
+
+# For have text compatible to the llm
+def json_to_llm_str(textJson):
+    str=''
+    for file,item in textJson.items():
+      str+=item + ' '
+    return str
+
+# Define the RE for extracting the contact details like number, mail , portfolio, website etc 
+def extract_contact_details(text):
+    # Regex patterns
+    # Phone numbers with at least 5 digits in any segment 
+    combined_phone_regex = re.compile(r'''
+    (?: 
+        #(?:(?:\+91[-.\s]?)?\d{5}[-.\s]?\d{5})|(?:\+?\d{1,3})?[-.\s()]?\d{5,}[-.\s()]?\d{5,}[-.\s()]?\d{1,9} | /^[\.-)( ]*([0-9]{3})[\.-)( ]*([0-9]{3})[\.-)( ]*([0-9]{4})$/ |
+        \+1\s\(\d{3}\)\s\d{3}-\d{4} |               # USA/Canada Intl +1 (XXX) XXX-XXXX
+        \(\d{3}\)\s\d{3}-\d{4} |                    # USA/Canada STD (XXX) XXX-XXXX
+        \(\d{3}\)\s\d{3}\s\d{4} |                   # USA/Canada (XXX) XXX XXXX
+        \(\d{3}\)\s\d{3}\s\d{3} |                   # USA/Canada (XXX) XXX XXX
+        \+1\d{10} |                                 # +1 XXXXXXXXXX
+        \d{10} |                                    # XXXXXXXXXX
+        \+44\s\d{4}\s\d{6} |                        # UK Intl +44 XXXX XXXXXX
+        \+44\s\d{3}\s\d{3}\s\d{4} |                 # UK Intl +44 XXX XXX XXXX
+        0\d{4}\s\d{6} |                             # UK STD 0XXXX XXXXXX
+        0\d{3}\s\d{3}\s\d{4} |                      # UK STD 0XXX XXX XXXX
+        \+44\d{10} |                                # +44 XXXXXXXXXX
+        0\d{10} |                                   # 0XXXXXXXXXX
+        \+61\s\d\s\d{4}\s\d{4} |                    # Australia Intl +61 X XXXX XXXX
+        0\d\s\d{4}\s\d{4} |                         # Australia STD 0X XXXX XXXX
+        \+61\d{9} |                                 # +61 XXXXXXXXX
+        0\d{9} |                                    # 0XXXXXXXXX
+        \+91\s\d{5}-\d{5} |                         # India Intl +91 XXXXX-XXXXX
+        \+91\s\d{4}-\d{6} |                         # India Intl +91 XXXX-XXXXXX
+        \+91\s\d{10} |                              # India Intl +91 XXXXXXXXXX
+        0\d{2}-\d{7} |                              # India STD 0XX-XXXXXXX
+        \+91\d{10} |                                # +91 XXXXXXXXXX
+        \+49\s\d{4}\s\d{8} |                        # Germany Intl +49 XXXX XXXXXXXX
+        \+49\s\d{3}\s\d{7} |                        # Germany Intl +49 XXX XXXXXXX
+        0\d{3}\s\d{8} |                             # Germany STD 0XXX XXXXXXXX
+        \+49\d{12} |                                # +49 XXXXXXXXXXXX
+        \+49\d{10} |                                # +49 XXXXXXXXXX
+        0\d{11} |                                   # 0XXXXXXXXXXX
+        \+86\s\d{3}\s\d{4}\s\d{4} |                 # China Intl +86 XXX XXXX XXXX
+        0\d{3}\s\d{4}\s\d{4} |                      # China STD 0XXX XXXX XXXX
+        \+86\d{11} |                                # +86 XXXXXXXXXXX
+        \+81\s\d\s\d{4}\s\d{4} |                    # Japan Intl +81 X XXXX XXXX
+        \+81\s\d{2}\s\d{4}\s\d{4} |                 # Japan Intl +81 XX XXXX XXXX
+        0\d\s\d{4}\s\d{4} |                         # Japan STD 0X XXXX XXXX
+        \+81\d{10} |                                # +81 XXXXXXXXXX
+        \+81\d{9} |                                 # +81 XXXXXXXXX
+        0\d{9} |                                    # 0XXXXXXXXX
+        \+55\s\d{2}\s\d{5}-\d{4} |                  # Brazil Intl +55 XX XXXXX-XXXX
+        \+55\s\d{2}\s\d{4}-\d{4} |                  # Brazil Intl +55 XX XXXX-XXXX
+        0\d{2}\s\d{4}\s\d{4} |                      # Brazil STD 0XX XXXX XXXX
+        \+55\d{11} |                                # +55 XXXXXXXXXXX
+        \+55\d{10} |                                # +55 XXXXXXXXXX
+        0\d{10} |                                   # 0XXXXXXXXXX
+        \+33\s\d\s\d{2}\s\d{2}\s\d{2}\s\d{2} |      # France Intl +33 X XX XX XX XX
+        0\d\s\d{2}\s\d{2}\s\d{2}\s\d{2} |           # France STD 0X XX XX XX XX
+        \+33\d{9} |                                 # +33 XXXXXXXXX
+        0\d{9} |                                    # 0XXXXXXXXX
+        \+7\s\d{3}\s\d{3}-\d{2}-\d{2} |             # Russia Intl +7 XXX XXX-XX-XX
+        8\s\d{3}\s\d{3}-\d{2}-\d{2} |               # Russia STD 8 XXX XXX-XX-XX
+        \+7\d{10} |                                 # +7 XXXXXXXXXX
+        8\d{10} |                                   # 8 XXXXXXXXXX
+        \+27\s\d{2}\s\d{3}\s\d{4} |                 # South Africa Intl +27 XX XXX XXXX
+        0\d{2}\s\d{3}\s\d{4} |                      # South Africa STD 0XX XXX XXXX
+        \+27\d{9} |                                 # +27 XXXXXXXXX
+        0\d{9} |                                    # 0XXXXXXXXX
+        \+52\s\d{3}\s\d{3}\s\d{4} |                 # Mexico Intl +52 XXX XXX XXXX
+        \+52\s\d{2}\s\d{4}\s\d{4} |                 # Mexico Intl +52 XX XXXX XXXX
+        01\s\d{3}\s\d{4} |                          # Mexico STD 01 XXX XXXX
+        \+52\d{10} |                                # +52 XXXXXXXXXX
+        01\d{7} |                                   # 01 XXXXXXX
+        \+234\s\d{3}\s\d{3}\s\d{4} |                # Nigeria Intl +234 XXX XXX XXXX
+        0\d{3}\s\d{3}\s\d{4} |                      # Nigeria STD 0XXX XXX XXXX
+        \+234\d{10} |                               # +234 XXXXXXXXXX
+        0\d{10} |                                   # 0XXXXXXXXXX
+        \+971\s\d\s\d{3}\s\d{4} |                   # UAE Intl +971 X XXX XXXX
+        0\d\s\d{3}\s\d{4} |                         # UAE STD 0X XXX XXXX
+        \+971\d{8} |                                # +971 XXXXXXXX
+        0\d{8} |                                    # 0XXXXXXXX
+        \+54\s9\s\d{3}\s\d{3}\s\d{4} |              # Argentina Intl +54 9 XXX XXX XXXX
+        \+54\s\d{1}\s\d{4}\s\d{4} |                 # Argentina Intl +54 X XXXX XXXX
+        0\d{3}\s\d{4} |                             # Argentina STD 0XXX XXXX
+        \+54\d{10} |                                # +54 9 XXXXXXXXXX
+        \+54\d{9} |                                 # +54 XXXXXXXXX
+        0\d{7} |                                    # 0XXXXXXX
+        \+966\s\d\s\d{3}\s\d{4} |                   # Saudi Intl +966 X XXX XXXX
+        0\d\s\d{3}\s\d{4} |                         # Saudi STD 0X XXX XXXX
+        \+966\d{8} |                                # +966 XXXXXXXX
+        0\d{8} |                                    # 0XXXXXXXX
+        \+1\d{10} |                                 # +1 XXXXXXXXXX
+        \+1\s\d{3}\s\d{3}\s\d{4} |                  # +1 XXX XXX XXXX
+        \d{5}\s\d{5} |                              # XXXXX XXXXX                              
+        \d{10} |                                    # XXXXXXXXXX
+        \+44\d{10} |                                # +44 XXXXXXXXXX
+        0\d{10} |                                   # 0XXXXXXXXXX
+        \+61\d{9} |                                 # +61 XXXXXXXXX
+        0\d{9} |                                    # 0XXXXXXXXX
+        \+91\d{10} |                                # +91 XXXXXXXXXX
+        \+49\d{12} |                                # +49 XXXXXXXXXXXX
+        \+49\d{10} |                                # +49 XXXXXXXXXX
+        0\d{11} |                                   # 0XXXXXXXXXXX
+        \+86\d{11} |                                # +86 XXXXXXXXXXX
+        \+81\d{10} |                                # +81 XXXXXXXXXX
+        \+81\d{9} |                                 # +81 XXXXXXXXX
+        0\d{9} |                                    # 0XXXXXXXXX
+        \+55\d{11} |                                # +55 XXXXXXXXXXX
+        \+55\d{10} |                                # +55 XXXXXXXXXX
+        0\d{10} |                                   # 0XXXXXXXXXX
+        \+33\d{9} |                                 # +33 XXXXXXXXX
+        0\d{9} |                                    # 0XXXXXXXXX
+        \+7\d{10} |                                 # +7 XXXXXXXXXX
+        8\d{10} |                                   # 8 XXXXXXXXXX
+        \+27\d{9} |                                 # +27 XXXXXXXXX
+        0\d{9} |                                    # 0XXXXXXXXX (South Africa STD)
+        \+52\d{10} |                                # +52 XXXXXXXXXX
+        01\d{7} |                                   # 01 XXXXXXX
+        \+234\d{10} |                               # +234 XXXXXXXXXX
+        0\d{10} |                                   # 0XXXXXXXXXX
+        \+971\d{8} |                                # +971 XXXXXXXX
+        0\d{8} |                                    # 0XXXXXXXX
+        \+54\s9\s\d{10} |                           # +54 9 XXXXXXXXXX
+        \+54\d{9} |                                 # +54 XXXXXXXXX
+        0\d{7} |                                    # 0XXXXXXX
+        \+966\d{8} |                                # +966 XXXXXXXX
+        0\d{8}                                      # 0XXXXXXXX
+        \+\d{3}-\d{3}-\d{4}
+    )  
+
+    ''',re.VERBOSE)
+    
+    # Email regex
+    email_regex = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b')
+    
+    # Links regex, updated to avoid conflicts with email domains
+    link_regex = re.compile(r'\b(?:https?:\/\/)?(?:www\.)[a-zA-Z0-9-]+\.(?:com|co\.in|co|io|org|net|edu|gov|mil|int|uk|us|in|de|au|app|tech|xyz|info|biz|fr|dev)\b')
+    
+    # Find all matches in the text
+    phone_numbers = [num for num in combined_phone_regex.findall(text) if len(num) >= 5]
+    print("phone_numbers--->",phone_numbers)
+    emails = email_regex.findall(text)
+    links_RE = [link for link in link_regex.findall(text) if len(link)>=11]
+    
+    # Remove profile links that might conflict with emails
+    links_RE = [link for link in links_RE if not any(email in link for email in emails)]
+    
+    return {
+        "phone_numbers": phone_numbers,
+        "emails": emails,
+        "links_RE": links_RE
+    }  
+
+# preprocessing the data 
+def process_extracted_text(extracted_text):
+    # Load JSON data
+    data = json.dumps(extracted_text, indent=4)
+    data = json.loads(data)
+
+    # Create a single dictionary to hold combined results
+    combined_results = {
+        "phone_numbers": [],
+        "emails": [],
+        "links_RE": []
+    }
+
+    # Process each text entry
+    for filename, text in data.items():
+        contact_details = extract_contact_details(text)
+        # Extend combined results with the details from this file
+        combined_results["phone_numbers"].extend(contact_details["phone_numbers"])
+        combined_results["emails"].extend(contact_details["emails"])
+        combined_results["links_RE"].extend(contact_details["links_RE"])
+
+    # Convert the combined results to JSON
+    combined_results_json = combined_results
+
+    # Print the final JSON results
+    print("Combined contact details in JSON format:")
+    print(combined_results_json)
+
+    return combined_results_json 
+
+# Process the model output for parsed result
+def process_resume_data(LLMdata,cont_data,extracted_text):
+    
+    # Initialize the processed data dictionary
+    processed_data = {            
+            "name": [LLMdata.get('Name', 'Not found')],
+            "contact_number": [LLMdata.get('Contact', 'Not found')],
+            "Designation":[LLMdata.get('Designation', 'Not found')],
+            "email": [LLMdata.get("Email", 'Not found')],
+            "Location": [LLMdata.get('Address', 'Not found')],
+            "Link": [LLMdata.get('Link', 'Not found')],
+            "Company":[LLMdata.get('Company', 'Not found')],
+            "extracted_text": extracted_text
+            }
+    processed_data['email'].extend(cont_data.get("emails", [])) 
+    processed_data['contact_number'].extend(cont_data.get("phone_numbers", []))
+    processed_data['Link'].extend(cont_data.get("links_RE", []))
+    return processed_data