<jupyter_start><jupyter_text><jupyter_code># installing pytesseract for optical char. recognition
pip install pytesseract
# installing gingerit
pip install gingerit
pip install googlemaps
# importing packages
import nltk
from nltk.corpus import stopwords
import spacy
import en_core_web_sm
import pytesseract as pt
from PIL import Image
from gingerit.gingerit import GingerIt
import requests
from bs4 import BeautifulSoup
from textblob import TextBlob
import urllib
import googlemaps
nltk.download("stopwords") # downloading nltk stop words
nltk.download("punkt")
# assigning variables
nlp = en_core_web_sm.load()
stop_words = stopwords.words("english")
extensions = ["jpg", "png", "jpeg"]
gmaps = googlemaps.Client(key='<KEY>')
# defining useful functions
def file_type(filename):# function for character optical recognition
    try:
        if filename.rsplit('.',1)[1].lower() in extensions:
            s = Image.open(filename)
            text = pt.image_to_string(s)
        else:
            text = filename
        return text
    except:
        print("Error working with file type")


def word(filename, final_type): # function to tokenize text 
    try:
        f = file_type(filename)
        tok_sent = nltk.sent_tokenize(f)
        tok_word = []
        for s in tok_sent:
            tok_word.append(nltk.word_tokenize(s))
        final_text = []
        for w in tok_word:
            if w not in stop_words:
                final_text.append(w)
        if final_type == 'sentence':
            return tok_sent
        elif final_type == 'word':
            return final_text
    except:
        print("Error tokenizing text")
# check for grammer and spelling errors and return the number of corrections
def check(filename):
    try:
        f = word(filename, 'sentence')
        corrections = 0
        for s in f:
            g = GingerIt()
            h = g.parse(s)
            corrections += len(h['corrections'])
        return corrections
    except:
        print("Error while checking grammer errors in text")
# function to return the name of the organization
def org(filename):
    try:
        f = file_type(filename)  
        doc = nlp(f)
        v = []
        for x in doc:
            if (x.ent_type_)  == 'ORG':
                v.append((x))
        comp = ""
        for i in v:
            comp = comp+' '+i.text
        return comp
    except:
        print("Error checking name of organization")

# function to check location
def location(filename):
    loc = word(filename,'word')
    pos = word(filename,'sentences')
    locations = ['lagos','abuja','oyo','rivers','abia','adamawa','akwaibom','anambra','bauchi',
                 'benue','borno','cross river','delta','ebonyi','edo','ekiti','enugu',
                 'gombe','imo','jigawa','kaduna','kano','katsina','kebbi','kogi','kwara',
                 'nassarawa','niger','ogun','ondo','osun','plateau','sokoto','taraba','dutse','zamfara','fct']
    place = []
    ff = []
    for i in loc:
        if type(i)=="str":
            if i.lower() in locations:
                ff.append(i)
                for j in pos:
                    if i.lower() in j.lower():
                        place.append(j) 
                    else:
                        place = place
        if type(i)=='list':
            for l in i:
                if l.lower() in locations:
                    ff.append(l)
                    for j in pos:
                        if l.lower() in j.lower():
                            place.append(j) 
                        else:
                            place = place
        else:
            ff = ff
    return place

# sentiment analysis

def percentage(part, whole): # calculate percentage
    if whole != 0:
        return round((100 * float(part)) / float(whole), 2)
    else:
        return "Error calculating percentage"     

def word_count(string):
    """function to return count of comments"""
    try:
        counts = dict()
        words = string.split()
        for word in words:
            if word in counts:
                counts[word] += 1
            else:
                counts[word] = 1
        return len(counts)
    except:
        print("Word count failed")
  
def add_to_word_list(strings):
    """function to add all comments to Wordlist"""
    global WordList
    try:
        k = 0
        while k < len(strings):
            if word_count(strings[k].text) > 1:
                WordList.append(strings[k].text)
            k += 1
    except:
        print("Add to WordList failed")


def search_item(search_term, next=False, page=0,  board=0):
    """function to search and return comments"""
    if next == False:
        page = requests.get("https://www.nairaland.com/search?q=" + urllib.parse.quote_plus(str(search_term)) + "&board="+str(board))
    else:
        page = requests.get("https://www.nairaland.com/search/"
                            + str(search_term) + "/0/"+str(board)+"/0/1" + str(page))
    soup = BeautifulSoup(page.content, 'html.parser')

    comments = soup.findAll("div", {"class": "narrow"})

    return comments


WordList = []
def sentiment(searchTerm):
    """function to evaluate sentiment"""
    try:
        j = 0
        board = 29
        while j < 10:
            if j == 0:
                nextItem = False
            else:
                nextItem = True
            commentsCurrent = search_item(searchTerm, nextItem, j,  board)
            add_to_word_list(commentsCurrent)
            j += 1
    except:
        print("Search failed")
        
    positive = 0
    negative = 0
    neutral = 0
    
    previous = []
    for tweet in WordList:
        if tweet in previous:
            continue
        previous.append(tweet)
        analysis = TextBlob(tweet)
        """evaluating polarity of comments"""
        polarity = analysis.sentiment.polarity

        if (analysis.sentiment.polarity == 0):
            neutral += 1
        elif (analysis.sentiment.polarity < 0.00):
            negative += 1
        elif (analysis.sentiment.polarity > 0.0):
            positive += 1
    
    noOfSearchTerms = positive + negative + neutral

    positive = percentage(positive, noOfSearchTerms)
    negative = percentage(negative, noOfSearchTerms)
    neutral = percentage(neutral, noOfSearchTerms)
    
    return positive, negative, neutral
# print("How people are reacting on " + searchTerm + " by analyzing " + str(noOfSearchTerms) + " comments from "
#       "on nairaland")

# if (negative >= 30):
#     print("There is a high percentage of negative comments about this Company online in regards to jobs")
# elif(negative >= 20 and negaative < 30):
#     print("There are some negative comments about this Company in regards to jobs" )
# elif (negative < 20):
#     print("There is a low percentage of negative comments about this Company online in regards to jobs")

def verify_address(address):    
    geocode_result = gmaps.geocode(address)
    if geocode_result != '[]':
        return 1
    else:
        return 0
def confidence_interval(filename, searchTerm, address ):
    if filename and searchTerm and address:
        correction = check(filename)
        positive, negative, neutral = sentiment(searchTerm)
        auth = verify_address(address)
        if auth:
            score_a = 5
        else:
            score_a = -5
            
        if negative < 20:
            score_n = 10
        elif negative >=20 and negative < 30:
            score_n = 5
        elif negative >= 30:
            score_n = 0
            
        if correction <= 5:
            score_c = 10
        elif correction > 5 and correction <= 10:
            score_c = 5
        elif correction > 10:
            score_c = 10
            
        confidence = ((score_a + score_n + score_c) / 30) * 10
        
        if confidence > 6:
            return "Based on logistics the job invite no be scam"
        if confidence >= 4 and confidence <= 6:
            return "The job invite shows elements of scam but not too sure"
        if confidence < 4:
            return "This is likely a scam"<jupyter_output><empty_output><jupyter_text>Test away<jupyter_code>filename = """Dear Akinola

Your online application has been received by us for the position of customer service representative . We believe you will be able to defend your CV and if need be choose from other option with the company.
You are hereby invited for an interview session. Your assessment will determine if you are fit for a job offer.

Date: Monday 21st of October, 2019
Time : 9am
Venue :2nd gate Chemline House, Obasa Road beside Forte Oil Filing Station. Oba - Akran Ikeja Lagos State.

Kindly respond to this email and confirm your availability. Come with a copy of this invite to gain access and an updated CV with a valid means of identification.

Best Regards

Human Resource Team
Kingsley (HRM)
09014033246

DISCLAIMER

Telcomm Link Team is committed to helping you secure worthwhile employment. We are a third-party recruiter and outsourcing agency.

Third-party Recruiters: Third-party Recruiters are agencies, organizations, or individuals recruiting candidates for specified positions, be it temporary, part-time, or full-time employments on behalf of others and not for their own needs. You understand and consent that in order to facilitate the recruitment process on your behalf, Telcomm Link Team may forward your details to clients who are interested in you, Or, in other cases forward your details to clients who are interested in your services"""

address = "no. 21 adesanya aguda, surulere"
searchTerm = 'shell'
check(filename)
verify_address(address)
sentiment(searchTerm)
confidence_interval(filename, searchTerm, address)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09317598768599282, 'fraction_numerical': 0.010466906105695228, 'mean_word_length': 2.263898191560616, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '87178', 'license': 'MIT', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2652, 'n_words': 916, 'path': 'confidence_interval.ipynb', 'repo_name': 'hngi/ML-TEAM3', 'size': '9779'}