<jupyter_start><jupyter_text>Confidence Levels of Google Cloud Vision API's transcriptions `merged_df` csv has the API transcriptions, human transcriptions, calculated error between the two and the confidence level for each submission `confidence_flag.py` file in the app is created from this notebook  Build csv with both api and human transcriptions<jupyter_code># Imports
import pandas as pd
import pylcs
import os.path
import seaborn as sns
from scipy import stats
# Bring in the human transcribed and Google API transcribed csv's
human = pd.read_csv('human_metrics.csv')
api = pd.read_csv('api_metrics.csv')
# Keep only the transcriptions and the story_ID
# Rename to specify api transcripption or human transcription

human = human[['story_id', 'transcription']].rename(columns={'transcription': 'human transcription'})
api = api[['story_id', 'transcription']].rename(columns={'transcription': 'api transcription'})
# There were 3 story_id's that were not correct
# That also made the datatype of the entire column an object - need to convert to int to merge

human['story_id'][55] = 3215
human['story_id'][64] = 3213
human['story_id'][104] = 5240

human['story_id'] = human['story_id'].astype(str).astype(int)
# Merge two tables on story_id
merged_df = pd.merge(human, api, on= 'story_id')
merged_df.head()<jupyter_output><empty_output><jupyter_text>Calculate error and add to df<jupyter_code>def calc_error(t1, t2):
    """Takes in two different transcriptions of a story, returns the error score"""

    # cut out the spaces in both -- the human transcripts use spaces visually
    # this may also eliminate real space errors but I think it's better
    t1 = t1.replace(" ", "")
    t2 = t2.replace(" ", "")

    # transcripts may differ in exact length
    avg_length = (len(t1) + len(t2)) / 2

    # result: average number of single-character errors per character
    return pylcs.levenshtein_distance(t1, t2) / avg_length
# Add an error column to have index locations to index to
merged_df['error'] = 0.0
# Calculate the error between the two transcriptions - populate error column
for i in range(merged_df.shape[0]):
    merged_df.loc[i, 'error'] = calc_error(merged_df.loc[i, 'human transcription'], merged_df.loc[i, 'api transcription'])

merged_df.head(10)<jupyter_output><empty_output><jupyter_text>Calculate confidence and add to df- Final method we used; calculate the confidence of each character in the entire submission, then return the average confidence- `confidence_flag.py` in the app uses the `image_confidence` method and calculates the confidence level at the character level and raises a flag is the confidence for a page is lower than 0.85.<jupyter_code>def image_confidence(image_path):
    '''
    Detects text in images and calculates the confidence level for each
    character. Returns a True boolean if the overall confidence for the
    page is less than 0.85. Otherwise, returns False

        Input: Path to file where image is stored
            One image per call: run function on each image in a submission
        Output: Boolean; True if confidence level for page is less than 0.85
                False if confidence is 0.85 or greater
    '''
    # Imports needed specificially for this function
    from google.cloud import vision
    import io
    from google.oauth2 import service_account

    # If image_path is local
    with io.open(image_path, 'rb') as image_file:
        content = image_file.read()
    image = vision.types.Image(content=content)

    # # If image_path is a uri
    # image = vision.types.Image()
    # image.source.image_uri = uri

    # Set language to english only
    language = vision.types.ImageContext(language_hints=['en-t-i0-handwrit'])

    # Connect to Google API client
    creds = service_account.Credentials.from_service_account_file('/Users/stevenchase/Desktop/Steven/Computer_Science/Lambda/labs/story_sqaud/Story Squad-6122da7459cf.json')
    client = vision.ImageAnnotatorClient(credentials=creds)
    response = client.document_text_detection(image=image, image_context=language)

    # List of confidence levels of each word
    symbol_confidences = []

    for page in response.full_text_annotation.pages:
        for block in page.blocks:
            for paragraph in block.paragraphs:
                for word in paragraph.words:
                    for symbol in word.symbols:
                        symbol_confidences.append(symbol.confidence)
                        
    # If there is no text on the page, needs to return a list
    if len(symbol_confidences) == 0:
        return [-1]
    else:
        # Return list of all of the confidences for each character
        return symbol_confidences
def multi_page(directory):
    '''
    Function calls image_confidence function for each page in a single submission
        Input: file path containing images of user's written stories
        Output: Float; Confidence in the transcription of the entire submission
    '''

    paths = []

    for item in os.listdir(directory):
        if item[-3:] == 'jpg':
            path = os.path.join(directory, item)
            paths.append(path)

    submission_confidence = []

    for image_path in paths:
        submission_confidence.extend(image_confidence(image_path))
    
    # Drop any '-1' (meaning no text detected) responses from confidence function
    submission_confidence = [x for x in submission_confidence if x != -1]

    # Check to make sure that submission_confidence isn't empty
    if len(submission_confidence) == 0:
        return 0
    else:
        confidence_level = sum(submission_confidence) / len(submission_confidence)
        return confidence_level
# Add a confidence column to be able to refer to the index
merged_df['confidence'] = 0.0
merged_df
# Root directory
root_directory = os.path.join('/Users/stevenchase/Desktop/Steven/Computer_Science/Lambda/labs/story_sqaud', 'Stories Dataset', "Transcribed Stories")
# Calculate the confidence for each transcription
for i in range(merged_df.shape[0]):
    # Build the directory path based off of the story_id
    directory = os.path.join(root_directory, str(merged_df['story_id'][i])[:2] + '--', str(merged_df['story_id'][i]))

    # Populate confidence column with call of multi_page function on each row
    merged_df.loc[i, 'confidence'] = multi_page(directory)

merged_df.head()
merged_df.to_csv('error_confidence_metrics.csv')<jupyter_output><empty_output><jupyter_text>Explore confidence metrics and relationship with calculated error- First Correlation metric: -0.25- Correlation after dropping the missing images: -0.46Final method of getting confidence level (by character for the entire submission)- First Correlation metric: -0.29- After dropping the missing images: -0.49<jupyter_code># Explore distribution of confidence 
merged_df['confidence'].describe()
# Scatter plot to compare relationship between error and confidence level
sns.regplot(x="error", y="confidence", data=merged_df);

ax = sns.distplot(merged_df['confidence'])
plt.title("Distribution of Confidence of API Transcription")
plt.show()
merged_df.corr()<jupyter_output><empty_output><jupyter_text>Explore outliers and reasons for bad correlation<jupyter_code># Copy of df to manipulate

confidence_df = merged_df[['story_id', 'error', 'confidence']]
confidence_df.head()<jupyter_output><empty_output><jupyter_text>Some stories had missing pages for the human transcription which resulted in inaccurate error calculations. Those were dropped below. Additionally, we found the following human transcription errors. We believe this was a small enough sample size that it didn't affect the correlation too much. But they are important to keep in mind so I will note them here for future iterations.- Story 5202: There is text added to a sentence by the child using a ^. It is placed out of order. There are also a few '[]' throughout the transcription. One is for something that is crossed out. The others are smudges on the paper. Typo 'Jlack' instead of 'Jack' on the second page.- Story 3109: Text added by ^ is out of place. Includes some text from an illustration. [] brackets where words are crossd out. - Story 3114: Carrot out of placeThe convention seems to be the text associated with a carrot (^) where the student is adding to a sentence is out of order. Words that are crossed out are indicated by []. Sometimes text in illustrations are included but most times they are not. The human transcriptions errors do not seem too drastic to affect the calculated errors.<jupyter_code># Drop stories that had known human error in the the transcriptions - missing pages
to_drop = [3213, 3215, 3240, 5104, 5109, 5262]
# Drop outliers to see the relationship better

for i in range(confidence_df.shape[0]):
    if confidence_df['story_id'][i] in to_drop:
        confidence_df = confidence_df.drop(i)
        print(i)
sns.regplot(x="error", y="confidence", data=confidence_df);
confidence_df.corr()<jupyter_output><empty_output><jupyter_text>Explore values that have a high confidence but also have a high error metric<jupyter_code># Create sub_df to explore the worst correlation data points
condition_1 = confidence_df['error'] >= 0.4
condition_2 = confidence_df['confidence'] >= 0.85

outlier_df = confidence_df[condition_1 & condition_2]
outlier_df<jupyter_output><empty_output><jupyter_text>Notes on the top 12 worst correlations:3128: Handwriting seemed fine. API transcription had a lot of words out of order which is what lead to the higher error metric. Could have been because some of it was faint handwriting.3216: Handwriting was fine, it was written front and back so you could faitly make out some of the words written on the back. Wonder if that threw google off. Might be able to clean that up3229: Transcription wasn't actually that bad. But it was a very short piece of writing so the error metric was heavily influenced by a few mistakes.3246: Some light writing3222: Faint handwriting5208: Page is crowded, the words are all pushed together and the letters are tall on the lines5241: IDK5122: Crossed out the first paragraph, google still tried to transcribe that5114: Some erase marks in few paragraphs, other than that good handwriting5119: IDK5110: Messy, different colors, crossed out material5102: Crossed out words<jupyter_code># Low confidence subset to explore any commonalities
low_confidence_df = confidence_df[confidence_df['confidence'] < 0.85]
low_confidence_df<jupyter_output><empty_output><jupyter_text>Notes on confidence below 0.853112: IDK3124, 5264, 5202: words pushed together5113: small writing, words are spaced but letters are crowded  Experimental work Explore different ways of calculating confidence levels, see if that effects the correlation Look at the first 25 samples- confidence by word per page- confidence by word entire submission at once- confidence by character entrie submission at onceWe ended up getting the best correlation for the first 25 samlpes by retrieving the confidence level of every character across the entire submission before aggregating a total confidence level - will use that method on the entire dataset of stories<jupyter_code># Drop the stories that have missing data
cleaned_df = merged_df.copy()

for i in range(cleaned_df.shape[0]):
    if cleaned_df['story_id'][i] in to_drop:
        cleanded_df = cleaned_df.drop(i)
        print(i)
# Subset of first 25 rows to experiment with
df_25 = cleaned_df[:25]
df_25.shape<jupyter_output><empty_output><jupyter_text>Confidence by word per page metrics<jupyter_code>sns.regplot(x="error", y="confidence", data=confidence_df[:25]);
confidence_df[:25].corr()<jupyter_output><empty_output><jupyter_text>Confidence by character by page<jupyter_code># Function to get submission confidence by character per page
def character_page_confidence(image_path):
    '''
    Detects document features in images and returns overall confidence level of transcribed text
        Input: Path to file where images are stored
        - Assuming 1 image per image_path - run this on each individual image in one submission
        - Code for both local image_path and remote image_path, comment out
            the apporopriate one
        Output: Float. Decimal format of confidence level of the transcription for the page
    '''

    from google.cloud import vision
    import io
    from google.oauth2 import service_account

    # If image_path is local
    with io.open(image_path, 'rb') as image_file:
        content = image_file.read()
    image = vision.types.Image(content=content)

    # # If image_path is a uri
    # image = vision.types.Image()
    # image.source.image_uri = uri

    # Set language to english only
    language = vision.types.ImageContext(language_hints=['en-t-i0-handwrit'])

    # Connect to Google API client
    creds = service_account.Credentials.from_service_account_file('/Users/stevenchase/Desktop/Steven/Computer_Science/Lambda/labs/story_sqaud/Story Squad-6122da7459cf.json')
    client = vision.ImageAnnotatorClient(credentials=creds)
    response = client.document_text_detection(image=image, image_context=language)

    # List of confidence levels of each word
    symbol_confidences = []

    for page in response.full_text_annotation.pages:
        for block in page.blocks:
            for paragraph in block.paragraphs:
                for word in paragraph.words:
                    # word_confidences.append(word.confidence)
                    for symbol in word.symbols:
                        symbol_confidences.append(symbol.confidence)
    # If there is no text on the page
    if len(symbol_confidences) == 0:
        return None
    else:
        # Calculate and return the overall confidence for the page
        page_confidence = sum(symbol_confidences) / len(symbol_confidences)

        return float(page_confidence)
def character_multi_page(directory):
    '''
    Function calls page_confidence function for each page in a single submission
        Input: file path containing images of user's written stories
        Output: Float; Confidence in the transcription of the entire submission
    '''
    
    paths = []

    for item in os.listdir(directory):
        if item[-3:] == 'jpg':
            path = os.path.join(directory, item)
            paths.append(path)

    submission_confidence = []

    for image_path in paths:
        submission_confidence.append(character_page_confidence(image_path))
    
    # Drop any 'None' responses from confidence function
    submission_confidence = [x for x in submission_confidence if x is not None]

    # Check to make sure that submission_confidence isn't empty
    if submission_confidence == 0:
        return 0
    else:
        confidence_level = sum(submission_confidence) / len(submission_confidence)
        return confidence_level
df_25['confidence'] = 0.0
# Root directory
root_directory = os.path.join('/Users/stevenchase/Desktop/Steven/Computer_Science/Lambda/labs/story_sqaud', 'Stories Dataset', "Transcribed Stories")
# Calculate the confidence for each transcription
for i in range(df_25.shape[0]):
    # Build the directory path based off of the story_id
    directory = os.path.join(root_directory, str(df_25['story_id'][i])[:2] + '--', str(df_25['story_id'][i]))

    # Populate confidence column with call of multi_page function on each row
    df_25.loc[i, 'confidence'] = character_multi_page(directory)

df_25.head()<jupyter_output><empty_output><jupyter_text>Confidence by character per page metrics<jupyter_code>sns.regplot(x="error", y="confidence", data=df_25);
df_25.corr()<jupyter_output><empty_output><jupyter_text>Confidence by character for the entire submission<jupyter_code># Function to get submission confidence by character per page
def character_submission_confidence(image_path):
    '''
    Detects document features in images and returns overall confidence level of transcribed text
        Input: Path to file where images are stored
        - Assuming 1 image per image_path - run this on each individual image in one submission
        - Code for both local image_path and remote image_path, comment out
            the apporopriate one
        Output: Float. Decimal format of confidence level of the transcription for the page
    '''

    from google.cloud import vision
    import io
    from google.oauth2 import service_account

    # If image_path is local
    with io.open(image_path, 'rb') as image_file:
        content = image_file.read()
    image = vision.types.Image(content=content)

    # # If image_path is a uri
    # image = vision.types.Image()
    # image.source.image_uri = uri

    # Set language to english only
    language = vision.types.ImageContext(language_hints=['en-t-i0-handwrit'])

    # Connect to Google API client
    creds = service_account.Credentials.from_service_account_file('/Users/stevenchase/Desktop/Steven/Computer_Science/Lambda/labs/story_sqaud/Story Squad-6122da7459cf.json')
    client = vision.ImageAnnotatorClient(credentials=creds)
    response = client.document_text_detection(image=image, image_context=language)

    # List of confidence levels of each word
    symbol_confidences = []

    for page in response.full_text_annotation.pages:
        for block in page.blocks:
            for paragraph in block.paragraphs:
                for word in paragraph.words:
                    # word_confidences.append(word.confidence)
                    for symbol in word.symbols:
                        symbol_confidences.append(symbol.confidence)
    # If there is no text on the page
    if len(symbol_confidences) == 0:
        return None
    else:
        # Return list of all of the confidences for each character
        return symbol_confidences
def character_multi_page_submission(directory):
    '''
    Function calls page_confidence function for each page in a single submission
        Input: file path containing images of user's written stories
        Output: Float; Confidence in the transcription of the entire submission
    '''
    
    paths = []

    for item in os.listdir(directory):
        if item[-3:] == 'jpg':
            path = os.path.join(directory, item)
            paths.append(path)

    submission_confidence = []

    for image_path in paths:
        submission_confidence.extend(character_submission_confidence(image_path))
    
    # Drop any 'None' responses from confidence function
    submission_confidence = [x for x in submission_confidence if x is not None]

    # Check to make sure that submission_confidence isn't empty
    if len(submission_confidence) == 0:
        return 0
    else:
        confidence_level = sum(submission_confidence) / len(submission_confidence)
        return confidence_level
# Create new df
df_25_character_submission = df_25.copy()
df_25_character_submission['confidence'] = 0.0
df_25_character_submission.head()
# Calculate the confidence for each transcription
for i in range(df_25_character_submission.shape[0]):
    # Build the directory path based off of the story_id
    directory = os.path.join(root_directory, str(df_25_character_submission['story_id'][i])[:2] + '--', str(df_25_character_submission['story_id'][i]))

    # Populate confidence column with call of multi_page function on each row
    df_25_character_submission.loc[i, 'confidence'] = character_multi_page_submission(directory)

df_25_character_submission.head()<jupyter_output><empty_output><jupyter_text>Confidence by character for entire submission at once<jupyter_code>sns.regplot(x="error", y="confidence", data=df_25_character_submission[:25]);
df_25_character_submission.corr()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07575989002596609, 'fraction_numerical': 0.015172343567028155, 'mean_word_length': 4.015832482124617, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '108656', 'license': 'MIT', 'n_tokens_mistral': 5739, 'n_tokens_neox': 5296, 'n_words': 1958, 'path': 'notebooks/transcription_confidence.ipynb', 'repo_name': 'stimsonc/scribble-stadium-ds', 'size': '19641'}