<jupyter_start><jupyter_text>Mask R-CNN - Statistics Extractor  Initialisation  Detecting Environment<jupyter_code>import sys
IN_COLAB = 'google.colab' in sys.modules
print("Executing in Google Colab" if IN_COLAB else "Executing locally")<jupyter_output><empty_output><jupyter_text>Getting all the needed files<jupyter_code>if IN_COLAB:
    import shutil
    shutil.rmtree('sample_data/', ignore_errors=True)
    import os
    GITHUB_REPO = "https://raw.githubusercontent.com/AdrienJaugey/Mask-R-CNN-Inference-Tool/stable/"
    files = ['datasetTools/datasetDivider.py', 'datasetTools/AnnotationAdapter.py',
             'datasetTools/ASAPAdapter.py', 'datasetTools/LabelMeAdapter.py', 'common_utils.py']
    for fileToDownload in files:
        url = GITHUB_REPO + fileToDownload
        !wget -qN $url
        if '/' in fileToDownload:
            destDir = fileToDownload.split('/')[0]
            fileName = fileToDownload.split('/')[1].replace('/', '')
            os.makedirs(destDir, exist_ok=True)
            !mv $fileName $fileToDownload<jupyter_output><empty_output><jupyter_text>Importing annotations file(s) to Google Colab<jupyter_code>import json
import os
import cv2
import numpy as np
from datasetTools import AnnotationAdapter as adapt
from datasetTools.datasetDivider import getBWCount
if IN_COLAB:
    howToGetAnnotationFiles = "Upload" #@param ["Upload", "From Google Drive"]
    os.makedirs('annotations', exist_ok=True)
if IN_COLAB and howToGetAnnotationFiles == "Upload":
    print("Please upload the annotation file(s) you want to get the stats of.")
    from google.colab import files
    import shutil
    uploaded = files.upload()
    for fileName in uploaded:
        if fileName.split('.')[-1] in adapt.ANNOTATION_FORMAT:
            shutil.move(fileName, "annotations/{}".format(fileName))
        else:
            os.remove(fileName)
if IN_COLAB and howToGetAnnotationFiles == "From Google Drive":
    from google.colab import drive
    drive.mount('/content/drive')
    pathToDrive = "'/content/drive/My Drive/"
    customPathInDrive = "" #@param {type:"string"}
    annotationFilePath = "" #@param{type:"string"}
    if annotationFilePath != "":
        pathToAnnotation = pathToDrive + customPathInDrive + annotationFilePath + "'"
        tempPath = "annotations/" + annotationFilePath
        print("Copying {} to {}".format(pathToAnnotation, tempPath))
        !cp -u $pathToAnnotation $tempPath
    else:
        pathToAnnotationFolder = "/content/drive/My Drive/" + customPathInDrive
        fileList = os.listdir(pathToAnnotationFolder)
        for dataFile in fileList:
            if dataFile.split('.')[-1] in adapt.ANNOTATION_FORMAT:
                pathToFile = pathToDrive + customPathInDrive + dataFile + "'"
                tempPath = 'annotations/' + dataFile
                print("Copying {} to {}".format(pathToFile, tempPath))
                !cp -u $pathToFile $tempPath<jupyter_output><empty_output><jupyter_text>Declaration of functions<jupyter_code>def getMaskArea(ptsMask):
    # Formating coordinates to be usable
    ptsMask = np.double(ptsMask)
    ptsMask = np.matrix.round(ptsMask)
    ptsMask = np.int32(ptsMask)

    xs = ptsMask[:, 0]
    ys = ptsMask[:, 1]

    xMin = np.amin(xs)
    xMax = np.amax(xs)
    yMin = np.amin(ys)
    yMax = np.amax(ys)

    ptsMask[:, 0] -= np.ones((ptsMask.shape[0]), dtype=np.uint8) * xMin
    ptsMask[:, 1] -= np.ones((ptsMask.shape[0]), dtype=np.uint8) * yMin

    mask = np.uint8(np.zeros((yMax - yMin, xMax - xMin)))
    cv2.fillPoly(mask, [ptsMask], 255)

    return getBWCount(mask)[1]
def getStatsFromAnnotation(annotationPath):
    adapter = None
    adapters = adapt.ANNOTATION_ADAPTERS
    adapterPriority = -1

    for a in adapters.values():
        if a.canRead(annotationPath):
            if a.getPriorityLevel() > adapterPriority:
                adapterPriority = a.getPriorityLevel()
                adapter = a

    masks = adapter.readFile(annotationPath)

    stats = {}

    for datasetClass, ptMask in masks:
        if type(datasetClass) is int:
            if datasetClass < len(classesInfo) and classesInfo[datasetClass]["id"] == datasetClass:
                maskClass = classesInfo[datasetClass]["name"]
            else:
                for classInfo in classesInfo:
                    if classInfo["id"] == datasetClass:
                        maskClass = classInfo["name"]
                        break
        else:
            maskClass = datasetClass.lower().strip(' ').replace(" ", "_")

        if maskClass not in stats:
            stats[maskClass] = {"count": 0, "area": 0}

        stats[maskClass]["count"] += 1
        stats[maskClass]["area"] += getMaskArea(ptMask)
    return stats<jupyter_output><empty_output><jupyter_text>Link between classes names and IDs<jupyter_code>classesInfo = [
    {"id":  0, "name": "cortex"},
    {"id":  1, "name": "medullaire"},
    {"id":  2, "name": "capsule"},
    {"id":  3, "name": "tubule_sain"},
    {"id":  4, "name": "tubule_atrophique"},
    {"id":  5, "name": "nsg"},
    {"id":  6, "name": "nsg_complet"},
    {"id":  7, "name": "nsg_partiel"},
    {"id":  8, "name": "pac"},
    {"id":  9, "name": "veine"},
    {"id": 10, "name": "vaisseau"},
    {"id": 11, "name": "intima"},
    {"id": 12, "name": "media"}
]<jupyter_output><empty_output><jupyter_text>Extraction of stats<jupyter_code>annotationsDir = "annotations/" #@param {type:"string"}
outputDir = "statistics/" #@param {type:"string"}
annotationsDir = os.path.normpath(annotationsDir)
saveStats = True #@param {type:"boolean"}
displayStats = False #@param {type:"boolean"}
if saveStats:
    os.makedirs(outputDir, exist_ok=True)
if saveStats or displayStats:
    files = os.listdir(annotationsDir)
    total = len(files)
    step = total // 4
    for idx, annotationFile in enumerate(files):
        if not displayStats and (idx + 1) % step == 0:
            print(f'\r{(idx + 1) / total:5.2%}', end="")
        fileExt = annotationFile.split('.')[-1]
        if os.path.isfile(os.path.join(annotationsDir, annotationFile)) and fileExt in adapt.ANNOTATION_FORMAT:
            imageName = annotationFile.split('.')[0]
            stats = getStatsFromAnnotation(os.path.join(annotationsDir, annotationFile))
            if displayStats:
                print("{} statistics :".format(imageName))
                for eltClass in stats:
                    print(" - {} : count = {}, area = {}".format(eltClass, stats[eltClass]["count"], stats[eltClass]["area"]))
                print()
            if saveStats:
                with open(os.path.join(outputDir, '{}_stats.json'.format(imageName)), 'w') as statFile:
                    json.dump(stats, statFile, indent="\t")
print("\nDone !")<jupyter_output><empty_output><jupyter_text>Exporting results<jupyter_code>if IN_COLAB and saveStats:
        !zip -qr statistics.zip statistics/
        print("Results can be downloaded on the Files tab on the left")
        print("Zip file name is : statistics.zip")
        from google.colab import files
        files.download('statistics.zip')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13651685393258428, 'fraction_numerical': 0.006882022471910112, 'mean_word_length': 3.092528735632184, 'pattern_counts': {'":': 31, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '562670', 'license': 'MIT', 'n_tokens_mistral': 2277, 'n_tokens_neox': 2197, 'n_words': 502, 'path': 'Mask_R_CNN_Stats_Extractor.ipynb', 'repo_name': 'AdrienJaugey/Custom-Mask-R-CNN-', 'size': '7120'}