<jupyter_start><jupyter_text>FunctionsSome utility functions for data analysis, preparation or visualization.<jupyter_code>import albumentations as albu<jupyter_output><empty_output><jupyter_text>RLE  RLE encoding<jupyter_code>def rle_encoding(x):
    '''
    x: numpy array of shape (height, width), 1 - mask, 0 - background
    Returns run length as list
    '''
    dots = np.where(x.T.flatten()==1)[0] # .T sets Fortran order down-then-right
    run_lengths = []
    prev = -2
    for b in dots:
        if (b>prev+1): run_lengths.extend((b+1, 0))
        run_lengths[-1] += 1
        prev = b
    
    return ' '.join(map(str, run_lengths))<jupyter_output><empty_output><jupyter_text>Deconding<jupyter_code>def rle_to_mask(rle_string, height, width):
    '''
    convert RLE(run length encoding) string to numpy array

    Parameters: 
    rle_string (str): string of rle encoded mask
    height (int): height of the mask
    width (int): width of the mask

    Returns: 
    numpy.array: numpy array of the mask
    '''
    
    rows, cols = height, width
    
    if rle_string == -1 or rle_string == "" or not type(rle_string) is str:
        return np.zeros((height, width))
    else:
        rle_numbers = [int(num_string) for num_string in rle_string.split(' ')]
        rle_pairs = np.array(rle_numbers).reshape(-1,2)
        img = np.zeros(rows*cols, dtype=np.uint8)
        for index, length in rle_pairs:
            index -= 1
            img[index:index+length] = 255
        img = img.reshape(cols,rows)
        img = img.T
        return img
    
# build the masks from the encoded pixels
# decode = lambda m: rle_to_mask(m, 1400, 2100)<jupyter_output><empty_output><jupyter_text>Image augmenation<jupyter_code>def get_training_augmentation():
    train_transform = [
        albu.HorizontalFlip(p=0.5),
        albu.VerticalFlip(p=0.5),
        albu.Blur(p=0.5),
        albu.ShiftScaleRotate(
            scale_limit=0.5,
            rotate_limit=0,
            shift_limit=0.1,
            p=0.5,
            border_mode=0
        ),
        albu.GridDistortion(p=0.5),        
    ]
    return albu.Compose(train_transform)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1135110294117647, 'fraction_numerical': 0.019301470588235295, 'mean_word_length': 2.7150170648464163, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '367133', 'license': 'MIT', 'n_tokens_mistral': 781, 'n_tokens_neox': 752, 'n_words': 166, 'path': 'notebooks/utils/functions.ipynb', 'repo_name': 'laurentgrenier/dl-handwritting-recognition', 'size': '2176'}