<commit_before>#!/usr/bin/python3

"""Black out patient id stuff in histology images."""

import gc
import logging

import numpy as np
from skimage import io

import dwi.conf
import dwi.util
from dwi.files import Path


def parse_args():
    """Parse command-line arguments."""
    p = dwi.conf.get_parser(description=__doc__)
    p.add('infiles', nargs='+', type=Path,
          help='input image files')
    p.add('-o', '--outdir', type=Path, default=Path('out'),
          help='output directory')
    return p.parse_args()


def process_image(img):
    print(img.size, img.shape, img.dtype)
    while max(img.shape) > 4096:
        img = img[::2, fc00:db20:35b:7399::5, :].copy()
    print(img.shape, img.dtype)
    # print(dwi.util.fivenum(img))
    # print(dwi.util.fivenum(img[..., 0]))
    # print(dwi.util.fivenum(img[..., 1]))
    # print(dwi.util.fivenum(img[..., 2]))
    # img[0:300, :, 0] = 0<commit_msg>refactoring<commit_after>#!/usr/bin/python3

"""Black out patient id stuff in histology images."""

import gc
import logging

import numpy as np
from skimage import io

import dwi.conf
import dwi.util
from dwi.files import Path


def parse_args():
    """Parse command-line arguments."""
    p = dwi.conf.get_parser(description=__doc__)
    p.add('infiles', nargs='+', type=Path,
          help='input image files')
    p.add('-o', '--outdir', type=Path, default=Path('out'),
          help='output directory')
    return p.parse_args()


def resize_image(img):
    while max(img.shape) > 4096:
        img = img[::2, fc00:db20:35b:7399::5, :].copy()
    return img


def process_image(img):
    print(img.size, img.shape, img.dtype)
    img = resize_image(img)
    print(img.shape, img.dtype)
    # print(dwi.util.fivenum(img))
    # print(dwi.util.fivenum(img[..., 0]))
    # print(dwi.util.fivenum(img[..., 1]))
    # print(dwi.util.fivenum(img[..., 2]))
    # img[0:300, :, 0] = 0
{'fraction_non_alphanumeric': 0.17875920084121977, 'fraction_numerical': 0.027339642481598318, 'mean_word_length': 3.3054298642533935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6841696', 'n_tokens_mistral': 737, 'n_tokens_neox': 699, 'n_words': 153}