<commit_before>''' All functions acting on the hits of one DUT are listed here'''
from __future__ import division

import logging
import os.path

import tables as tb
import numpy as np
from scipy.ndimage import median_filter

from pixel_clusterizer.clusterizer import HitClusterizer
from testbeam_analysis.tools import analysis_utils
from testbeam_analysis.tools.plot_utils import plot_noisy_pixels, plot_cluster_size


def remove_noisy_pixels(input_hits_file, n_pixel, output_hits_file=None, pixel_size=None, threshold=10.0, dut_name=None, plot=True, chunk_size=1000000):
    '''Removes noisy pixel from the data file containing the hit table.
    The hit table is read in chunks and for each chunk the noisy pixel are determined and removed.

    To call this function on 8 cores in parallel with chunk_size=1000000 the following RAM is needed:
    11 byte * 8 * 1000000 = 88 Mb

    Parameters
    ----------
    input_hits_file : string
        Input PyTables raw data file.
    n_pixel : tuple
        Total number of pixels per column and row.
    pixel_size : tuple
        Pixel dimension for column and row. If None, assuming square pixels.
    threshold : float
        The threshold for pixel masking. The threshold is given in units of sigma of the pixel noise (background subtracted).
    chunk_size : int
        Chunk size of the data when reading from file.
    '''
    logging.info('=== Removing noisy pixel in %s ===', input_hits_file)

    if not output_hits_file:
        output_hits_file = os.path.splitext(input_hits_file)[0] + '_noisy_pixels.h5'

    occupancy = None
    # calculating occupancy array
    with tb.open_file(input_hits_file, 'r') as input_file_h5:
        for hits, _ in analysis_utils.data_aligned_at_events(input_file_h5.root.Hits, chunk_size=chunk_size):
            col, row = hits['column'], hits['row']
            chunk_occ = analysis_utils.hist_2d_index(col - 1, row - 1, shape=n_pixel)
            if occupancy is None:
                occupancy = chunk_occ
            else:
                occupancy = occupancy + chunk_occ

    # run median filter across data, assuming 0 filling past the edges
    blurred = median_filter(occupancy.astype(np.int32), size=2, mode='constant', cval=0.0)
    difference = np.ma.masked_array(occupancy - blurred)

    std = np.ma.std(difference)
    abs_occ_threshold = threshold * std
    occupancy = np.ma.masked_where(difference > abs_occ_threshold, occupancy)
    logging.info('Removed %d hot pixels at threshold %.1f in %s', np.ma.count_masked(occupancy), threshold, input_hits_file)

    # generate tuple col / row array of hot pixels, do not use getmask()
    noisy_pixels_mask = np.ma.getmaskarray(occupancy)
    # generate pair of col / row arrays
    noisy_pixels = np.nonzero(noisy_pixels_mask)
    # check for any noisy pixels
    if noisy_pixels[0].shape[0] != 0:
        # map 2d array (col, row) to 1d array to increase selection speed
        noisy_pixels_1d = (noisy_pixels[0] + 1) * n_pixel[1] + (noisy_pixels[1] + 1)
    else:
        noisy_pixels_1d = []

    # storing putput files
    with tb.open_file(input_hits_file, 'r') as input_file_h5:
        with tb.open_file(output_hits_file, 'w') as out_file_h5:
            # creating new hit table without noisy pixels
            hit_table_out = out_file_h5.createTable(out_file_h5.root, name='Hits', description=input_file_h5.root.Hits.dtype, title='Selected not noisy hits for test beam analysis', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
            for hits, _ in analysis_utils.data_aligned_at_events(input_file_h5.root.Hits, chunk_size=chunk_size):
                # Select not noisy pixel
                hits_1d = hits['column'].astype(np.uint32) * n_pixel[1] + hits['row']  # change dtype to fit new number<commit_msg>API: allow adjusting the median filter size<commit_after>''' All functions acting on the hits of one DUT are listed here'''
from __future__ import division

import logging
import os.path

import tables as tb
import numpy as np
from scipy.ndimage import median_filter

from pixel_clusterizer.clusterizer import HitClusterizer
from testbeam_analysis.tools import analysis_utils
from testbeam_analysis.tools.plot_utils import plot_noisy_pixels, plot_cluster_size


def remove_noisy_pixels(input_hits_file, n_pixel, output_hits_file=None, pixel_size=None, threshold=10.0, filter_size=3, dut_name=None, plot=True, chunk_size=1000000):
    '''Removes noisy pixel from the data file containing the hit table.
    The hit table is read in chunks and for each chunk the noisy pixel are determined and removed.

    To call this function on 8 cores in parallel with chunk_size=1000000 the following RAM is needed:
    11 byte * 8 * 1000000 = 88 Mb

    Parameters
    ----------
    input_hits_file : string
        Input PyTables raw data file.
    n_pixel : tuple
        Total number of pixels per column and row.
    pixel_size : tuple
        Pixel dimension for column and row. If None, assuming square pixels.
    threshold : float
        The threshold for pixel masking. The threshold is given in units of sigma of the pixel noise (background subtracted). The lower the value the more pixels are masked.
    filter_size : scalar or tuple
        Adjust the median filter size by giving the number of columns and rows. The higher the value the more the background is smoothed and more pixels are masked.
    chunk_size : int
        Chunk size of the data when reading from file.
    '''
    logging.info('=== Removing noisy pixel in %s ===', input_hits_file)

    if not output_hits_file:
        output_hits_file = os.path.splitext(input_hits_file)[0] + '_noisy_pixels.h5'

    occupancy = None
    # calculating occupancy array
    with tb.open_file(input_hits_file, 'r') as input_file_h5:
        for hits, _ in analysis_utils.data_aligned_at_events(input_file_h5.root.Hits, chunk_size=chunk_size):
            col, row = hits['column'], hits['row']
            chunk_occ = analysis_utils.hist_2d_index(col - 1, row - 1, shape=n_pixel)
            if occupancy is None:
                occupancy = chunk_occ
            else:
                occupancy = occupancy + chunk_occ

    # run median filter across data, assuming 0 filling past the edges
    blurred = median_filter(occupancy.astype(np.int32), size=filter_size, mode='constant', cval=0.0)
    difference = np.ma.masked_array(occupancy - blurred)

    std = np.ma.std(difference)
    abs_occ_threshold = threshold * std
    occupancy = np.ma.masked_where(difference > abs_occ_threshold, occupancy)
    logging.info('Removed %d hot pixels at threshold %.1f in %s', np.ma.count_masked(occupancy), threshold, input_hits_file)

    # generate tuple col / row array of hot pixels, do not use getmask()
    noisy_pixels_mask = np.ma.getmaskarray(occupancy)
    # generate pair of col / row arrays
    noisy_pixels = np.nonzero(noisy_pixels_mask)
    # check for any noisy pixels
    if noisy_pixels[0].shape[0] != 0:
        # map 2d array (col, row) to 1d array to increase selection speed
        noisy_pixels_1d = (noisy_pixels[0] + 1) * n_pixel[1] + (noisy_pixels[1] + 1)
    else:
        noisy_pixels_1d = []

    # storing putput files
    with tb.open_file(input_hits_file, 'r') as input_file_h5:
        with tb.open_file(output_hits_file, 'w') as out_file_h5:
            # creating new hit table without noisy pixels
            hit_table_out = out_file_h5.createTable(out_file_h5.root, name='Hits', description=input_file_h5.root.Hits.dtype, title='Selected not noisy hits for test beam analysis', filters=tb.Filters(complib='blosc', complevel=5, fletcher32=False))
            for hits, _ in analysis_utils.data_aligned_at_events(input_file_h5.root.Hits, chunk_size=chunk_size):
                # Select not noisy pixel
                hits_1d = hits['column'].astype(np.uint32) * n_pixel[1] + hits['row']  # change dtype to fit new number
{'fraction_non_alphanumeric': 0.08051061678463094, 'fraction_numerical': 0.01744186046511628, 'mean_word_length': 3.3887964503605104, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2743607', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2384, 'n_words': 769}