<commit_before>"""
``FlowCal``'s Microsoft Excel User Interface.

This module contains functions to read, gate, and transform data from a set
of FCS files, as specified by an input Microsoft Excel file. This file
should contain the following tables:
    - **Instruments**: Describes the instruments used to acquire the
      samples listed in the other tables. Each instrument is specified by a
      row containing at least the following fields:

      - **ID**: Short string identifying the instrument. Will be referenced
        by samples in the other tables.
      - **Forward Scatter Channel**: Name of the forward scatter channel,
        as specified by the ``$PnN`` keyword in the associated FCS files.
      - **Side Scatter Channel**: Name of the side scatter channel, as
        specified by the ``$PnN`` keyword in the associated FCS files.
      - **Fluorescence Channels**: Name of the fluorescence channels in a
        comma-separated list, as specified by the ``$PnN`` keyword in the
        associated FCS files.
      - **Time Channel**: Name of the time channel, as specified by the
        ``$PnN`` keyword in the associated FCS files.

    - **Beads**: Describes the calibration beads samples that will be used
      to calibrate cell samples in the **Samples** table. The following
      information should be available for each beads sample:

      - **ID**: Short string identifying the beads sample. Will be
        referenced by cell samples in the **Samples** table.
      - **Instrument ID**: ID of the instrument used to acquire the sample.
        Must match one of the rows in the **Instruments** table.
      - **File Path**: Path of the FCS file containing the sample's data.
      - **<Fluorescence Channel Name> MEF Values**: The fluorescence in MEF
        of each bead subpopulation, as given by the manufacturer, as a
        comma-separated list of numbers. Any element of this list can be
        replaced with the word ``None``, in which case the corresponding
        subpopulation will not be used when fitting the beads fluorescence
        model. Note that the number of elements in this list (including
        the elements equal to ``None``) are the number of subpopulations
        that ``FlowCal`` will try to find.
      - **Gate fraction**: The fraction of events to keep from the sample
        after density-gating in the forward/side scatter channels.
      - **Clustering Channels**: The fluorescence channels used to identify
        the different bead subpopulations.

    - **Samples**: Describes the biological samples to be processed. The
      following information should be available for each sample:

      - **ID**: Short string identifying the sample. Will be used as part
        of the plot's filenames and in the **Histograms** table in the
        output Excel file.
      - **Instrument ID**: ID of the instrument used to acquire the sample.
        Must match one of the rows in the **Instruments** table.
      - **Beads ID**: ID of the beads sample used to convert data to
        calibrated MEF.
      - **File Path**: Path of the FCS file containing the sample's data.
      - **<Fluorescence Channel Name> Units**: Units to which the event
        list in the specified fluorescence channel should be converted, and
        all the subsequent plots and statistics should be reported. Should
        be one of the following: "Channel" (raw units), "a.u." or "RFI"
        (arbitrary units) or "MEF" (calibrated Molecules of Equivalent
        Fluorophore). If "MEF" is specified, the **Beads ID** should be
        populated, and should correspond to a beads sample with the
        **MEF Values** specified for the same channel.
      - **Gate fraction**: The fraction of events to keep from the sample
        after density-gating in the forward/side scatter channels.

Any columns other than the ones specified above can be present, but will be
ignored by ``FlowCal``.

"""

import collections
import os
import os.path
import platform
import re
import subprocess
import time
from Tkinter import Tk
from tkFileDialog import askopenfilename
import warnings

from matplotlib import pyplot as plt
import numpy as np
import pandas as pd

import FlowCal.io
import FlowCal.plot
import FlowCal.gate
import FlowCal.transform
import FlowCal.stats
import FlowCal.mef

# Regular expressions for headers that specify some fluorescence channel
re_mef_values = re.compile(r'^\s*(\S*)\s*MEF\s*Values\s*$')
re_units = re.compile(r'^\s*(\S*)\s*Units\s*$')

class ExcelUIException(Exception):
    """
    FlowCal Excel UI Error.

    """
    pass

def read_table(filename, sheetname, index_col=None):
    """
    Return the contents of an Excel table as a pandas DataFrame.

    Parameters
    ----------
    filename : str
        Name of the Excel file to read.
    sheetname : str or int
        Name or index of the sheet inside the Excel file to read.
    index_col : str, optional
        Column name or index to be used as row labels of the DataFrame. If
        None, default index will be used.

    Returns
    -------
    table : DataFrame
        A DataFrame containing the data in the specified Excel table. If
        `index_col` is not None, rows in which their `index_col` field
        is empty will not be present in `table`.

    Raises
    ------
    ValueError
        If `index_col` is specified and two rows contain the same
        `index_col` field.

    """
    # Catch sheetname as list or None
    if sheetname is None or hasattr(sheetname, '__iter__'):
        raise TypeError("sheetname should specify a single sheet")

    # Load excel table using pandas
    table = pd.read_excel(filename,
                          sheetname=sheetname,
                          index_col=index_col)
    # Eliminate rows whose index are null
    if index_col is not None:
        table = table[pd.notnull(table.index)]
    # Check for duplicated rows
    if table.index.has_duplicates:
        raise ValueError("sheet {} on file {} contains duplicated values "
                         "for column {}".format(sheetname, filename, index_col))

    return table

def write_workbook(filename, table_list, column_width=None):
    """
    Write an Excel workbook from a list of tables.

    Parameters
    ----------
    filename : str
        Name of the Excel file to write.
    table_list : list of ``(str, DataFrame)`` tuples
        Tables to be saved as individual sheets in the Excel table. Each
        tuple contains two values: the name of the sheet to be saved as a
        string, and the contents of the table as a DataFrame.
    column_width: int, optional
        The column width to use when saving the spreadsheet. If None,
        calculate width automatically from the maximum number of characters
        in each column.

    """
    # Modify default header format
    # Pandas' default header format is bold text with thin borders. Here we
    # use bold text only, without borders.
    # The format module is in pd.core.format in pandas<=0.18.0,
    # pd.formats.format in pandas>=0.18.1.
    try:
        format_module = pd.core.format
    except AttributeError as e:
        format_module = pd.formats.format
    old_header_style = format_module.header_style
    format_module.header_style = {"font": {"bold": True}}

    # Generate output writer object
    writer = pd.ExcelWriter(filename, engine='xlsxwriter')

    # Write tables
    for sheet_name, df in table_list:
        # Convert index names to regular columns
        df = df.reset_index()
        # Write to an Excel sheet
        df.to_excel(writer, sheet_name=sheet_name, index=False)
        # Set column width
        if column_width is None:
            for i, (col_name, column) in enumerate(df.iteritems()):
                # Get the maximum number of characters in a column
                max_chars_col = column.astype(str).str.len().max()
                max_chars_col = max(len(col_name), max_chars_col)
                # Write width
                writer.sheets[sheet_name].set_column(
                    i,
                    i,
                    width=1.*max_chars_col)
        else:
            writer.sheets[sheet_name].set_column(
                0,
                len(df.columns) - 1,
                width=column_width)

    # Write excel file
    writer.save()

    # Restore previous header format
    format_module.header_style = old_header_style

def process_beads_table(beads_table,
                        instruments_table,
                        base_dir=".",
                        verbose=False,
                        plot=False,
                        plot_dir=".",
                        full_output=False):
    """
    Process calibration bead samples, as specified by an input table.

    This function processes the entries in `beads_table`. For each row, the
    function does the following:
        - Load the FCS file specified in the field "File Path".
        - Transform the forward scatter/side scatter and fluorescence
          channels to RFI
        - Remove the 250 first and 100 last events.
        - Remove saturated events in the forward scatter and side scatter
          channels.
        - Apply density gating on the forward scatter/side scatter
          channels.
        - Generate a standard curve transformation function, for each
          fluorescence channel in which the associated MEF values are
          specified.
        - Generate forward/side scatter density plots and fluorescence
          histograms, and plots of the clustering and fitting steps of
          standard curve generation, if `plot` = True.
    
    Names of forward/side scatter and fluorescence channels are taken from
    `instruments_table`.

    Parameters
    ----------
    beads_table : DataFrame
        Table specifying beads samples to be processed. For more
        information about the fields required in this table, please consult
        the module's documentation.
    instruments_table : DataFrame
        Table specifying instruments. For more information about the fields
        required in this table, please consult the module's documentation.
    base_dir : str, optional
        Directory from where all the other paths are specified.
    verbose : bool, optional
        Whether to print information messages during the execution of this
        function.
    plot : bool, optional
        Whether to generate and save density/histogram plots of each
        sample, and each beads sample.
    plot_dir : str, optional
        Directory relative to `base_dir` into which plots are saved. If
        `plot` is False, this parameter is ignored.
    full_output : bool, optional
        Flag indicating whether to include an additional output, containing
        intermediate results from the generation of the MEF transformation
        functions.

    Returns
    -------
    beads_samples : list of FCSData objects
        A list of processed, gated, and transformed samples, as specified
        in `beads_table`, in the order of ``beads_table.index``.
    mef_transform_fxns : OrderedDict
        A dictionary of MEF transformation functions, indexed by
        ``beads_table.index``.
    mef_outputs : list
        A list with intermediate results of the generation of the MEF
        transformation functions, indexed by ``beads_table.index``. Only
        included if `full_output` is True.

    """
    # Initialize output variables
    beads_samples = []
    mef_transform_fxns = collections.OrderedDict()
    mef_outputs = []

    # Return empty structures if beads table is empty
    if beads_table.empty:
        if full_output:
            return beads_samples, mef_transform_fxns, mef_outputs
        else:
            return beads_samples, mef_transform_fxns

    if verbose:
        msg = "Processing Beads table ({} entries)".format(len(beads_table))
        print("")
        print(msg)
        print("="*len(msg))

    # Check that plotting directory exist, create otherwise
    if plot and not os.path.exists(os.path.join(base_dir, plot_dir)):
        os.makedirs(os.path.join(base_dir, plot_dir))

    # Extract header and channel names for which MEF values are specified.
    headers = list(beads_table.columns)
    mef_headers_all = [h for h in headers if re_mef_values.match(h)]
    mef_channels_all = [re_mef_values.search(h).group(1)
                        for h in mef_headers_all]

    # Iterate through table
    # We will look for a ExcelUIException on each iteration. If an exception
    # is caught, it will be stored in beads_samples.
    for beads_id, beads_row in beads_table.iterrows():
        try:
            ###
            # Instrument Data
            ###
            # Get the appropriate row in the instrument table
            instruments_row = instruments_table.loc[beads_row['Instrument ID']]
            # Scatter channels: Foward Scatter, Side Scatter
            sc_channels = [instruments_row['Forward Scatter Channel'],
                           instruments_row['Side Scatter Channel'],
                           ]
            # Fluorescence channels is a comma-separated list
            fl_channels = instruments_row['Fluorescence Channels'].split(',')
            fl_channels = [s.strip() for s in fl_channels]

            ###
            # Beads Data
            ###
            if verbose:
                print("\nBeads ID {}...".format(beads_id))
                print("Loading file \"{}\"...".format(beads_row['File Path']))

            # Attempt to open file
            filename = os.path.join(base_dir, beads_row['File Path'])
            try:
                beads_sample = FlowCal.io.FCSData(filename)
            except IOError:
                raise ExcelUIException("file \"{}\" not found".format(
                    beads_row['File Path']))
            # Check that the number of events is greater than 400
            if beads_sample.shape[0] < 400:
                raise ExcelUIException("number of events is lower than 400")

            ###
            # Transform
            ###
            if verbose:
                print("Performing data transformation...")
            # Transform FSC/SSC and fluorescence channels to linear scale
            beads_sample = FlowCal.transform.to_rfi(beads_sample,
                                                    sc_channels + fl_channels)

            # Parse clustering channels data
            cluster_channels = beads_row['Clustering Channels'].split(',')
            cluster_channels = [cc.strip() for cc in cluster_channels]

            ###
            # Gate
            ###
            if verbose:
                print("Performing gating...")
            # Remove first and last events. Transients in fluidics can make the
            # first few and last events slightly different from the rest.
            beads_sample_gated = FlowCal.gate.start_end(beads_sample,
                                                        num_start=250,
                                                        num_end=100)
            # Remove saturating events in forward/side scatter, if the FCS data
            # type is integer. The value of a saturating event is taken
            # automatically from `beads_sample_gated.range`.
            if beads_sample_gated.data_type == 'I':
                beads_sample_gated = FlowCal.gate.high_low(
                    beads_sample_gated,
                    channels=sc_channels)
            # Density gating
            beads_sample_gated, __, gate_contour = FlowCal.gate.density2d(
                data=beads_sample_gated,
                channels=sc_channels,
                gate_fraction=beads_row['Gate Fraction'],
                xscale='logicle',
                yscale='logicle',
                sigma=5.,
                full_output=True)

            # Plot forward/side scatter density plot and fluorescence histograms
            if plot:
                if verbose:
                    print("Plotting density plot and histogram...")
                # Density plot parameters
                density_params = {}
                density_params['mode'] = 'scatter'
                density_params["title"] = "{} ({:.1f}% retained)".format(
                    beads_id,
                    beads_sample_gated.shape[0] * 100. / beads_sample.shape[0])
                density_params['xscale'] = 'logicle'
                density_params['yscale'] = 'logicle'
                # Beads have a tight distribution, so axis limits will be set
                # from 0.75 decades below the 5th percentile to 0.75 decades
                # above the 95th percentile.
                density_params['xlim'] = \
                    (np.percentile(beads_sample_gated[:, sc_channels[0]],
                                   5) / (10**0.75),
                     np.percentile(beads_sample_gated[:, sc_channels[0]],
                                   95) * (10**0.75),
                     )
                density_params['ylim'] = \
                    (np.percentile(beads_sample_gated[:, sc_channels[1]],
                                   5) / (10**0.75),
                     np.percentile(beads_sample_gated[:, sc_channels[1]],
                                   95) * (10**0.75),
                     )
                # Beads have a tight distribution, so less smoothing should be
                # applied for visualization
                density_params['sigma'] = 5.
                # Histogram plot parameters
                hist_params = {'xscale': 'logicle'}
                # Plot
                figname = os.path.join(base_dir,
                                       plot_dir,
                                       "density_hist_{}.png".format(beads_id))
                plt.figure(figsize=(6,4))
                FlowCal.plot.density_and_hist(
                    beads_sample,
                    beads_sample_gated,
                    density_channels=sc_channels,
                    hist_channels=cluster_channels,
                    gate_contour=gate_contour,
                    density_params=density_params,
                    hist_params=hist_params,
                    savefig=figname)

            ###
            # Process MEF values
            ###
            # For each fluorescence channel, check whether a list of known MEF
            # values of the bead subpopulations is provided in `beads_row`. This
            # involves checking that a column named "[channel] MEF Values"
            # exists and is not empty. If so, store the name of the channel in
            # `mef_channels`, and the specified MEF values in `mef_values`.
            ###
            mef_values = []
            mef_channels = []
            for fl_channel in fl_channels:
                if fl_channel in mef_channels_all:
                    # Get header from channel name
                    mef_header = \
                        mef_headers_all[mef_channels_all.index(fl_channel)]
                    # Extract text. If empty, ignore.
                    mef_str = beads_row[mef_header]
                    if pd.isnull(mef_str):
                        continue
                    # Save channel name
                    mef_channels.append(fl_channel)
                    # Parse list of values
                    mef = mef_str.split(',')
                    mef = [int(e) if e.strip().isdigit() else np.nan
                           for e in mef]
                    mef_values.append(mef)
            mef_values = np.array(mef_values)

            # Obtain standard curve transformation
            if mef_channels:
                if verbose:
                    if len(mef_channels) == 1:
                        print("Calculating standard curve for channel {}..." \
                            .format(mef_channels[0]))
                    else:
                        print("Calculating standard curve for channels {}..." \
                            .format(", ".join(mef_channels)))

                mef_output = FlowCal.mef.get_transform_fxn(
                    beads_sample_gated,
                    mef_values,
                    mef_channels=mef_channels,
                    clustering_channels=cluster_channels,
                    verbose=False,
                    plot=plot,
                    plot_filename=beads_id,
                    plot_dir=os.path.join(base_dir, plot_dir),
                    full_output=full_output)

                if full_output:
                    mef_transform_fxn = mef_output.transform_fxn
                else:
                    mef_transform_fxn = mef_output

            else:
                mef_transform_fxn = None
                mef_output = None

        except ExcelUIException as e:
            # Print Exception message
            if verbose:
                print("ERROR: {}".format(str(e)))
            # Append exception to beads_samples array, and None to everything
            # else
            beads_samples.append(e)
            mef_transform_fxns[beads_id] = None
            if full_output:
                mef_outputs.append(None)<commit_msg>Excel UI process beads doesn't require plotting directory.<commit_after>"""
``FlowCal``'s Microsoft Excel User Interface.

This module contains functions to read, gate, and transform data from a set
of FCS files, as specified by an input Microsoft Excel file. This file
should contain the following tables:
    - **Instruments**: Describes the instruments used to acquire the
      samples listed in the other tables. Each instrument is specified by a
      row containing at least the following fields:

      - **ID**: Short string identifying the instrument. Will be referenced
        by samples in the other tables.
      - **Forward Scatter Channel**: Name of the forward scatter channel,
        as specified by the ``$PnN`` keyword in the associated FCS files.
      - **Side Scatter Channel**: Name of the side scatter channel, as
        specified by the ``$PnN`` keyword in the associated FCS files.
      - **Fluorescence Channels**: Name of the fluorescence channels in a
        comma-separated list, as specified by the ``$PnN`` keyword in the
        associated FCS files.
      - **Time Channel**: Name of the time channel, as specified by the
        ``$PnN`` keyword in the associated FCS files.

    - **Beads**: Describes the calibration beads samples that will be used
      to calibrate cell samples in the **Samples** table. The following
      information should be available for each beads sample:

      - **ID**: Short string identifying the beads sample. Will be
        referenced by cell samples in the **Samples** table.
      - **Instrument ID**: ID of the instrument used to acquire the sample.
        Must match one of the rows in the **Instruments** table.
      - **File Path**: Path of the FCS file containing the sample's data.
      - **<Fluorescence Channel Name> MEF Values**: The fluorescence in MEF
        of each bead subpopulation, as given by the manufacturer, as a
        comma-separated list of numbers. Any element of this list can be
        replaced with the word ``None``, in which case the corresponding
        subpopulation will not be used when fitting the beads fluorescence
        model. Note that the number of elements in this list (including
        the elements equal to ``None``) are the number of subpopulations
        that ``FlowCal`` will try to find.
      - **Gate fraction**: The fraction of events to keep from the sample
        after density-gating in the forward/side scatter channels.
      - **Clustering Channels**: The fluorescence channels used to identify
        the different bead subpopulations.

    - **Samples**: Describes the biological samples to be processed. The
      following information should be available for each sample:

      - **ID**: Short string identifying the sample. Will be used as part
        of the plot's filenames and in the **Histograms** table in the
        output Excel file.
      - **Instrument ID**: ID of the instrument used to acquire the sample.
        Must match one of the rows in the **Instruments** table.
      - **Beads ID**: ID of the beads sample used to convert data to
        calibrated MEF.
      - **File Path**: Path of the FCS file containing the sample's data.
      - **<Fluorescence Channel Name> Units**: Units to which the event
        list in the specified fluorescence channel should be converted, and
        all the subsequent plots and statistics should be reported. Should
        be one of the following: "Channel" (raw units), "a.u." or "RFI"
        (arbitrary units) or "MEF" (calibrated Molecules of Equivalent
        Fluorophore). If "MEF" is specified, the **Beads ID** should be
        populated, and should correspond to a beads sample with the
        **MEF Values** specified for the same channel.
      - **Gate fraction**: The fraction of events to keep from the sample
        after density-gating in the forward/side scatter channels.

Any columns other than the ones specified above can be present, but will be
ignored by ``FlowCal``.

"""

import collections
import os
import os.path
import platform
import re
import subprocess
import time
from Tkinter import Tk
from tkFileDialog import askopenfilename
import warnings

from matplotlib import pyplot as plt
import numpy as np
import pandas as pd

import FlowCal.io
import FlowCal.plot
import FlowCal.gate
import FlowCal.transform
import FlowCal.stats
import FlowCal.mef

# Regular expressions for headers that specify some fluorescence channel
re_mef_values = re.compile(r'^\s*(\S*)\s*MEF\s*Values\s*$')
re_units = re.compile(r'^\s*(\S*)\s*Units\s*$')

class ExcelUIException(Exception):
    """
    FlowCal Excel UI Error.

    """
    pass

def read_table(filename, sheetname, index_col=None):
    """
    Return the contents of an Excel table as a pandas DataFrame.

    Parameters
    ----------
    filename : str
        Name of the Excel file to read.
    sheetname : str or int
        Name or index of the sheet inside the Excel file to read.
    index_col : str, optional
        Column name or index to be used as row labels of the DataFrame. If
        None, default index will be used.

    Returns
    -------
    table : DataFrame
        A DataFrame containing the data in the specified Excel table. If
        `index_col` is not None, rows in which their `index_col` field
        is empty will not be present in `table`.

    Raises
    ------
    ValueError
        If `index_col` is specified and two rows contain the same
        `index_col` field.

    """
    # Catch sheetname as list or None
    if sheetname is None or hasattr(sheetname, '__iter__'):
        raise TypeError("sheetname should specify a single sheet")

    # Load excel table using pandas
    table = pd.read_excel(filename,
                          sheetname=sheetname,
                          index_col=index_col)
    # Eliminate rows whose index are null
    if index_col is not None:
        table = table[pd.notnull(table.index)]
    # Check for duplicated rows
    if table.index.has_duplicates:
        raise ValueError("sheet {} on file {} contains duplicated values "
                         "for column {}".format(sheetname, filename, index_col))

    return table

def write_workbook(filename, table_list, column_width=None):
    """
    Write an Excel workbook from a list of tables.

    Parameters
    ----------
    filename : str
        Name of the Excel file to write.
    table_list : list of ``(str, DataFrame)`` tuples
        Tables to be saved as individual sheets in the Excel table. Each
        tuple contains two values: the name of the sheet to be saved as a
        string, and the contents of the table as a DataFrame.
    column_width: int, optional
        The column width to use when saving the spreadsheet. If None,
        calculate width automatically from the maximum number of characters
        in each column.

    """
    # Modify default header format
    # Pandas' default header format is bold text with thin borders. Here we
    # use bold text only, without borders.
    # The format module is in pd.core.format in pandas<=0.18.0,
    # pd.formats.format in pandas>=0.18.1.
    try:
        format_module = pd.core.format
    except AttributeError as e:
        format_module = pd.formats.format
    old_header_style = format_module.header_style
    format_module.header_style = {"font": {"bold": True}}

    # Generate output writer object
    writer = pd.ExcelWriter(filename, engine='xlsxwriter')

    # Write tables
    for sheet_name, df in table_list:
        # Convert index names to regular columns
        df = df.reset_index()
        # Write to an Excel sheet
        df.to_excel(writer, sheet_name=sheet_name, index=False)
        # Set column width
        if column_width is None:
            for i, (col_name, column) in enumerate(df.iteritems()):
                # Get the maximum number of characters in a column
                max_chars_col = column.astype(str).str.len().max()
                max_chars_col = max(len(col_name), max_chars_col)
                # Write width
                writer.sheets[sheet_name].set_column(
                    i,
                    i,
                    width=1.*max_chars_col)
        else:
            writer.sheets[sheet_name].set_column(
                0,
                len(df.columns) - 1,
                width=column_width)

    # Write excel file
    writer.save()

    # Restore previous header format
    format_module.header_style = old_header_style

def process_beads_table(beads_table,
                        instruments_table,
                        base_dir=".",
                        verbose=False,
                        plot=False,
                        plot_dir=None,
                        full_output=False):
    """
    Process calibration bead samples, as specified by an input table.

    This function processes the entries in `beads_table`. For each row, the
    function does the following:
        - Load the FCS file specified in the field "File Path".
        - Transform the forward scatter/side scatter and fluorescence
          channels to RFI
        - Remove the 250 first and 100 last events.
        - Remove saturated events in the forward scatter and side scatter
          channels.
        - Apply density gating on the forward scatter/side scatter
          channels.
        - Generate a standard curve transformation function, for each
          fluorescence channel in which the associated MEF values are
          specified.
        - Generate forward/side scatter density plots and fluorescence
          histograms, and plots of the clustering and fitting steps of
          standard curve generation, if `plot` = True.
    
    Names of forward/side scatter and fluorescence channels are taken from
    `instruments_table`.

    Parameters
    ----------
    beads_table : DataFrame
        Table specifying beads samples to be processed. For more
        information about the fields required in this table, please consult
        the module's documentation.
    instruments_table : DataFrame
        Table specifying instruments. For more information about the fields
        required in this table, please consult the module's documentation.
    base_dir : str, optional
        Directory from where all the other paths are specified.
    verbose : bool, optional
        Whether to print information messages during the execution of this
        function.
    plot : bool, optional
        Whether to generate and save density/histogram plots of each
        sample, and each beads sample.
    plot_dir : str, optional
        Directory relative to `base_dir` into which plots are saved. If
        `plot` is False, this parameter is ignored. If ``plot==True`` and
        ``plot_dir is None``, plot without saving.
    full_output : bool, optional
        Flag indicating whether to include an additional output, containing
        intermediate results from the generation of the MEF transformation
        functions.

    Returns
    -------
    beads_samples : list of FCSData objects
        A list of processed, gated, and transformed samples, as specified
        in `beads_table`, in the order of ``beads_table.index``.
    mef_transform_fxns : OrderedDict
        A dictionary of MEF transformation functions, indexed by
        ``beads_table.index``.
    mef_outputs : list
        A list with intermediate results of the generation of the MEF
        transformation functions, indexed by ``beads_table.index``. Only
        included if `full_output` is True.

    """
    # Initialize output variables
    beads_samples = []
    mef_transform_fxns = collections.OrderedDict()
    mef_outputs = []

    # Return empty structures if beads table is empty
    if beads_table.empty:
        if full_output:
            return beads_samples, mef_transform_fxns, mef_outputs
        else:
            return beads_samples, mef_transform_fxns

    if verbose:
        msg = "Processing Beads table ({} entries)".format(len(beads_table))
        print("")
        print(msg)
        print("="*len(msg))

    # Check that plotting directory exist, create otherwise
    if plot and plot_dir is not None \
            and not os.path.exists(os.path.join(base_dir, plot_dir)):
        os.makedirs(os.path.join(base_dir, plot_dir))

    # Extract header and channel names for which MEF values are specified.
    headers = list(beads_table.columns)
    mef_headers_all = [h for h in headers if re_mef_values.match(h)]
    mef_channels_all = [re_mef_values.search(h).group(1)
                        for h in mef_headers_all]

    # Iterate through table
    # We will look for a ExcelUIException on each iteration. If an exception
    # is caught, it will be stored in beads_samples.
    for beads_id, beads_row in beads_table.iterrows():
        try:
            ###
            # Instrument Data
            ###
            # Get the appropriate row in the instrument table
            instruments_row = instruments_table.loc[beads_row['Instrument ID']]
            # Scatter channels: Foward Scatter, Side Scatter
            sc_channels = [instruments_row['Forward Scatter Channel'],
                           instruments_row['Side Scatter Channel'],
                           ]
            # Fluorescence channels is a comma-separated list
            fl_channels = instruments_row['Fluorescence Channels'].split(',')
            fl_channels = [s.strip() for s in fl_channels]

            ###
            # Beads Data
            ###
            if verbose:
                print("\nBeads ID {}...".format(beads_id))
                print("Loading file \"{}\"...".format(beads_row['File Path']))

            # Attempt to open file
            filename = os.path.join(base_dir, beads_row['File Path'])
            try:
                beads_sample = FlowCal.io.FCSData(filename)
            except IOError:
                raise ExcelUIException("file \"{}\" not found".format(
                    beads_row['File Path']))
            # Check that the number of events is greater than 400
            if beads_sample.shape[0] < 400:
                raise ExcelUIException("number of events is lower than 400")

            ###
            # Transform
            ###
            if verbose:
                print("Performing data transformation...")
            # Transform FSC/SSC and fluorescence channels to linear scale
            beads_sample = FlowCal.transform.to_rfi(beads_sample,
                                                    sc_channels + fl_channels)

            # Parse clustering channels data
            cluster_channels = beads_row['Clustering Channels'].split(',')
            cluster_channels = [cc.strip() for cc in cluster_channels]

            ###
            # Gate
            ###
            if verbose:
                print("Performing gating...")
            # Remove first and last events. Transients in fluidics can make the
            # first few and last events slightly different from the rest.
            beads_sample_gated = FlowCal.gate.start_end(beads_sample,
                                                        num_start=250,
                                                        num_end=100)
            # Remove saturating events in forward/side scatter, if the FCS data
            # type is integer. The value of a saturating event is taken
            # automatically from `beads_sample_gated.range`.
            if beads_sample_gated.data_type == 'I':
                beads_sample_gated = FlowCal.gate.high_low(
                    beads_sample_gated,
                    channels=sc_channels)
            # Density gating
            beads_sample_gated, __, gate_contour = FlowCal.gate.density2d(
                data=beads_sample_gated,
                channels=sc_channels,
                gate_fraction=beads_row['Gate Fraction'],
                xscale='logicle',
                yscale='logicle',
                sigma=5.,
                full_output=True)

            # Plot forward/side scatter density plot and fluorescence histograms
            if plot:
                if verbose:
                    print("Plotting density plot and histogram...")
                # Density plot parameters
                density_params = {}
                density_params['mode'] = 'scatter'
                density_params["title"] = "{} ({:.1f}% retained)".format(
                    beads_id,
                    beads_sample_gated.shape[0] * 100. / beads_sample.shape[0])
                density_params['xscale'] = 'logicle'
                density_params['yscale'] = 'logicle'
                # Beads have a tight distribution, so axis limits will be set
                # from 0.75 decades below the 5th percentile to 0.75 decades
                # above the 95th percentile.
                density_params['xlim'] = \
                    (np.percentile(beads_sample_gated[:, sc_channels[0]],
                                   5) / (10**0.75),
                     np.percentile(beads_sample_gated[:, sc_channels[0]],
                                   95) * (10**0.75),
                     )
                density_params['ylim'] = \
                    (np.percentile(beads_sample_gated[:, sc_channels[1]],
                                   5) / (10**0.75),
                     np.percentile(beads_sample_gated[:, sc_channels[1]],
                                   95) * (10**0.75),
                     )
                # Beads have a tight distribution, so less smoothing should be
                # applied for visualization
                density_params['sigma'] = 5.
                # Histogram plot parameters
                hist_params = {'xscale': 'logicle'}
                # Plot
                if plot_dir is not None:
                    figname = os.path.join(
                        base_dir,
                        plot_dir,
                        "density_hist_{}.png".format(beads_id))
                else:
                    figname = None
                plt.figure(figsize=(6,4))
                FlowCal.plot.density_and_hist(
                    beads_sample,
                    beads_sample_gated,
                    density_channels=sc_channels,
                    hist_channels=cluster_channels,
                    gate_contour=gate_contour,
                    density_params=density_params,
                    hist_params=hist_params,
                    savefig=figname)

            ###
            # Process MEF values
            ###
            # For each fluorescence channel, check whether a list of known MEF
            # values of the bead subpopulations is provided in `beads_row`. This
            # involves checking that a column named "[channel] MEF Values"
            # exists and is not empty. If so, store the name of the channel in
            # `mef_channels`, and the specified MEF values in `mef_values`.
            ###
            mef_values = []
            mef_channels = []
            for fl_channel in fl_channels:
                if fl_channel in mef_channels_all:
                    # Get header from channel name
                    mef_header = \
                        mef_headers_all[mef_channels_all.index(fl_channel)]
                    # Extract text. If empty, ignore.
                    mef_str = beads_row[mef_header]
                    if pd.isnull(mef_str):
                        continue
                    # Save channel name
                    mef_channels.append(fl_channel)
                    # Parse list of values
                    mef = mef_str.split(',')
                    mef = [int(e) if e.strip().isdigit() else np.nan
                           for e in mef]
                    mef_values.append(mef)
            mef_values = np.array(mef_values)

            # Obtain standard curve transformation
            if mef_channels:
                if verbose:
                    if len(mef_channels) == 1:
                        print("Calculating standard curve for channel {}..." \
                            .format(mef_channels[0]))
                    else:
                        print("Calculating standard curve for channels {}..." \
                            .format(", ".join(mef_channels)))

                mef_output = FlowCal.mef.get_transform_fxn(
                    beads_sample_gated,
                    mef_values,
                    mef_channels=mef_channels,
                    clustering_channels=cluster_channels,
                    verbose=False,
                    plot=plot,
                    plot_filename=beads_id,
                    plot_dir=os.path.join(base_dir, plot_dir) \
                             if plot_dir is not None else None,
                    full_output=full_output)

                if full_output:
                    mef_transform_fxn = mef_output.transform_fxn
                else:
                    mef_transform_fxn = mef_output

            else:
                mef_transform_fxn = None
                mef_output = None

        except ExcelUIException as e:
            # Print Exception message
            if verbose:
                print("ERROR: {}".format(str(e)))
            # Append exception to beads_samples array, and None to everything
            # else
            beads_samples.append(e)
            mef_transform_fxns[beads_id] = None
            if full_output:
                mef_outputs.append(None)
{'fraction_non_alphanumeric': 0.07236378994790933, 'fraction_numerical': 0.00403585339527899, 'mean_word_length': 1.854396892371576, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2597491', 'n_tokens_mistral': 11178, 'n_tokens_neox': 10368, 'n_words': 3880}