<commit_before>#!/usr/bin/env python

# Functions:
# list_all_samples
# list_common_samples
# peek_samples_hint
#
# align_matrices        Align a set of matrices.
# align_matrix          Align either expression or annotation matrices.
# align_express
# align_annot
# align_samples
#
# add_missing_samples
#
# get_samples           Get the samples from an express or annot matrix.
# get_express_samples
# get_annot_samples
#
# cmp_sample
# find_sample
# intersect_samples
# uniq_samples
# strip_nonalnum
#
# write_matrix
# read_express
# write_express
# read_annot
# write_annot

import os
import sys


class AnnotationMatrix:
    def __init__(self, name2annots, name_order):
        self.name2annots = name2annots.copy()
        self.name_order = name_order[:]


def list_all_samples(matrix_data, case_insensitive, hash_samples,
                     ignore_nonalnum):
    from genomicode import hashlib

    assert matrix_data

    # Get the samples that occur in any of the files.  Preserve the
    # order of the samples.
    all_samples = []
    for x in matrix_data:
        infile, outfile, matrix, header, samples = x
        all_samples.extend(samples)

    # Get rid of duplicate samples.
    duplicated = [False] * len(all_samples)
    all_samples_cmp = all_samples
    if case_insensitive:
        all_samples_cmp = [x.upper() for x in all_samples_cmp]
    if hash_samples:
        all_samples_cmp = [hashlib.hash_var(x) for x in all_samples_cmp]
    if ignore_nonalnum:
        all_samples_cmp = [strip_nonalnum(x) for x in all_samples_cmp]

    seen = {}
    for i, s in enumerate(all_samples_cmp):
        if s in seen:
            duplicated[i] = True
        seen[s] = 1

    all_samples = [
        all_samples[i] for i in range(len(all_samples)) if not duplicated[i]]
    return all_samples


def list_common_samples(matrix_data, case_insensitive, hash_samples,
                        ignore_nonalnum):
    assert matrix_data

    # Initialize the common samples.
    samples_hint = peek_samples_hint(matrix_data)
    assert samples_hint
    common = samples_hint

    # Get the samples that are in all of the data sets.  Keep track of
    # the order of the samples in the first file so we can resort them
    # later.
    first_samples = None
    for i, x in enumerate(matrix_data):
        infile, outfile, matrix, header, samples = x
        common = intersect_samples(
            common, samples, case_insensitive, hash_samples, ignore_nonalnum)
        if i == 0:
            first_samples = samples

    # Finally, order the annotations according to the first file given.
    assert first_samples
    samples = intersect_samples(
        first_samples, common, case_insensitive, hash_samples, ignore_nonalnum)

    return samples


def peek_samples_hint(matrix_data):
    # Figure out what the samples look like.  Look for an expression
    # file first.  If an expression file is not found, use the
    # annotation files.
    samples_hint = None
    x = [x for x in matrix_data if not isinstance(x[2], AnnotationMatrix)]
    if x:
        # Found an expression file.
        infile, outfile, matrix, header = x[0][:4]
        assert header is None
        samples_hint = get_express_samples(matrix)<commit_msg>bug fix<commit_after>#!/usr/bin/env python

# Functions:
# list_all_samples
# list_common_samples
# peek_samples_hint
#
# align_matrices        Align a set of matrices.
# align_matrix          Align either expression or annotation matrices.
# align_express
# align_annot
# align_samples
#
# add_missing_samples
#
# get_samples           Get the samples from an express or annot matrix.
# get_express_samples
# get_annot_samples
#
# cmp_sample
# find_sample
# intersect_samples
# uniq_samples
# strip_nonalnum
#
# write_matrix
# read_express
# write_express
# read_annot
# write_annot

import os
import sys


class AnnotationMatrix:
    def __init__(self, name2annots, name_order):
        self.name2annots = name2annots.copy()
        self.name_order = name_order[:]


def list_all_samples(matrix_data, case_insensitive, hash_samples,
                     ignore_nonalnum):
    from genomicode import hashlib

    assert matrix_data

    # Get the samples that occur in any of the files.  Preserve the
    # order of the samples.
    all_samples = []
    for x in matrix_data:
        infile, outfile, matrix, header, samples = x
        all_samples.extend(samples)

    # Get rid of duplicate samples.
    duplicated = [False] * len(all_samples)
    all_samples_cmp = all_samples
    if case_insensitive:
        all_samples_cmp = [x.upper() for x in all_samples_cmp]
    if hash_samples:
        all_samples_cmp = [hashlib.hash_var(x) for x in all_samples_cmp]
    if ignore_nonalnum:
        all_samples_cmp = [strip_nonalnum(x) for x in all_samples_cmp]

    seen = {}
    for i, s in enumerate(all_samples_cmp):
        if s in seen:
            duplicated[i] = True
        seen[s] = 1

    all_samples = [
        all_samples[i] for i in range(len(all_samples)) if not duplicated[i]]
    return all_samples


def list_common_samples(matrix_data, case_insensitive, hash_samples,
                        ignore_nonalnum):
    assert matrix_data

    # Initialize the common samples.
    samples_hint = peek_samples_hint(matrix_data)
    assert samples_hint
    common = samples_hint

    # Get the samples that are in all of the data sets.  Keep track of
    # the order of the samples in the first file so we can resort them
    # later.
    first_samples = None
    for i, x in enumerate(matrix_data):
        infile, outfile, matrix, header, samples = x
        common = intersect_samples(
            common, samples, case_insensitive, hash_samples, ignore_nonalnum)
        if i == 0:
            first_samples = samples

    # Finally, order the annotations according to the first file given.
    assert first_samples
    samples = intersect_samples(
        first_samples, common, case_insensitive, hash_samples, ignore_nonalnum)

    return samples


def peek_samples_hint(matrix_data):
    # Figure out what the samples look like.  Look for an expression
    # file first.  If an expression file is not found, use the
    # annotation files.
    samples_hint = None
    x = [x for x in matrix_data if not isinstance(x[2], AnnotationMatrix)]
    if x:
        # Found an expression file.
        infile, outfile, matrix, header = x[0][:4]
        #assert header is None
        x, samples_hint = get_express_samples(matrix)
{'fraction_non_alphanumeric': 0.06641952425084956, 'fraction_numerical': 0.002471424158171146, 'mean_word_length': 2.9554062309102016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4521522', 'n_tokens_mistral': 1983, 'n_tokens_neox': 1911, 'n_words': 609}