<commit_before>from collections import defaultdict
import gzip
import os
import argparse
import json

import numpy as np
from sklearn.decomposition import PCA

parser = argparse.ArgumentParser(description="PCA")
parser.add_argument('samples', help="All samples (comma separated)")
parser.add_argument('sample_ids', help="Sample IDs (comma separated")
parser.add_argument('--genes', nargs='+', help='filter genes')
args = parser.parse_args()


def isfloat(value):
    try:
        float(value)
        return True
    except ValueError:
        return False


def isgzipped(f):
    with open(f, 'rb') as rpkm_file:
        magic = rpkm_file.read(2)

    return magic == '\037\213'


samples = args.samples.split(',')
sample_ids = args.sample_ids.split(',')

if len(samples) != len(sample_ids):
    print '{"rc":"1"}'
    exit(1)

exp = []
allgenes = set()

for fname in samples:
    myopen = gzip.open if isgzipped(fname) else open

    with myopen(fname) as f:
        exp.append({gene_exp[0]: float(gene_exp[1]) for gene_exp in
                    (l.split('\t') for l in f) if len(gene_exp) == 2
                    and isfloat(gene_exp[1])})

        allgenes.update(exp[-1].keys())

if args.genes:
    allgenes = allgenes.intersection(args.genes)

# Default expression value is 0.
exp = np.array([[genemap.get(g, 0.) for g in allgenes] for genemap in exp])

pca = PCA(n_components=2)
transformed_data = pca.fit_transform(exp)

data = {'pca': {'flot': {'data': transformed_data.tolist(), 'xlabel': 'PC 1',
        'ylabel': 'PC 2', 'samples': sample_ids}}}

print json.dumps(data, separators=(',', ':'))
<commit_msg>PCA processor: changed 'n_components' parameter<commit_after>from collections import defaultdict
import gzip
import os
import argparse
import json

import numpy as np
from sklearn.decomposition import PCA

parser = argparse.ArgumentParser(description="PCA")
parser.add_argument('samples', help="All samples (comma separated)")
parser.add_argument('sample_ids', help="Sample IDs (comma separated")
parser.add_argument('--genes', nargs='+', help='filter genes')
args = parser.parse_args()


def isfloat(value):
    try:
        float(value)
        return True
    except ValueError:
        return False


def isgzipped(f):
    with open(f, 'rb') as rpkm_file:
        magic = rpkm_file.read(2)

    return magic == '\037\213'


samples = args.samples.split(',')
sample_ids = args.sample_ids.split(',')

if len(samples) != len(sample_ids):
    print '{"rc":"1"}'
    exit(1)

exp = []
allgenes = set()

for fname in samples:
    myopen = gzip.open if isgzipped(fname) else open

    with myopen(fname) as f:
        exp.append({gene_exp[0]: float(gene_exp[1]) for gene_exp in
                    (l.split('\t') for l in f) if len(gene_exp) == 2
                    and isfloat(gene_exp[1])})

        allgenes.update(exp[-1].keys())

if args.genes:
    allgenes = allgenes.intersection(args.genes)

# Default expression value is 0.
exp = np.array([[genemap.get(g, 0.) for g in allgenes] for genemap in exp])

pca = PCA(n_components=0.99, whiten=True)
transformed_data = pca.fit_transform(exp)

data = {'pca': {'flot': {'data': transformed_data.tolist(), 'xlabel': 'PC 1',
        'ylabel': 'PC 2', 'samples': sample_ids}}}

print json.dumps(data, separators=(',', ':'))

{'fraction_non_alphanumeric': 0.14902676399026765, 'fraction_numerical': 0.012165450121654502, 'mean_word_length': 3.7187948350071736, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3677642', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1103, 'n_words': 291}