<commit_before>/*eslint-env browser */
/*global require: false, module: false */

'use strict';

var _ = require('../underscore_ext');
var Rx = require('rx');
var xenaQuery = require('../xenaQuery');
var kmModel = require('../models/km');
var {reifyErrors, collectResults} = require('./errors');
var {setCohort, fetchDatasets, fetchSamples, fetchColumnData} = require('./common');
var {xenaFieldPaths, setFieldType} = require('../models/fieldSpec');
var {getColSpec} = require('../models/datasetJoins');
var {setNotifications} = require('../notifications');
var fetchSamplesFrom = require('../samplesFrom');
var fetch = require('../fieldFetch');

var identity = x => x;

var unionOfResults = resps => collectResults(resps, results => _.union(...results));

function cohortQuery(servers) {
	return Rx.Observable.zipArray(_.map(servers, s => reifyErrors(xenaQuery.all_cohorts(s), {host: s})))
			.flatMap(unionOfResults);
}

function fetchCohorts(serverBus, servers) {
	serverBus.onNext(['cohorts', cohortQuery(servers)]);
}

function exampleQuery(dsID) {
	return xenaQuery.dsID_fn(xenaQuery.dataset_field_examples)(dsID)
		.map(list => _.pluck(list, 'name'));
}

function fetchExamples(serverBus, state, dsID) {
	serverBus.onNext(['columnEdit-examples', exampleQuery(dsID)]);
}

// Normalization of fields from user input
function geneProbeMapLookup(settings, state) {
	const {host} = JSON.parse(settings.dsID),
		probemap = state.datasets[settings.dsID].probemap;
	return  xenaQuery.sparse_data_match_genes(host, probemap, settings.fields);
}

function probeLookup(settings) {
	const {host, name} = JSON.parse(settings.dsID);
	return xenaQuery.match_fields(host, name, settings.fields);
}

function mutationGeneLookup(settings) {
	const {host, name} = JSON.parse(settings.dsID);
	return xenaQuery.sparse_data_match_genes(host, name, settings.fields);
}

const fieldLookup = {
	'geneProbes': geneProbeMapLookup,
	'genes': geneProbeMapLookup,
	'mutation': mutationGeneLookup,
	'probes': probeLookup
};

function allFieldsLookup(settings, xenaFields, state) {
	var fieldSpecs = _.map(xenaFields, path => _.getIn(settings, path));
	return Rx.Observable.zipArray(
		_.map(fieldSpecs, fs => reifyErrors(
			(fieldLookup[fs.fieldType] || probeLookup)(fs, state)), {/*host info?*/}));
}

function normalizeFields(serverBus, state, id, settings) {
	var xenaFields = xenaFieldPaths(settings),
		lookup = allFieldsLookup(settings, xenaFields, state);
	serverBus.onNext(['normalize-fields', lookup, id, settings, xenaFields]);
}

var featuresInCohort = (datasets, features, cohort) =>
		_.pick(features, (f, dsID) => datasets[dsID].cohort === cohort);

var survivalVarsForCohorts = (cohort, datasets, features) =>
	_.map(cohort, ({name}) =>
			kmModel.pickSurvivalVars(featuresInCohort(datasets, features, name)));

var hasSurvFields  = vars => !!(vars.ev && vars.tte && vars.patient);

var probeFieldSpec = ({dsID, name}) => ({
	dsID,
	fetchType: 'xena', // maybe take from dataset meta instead of hard-coded
	valueType: 'float',
	fieldType: 'probes',
	fields: [name]
});

var codedFieldSpec = ({dsID, name}) => ({
	dsID,
	fetchType: 'xena', // maybe take from dataset meta instead of hard-coded
	valueType: 'coded',
	fieldType: 'clinical',
	fields: [name]
});

function mapToObj(keys, fn) {
	return _.object(keys, _.map(keys, fn));
}

var survFields = ['ev', 'tte', 'patient'];
var getFieldSpec = _.object(survFields,
		[probeFieldSpec, probeFieldSpec, codedFieldSpec]);

// XXX Note that we merge the 'patient' field across cohorts. This will
// be wrong in some cases, where 'patient' aliases, and right in other cases,
// such as TCGA patients in multiple cohorts.
function survivalFields(cohorts, datasets, features) {
	var vars = survivalVarsForCohorts(cohorts, datasets, features);
	return _.find(vars, hasSurvFields) ?  // at least one cohort w/surv data
		mapToObj(survFields, fname =>
				getColSpec(_.map(_.pluck(vars, fname), getFieldSpec[fname]), datasets))
		: null;
}

// If field set has changed, re-fetch.
function fetchSurvival(serverBus, state) {
	let {cohort, datasets, features, survival, cohortSamples} = state,
		fields = survivalFields(cohort, datasets, features),
		refetch = _.some(survFields,
				f => !_.isEqual(fields[f], _.getIn(survival, [f, 'field']))),

		queries = _.map(survFields, key => fetch(fields[key], cohortSamples)),
		collate = data => mapToObj(survFields,
				(k, i) => ({field: fields[k], data: data[i]}));

	// This could be optimized by grouping by server.
	refetch && serverBus.onNext([
			'km-survival-data', Rx.Observable.zipArray(...queries).map(collate)]);
}

var shouldSetCohort = state => (state.cohortPending && state.cohort !== state.cohortPending);<commit_msg>Handle km w/missing cohort data.<commit_after>/*eslint-env browser */
/*global require: false, module: false */

'use strict';

var _ = require('../underscore_ext');
var Rx = require('rx');
var xenaQuery = require('../xenaQuery');
var kmModel = require('../models/km');
var {reifyErrors, collectResults} = require('./errors');
var {setCohort, fetchDatasets, fetchSamples, fetchColumnData} = require('./common');
var {nullField, xenaFieldPaths, setFieldType} = require('../models/fieldSpec');
var {getColSpec} = require('../models/datasetJoins');
var {setNotifications} = require('../notifications');
var fetchSamplesFrom = require('../samplesFrom');
var fetch = require('../fieldFetch');

var identity = x => x;

var unionOfResults = resps => collectResults(resps, results => _.union(...results));

function cohortQuery(servers) {
	return Rx.Observable.zipArray(_.map(servers, s => reifyErrors(xenaQuery.all_cohorts(s), {host: s})))
			.flatMap(unionOfResults);
}

function fetchCohorts(serverBus, servers) {
	serverBus.onNext(['cohorts', cohortQuery(servers)]);
}

function exampleQuery(dsID) {
	return xenaQuery.dsID_fn(xenaQuery.dataset_field_examples)(dsID)
		.map(list => _.pluck(list, 'name'));
}

function fetchExamples(serverBus, state, dsID) {
	serverBus.onNext(['columnEdit-examples', exampleQuery(dsID)]);
}

// Normalization of fields from user input
function geneProbeMapLookup(settings, state) {
	const {host} = JSON.parse(settings.dsID),
		probemap = state.datasets[settings.dsID].probemap;
	return  xenaQuery.sparse_data_match_genes(host, probemap, settings.fields);
}

function probeLookup(settings) {
	const {host, name} = JSON.parse(settings.dsID);
	return xenaQuery.match_fields(host, name, settings.fields);
}

function mutationGeneLookup(settings) {
	const {host, name} = JSON.parse(settings.dsID);
	return xenaQuery.sparse_data_match_genes(host, name, settings.fields);
}

const fieldLookup = {
	'geneProbes': geneProbeMapLookup,
	'genes': geneProbeMapLookup,
	'mutation': mutationGeneLookup,
	'probes': probeLookup
};

function allFieldsLookup(settings, xenaFields, state) {
	var fieldSpecs = _.map(xenaFields, path => _.getIn(settings, path));
	return Rx.Observable.zipArray(
		_.map(fieldSpecs, fs => reifyErrors(
			(fieldLookup[fs.fieldType] || probeLookup)(fs, state)), {/*host info?*/}));
}

function normalizeFields(serverBus, state, id, settings) {
	var xenaFields = xenaFieldPaths(settings),
		lookup = allFieldsLookup(settings, xenaFields, state);
	serverBus.onNext(['normalize-fields', lookup, id, settings, xenaFields]);
}

var featuresInCohort = (datasets, features, cohort) =>
		_.pick(features, (f, dsID) => datasets[dsID].cohort === cohort);

var survivalVarsForCohorts = (cohort, datasets, features) =>
	_.map(cohort, ({name}) =>
			kmModel.pickSurvivalVars(featuresInCohort(datasets, features, name)));

var hasSurvFields  = vars => !!(vars.ev && vars.tte && vars.patient);

var probeFieldSpec = ({dsID, name}) => ({
	dsID,
	fetchType: 'xena', // maybe take from dataset meta instead of hard-coded
	valueType: 'float',
	fieldType: 'probes',
	fields: [name]
});

var codedFieldSpec = ({dsID, name}) => ({
	dsID,
	fetchType: 'xena', // maybe take from dataset meta instead of hard-coded
	valueType: 'coded',
	fieldType: 'clinical',
	fields: [name]
});

var checkNullField = fn => field => field ? fn(field) : nullField;

function mapToObj(keys, fn) {
	return _.object(keys, _.map(keys, fn));
}

var survFields = ['ev', 'tte', 'patient'];
var getFieldSpec = _.object(survFields,
		[probeFieldSpec, probeFieldSpec, codedFieldSpec].map(checkNullField));

// XXX Note that we merge the 'patient' field across cohorts. This will
// be wrong in some cases, where 'patient' aliases, and right in other cases,
// such as TCGA patients in multiple cohorts.
function survivalFields(cohorts, datasets, features) {
	var vars = survivalVarsForCohorts(cohorts, datasets, features);
	return _.find(vars, hasSurvFields) ?  // at least one cohort w/surv data
		mapToObj(survFields, fname =>
				getColSpec(_.map(_.pluck(vars, fname), getFieldSpec[fname]), datasets))
		: null;
}

// If field set has changed, re-fetch.
function fetchSurvival(serverBus, state) {
	let {cohort, datasets, features, survival, cohortSamples} = state,
		fields = survivalFields(cohort, datasets, features),
		refetch = _.some(survFields,
				f => !_.isEqual(fields[f], _.getIn(survival, [f, 'field']))),

		queries = _.map(survFields, key => fetch(fields[key], cohortSamples)),
		collate = data => mapToObj(survFields,
				(k, i) => ({field: fields[k], data: data[i]}));

	// This could be optimized by grouping by server.
	refetch && serverBus.onNext([
			'km-survival-data', Rx.Observable.zipArray(...queries).map(collate)]);
}

var shouldSetCohort = state => (state.cohortPending && state.cohort !== state.cohortPending);
{'fraction_non_alphanumeric': 0.1571548468317247, 'fraction_numerical': 0.0, 'mean_word_length': 5.529452054794521, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6846294', 'n_tokens_mistral': 3297, 'n_tokens_neox': 3116, 'n_words': 751}