<commit_before>/* @module DataSource */

var _ = require('lodash');
var highland = require('highland');
var httpErrors = require('./http-errors.js');
var internalContext = require('./context.json');
var JsonldMatcher = require('./jsonld-matcher.js');
var request = require('request');
var csv = require('csv-streamify');
var csvOptions = {objectMode: true, delimiter: '\t'};
var Utils = require('./utils.js');

/**
 * Used internally to create a new DataSource instance. See related
 * {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.html|DataSource}
 * from BridgeDb-Java.
 * @class
 * @alias dataSource
 * @memberof bridgeDb
 * @param {object} instance
 */
var DataSource = function(instance) {
  'use strict';

  /**
   * Get all biological data sources supported by BridgeDb, with all
   * available metadata, largely as specified in
   * {@link https://github.com/bridgedb/BridgeDb/blob/master/org.bridgedb.bio/resources/org/bridgedb/bio/datasources.txt|datasources.txt}.
   *
   * @return {Stream<array>} dataSourceStream
   * @return {object[]} dataSourceStream.dataSource
   * @return {string[]} dataSourceStream.dataSource.db Database name. See
   *                                                   {@link http://www.biopax.org/release/biopax-level3.owl#db|biopax:db}.
   * @return {string} dataSourceStream.dataSource.mainUrl See
   *                                                      {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.html#getMainUrl()|Java documentation}.
   * @return {string} dataSourceStream.dataSource.urlPattern See
   *                                                         {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.Builder.html#urlPattern(java.lang.String)|Java documentation}.
   * @return {string} dataSourceStream.dataSource.exampleIdentifier See
   *                                                                {@link http://identifiers.org/idot/exampleIdentifier|idot:exampleIdentifier}.
   * @return {string} dataSourceStream.dataSource.organism If the data source is for a single organism. See
   *                                                       {@link http://www.biopax.org/release/biopax-level3.owl#organism|biopax:organism}.
   * @return {string} dataSourceStream.dataSource.bridgeDbType Biological type, as used at BridgeDb. See
   *                                                           {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.Builder.html#type(java.lang.String)|Java documentation}.
   * @return {string} dataSourceStream.dataSource['@type'] Biological type, as used in GPML at WikiPathways and in PathVisio-Java.
   *                                                       See {@link http://vocabularies.wikipathways.org/gpml#Type|gpml:Type}.
   *                                                       And as used in Biopax. See the domain for
   *                                                       {@link http://www.biopax.org/release/biopax-level3.owl#entityReference|biopax:entityReference}.
   * @return {boolean} dataSourceStream.dataSource.isPrimary See Java documentation for
   *                                                         {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.html#isPrimary()|"isPrimary"} and for
   *                                                         {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.Builder.html#primary(boolean)|"primary" method}.
   * @return {string} dataSourceStream.dataSource.identifierPattern Regular expression for the identifiers from this data source.
   *                                                                See {@link http://identifiers.org/idot/identifierPattern|idot:identifierPattern}.
   * @return {string} dataSourceStream.dataSource.preferredPrefix Abbreviation as used by identifiers.org to identify a data source.
   *                                                              See {@link http://identifiers.org/idot/preferredPrefix|idot:preferredPrefix}.
   * @return {string[]} dataSourceStream.dataSource.alternatePrefix Abbreviation as used elsewhere to identify a data source, such
   *                                                                as at BridgeDb (bridgeDbSystemCode located both here and on its own).
   *                                                                See {@link http://identifiers.org/idot/alternatePrefix|idot:alternatePrefix}.
   * @return {string} dataSourceStream.dataSource.bridgeDbSystemCode See
   *                                                                 {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.html#getSystemCode()|Java documentation}.
   * @return {string} dataSourceStream.dataSource['@id'] Preferred {@link http://www.w3.org/TR/json-ld/#iris|IRI}
   *                                                     (usually a persistent URL) for identifying a data source.
   * @return {string[]} dataSourceStream.dataSource[owl:sameAs] Alternate IRI for identifying a data source.
   *                                                            See {@link http://www.w3.org/TR/owl-ref/#sameAs-def|owl:sameAs}.
   */
  var getAll = function() {
    function init() {
      var source = instance.config.dataSourcesUrl;
      return highland(
        request({
          url: source,
          withCredentials: false
        })
        .pipe(csv(csvOptions))
      )
      .map(function(array) {
        return {
          '@context':internalContext,
          db:_([array[0], array[10]])
              .uniq()
              .compact()
              .value(),
          bridgeDbSystemCode:array[1],
          mainUrl:array[2],
          urlPattern:array[3],
          exampleIdentifier:array[4],
          bridgeDbType:array[5],
          organism:array[6],
          isPrimary:Boolean(array[7]),
          miriamRootUrn:array[8],
          identifierPattern:array[9],
        };
      })
      .map(function(dataSource) {

        // remove empty properties

        return _.omit(dataSource, function(value) {
          return value === '' ||
            _.isNaN(value) ||
          _.isNull(value) ||
          _.isUndefined(value);
        });
      })
      .map(function(dataSource) {
        if (!!dataSource.miriamRootUrn &&
            dataSource.miriamRootUrn.indexOf('urn:miriam:') > -1) {

          dataSource.preferredPrefix =
            dataSource.miriamRootUrn.substring(11,
              dataSource.miriamRootUrn.length);
          dataSource['@id'] =
            'http://identifiers.org/' + dataSource.preferredPrefix;
          dataSource['owl:sameAs'] = dataSource['owl:sameAs'] || [];
          dataSource['owl:sameAs'].push(dataSource.miriamRootUrn);
        }
        delete dataSource.miriamRootUrn;
        return dataSource;
      })
      .map(function(dataSource) {
        if (!!dataSource.bridgeDbType) {
          dataSource['@type'] = [];
          if (dataSource.bridgeDbType === 'gene' ||
            dataSource.bridgeDbType === 'probe' ||
            dataSource.preferredPrefix === 'go') {

            dataSource['@type'].push('gpml:GeneProduct');
            dataSource['@type'].push('biopax:DnaReference');
          } else if (dataSource.bridgeDbType === 'rna') {
            dataSource['@type'].push('gpml:Rna');
            dataSource['@type'].push('biopax:RnaReference');
          } else if (dataSource.bridgeDbType === 'protein') {
            dataSource['@type'].push('gpml:Protein');
            dataSource['@type'].push('biopax:ProteinReference');
          } else if (dataSource.bridgeDbType === 'metabolite') {
            dataSource['@type'].push('gpml:Metabolite');
            dataSource['@type'].push('biopax:SmallMoleculeReference');
          } else if (dataSource.bridgeDbType === 'pathway') {
            dataSource['@type'].push('gpml:Pathway');
            dataSource['@type'].push('biopax:Pathway');
          }
        }

        dataSource.alternatePrefix = [
          dataSource.bridgeDbSystemCode
        ];

        return dataSource;
      })
      .collect();
    }

    return Utils._runOnce('dataSource', init);
  };

  function getOne(args) {
    // keys that can identify a data source
    var selectedKeys = [
      '@id',
      'preferredPrefix',
      'alternatePrefix',
      'db',
      'identifier'
    ];
    /*/
    return getAllFormattedForComparison()
    .flatFilter(function(dataSource) {
      console.log('dataSource');
      console.log(dataSource);
      return highland(highland.pairs(args))
      .filter(function(pair) {
        return selectedKeys.indexOf(pair[0]) > -1;
      })
      .flatFilter(function(pair) {
        return highland(dataSources).where(function(dataSource) {
          console.log('dataSource987');
          console.log(dataSource);
          return dataSource[pair[0]] === pair[1] ||
            _(dataSource[pair[0]]).intersection(pair[1]).length > 0;
        });
      });
    })
    .head();
    //*/
    //*
    return highland(highland.pairs(args))
    .flatMap(function(pair) {
      return JsonldMatcher._getOne(
        pair,
        getAll().sequence(),
        'dataSourcesFormattedForComparison',
        selectedKeys);
    })
    .compact();
    //*/
    /*/
    return getAllFormattedForComparison().collect()
    .flatMap(function(dataSources) {
      console.log('dataSources');
      console.log(dataSources);
      //return highland(dataSources);<commit_msg>It is not yet pretty, but I'm able to get the data source based on any key that will identify the data source via data-sources.txt<commit_after>/* @module DataSource */

var _ = require('lodash');
var highland = require('highland');
var httpErrors = require('./http-errors.js');
var internalContext = require('./context.json');
var JsonldMatcher = require('./jsonld-matcher.js');
var request = require('request');
var csv = require('csv-streamify');
var csvOptions = {objectMode: true, delimiter: '\t'};
var Utils = require('./utils.js');

/**
 * Used internally to create a new DataSource instance. See related
 * {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.html|DataSource}
 * from BridgeDb-Java.
 * @class
 * @alias dataSource
 * @memberof bridgeDb
 * @param {object} instance
 */
var DataSource = function(instance) {
  'use strict';

  /**
   * Get all biological data sources supported by BridgeDb, with all
   * available metadata, largely as specified in
   * {@link https://github.com/bridgedb/BridgeDb/blob/master/org.bridgedb.bio/resources/org/bridgedb/bio/datasources.txt|datasources.txt}.
   *
   * @return {Stream<array>} dataSourceStream
   * @return {object[]} dataSourceStream.dataSource
   * @return {string[]} dataSourceStream.dataSource.db Database name. See
   *                                                   {@link http://www.biopax.org/release/biopax-level3.owl#db|biopax:db}.
   * @return {string} dataSourceStream.dataSource.mainUrl See
   *                                                      {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.html#getMainUrl()|Java documentation}.
   * @return {string} dataSourceStream.dataSource.urlPattern See
   *                                                         {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.Builder.html#urlPattern(java.lang.String)|Java documentation}.
   * @return {string} dataSourceStream.dataSource.exampleIdentifier See
   *                                                                {@link http://identifiers.org/idot/exampleIdentifier|idot:exampleIdentifier}.
   * @return {string} dataSourceStream.dataSource.organism If the data source is for a single organism. See
   *                                                       {@link http://www.biopax.org/release/biopax-level3.owl#organism|biopax:organism}.
   * @return {string} dataSourceStream.dataSource.bridgeDbType Biological type, as used at BridgeDb. See
   *                                                           {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.Builder.html#type(java.lang.String)|Java documentation}.
   * @return {string} dataSourceStream.dataSource['@type'] Biological type, as used in GPML at WikiPathways and in PathVisio-Java.
   *                                                       See {@link http://vocabularies.wikipathways.org/gpml#Type|gpml:Type}.
   *                                                       And as used in Biopax. See the domain for
   *                                                       {@link http://www.biopax.org/release/biopax-level3.owl#entityReference|biopax:entityReference}.
   * @return {boolean} dataSourceStream.dataSource.isPrimary See Java documentation for
   *                                                         {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.html#isPrimary()|"isPrimary"} and for
   *                                                         {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.Builder.html#primary(boolean)|"primary" method}.
   * @return {string} dataSourceStream.dataSource.identifierPattern Regular expression for the identifiers from this data source.
   *                                                                See {@link http://identifiers.org/idot/identifierPattern|idot:identifierPattern}.
   * @return {string} dataSourceStream.dataSource.preferredPrefix Abbreviation as used by identifiers.org to identify a data source.
   *                                                              See {@link http://identifiers.org/idot/preferredPrefix|idot:preferredPrefix}.
   * @return {string[]} dataSourceStream.dataSource.alternatePrefix Abbreviation as used elsewhere to identify a data source, such
   *                                                                as at BridgeDb (bridgeDbSystemCode located both here and on its own).
   *                                                                See {@link http://identifiers.org/idot/alternatePrefix|idot:alternatePrefix}.
   * @return {string} dataSourceStream.dataSource.bridgeDbSystemCode See
   *                                                                 {@link http://bridgedb.org/apidoc/2.0/org/bridgedb/DataSource.html#getSystemCode()|Java documentation}.
   * @return {string} dataSourceStream.dataSource['@id'] Preferred {@link http://www.w3.org/TR/json-ld/#iris|IRI}
   *                                                     (usually a persistent URL) for identifying a data source.
   * @return {string[]} dataSourceStream.dataSource[owl:sameAs] Alternate IRI for identifying a data source.
   *                                                            See {@link http://www.w3.org/TR/owl-ref/#sameAs-def|owl:sameAs}.
   */
  var getAll = function() {
    function init() {
      var source = instance.config.dataSourcesUrl;
      return highland(
        request({
          url: source,
          withCredentials: false
        })
        .pipe(csv(csvOptions))
      )
      .map(function(array) {
        return {
          '@context':internalContext,
          db:_([array[0], array[10]])
              .uniq()
              .compact()
              .value(),
          bridgeDbSystemCode:array[1],
          mainUrl:array[2],
          urlPattern:array[3],
          exampleIdentifier:array[4],
          bridgeDbType:array[5],
          organism:array[6],
          isPrimary:Boolean(array[7]),
          miriamRootUrn:array[8],
          identifierPattern:array[9],
        };
      })
      .map(function(dataSource) {

        // remove empty properties

        return _.omit(dataSource, function(value) {
          return value === '' ||
            _.isNaN(value) ||
          _.isNull(value) ||
          _.isUndefined(value);
        });
      })
      .map(function(dataSource) {
        if (!!dataSource.miriamRootUrn &&
            dataSource.miriamRootUrn.indexOf('urn:miriam:') > -1) {

          dataSource.preferredPrefix =
            dataSource.miriamRootUrn.substring(11,
              dataSource.miriamRootUrn.length);
          dataSource['@id'] =
            'http://identifiers.org/' + dataSource.preferredPrefix;
          dataSource['owl:sameAs'] = dataSource['owl:sameAs'] || [];
          dataSource['owl:sameAs'].push(dataSource.miriamRootUrn);
        }
        delete dataSource.miriamRootUrn;
        return dataSource;
      })
      .map(function(dataSource) {
        if (!!dataSource.bridgeDbType) {
          dataSource['@type'] = [];
          if (dataSource.bridgeDbType === 'gene' ||
            dataSource.bridgeDbType === 'probe' ||
            dataSource.preferredPrefix === 'go') {

            dataSource['@type'].push('gpml:GeneProduct');
            dataSource['@type'].push('biopax:DnaReference');
          } else if (dataSource.bridgeDbType === 'rna') {
            dataSource['@type'].push('gpml:Rna');
            dataSource['@type'].push('biopax:RnaReference');
          } else if (dataSource.bridgeDbType === 'protein') {
            dataSource['@type'].push('gpml:Protein');
            dataSource['@type'].push('biopax:ProteinReference');
          } else if (dataSource.bridgeDbType === 'metabolite') {
            dataSource['@type'].push('gpml:Metabolite');
            dataSource['@type'].push('biopax:SmallMoleculeReference');
          } else if (dataSource.bridgeDbType === 'pathway') {
            dataSource['@type'].push('gpml:Pathway');
            dataSource['@type'].push('biopax:Pathway');
          }
        }

        dataSource.alternatePrefix = [
          dataSource.bridgeDbSystemCode
        ];

        return dataSource;
      })
      .collect();
    }

    return Utils._runOnce('dataSource', init);
  };

  function getOne(args) {
    // keys that can identify a data source
    var selectedKeys = [
      '@id',
      'preferredPrefix',
      'alternatePrefix',
      'db',
      'identifier'
    ];

    return highland(highland.pairs(args))
    .flatMap(function(pair) {
      return JsonldMatcher._getOne(
        pair,
        getAll().sequence(),
        'dataSourcesFormattedForComparison',
        selectedKeys);
    })
    .compact();
    //*/
    /*/
    return getAllFormattedForComparison().collect()
    .flatMap(function(dataSources) {
      console.log('dataSources');
      console.log(dataSources);
      //return highland(dataSources);
{'fraction_non_alphanumeric': 0.1267882494266681, 'fraction_numerical': 0.004259036802446216, 'mean_word_length': 2.0252725470763133, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 82, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2084560', 'n_tokens_mistral': 4938, 'n_tokens_neox': 4836, 'n_words': 875}