<commit_before>var d3 = require('d3')
  , csv = require('csv')
  , async = require('async')
  , _ = require('lodash')
  , http = require('http');

module.exports = (function(){

  var jsonLdContext = {
    id: '@id',
    bp: {
      '@id': 'http://www.biopax.org/release/biopax-level3.owl#',
      '@type': '@id'
    },
    dbId: {
      '@id': 'bp:id',
      '@type': 'xsd:string'
    },
    dbName: {
      '@id': 'bp:db',
      '@type': 'xsd:string'
    }
  };

  var providedDbId, providedDbName;

  function getBridgedbDatabaseMetadata(callbackOutside) {
    var that = this;
    if (this.bridgedbDatabaseMetadata) {
      return callbackOutside(null, this.bridgedbDatabaseMetadata);
    } 

    //*
    var options = {
      host: 'pointer.ucsf.edu',
      path: '/d3/r/data-sources/bridgedb-datasources.php',
      port: '80'
    };
    //*/

    /* disabling this one now, because it's not available via CORS
     * TODO get a version that works with CORS
    var options = {
      host: 'svn.bigcat.unimaas.nl',
      path: '/bridgedb/trunk/org.bridgedb.bio/resources/org/bridgedb/bio/datasources.txt',
      port: '80'
    };
    //*/

    var callback = function(response) {
      var str = '';
      response.on('data', function (chunk) {
        str += chunk;
      });

      response.on('end', function () {
        csv().from
        .string(
          str,
          {delimiter:'\t',
            // organism means "applicableOnlyToSpecies"
          columns:['dbName', 'systemCode', 'website', 'linkoutPattern', 'exampleIdentifier', 'type', 'organism', 'priority', 'miriamRootUrn', 'regex', 'officialName']} )
          .to.array( function(bridgedbDatabaseMetadata){
            _.forEach(bridgedbDatabaseMetadata, function(bridgedbDatabaseMetadataRow) {
              // TODO remove this code if the datasources.txt file gets corrected, making these corrections redundant.
              bridgedbDatabaseMetadataRow.miriamRootUrn = bridgedbDatabaseMetadataRow.miriamRootUrn.toLowerCase();
              if (!!bridgedbDatabaseMetadataRow.miriamRootUrn && bridgedbDatabaseMetadataRow.miriamRootUrn.indexOf('urn:miriam') > -1) {
                if (bridgedbDatabaseMetadataRow.miriamRootUrn === 'urn:miriam:obo.go') {
                  bridgedbDatabaseMetadataRow.miriamRootUrn = 'urn:miriam:go:';
                }
                if (bridgedbDatabaseMetadataRow.miriamRootUrn[bridgedbDatabaseMetadataRow.miriamRootUrn.length - 1] !== ':') {
                  bridgedbDatabaseMetadataRow.miriamRootUrn += ':';
                }
                bridgedbDatabaseMetadataRow.namespace = bridgedbDatabaseMetadataRow.miriamRootUrn.substring(11, bridgedbDatabaseMetadataRow.miriamRootUrn.length - 1);
              } else {
                delete bridgedbDatabaseMetadataRow.miriamRootUrn;
              }

              // remove items if empty

              if (!bridgedbDatabaseMetadataRow.exampleIdentifier) {
                delete bridgedbDatabaseMetadataRow.exampleIdentifier;
              }

              if (!bridgedbDatabaseMetadataRow.organism) {
                delete bridgedbDatabaseMetadataRow.organism;
              }

              if (!bridgedbDatabaseMetadataRow.regex) {
                delete bridgedbDatabaseMetadataRow.regex;
              }

              if (!!bridgedbDatabaseMetadataRow.type) {
                if (bridgedbDatabaseMetadataRow.type === 'gene' || bridgedbDatabaseMetadataRow.type === 'probe' || bridgedbDatabaseMetadataRow.namespace === 'go') {
                  bridgedbDatabaseMetadataRow.type = 'DnaReference';
                } else if (bridgedbDatabaseMetadataRow.type === 'rna') {
                  bridgedbDatabaseMetadataRow.type = 'RnaReference';
                } else if (bridgedbDatabaseMetadataRow.type === 'protein') {
                  bridgedbDatabaseMetadataRow.type = 'ProteinReference';
                } else if (bridgedbDatabaseMetadataRow.type === 'metabolite') {
                  bridgedbDatabaseMetadataRow.type = 'SmallMoleculeReference';
                } else if (bridgedbDatabaseMetadataRow.type === 'pathway') {
                  bridgedbDatabaseMetadataRow.type = 'Pathway';
                }
              }
            });
            that.bridgedbDatabaseMetadata = bridgedbDatabaseMetadata;
            callbackOutside(null, bridgedbDatabaseMetadata);
          });
      });
    };

    var req = http.request(options, callback);
    req.end();
  }

  function getEntityReferenceIdentifiersIri(bridgedbDatabaseMetadataForDbName, dbId, callback) {
    var iri = bridgedbDatabaseMetadataForDbName.linkoutPattern.replace('$id', dbId);
    return iri;
  }

  function getBridgedbApiHostAndPathByEntityReference(args, callback) {
    var result = {};
    result.host = 'webservice.bridgedb.org';
    if (!!args.bridgedbUri) {
      var bridgedbUriComponents = args.bridgedbUri.split(result.host);
      result.path = bridgedbUriComponents[bridgedbUriComponents.length - 1];

      var bridgedbPathComponents = result.path.split('/');
      var systemCode = bridgedbPathComponents[3];

      getBridgedbDatabaseMetadataForSystemCode(systemCode, function(err, bridgedbDatabaseMetadataRow) {
        providedDbName = bridgedbDatabaseMetadataRow.dbName;

        var dbId = bridgedbPathComponents[4];
        providedDbId = dbId;

        return callback(null, result);
      });
    } else if (!!args.organism && (!!args.systemCode || !!args.dbName) && typeof args.dbId !== 'undefined') {
      providedDbName = args.dbName;
      providedDbId = args.dbId;
      if (!!args.systemCode) {
        result.path = '/' + encodeURIComponent(args.organism) + '/xrefs/' + encodeURIComponent(args.systemCode) + '/' + encodeURIComponent(args.dbId);
        return callback(null, result);
      } else {
        getBridgedbSystemCode(args.dbName, function(err, systemCode) {
          result.path = '/' + encodeURIComponent(args.organism) + '/xrefs/' + encodeURIComponent(systemCode) + '/' + encodeURIComponent(args.dbId);
          return callback(null, result);
        });
      }
    } else {
      var entityReferenceIdCandidates = [
        args.id,
        args['@id'],
        args.identifier,
        args.identifiers,
        args.iri,
        args.uri,
        args.identifiersId
      ];

      var identifiersIdResults = _.filter(entityReferenceIdCandidates, function(entityReferenceIdCandidate) {
        var entityReferenceIdCandidateIsValid = false;
        if (!!entityReferenceIdCandidate) {
          entityReferenceIdCandidateIsValid = entityReferenceIdCandidate.indexOf('identifiers.org') > -1;
        }
        return entityReferenceIdCandidateIsValid;
      });
      if (!!identifiersIdResults && identifiersIdResults.length > 0) {
        var identifiersId = identifiersIdResults[0];
        var identifiersUriComponents = identifiersId.split('identifiers.org');
        var identifiersPath = identifiersUriComponents[identifiersUriComponents.length - 1];

        var identifiersPathComponents = identifiersPath.split('/');<commit_msg>add entityReference as accepted property name<commit_after>var d3 = require('d3')
  , csv = require('csv')
  , async = require('async')
  , _ = require('lodash')
  , http = require('http');

module.exports = (function(){

  var jsonLdContext = {
    id: '@id',
    bp: {
      '@id': 'http://www.biopax.org/release/biopax-level3.owl#',
      '@type': '@id'
    },
    dbId: {
      '@id': 'bp:id',
      '@type': 'xsd:string'
    },
    dbName: {
      '@id': 'bp:db',
      '@type': 'xsd:string'
    }
  };

  var providedDbId, providedDbName;

  function getBridgedbDatabaseMetadata(callbackOutside) {
    var that = this;
    if (this.bridgedbDatabaseMetadata) {
      return callbackOutside(null, this.bridgedbDatabaseMetadata);
    } 

    //*
    var options = {
      host: 'pointer.ucsf.edu',
      path: '/d3/r/data-sources/bridgedb-datasources.php',
      port: '80'
    };
    //*/

    /* disabling this one now, because it's not available via CORS
     * TODO get a version that works with CORS
    var options = {
      host: 'svn.bigcat.unimaas.nl',
      path: '/bridgedb/trunk/org.bridgedb.bio/resources/org/bridgedb/bio/datasources.txt',
      port: '80'
    };
    //*/

    var callback = function(response) {
      var str = '';
      response.on('data', function (chunk) {
        str += chunk;
      });

      response.on('end', function () {
        csv().from
        .string(
          str,
          {delimiter:'\t',
            // organism means "applicableOnlyToSpecies"
          columns:['dbName', 'systemCode', 'website', 'linkoutPattern', 'exampleIdentifier', 'type', 'organism', 'priority', 'miriamRootUrn', 'regex', 'officialName']} )
          .to.array( function(bridgedbDatabaseMetadata){
            _.forEach(bridgedbDatabaseMetadata, function(bridgedbDatabaseMetadataRow) {
              // TODO remove this code if the datasources.txt file gets corrected, making these corrections redundant.
              bridgedbDatabaseMetadataRow.miriamRootUrn = bridgedbDatabaseMetadataRow.miriamRootUrn.toLowerCase();
              if (!!bridgedbDatabaseMetadataRow.miriamRootUrn && bridgedbDatabaseMetadataRow.miriamRootUrn.indexOf('urn:miriam') > -1) {
                if (bridgedbDatabaseMetadataRow.miriamRootUrn === 'urn:miriam:obo.go') {
                  bridgedbDatabaseMetadataRow.miriamRootUrn = 'urn:miriam:go:';
                }
                if (bridgedbDatabaseMetadataRow.miriamRootUrn[bridgedbDatabaseMetadataRow.miriamRootUrn.length - 1] !== ':') {
                  bridgedbDatabaseMetadataRow.miriamRootUrn += ':';
                }
                bridgedbDatabaseMetadataRow.namespace = bridgedbDatabaseMetadataRow.miriamRootUrn.substring(11, bridgedbDatabaseMetadataRow.miriamRootUrn.length - 1);
              } else {
                delete bridgedbDatabaseMetadataRow.miriamRootUrn;
              }

              // remove items if empty

              if (!bridgedbDatabaseMetadataRow.exampleIdentifier) {
                delete bridgedbDatabaseMetadataRow.exampleIdentifier;
              }

              if (!bridgedbDatabaseMetadataRow.organism) {
                delete bridgedbDatabaseMetadataRow.organism;
              }

              if (!bridgedbDatabaseMetadataRow.regex) {
                delete bridgedbDatabaseMetadataRow.regex;
              }

              if (!!bridgedbDatabaseMetadataRow.type) {
                if (bridgedbDatabaseMetadataRow.type === 'gene' || bridgedbDatabaseMetadataRow.type === 'probe' || bridgedbDatabaseMetadataRow.namespace === 'go') {
                  bridgedbDatabaseMetadataRow.type = 'DnaReference';
                } else if (bridgedbDatabaseMetadataRow.type === 'rna') {
                  bridgedbDatabaseMetadataRow.type = 'RnaReference';
                } else if (bridgedbDatabaseMetadataRow.type === 'protein') {
                  bridgedbDatabaseMetadataRow.type = 'ProteinReference';
                } else if (bridgedbDatabaseMetadataRow.type === 'metabolite') {
                  bridgedbDatabaseMetadataRow.type = 'SmallMoleculeReference';
                } else if (bridgedbDatabaseMetadataRow.type === 'pathway') {
                  bridgedbDatabaseMetadataRow.type = 'Pathway';
                }
              }
            });
            that.bridgedbDatabaseMetadata = bridgedbDatabaseMetadata;
            callbackOutside(null, bridgedbDatabaseMetadata);
          });
      });
    };

    var req = http.request(options, callback);
    req.end();
  }

  function getEntityReferenceIdentifiersIri(bridgedbDatabaseMetadataForDbName, dbId, callback) {
    var iri = bridgedbDatabaseMetadataForDbName.linkoutPattern.replace('$id', dbId);
    return iri;
  }

  function getBridgedbApiHostAndPathByEntityReference(args, callback) {
    var result = {};
    result.host = 'webservice.bridgedb.org';
    if (!!args.bridgedbUri) {
      var bridgedbUriComponents = args.bridgedbUri.split(result.host);
      result.path = bridgedbUriComponents[bridgedbUriComponents.length - 1];

      var bridgedbPathComponents = result.path.split('/');
      var systemCode = bridgedbPathComponents[3];

      getBridgedbDatabaseMetadataForSystemCode(systemCode, function(err, bridgedbDatabaseMetadataRow) {
        providedDbName = bridgedbDatabaseMetadataRow.dbName;

        var dbId = bridgedbPathComponents[4];
        providedDbId = dbId;

        return callback(null, result);
      });
    } else if (!!args.organism && (!!args.systemCode || !!args.dbName) && typeof args.dbId !== 'undefined') {
      providedDbName = args.dbName;
      providedDbId = args.dbId;
      if (!!args.systemCode) {
        result.path = '/' + encodeURIComponent(args.organism) + '/xrefs/' + encodeURIComponent(args.systemCode) + '/' + encodeURIComponent(args.dbId);
        return callback(null, result);
      } else {
        getBridgedbSystemCode(args.dbName, function(err, systemCode) {
          result.path = '/' + encodeURIComponent(args.organism) + '/xrefs/' + encodeURIComponent(systemCode) + '/' + encodeURIComponent(args.dbId);
          return callback(null, result);
        });
      }
    } else {
      var entityReferenceIdCandidates = [
        args.entityReference,
        args.id,
        args['@id'],
        args.identifier,
        args.identifiers,
        args.iri,
        args.uri,
        args.identifiersId
      ];

      var identifiersIdResults = _.filter(entityReferenceIdCandidates, function(entityReferenceIdCandidate) {
        var entityReferenceIdCandidateIsValid = false;
        if (!!entityReferenceIdCandidate) {
          entityReferenceIdCandidateIsValid = entityReferenceIdCandidate.indexOf('identifiers.org') > -1;
        }
        return entityReferenceIdCandidateIsValid;
      });
      if (!!identifiersIdResults && identifiersIdResults.length > 0) {
        var identifiersId = identifiersIdResults[0];
        var identifiersUriComponents = identifiersId.split('identifiers.org');
        var identifiersPath = identifiersUriComponents[identifiersUriComponents.length - 1];

        var identifiersPathComponents = identifiersPath.split('/');
{'fraction_non_alphanumeric': 0.11663128096249116, 'fraction_numerical': 0.0028308563340410475, 'mean_word_length': 2.6196209016393444, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3650906', 'n_tokens_mistral': 4178, 'n_tokens_neox': 3969, 'n_words': 659}