<commit_before>#!/usr/bin/env node
var debug             = require('debug')('github:server');
var base              = require('taskcluster-base');
var api               = require('../lib/api');
var path              = require('path');
var Promise           = require('promise');
var exchanges         = require('../lib/exchanges');
var _                 = require('lodash');

/** Launch server */
var launch = async function(profile, customPublisher) {
  debug("Launching with profile: %s", profile);

  // Load configuration
  var cfg = base.config({
    defaults:     require('../config/defaults'),
    profile:      require('../config/' + profile),
    envs: [
      'pulse_username',
      'pulse_password',
      'taskclusterGithub_publishMetaData',
      'taskcluster_credentials_clientId',
      'taskcluster_credentials_accessToken',
      'aws_accessKeyId',
      'aws_secretAccessKey',
      'influx_connectionString',
      'webhook_secret'
    ],
    filename:     'taskcluster-github'
  });

  // Create a default stats drain, which just prints to stdout
  let statsDrain = {
      addPoint: (...args) => {debug("stats:", args)}
  }

  // Create InfluxDB connection for submitting statistics
  let influxConnectionString = cfg.get('influx:connectionString')
  if (influxConnectionString) {
      statsDrain = new base.stats.Influx({
        connectionString:   influxConnectionString,
        maxDelay:           cfg.get('influx:maxDelay'),
        maxPendingPoints:   cfg.get('influx:maxPendingPoints')
      });
  } else {
      debug("Missing influx_connectionString: stats collection disabled.")
  }

  // Start monitoring the process
  base.stats.startProcessUsageReporting({
    drain:      statsDrain,
    component:  cfg.get('taskclusterGithub:statsComponent'),
    process:    'server'
  });

  let validator = await base.validator({
    folder:           path.join(__dirname, '..', 'schemas'),
    constants:        require('../schemas/constants'),
    publish:          cfg.get('taskclusterGithub:publishMetaData') === 'true',
    schemaPrefix:     'github/v1/',
    aws:              cfg.get('aws')
  });

  let publisher = customPublisher
  let pulseCredentials = cfg.get('pulse')
  if (pulseCredentials.username && pulseCredentials.password) {
      publisher = await exchanges.setup({
        credentials:        pulseCredentials,
        exchangePrefix:     cfg.get('taskclusterGithub:exchangePrefix'),
        validator:          validator,
        referencePrefix:    'github/v1/exchanges.json',
        publish:            cfg.get('taskclusterGithub:publishMetaData') === 'true',
        aws:                cfg.get('aws'),
        drain:              statsDrain,
        component:          cfg.get('taskclusterGithub:statsComponent'),
        process:            'server'
      });
 } else {
    debug("Missing pulse credentials: leaving publisher undefined.")
 }

  // Create API router and publish reference if needed
  debug("Creating API router");

  let router = await api.setup({
    context:          {publisher, cfg},
    validator:        validator,
    authBaseUrl:      cfg.get('taskcluster:authBaseUrl'),
    credentials:      cfg.get('taskcluster:credentials'),
    publish:          cfg.get('taskclusterGithub:publishMetaData') === 'true',
    baseUrl:          cfg.get('server:publicUrl') + '/v1',
    referencePrefix:  'github/v1/api.json',
    aws:              cfg.get('aws'),
    component:        cfg.get('taskclusterGithub:statsComponent'),
    drain:            statsDrain
  });

  debug("Configuring app");

  // Create app
  var app = base.app({
    port:           Number(process.env.PORT || cfg.get('server:port')),
    env:            cfg.get('server:env'),
    forceSSL:       cfg.get('server:forceSSL'),
    trustProxy:     cfg.get('server:trustProxy')
  });
<commit_msg>Make publisher logic more robus.<commit_after>#!/usr/bin/env node
var debug             = require('debug')('github:server');
var base              = require('taskcluster-base');
var api               = require('../lib/api');
var path              = require('path');
var Promise           = require('promise');
var exchanges         = require('../lib/exchanges');
var _                 = require('lodash');

/** Launch server */
var launch = async function(profile, customPublisher) {
  debug("Launching with profile: %s", profile);

  // Load configuration
  var cfg = base.config({
    defaults:     require('../config/defaults'),
    profile:      require('../config/' + profile),
    envs: [
      'pulse_username',
      'pulse_password',
      'taskclusterGithub_publishMetaData',
      'taskcluster_credentials_clientId',
      'taskcluster_credentials_accessToken',
      'aws_accessKeyId',
      'aws_secretAccessKey',
      'influx_connectionString',
      'webhook_secret'
    ],
    filename:     'taskcluster-github'
  });

  // Create a default stats drain, which just prints to stdout
  let statsDrain = {
      addPoint: (...args) => {debug("stats:", args)}
  }

  // Create InfluxDB connection for submitting statistics
  let influxConnectionString = cfg.get('influx:connectionString')
  if (influxConnectionString) {
      statsDrain = new base.stats.Influx({
        connectionString:   influxConnectionString,
        maxDelay:           cfg.get('influx:maxDelay'),
        maxPendingPoints:   cfg.get('influx:maxPendingPoints')
      });
  } else {
      debug("Missing influx_connectionString: stats collection disabled.")
  }

  // Start monitoring the process
  base.stats.startProcessUsageReporting({
    drain:      statsDrain,
    component:  cfg.get('taskclusterGithub:statsComponent'),
    process:    'server'
  });

  let validator = await base.validator({
    folder:           path.join(__dirname, '..', 'schemas'),
    constants:        require('../schemas/constants'),
    publish:          cfg.get('taskclusterGithub:publishMetaData') === 'true',
    schemaPrefix:     'github/v1/',
    aws:              cfg.get('aws')
  });

  let publisher = customPublisher
  let pulseCredentials = cfg.get('pulse')
  if (publisher) {
    debug("Using a custom publiser instead of pulse")
  } else if (pulseCredentials.username && pulseCredentials.password) {
      publisher = await exchanges.setup({
        credentials:        pulseCredentials,
        exchangePrefix:     cfg.get('taskclusterGithub:exchangePrefix'),
        validator:          validator,
        referencePrefix:    'github/v1/exchanges.json',
        publish:            cfg.get('taskclusterGithub:publishMetaData') === 'true',
        aws:                cfg.get('aws'),
        drain:              statsDrain,
        component:          cfg.get('taskclusterGithub:statsComponent'),
        process:            'server'
      });
 } else {
    throw "Can't initialize pulse publisher: missing credentials"
 }

  // Create API router and publish reference if needed
  debug("Creating API router");

  let router = await api.setup({
    context:          {publisher, cfg},
    validator:        validator,
    authBaseUrl:      cfg.get('taskcluster:authBaseUrl'),
    credentials:      cfg.get('taskcluster:credentials'),
    publish:          cfg.get('taskclusterGithub:publishMetaData') === 'true',
    baseUrl:          cfg.get('server:publicUrl') + '/v1',
    referencePrefix:  'github/v1/api.json',
    aws:              cfg.get('aws'),
    component:        cfg.get('taskclusterGithub:statsComponent'),
    drain:            statsDrain
  });

  debug("Configuring app");

  // Create app
  var app = base.app({
    port:           Number(process.env.PORT || cfg.get('server:port')),
    env:            cfg.get('server:env'),
    forceSSL:       cfg.get('server:forceSSL'),
    trustProxy:     cfg.get('server:trustProxy')
  });

{'fraction_non_alphanumeric': 0.13201957753735188, 'fraction_numerical': 0.0010303967027305513, 'mean_word_length': 2.54242700729927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7005417', 'n_tokens_mistral': 2170, 'n_tokens_neox': 2137, 'n_words': 449}