<commit_before>package funnel
package agent

import knobs._
import java.net.URI
import java.io.File
import journal.Logger
import scalaz.{\/,-\/,\/-}
import scalaz.std.option._
import scalaz.concurrent.Task
import java.net.{InetAddress,URI}
import scalaz.syntax.applicative._
import scala.concurrent.duration._
import funnel.zeromq._, sockets._
import agent.zeromq._
import funnel.instruments._

object Main {
  private val log = Logger[Main.type]

  case class HttpConfig(host: String, port: Int)
  case class StatsdConfig(port: Int, prefix: String)
  case class ProxyConfig(host: String, port: Int)
  case class ZeromqConfig(socket: String, proxy: Option[ProxyConfig])
  case class NginxConfig(uri: String, frequency: Duration)
  case class MesosConfig(name: String,
                         uri: String,
                         frequency: Duration,
                         queries: List[String] = Nil,
                         checkfield: String)

  case class JmxConfig(
    name: String,
    uri: String,
    frequency: Duration,
    queries: List[String] = Nil,
    exclusions: List[String] = Nil)

  case class AgentConfig(
    systemCollection: Boolean,
    jvmCollection: Boolean
  )

  case class Options(
    agent: Option[AgentConfig],
    http: Option[HttpConfig],
    statsd: Option[StatsdConfig],
    zeromq: Option[ZeromqConfig],
    nginx: Option[NginxConfig],
    jmx: Option[JmxConfig],
    mesos: Option[MesosConfig]
  )

  def main(args: Array[String]): Unit = {
    log.info("Loading agent configuation from disk.")

    /**
     * Accepting argument on the command line is really just a
     * convenience for testing and ad-hoc ops trial of the agent.
     */
    val sources = args.toList.map(p =>
      Optional(FileResource(new File(p)))) :::
      Required(
        FileResource(new File("/usr/share/funnel-agent/etc/agent.cfg")) or
        ClassPathResource("oncue/agent.cfg")) :: Nil

    val config: Task[Config] = for {
      a <- knobs.loadImmutable(sources)
      b <- knobs.aws.config
    } yield a ++ b

    log.info(s"Input configuration file was: ${config.run}")

    /**
     * Create a typed set of options using knobs.
     * For the proxy host, try looking up the local address of this
     * particular node and fallback to the config, and then all
     * known interfaces host if that fails.<commit_msg>Fixing a issue with the order of loading configuration files into the agent. <commit_after>package funnel
package agent

import knobs._
import java.net.URI
import java.io.File
import journal.Logger
import scalaz.{\/,-\/,\/-}
import scalaz.std.option._
import scalaz.concurrent.Task
import java.net.{InetAddress,URI}
import scalaz.syntax.applicative._
import scala.concurrent.duration._
import funnel.zeromq._, sockets._
import agent.zeromq._
import funnel.instruments._

object Main {
  private val log = Logger[Main.type]

  case class HttpConfig(host: String, port: Int)
  case class StatsdConfig(port: Int, prefix: String)
  case class ProxyConfig(host: String, port: Int)
  case class ZeromqConfig(socket: String, proxy: Option[ProxyConfig])
  case class NginxConfig(uri: String, frequency: Duration)
  case class MesosConfig(name: String,
                         uri: String,
                         frequency: Duration,
                         queries: List[String] = Nil,
                         checkfield: String)

  case class JmxConfig(
    name: String,
    uri: String,
    frequency: Duration,
    queries: List[String] = Nil,
    exclusions: List[String] = Nil)

  case class AgentConfig(
    systemCollection: Boolean,
    jvmCollection: Boolean
  )

  case class Options(
    agent: Option[AgentConfig],
    http: Option[HttpConfig],
    statsd: Option[StatsdConfig],
    zeromq: Option[ZeromqConfig],
    nginx: Option[NginxConfig],
    jmx: Option[JmxConfig],
    mesos: Option[MesosConfig]
  )

  def main(args: Array[String]): Unit = {
    log.info("Loading agent configuation from disk.")

    /**
     * Accepting argument on the command line is really just a
     * convenience for testing and ad-hoc ops trial of the agent.
     *
     * Configs are loaded in order; LAST WRITER WINS, as configs
     * are reduced right to left.
     */
    val sources =
      List(Required(
        FileResource(new File("/usr/share/funnel-agent/etc/agent.cfg")) or
        ClassPathResource("oncue/agent.cfg"))) ++
      args.toList.map(p => Optional(FileResource(new File(p))))

    val config: Task[Config] = for {
      a <- knobs.loadImmutable(sources)
      b <- knobs.aws.config
    } yield a ++ b

    log.info(s"Input configuration file was: ${config.run}")

    /**
     * Create a typed set of options using knobs.
     * For the proxy host, try looking up the local address of this
     * particular node and fallback to the config, and then all
     * known interfaces host if that fails.
{'fraction_non_alphanumeric': 0.10115131578947369, 'fraction_numerical': 0.0, 'mean_word_length': 2.920225624496374, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3843677', 'n_tokens_mistral': 1477, 'n_tokens_neox': 1405, 'n_words': 448}