<commit_before>"""
    AbstractEvalMetric

The base class for all evaluation metrics. The sub-types should implement the following
interfaces:

* [`update!`](@ref)
* [`reset!`](@ref)
* [`get`](@ref)
"""
abstract AbstractEvalMetric

"""
    update!(metric, labels, preds)

Update and accumulate metrics.

# Arguments:
* `metric::AbstractEvalMetric`: the metric object.
* `labels::Vector{NDArray}`: the labels from the data provider.
* `preds::Vector{NDArray}`: the outputs (predictions) of the network.
"""
function update!{T <: AbstractEvalMetric}(metric :: T, labels :: Vector{NDArray}, preds :: Vector{NDArray})
  if length(labels) != length(preds)
    Base.warn_once(
      "The number of labels ($(length(labels))) does not correspond to the\
      number of outputs ($(length(preds))). The calculated metric might not be accuracte.")
  end
  for (label, pred) in zip(labels, preds)
    _update_single_output(metric, label, pred)
  end
end

"""
    reset!(metric)

Reset the accumulation counter.
"""
function reset!(metric :: AbstractEvalMetric)
  throw(MethodError(reset!, (typeof(metric),)))
end


import Base: get
"""
    get(metric)

Get the accumulated metrics.

Returns `Vector{Tuple{Base.Symbol, Real}}`, a list of name-value pairs.
For example, `[(:accuracy, 0.9)]`.
"""
function get(metric :: AbstractEvalMetric)
  throw(MethodError(get, (typeof(metric),)))
end

"""
    MultiMetric(metrics::Vector{AbstractEvalMetric})

Combine multiple metrics in one and get a result for all of them.

# Usage
To calculate both mean-squared error [`Accuracy`](@ref) and log-loss [`ACE`](@ref):
```julia
  mx.fit(..., eval_metric = mx.MultiMetric([mx.Accuracy(), mx.ACE()]))
```
"""
type MultiMetric <: mx.AbstractEvalMetric
    metrics :: Vector{mx.AbstractEvalMetric}
end

function update!(metric :: MultiMetric, labels :: Vector{NDArray}, preds :: Vector{NDArray})
    for m in metric.metrics
        update!(m, labels, preds)
    end
    return nothing
end

function reset!(metric :: MultiMetric)
    map(reset!, metric.metrics)
    return nothing
end

function get(metric :: MultiMetric)
    mapreduce(get, append!, metric.metrics)
end

"""<commit_msg>add SeqMetric to apply different metrics for multi-output<commit_after>"""
    AbstractEvalMetric

The base class for all evaluation metrics. The sub-types should implement the following
interfaces:

* [`update!`](@ref)
* [`reset!`](@ref)
* [`get`](@ref)
"""
abstract AbstractEvalMetric

"""
    update!(metric, labels, preds)

Update and accumulate metrics.

# Arguments:
* `metric::AbstractEvalMetric`: the metric object.
* `labels::Vector{NDArray}`: the labels from the data provider.
* `preds::Vector{NDArray}`: the outputs (predictions) of the network.
"""
function update!{T <: AbstractEvalMetric}(metric :: T, labels :: Vector{NDArray}, preds :: Vector{NDArray})
  if length(labels) != length(preds)
    Base.warn_once(
      "The number of labels ($(length(labels))) does not correspond to the\
      number of outputs ($(length(preds))). The calculated metric might not be accuracte.")
  end
  for (label, pred) in zip(labels, preds)
    _update_single_output(metric, label, pred)
  end
end

"""
    reset!(metric)

Reset the accumulation counter.
"""
function reset!(metric :: AbstractEvalMetric)
  throw(MethodError(reset!, (typeof(metric),)))
end


import Base: get
"""
    get(metric)

Get the accumulated metrics.

Returns `Vector{Tuple{Base.Symbol, Real}}`, a list of name-value pairs.
For example, `[(:accuracy, 0.9)]`.
"""
function get(metric :: AbstractEvalMetric)
  throw(MethodError(get, (typeof(metric),)))
end

"""
    MultiMetric(metrics::Vector{AbstractEvalMetric})

Combine multiple metrics in one and get a result for all of them.

# Usage
To calculate both mean-squared error [`Accuracy`](@ref) and log-loss [`ACE`](@ref):
```julia
  mx.fit(..., eval_metric = mx.MultiMetric([mx.Accuracy(), mx.ACE()]))
```
"""
type MultiMetric <: mx.AbstractEvalMetric
    metrics :: Vector{mx.AbstractEvalMetric}
end

function update!(metric :: MultiMetric, labels :: Vector{NDArray}, preds :: Vector{NDArray})
    for m in metric.metrics
        update!(m, labels, preds)
    end
    return nothing
end

function reset!(metric :: MultiMetric)
    map(reset!, metric.metrics)
    return nothing
end

function get(metric :: MultiMetric)
    mapreduce(get, append!, metric.metrics)
end

"""
    SeqMetric(metrics::Vector{AbstractEvalMetric})

Apply a different metric to each output. This is especially useful for `mx.Group`.

# Usage
Calculate accuracy [`Accuracy`](@ref) for the first output
and log-loss [`ACE`](@ref) for the second output:
```julia
  mx.fit(..., eval_metric = mx.SeqMetric([mx.Accuracy(), mx.ACE()]))
```
"""
type SeqMetric <: mx.AbstractEvalMetric
    metrics :: Vector{mx.AbstractEvalMetric}
end

function update!(metric :: SeqMetric, labels :: Vector{NDArray}, preds :: Vector{NDArray})
    @assert length(metric.metrics) == length(labels)
    @assert length(metric.metrics) == length(preds)
    for (m, l, p) in zip(metric.metrics, labels, preds)
        update!(m, [l], [p])
    end
    return nothing
end

function reset!(metric :: SeqMetric)
    map(reset!, metric.metrics)
    return nothing
end

function get(metric :: SeqMetric)
    mapreduce(get, append!, metric.metrics)
end

"""
{'fraction_non_alphanumeric': 0.1462675255778704, 'fraction_numerical': 0.0007578628268283441, 'mean_word_length': 4.738043478260869, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491823', 'n_tokens_mistral': 1643, 'n_tokens_neox': 1604, 'n_words': 484}