<commit_before>// Copyright: 2014, 2015, Ableton AG, Berlin. All rights reserved.
/*!
 * @file
 */

#pragma once

#include <atria/xform/transducer_impl.hpp>
#include <atria/xform/state_wrapper.hpp>
#include <atria/xform/skip.hpp>
#include <atria/estd/tuple_hash.hpp>
#include <atria/prelude/tuplify.hpp>
#include <atria/estd/functional.hpp>
#include <unordered_set>

namespace atria {
namespace xform {

namespace detail {

struct distinct_rf_gen
{
  template <typename ReducingFnT>
  struct apply
  {
    ReducingFnT step;

    template <typename ...InputTs>
    using cache_t = std::unordered_set<
        estd::decay_t<
          decltype(tuplify(std::declval<InputTs>()...))> >;

    template <typename StateT, typename ...InputTs>
    auto operator() (StateT&& s, InputTs&& ...is)
      -> decltype(
        wrap_state(
          call(step,
               state_unwrap(std::forward<StateT>(s)),
               std::forward<InputTs>(is)...),
          cache_t<InputTs...>{}))
    {
      auto cache = state_data(std::forward<StateT>(s), []{
          return cache_t<InputTs...>{};
        });

      auto unique = cache.insert(tuplify(is...)).second;
      return wrap_state(
          unique
            ? call(step,
                   state_unwrap(std::forward<StateT>(s)),
                   std::forward<InputTs>(is)...)
            : skip(step,
                   state_unwrap(std::forward<StateT>(s)),
                   std::forward<InputTs>(is)...),
          std::move(cache));
    }
  };
};

} // namespace detail

using distinct_t = transducer_impl<detail::distinct_rf_gen>;

/*!
 * Equivalent to clojure/distinct$0
 */
constexpr distinct_t distinct {};

} // namespace xform<commit_msg>xform: more performance `distinct` avoiding union types<commit_after>// Copyright: 2014, 2015, Ableton AG, Berlin. All rights reserved.
/*!
 * @file
 */

#pragma once

#include <atria/xform/transducer_impl.hpp>
#include <atria/xform/state_wrapper.hpp>
#include <atria/xform/with_state.hpp>
#include <atria/estd/tuple_hash.hpp>
#include <atria/prelude/tuplify.hpp>
#include <atria/estd/functional.hpp>
#include <atria/meta/copy_traits.hpp>
#include <unordered_set>

namespace atria {
namespace xform {

namespace detail {

struct distinct_rf_gen
{
  template <typename ReducingFnT>
  struct apply
  {
    ReducingFnT step;

    template <typename ...InputTs>
    using cache_t = std::unordered_set<
        estd::decay_t<decltype(tuplify(std::declval<InputTs>()...))> >;

    template <typename StateT, typename ...InputTs>
    auto operator() (StateT&& s, InputTs&& ...is)
      -> decltype(wrap_state(step(state_unwrap(s), is...), cache_t<InputTs...>{}))
    {
      using result_t   = decltype(wrap_state(step(state_unwrap(s), is...), cache_t<InputTs...>{}));
      using complete_t = decltype(state_complete(s));

      using wrapped_t   = meta::copy_decay_t<StateT, estd::decay_t<result_t>>;
      using unwrapped_t = meta::copy_decay_t<StateT, estd::decay_t<complete_t>>;

      return with_state(
        std::forward<StateT>(s),
        [&] (unwrapped_t&& st) {
          return wrap_state(
            step(state_unwrap(std::forward<unwrapped_t>(st)),
                 std::forward<InputTs>(is)...),
            cache_t<InputTs...>{ tuplify(is...) });
        },
        [&] (wrapped_t st) {
          auto& cache = state_wrapper_data(st);
          auto unique = cache.insert(tuplify(is...)).second;
          return !unique
            ? st
            : wrap_state(
              step(std::move(state_unwrap(st)),
                   std::forward<InputTs>(is)...),
              std::move(cache));
        });
    }
  };
};

} // namespace detail

using distinct_t = transducer_impl<detail::distinct_rf_gen>;

/*!
 * Equivalent to clojure/distinct$0
 */
constexpr distinct_t distinct {};

} // namespace xform
{'fraction_non_alphanumeric': 0.15493326354357498, 'fraction_numerical': 0.004710808688824915, 'mean_word_length': 2.7655172413793103, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2964639', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1260, 'n_words': 223}