<commit_before>require "ostruct"

class BaseParameterParser

  # The fields listed here are the only ones which the search results can be
  # ordered by.  These are listed and validated explicitly because
  # sorting by arbitrary fields can be expensive in terms of memory usage in
  # elasticsearch, and because elasticsearch gives fairly obscure error
  # messages if undefined sort fields are used.
  ALLOWED_SORT_FIELDS = %w(
    last_update
    public_timestamp
  )

  # The fields listed here are the only ones which can be used to filter by.
  ALLOWED_FILTER_FIELDS = %w(
    document_type
    format
    organisations
    section
    specialist_sectors
  )

  # Incoming filter fields will have their names transformed according to the
  # following mapping. Fields not listed here will be passed through unchanged.
  FILTER_NAME_MAPPING = {
    "document_type" => "_type",
  }

  # The fields listed here are the only ones which can be used to calculated
  # facets for.  This should be a subset of ALLOWED_FILTER_FIELDS
  ALLOWED_FACET_FIELDS = %w(
    format
    organisations
    section
    specialist_sectors
  )

  # The fields for which facet examples are allowed to be requested.
  # This is locked down because these can only be requested with the current
  # version of elasticsearch by performing a separate query for each facet
  # option.  This is done using the msearch API to perform many queries
  # together, but is still potentially expensive.  They could be efficiently
  # calculated with the top-documents aggregator in elasticsearch 1.3, so this
  # restriction could be relaxed in future.
  ALLOWED_FACET_EXAMPLE_FIELDS = %w(
    format
    organisations
    section
    specialist_sectors
  )

  # The keys by which facet values can be sorted (using the "order" option).
  # Multiple can be supplied, separated by colons - items which are equal
  # according to the first option are sorted by the next key, etc.  keys can be
  # preceded with a "-" to sort in descending order.<commit_msg>Allow filtering and facetting by "manual"<commit_after>require "ostruct"

class BaseParameterParser

  # The fields listed here are the only ones which the search results can be
  # ordered by.  These are listed and validated explicitly because
  # sorting by arbitrary fields can be expensive in terms of memory usage in
  # elasticsearch, and because elasticsearch gives fairly obscure error
  # messages if undefined sort fields are used.
  ALLOWED_SORT_FIELDS = %w(
    last_update
    public_timestamp
  )

  # The fields listed here are the only ones which can be used to filter by.
  ALLOWED_FILTER_FIELDS = %w(
    document_type
    format
    manual
    organisations
    section
    specialist_sectors
  )

  # Incoming filter fields will have their names transformed according to the
  # following mapping. Fields not listed here will be passed through unchanged.
  FILTER_NAME_MAPPING = {
    "document_type" => "_type",
  }

  # The fields listed here are the only ones which can be used to calculated
  # facets for.  This should be a subset of ALLOWED_FILTER_FIELDS
  ALLOWED_FACET_FIELDS = %w(
    format
    manual
    organisations
    section
    specialist_sectors
  )

  # The fields for which facet examples are allowed to be requested.
  # This is locked down because these can only be requested with the current
  # version of elasticsearch by performing a separate query for each facet
  # option.  This is done using the msearch API to perform many queries
  # together, but is still potentially expensive.  They could be efficiently
  # calculated with the top-documents aggregator in elasticsearch 1.3, so this
  # restriction could be relaxed in future.
  ALLOWED_FACET_EXAMPLE_FIELDS = %w(
    format
    organisations
    section
    specialist_sectors
  )

  # The keys by which facet values can be sorted (using the "order" option).
  # Multiple can be supplied, separated by colons - items which are equal
  # according to the first option are sorted by the next key, etc.  keys can be
  # preceded with a "-" to sort in descending order.
{'fraction_non_alphanumeric': 0.03957010258915486, 'fraction_numerical': 0.0009770395701025891, 'mean_word_length': 3.4705240174672487, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4097616', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1087, 'n_words': 528}