<commit_before>#############################
# YASS configuration schema #
#############################

# Definition reference: http://docs.python-cerberus.org/en/stable/

data:
  type: dict
  required: True
  schema:
    # project's root folder, data will be loaded and saved here
    # can be an absolute or relative path
    root_folder:
      type: string
      required: True
    # recordings filename, relative to root folder
    recordings:
      type: string
      required: True
    # channel geometry filename, relative to root folder
    geometry:
      type: string
      required: True
    stimulus:
      type: string
      required: False
      default: None
    triggers:
      type: string
      required: False
      default: None

resources:
  type: dict
  required: True
  schema:
    multi_processing: 
      type: [integer, string]
      required: True
      default: 0
    n_processors: 
      type: [integer, string]
      required: True
      default: 1
    n_sec_chunk: 
      type: [integer, string]
      required: True
      default: 2
    n_sec_chunk_gpu:
      type: [integer, string]
      required: True
      default: 1

recordings:
  type: dict
  required: True
  schema:
    # precision of the recording – must be a valid numpy dtype
    dtype:
      type: string
      required: True
    # recording rate (in Hz)
    sampling_rate:
      type: integer
      required: True
    # number of channels
    n_channels:
      type: integer
      required: True
    # channels spatial radius
    spatial_radius:
      type: integer
      required: True
    # temporal length of wavforms in ms
    spike_size_ms:
      type: float
      required: True

preprocess:
  type: dict
  default:
    apply_filter: True
    dtype: float64
    filter:
      order: 3
      low_pass_freq: 300
      high_factor: 0.1
  schema:
    # apply butterworth filter in the preprocessing step?
    apply_filter:
      type: boolean
      default: True
    # output dtype for transformed data
    dtype:
      type: string
      default: float64
    filter:
      type: dict
      default:
        order: 3
        low_pass_freq: 300
        high_factor: 0.1
      schema:
        # Order of Butterworth filter
        order:
          type: integer
          default: 3
        # Low pass frequency (Hz)
        low_pass_freq:
          type: integer
          default: 300
        # High pass factor (proportion of sampling rate)
        high_factor:
          type: float
          default: 0.1
detect:
  type: dict
  default:
    method: nn
    temporal_features: 3
    neural_network_detector:
      filename: detect_nn1.ckpt
      threshold_spike: 0.5
    neural_network_autoencoder:
      filename: ae_nn1.ckpt
    threshold_detector:
      std_factor: 4
    run_sec_chunk: full

  schema:
    # 'nn' for neural net detction, 'threshold' for amplitude threshold detection
    method:
      type: string
      required: True
      allowed: [threshold, nn]
    # number of features in the temporal dimension to use when applying
    # dimensionality reduction
    temporal_features:
      type: integer
      default: 3
    neural_network_detector:
      type: dict
      default:
        # model name, can be any of the models included in yass (detectnet1.ckpt),
        # a relative folder to data.root_fodler (e.g.
        # $ROOT_FOLDER/models/mymodel.ckpt) or an absolute path to a model
        # (e.g. /path/to/my/model.ckpt). In the same folder as your model, there
        # must be a yaml file with the number and size of the filters, the file
        # should be named exactly as your model but with yaml extension
        # see yass/src/assets/models/ for an example
        filename: detect_nn1.ckpt
        # Threshold for spike event detection
        threshold_spike: 0.5
    neural_network_autoencoder:
      type: dict
      default:
        # same rules apply as in neural_network_detector.filename but no
        # yaml file is needed
        filename: ae_nn1.ckpt
    threshold_detector:
      type: dict
      default:
        std_factor: 4

cluster:
  type: dict
  default:
    max_n_spikes: 10000
    min_spikes: 100
    prior:
      beta: 1<commit_msg>add new param into config schema<commit_after>#############################
# YASS configuration schema #
#############################

# Definition reference: http://docs.python-cerberus.org/en/stable/

data:
  type: dict
  required: True
  schema:
    # project's root folder, data will be loaded and saved here
    # can be an absolute or relative path
    root_folder:
      type: string
      required: True
    # recordings filename, relative to root folder
    recordings:
      type: string
      required: True
    # channel geometry filename, relative to root folder
    geometry:
      type: string
      required: True
    stimulus:
      type: string
      required: False
      default: None
    triggers:
      type: string
      required: False
      default: None

resources:
  type: dict
  required: True
  schema:
    multi_processing: 
      type: [integer, string]
      required: True
      default: 0
    n_processors: 
      type: [integer, string]
      required: True
      default: 1
    n_sec_chunk: 
      type: [integer, string]
      required: True
      default: 2
    n_sec_chunk_gpu:
      type: [integer, string]
      required: True
      default: 1

recordings:
  type: dict
  required: True
  schema:
    # precision of the recording – must be a valid numpy dtype
    dtype:
      type: string
      required: True
    # recording rate (in Hz)
    sampling_rate:
      type: integer
      required: True
    # number of channels
    n_channels:
      type: integer
      required: True
    # channels spatial radius
    spatial_radius:
      type: integer
      required: True
    # temporal length of wavforms in ms
    spike_size_ms:
      type: float
      required: True

preprocess:
  type: dict
  default:
    apply_filter: True
    dtype: float64
    filter:
      order: 3
      low_pass_freq: 300
      high_factor: 0.1
  schema:
    # apply butterworth filter in the preprocessing step?
    apply_filter:
      type: boolean
      default: True
    # output dtype for transformed data
    dtype:
      type: string
      default: float64
    filter:
      type: dict
      default:
        order: 3
        low_pass_freq: 300
        high_factor: 0.1
      schema:
        # Order of Butterworth filter
        order:
          type: integer
          default: 3
        # Low pass frequency (Hz)
        low_pass_freq:
          type: integer
          default: 300
        # High pass factor (proportion of sampling rate)
        high_factor:
          type: float
          default: 0.1
detect:
  type: dict
  default:
    method: nn
    temporal_features: 3
    neural_network_detector:
      filename: detect_nn1.ckpt
      threshold_spike: 0.5
    neural_network_autoencoder:
      filename: ae_nn1.ckpt
    threshold_detector:
      std_factor: 4
    run_sec_chunk: full

  schema:
    # 'nn' for neural net detction, 'threshold' for amplitude threshold detection
    method:
      type: string
      required: True
      allowed: [threshold, nn]
    # number of features in the temporal dimension to use when applying
    # dimensionality reduction
    temporal_features:
      type: integer
      default: 3
    neural_network_detector:
      type: dict
      default:
        # model name, can be any of the models included in yass (detectnet1.ckpt),
        # a relative folder to data.root_fodler (e.g.
        # $ROOT_FOLDER/models/mymodel.ckpt) or an absolute path to a model
        # (e.g. /path/to/my/model.ckpt). In the same folder as your model, there
        # must be a yaml file with the number and size of the filters, the file
        # should be named exactly as your model but with yaml extension
        # see yass/src/assets/models/ for an example
        filename: detect_nn1.ckpt
        # Threshold for spike event detection
        threshold_spike: 0.5
    neural_network_autoencoder:
      type: dict
      default:
        # same rules apply as in neural_network_detector.filename but no
        # yaml file is needed
        filename: ae_nn1.ckpt
    threshold_detector:
      type: dict
      default:
        std_factor: 4

cluster:
  type: dict
  default:
    max_n_sec_chunk: 600
    min_spikes: 100
    prior:
      beta: 1
{'fraction_non_alphanumeric': 0.07282246549262256, 'fraction_numerical': 0.011185149928605425, 'mean_word_length': 2.1597744360902253, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7538639', 'n_tokens_mistral': 2481, 'n_tokens_neox': 2394, 'n_words': 844}