<commit_before>#!/usr/bin/env python
# -*- c-file-style: "sourcery" -*-
#
# Use and distribution of this software and its source code is governed
# by the terms and conditions defined in the "LICENSE" file that is part
# of this source code package.
#
"""
UNDER CONSTRUCTION: Driver for training the covariate shift detector on the command line.

Usage:
  shift_detect [options] DATASETS PREPROCESS

Arguments:

  DATASETS                    comma separated list of datasets in importance order
  PREPROCESS                  comma separated list of preprocessing directives for each dataset specified

Options:
  -h, --help                  show this help message and exit
  -v, --version               show version and exit

  -g, --debug                 operate in debug mode
  -b, --benchmark             operate in benchmark mode

  -m, --host M                database host name
  -p, --port P                database port
  -d, --database D            database name

  -o, --test                  use fixed gaussian sample data against covariate shift detection algorithm
  -j, --sudden                apply abrupt covariate shift detection algorithm on samples from test interval
  -r, --gradual               apply gradual covariate shift detection algorithm on samples from test interval(s)

  -c, --collection C          run detector on named collection

  -w, --win1start W           ISO8601 start date-time for reference interval sample
  -x, --win1end X             ISO8601 end date-time for reference interval sample
  -y, --win2start Y           ISO8601 start date-time for test interval sample
  -z, --win2end Z             ISO8601 end date-time for test interval sample
  -n, --subseqlen N           window size used for interval subsequences when generating Hankel matrix
  -e, --granularity E         for any datetime fields, only use values at the granularity specified by E
  -u, --samplesize U          sample at most U observations from the population defined by each interval; no sampling if U=0

  -a, --alpha A               use A as the alpha relative parameter (ULSIF {a = 0}, RULSIF {a > 0} )
  -s, --sigma S               use G as the gaussian width parameter override, sigma
  -l, --lambda L              use L as the regularization parameter override, lamda
  -k, --kernels K             use K number of kernels basis functions as an override
  -f, --folds F               use F number of cross validation folds, where leave one out CV is used if {f = 0}

  -1, --scatter               render scatter plot of reference and test samples of dataset1 vs dataset2
  -2, --scatterhistogram      render scatter + histogram plot of dataset1 vs dataset2
  -3, --scatterkernel         render scatter plot of reference and test samples of dataset1 in Hilbert space

  -4, --scatterratio          render scatter plot of reference and test samples of dataset1 vs density ratio
  -5, --timeseries            render time series of entire sample of dataset1,...,datasetN against change point score
  -6, --perfline              render line chart of compute performance for the specified datasets

  --dataset1 1                run detector on named data set field name from the specified collection; treated as R^d=1
  --dataset2 2                run detector on named data set field name from the specified collection; treated as R^d=2
  --dataset3 3                run detector on named data set field name from the specified collection; treated as R^d=3
  --dataset4 4                run detector on named data set field name from the specified collection; treated as R^d=4
  --dataset5 5                run detector on named data set field name from the specified collection; treated as R^d=5
  --dataset6 6                run detector on named data set field name from the specified collection; treated as R^d=6
  --dataset7 7                run detector on named data set field name from the specified collection; treated as R^d=7
  --dataset8 8                run detector on named data set field name from the specified collection; treated as R^d=8
  --dataset9 9                run detector on named data set field name from the specified collection; treated as R^d=9
  --dataset0 0                run detector on named data set field name from the specified collection; treated as R^d=10

  --preprocessCategorical 1   preprocess categorical variables using [NONE, ONEHOT, BINARIZE_LABEL]
  --preprocessOrdinal 2       preprocess ordinal     variables using [NONE, BINARIZE_LABEL]
  --preprocessContinuous 3    preprocess continuous  variables using [NONE, STANDARDIZE, RESCALE, NORMALIZE]

"""
from __future__             import print_function
from docopt                 import docopt<commit_msg>Trimmed design of command-line driver<commit_after>#!/usr/bin/env python
# -*- c-file-style: "sourcery" -*-
#
# Use and distribution of this software and its source code is governed
# by the terms and conditions defined in the "LICENSE" file that is part
# of this source code package.
#
"""
UNDER CONSTRUCTION: Driver for training the covariate shift detector on the command line.

Usage:
  shift_detect [options] DATASETS PREPROCESS

Arguments:

  DATASETS                    comma separated list of datasets in importance order
  PREPROCESS                  comma separated list of preprocessing directives for each dataset specified

Options:
  -h, --help                  show this help message and exit
  -v, --version               show version and exit

  -g, --debug                 operate in debug mode
  -b, --benchmark             operate in benchmark mode

  -m, --host M                database host name
  -p, --port P                database port
  -d, --database D            database name
  -c, --collection C          run detector on named database collection

  -o, --test                  use fixed gaussian sample data against covariate shift detection algorithm

  -a, --alpha A               use A as the alpha relative parameter (ULSIF {a = 0}, RULSIF {a > 0} )
  -s, --sigma S               use G as the gaussian width parameter override, sigma
  -l, --lambda L              use L as the regularization parameter override, lamda
  -k, --kernels K             use K number of kernels basis functions as an override
  -f, --folds F               use F number of cross validation folds, where leave one out CV is used if {f = 0}

  -1, --scatter               render scatter plot of reference and test samples of dataset1 vs dataset2
  -2, --scatterhistogram      render scatter + histogram plot of dataset1 vs dataset2
  -3, --scatterkernel         render scatter plot of reference and test samples of dataset1 in Hilbert space

  -4, --scatterratio          render scatter plot of reference and test samples of dataset1 vs density ratio
  -5, --perfline              render line chart of compute performance for the specified datasets

  --dataset1 1                run detector on named data set field name from the specified collection; treated as R^d=1
  --dataset2 2                run detector on named data set field name from the specified collection; treated as R^d=2
  --dataset3 3                run detector on named data set field name from the specified collection; treated as R^d=3
  --dataset4 4                run detector on named data set field name from the specified collection; treated as R^d=4
  --dataset5 5                run detector on named data set field name from the specified collection; treated as R^d=5
  --dataset6 6                run detector on named data set field name from the specified collection; treated as R^d=6
  --dataset7 7                run detector on named data set field name from the specified collection; treated as R^d=7
  --dataset8 8                run detector on named data set field name from the specified collection; treated as R^d=8
  --dataset9 9                run detector on named data set field name from the specified collection; treated as R^d=9
  --dataset0 0                run detector on named data set field name from the specified collection; treated as R^d=10

  --preprocessCategorical 1   preprocess categorical variables using [NONE, ONEHOT, BINARIZE_LABEL]
  --preprocessOrdinal 2       preprocess ordinal     variables using [NONE, BINARIZE_LABEL]
  --preprocessContinuous 3    preprocess continuous  variables using [NONE, STANDARDIZE, RESCALE, NORMALIZE]

"""
from __future__             import print_function
from docopt                 import docopt
{'fraction_non_alphanumeric': 0.05385340560923874, 'fraction_numerical': 0.014023096865425407, 'mean_word_length': 2.2780996523754347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7305719', 'n_tokens_mistral': 2165, 'n_tokens_neox': 2031, 'n_words': 1090}