<commit_before>
          
   STATSD  
          



VERSION ==>                       #0.8.6

STATSD PROTOCOL ==>               #See its doc

GOAL ==>                          #Multiple parts:
                                  #  - a client is meant to send statsd protocol packets
                                  #  - server receiving statsd protocol packets
                                  #     - protocol-agnostic. Built-in: UDP, TCP
                                  #  - daemon aggregating and normalizing packets received by server, into metrics
                                  #  - "backends" receive daemon's metrics at regular intervals (flushes)
                                  #    for storing, aggregating, visualizing, etc.
                                  #     - built-in: Graphite, console
                                  #  - second TCP server receiving administration instructions
                                  #  - basic health monitoring
                                  #  - debug|logs are printed to stdout (but can redirect to syslog)
                                  #Metrics:
                                  #  - based on statsd protocol
                                  #     - counters, gauges, sets, timers
                                  #     - does not support TYPE "h", but supports histograms
                                  #     - compute aggregate data

DATE_NUM                          #ms since Epoch


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:            SERVERS            :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


statsd CONFIG_FILE                #Starts servers

CONFIG_FILE                       #Must be JavaScript OBJ (run with eval())
CONF.automaticConfigReload        #BOOL (def: true). If CONFIG_FILE is updated, reload CONF

CONF.servers                      #Servers receiving metrics packets. OBJ_ARR:
                                  #  - server 'PATH'
                                  #  - other config properties passed to servers
                                  #Points to Node.js file exporting a start(OBJ, FUNC(BUFFER, OBJ2))->BOOL
                                  #  - starts the server
                                  #  - if CONF.servers not specified, OBJ is CONF
                                  #  - FUNC(BUFFER, OBJ2) should be called on new packets
                                  #     - BUFFER is statsd protocol packet
                                  #     - OBJ2 is arbitrary protocol-specific, only passed to 'packet' event
                                  #  - BOOL is success
                                  #Available ones:
                                  #  - './servers/udp.js' (def): UDP with CONF:
                                  #     - address STR (def: undefined)
                                  #     - port NUM (def: 8125)
                                  #     - address_ipv6 BOOL (def: true)
                                  #  - './servers/tcp.js': TCP with CONF:
                                  #     - address STR (def: undefined)
                                  #     - port NUM (def: 8125)
                                  #     - socket 'PATH': alternative to port
                                  #     - socket_mod UINT: does chmod() on socket
                                  #  - any absolute PATH

CONF.title                        #STR|false. Sets process.title (def: "statsd ...")

LOAD BALANCING ==>                #A load balancer ("cluster proxy") is available.
                                  #See documentation if ever needs it


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:             FLUSH             :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


FLUSH ==>                         #At regular interval:
                                  #  - retrieve METRICS since last flush
                                  #  - emit EVENT_EMITTER 'flush' with METRICS, to communicate to backends
                                  #  - resets all METRICS to 0
CONF.flushInterval                #NUM (in ms, def: 10s)
                                  #Also flushes on process exit

CONF.deleteIdleStats              #BOOL (def: false). Do not send metrics when 0 since last flush
CONF.deleteCounters
CONF.deleteGauges
CONF.deleteTimers
CONF.deleteSets                   #Same for TYPE "c|g|ms|s" only
CONF.gaugesMaxTTL                 #UINT (def: 1).
                                  #When CONF.deleteGauges true, still keep sending new gauge metrics for UINT flushes
MGT_CMD delcounters|deltimers|    #Deletes METRICS.counters|timers|gauges.MNAME
 delgauges MNAME...               #MNAME can be .* or MNAME.*

CONF.keyFlush.interval            #NUM (in ms, def: 0)
                                  #Every NUMms, print several lines where each line is "TIMESTAMP count=NUM key=MNAME"
                                  #  - NUM is number of metrics MNAME sent since last time
                                  #  - sorted by NUM
CONF.keyFlush.log                 #'PATH' to write to (def: stdout)
CONF.keyFlush.percent             #NUM (def: 100): only print the NUM% most used metrics


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:            METRICS            :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


METRICS                           #OBJ passed to backends, aggregating and normalizing client packets sent since last flush

METRICS.counters
 ["statsd.packets_received"]      #NUM. Total number of packets sent
METRICS.counters
 ["statsd.metrics_received"]      #NUM. Total number of metric messages sent
METRICS.counters
 ["statsd.bad_lines_seen"]        #NUM. Total number of metric messages with invalid syntax
METRICS.gauges
 ["statsd.timestamp_lag"]         #NUMms since last flush, minus flushInterval. I.e. computes how accurately flush happens
METRICS.statsd_metrics.
 processing_time                  #NUM (in ms) that it took to compute TIMER_DATA.*

METRICS.counters.MNAME            #NUM. For "c|m" metrics
                                  #If @SAMPLE_RATE, multiplied by its inverse
METRICS.counter_rates.MNAME       #Like counters.MNAME, but as rate per second (i.e. divided by CONF.flushInterval / 1000)

METRICS.gauges.MNAME              #NUM. For "g" metrics

METRICS.sets.MNAME                #SET of "s" metrics values

METRICS.timers.MNAME              #NUM_ARR. For "ms" metrics
METRICS.timer_counters.MNAME      #NUM. For "ms" metrics
                                  #If @SAMPLE_RATE, multiplied by its inverse

METRICS.timer_data.MNAME          #TIMER_DATA. Aggregated information for "ms" metrics since last flush
TIMER_DATA.count                  #Like METRICS.timer_counters.MNAME
TIMER_DATA.count_ps               #Like METRICS.timer_counters.MNAME, but as rate per second (i.e. divided by CONF.flushInterval / 1000)
TIMER_DATA.lower                  #Min
TIMER_DATA.upper                  #Max
TIMER_DATA.sum                    #Sum of each value
TIMER_DATA.sum_squares            #Sum of each (value ** 2)
TIMER_DATA.mean                   #Mean
TIMER_DATA.median                 #Median
TIMER_DATA.std                    #Standard deviation

TIMER_DATA.count|lower|upper|mean|
 sum|sum_squares_NUM              #Like TIMER_DATA.* for only up to a specific percentile NUM
CONF.percentThreshold             #0-100[_ARR] (def: 90). Percentiles used with TIMER_DATA.*_NUM
METRICS.pctThreshold              #NUM_ARR from CONF.percentThreshold

TIMER_DATA.histogram.bin_NUM      #0-1. Histogram of "ms" metrics across flush periods.
                                  #NUM "." is replaced by "_"
CONF.histogram                    #Defines histogram's bins. OBJ_ARR (def: []):
                                  #  - metric 'MNAME': '' for all
                                  #  - bins NUM|'inf'_ARR:
                                  #     - increasing
                                  #     - first NUM is assumed to be 0
METRICS.histogram                 #OBJ_ARR from CONF.histogram

CONF.calculatedTimerMetrics       #'VAR'_ARR. Whitelist TIMER_DATA.VAR. Others will be undefined
                                  #'VAR' can end with '_percent' for TIMER_DATA.*_NUM

CONF.keyNameSanitize              #BOOL (def: true).
                                  #Slugify MNAME:
                                  #  - spaces -> _
                                  #  - / -> -
                                  #  - remove anything but \w-.

CONF.prefixStats                  #STR prefixing keys in METRICS.counters|gauges["statsd..."] (def: 'statsd').


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:           BACKENDS            :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


CONF.backends                     #Backends, i.e. processes receiving metrics.
                                  #'PATH'_ARR to Node.js file exporting a init(DATE_NUM, CONF, EVENT_EMITTER, LOGGER)->BOOL
                                  #  - LOGGER has single method log('MESSAGE', 'TYPE') (for logs|debugs)
                                  #  - BOOL is success
                                  #Available built-in backends:
                                  #  - './backends/graphite.js' (def)
                                  #  - './backends/console.js':
                                  #     - print on stdout
                                  #     - CONF.console.prettyprint BOOL (def: false): add colors and max depth 5
                                  #  - './backends/repeater.js'
                                  #     - echo raw statsd packet
                                  #     - CONF:
                                  #        - repeaterProtocol 'udp4' (def), 'udp6' or 'tcp'
                                  #        - repeater OBJ_ARR: host STR, port NUM
                                  #        - debug BOOL (def: false)
                                  #  - any absolute PATH
                                  #More at https://github.com/statsd/statsd/blob/master/docs/backend.md
EVENT_EMITTER.on
 ('flush', FUNC(DATE_NUM,METRICS))#On each flush, i.e. should handle new metrics
EVENT_EMITTER.on('status',
 FUNC(ERROR, 'VAR', 'VAR2', VAL)) #On MGT_CMD stats, i.e. to return stats that should be printed then
EVENT_EMITTER.on
 ('packet', FUNC(BUFFER, OBJ2))   #On new raw statsd packet


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:          MANAGEMENT           :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


MANAGEMENT SERVER ==>             #TCP server meant for administation.
                                  #Messages "MGT_CMD [ARG...]"
                                  #"Prints" means send back in TCP

CONF.mgmt_address                 #Server address (def: undefined)
CONF.mgmt_port                    #server port (def: 8126)

MGT_CMD config                    #Prints CONF
MGT_CMD help                      #Prints available MGT_CMD
MGT_CMD quit                      #Stop management server

MGT_CMD counters|timers|gauges    #Prints METRICS.counters|timers|gauges

MGT_CMD stats                     #Prints some information since load time:
                                  #  - number of ms
                                  #  - time since last packet
                                  #  - number of metric messages with invalid syntax
                                  #Emit 'status' event to let backends return values they want to print
                                  #  - prints 'VAR.VAR2: VAL'
                                  #  - if VAR2 starts with 'last_', VAL is a DATE_NUM, and the time interval is printed instead


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:            LOGGING            :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


CONF.log.backend                  #Where to print debug|error messages
                                  #Can be:
                                  #  - 'stdout' (def): prefixed with "DEBUG: "
                                  #  - 'syslog':
                                  #     - use modern-syslog
                                  #     - also uses CONF.log:
                                  #        - level STR (def: 'LOG_INFO')
                                  #        - application STR (def: 'statsd')

CONF.dumpMessages                 #BOOL (def: false). Print each metric message sent
CONF.debug                        #BOOL (def: false). Verbose


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:       HEALTH MONITORING       :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


MGT_CMD health                    #Prints whether monitored machine is "up|down"
                                  #Value depends only on MGT_CMD health up|down and CONF.healthStatus,
                                  #i.e. health checking must be implemented separately.
MGT_CMD health up|down            #Modifies health status
CONF.healthStatus                 #STR (def: 'up'). Initial health status
<commit_msg>no commit message<commit_after>
          
   STATSD  
          



VERSION ==>                       #0.9.0

STATSD PROTOCOL ==>               #See its doc

GOAL ==>                          #Multiple parts:
                                  #  - a client is meant to send statsd protocol packets
                                  #  - server receiving statsd protocol packets
                                  #     - protocol-agnostic. Built-in: UDP, TCP
                                  #  - daemon aggregating and normalizing packets received by server, into metrics
                                  #  - "backends" receive daemon's metrics at regular intervals (flushes)
                                  #    for storing, aggregating, visualizing, etc.
                                  #     - built-in: Graphite, console
                                  #  - second TCP server receiving administration instructions
                                  #  - basic health monitoring
                                  #  - debug|logs are printed to stdout (but can redirect to syslog)
                                  #Metrics:
                                  #  - based on statsd protocol
                                  #     - counters, gauges, sets, timers
                                  #     - does not support TYPE "h", but supports histograms
                                  #     - compute aggregate data

DATE_NUM                          #ms since Epoch


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:            SERVERS            :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


statsd CONFIG_FILE                #Starts servers

CONFIG_FILE                       #Must be JavaScript OBJ (run with eval())
CONF.automaticConfigReload        #BOOL (def: true). If CONFIG_FILE is updated, reload CONF

CONF.servers                      #Servers receiving metrics packets. OBJ_ARR:
                                  #  - server 'PATH'
                                  #  - other config properties passed to servers
                                  #Points to Node.js file exporting a start(OBJ, FUNC(BUFFER, OBJ2))->BOOL
                                  #  - starts the server
                                  #  - if CONF.servers not specified, OBJ is CONF
                                  #  - FUNC(BUFFER, OBJ2) should be called on new packets
                                  #     - BUFFER is statsd protocol packet
                                  #     - OBJ2 is arbitrary protocol-specific, only passed to 'packet' event
                                  #  - BOOL is success
                                  #Available ones:
                                  #  - './servers/udp.js' (def): UDP with CONF:
                                  #     - address STR (def: undefined)
                                  #     - port NUM (def: 8125)
                                  #     - address_ipv6 BOOL (def: true)
                                  #  - './servers/tcp.js': TCP with CONF:
                                  #     - address STR (def: undefined)
                                  #     - port NUM (def: 8125)
                                  #     - socket 'PATH': alternative to port
                                  #     - socket_mod UINT: does chmod() on socket
                                  #  - any absolute PATH

CONF.title                        #STR|false. Sets process.title (def: "statsd ...")

LOAD BALANCING ==>                #A load balancer ("cluster proxy") is available.
                                  #See documentation if ever needs it


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:             FLUSH             :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


FLUSH ==>                         #At regular interval:
                                  #  - retrieve METRICS since last flush
                                  #  - emit EVENT_EMITTER 'flush' with METRICS, to communicate to backends
                                  #  - resets all METRICS to 0
CONF.flushInterval                #NUM (in ms, def: 10s)
                                  #Also flushes on process exit

CONF.deleteIdleStats              #BOOL (def: false). Do not send metrics when 0 since last flush
CONF.deleteCounters
CONF.deleteGauges
CONF.deleteTimers
CONF.deleteSets                   #Same for TYPE "c|g|ms|s" only
CONF.gaugesMaxTTL                 #UINT (def: 1).
                                  #When CONF.deleteGauges true, still keep sending new gauge metrics for UINT flushes
MGT_CMD delcounters|deltimers|    #Deletes METRICS.counters|timers|gauges.MNAME
 delgauges MNAME...               #MNAME can be .* or MNAME.*

CONF.keyFlush.interval            #NUM (in ms, def: 0)
                                  #Every NUMms, print several lines where each line is "TIMESTAMP count=NUM key=MNAME"
                                  #  - NUM is number of metrics MNAME sent since last time
                                  #  - sorted by NUM
CONF.keyFlush.log                 #'PATH' to write to (def: stdout)
CONF.keyFlush.percent             #NUM (def: 100): only print the NUM% most used metrics


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:            METRICS            :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


METRICS                           #OBJ passed to backends, aggregating and normalizing client packets sent since last flush

METRICS.counters
 ["statsd.packets_received"]      #NUM. Total number of packets sent
METRICS.counters
 ["statsd.metrics_received"]      #NUM. Total number of metric messages sent
METRICS.counters
 ["statsd.bad_lines_seen"]        #NUM. Total number of metric messages with invalid syntax
METRICS.gauges
 ["statsd.timestamp_lag"]         #NUMms since last flush, minus flushInterval. I.e. computes how accurately flush happens
METRICS.statsd_metrics.
 processing_time                  #NUM (in ms) that it took to compute TIMER_DATA.*

METRICS.counters.MNAME            #NUM. For "c|m" metrics
                                  #If @SAMPLE_RATE, multiplied by its inverse
METRICS.counter_rates.MNAME       #Like counters.MNAME, but as rate per second (i.e. divided by CONF.flushInterval / 1000)

METRICS.gauges.MNAME              #NUM. For "g" metrics

METRICS.sets.MNAME                #SET of "s" metrics values

METRICS.timers.MNAME              #NUM_ARR. For "ms" metrics
METRICS.timer_counters.MNAME      #NUM. For "ms" metrics
                                  #If @SAMPLE_RATE, multiplied by its inverse

METRICS.timer_data.MNAME          #TIMER_DATA. Aggregated information for "ms" metrics since last flush
TIMER_DATA.count                  #Like METRICS.timer_counters.MNAME
TIMER_DATA.count_ps               #Like METRICS.timer_counters.MNAME, but as rate per second (i.e. divided by CONF.flushInterval / 1000)
TIMER_DATA.lower                  #Min
TIMER_DATA.upper                  #Max
TIMER_DATA.sum                    #Sum of each value
TIMER_DATA.sum_squares            #Sum of each (value ** 2)
TIMER_DATA.mean                   #Mean
TIMER_DATA.median                 #Median
TIMER_DATA.std                    #Standard deviation

TIMER_DATA.count|lower|upper|mean|
 sum|sum_squares_NUM              #Like TIMER_DATA.* for only up to a specific percentile NUM
CONF.percentThreshold             #0-100[_ARR] (def: 90). Percentiles used with TIMER_DATA.*_NUM
METRICS.pctThreshold              #NUM_ARR from CONF.percentThreshold

TIMER_DATA.histogram.bin_NUM      #0-1. Histogram of "ms" metrics across flush periods.
                                  #NUM "." is replaced by "_"
CONF.histogram                    #Defines histogram's bins. OBJ_ARR (def: []):
                                  #  - metric 'MNAME': '' for all
                                  #  - bins NUM|'inf'_ARR:
                                  #     - increasing
                                  #     - first NUM is assumed to be 0
METRICS.histogram                 #OBJ_ARR from CONF.histogram

CONF.calculatedTimerMetrics       #'VAR'_ARR. Whitelist TIMER_DATA.VAR. Others will be undefined
                                  #'VAR' can end with '_percent' for TIMER_DATA.*_NUM

CONF.keyNameSanitize              #BOOL (def: true).
                                  #Slugify MNAME:
                                  #  - spaces -> _
                                  #  - / -> -
                                  #  - remove anything but \w-.;=

CONF.prefixStats                  #STR prefixing keys in METRICS.counters|gauges["statsd..."] (def: 'statsd').


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:           BACKENDS            :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


CONF.backends                     #Backends, i.e. processes receiving metrics.
                                  #'PATH'_ARR to Node.js file exporting a init(DATE_NUM, CONF, EVENT_EMITTER, LOGGER)->BOOL
                                  #  - LOGGER has single method log('MESSAGE', 'TYPE') (for logs|debugs)
                                  #  - BOOL is success
                                  #Available built-in backends:
                                  #  - './backends/graphite.js' (def)
                                  #  - './backends/console.js':
                                  #     - print on stdout
                                  #     - CONF.console.prettyprint BOOL (def: false): add colors and max depth 5
                                  #  - './backends/repeater.js'
                                  #     - echo raw statsd packet
                                  #     - CONF:
                                  #        - repeaterProtocol 'udp4' (def), 'udp6' or 'tcp'
                                  #        - repeater OBJ_ARR: host STR, port NUM
                                  #        - debug BOOL (def: false)
                                  #  - any absolute PATH
                                  #More at https://github.com/statsd/statsd/blob/master/docs/backend.md
EVENT_EMITTER.on
 ('flush', FUNC(DATE_NUM,METRICS))#On each flush, i.e. should handle new metrics
EVENT_EMITTER.on('status',
 FUNC(ERROR, 'VAR', 'VAR2', VAL)) #On MGT_CMD stats, i.e. to return stats that should be printed then
EVENT_EMITTER.on
 ('packet', FUNC(BUFFER, OBJ2))   #On new raw statsd packet


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:          MANAGEMENT           :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


MANAGEMENT SERVER ==>             #TCP server meant for administation.
                                  #Messages "MGT_CMD [ARG...]"
                                  #"Prints" means send back in TCP

CONF.mgmt_address                 #Server address (def: undefined)
CONF.mgmt_port                    #server port (def: 8126)

MGT_CMD config                    #Prints CONF
MGT_CMD help                      #Prints available MGT_CMD
MGT_CMD quit                      #Stop management server

MGT_CMD counters|timers|gauges    #Prints METRICS.counters|timers|gauges

MGT_CMD stats                     #Prints some information since load time:
                                  #  - number of ms
                                  #  - time since last packet
                                  #  - number of metric messages with invalid syntax
                                  #Emit 'status' event to let backends return values they want to print
                                  #  - prints 'VAR.VAR2: VAL'
                                  #  - if VAR2 starts with 'last_', VAL is a DATE_NUM, and the time interval is printed instead


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:            LOGGING            :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


CONF.log.backend                  #Where to print debug|error messages
                                  #Can be:
                                  #  - 'stdout' (def): prefixed with "DEBUG: "
                                  #  - 'syslog':
                                  #     - use modern-syslog
                                  #     - also uses CONF.log:
                                  #        - level STR (def: 'LOG_INFO')
                                  #        - application STR (def: 'statsd')

CONF.dumpMessages                 #BOOL (def: false). Print each metric message sent
CONF.debug                        #BOOL (def: false). Verbose


                                             /=+===============================+=\
                                            /  :                               :  \
                                            )==:       HEALTH MONITORING       :==(
                                            \  :_______________________________:  /
                                             \=+===============================+=/


MGT_CMD health                    #Prints whether monitored machine is "up|down"
                                  #Value depends only on MGT_CMD health up|down and CONF.healthStatus,
                                  #i.e. health checking must be implemented separately.
MGT_CMD health up|down            #Modifies health status
CONF.healthStatus                 #STR (def: 'up'). Initial health status

{'fraction_non_alphanumeric': 0.11869610661092239, 'fraction_numerical': 0.0034622419649856283, 'mean_word_length': 0.9869556752547213, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 408, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2498449', 'n_tokens_mistral': 8953, 'n_tokens_neox': 8428, 'n_words': 1876}