<commit_before>'use strict';

// Simple and configurable map and reduce dataflow transforms

const _ = require('underscore');
const dbclient = require('abacus-dbclient');
const batch = require('abacus-batch');
const retry = require('abacus-retry');
const breaker = require('abacus-breaker');
const yieldable = require('abacus-yieldable');
const throttle = require('abacus-throttle');
const transform = require('abacus-transform');
const request = require('abacus-request');
const urienv = require('abacus-urienv');
const router = require('abacus-router');
const partition = require('abacus-partition');
const lockcb = require('abacus-lock');
const lru = require('abacus-lrucache');
const oauth = require('abacus-oauth');
const vcapenv = require('abacus-vcapenv');
const cluster = require('abacus-cluster');
const dedupe = require('abacus-dedupe');
const seqid = require('abacus-seqid');
const url = require('url');

const extend = _.extend;
const map = _.map;
const object = _.object;
const filter = _.filter;
const find = _.find;
const pairs = _.pairs;
const zip = _.zip;
const last = _.last;
const reduce = _.reduce;
const memoize = _.memoize;
const omit = _.omit;
const range = _.range;
const uniq = _.uniq;
const pick = _.pick;

const tmap = yieldable(transform.map);
const treduce = yieldable(transform.reduce);

const brequest = yieldable(throttle(retry(breaker(batch(request)))));<commit_msg>Configurable retry post to dataflow sink<commit_after>'use strict';

// Simple and configurable map and reduce dataflow transforms

const _ = require('underscore');
const dbclient = require('abacus-dbclient');
const batch = require('abacus-batch');
const retry = require('abacus-retry');
const breaker = require('abacus-breaker');
const yieldable = require('abacus-yieldable');
const throttle = require('abacus-throttle');
const transform = require('abacus-transform');
const request = require('abacus-request');
const urienv = require('abacus-urienv');
const router = require('abacus-router');
const partition = require('abacus-partition');
const lockcb = require('abacus-lock');
const lru = require('abacus-lrucache');
const oauth = require('abacus-oauth');
const vcapenv = require('abacus-vcapenv');
const cluster = require('abacus-cluster');
const dedupe = require('abacus-dedupe');
const seqid = require('abacus-seqid');
const url = require('url');

const extend = _.extend;
const map = _.map;
const object = _.object;
const filter = _.filter;
const find = _.find;
const pairs = _.pairs;
const zip = _.zip;
const last = _.last;
const reduce = _.reduce;
const memoize = _.memoize;
const omit = _.omit;
const range = _.range;
const uniq = _.uniq;
const pick = _.pick;

const tmap = yieldable(transform.map);
const treduce = yieldable(transform.reduce);

// Return the configured number of retries
const retries = process.env.SINK_RETRIES ?
  parseInt(process.env.SINK_RETRIES) : 5;

const brequest = yieldable(retry(breaker(batch(throttle(request))), retries));
{'fraction_non_alphanumeric': 0.1461356733040837, 'fraction_numerical': 0.0003374957813027337, 'mean_word_length': 5.909090909090909, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6791884', 'n_tokens_mistral': 950, 'n_tokens_neox': 929, 'n_words': 256}