<commit_before>var crypto = require('crypto')
  , url = require('url')
  , _ = require('underscore')
  , isUrl = require('is-url')
  , Ignore = require("fstream-ignore")
  , semver = require('semver')
  , glob = require('glob')
  , minimatch = require('minimatch')
  , querystring = require('querystring')
  , cookie = require('cookie')
  , EventEmitter = require('events').EventEmitter
  , util = require('util')
  , request = require('request')
  , path = require('path')
  , mime = require('mime')
  , rimraf = require('rimraf')
  , mkdirp = require('mkdirp')
  , async = require('async')
  , fs = require('fs')
  , zlib = require('zlib')
  , tar = require('tar')
  , once = require('once')
  , concat = require('concat-stream')
  , jsonld = require('jsonld')
  , clone = require('clone')
  , publish = require('./lib/publish')
  , pubmed = require('./plugin/pubmed').pubmed
  , oapmc = require('./plugin/oapmc')
  , binaryCSV = require('binary-csv')
  , split = require('split')
  , temp = require('temp')
  , githubUrl = require('github-url')
  , previewTabularData = require('preview-tabular-data').preview
  , jsonldContextInfer = require('jsonld-context-infer')
  , targz = require('tar.gz');

var conf = require('rc')('ldpm', {protocol: 'https', port: 443, hostname: 'registry.standardanalytics.io', strictSSL: false, sha:true});

mime.define({
  'application/ld+json': ['jsonld'],
  'application/x-ldjson': ['ldjson', 'ldj'],
  'application/x-gzip': ['gz', 'gzip', 'tgz'] //tar.gz won't work
});
<commit_msg>rm tar.gz deps<commit_after>var crypto = require('crypto')
  , url = require('url')
  , _ = require('underscore')
  , isUrl = require('is-url')
  , Ignore = require("fstream-ignore")
  , semver = require('semver')
  , glob = require('glob')
  , minimatch = require('minimatch')
  , querystring = require('querystring')
  , cookie = require('cookie')
  , EventEmitter = require('events').EventEmitter
  , util = require('util')
  , request = require('request')
  , path = require('path')
  , mime = require('mime')
  , rimraf = require('rimraf')
  , mkdirp = require('mkdirp')
  , async = require('async')
  , fs = require('fs')
  , zlib = require('zlib')
  , tar = require('tar')
  , once = require('once')
  , concat = require('concat-stream')
  , jsonld = require('jsonld')
  , clone = require('clone')
  , publish = require('./lib/publish')
  , pubmed = require('./plugin/pubmed').pubmed
  , oapmc = require('./plugin/oapmc')
  , binaryCSV = require('binary-csv')
  , split = require('split')
  , temp = require('temp')
  , githubUrl = require('github-url')
  , previewTabularData = require('preview-tabular-data').preview
  , jsonldContextInfer = require('jsonld-context-infer');

var conf = require('rc')('ldpm', {protocol: 'https', port: 443, hostname: 'registry.standardanalytics.io', strictSSL: false, sha:true});

mime.define({
  'application/ld+json': ['jsonld'],
  'application/x-ldjson': ['ldjson', 'ldj'],
  'application/x-gzip': ['gz', 'gzip', 'tgz'] //tar.gz won't work
});

{'fraction_non_alphanumeric': 0.20485533754572663, 'fraction_numerical': 0.0019953441968739607, 'mean_word_length': 4.479052823315119, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '150353', 'n_tokens_mistral': 1029, 'n_tokens_neox': 1008, 'n_words': 190}