<commit_before>var os = require('os')
var fs = require('fs')
var path = require('path')
var test = require('tape')
var spawn = require('tape-spawn')
var helpers = require('./helpers')

var tmp = os.tmpdir()
var dat = path.resolve(__dirname + '/../cli.js')
var dat1 = path.join(tmp, 'dat-1')

helpers.onedat(dat1)

test('dat write', function (t) {
  var st = spawn(t, "echo 'hello world' | " + dat + ' write test-file.txt -', {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
})

test('dat cat after write', function (t) {
  datCatEquals(t, 'test-file.txt', /hello world/)
})

test('dat write to dataset', function (t) {
  var st = spawn(t, "echo 'hello world' | " + dat + ' write -d my-dataset test-file.txt -', {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
})

test('dat cat after write to dataset', function (t) {
  datCatEquals(t, 'test-file.txt', /hello world/, '-d my-dataset')
})

test('dat write to new dataset', function (t) {
  var st = spawn(t, "echo 'goodbye world' | " + dat + ' write -d my-dataset-2 test-file.txt -', {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
})

test('dat cat after write to dataset 2', function (t) {
  datCatEquals(t, 'test-file.txt', /goodbye world/, '-d my-dataset-2')
})

test('dat overwrite to dataset 2', function (t) {
  var st = spawn(t, "echo 'goodbye mars' | " + dat + ' write -d my-dataset-2 test-file.txt -', {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
})

test('dat cat after overwrite to dataset 2', function (t) {
  datCatEquals(t, 'test-file.txt', /goodbye mars/,  '-d my-dataset-2')
})

/** from file **/

var blobPath = path.resolve(__dirname + '/fixtures/blob.txt')

test('dat write from file', function (t) {
  datWrite(t, blobPath, '-d my-dataset-2')
})

test('dat cat after write from file', function (t) {
  contents = fs.readFileSync(blobPath).toString()
  datCatEquals(t, blobPath, contents, '-d my-dataset-2')
})

test('dat write from file with new name', function (t) {
  datWrite(t, blobPath, '-d my-dataset-2 --name=new-name.txt')
})

test('dat cat after write from file with new name', function (t) {
  contents = fs.readFileSync(blobPath).toString()
  datCatEquals(t, 'new-name.txt', contents, '-d my-dataset-2')
})

test('dat write from file with new name with abbr', function (t) {
  datWrite(t, blobPath, '-d my-dataset-2 -n new-name-abbr.txt')
})

test('dat cat after write from file with new name with abbr', function (t) {
  contents = fs.readFileSync(blobPath).toString()
  datCatEquals(t, 'new-name-abbr.txt', contents, '-d my-dataset-2')
})

function datWrite(t, blobPath, ext) {
  var cmd =  ' write '  + blobPath
  if (ext) {
    cmd = cmd + ' ' + ext
  }
  var st = spawn(t, dat + cmd , {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
}

function datCatEquals(t, key, contents, ext) {
  var cmd = ' cat ' + key
  if (ext) {
    cmd = cmd + ' ' + ext
  }
  var st = spawn(t, dat + cmd, {cwd: dat1})
  st.stderr.empty()
  st.stdout.match(contents)
  st.end()
}<commit_msg>Standard style<commit_after>var os = require('os')
var fs = require('fs')
var path = require('path')
var test = require('tape')
var spawn = require('tape-spawn')
var helpers = require('./helpers')

var tmp = os.tmpdir()
var dat = path.resolve(__dirname + '/../cli.js')
var dat1 = path.join(tmp, 'dat-1')

helpers.onedat(dat1)

test('dat write', function (t) {
  var st = spawn(t, "echo 'hello world' | " + dat + ' write test-file.txt -', {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
})

test('dat cat after write', function (t) {
  datCatEquals(t, 'test-file.txt', /hello world/)
})

test('dat write to dataset', function (t) {
  var st = spawn(t, "echo 'hello world' | " + dat + ' write -d my-dataset test-file.txt -', {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
})

test('dat cat after write to dataset', function (t) {
  datCatEquals(t, 'test-file.txt', /hello world/, '-d my-dataset')
})

test('dat write to new dataset', function (t) {
  var st = spawn(t, "echo 'goodbye world' | " + dat + ' write -d my-dataset-2 test-file.txt -', {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
})

test('dat cat after write to dataset 2', function (t) {
  datCatEquals(t, 'test-file.txt', /goodbye world/, '-d my-dataset-2')
})

test('dat overwrite to dataset 2', function (t) {
  var st = spawn(t, "echo 'goodbye mars' | " + dat + ' write -d my-dataset-2 test-file.txt -', {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
})

test('dat cat after overwrite to dataset 2', function (t) {
  datCatEquals(t, 'test-file.txt', /goodbye mars/, '-d my-dataset-2')
})

/** from file **/

var blobPath = path.resolve(__dirname + '/fixtures/blob.txt')

test('dat write from file', function (t) {
  datWrite(t, blobPath, '-d my-dataset-2')
})

test('dat cat after write from file', function (t) {
  var contents = fs.readFileSync(blobPath).toString()
  datCatEquals(t, blobPath, contents, '-d my-dataset-2')
})

test('dat write from file with new name', function (t) {
  datWrite(t, blobPath, '-d my-dataset-2 --name=new-name.txt')
})

test('dat cat after write from file with new name', function (t) {
  var contents = fs.readFileSync(blobPath).toString()
  datCatEquals(t, 'new-name.txt', contents, '-d my-dataset-2')
})

test('dat write from file with new name with abbr', function (t) {
  datWrite(t, blobPath, '-d my-dataset-2 -n new-name-abbr.txt')
})

test('dat cat after write from file with new name with abbr', function (t) {
  var contents = fs.readFileSync(blobPath).toString()
  datCatEquals(t, 'new-name-abbr.txt', contents, '-d my-dataset-2')
})

function datWrite (t, blobPath, ext) {
  var cmd = ' write ' + blobPath
  if (ext) {
    cmd = cmd + ' ' + ext
  }
  var st = spawn(t, dat + cmd, {cwd: dat1})
  st.stdout.empty()
  st.stderr.match(/Done writing binary data/)
  st.end()
}

function datCatEquals (t, key, contents, ext) {
  var cmd = ' cat ' + key
  if (ext) {
    cmd = cmd + ' ' + ext
  }
  var st = spawn(t, dat + cmd, {cwd: dat1})
  st.stderr.empty()
  st.stdout.match(contents)
  st.end()
}
{'fraction_non_alphanumeric': 0.1826210376912159, 'fraction_numerical': 0.006938968616937391, 'mean_word_length': 3.572458543619322, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5674337', 'n_tokens_mistral': 2310, 'n_tokens_neox': 2302, 'n_words': 670}