<commit_before>const sinon = require('sinon')
const chai = require('chai')
const { expect } = chai
const SandboxedModule = require('sandboxed-module')
const Errors = require('../../../app/js/Errors')

chai.use(require('sinon-chai'))
chai.use(require('chai-as-promised'))

const modulePath = '../../../app/js/FSPersistorManager.js'

describe('FSPersistorManagerTests', function() {
  const stat = { size: 4, isFile: sinon.stub().returns(true) }
  const fd = 1234
  const readStream = 'readStream'
  const writeStream = 'writeStream'
  const remoteStream = 'remoteStream'
  const tempFile = '/tmp/potato.txt'
  const location = '/foo'
  const error = new Error('guru meditation error')

  const files = ['animals/wombat.tex', 'vegetables/potato.tex']
  const filteredFilenames = ['animals_wombat.tex', 'vegetables_potato.tex']
  let fs, rimraf, stream, LocalFileWriter, FSPersistorManager, glob

  beforeEach(function() {
    fs = {
      createReadStream: sinon.stub().returns(readStream),
      createWriteStream: sinon.stub().returns(writeStream),
      unlink: sinon.stub().yields(),
      open: sinon.stub().yields(null, fd),
      stat: sinon.stub().yields(null, stat)
    }
    glob = sinon.stub().yields(null, files)
    rimraf = sinon.stub().yields()
    stream = { pipeline: sinon.stub().yields() }
    LocalFileWriter = {<commit_msg>Return full path in stubbed 'glob' method<commit_after>const sinon = require('sinon')
const chai = require('chai')
const { expect } = chai
const SandboxedModule = require('sandboxed-module')
const Errors = require('../../../app/js/Errors')

chai.use(require('sinon-chai'))
chai.use(require('chai-as-promised'))

const modulePath = '../../../app/js/FSPersistorManager.js'

describe('FSPersistorManagerTests', function() {
  const stat = { size: 4, isFile: sinon.stub().returns(true) }
  const fd = 1234
  const readStream = 'readStream'
  const writeStream = 'writeStream'
  const remoteStream = 'remoteStream'
  const tempFile = '/tmp/potato.txt'
  const location = '/foo'
  const error = new Error('guru meditation error')

  const files = ['animals/wombat.tex', 'vegetables/potato.tex']
  const globs = [`${location}/${files[0]}`, `${location}/${files[1]}`]
  const filteredFilenames = ['animals_wombat.tex', 'vegetables_potato.tex']
  let fs, rimraf, stream, LocalFileWriter, FSPersistorManager, glob

  beforeEach(function() {
    fs = {
      createReadStream: sinon.stub().returns(readStream),
      createWriteStream: sinon.stub().returns(writeStream),
      unlink: sinon.stub().yields(),
      open: sinon.stub().yields(null, fd),
      stat: sinon.stub().yields(null, stat)
    }
    glob = sinon.stub().yields(null, globs)
    rimraf = sinon.stub().yields()
    stream = { pipeline: sinon.stub().yields() }
    LocalFileWriter = {
{'fraction_non_alphanumeric': 0.1652892561983471, 'fraction_numerical': 0.004311893639956881, 'mean_word_length': 4.174721189591078, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4612751', 'n_tokens_mistral': 959, 'n_tokens_neox': 941, 'n_words': 190}