<commit_before>fs = require 'fs'
path = require 'path'
wrench = require 'wrench'
PathScanner = require '../src/path-scanner'

describe "PathScanner", ->
  rootPath = null
  paths = null

  createPathCollector = ->
    paths = []
    pathHandler = jasmine.createSpy()
    pathHandler.andCallFake (p) ->
      paths.push(p)
    pathHandler

  describe "a non-git directory with many files", ->
    beforeEach ->
      rootPath = fs.realpathSync("spec/fixtures/many-files")

    it 'lists all non-hidden files', ->
      scanner = new PathScanner(rootPath)
      scanner.on('path-found', pathHandler = createPathCollector())
      scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

      runs ->
        scanner.scan()

      waitsFor ->
        pathHandler.callCount > 0

      waitsFor ->
        finishedHandler.callCount > 0

      runs ->
        # symlink-to-file1.txt is a file on windows
        if process.platform is 'win32'
          expect(paths.length).toBe 16
        else
          expect(paths.length).toBe 15

        expect(paths).toContain path.join(rootPath, 'file1.txt')
        expect(paths).toContain path.join(rootPath, 'dir', 'file7_ignorable.rb')

    describe "including file paths", ->
      it "lists only paths specified by file pattern", ->
        scanner = new PathScanner(rootPath, inclusions: ['*.js'])
        scanner.on('path-found', pathHandler = createPathCollector())
        scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

        runs ->
          scanner.scan()

        waitsFor ->
          finishedHandler.callCount > 0

        runs ->
          expect(paths.length).toBe 4
          expect(paths).toContain path.join(rootPath, 'newdir', 'deep_dir.js')
          expect(paths).toContain path.join(rootPath, 'sample.js')

      it "explicit inclusions override exclusions", ->
        scanner = new PathScanner(rootPath, inclusions: ['dir'], exclusions: ['dir'])
        scanner.on('path-found', pathHandler = createPathCollector())
        scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

        runs ->
          scanner.scan()

        waitsFor ->
          finishedHandler.callCount > 0

        runs ->
          expect(paths).toContain path.join(rootPath, 'dir', 'file7_ignorable.rb')

      it "lists only paths specified by a deep dir", ->
        scanner = new PathScanner(rootPath, inclusions: [path.join('.root', 'subdir')+path.sep], includeHidden: true)
        scanner.on('path-found', pathHandler = createPathCollector())
        scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

        runs ->
          scanner.scan()

        waitsFor ->
          finishedHandler.callCount > 0

        runs ->
          expect(paths).toContain path.join(rootPath, '.root', 'subdir', 'file1.txt')
          expect(paths).not.toContain path.join(rootPath, '.root', 'file3.txt')

      dirs = ['dir', "dir#{path.sep}", "dir#{path.sep}*", "dir#{path.sep}**"]
      for dir in dirs
        ((dir) ->
          it "lists only paths specified in #{dir}", ->
            scanner = new PathScanner(rootPath, inclusions: [dir])
            scanner.on('path-found', pathHandler = createPathCollector())
            scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

            runs ->
              scanner.scan()

            waitsFor ->
              finishedHandler.callCount > 0

            runs ->
              expect(paths.length).toBe 1
              expect(paths).toContain path.join(rootPath, 'dir', 'file7_ignorable.rb')
        )(dir)

  describe "with a git repo", ->
    beforeEach ->
      rootPath = fs.realpathSync("spec/fixtures/git")
      wrench.copyDirSyncRecursive(path.join(rootPath, 'git.git'), path.join(rootPath, '.git'))
      wrench.rmdirSyncRecursive(path.join(rootPath, 'git.git'))
      fs.writeFileSync(path.join(rootPath, 'ignored.txt'), "This must be added in the spec because the file can't be checked in!")

    afterEach ->
      wrench.copyDirSyncRecursive(path.join(rootPath, '.git'), path.join(rootPath, 'git.git'))
      wrench.rmdirSyncRecursive(path.join(rootPath, '.git'))

    it "excludes files specified with .gitignore", ->
      scanner = new PathScanner(rootPath, excludeVcsIgnores: true)
      scanner.on('path-found', pathHandler = createPathCollector())
      scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

      runs ->
        scanner.scan()

      waitsFor ->
        finishedHandler.callCount > 0

      runs ->
        expect(paths.length).toBe 2
        expect(paths).not.toContain path.join(rootPath, 'ignored.txt')

    it "includes files matching .gitignore patterns when excludeVcsIgnores == false", ->
      scanner = new PathScanner(rootPath, excludeVcsIgnores: false)
      scanner.on('path-found', pathHandler = createPathCollector())
      scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())
<commit_msg>Add a spec for negation<commit_after>fs = require 'fs'
path = require 'path'
wrench = require 'wrench'
PathScanner = require '../src/path-scanner'

describe "PathScanner", ->
  rootPath = null
  paths = null

  createPathCollector = ->
    paths = []
    pathHandler = jasmine.createSpy()
    pathHandler.andCallFake (p) ->
      paths.push(p)
    pathHandler

  describe "a non-git directory with many files", ->
    beforeEach ->
      rootPath = fs.realpathSync("spec/fixtures/many-files")

    it 'lists all non-hidden files', ->
      scanner = new PathScanner(rootPath)
      scanner.on('path-found', pathHandler = createPathCollector())
      scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

      runs ->
        scanner.scan()

      waitsFor ->
        pathHandler.callCount > 0

      waitsFor ->
        finishedHandler.callCount > 0

      runs ->
        # symlink-to-file1.txt is a file on windows
        if process.platform is 'win32'
          expect(paths.length).toBe 16
        else
          expect(paths.length).toBe 15

        expect(paths).toContain path.join(rootPath, 'file1.txt')
        expect(paths).toContain path.join(rootPath, 'dir', 'file7_ignorable.rb')

    describe "including file paths", ->
      it "lists only paths specified by file pattern", ->
        scanner = new PathScanner(rootPath, inclusions: ['*.js'])
        scanner.on('path-found', pathHandler = createPathCollector())
        scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

        runs ->
          scanner.scan()

        waitsFor ->
          finishedHandler.callCount > 0

        runs ->
          expect(paths.length).toBe 4
          expect(paths).toContain path.join(rootPath, 'newdir', 'deep_dir.js')
          expect(paths).toContain path.join(rootPath, 'sample.js')

      it "explicit inclusions override exclusions", ->
        scanner = new PathScanner(rootPath, inclusions: ['dir'], exclusions: ['dir'])
        scanner.on('path-found', pathHandler = createPathCollector())
        scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

        runs ->
          scanner.scan()

        waitsFor ->
          finishedHandler.callCount > 0

        runs ->
          expect(paths).toContain path.join(rootPath, 'dir', 'file7_ignorable.rb')

      it "lists only paths specified by a deep dir", ->
        scanner = new PathScanner(rootPath, inclusions: [path.join('.root', 'subdir')+path.sep], includeHidden: true)
        scanner.on('path-found', pathHandler = createPathCollector())
        scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

        runs ->
          scanner.scan()

        waitsFor ->
          finishedHandler.callCount > 0

        runs ->
          expect(paths).toContain path.join(rootPath, '.root', 'subdir', 'file1.txt')
          expect(paths).not.toContain path.join(rootPath, '.root', 'file3.txt')

      dirs = ['dir', "dir#{path.sep}", "dir#{path.sep}*", "dir#{path.sep}**"]
      for dir in dirs
        ((dir) ->
          it "lists only paths specified in #{dir}", ->
            scanner = new PathScanner(rootPath, inclusions: [dir])
            scanner.on('path-found', pathHandler = createPathCollector())
            scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

            runs ->
              scanner.scan()

            waitsFor ->
              finishedHandler.callCount > 0

            runs ->
              expect(paths.length).toBe 1
              expect(paths).toContain path.join(rootPath, 'dir', 'file7_ignorable.rb')
        )(dir)

    describe "excluding file paths with the not operator", ->
      it "lists only paths specified by file pattern", ->
        scanner = new PathScanner(rootPath, inclusions: ['!*.js'])
        scanner.on('path-found', pathHandler = createPathCollector())
        scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

        runs ->
          scanner.scan()

        waitsFor ->
          finishedHandler.callCount > 0

        runs ->
          expect(paths).not.toContain path.join(rootPath, 'newdir', 'deep_dir.js')
          expect(paths).not.toContain path.join(rootPath, 'sample.js')
          expect(paths).toContain path.join(rootPath, 'sample.txt')

  describe "with a git repo", ->
    beforeEach ->
      rootPath = fs.realpathSync("spec/fixtures/git")
      wrench.copyDirSyncRecursive(path.join(rootPath, 'git.git'), path.join(rootPath, '.git'))
      wrench.rmdirSyncRecursive(path.join(rootPath, 'git.git'))
      fs.writeFileSync(path.join(rootPath, 'ignored.txt'), "This must be added in the spec because the file can't be checked in!")

    afterEach ->
      wrench.copyDirSyncRecursive(path.join(rootPath, '.git'), path.join(rootPath, 'git.git'))
      wrench.rmdirSyncRecursive(path.join(rootPath, '.git'))

    it "excludes files specified with .gitignore", ->
      scanner = new PathScanner(rootPath, excludeVcsIgnores: true)
      scanner.on('path-found', pathHandler = createPathCollector())
      scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

      runs ->
        scanner.scan()

      waitsFor ->
        finishedHandler.callCount > 0

      runs ->
        expect(paths.length).toBe 2
        expect(paths).not.toContain path.join(rootPath, 'ignored.txt')

    it "includes files matching .gitignore patterns when excludeVcsIgnores == false", ->
      scanner = new PathScanner(rootPath, excludeVcsIgnores: false)
      scanner.on('path-found', pathHandler = createPathCollector())
      scanner.on('finished-scanning', finishedHandler = jasmine.createSpy())

{'fraction_non_alphanumeric': 0.13268669366230343, 'fraction_numerical': 0.004426028816272719, 'mean_word_length': 2.650739085596425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4096349', 'n_tokens_mistral': 3247, 'n_tokens_neox': 3134, 'n_words': 669}