<commit_before>'use strict'

const assert = require('assert')

const Scrapers = require('../scrapers')

function makeOnCreatePageMock(results) {
    const callback = (query, phantomObjects, appObjects) => {
        return new Promise(resolve => {
            phantomObjects
                .instancePromise
                .then(instance => instance.exit())
            resolve(results)
        })
    }

    return callback
}

const onCreatePageMockEmpty = makeOnCreatePageMock([])

const onCreatePageMockA = makeOnCreatePageMock([
    ['Abot', ' Whitespace and @spam eggs http://domain.com  '],
    ['Bbotty', 'Should not present in the output'],
    ['Cbot', 'Description will be overriden'],
    ['Dbot', 'Both name and description will be overriden'],
])

const onCreatePageMockB = makeOnCreatePageMock([
    ['Cbot', 'New description'],
    ['dbot', 'New name and description'],
])

const stubFunction = () => undefined

describe('Scrapers', () => {
    const appObjectsMock = {
        config: {
            scrapers: ['scraperA', 'scraperB'],
            message: {
                maxLines: 30,
                descriptionMaxLength: 100,
            },
        },

        logger: {
            debug: stubFunction,
            info: stubFunction,
        },
    }

    describe('#find()', () => {
        it('should merge empty results', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockEmpty,
                scraperB: onCreatePageMockEmpty,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))
                    assert.strictEqual(results.length, 0)
                })
                .then(done)
        })

        it('should override previous descriptions', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasOldDescriptions =
                        results.includes('@cbot — Description will be overriden') ||
                        results.includes('@dbot — Both name and description will be overriden')

                    const hasNewDescriptions =
                        results.includes('@cbot — New description') &&
                        results.includes('@dbot — New name and description')

                    assert.ok(!hasOldDescriptions)
                    assert.ok(hasNewDescriptions)
                })
                .then(done)
        })

        it('should have case-insensitive bot names', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasOldNames = results
                        .filter(line => line.startsWith('@Dbot'))
                        .length > 0

                    const hasNewName = results
                        .filter(line => line.startsWith('@dbot'))
                        .length === 1

                    assert.ok(!hasOldNames)
                    assert.ok(hasNewName)
                })
                .then(done)
        })

        it('should contain only bot names with postfix "bot"', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const nameHasPostfixBot = results
                        .filter(line => /^@.*bot — .*$/.test(line))
                        .length === results.length

                    assert.ok(nameHasPostfixBot)
                })
                .then(done)
        })

        it('should trim description whitespace', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasWhitespace = results
                        .filter(line =>
                            line.startsWith(' ') ||
                            line.endsWith(' ') ||
                            line.startsWith('\t') ||
                            line.endsWith('\t'))
                        .length > 0

                    assert.ok(!hasWhitespace)
                })
                .then(done)
        })

        it('should remove URLs from description', () => {
            assert.ok(false)
        })

        it('should remove @ from description', () => {
            assert.ok(false)
        })

        it('should remove new lines from description', () => {
            assert.ok(false)
        })

        it('should sort by bot names', () => {
            assert.ok(false)
        })<commit_msg>add more tests<commit_after>'use strict'

const assert = require('assert')

const Scrapers = require('../scrapers')

function makeOnCreatePageMock(results) {
    const callback = (query, phantomObjects, appObjects) => {
        return new Promise(resolve => {
            phantomObjects
                .instancePromise
                .then(instance => instance.exit())
            resolve(results)
        })
    }

    return callback
}

const onCreatePageMockEmpty = makeOnCreatePageMock([])

const onCreatePageMockA = makeOnCreatePageMock([
    ['Abot', ' Whitespace and @spam https://eggs.foo\nftp://bar http://domain.com \t '],
    ['Bbotty', 'Should not present in the output'],
    ['Cbot', 'Description will be overriden'],
    ['Dbot', 'Both name and description will be overriden'],
])

const onCreatePageMockB = makeOnCreatePageMock([
    ['Cbot', 'New description'],
    ['dbot', 'New name and description'],
])

const stubFunction = () => undefined

describe('Scrapers', () => {
    const appObjectsMock = {
        config: {
            scrapers: ['scraperA', 'scraperB'],
            message: {
                maxLines: 30,
                descriptionMaxLength: 100,
            },
        },

        logger: {
            debug: stubFunction,
            info: stubFunction,
        },
    }

    describe('#find()', () => {
        it('should merge empty results', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockEmpty,
                scraperB: onCreatePageMockEmpty,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))
                    assert.strictEqual(results.length, 0)
                })
                .then(done)
        })

        it('should override previous descriptions', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasOldDescriptions =
                        results.includes('@cbot — Description will be overriden') ||
                        results.includes('@dbot — Both name and description will be overriden')

                    const hasNewDescriptions =
                        results.includes('@cbot — New description') &&
                        results.includes('@dbot — New name and description')

                    assert.ok(!hasOldDescriptions)
                    assert.ok(hasNewDescriptions)
                })
                .then(done)
        })

        it('should have case-insensitive bot names', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasOldNames = results
                        .filter(line => line.startsWith('@Dbot'))
                        .length > 0

                    const hasNewName = results
                        .filter(line => line.startsWith('@dbot'))
                        .length === 1

                    assert.ok(!hasOldNames)
                    assert.ok(hasNewName)
                })
                .then(done)
        })

        it('should contain only bot names with postfix "bot"', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const nameHasPostfixBot = results
                        .filter(line => /^@.*bot — .*$/.test(line))
                        .length === results.length

                    assert.ok(nameHasPostfixBot)
                })
                .then(done)
        })

        it('should trim description whitespace', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasWhitespace = results
                        .filter(line =>
                            line.startsWith(' ') ||
                            line.endsWith(' ') ||
                            line.startsWith('\t') ||
                            line.endsWith('\t'))
                        .length > 0

                    assert.ok(!hasWhitespace)
                })
                .then(done)
        })

        it('should remove URLs from description', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasUrls = results
                        .filter(line =>
                            line.includes('https://') ||
                            line.includes('http://') ||
                            line.includes('ftp://'))
                        .length > 0

                    assert.ok(!hasUrls)
                })
                .then(done)
        })

        it('should remove @ from description', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasAtChar = results
                        .filter(line => line
                            .slice(1)
                            .includes('@'))
                        .length > 0

                    assert.ok(!hasAtChar)
                })
                .then(done)
        })

        it('should remove new lines from description', done => {
            const createPageCallbacks = {
                scraperA: onCreatePageMockA,
                scraperB: onCreatePageMockB,
            }

            const scrapers = new Scrapers(appObjectsMock, createPageCallbacks)
            scrapers
                .find('query')
                .then(results => {
                    assert.ok(Array.isArray(results))

                    const hasNewLines = results
                        .filter(line => line
                            .includes('\n'))
                        .length > 0

                    assert.ok(!hasNewLines)
                })
                .then(done)
        })

        it('should sort by bot names', () => {
            assert.ok(false)
        })
{'fraction_non_alphanumeric': 0.10020830233596191, 'fraction_numerical': 0.0016366612111292963, 'mean_word_length': 1.2673300725248777, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 64, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4641353', 'n_tokens_mistral': 3397, 'n_tokens_neox': 3171, 'n_words': 666}