<commit_before>const tester = require("tester")
    , Lien = require("lien")
    , scrapeIt = require("..")
    ;

const PORT = 9000
    , HOST = "http://localhost"
    , URL = HOST + ":" + PORT
    ;

tester.describe("scrape-it", t => {
    t.it("start the server", cb => {
        new Lien({
            port: 9000
          , public: `${__dirname}/public`
        }).on("load", cb);
    });
    t.it("scrape simple data", cb => {
        scrapeIt(URL, {
            title: "h1.title"
          , description: ".description"
          , date: {
                selector: ".date"
              , convert: x => new Date(x)
            }
        }, (err, { data }) => {
            t.expect(err).toBe(null);
            t.expect(data).toEqual({
                title: "Title"
              , description: "Lorem ipsum"
              , date: new Date("1988-01-01")
            });
            cb();
        });
    });
    t.it("scrape lists", cb => {
        scrapeIt(URL, {
            features: {
                listItem: ".features > li"
            }
        }, (err, { data }) => {
            t.expect(err).toBe(null);
            t.expect(data).toEqual({
                features: [
                    "1"
                  , "2"
                  , "3"
                  , "4"
                  , "5"
                  , "6"
                ]
            });
            cb();
        });
    });
    t.it("promise version", cb => {
        scrapeIt(URL, {
            features: {
                listItem: ".features > li"
            }
        }).then(({ data }) => {
            t.expect(data).toEqual({
                features: [
                    "1"
                  , "2"
                  , "3"
                  , "4"
                  , "5"
                  , "6"
                ]<commit_msg>Add unit test for converting lists<commit_after>const tester = require("tester")
    , Lien = require("lien")
    , scrapeIt = require("..")
    ;

const PORT = 9000
    , HOST = "http://localhost"
    , URL = HOST + ":" + PORT
    ;

tester.describe("scrape-it", t => {
    t.it("start the server", cb => {
        new Lien({
            port: 9000
          , public: `${__dirname}/public`
        }).on("load", cb);
    });
    t.it("scrape simple data", cb => {
        scrapeIt(URL, {
            title: "h1.title"
          , description: ".description"
          , date: {
                selector: ".date"
              , convert: x => new Date(x)
            }
        }, (err, { data }) => {
            t.expect(err).toBe(null);
            t.expect(data).toEqual({
                title: "Title"
              , description: "Lorem ipsum"
              , date: new Date("1988-01-01")
            });
            cb();
        });
    });
    t.it("scrape lists", cb => {
        scrapeIt(URL, {
            features: {
                listItem: ".features > li"
            }
        }, (err, { data }) => {
            t.expect(err).toBe(null);
            t.expect(data).toEqual({
                features: [
                    "1"
                  , "2"
                  , "3"
                  , "4"
                  , "5"
                  , "6"
                ]
            });
            cb();
        });
    });
    t.it("scrape and convert lists", cb => {
        scrapeIt(URL, {
            features: {
                listItem: ".features > li",
                convert: x => parseInt(x, 10)
            }
        }, (err, { data }) => {
            t.expect(err).toBe(null);
            t.expect(data).toEqual({
                features: [1, 2, 3, 4, 5, 6]
            });
            cb();
        });
    });
    t.it("promise version", cb => {
        scrapeIt(URL, {
            features: {
                listItem: ".features > li"
            }
        }).then(({ data }) => {
            t.expect(data).toEqual({
                features: [
                    "1"
                  , "2"
                  , "3"
                  , "4"
                  , "5"
                  , "6"
                ]
{'fraction_non_alphanumeric': 0.161409561360276, 'fraction_numerical': 0.016264169541646133, 'mean_word_length': 0.9319371727748691, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1721484', 'n_tokens_mistral': 1239, 'n_tokens_neox': 1159, 'n_words': 223}