<commit_before>#!/usr/bin/env node

function http_request(url, callback) {

    function parse_dom(input) {
        require("jsdom").env(input, {features: false}, callback)
    }

    var http_proxy = process.env["http_proxy"]
    if (http_proxy) {
        (function () {
            require('request')(
                {
                    url: url,
                    proxy: http_proxy,
                },
                function(error, response, html) {
                    if (!error && response.statusCode == 200) {
                        parse_dom(html)
                    }
                })
         })()
    } else {
        parse_dom(url)
    }
}

function getopt(args) {
    var set = {}

    function is_option(str) {
        if (str.length === 2 && str[0] === '-') {
            return true
        } else {
            return false
        }
    }

    (function filter_option(index) {
        var arg
        for (; index < args.length; index++) {
            arg = args[index]
            if (is_option(arg)) {
                if (index + 1 < args.length && !is_option(args[index+1])) {
                   set[arg] = args[++index]
                } else {
                   set[arg] = null
                }
            }
        }
    })(0)

    return set
}

function print_index(index) {
    console.log("title:\t", index.title)
    console.log("author:\t", index.author)
    console.log("cover:\t", index.cover)
    console.log("==== Table of Contecnts ====")
    index.toc.forEach(function (item) {
        console.log(item.name, " => ", item.url)
    })
}

function read_index(url, further_operation) {
    http_request(url,
        function (errors, window) {
            function extractNovelInfo(parentNode) {
                var rc = {}
                var pat = /^ (.+):(.+)$/
                for (var c = parentNode.firstChild; c !== null; c = c.nextSibling) {
                    if (c.nodeType === window.Node.TEXT_NODE) {
                        var tmp = pat.exec(c.data)
                        if (tmp !== null) {
                            var key = tmp[1]
                            var value = tmp[2]
                            switch (key) {
                                case "作者":
                                    rc.author = value; break;
                                case "状态":
                                    rc.status = value; break;
                                case "简介":
                                    rc.brief = value; break;
                            }
                        }
                    }
                }
                return rc
            }

            function get_toc(toc) {
                var rc = []
                var list = toc.getElementsByClassName("zl")
                for (var i = 0; i < list.length; i++) {
                    var li = list[i]
                    var a = li.firstChild
                    var text = a.firstChild.data
                    var link = a.href
                    rc.push({name: text, url: link})
                }
                return rc
            }

            var document = window.document
            var tit = document.getElementsByClassName("tit")[0]
            var header = extractNovelInfo(tit.parentNode)
            var index = {
                "title": tit.firstChild.firstChild.data,
                "cover": tit.nextSibling.src,
                "author": header.author,
                "toc": get_toc(document.getElementsByClassName("tit")[1]
                    .parentNode.children[1])
            }
            window.close();
            further_operation(index)
       })
}

function save_chapter(html, destdir, filename) {
    var fs = require("fs")
    if (destdir[destdir.length - 1] !== '/') {
        destdir = destdir + '/'
    }
    fs.writeFileSync(destdir + filename, html)
}

function fetch_chapter(args) {
    var item = args.item
    var dest = args.dest
    var chain_func = args.chain_func
    http_request(
        item.url,
        function (errors, window) {
            function removeNode(node) {
                node.parentNode.removeChild(node)
            }
            function removeNodes(nodes) {
                var to_delete = []
                for (var i = 0; i < nodes.length; i++) {
                    to_delete.push(nodes[i])
                }
                for (var i = 0; i < to_delete.length; i++) {
                    removeNode(to_delete[i])
                }
            }

            var document = window.document
            var content = document.getElementById("content")
            removeNodes(content.getElementsByTagName("script"))
            removeNodes(content.getElementsByClassName("bad"))
            removeNode(content.lastChild)
            removeNode(content.lastChild)
            removeNode(content.lastChild)
            removeNode(content.lastChild)

            var html = [
        '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"',
        '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">',
        '<html xmlns="http://www.w3.org/1999/xhtml">',
        '<head>',
        '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">',
        '<title>',
        item.name,
        '</title></head>',
        '</head>',
        '<body>',
        content.innerHTML,
        '</body></html>'].join("")
            if (!dest) {
                console.log(html)
            } else {
                (function() {
                    var filename
                    var url = decodeURI(item.url)
                    var buffer = []

                    if (url[url.length-1] == '/') {
                        url = url.slice(0, url.length - 1)
                    }
                    for (var i = url.length - 1; i >= 0; i--) {
                        if (url[i] !== '/') {
                            buffer.push(url[i])
                        } else {
                            break
                        }
                    }
                    chain_func(html, buffer.reverse().join(""), args)
                })();
            }
        })
}

function download_index(index, dir, start) {
    var count = -1
    var toc = index.toc
    var length = toc.length
    start && (count += (start - 1))

    var args = {
        generator : function() {
           return toc[++count]
        },
        dest: dir,
        chain_func : function(html, filename, args) {
            save_chapter(html, args.dest, filename)
            fetch_next()
        }
    }

    function fetch_next() {
        args.item = args.generator()
        if (args.item) {
            console.log("fetch ", count+1, '/', length, " : ", args.item.name)
            fetch_chapter(args)
        }
    }

    fetch_next()
}

(function main() {
   var options = getopt(process.argv)<commit_msg>fix fetching rules for [cover url] and [author]<commit_after>#!/usr/bin/env node

function http_request(url, callback) {

    function parse_dom(input) {
        require("jsdom").env(input,  {features: false} , callback)
    }

    var http_proxy = process.env["http_proxy"]
    if (http_proxy) {
        (function () {
            require('request')(
                {
                    url: url,
                    proxy: http_proxy,
                },
                function(error, response, html) {
                    if (!error && response.statusCode == 200) {
                        parse_dom(html)
                    }
                })
         })()
    } else {
        parse_dom(url)
    }
}

function getopt(args) {
    var set = {}

    function is_option(str) {
        if (str.length === 2 && str[0] === '-') {
            return true
        } else {
            return false
        }
    }

    (function filter_option(index) {
        var arg
        for (; index < args.length; index++) {
            arg = args[index]
            if (is_option(arg)) {
                if (index + 1 < args.length && !is_option(args[index+1])) {
                   set[arg] = args[++index]
                } else {
                   set[arg] = null
                }
            }
        }
    })(0)

    return set
}

function print_index(index) {
    console.log("title:\t", index.title)
    console.log("author:\t", index.author)
    console.log("cover:\t", index.cover)
    console.log("==== Table of Contecnts ====")
    index.toc.forEach(function (item) {
        console.log(item.name, " => ", item.url)
    })
}

function read_index(url, further_operation) {
    http_request(url,
        function (errors, window) {
            function extractNovelInfo(parentNode) {
                var rc = {}
                var pat = /^\s*(.+):(.+)$/
                for (var c = parentNode.firstChild; c !== null; c = c.nextSibling) {
                    if (c.nodeType === window.Node.TEXT_NODE) {
                        var tmp = pat.exec(c.data)
                        if (tmp !== null) {
                            var key = tmp[1]
                            var value = tmp[2]
                            switch (key) {
                                case "作者":
                                    rc.author = value; break;
                                case "状态":
                                    rc.status = value; break;
                                case "简介":
                                    rc.brief = value; break;
                            }
                        }
                    }
                }
                return rc
            }

            function get_toc(toc) {
                var rc = []
                var list = toc.getElementsByClassName("zl")
                for (var i = 0; i < list.length; i++) {
                    var li = list[i]
                    var a = li.firstChild
                    var text = a.firstChild.data
                    var link = a.href
                    rc.push({name: text, url: link})
                }
                return rc
            }

            var document = window.document
            var tit = document.getElementsByClassName("tit")[0]
            var header = extractNovelInfo(tit.parentNode)
            var index = {
                "title": tit.firstChild.firstChild.data,
                "cover": (function(brother) { // search for img
                    for (var x = brother.nextSibling; x; x = x.nextSibling) {
                        if (x.nodeType === window.Node.ELEMENT_NODE
                            && x.tagName === 'IMG')
                            return x
                    }
                })(tit).src,
                "author": header.author,
                "toc": get_toc(document.getElementsByClassName("tit")[1]
                    .parentNode.children[1])
            }
            window.close();
            further_operation(index)
       })
}

function save_chapter(html, destdir, filename) {
    var fs = require("fs")
    if (destdir[destdir.length - 1] !== '/') {
        destdir = destdir + '/'
    }
    fs.writeFileSync(destdir + filename, html)
}

function fetch_chapter(args) {
    var item = args.item
    var dest = args.dest
    var chain_func = args.chain_func
    http_request(
        item.url,
        function (errors, window) {
            function removeNode(node) {
                node.parentNode.removeChild(node)
            }
            function removeNodes(nodes) {
                var to_delete = []
                for (var i = 0; i < nodes.length; i++) {
                    to_delete.push(nodes[i])
                }
                for (var i = 0; i < to_delete.length; i++) {
                    removeNode(to_delete[i])
                }
            }

            var document = window.document
            var content = document.getElementById("content")
            removeNodes(content.getElementsByTagName("script"))
            removeNodes(content.getElementsByClassName("bad"))
            removeNode(content.lastChild)
            removeNode(content.lastChild)
            removeNode(content.lastChild)
            removeNode(content.lastChild)

            var html = [
        '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"',
        '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">',
        '<html xmlns="http://www.w3.org/1999/xhtml">',
        '<head>',
        '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">',
        '<title>',
        item.name,
        '</title></head>',
        '</head>',
        '<body>',
        content.innerHTML,
        '</body></html>'].join("")
            if (!dest) {
                console.log(html)
            } else {
                (function() {
                    var filename
                    var url = decodeURI(item.url)
                    var buffer = []

                    if (url[url.length-1] == '/') {
                        url = url.slice(0, url.length - 1)
                    }
                    for (var i = url.length - 1; i >= 0; i--) {
                        if (url[i] !== '/') {
                            buffer.push(url[i])
                        } else {
                            break
                        }
                    }
                    chain_func(html, buffer.reverse().join(""), args)
                })();
            }
        })
}

function download_index(index, dir, start) {
    var count = -1
    var toc = index.toc
    var length = toc.length
    start && (count += (start - 1))

    console.log("write index.json")
    require("fs").writeFileSync(dir + "/index.json", JSON.stringify(index))

    var args = {
        generator : function() {
           return toc[++count]
        },
        dest: dir,
        chain_func : function(html, filename, args) {
            save_chapter(html, args.dest, filename)
            fetch_next()
        }
    }

    function fetch_next() {
        args.item = args.generator()
        if (args.item) {
            console.log("fetch ", count+1, '/', length, " : ", args.item.name)
            fetch_chapter(args)
        }
    }

    fetch_next()
}

(function main() {
   var options = getopt(process.argv)
{'fraction_non_alphanumeric': 0.12635225906809022, 'fraction_numerical': 0.005232270381107262, 'mean_word_length': 1.1709900230237913, 'pattern_counts': {'":': 14, '<': 35, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7596881', 'n_tokens_mistral': 3991, 'n_tokens_neox': 3775, 'n_words': 772}