<commit_before>#!/usr/bin/env node

/* globals showdown */
/* jshint laxcomma:true, strict:false, -W116 */

var fs = require("fs-extra")
,   pth = require("path")
,   nopt = require("nopt")
,   cwd = process.cwd()
,   jn = pth.join
,   dn = __dirname
,   resDir = jn(dn, "res")
,   rfs = function (path) { return fs.readFileSync(path, "utf8"); }
,   wfs = function (path, content) { fs.writeFileSync(path, content, { encoding: "utf8" }); }
,   rjson = function (path) { return JSON.parse(rfs(path)); }
,   wjson = function (path, obj) { wfs(path, JSON.stringify(obj, null, 2)); }
,   tmpl = rfs(jn(resDir, "template.html"))
,   knownOpts = {
                    input:      String
                ,   output:     String
                ,   help:       Boolean
                ,   spec:       String
                ,   version:    Boolean
                ,   markdown:   Boolean
                ,   description:String
                ,   failures:   Boolean
                }
,   shortHands = {
                    i:      ["--input"]
                ,   o:      ["--output"]
                ,   h:      ["--help"]
                ,   s:      ["--spec"]
                ,   v:      ["--version"]
                ,   m:      ["--markdown"]
                ,   d:      ["--description"]
                ,   f:      ["--failures"]
                }
,   out = {
        ua: []
    ,   results: {}
    }
,   lessThanTwo = []
,   all = []
,   completeFail = []
,   err = function (str) {
        console.error("[ERROR] " + str);
        process.exit(1);
    }
,   esc = function (str) {
        if (str === undefined || str === "ManualCheckNeeded") return "-";
        if (!str) return str;
        return str.replace(/&/g, "&amp;")
                  .replace(/</g, "&lt;")
                  .replace(/>/g, "&gt;")
                  .replace("_constructor", "constructor")
        ;
    }
,   interpolate = function (data) {
        return tmpl.replace(/\{\{(\w+)\}\}/g, function (m, p1) {
            return data[p1] !== undefined ? data[p1] : "@@@ERROR@@@";
        });
    }
,   cells = function (data) {
        var res = "";
        for (var i = 0, n = out.ua.length; i < n; i++) res += "<td class='" + data[out.ua[i]] + "'>" + esc(data[out.ua[i]]) + "</td>";
        return res;
    }
,   reports = []
,   subtestsPerTest = {}
,   consolidated = {}
,   totalSubtests = 0
,   uaPass = {}
,   copyFiles = "analysis.css jquery.min.js sticky-headers.js bootstrap.min.css".split(" ")
,   filter = {}
,   showdown = require("showdown")
,   messages = function (data) {
        var res = "";
        for (var i = 0, n = out.ua.length; i < n; i++) {
            if (data.hasOwnProperty(out.ua[i])) {
              var message = options.markdown ? markdown.makeHtml(data[out.ua[i]]) : esc(data[out.ua[i]]) ;
              res += "<tr class='message'><td class='ua'>"+out.ua[i]+":</td><td> "+message+"</td></tr>\n";
            }
        }
        if (res !== "") {
            res = "<tr class='messages'><td colspan='" + (n+1) + "'><table>"+res+"</table></td></tr>\n";
        }
        return res;
    }
;

showdown.extension('strip', function() {
    return [
    { type: 'output',
        regex: /<p>/,
        replace: ''
    },
    { type: 'output',
        regex: /<\/p>$/,
        replace: ''
    }
    ];
});

var markdown = new showdown.Converter({ extensions: [ 'strip' ] });


// parse the command line options so we know the input directory if any
var parsed = nopt(knownOpts, shortHands)
,   defaults = {} ;

// Read in a defaults file if there is one
if (fs.existsSync(jn(parsed.input || cwd, "wptreport.json"))) defaults = require(jn( parsed.input || cwd, "wptreport.json"));

// now set the options, preferring command line, then options from file, then built-in defaults
var options = {
        input:      parsed.input || cwd
    ,   output:     parsed.output || defaults['output'] || cwd
    ,   help:       parsed.help || false
    ,   version:    parsed.version || false
    ,   spec:       parsed.spec || defaults['spec'] || ""
    ,   failures:   parsed.failures || defaults['failures'] || false
    ,   markdown:   parsed.markdown || defaults['markdown'] || false
    }
,   prefix = options.spec ? options.spec + ": " : ""
;

if (options.help) {
    console.log([
        "wptreport [--input /path/to/dir] [--output /path/to/dir] [--spec SpecName]"
    ,   ""
    ,   "   Generate nice-looking reports of various types based on test run reports coming"
    ,   "   out of Web Platform Tests."
    ,   ""
    ,   "   --input, -i  <directory> that contains all the JSON. JSON files must match the pattern"
    ,   "                \\w{2}\\d{d}\\.json. Defaults to the current directory. This is also where"
    ,   "                the filter.js and wptreport.json are found, if any."
    ,   "   --output*, -o <directory> where the generated reports are stored. Defaults to the current"
    ,   "                directory."
    ,   "   --failures*, -f to include any failure message text"
    ,   "   --markdown*, -m to interpret subtest name as Markdown"<commit_msg>Fixed incorporation of description default and usage message.<commit_after>#!/usr/bin/env node

/* globals showdown */
/* jshint laxcomma:true, strict:false, -W116 */

var fs = require("fs-extra")
,   pth = require("path")
,   nopt = require("nopt")
,   cwd = process.cwd()
,   jn = pth.join
,   dn = __dirname
,   resDir = jn(dn, "res")
,   rfs = function (path) { return fs.readFileSync(path, "utf8"); }
,   wfs = function (path, content) { fs.writeFileSync(path, content, { encoding: "utf8" }); }
,   rjson = function (path) { return JSON.parse(rfs(path)); }
,   wjson = function (path, obj) { wfs(path, JSON.stringify(obj, null, 2)); }
,   tmpl = rfs(jn(resDir, "template.html"))
,   knownOpts = {
                    input:      String
                ,   output:     String
                ,   help:       Boolean
                ,   spec:       String
                ,   version:    Boolean
                ,   markdown:   Boolean
                ,   description:String
                ,   failures:   Boolean
                }
,   shortHands = {
                    i:      ["--input"]
                ,   o:      ["--output"]
                ,   h:      ["--help"]
                ,   s:      ["--spec"]
                ,   v:      ["--version"]
                ,   m:      ["--markdown"]
                ,   d:      ["--description"]
                ,   f:      ["--failures"]
                }
,   out = {
        ua: []
    ,   results: {}
    }
,   lessThanTwo = []
,   all = []
,   completeFail = []
,   err = function (str) {
        console.error("[ERROR] " + str);
        process.exit(1);
    }
,   esc = function (str) {
        if (str === undefined || str === "ManualCheckNeeded") return "-";
        if (!str) return str;
        return str.replace(/&/g, "&amp;")
                  .replace(/</g, "&lt;")
                  .replace(/>/g, "&gt;")
                  .replace("_constructor", "constructor")
        ;
    }
,   interpolate = function (data) {
        return tmpl.replace(/\{\{(\w+)\}\}/g, function (m, p1) {
            return data[p1] !== undefined ? data[p1] : "@@@ERROR@@@";
        });
    }
,   cells = function (data) {
        var res = "";
        for (var i = 0, n = out.ua.length; i < n; i++) res += "<td class='" + data[out.ua[i]] + "'>" + esc(data[out.ua[i]]) + "</td>";
        return res;
    }
,   reports = []
,   subtestsPerTest = {}
,   consolidated = {}
,   totalSubtests = 0
,   uaPass = {}
,   copyFiles = "analysis.css jquery.min.js sticky-headers.js bootstrap.min.css".split(" ")
,   filter = {}
,   showdown = require("showdown")
,   messages = function (data) {
        var res = "";
        for (var i = 0, n = out.ua.length; i < n; i++) {
            if (data.hasOwnProperty(out.ua[i])) {
              var message = options.markdown ? markdown.makeHtml(data[out.ua[i]]) : esc(data[out.ua[i]]) ;
              res += "<tr class='message'><td class='ua'>"+out.ua[i]+":</td><td> "+message+"</td></tr>\n";
            }
        }
        if (res !== "") {
            res = "<tr class='messages'><td colspan='" + (n+1) + "'><table>"+res+"</table></td></tr>\n";
        }
        return res;
    }
;

showdown.extension('strip', function() {
    return [
    { type: 'output',
        regex: /<p>/,
        replace: ''
    },
    { type: 'output',
        regex: /<\/p>$/,
        replace: ''
    }
    ];
});

var markdown = new showdown.Converter({ extensions: [ 'strip' ] });


// parse the command line options so we know the input directory if any
var parsed = nopt(knownOpts, shortHands)
,   defaults = {} ;

// Read in a defaults file if there is one
if (fs.existsSync(jn(parsed.input || cwd, "wptreport.json"))) defaults = require(jn( parsed.input || cwd, "wptreport.json"));

// now set the options, preferring command line, then options from file, then built-in defaults
var options = {
        input:      parsed.input || cwd
    ,   output:     parsed.output || defaults['output'] || cwd
    ,   help:       parsed.help || false
    ,   version:    parsed.version || false
    ,   spec:       parsed.spec || defaults['spec'] || ""
    ,   failures:   parsed.failures || defaults['failures'] || false
    ,   markdown:   parsed.markdown || defaults['markdown'] || false
    ,   description:parsed.description || defaults['description'] || ""
    }
,   prefix = options.spec ? options.spec + ": " : ""
;

if (options.help) {
    console.log([
        "wptreport [--input /path/to/dir] [--output /path/to/dir] [--spec SpecName]"
    ,   ""
    ,   "   Generate nice-looking reports of various types based on test run reports coming"
    ,   "   out of Web Platform Tests."
    ,   ""
    ,   "   --input, -i  <directory> that contains all the JSON. JSON files must match the pattern"
    ,   "                \\w{2}\\d{d}\\.json. Defaults to the current directory. This is also where"
    ,   "                the filter.js and wptreport.json are found, if any."
    ,   "   --output*, -o <directory> where the generated reports are stored. Defaults to the current"
    ,   "                directory."
    ,   "   --failures*, -f to include any failure message text"
    ,   "   --markdown*, -m to interpret subtest name as Markdown"
    ,   "   --description*, -d description file to use to annotation the report."
{'fraction_non_alphanumeric': 0.17528458421763457, 'fraction_numerical': 0.002894076789504148, 'mean_word_length': 1.8064428803465078, 'pattern_counts': {'":': 4, '<': 45, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5839332', 'n_tokens_mistral': 3352, 'n_tokens_neox': 3278, 'n_words': 804}