<commit_before>#!/usr/bin/env node

var fs = require("fs-extra");

var SOURCE_DIR = "src/",
    LANGUAGE_EMOJI_MAPPER = {
        "af": "🇿🇦",
        "ar": "🇸🇦",
        "bg": "🇧🇬",
        "cs": "🇨🇿",
        "da": "🇩🇰",
        "de": "🇩🇪",
        "el": "🇬🇷",
        "en": "🇺🇸",
        "es": "🇪🇸",
        "et": "🇪🇪",
        "fa": "🇮🇷",
        "fj": "🇫🇯",
        "fr": "🇫🇷",
        "it": "🇮🇹",
        "lt": "🇱🇹",
        "he": "🇮🇱",
        "hr": "🇭🇷",
        "hu": "🇭🇺",
        "in": "🇮🇩",
        "ka": "🇬🇪",
        "ko": "🇰🇷",
        "lo": "🇱🇦",
        "mk": "🇲🇰",
        "mn": "🇲🇳",
        "ms": "🇲🇾",
        "no": "🇳🇴",
        "ne": "🇳🇵",
        "ja": "🇯🇵",
        "pl": "🇵🇱",
        "pt": "🇵🇹",
        "ro": "🇷🇴",
        "ru": "🇷🇺",
        "sk": "🇸🇰",
        "sl": "🇸🇮",
        "sr": "🇷🇸",
        "st": "🇿🇦",
        "sv": "🇸🇪",
        "sw": "🇰🇪",
        "ta": "🇮🇳",
        "tl": "🇵🇭",
        "th": "🇹🇭",
        "tr": "🇹🇷",
        "uk": "🇺🇦",
        "vi": "🇻🇳",
        "zh": "🇨🇳"
    },
    SOURCE_EXTENSION = "md",
    SOURCE_EXTENSION_BIBLE = "bible",
    MD_COMPLETE_THRESHOLD = 300;

var getQuarterlyPrefix = function(d) {
    d = d || new Date();
    var quarterIndex = (Math.ceil((d.getMonth()+1)/3));
    return d.getFullYear() + "-0" + quarterIndex;
};

var l = {
    "en": {
        "2017-03": {
            "01": {
                "complete": "complete", // incomplete
                "01": {
                    "text": "complete",
                    "bible": "complete"
                }
            }
        }
    }
};

/**
 *
 * @param lessonDir
 * @param language
 * @param quarterly
 * @param week
 */
var scanAndReturnIfWeekIsComplete = function(weeksDir, week){
    // scan all md files
    // compare if amount of MD that are completed (size is more than X) is more or less of MD that are incomplete
    // also check if Bible is not there but marked as completed

    var days = fs.readdirSync(weeksDir),
        ret = {};

    var _complete = 0, _incomplete = 0;

    for (var _day = 0; _day < days.length; _day++){
        var extension = days[_day].split(".").pop();
        if (extension !== SOURCE_EXTENSION) continue;

        var sourceSize = Buffer.byteLength(fs.readFileSync(weeksDir + "/" + days[_day]), 'utf8'),
            sourceBible = false;

        try {
            fs.lstatSync(weeksDir + "/" + days[_day] + "." + SOURCE_EXTENSION_BIBLE);
            sourceBible = true;
        } catch (err) {}

        ret[days[_day]] = {
            "text": sourceSize > MD_COMPLETE_THRESHOLD,
            "bible": sourceBible
        };

        var _ = (sourceSize > MD_COMPLETE_THRESHOLD) ? _complete++ : _incomplete++;
    }

    ret["complete"] = _complete > _incomplete;

    return ret;
};

var summaryMatrixToShortMd = function(summaryMatrix){
    var lessons = [];
    var output = "#### Summary\n\n";
    output += "##### " + getQuarterlyPrefix() + "\n\n";


    // Taking English to build header
    output += "|  |";
    var _separator = "\n| ------------ |";

    for (var lesson in summaryMatrix["en"][getQuarterlyPrefix()]){
        lessons.push(lesson);
    }

    lessons.sort(function(x, y){
        return parseInt(x) - parseInt(y);
    });

    for (var i = 0; i < lessons.length; i++){
        output += " " + lessons[i] + " |";
        _separator += " ------------ |";
    }

    output += _separator + "\n";
<commit_msg>Updated summary.js<commit_after>#!/usr/bin/env node

var fs = require("fs-extra");

var SOURCE_DIR = "src/",
    LANGUAGE_EMOJI_MAPPER = {
        "af": "🇿🇦",
        "ar": "🇸🇦",
        "bg": "🇧🇬",
        "cs": "🇨🇿",
        "da": "🇩🇰",
        "de": "🇩🇪",
        "el": "🇬🇷",
        "en": "🇺🇸",
        "es": "🇪🇸",
        "et": "🇪🇪",
        "fa": "🇮🇷",
        "fj": "🇫🇯",
        "fr": "🇫🇷",
        "it": "🇮🇹",
        "lt": "🇱🇹",
        "he": "🇮🇱",
        "hr": "🇭🇷",
        "hu": "🇭🇺",
        "in": "🇮🇩",
        "ka": "🇬🇪",
        "ko": "🇰🇷",
        "lo": "🇱🇦",
        "mk": "🇲🇰",
        "mn": "🇲🇳",
        "ms": "🇲🇾",
        "no": "🇳🇴",
        "ne": "🇳🇵",
        "ja": "🇯🇵",
        "pl": "🇵🇱",
        "pt": "🇵🇹",
        "ro": "🇷🇴",
        "ru": "🇷🇺",
        "sk": "🇸🇰",
        "sl": "🇸🇮",
        "sr": "🇷🇸",
        "st": "🇿🇦",
        "sv": "🇸🇪",
        "sw": "🇰🇪",
        "ta": "🇮🇳",
        "tl": "🇵🇭",
        "th": "🇹🇭",
        "tr": "🇹🇷",
        "uk": "🇺🇦",
        "vi": "🇻🇳",
        "zh": "🇨🇳"
    },
    SOURCE_EXTENSION = "md",
    SOURCE_EXTENSION_BIBLE = "bible",
    MD_COMPLETE_THRESHOLD = 300;

var getQuarterlyPrefix = function(d) {
    d = d || new Date();
    var quarterIndex = (Math.ceil((d.getMonth()+1)/3));
    return d.getFullYear() + "-0" + quarterIndex;
};

var l = {
    "en": {
        "2017-03": {
            "01": {
                "complete": "complete", // incomplete
                "01": {
                    "text": "complete",
                    "bible": "complete"
                }
            }
        }
    }
};

/**
 *
 * @param lessonDir
 * @param language
 * @param quarterly
 * @param week
 */
var scanAndReturnIfWeekIsComplete = function(weeksDir, week){
    // scan all md files
    // compare if amount of MD that are completed (size is more than X) is more or less of MD that are incomplete
    // also check if Bible is not there but marked as completed

    var days = fs.readdirSync(weeksDir),
        ret = {};

    var _complete = 0, _incomplete = 0;

    for (var _day = 0; _day < days.length; _day++){
        var extension = days[_day].split(".").pop();
        if (extension !== SOURCE_EXTENSION) continue;

        var sourceSize = Buffer.byteLength(fs.readFileSync(weeksDir + "/" + days[_day]), 'utf8'),
            sourceBible = false;

        try {
            fs.lstatSync(weeksDir + "/" + days[_day] + "." + SOURCE_EXTENSION_BIBLE);
            sourceBible = true;
        } catch (err) {}

        ret[days[_day]] = {
            "text": sourceSize > MD_COMPLETE_THRESHOLD,
            "bible": sourceBible
        };

        var _ = (sourceSize > MD_COMPLETE_THRESHOLD) ? _complete++ : _incomplete++;
    }

    ret["complete"] = _complete > _incomplete;

    return ret;
};

var summaryMatrixToShortMd = function(summaryMatrix){
    var lessons = [];
    var output = "#### Current summary of lessons for " + getQuarterlyPrefix() + "\n\n";
    output += "##### " + getQuarterlyPrefix() + "\n\n";


    // Taking English to build header
    output += "|  |";
    var _separator = "\n| ------------ |";

    for (var lesson in summaryMatrix["en"][getQuarterlyPrefix()]){
        lessons.push(lesson);
    }

    lessons.sort(function(x, y){
        return parseInt(x) - parseInt(y);
    });

    for (var i = 0; i < lessons.length; i++){
        output += " " + lessons[i] + " |";
        _separator += " ------------ |";
    }

    output += _separator + "\n";

{'fraction_non_alphanumeric': 0.22372437854339294, 'fraction_numerical': 0.006105538595726123, 'mean_word_length': 1.7022780832678712, 'pattern_counts': {'":': 108, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1004012', 'n_tokens_mistral': 2983, 'n_tokens_neox': 2778, 'n_words': 565}