<commit_before>var async = require('../../dist/async');
var fs = require('fs-extra');
var path = require('path');

var $ = require('cheerio');
var _ = require('lodash');

var docsDir = path.join(__dirname, '../../docs');
var pageTitle = 'Methods:';

var docFilename = 'docs.html';
var mainModuleFile = 'module-async.html';
var mainSectionId = '#main';
var mainScrollableSection = '#main-container';
var sectionTitleClass = '.page-title';

var HTMLFileBegin = '<!DOCTYPE html>\n<html lang="en">\n<head>\n';
var HTMLFileHeadBodyJoin = '</head>\n<body>';
var HTMLFileEnd = '</body>';

var additionalFooterText = ' Documentation has been modified from the original. ' +
    ' For more information, please see the <a href="https://github.com/caolan/async">async</a> repository.';

function generateHTMLFile(filename, $page, callback) {
    var methodName = filename.match(/\/(\w+)\.js\.html$/);
    if (methodName) {
        var $thisMethodDocLink = $page.find('#toc').find('a[href="'+docFilename+'#'+methodName[1]+'"]');
        $thisMethodDocLink.parent().addClass('active');
    }

    // generate an HTML file from a cheerio object
    var HTMLdata = HTMLFileBegin + $page.find('head').html()
        + HTMLFileHeadBodyJoin + $page.find('body').html()
        + HTMLFileEnd;

    fs.writeFile(filename, HTMLdata, callback);
}

function extractModuleFiles(files) {
    return _.filter(files, function(file) {
        return _.startsWith(file, 'module') && file !== mainModuleFile;
    });
}

function getSearchableInfo($page, callback) {
    var $sourceLinks = $page.find('a[href$=".js.html"]');
    var sourceFiles = $sourceLinks.map(function() {
        return $(this).attr('href');
    }).toArray().sort();

    var $methodLinks = $page.find('nav').find('a');
    var methods = $methodLinks.map(function() {
        var methodName = $(this).text();
        return (methodName === 'Home' ? null : methodName);
    }).toArray().sort();

    fs.mkdirsSync(path.join(docsDir, 'data'));
    async.parallel([
        function(fileCallback) {
            fs.writeJson(path.join(docsDir, 'data/sourceFiles.json'), sourceFiles, fileCallback);
        },
        function(fileCallback) {
            fs.writeJson(path.join(docsDir, 'data/methodNames.json'), methods, fileCallback);
        }
    ], callback);
}

function combineFakeModules(files, callback) {
    var moduleFiles = extractModuleFiles(files);

    fs.readFile(path.join(docsDir, mainModuleFile), 'utf8', function(err, mainModuleData) {
        if (err) return callback(err);

        var $mainPage = $(mainModuleData);
        // each 'module' (category) has a separate page, with all of the
        // important information in a 'main' div. Combine all of these divs into
        // one on the actual module page (async)
        async.eachSeries(moduleFiles, function(file, fileCallback) {
            fs.readFile(path.join(docsDir, file), 'utf8', function(err, moduleData) {
                if (err) return fileCallback(err);
                var $modulePage = $(moduleData);
                var moduleName = $modulePage.find(sectionTitleClass).text();
                $modulePage.find(sectionTitleClass).attr('id', moduleName.toLowerCase());
                $mainPage.find(mainScrollableSection).append($modulePage.find(mainScrollableSection).html());
                return fileCallback();
            });
        }, function(err) {
            if (err) return callback(err);

            getSearchableInfo($mainPage, function(err) {
                if (err) return callback(err);
                generateHTMLFile(path.join(docsDir, docFilename), $mainPage, callback);
            });
        });
    });
}

function applyPreCheerioFixes(data) {


    var rIncorrectCFText = />ControlFlow</g;
    var fixedCFText = '>Control Flow<';

    var rIncorrectModuleText = />module:(\w+)\.(\w+)</g;

    // the heading needs additional padding at the top so it doesn't get cutoff
    return data
        // for JSDoc to work, the module needs to be labelled 'ControlFlow', while
        // on the page it should appear as 'Control Flow'
        .replace(rIncorrectCFText, fixedCFText)
        // for return types, JSDoc doesn't allow replacing the link text, so it
        // needs to be done here
        .replace(rIncorrectModuleText, function(match, moduleName, methodName) {
            return '>'+methodName+'<';
        });
}


function scrollSpyFix($page, $nav) {
    // move everything into one big ul (for Bootstrap scroll-spy)
    var $ul = $nav.children('ul');
    $ul.addClass('nav').addClass('methods');
    $ul.find('.methods').each(function() {
        var $methodsList = $(this);
        var $methods = $methodsList.find('[data-type="method"]');
        var $parentLi = $methodsList.parent();

        $methodsList.remove();
        $methods.remove();
        $parentLi.after($methods);
        $parentLi.addClass('toc-header');

    });

    $page.find('[data-type="method"]').each(function() {
        $(this).addClass("toc-method");
    });

    $page.find('[id^="."]').each(function() {
        var $ele = $(this);
        var id = $(this).attr('id');
        $ele.attr('id', id.replace('.', ''));
    });
}

function fixToc(file, $page, moduleFiles) {
    // remove `async` listing from toc
    $page.find('a[href="'+mainModuleFile+'"]').parent().remove();

    // change toc title
    var $nav = $page.find('nav');
    $nav.attr('id', 'toc');
    $nav.children('h3').text(pageTitle);
    $nav.children('h2').remove();

    scrollSpyFix($page, $nav);

    var prependFilename = (file === docFilename) ? '' : docFilename;
    // make everything point to the same 'docs.html' page
    _.each(moduleFiles, function(filename) {
        $page.find('[href^="'+filename+'"]').each(function() {
            var $ele = $(this);
            var href = $ele.attr('href');<commit_msg>docs: using jQuery/cheerio builtins for adding class<commit_after>var async = require('../../dist/async');
var fs = require('fs-extra');
var path = require('path');

var $ = require('cheerio');
var _ = require('lodash');

var docsDir = path.join(__dirname, '../../docs');
var pageTitle = 'Methods:';

var docFilename = 'docs.html';
var mainModuleFile = 'module-async.html';
var mainSectionId = '#main';
var mainScrollableSection = '#main-container';
var sectionTitleClass = '.page-title';

var HTMLFileBegin = '<!DOCTYPE html>\n<html lang="en">\n<head>\n';
var HTMLFileHeadBodyJoin = '</head>\n<body>';
var HTMLFileEnd = '</body>';

var additionalFooterText = ' Documentation has been modified from the original. ' +
    ' For more information, please see the <a href="https://github.com/caolan/async">async</a> repository.';

function generateHTMLFile(filename, $page, callback) {
    var methodName = filename.match(/\/(\w+)\.js\.html$/);
    if (methodName) {
        var $thisMethodDocLink = $page.find('#toc').find('a[href="'+docFilename+'#'+methodName[1]+'"]');
        $thisMethodDocLink.parent().addClass('active');
    }

    // generate an HTML file from a cheerio object
    var HTMLdata = HTMLFileBegin + $page.find('head').html()
        + HTMLFileHeadBodyJoin + $page.find('body').html()
        + HTMLFileEnd;

    fs.writeFile(filename, HTMLdata, callback);
}

function extractModuleFiles(files) {
    return _.filter(files, function(file) {
        return _.startsWith(file, 'module') && file !== mainModuleFile;
    });
}

function getSearchableInfo($page, callback) {
    var $sourceLinks = $page.find('a[href$=".js.html"]');
    var sourceFiles = $sourceLinks.map(function() {
        return $(this).attr('href');
    }).toArray().sort();

    var $methodLinks = $page.find('nav').find('a');
    var methods = $methodLinks.map(function() {
        var methodName = $(this).text();
        return (methodName === 'Home' ? null : methodName);
    }).toArray().sort();

    fs.mkdirsSync(path.join(docsDir, 'data'));
    async.parallel([
        function(fileCallback) {
            fs.writeJson(path.join(docsDir, 'data/sourceFiles.json'), sourceFiles, fileCallback);
        },
        function(fileCallback) {
            fs.writeJson(path.join(docsDir, 'data/methodNames.json'), methods, fileCallback);
        }
    ], callback);
}

function combineFakeModules(files, callback) {
    var moduleFiles = extractModuleFiles(files);

    fs.readFile(path.join(docsDir, mainModuleFile), 'utf8', function(err, mainModuleData) {
        if (err) return callback(err);

        var $mainPage = $(mainModuleData);
        // each 'module' (category) has a separate page, with all of the
        // important information in a 'main' div. Combine all of these divs into
        // one on the actual module page (async)
        async.eachSeries(moduleFiles, function(file, fileCallback) {
            fs.readFile(path.join(docsDir, file), 'utf8', function(err, moduleData) {
                if (err) return fileCallback(err);
                var $modulePage = $(moduleData);
                var moduleName = $modulePage.find(sectionTitleClass).text();
                $modulePage.find(sectionTitleClass).attr('id', moduleName.toLowerCase());
                $mainPage.find(mainScrollableSection).append($modulePage.find(mainScrollableSection).html());
                return fileCallback();
            });
        }, function(err) {
            if (err) return callback(err);

            getSearchableInfo($mainPage, function(err) {
                if (err) return callback(err);
                generateHTMLFile(path.join(docsDir, docFilename), $mainPage, callback);
            });
        });
    });
}

function applyPreCheerioFixes(data) {


    var rIncorrectCFText = />ControlFlow</g;
    var fixedCFText = '>Control Flow<';

    var rIncorrectModuleText = />module:(\w+)\.(\w+)</g;

    // the heading needs additional padding at the top so it doesn't get cutoff
    return data
        // for JSDoc to work, the module needs to be labelled 'ControlFlow', while
        // on the page it should appear as 'Control Flow'
        .replace(rIncorrectCFText, fixedCFText)
        // for return types, JSDoc doesn't allow replacing the link text, so it
        // needs to be done here
        .replace(rIncorrectModuleText, function(match, moduleName, methodName) {
            return '>'+methodName+'<';
        });
}


function scrollSpyFix($page, $nav) {
    // move everything into one big ul (for Bootstrap scroll-spy)
    var $ul = $nav.children('ul');
    $ul.addClass('nav').addClass('methods');
    $ul.find('.methods').each(function() {
        var $methodsList = $(this);
        var $methods = $methodsList.find('[data-type="method"]');
        var $parentLi = $methodsList.parent();

        $methodsList.remove();
        $methods.remove();
        $parentLi.after($methods);
        $parentLi.addClass('toc-header');

    });

    $page.find('[data-type="method"]').addClass("toc-method");

    $page.find('[id^="."]').each(function() {
        var $ele = $(this);
        var id = $(this).attr('id');
        $ele.attr('id', id.replace('.', ''));
    });
}

function fixToc(file, $page, moduleFiles) {
    // remove `async` listing from toc
    $page.find('a[href="'+mainModuleFile+'"]').parent().remove();

    // change toc title
    var $nav = $page.find('nav');
    $nav.attr('id', 'toc');
    $nav.children('h3').text(pageTitle);
    $nav.children('h2').remove();

    scrollSpyFix($page, $nav);

    var prependFilename = (file === docFilename) ? '' : docFilename;
    // make everything point to the same 'docs.html' page
    _.each(moduleFiles, function(filename) {
        $page.find('[href^="'+filename+'"]').each(function() {
            var $ele = $(this);
            var href = $ele.attr('href');
{'fraction_non_alphanumeric': 0.16813553521006247, 'fraction_numerical': 0.0008556515786771626, 'mean_word_length': 3.027567195037905, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2113329', 'n_tokens_mistral': 3504, 'n_tokens_neox': 3433, 'n_words': 784}