<commit_before>﻿// Imports.
var fs = require('fs'),
    xml2js = require('xml2js'),
    _ = require('lodash'),
    Promise = require('promise'),
    glob = require('glob'),
    upath = require('upath'),
    extend = require('extend'),
    minimatch = require('minimatch');

// Initialize variables.
var nrOfCheckedFiles = 0,
    nrOfIncludedFiles = 0,
    nrOfPhysicalFiles = 0,
    nrOfTargetedFiles = 0,
    nrOfNotIncludedFiles = 0,
    notIncludedFiles = [],
    maxFilesToShowDefault = 20,
    options = {},
    basePath;

// Polyfills for ES6 functions.
if (!String.prototype.startsWith) {
    String.prototype.startsWith = function(prefix) {
        return this.indexOf(prefix) === 0;
    }
}

// Source: https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
if (!String.prototype.endsWith) {
    String.prototype.endsWith = function (searchString, position) {
        var subjectString = this.toString();
        if (position === undefined || position > subjectString.length) {
            position = subjectString.length;
        }
        position -= searchString.length;
        var lastIndex = subjectString.indexOf(searchString, position);
        return lastIndex !== -1 && lastIndex === position;
    };
}

/**
 * Print a debug message to the console (only if debug flag is set).
 * @param {string} The message to print.
 */
function debug(message) {
    if (options.verbose) {
        console.log(message);
    }
}

/**
 * Return the path of a given filename.
 * @param {string} The full path to a file, for instance '../MyApp.Web/App.Web.csproj'.
 * @returns {string} Only the path to the file excluding the filename, for instance 'App.Web.csproj'
 */
function getBasePath(pathAndFile) {
    if (!pathAndFile) {
        throw new Error('getBasePath: Supplied path is empty!');
    }
    // Remove a potential end slash from path to prevent errors 
    if (pathAndFile.endsWith('/')) {
        pathAndFile.substring(0, pathAndFile.length - 1)
    }
    var index = pathAndFile.lastIndexOf('/');
    return index === -1 ? "" : pathAndFile.substring(0, index + 1);
}


/**
 * @param {string} csProjFilename 
 * @returns {string} All files from the given cs project file.
 */
function readCsProjeFile(csProjFilename) {
    var fileIncludes = [];
    var parser = new xml2js.Parser();
    return new Promise(function(resolve, reject) {
        fs.readFile(csProjFilename, function (err, data) {
            if (err) {
                reject('File \'' + csProjFilename + '\' not found.');
                return;
            }
            parser.parseString(data, function (err, result) {
                var itemgroups = result.Project.ItemGroup;
                if (!itemgroups || itemgroups.length === 0) {
                    reject('No item groups found in ' + csProjFilename);
                }
                _.each(result.Project.ItemGroup, function(itemGroup, groupKey, groupList) {
                    if ((!itemGroup.Content || itemGroup.Content.length === 0)) {             
                        debug('Warning: No <Content> tags found in <ItemGroup> nr ' + groupKey + '.');
                        // When we are at the last group  iteration then filling the 'fileIncludes' is done, so 'return' it.
                        if (groupKey === groupList.length - 1) {
                            resolve(fileIncludes);
                        }
                    } else {
                        debug('Scanning ' + itemGroup.Content.length + ' <Content> tags found in <ItemGroup> nr ' + groupKey + '.');
                    }
                    _.each(itemGroup.Content, function(content, contentKey, contentList) {
                        // 6. Filter uit de fileContents alle <Content Include="x"> uit alle <ItemGroups> en sla elk van deze 'x'-en op in een array 'includedFiles'
                        if (content && content.$ && content.$.Include /* && content.$.Include.endsWith('.js') */) {
                            fileIncludes.push(content.$.Include);
                        }
                        // When we are at the last item of the last group in the (nested) iteration.
                        if (contentKey === contentList.length - 1 && groupKey === groupList.length - 1) {
                            // Then filling the 'fileIncludes' is done, so 'return' it.
                            resolve(fileIncludes);
                        }
                    });
                });
            });
        });
    });
}<commit_msg>Support linked files<commit_after>﻿// Imports.
var fs = require('fs'),
    xml2js = require('xml2js'),
    _ = require('lodash'),
    Promise = require('promise'),
    glob = require('glob'),
    upath = require('upath'),
    extend = require('extend'),
    minimatch = require('minimatch');

// Initialize variables.
var nrOfCheckedFiles = 0,
    nrOfIncludedFiles = 0,
    nrOfPhysicalFiles = 0,
    nrOfTargetedFiles = 0,
    nrOfNotIncludedFiles = 0,
    notIncludedFiles = [],
    maxFilesToShowDefault = 20,
    options = {},
    basePath;

// Polyfills for ES6 functions.
if (!String.prototype.startsWith) {
    String.prototype.startsWith = function(prefix) {
        return this.indexOf(prefix) === 0;
    }
}

// Source: https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith
if (!String.prototype.endsWith) {
    String.prototype.endsWith = function (searchString, position) {
        var subjectString = this.toString();
        if (position === undefined || position > subjectString.length) {
            position = subjectString.length;
        }
        position -= searchString.length;
        var lastIndex = subjectString.indexOf(searchString, position);
        return lastIndex !== -1 && lastIndex === position;
    };
}

/**
 * Print a debug message to the console (only if debug flag is set).
 * @param {string} The message to print.
 */
function debug(message) {
    if (options.verbose) {
        console.log(message);
    }
}

/**
 * Return the path of a given filename.
 * @param {string} The full path to a file, for instance '../MyApp.Web/App.Web.csproj'.
 * @returns {string} Only the path to the file excluding the filename, for instance 'App.Web.csproj'
 */
function getBasePath(pathAndFile) {
    if (!pathAndFile) {
        throw new Error('getBasePath: Supplied path is empty!');
    }
    // Remove a potential end slash from path to prevent errors 
    if (pathAndFile.endsWith('/')) {
        pathAndFile.substring(0, pathAndFile.length - 1)
    }
    var index = pathAndFile.lastIndexOf('/');
    return index === -1 ? "" : pathAndFile.substring(0, index + 1);
}


/**
 * @param {string} csProjFilename 
 * @returns {string} All files from the given cs project file.
 */
function readCsProjeFile(csProjFilename) {
    var fileIncludes = [];
    var parser = new xml2js.Parser();
    return new Promise(function(resolve, reject) {
        fs.readFile(csProjFilename, function (err, data) {
            if (err) {
                reject('File \'' + csProjFilename + '\' not found.');
                return;
            }
            parser.parseString(data, function (err, result) {
                var itemgroups = result.Project.ItemGroup;
                if (!itemgroups || itemgroups.length === 0) {
                    reject('No item groups found in ' + csProjFilename);
                }
                _.each(result.Project.ItemGroup, function(itemGroup, groupKey, groupList) {
                    if ((!itemGroup.Content || itemGroup.Content.length === 0)) {             
                        debug('Warning: No <Content> tags found in <ItemGroup> nr ' + groupKey + '.');
                        // When we are at the last group  iteration then filling the 'fileIncludes' is done, so 'return' it.
                        if (groupKey === groupList.length - 1) {
                            resolve(fileIncludes);
                        }
                    } else {
                        debug('Scanning ' + itemGroup.Content.length + ' <Content> tags found in <ItemGroup> nr ' + groupKey + '.');
                    }
                    _.each(itemGroup.Content, function(content, contentKey, contentList) {
                        // 6. Filter uit de fileContents alle <Content Include="x"> uit alle <ItemGroups> en sla elk van deze 'x'-en op in een array 'includedFiles'
                         if (content && content.Link) {
                            fileIncludes.push(content.Link[0]);
                        } else
                        if (content && content.$ && content.$.Include /* && content.$.Include.endsWith('.js') */) {
                            fileIncludes.push(content.$.Include);
                        }
                        // When we are at the last item of the last group in the (nested) iteration.
                        if (contentKey === contentList.length - 1 && groupKey === groupList.length - 1) {
                            // Then filling the 'fileIncludes' is done, so 'return' it.
                            resolve(fileIncludes);
                        }
                    });
                });
            });
        });
    });
}
{'fraction_non_alphanumeric': 0.12366876765920452, 'fraction_numerical': 0.005324929363181917, 'mean_word_length': 1.929958611907036, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2568761', 'n_tokens_mistral': 2536, 'n_tokens_neox': 2413, 'n_words': 673}