<commit_before>'use strict';

var fs = require('fs');
var path = require('path');
var dir = require('node-dir');
var q = require('q');
var cli = require('commander');

var getLicenseType = function (info) {
    if (info.licenses) {
        if (typeof info.licenses[0] === 'object') {
            return info.licenses[0].type;
        }

        return info.licenses[0];
    }

    if (info.license) {
        if (typeof info.license === 'object') {
            return info.license.type;
        }

        return info.license;
    }

    return '';
};

var inferFromLicense = function (licenseText) {
    var lines = licenseText.split('\n');

    if (/MIT/g.test(lines[0])) {
        return 'MIT';
    }

    if (/Apache/gi.test(lines[0])) {
        return 'Apache, v2';
    }

    return 'UNKNOWN';
};

var findLicenses = function (uniqueOnly, includeLicenseText) {
    var deferred = q.defer();

    var modulesDir = path.join(process.cwd(), 'node_modules');

    dir.files(modulesDir, function (err, files) {
        var licenses = [];

        files.forEach(function (file) {
            if (/license/gi.test(file)) {
                var currPath = file.split(path.sep);
                var currModule = currPath[currPath.lastIndexOf('node_modules') + 1];

                var moduleInfo = {};

                try {
                    moduleInfo = require(path.join(path.dirname(file), 'package.json'));
                } catch (e) {
                    console.log('No package.json found for ' + currModule + '.');
                }

                var licenseText = fs.readFileSync(file, 'utf8');

                if (uniqueOnly) {
                    var isUnique = licenses.filter(function (license) {
                        return license.module === currModule && license.version === moduleInfo.version;
                    }).length === 0;

                    if (!isUnique) {
                        return;
                    }
                }

                licenses.push({
                    'module': currModule,
                    'version': moduleInfo.version,
                    'license': getLicenseType(moduleInfo) || inferFromLicense(licenseText),
                    'license_file': file,
                    'content': includeLicenseText ? licenseText : ''
                });
            }
        });

        deferred.resolve(licenses);
    });

    return deferred.promise;
};

var createReadable = function (licenses) {
    var text = '';

    text += 'Total Licenses: ' + licenses.length + '\n';
    text += '=========================' + '\n\n';

    licenses.forEach(function (license) {
        text += 'Module: ' + license.module + '\n';
        text += 'Version: ' + license.version + '\n';
        text += 'License: ' + license.license + '\n';

        if (license.content !== '') {
            text += 'License Text:\n' + license.content + '\n';
        }

        text += '\n';
    });

    return text;
};

cli
    .version('0.0.1', '-v, --version')
    .option('-t, --text', 'Output readable text')
    .option('-l, --license-text', 'Include license file contents')
    .option('-u, --unique', 'Only include unique dependencies')
    .parse(process.argv);

if (cli.text) {
    findLicenses(cli.unique, cli.licenseText).then(function (data) {
        fs.writeFile(path.join(process.cwd(), 'licensure.txt'), createReadable(data), function (err) {
            if (err) {
                throw err;
            }

            console.log('Finished processing ' + data.length + ' licenses.');
            console.log('License information store in licensure.txt.');
        });
    });
} else {
    findLicenses(cli.unique, cli.licenseText).then(function (data) {
        fs.writeFile(path.join(process.cwd(), 'licensure.json'), JSON.stringify(data, null, 4), function (err) {
            if (err) {
                throw err;
            }

            console.log('Finished processing ' + data.length + ' licenses.');
            console.log('License information stored in licensure.json.');
        });
    });
}
<commit_msg>Add more license inference rules<commit_after>'use strict';

var fs = require('fs');
var path = require('path');
var dir = require('node-dir');
var q = require('q');
var cli = require('commander');

var getLicenseType = function (info) {
    if (info.licenses) {
        if (typeof info.licenses[0] === 'object') {
            return info.licenses[0].type;
        }

        return info.licenses[0];
    }

    if (info.license) {
        if (typeof info.license === 'object') {
            return info.license.type;
        }

        return info.license;
    }

    return '';
};

var inferFromLicense = function (licenseText) {
    var lines = licenseText.split('\n');

    if (/MIT/g.test(lines[0])) {
        return 'MIT';
    }

    if (/Apache/gi.test(lines[0])) {
        return 'Apache, v2';
    }

    if (/BSD/g.test(lines[0])) {
        return 'BSD';
    }

    if (/Permission is hereby granted, free of charge,/g.test(licenseText)) {
        return 'MIT';
    }

    if (/Redistribution and use in source and binary forms,/g.test(licenseText)) {
        return 'BSD';
    }

    return 'UNKNOWN';
};

var findLicenses = function (uniqueOnly, includeLicenseText) {
    var deferred = q.defer();

    var modulesDir = path.join(process.cwd(), 'node_modules');

    dir.files(modulesDir, function (err, files) {
        var licenses = [];

        files.forEach(function (file) {
            if (/license/gi.test(file)) {
                var currPath = file.split(path.sep);
                var currModule = currPath[currPath.lastIndexOf('node_modules') + 1];

                var moduleInfo = {};

                try {
                    moduleInfo = require(path.join(path.dirname(file), 'package.json'));
                } catch (e) {
                    console.log('No package.json found for ' + currModule + '.');
                }

                var licenseText = fs.readFileSync(file, 'utf8');

                if (uniqueOnly) {
                    var isUnique = licenses.filter(function (license) {
                        return license.module === currModule && license.version === moduleInfo.version;
                    }).length === 0;

                    if (!isUnique) {
                        return;
                    }
                }

                licenses.push({
                    'module': currModule,
                    'version': moduleInfo.version,
                    'license': getLicenseType(moduleInfo) || inferFromLicense(licenseText),
                    'license_file': file,
                    'content': includeLicenseText ? licenseText : ''
                });
            }
        });

        deferred.resolve(licenses);
    });

    return deferred.promise;
};

var createReadable = function (licenses) {
    var text = '';

    text += 'Total Licenses: ' + licenses.length + '\n';
    text += '=========================' + '\n\n';

    licenses.forEach(function (license) {
        text += 'Module: ' + license.module + '\n';
        text += 'Version: ' + license.version + '\n';
        text += 'License: ' + license.license + '\n';

        if (license.content !== '') {
            text += 'License Text:\n' + license.content + '\n';
        }

        text += '\n';
    });

    return text;
};

cli
    .version('0.0.1', '-v, --version')
    .option('-t, --text', 'Output readable text')
    .option('-l, --license-text', 'Include license file contents')
    .option('-u, --unique', 'Only include unique dependencies')
    .parse(process.argv);

if (cli.text) {
    findLicenses(cli.unique, cli.licenseText).then(function (data) {
        fs.writeFile(path.join(process.cwd(), 'licensure.txt'), createReadable(data), function (err) {
            if (err) {
                throw err;
            }

            console.log('Finished processing ' + data.length + ' licenses.');
            console.log('License information store in licensure.txt.');
        });
    });
} else {
    findLicenses(cli.unique, cli.licenseText).then(function (data) {
        fs.writeFile(path.join(process.cwd(), 'licensure.json'), JSON.stringify(data, null, 4), function (err) {
            if (err) {
                throw err;
            }

            console.log('Finished processing ' + data.length + ' licenses.');
            console.log('License information stored in licensure.json.');
        });
    });
}

{'fraction_non_alphanumeric': 0.15782618991238456, 'fraction_numerical': 0.00319677954061094, 'mean_word_length': 1.8231951871657754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3256812', 'n_tokens_mistral': 2443, 'n_tokens_neox': 2334, 'n_words': 519}