<commit_before>
/*
Copyright (c) 2013, Yahoo! Inc. All rights reserved.
Code licensed under the BSD License:
http://yuilibrary.com/license/
*/

var UNKNOWN = 'UNKNOWN';
var data = {};
var fs = require('fs');
var path = require('path');
var read = require('read-installed');
var treeify = require('treeify');
var license = require('./license');

var flatten = function(json) {
    var moduleInfo = {licenses: UNKNOWN},
        licenseData, files;

    data[json.name + '@' + json.version] = moduleInfo;

    if (json.repository) {
        if (typeof json.repository === 'object' && typeof json.repository.url === 'string') {
            moduleInfo.repository = json.repository.url.replace('git://github.com', 'https://github.com').replace('.git', '');
            moduleInfo.repository = json.repository.url.replace('<EMAIL>:', 'https://github.com/').replace('.git', '');
        }
    }
    if (json.url) {
        if (typeof json.url === 'object') {
            moduleInfo.url = json.url.web;
        }
    }

    licenseData = json.license || json.licenses || undefined;
    if (licenseData) {
        if (Array.isArray(licenseData) && licenseData.length > 0) {
            moduleInfo.licenses = licenseData.map(function(license){
                if (typeof license === 'object') {
                    return license.type;
                } else if (typeof license === 'string') {
                    return license;
                }
            });
        } else if (typeof licenseData === 'object' && licenseData.type) {
            moduleInfo.licenses = licenseData.type;
        } else if (typeof licenseData === 'string') {
            moduleInfo.licenses = licenseData;
        }
    } else if (license(json.readme)) {
        moduleInfo.licenses = license(json.readme);
    } else {
        files = fs.readdirSync(json.path).filter(function(filename) {
            filename = filename.toUpperCase();
            return filename.indexOf('LICENSE') > -1 || filename.indexOf('LICENCE') > -1 ;
        });

        files.forEach(function(filename) {
            moduleInfo.licenses = license(fs.readFileSync(path.join(json.path, filename), {encoding: 'utf8'}));
        });
    }

    if (Array.isArray(moduleInfo.licenses)) {
        if (moduleInfo.licenses.length === 1) {
            moduleInfo.licenses = moduleInfo.licenses[0];
        }
    }

    if (json.dependencies) {
        Object.keys(json.dependencies).forEach(function(name) {
            var childDependency = json.dependencies[name],
                dependencyId = childDependency.name + '@' + childDependency.version;
            if (data[dependencyId]) { // already exists
                return;
            }
            flatten(childDependency);
        });
    }
};

exports.init = function(options, callback) {

    console.log('scanning' , options.start);

    read(options.start, function(err, json) {
        flatten(json);
        var sorted = {};
        Object.keys(data).sort().forEach(function(item) {<commit_msg>Added checking that the license file is a normal file, because otherwise an appropriately named directory crashes the license-checker through EISDIR, illegal operation on a directory.<commit_after>
/*
Copyright (c) 2013, Yahoo! Inc. All rights reserved.
Code licensed under the BSD License:
http://yuilibrary.com/license/
*/

var UNKNOWN = 'UNKNOWN';
var data = {};
var fs = require('fs');
var path = require('path');
var read = require('read-installed');
var treeify = require('treeify');
var license = require('./license');

var flatten = function(json) {
    var moduleInfo = {licenses: UNKNOWN},
        licenseData, files, licenseFile;

    data[json.name + '@' + json.version] = moduleInfo;

    if (json.repository) {
        if (typeof json.repository === 'object' && typeof json.repository.url === 'string') {
            moduleInfo.repository = json.repository.url.replace('git://github.com', 'https://github.com').replace('.git', '');
            moduleInfo.repository = json.repository.url.replace('git@github.com:', 'https://github.com/').replace('.git', '');
        }
    }
    if (json.url) {
        if (typeof json.url === 'object') {
            moduleInfo.url = json.url.web;
        }
    }

    licenseData = json.license || json.licenses || undefined;
    if (licenseData) {
        if (Array.isArray(licenseData) && licenseData.length > 0) {
            moduleInfo.licenses = licenseData.map(function(license){
                if (typeof license === 'object') {
                    return license.type;
                } else if (typeof license === 'string') {
                    return license;
                }
            });
        } else if (typeof licenseData === 'object' && licenseData.type) {
            moduleInfo.licenses = licenseData.type;
        } else if (typeof licenseData === 'string') {
            moduleInfo.licenses = licenseData;
        }
    } else if (license(json.readme)) {
        moduleInfo.licenses = license(json.readme);
    } else {
        files = fs.readdirSync(json.path).filter(function(filename) {
            filename = filename.toUpperCase();
            return filename.indexOf('LICENSE') > -1 || filename.indexOf('LICENCE') > -1 ;
        });

        files.forEach(function(filename) {
            licenseFile = path.join(json.path, filename);
            // Checking that the file is in fact a normal file and not a directory for example.
            if (fs.lstatSync(licenseFile).isFile()) {
                moduleInfo.licenses = license(fs.readFileSync(licenseFile, {encoding: 'utf8'}));
            }
        });
    }

    if (Array.isArray(moduleInfo.licenses)) {
        if (moduleInfo.licenses.length === 1) {
            moduleInfo.licenses = moduleInfo.licenses[0];
        }
    }

    if (json.dependencies) {
        Object.keys(json.dependencies).forEach(function(name) {
            var childDependency = json.dependencies[name],
                dependencyId = childDependency.name + '@' + childDependency.version;
            if (data[dependencyId]) { // already exists
                return;
            }
            flatten(childDependency);
        });
    }
};

exports.init = function(options, callback) {

    console.log('scanning' , options.start);

    read(options.start, function(err, json) {
        flatten(json);
        var sorted = {};
        Object.keys(data).sort().forEach(function(item) {
{'fraction_non_alphanumeric': 0.14461970951116665, 'fraction_numerical': 0.0031235358425737935, 'mean_word_length': 2.3233004670472237, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7266296', 'n_tokens_mistral': 1777, 'n_tokens_neox': 1723, 'n_words': 378}