<commit_before>'use strict';

var fs = require('fs');
var path = require('path');

var glob = require('glob');
var async = require('async');
var walker = require('filewalker');

var utils = require('./utils');


module.exports = {
    walk: walk,
    write: write
};

function walk(root, cb) {
    glob(path.join(root, '/*/package.json'), function(err, files) {
        if(err) {
            return console.error(err);
        }

        async.map(files, function(file, cb) {
            var d = require('../' + file);
            var repoUrl = d.repositories && d.repositories[0] && d.repositories[0].url;

            if(!repoUrl && d.repository) {
                if(Array.isArray(d.repository)) {
                    d.repository = d.repository[0];
                }

                repoUrl = d.repository.url? d.repository.url: d.repository;
            }
            if(!repoUrl && d.homepage) {
                repoUrl = d.homepage;
            }

            if(!repoUrl) {
                console.warn(d.name + ' is missing a repo!');

                return cb();
            }

            var author = d.author && d.author.name;

            if(!author && d.maintainers) {
                author = d.maintainers[0].name;
            }

            var gh = utils.parseGh(repoUrl);
            var ret = {
                author: author,
                name: d.name,
                version: d.version,
                description: d.description,
                homepage: d.homepage,
                keywords: d.keywords,
                github: 'https://github.com/' + gh.user + '/' + gh.repo<commit_msg>walker - Parse author correctly in case the field is string<commit_after>'use strict';

var fs = require('fs');
var path = require('path');

var glob = require('glob');
var async = require('async');
var walker = require('filewalker');

var utils = require('./utils');


module.exports = {
    walk: walk,
    write: write
};

function walk(root, cb) {
    glob(path.join(root, '/*/package.json'), function(err, files) {
        if(err) {
            return console.error(err);
        }

        async.map(files, function(file, cb) {
            var d = require('../' + file);
            var repoUrl = d.repositories && d.repositories[0] && d.repositories[0].url;

            if(!repoUrl && d.repository) {
                if(Array.isArray(d.repository)) {
                    d.repository = d.repository[0];
                }

                repoUrl = d.repository.url? d.repository.url: d.repository;
            }
            if(!repoUrl && d.homepage) {
                repoUrl = d.homepage;
            }

            if(!repoUrl) {
                console.warn(d.name + ' is missing a repo!');

                return cb();
            }

            var author = d.author;

            if(author && author.name) {
                author = author.name;
            }

            if(!author && d.maintainers) {
                author = d.maintainers[0].name;
            }

            var gh = utils.parseGh(repoUrl);
            var ret = {
                author: author,
                name: d.name,
                version: d.version,
                description: d.description,
                homepage: d.homepage,
                keywords: d.keywords,
                github: 'https://github.com/' + gh.user + '/' + gh.repo
{'fraction_non_alphanumeric': 0.13958333333333334, 'fraction_numerical': 0.002380952380952381, 'mean_word_length': 1.4586686174103878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3570550', 'n_tokens_mistral': 994, 'n_tokens_neox': 926, 'n_words': 205}