<commit_before>"use strict";

var URL     = require('url');
var request = require('request');
var userAgent = require('../util/userAgent');
var log     = require('../log');
var moment = require('moment');
var q = require('q');

function commitMessages(options) {
    var deferred = q.defer();

    // TODO: better argument check and url fix, or just use npm project name
    var repoURL = URL.parse(options.repo.replace(/\.git$/, ''));
    var project = repoURL.pathname;

    if (!project) {
        deferred.reject(new Error('that\'s no github url i know of'));
    }

    // remove leading /'s
    project = project.replace(/^\//g, '');

    var url = 'https://api.github.com/repos/' + project + '/commits?per_page=250';
    log.debug('requesting: ' + url);

    var headers = { 'User-Agent': userAgent };
    if (process.env.GITHUB_TOKEN) {
        headers.Authorization = 'token ' + process.env.GITHUB_TOKEN;
    }

    request({uri: url, json: true, headers: headers}, function (err, res, data) {
        log.debug('complete: ' + url);

        if (err) {
            return deferred.reject(err);
        }

        if (!data.map) {
            return deferred.reject('Unknown Github Repo: ' + options.repo + '. ' + (data.message || JSON.stringify(data)));
        }

        var changes = data.map(function(change){
            var sha = change.sha;<commit_msg>fix: GitHub URL parsing (#36)<commit_after>"use strict";

var URL     = require('url');
var request = require('request');
var userAgent = require('../util/userAgent');
var log     = require('../log');
var moment = require('moment');
var q = require('q');

function commitMessages(options) {
    var deferred = q.defer();

    var massagedRepo = options.repo;
    massagedRepo = massagedRepo.replace('git+https://', 'https://');
    massagedRepo = massagedRepo.replace('git://', 'https://');
    // Strip out .git and also any extra URL due to monorepos
    massagedRepo = massagedRepo.replace(
        /(https:\/\/github.com\/[^\/]*\/[^\/\.]*).*/,
        "$1"
    );
    var repoURL = URL.parse(massagedRepo);
    var project = repoURL.pathname;

    if (!project) {
        deferred.reject(new Error('that\'s no github url i know of'));
    }

    // remove leading /'s
    project = project.replace(/^\//g, '');

    var url = 'https://api.github.com/repos/' + project + '/commits?per_page=250';
    log.debug('requesting: ' + url);

    var headers = { 'User-Agent': userAgent };
    if (process.env.GITHUB_TOKEN) {
        headers.Authorization = 'token ' + process.env.GITHUB_TOKEN;
    }

    request({uri: url, json: true, headers: headers}, function (err, res, data) {
        log.debug('complete: ' + url);

        if (err) {
            return deferred.reject(err);
        }

        if (!data.map) {
            return deferred.reject('Unknown Github Repo: ' + options.repo + '. ' + (data.message || JSON.stringify(data)));
        }

        var changes = data.map(function(change){
            var sha = change.sha;
{'fraction_non_alphanumeric': 0.17405801933977993, 'fraction_numerical': 0.0030010003334444814, 'mean_word_length': 2.5842293906810037, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1266762', 'n_tokens_mistral': 945, 'n_tokens_neox': 918, 'n_words': 226}