<commit_before>cozydb = require 'cozydb'
requestJson = require 'request-json'
request = require 'request'
moment = require 'moment'
cheerio = require 'cheerio'
fs = require 'fs'
async = require 'async'

fetcher = require '../lib/fetcher'
localization = require '../lib/localization_manager'


log = require('printit')
    prefix: "Github"
    date: true


# Models

Commit = require '../models/commit'

# Konnector

module.exports =

    name: "Github Commits"
    slug: "githubcommits"
    description: 'konnector description github commits'
    vendorLink: "https://www.github.com/"

    fields:
        login: "text"
        password: "password"
    models:
        commit: Commit

    # Define model requests.
    init: (callback) ->
        callback()

    fetch: (requiredFields, callback) ->
        log.info "Import started"

        fetcher.new()
            .use(getEvents)
            .use(buildCommitDateHash)
            .use(logCommits)
            .args(requiredFields, {}, {})
            .fetch (err, fields, commits, data) ->
                log.info "Import finished"
                notifContent = null
                if commits and commits.numImportedCommits > 0
                    localizationKey = 'notification commits'
                    options = smart_count: commits.numImportedCommits
                    notifContent = localization.t localizationKey, options

                callback err, notifContent


# Get latest events list to know wich commits were pushed.
getEvents = (requiredFields, commits, data, next) ->
    client = requestJson.createClient 'https://api.github.com'
    username = requiredFields.login
    pass = <PASSWORD>

    client.setBasicAuth username, pass

    path = "users/#{username}/events?page="

    data.commits = []
    log.info "Fetch commits sha from events..."
    async.eachSeries [1..10], (page, callback) ->

        client.get path + page, (err, res, events) ->
            unless err?
                unless events.message is 'Bad credentials'
                    log.info "Fetch events page #{page}..."
                    for event in events
                        if event.type is 'PushEvent'
                            for commit in event.payload.commits
                                data.commits.push commit
                    callback()
                else
                    callback 'bad credentials'
            else
                log.error err
                callback()
    , (err) ->
        log.info "All events data fetched."
        next(err)


# Build hash listing all already downloaded commits.
buildCommitDateHash = (requiredFields, entries, data, next) ->
    entries.commitHash = {}
    Commit.all (err, commits) ->
        if err
            log.error err
            next 'request error'
        else
            for commit in commits
                entries.commitHash[commit.sha] = true
            next()


# Retrieve and save non existing commits one by one.
logCommits = (requiredFields, entries, data, next) ->
    client = requestJson.createClient 'https://api.github.com'
    username = requiredFields.login
    pass = <PASSWORD>

    numImportedCommits = 0
    client.setBasicAuth username, pass

    async.eachSeries data.commits, (commit, callback) ->
        path = commit.url.substring 'https://api.github.com/'.length

        if commit? and entries.commitHash[commit.sha]
            log.info "Commit #{commit.sha} not saved: already exists."
            callback()

        else
            client.get path, (err, res, commit) ->

                if err
                    log.error err
                    callback()

                else if not commit? or not commit.commit? or not commit.author?
                    if commit?
                        log.info "Commit not saved: no metadata."
                    else
                        log.info "Commit #{commit.sha} not saved: no metadata."
                    callback()

                else if commit.author.login isnt username
                    log.info "Commit #{commit.sha} not saved: " + \
                             "user is not author (#{commit.author.login})."
                    callback('bad credentials')

                else<commit_msg>Fix Github username check case-sensitiveness (#476)<commit_after>cozydb = require 'cozydb'
requestJson = require 'request-json'
request = require 'request'
moment = require 'moment'
cheerio = require 'cheerio'
fs = require 'fs'
async = require 'async'

fetcher = require '../lib/fetcher'
localization = require '../lib/localization_manager'


log = require('printit')
    prefix: "Github"
    date: true


# Models

Commit = require '../models/commit'

# Konnector

module.exports =

    name: "Github Commits"
    slug: "githubcommits"
    description: 'konnector description github commits'
    vendorLink: "https://www.github.com/"

    fields:
        login: "text"
        password: "password"
    models:
        commit: Commit

    # Define model requests.
    init: (callback) ->
        callback()

    fetch: (requiredFields, callback) ->
        log.info "Import started"

        fetcher.new()
            .use(getEvents)
            .use(buildCommitDateHash)
            .use(logCommits)
            .args(requiredFields, {}, {})
            .fetch (err, fields, commits, data) ->
                log.info "Import finished"
                notifContent = null
                if commits and commits.numImportedCommits > 0
                    localizationKey = 'notification commits'
                    options = smart_count: commits.numImportedCommits
                    notifContent = localization.t localizationKey, options

                callback err, notifContent


# Get latest events list to know wich commits were pushed.
getEvents = (requiredFields, commits, data, next) ->
    client = requestJson.createClient 'https://api.github.com'
    username = requiredFields.login
    pass = <PASSWORD>

    client.setBasicAuth username, pass

    path = "users/#{username}/events?page="

    data.commits = []
    log.info "Fetch commits sha from events..."
    async.eachSeries [1..10], (page, callback) ->

        client.get path + page, (err, res, events) ->
            unless err?
                unless events.message is 'Bad credentials'
                    log.info "Fetch events page #{page}..."
                    for event in events
                        if event.type is 'PushEvent'
                            for commit in event.payload.commits
                                data.commits.push commit
                    callback()
                else
                    callback 'bad credentials'
            else
                log.error err
                callback()
    , (err) ->
        log.info "All events data fetched."
        next(err)


# Build hash listing all already downloaded commits.
buildCommitDateHash = (requiredFields, entries, data, next) ->
    entries.commitHash = {}
    Commit.all (err, commits) ->
        if err
            log.error err
            next 'request error'
        else
            for commit in commits
                entries.commitHash[commit.sha] = true
            next()


# Retrieve and save non existing commits one by one.
logCommits = (requiredFields, entries, data, next) ->
    client = requestJson.createClient 'https://api.github.com'
    username = requiredFields.login
    pass = <PASSWORD>Fields.password

    numImportedCommits = 0
    client.setBasicAuth username, pass

    async.eachSeries data.commits, (commit, callback) ->
        path = commit.url.substring 'https://api.github.com/'.length

        if commit? and entries.commitHash[commit.sha]
            log.info "Commit #{commit.sha} not saved: already exists."
            callback()

        else
            client.get path, (err, res, commit) ->

                if err
                    log.error err
                    callback()

                else if not commit? or not commit.commit? or not commit.author?
                    if commit?
                        log.info "Commit not saved: no metadata."
                    else
                        log.info "Commit #{commit.sha} not saved: no metadata."
                    callback()

                else if commit.author.login.toLowerCase() isnt
                username.toLowerCase()
                    log.info "Commit #{commit.sha} not saved: " + \
                             "user is not author (#{commit.author.login})."
                    callback('parsing error')

                else
{'fraction_non_alphanumeric': 0.09583284697093498, 'fraction_numerical': 0.0015174506828528073, 'mean_word_length': 1.6994328922495274, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6330175', 'n_tokens_mistral': 2272, 'n_tokens_neox': 2136, 'n_words': 648}