<commit_before>import qs from 'querystring'
import fetch from 'unfetch'

export function getGistInfo(url, user = null, pass = null) {
    return Promise.resolve().then(() => {
        let obj = new URL(url)
        let [id] = obj.pathname.match(/[^\/]+/g).slice(-1)
        let host = obj.hostname !== 'gist.github.com' ? `${obj.hostname}/api` : 'api.github.com'
        let client = new GitHub({user, pass, host})

        return {id, host, client}
    }).then(({id, host, client}) => client.getGist(id).then(gist => {
        let user = gist.owner.login
        let avatar = gist.owner.avatar_url
        let file = gist.files[Object.keys(gist.files)[0]]
        if (file == null) return Promise.reject(new Error('File not found'))

        return Promise.resolve().then(() => {
            if (!file.truncated) return file.content

            return fetch(file.raw_url).then(res => {
                if (!res.ok) return Promise.reject(new Error('Could not retrieve file'))
                return res.text()
            })
        }).then(content => ({id, host, user, avatar, client, content}))
    }))
}

export default class GitHub {
    constructor({user = null, pass = null, host = 'api.github.com'} = {}) {
        this.host = host
        this.setAuthorization(user, pass)
    }

    setAuthorization(user = null, pass = null) {
        if (user == null) this.authorization = null
        else this.authorization = atob(`${user}:${pass}`)
    }

    makeHeaders() {
        let headers = {
            'If-Modified-Since': 'Mon, 26 Jul 1997 05:00:00 GMT',
        }

        if (this.authorization != null) {
            headers.Authorization = `Basic ${this.authorization}`
        }

        return headers
    }

    getUser() {
        return fetch(`https://${this.host}/user`, {
            headers: this.makeHeaders()
        }).then(res => {
            if (!res.ok) return Promise.reject(new Error(res.statusText))
            return res.json()
        })
    }

    getGist(id) {
        return fetch(`https://${this.host}/gists/${id}`, {
            headers: this.makeHeaders()
        }).then(res => {
            if (!res.ok) return Promise.reject(new Error(res.statusText))
            return res.json()
        })
    }<commit_msg>Fix base64 encoding<commit_after>import qs from 'querystring'
import fetch from 'unfetch'

export function getGistInfo(url, user = null, pass = null) {
    return Promise.resolve().then(() => {
        let obj = new URL(url)
        let [id] = obj.pathname.match(/[^\/]+/g).slice(-1)
        let host = obj.hostname !== 'gist.github.com' ? `${obj.hostname}/api` : 'api.github.com'
        let client = new GitHub({user, pass, host})

        return {id, host, client}
    }).then(({id, host, client}) => client.getGist(id).then(gist => {
        let user = gist.owner.login
        let avatar = gist.owner.avatar_url
        let file = gist.files[Object.keys(gist.files)[0]]
        if (file == null) return Promise.reject(new Error('File not found'))

        return Promise.resolve().then(() => {
            if (!file.truncated) return file.content

            return fetch(file.raw_url).then(res => {
                if (!res.ok) return Promise.reject(new Error('Could not retrieve file'))
                return res.text()
            })
        }).then(content => ({id, host, user, avatar, client, content}))
    }))
}

export default class GitHub {
    constructor({user = null, pass = null, host = 'api.github.com'} = {}) {
        this.host = host
        this.setAuthorization(user, pass)
    }

    setAuthorization(user = null, pass = null) {
        if (user == null) this.authorization = null
        else this.authorization = btoa(`${user}:${pass}`)
    }

    makeHeaders() {
        let headers = {
            'If-Modified-Since': 'Mon, 26 Jul 1997 05:00:00 GMT',
        }

        if (this.authorization != null) {
            headers.Authorization = `Basic ${this.authorization}`
        }

        return headers
    }

    getUser() {
        return fetch(`https://${this.host}/user`, {
            headers: this.makeHeaders()
        }).then(res => {
            if (!res.ok) return Promise.reject(new Error(res.statusText))
            return res.json()
        })
    }

    getGist(id) {
        return fetch(`https://${this.host}/gists/${id}`, {
            headers: this.makeHeaders()
        }).then(res => {
            if (!res.ok) return Promise.reject(new Error(res.statusText))
            return res.json()
        })
    }
{'fraction_non_alphanumeric': 0.1590607000443066, 'fraction_numerical': 0.00664599025254763, 'mean_word_length': 2.1116471399035146, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3983869', 'n_tokens_mistral': 1386, 'n_tokens_neox': 1343, 'n_words': 314}