<commit_before>import json
import requests
from copy import copy
from urllib import urlencode

from settings import client_id, client_secret


def build_authentication_request_url():
    url = 'https://accounts.google.com/o/oauth2/auth'
    params = {
        'response_type': 'code',
        'client_id': client_id,
        'scope': 'https://www.googleapis.com/auth/youtube.readonly',
        'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob',
    }
    return url + '?' + urlencode(params)


authentication_request_url = build_authentication_request_url()


class APIClient(object):
    access_token = None
    refresh_token = None

    def __init__(self, code):
        self.request_token(code)

    def request_token(self, code=None):
        if code is None and self.refresh_token is None:
            raise Exception('The `code` argument is required on the first '
                            'call to request_token')

        # Assemble POST data

        post_data = {
            'client_id': client_id,
            'client_secret': client_secret,
        }
        if code is None:
            post_data['refresh_token'] = self.refresh_token
            post_data['grant_type'] = 'refresh_token'
        else:
            post_data['code'] = code
            post_data['redirect_uri'] = 'urn:ietf:wg:oauth:2.0:oob'
            post_data['grant_type'] = 'authorization_code'

        # Make token request

        response = requests.post(
            'https://accounts.google.com/o/oauth2/token', data=post_data)
        j = json.loads(response.text)
        if 'error' in j:
            raise Exception(
                '{}: {}'.format(j['error'], j['error_description']))

        # Extract token(s)

        self.access_token = j['access_token']
        if code is not None:
            self.refresh_token = j['refresh_token']

    def get(self, path, params, process):
        nextPageToken = True
        while nextPageToken is not None:
            url = 'https://www.googleapis.com/youtube/v3' + path
            headers = {'Authorization': 'Bearer ' + self.access_token}

            page_params = copy(params)

            if isinstance(nextPageToken, basestring):
                page_params['pageToken'] = nextPageToken
            j = json.loads(
                requests.get(url, params=page_params, headers=headers).text)
            if 'error' in j:
                raise Exception(j['error'])
            for item in j['items']:
                process(item)

            nextPageToken = j.get('nextPageToken', None)<commit_msg>Implement token refreshing<commit_after>import json
import requests
from copy import copy
from urllib import urlencode

from settings import client_id, client_secret


def build_authentication_request_url():
    url = 'https://accounts.google.com/o/oauth2/auth'
    params = {
        'response_type': 'code',
        'client_id': client_id,
        'scope': 'https://www.googleapis.com/auth/youtube.readonly',
        'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob',
    }
    return url + '?' + urlencode(params)


authentication_request_url = build_authentication_request_url()


class APIClient(object):
    access_token = None
    refresh_token = None

    def retrieve_stored_tokens(self):
        try:
            with open('tokens.txt', 'r') as token_file:
                tokens = json.load(token_file)

            self.access_token = tokens['access']
            self.refresh_token = tokens['refresh']
        except IOError as e:
            if e.errno != 2:  # 'No such file or directory'
                raise()

    def store_tokens(self):
        with open('tokens.txt', 'w') as token_file:
            json.dump({
                'access': self.access_token,
                'refresh': self.refresh_token,
            }, token_file)

    def get_token_pair(self, code=None):
        if code is None:
            retrieve_stored_tokens()

        # Assemble POST data

        post_data = {
            'client_id': client_id,
            'client_secret': client_secret,
            'code': code,
            'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob',
            'grant_type': 'authorization_code',
        }

        # Make token request

        response = requests.post(
            'https://accounts.google.com/o/oauth2/token', data=post_data)
        j = json.loads(response.text)
        if 'error' in j:
            raise Exception(
                '{}: {}'.format(j['error'], j['error_description']))

        # Extract token(s)

        self.access_token = j['access_token']
        self.refresh_token = j['refresh_token']

        self.store_tokens()

    def refresh(self):
        if self.refresh_token is None:
            raise Exception('You must call get_token_pair first.')

        # Assemble POST data

        post_data = {
            'client_id': client_id,
            'client_secret': client_secret,
            'refresh_token': self.refresh_token,
            'grant_type': 'refresh_token',
        }

        # Make token request

        response = requests.post(
            'https://accounts.google.com/o/oauth2/token', data=post_data)
        j = json.loads(response.text)
        if 'error' in j:
            raise Exception(
                '{}: {}'.format(j['error'], j['error_description']))

        # Extract token(s)

        self.access_token = j['access_token']

    def get(self, path, params, process):
        nextPageToken = True
        while nextPageToken is not None:
            url = 'https://www.googleapis.com/youtube/v3' + path
            headers = {'Authorization': 'Bearer ' + self.access_token}

            page_params = copy(params)

            if isinstance(nextPageToken, basestring):
                page_params['pageToken'] = nextPageToken

            refresh_tries = 0
            while True:
                response = requests.get(url, params=page_params,
                                        headers=headers)
                if response.status_code == 401:
                    if refresh_tries < 2:
                        self.refresh()
                        refresh_tries += 1
                    else:
                        raise Exception('Something went wrong')
                else:
                    break

            j = json.loads(response.text)

            if 'error' in j:
                raise Exception(j['error'])
            for item in j['items']:
                process(item)

            nextPageToken = j.get('nextPageToken', None)
{'fraction_non_alphanumeric': 0.10982747997535429, 'fraction_numerical': 0.0033887861983980284, 'mean_word_length': 1.9327009936766035, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 9, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2693339', 'n_tokens_mistral': 1884, 'n_tokens_neox': 1803, 'n_words': 403}