<commit_before>import base64
import json
import re
import string
import sys
import unicodedata
import time
import BaseHTTPServer
import SocketServer
from urlparse import urlparse
from rauth import OAuth1Service, OAuth2Service

rdio_client_id=''
rdio_client_secret=''
spotify_client_id=''
spotify_client_secret=''

page_size = 50
redirect_port = 8123

def normalize_text(data):
    return re.sub(r'- .*$', '', re.sub(r'[\(\[][^)]*[\)\]]', '', unicodedata.normalize('NFKD', data.lower()).encode('ASCII', 'ignore'))).strip()

def get_sessions():
    redirect_uri = 'http://localhost:%d/' % redirect_port
    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
        def do_GET(s):
            Handler.code = urlparse(s.path).query.split('=')[1]
            s.send_response(200)
            s.send_header("Content-type", "text/html")
            s.end_headers()
            s.wfile.write('<h3>Code received successfully</h3>')
            s.wfile.write('<p>Please look back at the terminal.</p>')
    httpd = SocketServer.TCPServer(("", redirect_port), Handler)
    def wait_for_code():
        httpd.handle_request()
        sys.stdout.flush()
        return Handler.code

    rdio = OAuth2Service(
          name='rdio',
          client_id=rdio_client_id,
          client_secret=rdio_client_secret,
          authorize_url='https://www.rdio.com/oauth2/authorize',
          access_token_url='https://services.rdio.com/oauth2/token',
          base_url='https://services.rdio.com/api/1/',)

    params={'response_type': 'code', 'redirect_uri': redirect_uri}
    rdio_authorize_url = rdio.get_authorize_url(**params)

    print 'Visit this URL in your browser: ' + rdio_authorize_url
    rdio_pin = wait_for_code()

    rdio_session = rdio.get_auth_session(method='POST',
                                         data={'code': rdio_pin, 
                                               'grant_type': 'authorization_code',
                                               'redirect_uri': redirect_uri,},
                                         headers={'Authorization': 'Basic ' + base64.b64encode(rdio_client_id + ":" + rdio_client_secret)},
                                         decoder=json.loads)

    spotify = OAuth2Service(
          name='spotify',
          client_id=spotify_client_id,
          client_secret=spotify_client_secret,
          authorize_url='https://accounts.spotify.com/authorize/',
          access_token_url='https://accounts.spotify.com/api/token',
          base_url='https://api.spotify.com',)

    params={'scope':'user-library-modify user-library-read playlist-read-private playlist-modify-public user-follow-modify', \
            'response_type': 'code', 'redirect_uri': redirect_uri}
    spotify_authorize_url = spotify.get_authorize_url(**params)

    print 'Visit this URL in your browser: ' + spotify_authorize_url
    spotify_pin = wait_for_code()

    spotify_session = spotify.get_auth_session(method='POST',
                                               data={'code': spotify_pin, 
                                                     'grant_type': 'authorization_code',
                                                     'redirect_uri': redirect_uri,},
                                               headers={'Authorization': 'Basic ' + base64.b64encode(spotify_client_id + ":" + spotify_client_secret)},
                                               decoder=json.loads)

    spotify_refresh_token = spotify_session.access_token_response.json()['refresh_token']
    def retry_if_possible(response):
        if response.status_code == 429 and response.headers['retry-after']:
            time.sleep(float(response.headers['retry-after']))
            return True
        if response.status_code == 401:
            refresh = spotify.get_raw_access_token(data={'refresh_token':spotify_refresh_token,
                                                         'grant_type': 'refresh_token'})
            spotify_session.access_token = refresh.json()['access_token']
            return True
        return False

    def spotify_get(url, **kwargs):
        while True:
            response = spotify_session.orig_get(url, **kwargs)
            if retry_if_possible(response):
                continue
            return response
    spotify_session.orig_get = spotify_session.get
    spotify_session.get = spotify_get<commit_msg>Retry after receiving an HTTP 50x response from Spotify<commit_after>import base64
import json
import re
import string
import sys
import unicodedata
import time
import BaseHTTPServer
import SocketServer
from urlparse import urlparse
from rauth import OAuth1Service, OAuth2Service

rdio_client_id=''
rdio_client_secret=''
spotify_client_id=''
spotify_client_secret=''

page_size = 50
redirect_port = 8123

def normalize_text(data):
    return re.sub(r'- .*$', '', re.sub(r'[\(\[][^)]*[\)\]]', '', unicodedata.normalize('NFKD', data.lower()).encode('ASCII', 'ignore'))).strip()

def get_sessions():
    redirect_uri = 'http://localhost:%d/' % redirect_port
    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
        def do_GET(s):
            Handler.code = urlparse(s.path).query.split('=')[1]
            s.send_response(200)
            s.send_header("Content-type", "text/html")
            s.end_headers()
            s.wfile.write('<h3>Code received successfully</h3>')
            s.wfile.write('<p>Please look back at the terminal.</p>')
    httpd = SocketServer.TCPServer(("", redirect_port), Handler)
    def wait_for_code():
        httpd.handle_request()
        sys.stdout.flush()
        return Handler.code

    rdio = OAuth2Service(
          name='rdio',
          client_id=rdio_client_id,
          client_secret=rdio_client_secret,
          authorize_url='https://www.rdio.com/oauth2/authorize',
          access_token_url='https://services.rdio.com/oauth2/token',
          base_url='https://services.rdio.com/api/1/',)

    params={'response_type': 'code', 'redirect_uri': redirect_uri}
    rdio_authorize_url = rdio.get_authorize_url(**params)

    print 'Visit this URL in your browser: ' + rdio_authorize_url
    rdio_pin = wait_for_code()

    rdio_session = rdio.get_auth_session(method='POST',
                                         data={'code': rdio_pin, 
                                               'grant_type': 'authorization_code',
                                               'redirect_uri': redirect_uri,},
                                         headers={'Authorization': 'Basic ' + base64.b64encode(rdio_client_id + ":" + rdio_client_secret)},
                                         decoder=json.loads)

    spotify = OAuth2Service(
          name='spotify',
          client_id=spotify_client_id,
          client_secret=spotify_client_secret,
          authorize_url='https://accounts.spotify.com/authorize/',
          access_token_url='https://accounts.spotify.com/api/token',
          base_url='https://api.spotify.com',)

    params={'scope':'user-library-modify user-library-read playlist-read-private playlist-modify-public user-follow-modify', \
            'response_type': 'code', 'redirect_uri': redirect_uri}
    spotify_authorize_url = spotify.get_authorize_url(**params)

    print 'Visit this URL in your browser: ' + spotify_authorize_url
    spotify_pin = wait_for_code()

    spotify_session = spotify.get_auth_session(method='POST',
                                               data={'code': spotify_pin, 
                                                     'grant_type': 'authorization_code',
                                                     'redirect_uri': redirect_uri,},
                                               headers={'Authorization': 'Basic ' + base64.b64encode(spotify_client_id + ":" + spotify_client_secret)},
                                               decoder=json.loads)

    spotify_refresh_token = spotify_session.access_token_response.json()['refresh_token']
    def retry_if_possible(response):
        if response.status_code == 429 and response.headers['retry-after']:
            time.sleep(float(response.headers['retry-after']))
            return True
        if response.status_code == 401:
            refresh = spotify.get_raw_access_token(data={'refresh_token':spotify_refresh_token,
                                                         'grant_type': 'refresh_token'})
            spotify_session.access_token = refresh.json()['access_token']
            return True
        if response.status_code / 100 == 5:
            return True
        return False

    def spotify_get(url, **kwargs):
        while True:
            response = spotify_session.orig_get(url, **kwargs)
            if retry_if_possible(response):
                continue
            return response
    spotify_session.orig_get = spotify_session.get
    spotify_session.get = spotify_get
{'fraction_non_alphanumeric': 0.11435990065477535, 'fraction_numerical': 0.008579814856626778, 'mean_word_length': 2.236755571793935, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 12, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5542073', 'n_tokens_mistral': 2558, 'n_tokens_neox': 2470, 'n_words': 409}