<commit_before>#!/usr/bin/env python3
from __future__ import print_function

import argparse
import base64
import hashlib
import json
import os
import requests
import sys
import time

class Error(Exception):
  pass

def sleep_time(attempt):
  if attempt <= 0:
    raise Exception('Unexpected')
  if attempt == 1:
    return 0
  if attempt == 2:
    return 15
  if attempt == 3:
    return 60
  if attempt == 4:
    return 90
  if attempt == 5:
    return 300
  return 1200

def retry(func_in):
  def func_out(*args, **kwargs):
    retry_max = 10
    i = 0
    while True:
      i = i + 1
      try:
        return func_in(*args, **kwargs)
      except Error as err:
        # Treat Errors as fatal and do not retry.
        # Also explicitly flush message to avoid "no output" issue on some CIs.
        print('Error:\n  {}'.format(err))
        sys.stdout.flush()
        raise err
      except Exception as exc:
        if i > retry_max:
          raise exc
        print('Operation failed. Exception:\n  {}'.format(exc))
        sec = sleep_time(i)
        print('Retry #{} (of {}) after {} seconds'.format(i, retry_max, sec))
        sys.stdout.flush()
        time.sleep(sec)
  return func_out

# http://stackoverflow.com/a/16696317/2288008
@retry
def download_file(url, local_file, auth, chunk_size=1024):
  print('Downloading:\n  {}\n  -> {}'.format(url, local_file))
  r = requests.get(url, stream=True, auth=auth)
  if not r.ok:
    raise Exception('Downloading failed')
  with open(local_file, 'wb') as f:
    for chunk in r.iter_content(chunk_size=chunk_size):
      if chunk:
        f.write(chunk)

class Github:
  def __init__(self, username, password, repo_owner, repo):
    self.repo_owner = repo_owner
    self.repo = repo
    self.auth = requests.auth.HTTPBasicAuth(username, password)
    self.simple_request()

  @retry
  def simple_request(self):
    print('Processing simple request')
    r = requests.get('https://api.github.com', auth=self.auth)
    if not r.ok:
      sys.exit('Simple request fails. Check your password.')

    limit = int(r.headers['X-RateLimit-Remaining'])
    print('GitHub Limit: {}'.format(limit))
    if limit == 0:
      raise Exception('GitHub limit is 0')
    print('Simple request pass')

  @retry
  def get_release_by_tag(self, tagname):
    print('Get release-id by tag `{}`'.format(tagname))
    # https://developer.github.com/v3/repos/releases/#get-a-release-by-tag-name
    # GET /repos/:owner/:repo/releases/tags/:tag

    url = 'https://api.github.com/repos/{}/{}/releases/tags/{}'.format(
        self.repo_owner,
        self.repo,
        tagname
    )

    r = requests.get(url, auth=self.auth)
    if r.status_code == 404:
        # Create release if not exists
        # https://developer.github.com/v3/repos/releases/#create-a-release
        # POST /repos/:owner/:repo/releases

        post_url = 'https://api.github.com/repos/{}/{}/releases'.format(
            self.repo_owner,
            self.repo,
        )
        tag_data = "{" + '"tag_name": "{}"'.format(tagname) + "}"
        r = requests.post(post_url, data=tag_data, auth=self.auth)
        if not r.status_code == 201:
            raise Error("Unexpected status code: {}".format(r.status_code))
        if not r.ok:
            raise Error("Can't create release tag {}".format(tagname))
        r = requests.get(url, auth=self.auth)

    if not r.ok:
        raise Exception(
            'Get release id failed. Status code: {}. Requested url: {}'.format(
                r.status_code, url))

    release_id = r.json()['id']
    upload_url = r.json()['upload_url']
    uri_template_vars = '{?name,label}'
    if uri_template_vars not in upload_url:
        raise Exception('Unsupported upload URI template: {}'.format(upload_url))
    upload_url = upload_url.replace(uri_template_vars, '?name={}')
    print('Release id: {}'.format(release_id))
    print('Upload URL: {}'.format(upload_url))
    return release_id, upload_url

  @retry
  def find_asset_id_by_name(self, release_id, name):
    # https://developer.github.com/v3/repos/releases/#list-assets-for-a-release
    # GET /repos/:owner/:repo/releases/:id/assets
<commit_msg>Upload script: Handle 404 code<commit_after>#!/usr/bin/env python3
from __future__ import print_function

import argparse
import base64
import hashlib
import json
import os
import requests
import sys
import time

class Error(Exception):
  pass

def sleep_time(attempt):
  if attempt <= 0:
    raise Exception('Unexpected')
  if attempt == 1:
    return 0
  if attempt == 2:
    return 15
  if attempt == 3:
    return 60
  if attempt == 4:
    return 90
  if attempt == 5:
    return 300
  return 1200

def retry(func_in):
  def func_out(*args, **kwargs):
    retry_max = 10
    i = 0
    while True:
      i = i + 1
      try:
        return func_in(*args, **kwargs)
      except Error as err:
        # Treat Errors as fatal and do not retry.
        # Also explicitly flush message to avoid "no output" issue on some CIs.
        print('Error:\n  {}'.format(err))
        sys.stdout.flush()
        raise err
      except Exception as exc:
        if i > retry_max:
          raise exc
        print('Operation failed. Exception:\n  {}'.format(exc))
        sec = sleep_time(i)
        print('Retry #{} (of {}) after {} seconds'.format(i, retry_max, sec))
        sys.stdout.flush()
        time.sleep(sec)
  return func_out

# http://stackoverflow.com/a/16696317/2288008
@retry
def download_file(url, local_file, auth, chunk_size=1024):
  print('Downloading:\n  {}\n  -> {}'.format(url, local_file))
  r = requests.get(url, stream=True, auth=auth)
  if not r.ok:
    raise Exception('Downloading failed')
  with open(local_file, 'wb') as f:
    for chunk in r.iter_content(chunk_size=chunk_size):
      if chunk:
        f.write(chunk)

class Github:
  def __init__(self, username, password, repo_owner, repo):
    self.repo_owner = repo_owner
    self.repo = repo
    self.username = username
    self.auth = requests.auth.HTTPBasicAuth(username, password)
    self.simple_request()

  @retry
  def simple_request(self):
    print('Processing simple request')
    r = requests.get('https://api.github.com', auth=self.auth)
    if not r.ok:
      sys.exit('Simple request fails. Check your password.')

    limit = int(r.headers['X-RateLimit-Remaining'])
    print('GitHub Limit: {}'.format(limit))
    if limit == 0:
      raise Exception('GitHub limit is 0')
    print('Simple request pass')

  @retry
  def get_release_by_tag(self, tagname):
    print('Get release-id by tag `{}`'.format(tagname))
    # https://developer.github.com/v3/repos/releases/#get-a-release-by-tag-name
    # GET /repos/:owner/:repo/releases/tags/:tag

    url = 'https://api.github.com/repos/{}/{}/releases/tags/{}'.format(
        self.repo_owner,
        self.repo,
        tagname
    )

    r = requests.get(url, auth=self.auth)
    if r.status_code == 404:
        # Create release if not exists
        # https://developer.github.com/v3/repos/releases/#create-a-release
        # POST /repos/:owner/:repo/releases

        post_url = 'https://api.github.com/repos/{}/{}/releases'.format(
            self.repo_owner,
            self.repo,
        )
        tag_data = "{" + '"tag_name": "{}"'.format(tagname) + "}"
        r = requests.post(post_url, data=tag_data, auth=self.auth)
        if r.status_code == 404:
            repo = "https://github.com/{}/{}".format(self.repo_owner, self.repo)
            raise Error(
                "Repository not found {} or user {} have no access to it".
                    format(repo, self.username)
            )
        if not r.status_code == 201:
            raise Error("Unexpected status code: {}".format(r.status_code))
        if not r.ok:
            raise Error("Can't create release tag {}".format(tagname))
        r = requests.get(url, auth=self.auth)

    if not r.ok:
        raise Exception(
            'Get release id failed. Status code: {}. Requested url: {}'.format(
                r.status_code, url))

    release_id = r.json()['id']
    upload_url = r.json()['upload_url']
    uri_template_vars = '{?name,label}'
    if uri_template_vars not in upload_url:
        raise Exception('Unsupported upload URI template: {}'.format(upload_url))
    upload_url = upload_url.replace(uri_template_vars, '?name={}')
    print('Release id: {}'.format(release_id))
    print('Upload URL: {}'.format(upload_url))
    return release_id, upload_url

  @retry
  def find_asset_id_by_name(self, release_id, name):
    # https://developer.github.com/v3/repos/releases/#list-assets-for-a-release
    # GET /repos/:owner/:repo/releases/:id/assets

{'fraction_non_alphanumeric': 0.13722543352601155, 'fraction_numerical': 0.013872832369942197, 'mean_word_length': 2.6954293037163604, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 13, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5543636', 'n_tokens_mistral': 2912, 'n_tokens_neox': 2765, 'n_words': 686}