<commit_before>#!/usr/bin/env python

# FIXME: Make this work under Python 2

import argparse
import csv
import doctest
import json
import os
import pickle
import sys
import time
import urllib.request

from collections import namedtuple
from datetime import datetime


class TargetNotFound(Exception):
    pass


def ls_github(url, cache_ttl=None):

    # FIXME: Move cache to a class or find a package.
    cache_name = "github.pickle"

    def load_cache():
        try:
            cache = pickle.load(open(cache_name, 'rb'))
        except IOError:
            cache = {}
        return cache

    def save_cache(cache):
        pickle.dump(cache, open(cache_name, 'wb'))

    cache = load_cache()
    if url in cache and (cache_ttl is None or (
        (datetime.now()-cache[url]['timestamp']).total_seconds() <
            cache_ttl)):
        data = cache[url]['data']
    else:
        while True:
            data = json.loads(urllib.request.urlopen(url).read().decode())
            if "message" in data:
                print("Warning: {}".format(data["message"]))
                time.sleep(1)
                continue
            else:
                break
        cache[url] = {
            'timestamp': datetime.now(),
            'data': data
        }
    save_cache(cache)

    return data

_Version = namedtuple("Version", ("version", "commits", "hash"))


class Version(_Version):
    """
    >>> v = Version("v0.0.4-44-g0cd842f")
    >>> v
    Version(version='v0.0.4', commits=44, hash='0cd842f')
    >>> str(v)
    'v0.0.4-44-g0cd842f'
    """

    def __new__(cls, value):
        version, commits, githash = value.split('-')
        commits = int(commits)
        assert githash[0] == 'g'
        return _Version.__new__(cls, version, commits, githash[1:])

    def __str__(self):
        return "%s-%i-g%s" % self


doctest.testmod()


def parse_args():

    parser = argparse.ArgumentParser(
            description='Download prebuilt firmware')
    parser.add_argument('--user',
            help='Github user to download from.', default="timvideos")
    parser.add_argument('--rev',
            help='Get a specific version.')
    parser.add_argument('--platform',
            help='Get for a specific platform (board + expansion boards configuration).')
    parser.add_argument('--board',
            help='Alias for --platform.', dest="platform")
    parser.add_argument('--channel',
            help="Get latest version from in a specific channel ().",
            default="unstable")
    parser.add_argument('--tag',
            help='Alias for --channel.', dest="channel")
    parser.add_argument('--latest', dest="channel", action="store_const",
            help="Get the latest version.",
            const="unstable")
    parser.add_argument('--branch',
            help="Branch to download from.", default="master")
    parser.add_argument('--target',
            help="Target to download from.", default="hdmi2usb")
    parser.add_argument('--firmware',
            help="Firmware to download from.", default="firmware")
    parser.add_argument('--arch', default="lm32",
            help="Soft-CPU architecture to download from.")

    args = parser.parse_args()

    assert args.platform
    assert args.rev or args.channel
    assert args.target

    return args


def mk_url(user, branch):

    details = {
        "owner": user,<commit_msg>3 options for channel?!<commit_after>#!/usr/bin/env python

# FIXME: Make this work under Python 2

import argparse
import csv
import doctest
import json
import os
import pickle
import sys
import time
import urllib.request

from collections import namedtuple
from datetime import datetime


class TargetNotFound(Exception):
    pass


def ls_github(url, cache_ttl=None):

    # FIXME: Move cache to a class or find a package.
    cache_name = "github.pickle"

    def load_cache():
        try:
            cache = pickle.load(open(cache_name, 'rb'))
        except IOError:
            cache = {}
        return cache

    def save_cache(cache):
        pickle.dump(cache, open(cache_name, 'wb'))

    cache = load_cache()
    if url in cache and (cache_ttl is None or (
        (datetime.now()-cache[url]['timestamp']).total_seconds() <
            cache_ttl)):
        data = cache[url]['data']
    else:
        while True:
            data = json.loads(urllib.request.urlopen(url).read().decode())
            if "message" in data:
                print("Warning: {}".format(data["message"]))
                time.sleep(1)
                continue
            else:
                break
        cache[url] = {
            'timestamp': datetime.now(),
            'data': data
        }
    save_cache(cache)

    return data

_Version = namedtuple("Version", ("version", "commits", "hash"))


class Version(_Version):
    """
    >>> v = Version("v0.0.4-44-g0cd842f")
    >>> v
    Version(version='v0.0.4', commits=44, hash='0cd842f')
    >>> str(v)
    'v0.0.4-44-g0cd842f'
    """

    def __new__(cls, value):
        version, commits, githash = value.split('-')
        commits = int(commits)
        assert githash[0] == 'g'
        return _Version.__new__(cls, version, commits, githash[1:])

    def __str__(self):
        return "%s-%i-g%s" % self


doctest.testmod()


def parse_args():

    parser = argparse.ArgumentParser(
            description='Download prebuilt firmware')
    parser.add_argument('--user',
            help='Github user to download from.', default="timvideos")
    parser.add_argument('--rev',
            help='Get a specific version.')
    parser.add_argument('--platform',
            help='Get for a specific platform (board + expansion boards configuration).')
    parser.add_argument('--board',
            help='Alias for --platform.', dest="platform")

    parser.add_argument('--channel',
            help="Get latest version from in a specific channel ().",
            default="unstable")
    parser.add_argument('--tag',
            help='Alias for --channel.', dest="channel")
    parser.add_argument('--latest', dest="channel", action="store_const",
            help="Get the latest version.",
            const="unstable")

    parser.add_argument('--branch',
            help="Branch to download from.", default="master")
    parser.add_argument('--target',
            help="Target to download from.", default="hdmi2usb")
    parser.add_argument('--firmware',
            help="Firmware to download from.", default="firmware")
    parser.add_argument('--arch', default="lm32",
            help="Soft-CPU architecture to download from.")

    args = parser.parse_args()

    assert args.platform
    assert args.rev or args.channel
    assert args.target

    return args


def mk_url(user, branch):

    details = {
        "owner": user,
{'fraction_non_alphanumeric': 0.13691354381823548, 'fraction_numerical': 0.010179994098554145, 'mean_word_length': 2.416834677419355, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1987194', 'n_tokens_mistral': 2031, 'n_tokens_neox': 1944, 'n_words': 492}