<commit_before># -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: <NAME> <<EMAIL>>
#           see also AUTHORS file
#
# :copyright:   (c) 2015, The Tor Project, Inc.
#               (c) 2015, <NAME>
#
# :license: This is Free Software. See LICENSE for license information.
#

import os
import sys
import re
import hashlib

from libsaas.services import github
import gnupg
import gettor.core


def get_file_sha256(file):
    """Get the sha256 of a file.

    :param: file (string) the path of the file.

    :return: (string) the sha256 hash.

    """
    # as seen on the internetz
    BLOCKSIZE = 65536
    hasher = hashlib.sha256()
    with open(file, 'rb') as afile:
        buf = afile.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(BLOCKSIZE)
    return hasher.hexdigest()


def get_bundle_info(file, osys):
    """Get the os, arch and lc from a bundle string.

    :param: file (string) the name of the file.
    :param: osys (string) the OS.

    :raise: ValueError if the bundle doesn't have a valid bundle format.

    :return: (list) the os, arch and lc.

    """
    if(osys == 'windows'):
        m = re.search(
            'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe',
            file)
        if m:
            lc = m.group(1)
            return 'windows', '32/64', lc
    elif(osys == 'linux'):
        m = re.search(
            'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
            file)
        if m:
            arch = m.group(1)
            lc = m.group(2)
            return 'linux', arch, lc
    elif(osys == 'osx'):
        m = re.search(
            'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg',
            file)
        if m:
            os = 'osx'
            arch = m.group(1)
            lc = m.group(2)
            return 'osx', arch, lc

if __name__ == '__main__':

    # this script should be called after fetching the latest Tor Browser,
    # and specifying the latest version
    version = sys.argv[1]

    # the token allow us to run this script without GitHub user/pass
    gh_token = '<PASSWORD>'

    # path to the fingerprint that signed the packages
    tb_key = 'upload/torbrowser-key.asc'

    # import key fingerprint
    gpg = gnupg.GPG()
    key_data = open(tb_key).read()
    import_result = gpg.import_keys(key_data)
    fp = import_result.results[0]['fingerprint']

    # make groups of four characters to make fingerprint more readable
    # e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
    readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))

    # we should have previously created a repository on GitHub where we
    # want to push the files using a SSH key (to avoid using user/pass)
    remote = 'origin'
    branch = 'master'
    user = 'ilv'
    repo = 'gettor'
    path = 'upload'
    raw_content = 'https://raw.githubusercontent.com/%s/%s/%s/%s/' %\
                  (user, repo, branch, path)

    # steps: 1) create dir for the new version, copy the latest Tor Browser
    # files there, add the files via git, make a commit for the new version
    # and push the changes (easier with system calls)
    # os.system('mkdir %s' % version)
    # os.system('cp latest/* %s' % version)
    # os.system('git add %s;git commit -m "%s"' % (version, version))
    # os.system('cd %s;git push %s %s' % (version, remote, branch))

    gh = github.GitHub(gh_token, None)
    repocontent = gh.repo(
        user,
        repo
    ).contents().get('%s/%s' % (path, version))

    core = gettor.core.Core(
        '/path/to/gettor/core.cfg'
    )

    # erase old links, if any
    core.create_links_file('GitHub', readable_fp)
<commit_msg>Uncommented system calls for git<commit_after># -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: <NAME> <<EMAIL>>
#           see also AUTHORS file
#
# :copyright:   (c) 2015, The Tor Project, Inc.
#               (c) 2015, <NAME>
#
# :license: This is Free Software. See LICENSE for license information.
#

import os
import sys
import re
import hashlib

from libsaas.services import github
import gnupg
import gettor.core


def get_file_sha256(file):
    """Get the sha256 of a file.

    :param: file (string) the path of the file.

    :return: (string) the sha256 hash.

    """
    # as seen on the internetz
    BLOCKSIZE = 65536
    hasher = hashlib.sha256()
    with open(file, 'rb') as afile:
        buf = afile.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(BLOCKSIZE)
    return hasher.hexdigest()


def get_bundle_info(file, osys):
    """Get the os, arch and lc from a bundle string.

    :param: file (string) the name of the file.
    :param: osys (string) the OS.

    :raise: ValueError if the bundle doesn't have a valid bundle format.

    :return: (list) the os, arch and lc.

    """
    if(osys == 'windows'):
        m = re.search(
            'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe',
            file)
        if m:
            lc = m.group(1)
            return 'windows', '32/64', lc
    elif(osys == 'linux'):
        m = re.search(
            'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
            file)
        if m:
            arch = m.group(1)
            lc = m.group(2)
            return 'linux', arch, lc
    elif(osys == 'osx'):
        m = re.search(
            'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg',
            file)
        if m:
            os = 'osx'
            arch = m.group(1)
            lc = m.group(2)
            return 'osx', arch, lc

if __name__ == '__main__':

    # this script should be called after fetching the latest Tor Browser,
    # and specifying the latest version
    version = sys.argv[1]

    # the token allow us to run this script without GitHub user/pass
    gh_token = 'secret-token'

    # path to the fingerprint that signed the packages
    tb_key = 'upload/torbrowser-key.asc'

    # import key fingerprint
    gpg = gnupg.GPG()
    key_data = open(tb_key).read()
    import_result = gpg.import_keys(key_data)
    fp = import_result.results[0]['fingerprint']

    # make groups of four characters to make fingerprint more readable
    # e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
    readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))

    # we should have previously created a repository on GitHub where we
    # want to push the files using a SSH key (to avoid using user/pass)
    remote = 'origin'
    branch = 'master'
    user = 'ilv'
    repo = 'gettor'
    path = 'upload'
    raw_content = 'https://raw.githubusercontent.com/%s/%s/%s/%s/' %\
                  (user, repo, branch, path)

    # steps: 1) create dir for the new version, copy the latest Tor Browser
    # files there, add the files via git, make a commit for the new version
    # and push the changes (easier with system calls)
    os.system('mkdir %s' % version)
    os.system('cp latest/* %s' % version)
    os.system('git add %s;git commit -m "%s"' % (version, version))
    os.system('cd %s;git push %s %s' % (version, remote, branch))

    gh = github.GitHub(gh_token, None)
    repocontent = gh.repo(
        user,
        repo
    ).contents().get('%s/%s' % (path, version))

    core = gettor.core.Core(
        '/path/to/gettor/core.cfg'
    )

    # erase old links, if any
    core.create_links_file('GitHub', readable_fp)

{'fraction_non_alphanumeric': 0.13394397982211603, 'fraction_numerical': 0.019115890083632018, 'mean_word_length': 2.184277261200338, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326776', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2493, 'n_words': 796}