<commit_before># dbus-python system package is needed, so create a virtualenv with --system-site-packages
# Packages needed: boxsdk>=2.0.0a9 oauth2client keyring

import os
import sys
import json
import webbrowser
import argparse
import getpass
import hashlib
import base64

# Import two classes from the boxsdk module - Client and OAuth2
import boxsdk
from boxsdk import Client
# from boxsdk.util.multipart_stream import MultipartStream
UPLOAD_URL = boxsdk.config.API.UPLOAD_URL

import oauth2client
from oauth2client.contrib.keyring_storage import Storage
from oauth2client import tools

class FolderTraversalException(Exception):
    pass

class BoxAPI:
    def __init__(self):
        storage = Storage('klab_box_sync', getpass.getuser())
        self.credentials = storage.get()
        if self.credentials == None:
            parser = argparse.ArgumentParser(parents=[tools.argparser])
            flags = parser.parse_args()
            flow = oauth2client.client.flow_from_clientsecrets('client_secrets.json', scope='', redirect_uri = 'http://localhost:8080')
            self.credentials = tools.run_flow(flow, storage, flags)

        # We need to check if our access has expired here, because we are
        # authorizing outside of boxsdk and boxsdk will not be able to refresh later
        # if the token has expired
        if self.credentials.access_token_expired:
            self.credentials.get_access_token()

        self.client = Client(self.credentials)

        self.root_folder = self.client.folder( folder_id = '0' )

    def find_folder_by_name( self, search_folder, name, limit = 1000 ):
        folders = [ f for f in search_folder.get_items( limit = limit ) if f['name'] == name ]
        if len( folders ) != 1:
            raise FolderTraversalException()
        return folders[0]['id']

    def find_folder_path( self, folder_path, limit = 1000 ):
        current_folder_id = '0'
        for folder_name in os.path.normpath(folder_path).split(os.path.sep):
            if len(folder_name) > 0:
                current_folder_id = self.find_folder_by_name( self.client.folder( folder_id = current_folder_id ), folder_name, limit = limit )
        return current_folder_id

    def upload( self,
                destination_folder_id,
                source_path,
                preflight_check = True,
    ):
        file_size = os.stat(source_path).st_size
        if file_size >= 55000000:
            self.chunked_upload( destination_folder_id, source_path, preflight_check = preflight_check )
        else:
            self.client.folder( folder_id = destination_folder_id ).upload( file_path = source_path, preflight_check = preflight_check, preflight_expected_size = file_size )

    def chunked_upload( self,
                        destination_folder_id,
                        source_path,
                        preflight_check = True,
    ):
        destination_folder = self.client.folder( folder_id = destination_folder_id )
        file_size = os.stat(source_path).st_size
        if preflight_check:
            destination_folder.preflight_check( size = file_size, name = os.path.basename(source_path) )

        url = '{0}/files/upload_sessions'.format(UPLOAD_URL)

        data = json.dumps({
            'folder_id' : destination_folder_id,
            'file_size' : file_size,
            'file_name' : os.path.basename(source_path),
        })

        json_response = self.client.session.post(url, data=data, expect_json_response=True)
        abort_url = json_response.json()['session_endpoints']['abort']
        upload_responses = {
            'create' : json_response.json(),
            'parts' : {},
        }

        try:
            session_id = json_response.json()['id']
            part_size =  json_response.json()['part_size']

            for part_n in range( json_response.json()['total_parts'] ):
                start_byte = part_n * part_size
                url = '{0}/files/upload_sessions/{1}'.format( UPLOAD_URL, session_id )

                headers = {
                    'content-range' : 'bytes {0}-{1}/{2}'.format( start_byte, start_byte + part_size - 1, file_size ),
                    'content-type' : 'application/octet-stream',
                }

                sha1 = hashlib.sha1()

                with open( source_path, 'rb' ) as f:
                    f.seek( start_byte )
                    data = f.read( part_size - 1 )
                    sha1.update(data)

                headers['digest'] = 'sha=' + base64.b64encode(sha1.digest()).decode()
                print( part_n, part_size, headers['digest'] )

                part_response = self.client.session.put(url, headers = headers, data = data, expect_json_response = True)

                upload_responses['parts'][part_n] = part_response

            # Commit
            url = '{0}/files/upload_sessions/{1}/commit'.format( UPLOAD_URL, session_id )
            data = json.dumps({
                'parts' : [ upload_responses['parts'][part_n] for part_n in range( json_response.json()['total_parts'] ) ],
            })
            commit_response = self.client.session.post(url, data=data, expect_json_response=True)
            upload_responses['commit'] = commit_response
        except:<commit_msg>Chunked uploads working<commit_after># dbus-python system package is needed, so create a virtualenv with --system-site-packages
# Packages needed: boxsdk>=2.0.0a9 oauth2client keyring

import os
import sys
import json
import webbrowser
import argparse
import getpass
import hashlib
import base64

# Import two classes from the boxsdk module - Client and OAuth2
import boxsdk
from boxsdk import Client
# from boxsdk.util.multipart_stream import MultipartStream
UPLOAD_URL = boxsdk.config.API.UPLOAD_URL

import oauth2client
from oauth2client.contrib.keyring_storage import Storage
from oauth2client import tools

class FolderTraversalException(Exception):
    pass

class BoxAPI:
    def __init__(self):
        storage = Storage('klab_box_sync', getpass.getuser())
        self.credentials = storage.get()
        if self.credentials == None:
            parser = argparse.ArgumentParser(parents=[tools.argparser])
            flags = parser.parse_args()
            flow = oauth2client.client.flow_from_clientsecrets('client_secrets.json', scope='', redirect_uri = 'http://localhost:8080')
            self.credentials = tools.run_flow(flow, storage, flags)

        # We need to check if our access has expired here, because we are
        # authorizing outside of boxsdk and boxsdk will not be able to refresh later
        # if the token has expired
        if self.credentials.access_token_expired:
            self.credentials.get_access_token()

        self.client = Client(self.credentials)

        self.root_folder = self.client.folder( folder_id = '0' )

    def find_folder_by_name( self, search_folder, name, limit = 1000 ):
        folders = [ f for f in search_folder.get_items( limit = limit ) if f['name'] == name ]
        if len( folders ) != 1:
            raise FolderTraversalException()
        return folders[0]['id']

    def find_folder_path( self, folder_path, limit = 1000 ):
        current_folder_id = '0'
        for folder_name in os.path.normpath(folder_path).split(os.path.sep):
            if len(folder_name) > 0:
                current_folder_id = self.find_folder_by_name( self.client.folder( folder_id = current_folder_id ), folder_name, limit = limit )
        return current_folder_id

    def upload( self,
                destination_folder_id,
                source_path,
                preflight_check = True,
    ):
        file_size = os.stat(source_path).st_size
        if file_size >= 55000000:
            self.chunked_upload( destination_folder_id, source_path, preflight_check = preflight_check )
        else:
            self.client.folder( folder_id = destination_folder_id ).upload( file_path = source_path, preflight_check = preflight_check, preflight_expected_size = file_size )

    def chunked_upload( self,
                        destination_folder_id,
                        source_path,
                        preflight_check = True,
    ):
        destination_folder = self.client.folder( folder_id = destination_folder_id )
        file_size = os.stat(source_path).st_size
        if preflight_check:
            destination_folder.preflight_check( size = file_size, name = os.path.basename(source_path) )

        url = '{0}/files/upload_sessions'.format(UPLOAD_URL)

        data = json.dumps({
            'folder_id' : destination_folder_id,
            'file_size' : file_size,
            'file_name' : os.path.basename(source_path),
        })

        json_response = self.client.session.post(url, data=data, expect_json_response=True)
        abort_url = json_response.json()['session_endpoints']['abort']
        upload_responses = {
            'create' : json_response.json(),
            'parts' : {},
        }

        try:
            session_id = json_response.json()['id']
            part_size = json_response.json()['part_size']

            total_sha = hashlib.sha1()

            for part_n in range( json_response.json()['total_parts'] ):
                start_byte = part_n * part_size
                url = '{0}/files/upload_sessions/{1}'.format( UPLOAD_URL, session_id )

                headers = {
                    'content-type' : 'application/octet-stream',
                }

                sha1 = hashlib.sha1()

                with open( source_path, 'rb' ) as f:
                    f.seek( start_byte )
                    data = f.read( part_size )
                    sha1.update(data)
                    total_sha.update(data)

                headers['digest'] = 'sha=' + base64.b64encode(sha1.digest()).decode()
                headers['content-range'] = 'bytes {0}-{1}/{2}'.format( start_byte, start_byte + len(data) - 1, file_size )
                part_response = self.client.session.put(url, headers = headers, data = data, expect_json_response = True)

                upload_responses['parts'][part_n] = part_response.json()['part']

            # Commit
            url = '{0}/files/upload_sessions/{1}/commit'.format( UPLOAD_URL, session_id )
            data = json.dumps({
                'parts' : [ upload_responses['parts'][part_n] for part_n in range( json_response.json()['total_parts'] ) ],
            })
            headers = {}
            headers['digest'] = 'sha=' + base64.b64encode(total_sha.digest()).decode()
            commit_response = self.client.session.post(url, headers=headers, data=data, expect_json_response=True)
            upload_responses['commit'] = commit_response.json()
        except:
{'fraction_non_alphanumeric': 0.10061601642710473, 'fraction_numerical': 0.010640283740899757, 'mean_word_length': 2.403748411689962, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4514509', 'n_tokens_mistral': 3201, 'n_tokens_neox': 3050, 'n_words': 670}