<jupyter_start><jupyter_text>Local Setup<jupyter_code>repo = Repository('/Users/rick/projects/tensorwerk/hangar/dev/dota/')
repo.init(user_name='<NAME>', user_email='<EMAIL>', remove_old=True)
co = repo.checkout(write=True)
testData = np.loadtxt('/Users/rick/projects/tensorwerk/hangar/dev/data/dota2Dataset/dota2Test.csv', delimiter=',', dtype=np.uint8)
trainData = np.loadtxt('/Users/rick/projects/tensorwerk/hangar/dev/data/dota2Dataset/dota2Train.csv', delimiter=',', dtype=np.uint16)
testName = 'test'
testPrototype = testData[0]
trainName = 'train'
trainPrototype = trainData[0]
co.datasets.init_dataset(name=testName, prototype=testPrototype, samples_are_named=False)

testDset = co.datasets[testName]
pbar = tqdm(total=testData.shape[0])
with testDset as ds:
    for gameIdx, gameData in enumerate(testData):
        if (gameIdx % 500 == 0):
            pbar.update(500)
        ds.add(gameData)
pbar.close()

co.commit('initial commit on master with test data')
repo.create_branch('add-train')

co.close()
co = repo.checkout(write=True, branch_name='add-train')

co.datasets.init_dataset(name=trainName, prototype=trainPrototype, samples_are_named=False)
trainDset = co.datasets[trainName]

pbar = tqdm(total=trainData.shape[0])
with trainDset as dt:
    for gameIdx, gameData in enumerate(trainData):
        if (gameIdx % 500 == 0):
            pbar.update(500)
        dt.add(gameData)
pbar.close()

co.commit('added training data on another branch')
co.close()
repo.log()<jupyter_output>* 06d4bffed975ce2ccd4f89ea6bd628df64ddc18d ([1;31madd-train[m) : added training data on another branch
* 829210950e8b8d0efcbfc8842c61497b11766cf4 ([1;31mmaster[m) : initial commit on master with test data<jupyter_text>Push to remote server running on google cloud at `172.16.17.32:50051`<jupyter_code>repo.add_remote('gcloud', '172.16.17.32:50051')<jupyter_output><empty_output><jupyter_text>Push the master branch<jupyter_code>repo.push('gcloud', 'master')<jupyter_output><empty_output><jupyter_text>Push the training branch<jupyter_code>repo.push('gcloud', 'add-train')<jupyter_output><empty_output><jupyter_text>Now Restart and See How this Works! `hangar clone 172.16.17.32:50051`<jupyter_code>repo = Repository('/Users/rick/projects/tensorwerk/hangar/dev/dota-clone/')
repo.list_branch_names()
repo.log()
repo.fetch('origin', 'add-train')
repo.list_branch_names()
repo.log(branch_name='origin/add-train')<jupyter_output><empty_output><jupyter_text>merge the commits and push!<jupyter_code>co = repo.checkout(write=True, branch_name='master')
co.reset_staging_area()
repo.merge('ff merge', 'master', 'origin/add-train')
repo.log()
repo.push('origin', 'master')<jupyter_output><empty_output><jupyter_text>Check that it worked!<jupyter_code>repo.fetch('origin', 'master')
repo.summary()<jupyter_output><empty_output><jupyter_text>Try Cloning in a fresh repo just to be sure!<jupyter_code>repo = Repository('/Users/rick/projects/tensorwerk/hangar/dev/dota-clone-2/')
repo.clone('<NAME>', '<EMAIL>', remote_address='172.16.17.32:50051')
repo.log()
co = repo.checkout()
co.datasets
co.datasets['train']
repo.summary()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15079365079365079, 'fraction_numerical': 0.042222222222222223, 'mean_word_length': 7.10025706940874, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '573081', 'license': 'MIT', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1177, 'n_words': 179, 'path': 'DLWorkFlow/demo-3-remote.ipynb', 'repo_name': 'hhsecond/talks', 'size': '3188'}