<commit_before>#!/usr/bin/python
import sys, os, subprocess, shutil, errno, datetime, unittest
import sync

def test_sync():

  test_config = {
    "mount": "/tmp/sync",
    "actives": {
      "Devastator" : [
        "Scrapper",
        "Hook",
        "Bonecrusher",
        "Longhaul",
        "Mixmaster",
        "Scavenger"
      ],
      "Voltron": [
        "Black",
        "Blue",
        "Green",
        "Red",
        "Yellow"
      ]
    }
  }

  if os.path.isdir(test_config['mount']):
    shutil.rmtree(test_config['mount'])

  # setup: create tmp filesystem
  for k, v in test_config['actives'].iteritems():
    master = os.sep.join([test_config['mount'], k])
    os.makedirs(master)

    f = open(os.sep.join([master, 'test']), 'w')
    f.write(master)
    f.close()

    for v2 in v:
      slave = os.sep.join([test_config['mount'], v2])
      os.makedirs(slave)

  # execute
  out = sync.sync(test_config)

  # validate
  for k, v in test_config['actives'].iteritems():
    master = os.sep.join([test_config['mount'], k])

    for v2 in v:
      synced_filename = os.sep.join([test_config['mount'], v2, 'test'])
      f = open(synced_filename, 'r')

      assert(os.path.isfile(synced_filename))
      assert(f.read() == master)

      f.close()

  # teardown
  shutil.rmtree(test_config['mount'], True)

if __name__ == "__main__":

  command = sys.argv[1]
  if command == "sync":
    sync()
  elif command == "test":
    test_sync()
<commit_msg>better.<commit_after>#!/usr/bin/python
import sys, os, subprocess, shutil, errno, datetime, unittest
import sync

def test_sync():

  test_config = {
    "mount": "/tmp/sync",
    "actives": {
      "Devastator" : [
        "Scrapper",
        "Hook",
        "Bonecrusher",
        "Longhaul",
        "Mixmaster",
        "Scavenger"
      ],
      "Voltron": [
        "Black",
        "Blue",
        "Green",
        "Red",
        "Yellow"
      ]
    }
  }

  if os.path.isdir(test_config['mount']):
    shutil.rmtree(test_config['mount'])

  # setup: create tmp filesystem
  for k, v in test_config['actives'].iteritems():
    master = os.sep.join([test_config['mount'], k])
    os.makedirs(master)

    f = open(os.sep.join([master, 'test']), 'w')
    f.write(master)
    f.close()

    for v2 in v:
      slave = os.sep.join([test_config['mount'], v2])
      os.makedirs(slave)

  # execute
  out = sync.sync(test_config)

  # validate
  for k, v in test_config['actives'].iteritems():
    master = os.sep.join([test_config['mount'], k])

    for v2 in v:
      synced_filename = os.sep.join([test_config['mount'], v2, 'test'])
      f = open(synced_filename, 'r')

      assert(os.path.isfile(synced_filename))
      assert(f.read() == master)

      f.close()

  # teardown
  shutil.rmtree(test_config['mount'], True)

if __name__ == "__main__":
  test_sync()

{'fraction_non_alphanumeric': 0.17159971811134603, 'fraction_numerical': 0.003171247357293869, 'mean_word_length': 2.54875, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4840523', 'n_tokens_mistral': 1045, 'n_tokens_neox': 1014, 'n_words': 197}