<commit_before>#!/usr/bin/python

import subprocess
import sys

def RunCommand(command):
  run = subprocess.Popen(command, shell=True)
  output = run.communicate()
  if run.returncode:
    print "Non-zero return code: " + str(run.returncode) + " => exiting!"
    sys.exit(1)

def list_of_experiments():
  experiments = []
  configure_file = open("configure")
  list_start = False
  for line in configure_file.read().split("\n"):
    if line == 'EXPERIMENT_LIST="':
      list_start = True
    elif line == '"':
      list_start = False
    elif list_start:
      currently_broken = ["csm"]
      experiment = line[4:]
      if experiment not in currently_broken:
        experiments.append(experiment)
  return experiments

def main():
  base_command = "./configure --enable-internal-stats"
  test_build(base_command)
  for experiment_name in list_of_experiments():
    test_build("%s --enable-experimental --enable-%s" % (base_command,
      experiment_name))

def test_build(configure_command):
  print "\033[34m\033[47mTesting %s\033[0m" % (configure_command)
  RunCommand(configure_command)
  RunCommand("make clean")
  RunCommand("make")

if __name__ == "__main__":
  main()
<commit_msg>all_builds.py: support for sharding builds<commit_after>#!/usr/bin/python

import getopt
import subprocess
import sys

LONG_OPTIONS = ["shard=", "shards="]
BASE_COMMAND = "./configure --enable-internal-stats --enable-experimental"

def RunCommand(command):
  run = subprocess.Popen(command, shell=True)
  output = run.communicate()
  if run.returncode:
    print "Non-zero return code: " + str(run.returncode) + " => exiting!"
    sys.exit(1)

def list_of_experiments():
  experiments = []
  configure_file = open("configure")
  list_start = False
  for line in configure_file.read().split("\n"):
    if line == 'EXPERIMENT_LIST="':
      list_start = True
    elif line == '"':
      list_start = False
    elif list_start:
      currently_broken = ["csm"]
      experiment = line[4:]
      if experiment not in currently_broken:
        experiments.append(experiment)
  return experiments

def main(argv):
  # Parse arguments
  options = {"--shard": 0, "--shards": 1}
  o, _ = getopt.getopt(argv[1:], None, LONG_OPTIONS)
  options.update(o)

  # Shard experiment list
  shard = int(options["--shard"])
  shards = int(options["--shards"])
  experiments = list_of_experiments()
  configs = [BASE_COMMAND]
  configs += ["%s --enable-%s" % (BASE_COMMAND, e) for e in experiments]
  my_configs = zip(configs, range(len(configs)))
  my_configs = filter(lambda x: x[1] % shards == shard, my_configs)
  my_configs = [e[0] for e in my_configs]

  # Run configs for this shard
  for config in my_configs:
    test_build(config)

def test_build(configure_command):
  print "\033[34m\033[47mTesting %s\033[0m" % (configure_command)
  RunCommand(configure_command)
  RunCommand("make clean")
  RunCommand("make")

if __name__ == "__main__":
  main(sys.argv)

{'fraction_non_alphanumeric': 0.13678121810139504, 'fraction_numerical': 0.012589316093909493, 'mean_word_length': 3.9915110356536503, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3982863', 'n_tokens_mistral': 1006, 'n_tokens_neox': 987, 'n_words': 226}