<commit_before>import os, tempfile, urllib, zipfile, shutil, json
import subprocess32 as subprocess
from contextlib import contextmanager

WORKING_DIR = os.path.dirname(os.path.realpath(__file__))
CORPUS_INFO = None
with open(os.path.join(WORKING_DIR, 'corpus.json')) as f:
  CORPUS_INFO = json.loads(f.read())
CORPUS_DIR = os.path.join(WORKING_DIR, "corpus")

LOG_FILE = None

def write_log(line):
  global LOG_FILE
  if LOG_FILE:
    LOG_FILE.write(line)
    LOG_FILE.flush()

def run_cmd(cmd):
  stats = {'output': ''}

  write_log("Running command '{}'\n".format(' '.join(cmd)))

  process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  for line in iter(process.stdout.readline, b''):
    stats['output'] = stats['output'] + line
    write_log(line)
  process.stdout.close()
  process.wait()

  if process.returncode != 0:
    print "Error running command '{}', check corpus.log for details.".format(' '.join(cmd))

  write_log("\n\n")

  return stats

@contextmanager
def cd(newdir):
  prevdir = os.getcwd()
  os.chdir(os.path.expanduser(newdir))
  try:
    yield
  finally:
    os.chdir(prevdir)

def git_update(project):
  if project['git-ref'] not in run_cmd(['git', 'rev-parse', 'HEAD'])['output']:
    print "Checking out git ref %s." % project['git-ref']
    run_cmd(['git', 'fetch'])
    run_cmd(['git', 'reset', '--hard'])<commit_msg>Have fetch_corpus complain if a corpus project URL changes<commit_after>import os, tempfile, urllib, zipfile, shutil, json
import subprocess32 as subprocess
from contextlib import contextmanager

WORKING_DIR = os.path.dirname(os.path.realpath(__file__))
CORPUS_INFO = None
with open(os.path.join(WORKING_DIR, 'corpus.json')) as f:
  CORPUS_INFO = json.loads(f.read())
CORPUS_DIR = os.path.join(WORKING_DIR, "corpus")

LOG_FILE = None

def write_log(line):
  global LOG_FILE
  if LOG_FILE:
    LOG_FILE.write(line)
    LOG_FILE.flush()

def run_cmd(cmd):
  stats = {'output': ''}

  write_log("Running command '{}'\n".format(' '.join(cmd)))

  process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  for line in iter(process.stdout.readline, b''):
    stats['output'] = stats['output'] + line
    write_log(line)
  process.stdout.close()
  process.wait()

  if process.returncode != 0:
    print "Error running command '{}', check corpus.log for details.".format(' '.join(cmd))

  write_log("\n\n")

  return stats

@contextmanager
def cd(newdir):
  prevdir = os.getcwd()
  os.chdir(os.path.expanduser(newdir))
  try:
    yield
  finally:
    os.chdir(prevdir)

def git_update(project):
  if project['git-url'] != run_cmd(['git', 'remote', '-v']):
    print "git_url for {} has changed. Please delete the directory to redownload.".format(project['name'])
    return
  if project['git-ref'] not in run_cmd(['git', 'rev-parse', 'HEAD'])['output']:
    print "Checking out git ref %s." % project['git-ref']
    run_cmd(['git', 'fetch'])
    run_cmd(['git', 'reset', '--hard'])
{'fraction_non_alphanumeric': 0.16330645161290322, 'fraction_numerical': 0.0020161290322580645, 'mean_word_length': 4.523191094619666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3462265', 'n_tokens_mistral': 1077, 'n_tokens_neox': 1055, 'n_words': 240}