<commit_before>import json
from operator import itemgetter
import os
import subprocess
import sys

from twitter.common import log
from twitter.common.contextutil import open_zip
from twitter.common.dirutil import Fileset, safe_mkdtemp
from twitter.common.git import branch, git


class CommandLineBuilder(object):

  class Error(Exception): pass

  @classmethod
  def call(cls, cmd):
    log.info('Running: %s' % cmd)
    po = subprocess.Popen(cmd, shell=True,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=sys.stdin)
    so, se = po.communicate()
    for line in so.splitlines():
      log.info('stdout: %s' % line)
    for line in se.splitlines():
      log.info('stderr: %s' % line)
    rc = po.returncode
    if rc != 0:
      log.error('Command returned non-zero (%d) exit code: %s' % (rc, cmd))
    return rc, so, se

  @classmethod
  def check_call(cls, cmd, *args, **kw):
    rc, _, _ = cls.call(cmd, *args, **kw)
    if rc != 0:
      raise cls.Error('Command Failed: %s' % cmd)

  def __init__(self, build_spec):
    self.test_command = build_spec.get('test_command')
    self.build_command = build_spec.get('build_command')
    self.artifact_globs = build_spec['artifact_globs']

  @property
  def artifacts(self):
    artifacts = set()
    for glob in self.artifact_globs:
      artifacts.update(Fileset.zglobs(glob)())
    return artifacts

  def run(self):
    for command in (self.test_command, self.build_command):
      if command is not None:
        self.check_call(command)


class GitBuilder(CommandLineBuilder):

  def __init__(self, build_spec):
    super(GitBuilder, self).__init__(build_spec)
    self.gitsha = build_spec['gitsha']
    self._repo = git.Repo(os.getcwd())
    self.resolved_gitsha = self._repo.rev_parse(self.gitsha).hexsha

  def validate_repo(self, require_origin_master=False):
    if self._repo.is_dirty():
      raise self.Error('Can not run the build from a dirty repository!')
    sha = self.resolved_gitsha
    if require_origin_master:
      for commit in self._repo.iter_commits(rev='origin/master'):
        if sha == commit.hexsha:
          break
      else:
        raise self.Error('Non-master shas can not be candidates for releases!')
    return sha

  def run(self):<commit_msg>Better error message for dirty repo referenced in BuildSpec in aurora config<commit_after>import json
from operator import itemgetter
import os
import subprocess
import sys

from twitter.common import log
from twitter.common.contextutil import open_zip
from twitter.common.dirutil import Fileset, safe_mkdtemp
from twitter.common.git import branch, git


class CommandLineBuilder(object):

  class Error(Exception): pass

  @classmethod
  def call(cls, cmd):
    log.info('Running: %s' % cmd)
    po = subprocess.Popen(cmd, shell=True,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=sys.stdin)
    so, se = po.communicate()
    for line in so.splitlines():
      log.info('stdout: %s' % line)
    for line in se.splitlines():
      log.info('stderr: %s' % line)
    rc = po.returncode
    if rc != 0:
      log.error('Command returned non-zero (%d) exit code: %s' % (rc, cmd))
    return rc, so, se

  @classmethod
  def check_call(cls, cmd, *args, **kw):
    rc, _, _ = cls.call(cmd, *args, **kw)
    if rc != 0:
      raise cls.Error('Command Failed: %s' % cmd)

  def __init__(self, build_spec):
    self.test_command = build_spec.get('test_command')
    self.build_command = build_spec.get('build_command')
    self.artifact_globs = build_spec['artifact_globs']

  @property
  def artifacts(self):
    artifacts = set()
    for glob in self.artifact_globs:
      artifacts.update(Fileset.zglobs(glob)())
    return artifacts

  def run(self):
    for command in (self.test_command, self.build_command):
      if command is not None:
        self.check_call(command)


class GitBuilder(CommandLineBuilder):

  def __init__(self, build_spec):
    super(GitBuilder, self).__init__(build_spec)
    self.gitsha = build_spec['gitsha']
    self._repo = git.Repo(os.getcwd())
    self.resolved_gitsha = self._repo.rev_parse(self.gitsha).hexsha

  def validate_repo(self, require_origin_master=False):
    if self._repo.is_dirty():
      raise self.Error('Can not run the build from a dirty repository! Commit or stash your changes'
                       ' to proceed, or remove gitsha from BuildSpec in aurora config to build'
                       ' current source.')
    sha = self.resolved_gitsha
    if require_origin_master:
      for commit in self._repo.iter_commits(rev='origin/master'):
        if sha == commit.hexsha:
          break
      else:
        raise self.Error('Non-master shas can not be candidates for releases!')
    return sha

  def run(self):
{'fraction_non_alphanumeric': 0.11153358681875793, 'fraction_numerical': 0.0008449514152936206, 'mean_word_length': 3.408752327746741, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5924845', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1503, 'n_words': 407}