<commit_before>#!/usr/bin/env python

import errno
import glob
import os
import subprocess
import sys
import tempfile

from lib.util import *


TARGET_PLATFORM = {
  'cygwin': 'win32',
  'darwin': 'darwin',
  'linux2': 'linux',
  'win32': 'win32',
}[sys.platform]

SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
DIST_DIR = os.path.join(SOURCE_ROOT, 'dist')
DIST_NAME = 'atom-shell-{0}-{1}.zip'.format(get_atom_shell_version(),
                                            TARGET_PLATFORM)


def main():
  if not dist_newer_than_head():
    create_dist = os.path.join(SOURCE_ROOT, 'script', 'create-dist.py')
    subprocess.check_call([sys.executable, create_dist])
  upload()


def dist_newer_than_head():
  with scoped_cwd(SOURCE_ROOT):
    try:
      head_time = subprocess.check_output(['git', 'log', '--pretty=format:%at',
                                           '-n', '1']).strip()
      dist_time = os.path.getmtime(os.path.join(DIST_DIR, DIST_NAME))
    except OSError as e:
      if e.errno != errno.ENOENT:
        raise
      return False

  return dist_time > int(head_time)


def upload():
  os.chdir(DIST_DIR)
  bucket, access_key, secret_key = s3_config()

  version = get_atom_shell_version()
  s3put(bucket, access_key, secret_key, DIST_DIR,
        'atom-shell/{0}'.format(version), [DIST_NAME])
  s3put(bucket, access_key, secret_key, DIST_DIR,
        'atom-shell/dist/{0}'.format(version), glob.glob('node-*.tar.gz'))

  update_version(bucket, access_key, secret_key)


def s3_config():
  config = (os.environ.get('ATOM_SHELL_S3_BUCKET', ''),
            os.environ.get('ATOM_SHELL_S3_ACCESS_KEY', ''),
            os.environ.get('ATOM_SHELL_S3_SECRET_KEY', ''))
  message = ('Error: Please set the $ATOM_SHELL_S3_BUCKET, '
             '$ATOM_SHELL_S3_ACCESS_KEY, and '
             '$ATOM_SHELL_S3_SECRET_KEY environment variables')
  assert all(len(c) for c in config), message
  return config


def update_version(bucket, access_key, secret_key):
  prefix = os.path.join(SOURCE_ROOT, 'dist')
  version = os.path.join(prefix, 'version')
  s3put(bucket, access_key, secret_key, prefix, 'atom-shell', [ version ])


def s3put(bucket, access_key, secret_key, prefix, key_prefix, files):
  args = [
    's3put',
    '--bucket', bucket,
    '--access_key', access_key,
    '--secret_key', secret_key,
    '--prefix', prefix,
    '--key_prefix', key_prefix,
    '--grant', 'public-read'
  ] + files

  subprocess.check_call(args)


if __name__ == '__main__':
  import sys
  sys.exit(main())
<commit_msg>Add flag for upload script to skip updating version.<commit_after>#!/usr/bin/env python

import argparse
import errno
import glob
import os
import subprocess
import sys
import tempfile

from lib.util import *


TARGET_PLATFORM = {
  'cygwin': 'win32',
  'darwin': 'darwin',
  'linux2': 'linux',
  'win32': 'win32',
}[sys.platform]

ATOM_SHELL_VRESION = get_atom_shell_version()

SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
DIST_DIR = os.path.join(SOURCE_ROOT, 'dist')
DIST_NAME = 'atom-shell-{0}-{1}.zip'.format(ATOM_SHELL_VRESION, TARGET_PLATFORM)


def main():
  args = parse_args()

  if not dist_newer_than_head():
    create_dist = os.path.join(SOURCE_ROOT, 'script', 'create-dist.py')
    subprocess.check_call([sys.executable, create_dist])

  bucket, access_key, secret_key = s3_config()
  upload(bucket, access_key, secret_key)
  if not args.no_update_version:
    update_version(bucket, access_key, secret_key)


def parse_args():
  parser = argparse.ArgumentParser(description='upload distribution file')
  parser.add_argument('-n', '--no-update-version',
                      help='Do not update the latest version file',
                      action='store_false')
  return parser.parse_args()


def dist_newer_than_head():
  with scoped_cwd(SOURCE_ROOT):
    try:
      head_time = subprocess.check_output(['git', 'log', '--pretty=format:%at',
                                           '-n', '1']).strip()
      dist_time = os.path.getmtime(os.path.join(DIST_DIR, DIST_NAME))
    except OSError as e:
      if e.errno != errno.ENOENT:
        raise
      return False

  return dist_time > int(head_time)


def upload(bucket, access_key, secret_key, version=ATOM_SHELL_VRESION):
  os.chdir(DIST_DIR)

  s3put(bucket, access_key, secret_key, DIST_DIR,
        'atom-shell/{0}'.format(version), [DIST_NAME])
  s3put(bucket, access_key, secret_key, DIST_DIR,
        'atom-shell/dist/{0}'.format(version), glob.glob('node-*.tar.gz'))


def update_version(bucket, access_key, secret_key):
  prefix = os.path.join(SOURCE_ROOT, 'dist')
  version = os.path.join(prefix, 'version')
  s3put(bucket, access_key, secret_key, prefix, 'atom-shell', [version])


def s3_config():
  config = (os.environ.get('ATOM_SHELL_S3_BUCKET', ''),
            os.environ.get('ATOM_SHELL_S3_ACCESS_KEY', ''),
            os.environ.get('ATOM_SHELL_S3_SECRET_KEY', ''))
  message = ('Error: Please set the $ATOM_SHELL_S3_BUCKET, '
             '$ATOM_SHELL_S3_ACCESS_KEY, and '
             '$ATOM_SHELL_S3_SECRET_KEY environment variables')
  assert all(len(c) for c in config), message
  return config


def s3put(bucket, access_key, secret_key, prefix, key_prefix, files):
  args = [
    's3put',
    '--bucket', bucket,
    '--access_key', access_key,
    '--secret_key', secret_key,
    '--prefix', prefix,
    '--key_prefix', key_prefix,
    '--grant', 'public-read'
  ] + files

  subprocess.check_call(args)


if __name__ == '__main__':
  import sys
  sys.exit(main())

{'fraction_non_alphanumeric': 0.14393939393939395, 'fraction_numerical': 0.009018759018759018, 'mean_word_length': 3.571310799670239, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '717112', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1982, 'n_words': 376}