<commit_before>#!/usr/bin/env python

import os
import sys
import glob
import gzip
import json
import time
import pipes
import base64
import shutil
import urllib2
import subprocess

SOURCES = (
  glob.glob('src/backend/*.sk') +
  glob.glob('src/core/*.sk') +
  glob.glob('src/driver/*.sk') +
  glob.glob('src/frontend/*.sk') +
  glob.glob('src/lib/*.sk') +
  glob.glob('src/middle/*.sk') +
  glob.glob('tests/*.sk')
)

PUBLIC_CPP_FILES = [
  'src/cpp/skew.cpp',
  'src/cpp/skew.h',
]

FLAGS = [
  '--inline-functions',
  '--verbose',
  '--message-limit=0',
]

CPP_FLAGS = [
  '-std=c++11',
  '-Wall',
  '-Wextra',
  '-Wno-switch',
  '-Wno-unused-parameter',
  '-Wno-unused-variable',
  '-include', 'src/cpp/skew.h',
  '-include', 'src/cpp/support.h',
]

CPP_DEBUG_FLAGS = [
  'src/cpp/skew.cpp',
  'src/cpp/support.cpp',
]

CPP_RELEASE_FLAGS = [
  '-O3',
  '-DNDEBUG',
  '-fomit-frame-pointer',
  '-include', 'src/cpp/skew.cpp',
  '-include', 'src/cpp/support.cpp',
  '-include', 'src/cpp/fast.cpp',
]

CPP_GC_FLAGS = [
  '-DSKEW_GC_MARK_AND_SWEEP',
]

node_binary = None
jobs = {}

################################################################################

def job(fn):
  jobs[fn.__name__] = fn
  return fn

def run(args, exit_on_failure=True, **kwargs):
  # Print the command for debugging so that it can be copied and pasted into a terminal directly
  print ' '.join(map(pipes.quote, args))

  # Start the process
  process = subprocess.Popen(args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, **kwargs)
  try:
    process.wait()

    # Abort on failure
    if exit_on_failure and process.returncode:
      print 'error: command exited with code %d' % process.returncode
      sys.exit(1)

    return process.returncode

  # Ensure the process is terminated
  finally:
    try:
      process.terminate()<commit_msg>update "build.py" for python 3<commit_after>#!/usr/bin/env python

import os
import sys
import glob
import gzip
import json
import time
import pipes
import base64
import shutil
import subprocess

SOURCES = (
  glob.glob('src/backend/*.sk') +
  glob.glob('src/core/*.sk') +
  glob.glob('src/driver/*.sk') +
  glob.glob('src/frontend/*.sk') +
  glob.glob('src/lib/*.sk') +
  glob.glob('src/middle/*.sk') +
  glob.glob('tests/*.sk')
)

PUBLIC_CPP_FILES = [
  'src/cpp/skew.cpp',
  'src/cpp/skew.h',
]

FLAGS = [
  '--inline-functions',
  '--verbose',
  '--message-limit=0',
]

CPP_FLAGS = [
  '-std=c++11',
  '-Wall',
  '-Wextra',
  '-Wno-switch',
  '-Wno-unused-parameter',
  '-Wno-unused-variable',
  '-include', 'src/cpp/skew.h',
  '-include', 'src/cpp/support.h',
]

CPP_DEBUG_FLAGS = [
  'src/cpp/skew.cpp',
  'src/cpp/support.cpp',
]

CPP_RELEASE_FLAGS = [
  '-O3',
  '-DNDEBUG',
  '-fomit-frame-pointer',
  '-include', 'src/cpp/skew.cpp',
  '-include', 'src/cpp/support.cpp',
  '-include', 'src/cpp/fast.cpp',
]

CPP_GC_FLAGS = [
  '-DSKEW_GC_MARK_AND_SWEEP',
]

node_binary = None
jobs = {}

################################################################################

def job(fn):
  jobs[fn.__name__] = fn
  return fn

def run(args, exit_on_failure=True, **kwargs):
  # Print the command for debugging so that it can be copied and pasted into a terminal directly
  print(' '.join(map(pipes.quote, args)))

  # Start the process
  process = subprocess.Popen(args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, **kwargs)
  try:
    process.wait()

    # Abort on failure
    if exit_on_failure and process.returncode:
      print('error: command exited with code %d' % process.returncode)
      sys.exit(1)

    return process.returncode

  # Ensure the process is terminated
  finally:
    try:
      process.terminate()
{'fraction_non_alphanumeric': 0.2131192161132281, 'fraction_numerical': 0.004354926510615134, 'mean_word_length': 3.5482673267326734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4052746', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1361, 'n_words': 270}