<commit_before>#!/usr/bin/env python

"""
Building the main components of sagews from source, ensuring that all
important (usually security-related) options are compiled in.

The components of sagews are:

    * python -- language (used mainly for admin control and sage)
    * node -- language (for web server)
    * nginx -- static web server
    * haproxy -- proxy and load ballancer
    * tinc -- p2p vpn
    * protobuf -- google's messaging api and format
    * cassandra -- distributed p2p database
    * sage (which we do not build here, yet -- perhaps we should)

This supports OS X and Ubuntu 12.04 on AMD and Intel.
"""

import logging, os, shutil, subprocess, sys, time

# Enable logging
logging.basicConfig()
log = logging.getLogger('')
log.setLevel(logging.DEBUG)   # WARNING, INFO

OS     = os.uname()[0]
PWD    = os.path.abspath('.')
DATA   = os.path.abspath('data')
SRC    = os.path.abspath('src')
PATCHES= os.path.join(SRC, 'patches')
BUILD  = os.path.abspath(os.path.join(DATA, 'build'))
TARGET = os.path.abspath(os.path.join(DATA, 'local'))

NODE_MODULES = [
    'commander', 'start-stop-daemon', 'winston', 'sockjs', 'helenus',
    'sockjs-client-ws', 'coffee-script', 'node-uuid', 'browserify@1.16.4', 'uglify-js2',
    'passport', 'passport-github', 'express', 'nodeunit', 'validator', 'async',
    'password-hash', 'emailjs', 'cookies', 'htmlparser', 'mime', 'pty.js', 'posix',
    'mkdirp', 'walk', 'temp', 'portfinder', 'googlediff', 'formidable@latest'
    ]

PYTHON_PACKAGES = [
    'ipython','readline', # a usable command line  (ipython uses readline)
    'python-daemon',      # daemonization of python modules
    'paramiko',           # ssh2 implementation in python
    'cql',                # interface to Cassandra
    ]

if not os.path.exists(BUILD):
    os.makedirs(BUILD)

os.environ['PATH'] = os.path.join(TARGET, 'bin') + ':' + os.environ['PATH']
os.environ['LD_LIBRARY_PATH'] = os.path.join(TARGET, 'lib') + ':' + os.environ.get('LD_LIBRARY_PATH','')

# number of cpus<commit_msg>Add package needed for markdown decorator support.<commit_after>#!/usr/bin/env python

"""
Building the main components of sagews from source, ensuring that all
important (usually security-related) options are compiled in.

The components of sagews are:

    * python -- language (used mainly for admin control and sage)
    * node -- language (for web server)
    * nginx -- static web server
    * haproxy -- proxy and load ballancer
    * tinc -- p2p vpn
    * protobuf -- google's messaging api and format
    * cassandra -- distributed p2p database
    * sage (which we do not build here, yet -- perhaps we should)

This supports OS X and Ubuntu 12.04 on AMD and Intel.
"""

import logging, os, shutil, subprocess, sys, time

# Enable logging
logging.basicConfig()
log = logging.getLogger('')
log.setLevel(logging.DEBUG)   # WARNING, INFO

OS     = os.uname()[0]
PWD    = os.path.abspath('.')
DATA   = os.path.abspath('data')
SRC    = os.path.abspath('src')
PATCHES= os.path.join(SRC, 'patches')
BUILD  = os.path.abspath(os.path.join(DATA, 'build'))
TARGET = os.path.abspath(os.path.join(DATA, 'local'))

NODE_MODULES = [
    'commander', 'start-stop-daemon', 'winston', 'sockjs', 'helenus',
    'sockjs-client-ws', 'coffee-script', 'node-uuid', 'browserify@1.16.4', 'uglify-js2',
    'passport', 'passport-github', 'express', 'nodeunit', 'validator', 'async',
    'password-hash', 'emailjs', 'cookies', 'htmlparser', 'mime', 'pty.js', 'posix',
    'mkdirp', 'walk', 'temp', 'portfinder', 'googlediff', 'formidable@latest'
    ]

PYTHON_PACKAGES = [
    'ipython','readline', # a usable command line  (ipython uses readline)
    'python-daemon',      # daemonization of python modules
    'paramiko',           # ssh2 implementation in python
    'cql',                # interface to Cassandra
    'markdown2'
    ]

if not os.path.exists(BUILD):
    os.makedirs(BUILD)

os.environ['PATH'] = os.path.join(TARGET, 'bin') + ':' + os.environ['PATH']
os.environ['LD_LIBRARY_PATH'] = os.path.join(TARGET, 'lib') + ':' + os.environ.get('LD_LIBRARY_PATH','')

# number of cpus
{'fraction_non_alphanumeric': 0.157997557997558, 'fraction_numerical': 0.006593406593406593, 'mean_word_length': 3.6282485875706216, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1016994', 'n_tokens_mistral': 1434, 'n_tokens_neox': 1389, 'n_words': 372}