<commit_before>import base64
import hashlib
import json
import os.path
import random
import subprocess
import sys
import tempfile
from types import IntType, StringType

from werkzeug import Request

from sdklib import node_path



def file_data(filename):
    '''
    Get file data in base64 for upload
    '''
    with open(filename, 'r') as fh:
        data = fh.read()
        data64 = base64.b64encode(data)
        return data64


def hash(content):
    h = hashlib.sha1()
    h.update(content)
    return h


def compile_coffeescript(filename):
    return node_command(['coffee-script', 'bin', 'coffee'], ['--print', filename])


def compile_less(filename):
    return node_command(['less', 'bin', 'lessc'], [filename])


def compile_stylus(filename):
    return node_command(['stylus', 'bin', 'stylus'], stdin=filename)


def node_command(command, args=[], stdin=None):
    '''
    convenience method to call node and additional node commands installed
    from npm such as coffeescript or less.

    command is the relative path list to the command
    args are the additional parameters to pass after the command
    '''
    # these are 64bit binaries
    if sys.platform.startswith('linux'):
        node_name = 'node-linux'
    elif sys.platform.startswith('darwin'):
        node_name = 'node-darwin'
    node = os.path.join(node_path, node_name)
    command = os.path.join(node_path, *command)
    output = tempfile.TemporaryFile()
    cmd_args = [node, command] + args
    if stdin is None:
        subprocess.call(cmd_args, stdout=output)
    else:
        stdinfh = open(stdin, 'r')
        subprocess.call(cmd_args, stdin=stdinfh, stdout=output)
        stdinfh.close()
    output.seek(0)
    return output.read()


def api_parse(default):
    '''
    Decorator to take web response data (at this point it should already be 
    parsed as a JSON string) and remove the 'success' and 'result' keys, 
    returning the default parameter if 'success' if False
    '''
    def json_result(*args, **kwargs):
        result = fn(*args, **kwargs)
        return json.loads(result.content)
    return json_result


class PathUtil(object):
    '''
    Used to get path names for use in templates.

    Usage:
    
    path = PathUtil(wsgi.environ)
    path() full path
    path(n) n segment, 0 based index
    '''

    def __init__(self, environ):
        path = environ.get('PATH_INFO')
        self.path = path
        self.paths = path.strip('/').split('/')
        self.placeholders = {}

    def __call__(self, index=None):
        if index is None:
            return self.path
        elif type(index) is IntType and len(self.paths) > index:
            return self.paths[index]
        elif type(index) is StringType:
            return self.placeholders.get(index, '')
        return ''

    def add_placeholders(self, placeholders):
        self.placeholders = {
            k: v for k, v in placeholders.items() if not k.startswith('_')
        }


class GetUtil(object):
    '''
    Used to query for GET variables
    '''

    def __init__(self, environ):
        req = Request(environ)
        self.args = req.args

    def __call__(self, name):
        return self.args.get(name, '')

    def list(self, name):
        return self.args.getlist(name)
        

class StaticUtil(object):
    '''
    Used to output the static file path with a cache controlling GET

    Locally this always returns current timestamp for GET['t'].
    Production should be unique per upload.
    '''

    def __call__(self, path):
        cache = '%10x' % random.randrange(256**5)
        if not path.startswith('/'):
            path = '/' + path
        full_path = '/static{path}?t={cache}'.format(path=path, cache=cache)
        return full_path<commit_msg>static cache GET changed to c<commit_after>import base64
import hashlib
import json
import os.path
import random
import subprocess
import sys
import tempfile
from types import IntType, StringType

from werkzeug import Request

from sdklib import node_path



def file_data(filename):
    '''
    Get file data in base64 for upload
    '''
    with open(filename, 'r') as fh:
        data = fh.read()
        data64 = base64.b64encode(data)
        return data64


def hash(content):
    h = hashlib.sha1()
    h.update(content)
    return h


def compile_coffeescript(filename):
    return node_command(['coffee-script', 'bin', 'coffee'], ['--print', filename])


def compile_less(filename):
    return node_command(['less', 'bin', 'lessc'], [filename])


def compile_stylus(filename):
    return node_command(['stylus', 'bin', 'stylus'], stdin=filename)


def node_command(command, args=[], stdin=None):
    '''
    convenience method to call node and additional node commands installed
    from npm such as coffeescript or less.

    command is the relative path list to the command
    args are the additional parameters to pass after the command
    '''
    # these are 64bit binaries
    if sys.platform.startswith('linux'):
        node_name = 'node-linux'
    elif sys.platform.startswith('darwin'):
        node_name = 'node-darwin'
    node = os.path.join(node_path, node_name)
    command = os.path.join(node_path, *command)
    output = tempfile.TemporaryFile()
    cmd_args = [node, command] + args
    if stdin is None:
        subprocess.call(cmd_args, stdout=output)
    else:
        stdinfh = open(stdin, 'r')
        subprocess.call(cmd_args, stdin=stdinfh, stdout=output)
        stdinfh.close()
    output.seek(0)
    return output.read()


def api_parse(default):
    '''
    Decorator to take web response data (at this point it should already be 
    parsed as a JSON string) and remove the 'success' and 'result' keys, 
    returning the default parameter if 'success' if False
    '''
    def json_result(*args, **kwargs):
        result = fn(*args, **kwargs)
        return json.loads(result.content)
    return json_result


class PathUtil(object):
    '''
    Used to get path names for use in templates.

    Usage:
    
    path = PathUtil(wsgi.environ)
    path() full path
    path(n) n segment, 0 based index
    '''

    def __init__(self, environ):
        path = environ.get('PATH_INFO')
        self.path = path
        self.paths = path.strip('/').split('/')
        self.placeholders = {}

    def __call__(self, index=None):
        if index is None:
            return self.path
        elif type(index) is IntType and len(self.paths) > index:
            return self.paths[index]
        elif type(index) is StringType:
            return self.placeholders.get(index, '')
        return ''

    def add_placeholders(self, placeholders):
        self.placeholders = {
            k: v for k, v in placeholders.items() if not k.startswith('_')
        }


class GetUtil(object):
    '''
    Used to query for GET variables
    '''

    def __init__(self, environ):
        req = Request(environ)
        self.args = req.args

    def __call__(self, name):
        return self.args.get(name, '')

    def list(self, name):
        return self.args.getlist(name)
        

class StaticUtil(object):
    '''
    Used to output the static file path with a cache controlling GET

    Locally this always returns current timestamp for GET['t'].
    Production should be unique per upload.
    '''

    def __call__(self, path):
        cache = '%10x' % random.randrange(256**5)
        if not path.startswith('/'):
            path = '/' + path
        full_path = '/static{path}?c={cache}'.format(path=path, cache=cache)
        return full_path
{'fraction_non_alphanumeric': 0.11002120890774125, 'fraction_numerical': 0.006097560975609756, 'mean_word_length': 2.7130905511811023, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2064017', 'n_tokens_mistral': 2306, 'n_tokens_neox': 2245, 'n_words': 694}