<commit_before>from __future__ import print_function

import os, json, socket

join = os.path.join

def cmd(s):
    print(s)
    if os.system(s):
        raise RuntimeError

def chdir():
    os.chdir(os.path.split(os.path.abspath(__file__))[0])

def base_url(port=None):
    info_file = join(os.environ['SMC'], 'info.json')
    info = json.loads(open(info_file).read())
    if port is None:
        write_base_url = False
    else:
        write_base_url = True
        port = get_ports()['hub']
    base_url = "/{project_id}/port/{port}".format(project_id=info['project_id'], port=port)
    if write_base_url:
        open("../../data/base_url",'w').write(base_url)
    return base_url

def get_open_port():    # http://stackoverflow.com/questions/2838244/get-open-tcp-port-in-python
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(("",0))
    s.listen(1)
    port = s.getsockname()[1]
    s.close()
    return port

def get_ports():
    P = os.path.split(os.path.abspath(__file__))[0]
    path = join(P, 'ports')
    if not os.path.exists(path):
        os.mkdir(path)
    ports = {'hub':0, 'hub-api':0, 'hub-share':0, 'hub-share-2':0}
    for x in ports.keys():
        file = join(path, x)
        if os.path.exists(file):
            ports[x] = int(open(file).read())
        else:
            ports[x] = get_open_port()
            open(file,'w').write(str(ports[x]))
    return ports
<commit_msg>woops; got the previous change to dev/project/util.py backward<commit_after>from __future__ import print_function

import os, json, socket

join = os.path.join

def cmd(s):
    print(s)
    if os.system(s):
        raise RuntimeError

def chdir():
    os.chdir(os.path.split(os.path.abspath(__file__))[0])

def base_url(port=None):
    info_file = join(os.environ['SMC'], 'info.json')
    info = json.loads(open(info_file).read())
    if port is None:
        write_base_url = True
        port = get_ports()['hub']
    else:
        write_base_url = False
    base_url = "/{project_id}/port/{port}".format(project_id=info['project_id'], port=port)
    if write_base_url:
        open("../../data/base_url",'w').write(base_url)
    return base_url

def get_open_port():    # http://stackoverflow.com/questions/2838244/get-open-tcp-port-in-python
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(("",0))
    s.listen(1)
    port = s.getsockname()[1]
    s.close()
    return port

def get_ports():
    P = os.path.split(os.path.abspath(__file__))[0]
    path = join(P, 'ports')
    if not os.path.exists(path):
        os.mkdir(path)
    ports = {'hub':0, 'hub-api':0, 'hub-share':0, 'hub-share-2':0}
    for x in ports.keys():
        file = join(path, x)
        if os.path.exists(file):
            ports[x] = int(open(file).read())
        else:
            ports[x] = get_open_port()
            open(file,'w').write(str(ports[x]))
    return ports

{'fraction_non_alphanumeric': 0.16775975146703487, 'fraction_numerical': 0.01173627890921643, 'mean_word_length': 2.83841059602649, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7356273', 'n_tokens_mistral': 1072, 'n_tokens_neox': 1061, 'n_words': 191}