<commit_before># coding: utf-8
import sys
import os
import subprocess

realpath = os.path.dirname(os.path.realpath(__file__))
datadir = "../bin"

def word2vec(train, output, size=100, window=5, sample=0, hs=1, negative=0, threads=4,
             min_count=5, alpha=0.025, debug=2, binary=1, cbow=0,
             save_vocab=None, read_vocab=None, verbose=False):
    process = [realpath + '/' + datadir + '/' + 'word2vec']
    args = ['-train', '-output', '-size', '-window', '-sample', '-hs', '-negative', '-threads',
            '-min-count', '-alpha', '-debug', '-binary', '-cbow']
    values = [train, output, size, window, sample, hs, negative, threads,
              min_count, alpha, debug, binary, cbow]
    for arg, value in zip(args, values):
        process.append(arg)
        process.append(str(value))
    if save_vocab is not None:
        process.append('-save-vocab')
        process.append(str(save_vocab))
    if read_vocab is not None:
        process.append('-read-vocab')
        process.append(str(read_vocab))

    proc = subprocess.Popen(process, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    if verbose:
        for line in proc.stdout:
            sys.stdout.write(line)
            sys.stdout.flush()

    out, err = proc.communicate()
    if 'ERROR:' in out:
        raise Exception(out)


def word2clusters(train, output, classes, size=100, window=5, sample=0, hs=1, negative=0, threads=4,
                  min_count=5, alpha=0.025, debug=2, binary=0, cbow=0,
                  save_vocab=None, read_vocab=None, verbose=False):
    process = [realpath + datadir + '/' +'word2vec']
    args = ['-train', '-output', '-size', '-window', '-sample', '-hs', '-negative', '-threads',
            '-min-count', '-alpha', '-classes', '-debug', '-binary', '-cbow']
    values = [train, output, size, window, sample, hs, negative, threads,
              min_count, alpha, classes, debug, binary, cbow]
    for arg, value in zip(args, values):
        process.append(arg)
        process.append(str(value))
    if save_vocab is not None:
        process.append('-save-vocab')
        process.append(str(save_vocab))
    if read_vocab is not None:
        process.append('-read-vocab')
        process.append(str(read_vocab))

    proc = subprocess.Popen(process, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    if verbose:
        for line in proc.stdout:
            sys.stdout.write(line)
            sys.stdout.flush()

    out, err = proc.communicate()
    if 'ERROR:' in out:
        raise Exception(out)


def word2phase(train, output, min_count=5, threshold=100, debug=2, verbose=False):
    process = [realpath + datadir + '/' +'word2phrase']
    args = ['-train', '-output', '-min-count', '-threshold', '-debug']<commit_msg>using os.path.join<commit_after># coding: utf-8
import sys
import os
import subprocess

realpath = os.path.dirname(os.path.realpath(__file__))
datadir = "../bin"

def word2vec(train, output, size=100, window=5, sample=0, hs=1, negative=0, threads=4,
             min_count=5, alpha=0.025, debug=2, binary=1, cbow=0,
             save_vocab=None, read_vocab=None, verbose=False):
    process = [os.path.join(realpath, datadir, word2vec)]
    args = ['-train', '-output', '-size', '-window', '-sample', '-hs', '-negative', '-threads',
            '-min-count', '-alpha', '-debug', '-binary', '-cbow']
    values = [train, output, size, window, sample, hs, negative, threads,
              min_count, alpha, debug, binary, cbow]
    for arg, value in zip(args, values):
        process.append(arg)
        process.append(str(value))
    if save_vocab is not None:
        process.append('-save-vocab')
        process.append(str(save_vocab))
    if read_vocab is not None:
        process.append('-read-vocab')
        process.append(str(read_vocab))

    proc = subprocess.Popen(process, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    if verbose:
        for line in proc.stdout:
            sys.stdout.write(line)
            sys.stdout.flush()

    out, err = proc.communicate()
    if 'ERROR:' in out:
        raise Exception(out)


def word2clusters(train, output, classes, size=100, window=5, sample=0, hs=1, negative=0, threads=4,
                  min_count=5, alpha=0.025, debug=2, binary=0, cbow=0,
                  save_vocab=None, read_vocab=None, verbose=False):
    process = [os.path.join(realpath, datadir, word2vec)]
    args = ['-train', '-output', '-size', '-window', '-sample', '-hs', '-negative', '-threads',
            '-min-count', '-alpha', '-classes', '-debug', '-binary', '-cbow']
    values = [train, output, size, window, sample, hs, negative, threads,
              min_count, alpha, classes, debug, binary, cbow]
    for arg, value in zip(args, values):
        process.append(arg)
        process.append(str(value))
    if save_vocab is not None:
        process.append('-save-vocab')
        process.append(str(save_vocab))
    if read_vocab is not None:
        process.append('-read-vocab')
        process.append(str(read_vocab))

    proc = subprocess.Popen(process, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    if verbose:
        for line in proc.stdout:
            sys.stdout.write(line)
            sys.stdout.flush()

    out, err = proc.communicate()
    if 'ERROR:' in out:
        raise Exception(out)


def word2phase(train, output, min_count=5, threshold=100, debug=2, verbose=False):
    process = [realpath + datadir + '/' +'word2phrase']
    args = ['-train', '-output', '-min-count', '-threshold', '-debug']
{'fraction_non_alphanumeric': 0.16046259486808817, 'fraction_numerical': 0.01590169859053126, 'mean_word_length': 2.8598326359832638, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3284399', 'n_tokens_mistral': 1883, 'n_tokens_neox': 1824, 'n_words': 434}