<commit_before>"""
To upload a new version:
1. make clean
2. git tag a new version: git tag v1.x.x
3. python setup.py sdist
4. python setup.py sdist register upload

Note in binary files (`data_files`) from `setup()` below:

- When you do `python setup.py install` it will place the
binary files (`word2vec`, `word2phrase`, ...) in: `$(pwd)/bin`
So you have to export that to the `$PATH`
- When you do `pip install word2vec` it will place them in the correct
`{{ environtment }}/bin` directory so users dont need to do that.
"""

from setuptools import setup
from setuptools import find_packages
from  setuptools.command.install import install as _install
from Cython.Build import cythonize

import os
import sys
import subprocess

import versioneer


class install(_install):
    def run(self):
        this_dir = os.path.dirname(os.path.realpath(__file__))
        self.C_SOURCE = os.path.join(this_dir, 'word2vec', 'c')
        self.BIN_DIR = 'bin'
        if not os.path.exists(self.BIN_DIR):
            os.makedirs(self.BIN_DIR)

        if sys.platform == 'win32':
            self.compile_c('win32/word2vec.c', 'word2vec.exe')
            self.compile_c('win32/word2phrase.c', 'word2phrase.exe')
            self.compile_c('win32/distance.c', 'word2vec-distance.exe')
            self.compile_c('win32/word-analogy.c', 'word2vec-word-analogy.exe')
            self.compile_c('win32/compute-accuracy.c', 'word2vec-compute-accuracy.exe')
        else:
            self.compile_c('word2vec.c', 'word2vec')
            self.compile_c('word2phrase.c', 'word2phrase')
            self.compile_c('distance.c', 'word2vec-distance')
            self.compile_c('word-analogy.c', 'word2vec-word-analogy')
            self.compile_c('compute-accuracy.c', 'word2vec-compute-accuracy')
            self.compile_c('word2vec-sentence2vec.c', 'word2vec-doc2vec')

        _install.run(self)

    def compile_c(self, source, target):
        CC = 'gcc'

        DEFAULT_CFLAGS = '-lm -pthread -O3 -Wall -march=native -funroll-loops'
        DEFAULT_CFLAGS += ' -Wno-unused-result'
        if sys.platform == 'darwin':
            DEFAULT_CFLAGS += ' -I/usr/include/malloc'
        if sys.platform == 'win32':
            DEFAULT_CFLAGS = '-O2 -Wall -funroll-loops'
        CFLAGS = os.environ.get('CFLAGS', DEFAULT_CFLAGS)

        source_path = os.path.join(self.C_SOURCE, source)
        target_path = os.path.join(self.BIN_DIR, target)<commit_msg>Change binaries creation on windows<commit_after>"""
To upload a new version:
1. make clean
2. git tag a new version: git tag v1.x.x
3. python setup.py sdist
4. python setup.py sdist register upload

Note in binary files (`data_files`) from `setup()` below:

- When you do `python setup.py install` it will place the
binary files (`word2vec`, `word2phrase`, ...) in: `$(pwd)/bin`
So you have to export that to the `$PATH`
- When you do `pip install word2vec` it will place them in the correct
`{{ environtment }}/bin` directory so users dont need to do that.
"""

from setuptools import setup
from setuptools import find_packages
from  setuptools.command.install import install as _install
from Cython.Build import cythonize

import os
import sys
import subprocess

import versioneer


class install(_install):
    def run(self):
        this_dir = os.path.dirname(os.path.realpath(__file__))
        self.C_SOURCE = os.path.join(this_dir, 'word2vec', 'c')
        self.TARGET_DIR = 'bin'

        if not os.path.exists(self.TARGET_DIR):
            os.makedirs(self.TARGET_DIR)

        if sys.platform == 'win32':
            self.TARGET_DIR = 'Scripts'
            self.compile_c('win32/word2vec.c', 'word2vec.exe')
            self.compile_c('win32/word2phrase.c', 'word2phrase.exe')
            self.compile_c('win32/distance.c', 'word2vec-distance.exe')
            self.compile_c('win32/word-analogy.c', 'word2vec-word-analogy.exe')
            self.compile_c('win32/compute-accuracy.c', 'word2vec-compute-accuracy.exe')
        else:
            self.compile_c('word2vec.c', 'word2vec')
            self.compile_c('word2phrase.c', 'word2phrase')
            self.compile_c('distance.c', 'word2vec-distance')
            self.compile_c('word-analogy.c', 'word2vec-word-analogy')
            self.compile_c('compute-accuracy.c', 'word2vec-compute-accuracy')
            self.compile_c('word2vec-sentence2vec.c', 'word2vec-doc2vec')

        _install.run(self)

    def compile_c(self, source, target):
        CC = 'gcc'

        DEFAULT_CFLAGS = '-lm -pthread -O3 -Wall -march=native -funroll-loops'
        DEFAULT_CFLAGS += ' -Wno-unused-result'
        if sys.platform == 'darwin':
            DEFAULT_CFLAGS += ' -I/usr/include/malloc'
        if sys.platform == 'win32':
            DEFAULT_CFLAGS = '-O2 -Wall -funroll-loops'
        CFLAGS = os.environ.get('CFLAGS', DEFAULT_CFLAGS)

        source_path = os.path.join(self.C_SOURCE, source)
        target_path = os.path.join(self.TARGET_DIR, target)
{'fraction_non_alphanumeric': 0.12916246215943492, 'fraction_numerical': 0.017356205852674066, 'mean_word_length': 2.871875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2779052', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1623, 'n_words': 383}