<commit_before>
"""Makefile for the preprocess project.

${common_task_list}

See `mk -h' for options.
"""

import sys
import os
from os.path import join, dirname, normpath, abspath, exists, basename
import re
import webbrowser

from mklib.common import MkError
from mklib import Task
from mklib import sh



class bugs(Task):
    """Open bug database page."""
    def make(self):
        webbrowser.open("http://code.google.com/p/preprocess/issues/list")

class site(Task):
    """Open the Google Code project page."""
    def make(self):
        webbrowser.open("http://code.google.com/p/preprocess/")

class play(Task):
    def make(self):
        sh.run("python setup.py install")
        sh.run("/Library/Frameworks/Python.framework/Versions/2.5/bin/preprocess foo.txt")


class clean(Task):
    """Clean generated files and dirs."""
    def make(self):
        patterns = [
            "dist",
            "build",
            "MANIFEST",
            "*.pyc",
            "lib/*.pyc",
        ]
        for pattern in patterns:
            p = join(self.dir, pattern)
            for path in glob(p):
                sh.rm(path, log=self.log)

class test(Task):
    """Run all tests (except known failures)."""
    def make(self):
        for ver, python in self._gen_pythons():
            if ver < (2,3):
                # Don't support Python < 2.3.
                continue
            elif ver >= (3,0):
                # Don't (yet) support Python 3.
                continue
            ver_str = "%s.%s" % ver
            print "-- test with Python %s (%s)" % (ver_str, python)
            assert ' ' not in python
            sh.run_in_dir("%s test.py" % python,
                          join(self.dir, "test"))

    def _python_ver_from_python(self, python):
        assert ' ' not in python
        o = os.popen('''%s -c "import sys; print(sys.version)"''' % python)
        ver_str = o.read().strip()
        ver_bits = re.split("\.|[^\d]", ver_str, 2)[:2]
        ver = tuple(map(int, ver_bits))
        return ver

    def _gen_python_names(self):
        yield "python"
        for ver in [(2,3), (2,4), (2,5), (2,6), (2,7), (3,0), (3,1)]:
            yield "python%d.%d" % ver

    def _gen_pythons(self):
        sys.path.insert(0, join(self.dir, "externals", "which"))
        import which
        python_from_ver = {}
        for name in self._gen_python_names():
            for python in which.whichall(name):
                ver = self._python_ver_from_python(python)
                if ver not in python_from_ver:
                    python_from_ver[ver] = python
        for ver, python in sorted(python_from_ver.items()):
            yield ver, python
        

class sdist(Task):
    """python setup.py sdist"""
    def make(self):
        sh.run_in_dir("%spython setup.py sdist -f --formats zip"
                        % _setup_command_prefix(),
                      self.dir, self.log.debug)

class webdist(Task):
    """Build a web dist package for trentm.com/projects/
    
    "Web dist" packages are zip files with '.web' extension. All files in
    the zip must be under a dir named after the project. There must be a
    webinfo.xml file at <projname>/webinfo.xml. This file is "defined"
    by the parsing in trentm.com/build.py.
    """ 
    deps = ["docs"]

    def results(self):
        yield join(self.dir, "dist", "preprocess-%s.web" % _get_version())

    def make(self):
        assert sys.platform != "win32", "'webdist' not implemented for win32"
        build_dir = join(self.dir, "build", "webdist")
        zip_dir = join(build_dir, "preprocess")
        if exists(build_dir):
            sh.rm(build_dir)
        os.makedirs(zip_dir)

        # Copy the webdist bits to the build tree.
        manifest = [
            "src/trentm.com/project-info.xml",
            "src/trentm.com/index.markdown",
            "LICENSE.txt",
            "lib/preprocess.py",
            "src/trentm.com/logo.jpg",
        ]
        for src in manifest:
            sh.cp(src, dstdir=zip_dir, log=self.log.info)

        # Zip up the webdist contents.
        dist_dir = join(self.dir, "dist")
        bit = abspath(join(dist_dir, "preprocess-%s.web" % _get_version()))
        if exists(bit):
            os.remove(bit)
        if not exists(dist_dir):
            os.makedirs(dist_dir)
        sh.run_in_dir("zip -r %s preprocess" % bit, build_dir, self.log.info)

class pypi_upload(Task):
    """Upload release to pypi."""
    def make(self):
        tasks = (sys.platform == "win32"
                 and "sdist --formats zip bdist_wininst upload"<commit_msg>fix for 'mk webdist'<commit_after>
"""Makefile for the preprocess project.

${common_task_list}

See `mk -h' for options.
"""

import sys
import os
from os.path import join, dirname, normpath, abspath, exists, basename
import re
import webbrowser

from mklib.common import MkError
from mklib import Task
from mklib import sh



class bugs(Task):
    """Open bug database page."""
    def make(self):
        webbrowser.open("http://code.google.com/p/preprocess/issues/list")

class site(Task):
    """Open the Google Code project page."""
    def make(self):
        webbrowser.open("http://code.google.com/p/preprocess/")

class play(Task):
    def make(self):
        sh.run("python setup.py install")
        sh.run("/Library/Frameworks/Python.framework/Versions/2.5/bin/preprocess foo.txt")


class clean(Task):
    """Clean generated files and dirs."""
    def make(self):
        patterns = [
            "dist",
            "build",
            "MANIFEST",
            "*.pyc",
            "lib/*.pyc",
        ]
        for pattern in patterns:
            p = join(self.dir, pattern)
            for path in glob(p):
                sh.rm(path, log=self.log)

class test(Task):
    """Run all tests (except known failures)."""
    def make(self):
        for ver, python in self._gen_pythons():
            if ver < (2,3):
                # Don't support Python < 2.3.
                continue
            elif ver >= (3,0):
                # Don't (yet) support Python 3.
                continue
            ver_str = "%s.%s" % ver
            print "-- test with Python %s (%s)" % (ver_str, python)
            assert ' ' not in python
            sh.run_in_dir("%s test.py" % python,
                          join(self.dir, "test"))

    def _python_ver_from_python(self, python):
        assert ' ' not in python
        o = os.popen('''%s -c "import sys; print(sys.version)"''' % python)
        ver_str = o.read().strip()
        ver_bits = re.split("\.|[^\d]", ver_str, 2)[:2]
        ver = tuple(map(int, ver_bits))
        return ver

    def _gen_python_names(self):
        yield "python"
        for ver in [(2,3), (2,4), (2,5), (2,6), (2,7), (3,0), (3,1)]:
            yield "python%d.%d" % ver

    def _gen_pythons(self):
        sys.path.insert(0, join(self.dir, "externals", "which"))
        import which
        python_from_ver = {}
        for name in self._gen_python_names():
            for python in which.whichall(name):
                ver = self._python_ver_from_python(python)
                if ver not in python_from_ver:
                    python_from_ver[ver] = python
        for ver, python in sorted(python_from_ver.items()):
            yield ver, python
        

class sdist(Task):
    """python setup.py sdist"""
    def make(self):
        sh.run_in_dir("%spython setup.py sdist -f --formats zip"
                        % _setup_command_prefix(),
                      self.dir, self.log.debug)

class webdist(Task):
    """Build a web dist package for trentm.com/projects/
    
    "Web dist" packages are zip files with '.web' extension. All files in
    the zip must be under a dir named after the project. There must be a
    webinfo.xml file at <projname>/webinfo.xml. This file is "defined"
    by the parsing in trentm.com/build.py.
    """ 
    deps = ["docs"]

    def results(self):
        yield join(self.dir, "dist", "preprocess-%s.web" % _get_version())

    def make(self):
        assert sys.platform != "win32", "'webdist' not implemented for win32"
        build_dir = join(self.dir, "build", "webdist")
        zip_dir = join(build_dir, "preprocess")
        if exists(build_dir):
            sh.rm(build_dir)
        os.makedirs(zip_dir)

        # Copy the webdist bits to the build tree.
        manifest = [
            "src/trentm.com/project-info.xml",
            "src/trentm.com/index.markdown",
            "LICENSE.txt",
            "lib/preprocess.py",
            "src/trentm.com/logo.gif",
        ]
        for src in manifest:
            sh.cp(src, dstdir=zip_dir, log=self.log.info)

        # Zip up the webdist contents.
        dist_dir = join(self.dir, "dist")
        bit = abspath(join(dist_dir, "preprocess-%s.web" % _get_version()))
        if exists(bit):
            os.remove(bit)
        if not exists(dist_dir):
            os.makedirs(dist_dir)
        sh.run_in_dir("zip -r %s preprocess" % bit, build_dir, self.log.info)

class pypi_upload(Task):
    """Upload release to pypi."""
    def make(self):
        tasks = (sys.platform == "win32"
                 and "sdist --formats zip bdist_wininst upload"
{'fraction_non_alphanumeric': 0.13328991642244653, 'fraction_numerical': 0.006946705741886465, 'mean_word_length': 1.9935022742040285, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6381038', 'n_tokens_mistral': 2969, 'n_tokens_neox': 2886, 'n_words': 772}