<commit_before># coding: utf-8
# Copyright 2013 The Font Bakery Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# See AUTHORS.txt for the list of Authors and LICENSE.txt for the License.

from __future__ import print_function
import os
import sys
import glob
import subprocess
import codecs
from flask.ext.rq import job
import plistlib
from .utils import RedisFd
import re
import yaml
from fontTools import ttLib
from fontaine.ext.subsets import Extension as SubsetExtension


def run(command, cwd, log):
    """ Wrapper for subprocess.Popen with custom logging support.

        :param command: shell command to run, required
        :param cwd: - current working dir, required
        :param log: - logging object with .write() method, required

    """
    # print the command on the worker console
    print("[%s]:%s" % (cwd, command))
    # log the command
    log.write('\n$ %s\n' % command)
    # Start the command
    env = os.environ.copy()
    env.update({'PYTHONPATH': os.pathsep.join(sys.path)})
    p = subprocess.Popen(command, shell=True, cwd=cwd, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE, close_fds=True, env=env)
    while True:
        # Read output and errors
        stdout = p.stdout.readline()
        stderr = p.stderr.readline()
        # Log output
        log.write(stdout)
        # Log error
        if stderr:
            # print the error on the worker console
            print(stderr, end='')
            # log error
            log.write(stderr, prefix='Error: ')
        # If no output and process no longer running, stop
        if not stdout and not stderr and p.poll() is not None:
            break
    # if the command did not exit cleanly (with returncode 0)
    if p.returncode:
        msg = 'Fatal: Exited with return code %s \n' % p.returncode
        # Log the exit status
        log.write(msg)
        # Raise an error on the worker
        raise StandardError(msg)


def prun(command, cwd, log=None):
    """
    Wrapper for subprocess.Popen that capture output and return as result

        :param command: shell command to run
        :param cwd: current working dir
        :param log: loggin object with .write() method

    """
    # print the command on the worker console
    print("[%s]:%s" % (cwd, command))
    env = os.environ.copy()
    env.update({'PYTHONPATH': os.pathsep.join(sys.path)})
    p = subprocess.Popen(command, shell=True, cwd=cwd,
                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                         close_fds=True, env=env)
    stdout = p.communicate()[0]
    if log:
        log.write('$ %s' % command)
        log.write(stdout)
    return stdout


@job
def project_git_sync(project):
    """
    Sync _in git repo, or download it if it doesn't yet exist.

    :param project: :class:`~bakery.models.Project` instance
    :param log: :class:`~bakery.utils.RedisFd` as log
    """
    from .app import db, app
    project.is_ready = False
    db.session.add(project)
    db.session.commit()
    db.session.refresh(project)

    _in = os.path.join(app.config['DATA_ROOT'], '%(login)s/%(id)s.in/' % project)
    # Create the incoming repo directory (_in) if it doesn't exist
    if not os.path.exists(_in):
        # log.write('Creating Incoming Directory\n', prefix='### ')
        prun('mkdir -p %s' % _in, cwd=app.config['DATA_ROOT'])
    # Update _in if it already exists with a .git directory
    if os.path.exists(os.path.join(_in, '.git')):
        # log.write('Sync Git Repository\n', prefix='### ')
        # remove anything in the _in directory that isn't checked in
        prun('git reset --hard', cwd=_in)
        prun('git clean --force', cwd=_in)
        # pull from origin master branch
        prun('git pull origin master', cwd=_in)
    # Since it doesn't exist as a git repo, get the _in repo
    else:
        # clone the repository
        # log.write('Copying Git Repository\n', prefix='### ')
        try:
            # TODO in the future, use http://schacon.github.io/git/git-ls-remote.html to validate the URL string
            # http://stackoverflow.com/questions/9610131/how-to-check-the-validity-of-a-remote-git-repository-url
            prun('git clone --depth=100 --quiet --branch=master %(clone)s .' % project, cwd=_in)
        # if the clone action didn't work, just copy it
        except:
            # if this is a file URL, copy the files, and set up the _in directory as a git repo
            if project.clone[:7] == "file://":
                # cp recursively, keeping all attributes, not following symlinks, not deleting existing files, verbosely
                prun('cp -a %(clone)s .' % project, cwd=_in)
                #
                prun('git init .', cwd=_in)
                prun('git add *', cwd=_in)
                msg = "Initial commit made automatically by Font Bakery"
                prun('git commit -a -m "%s"' % msg, cwd=_in)
        # Now we have it, create an initial project state
        finally:
            config = project.config

    # set project state as ready after sync is done
    project.is_ready = True
    db.session.add(project)
    db.session.commit()


def copy_ufo_files(project, build, log):
    from .app import app
    config = project.config

    param = {'login': project.login, 'id': project.id,
             'revision': build.revision, 'build': build.id}

    _in = os.path.join(app.config['DATA_ROOT'], '%(login)s/%(id)s.in/' % param)
    _out = os.path.join(app.config['DATA_ROOT'], '%(login)s/%(id)s.out/%(build)s.%(revision)s/' % param)
    _out_src = os.path.join(app.config['DATA_ROOT'], '%(login)s/%(id)s.out/%(build)s.%(revision)s/sources/' % param)

    log.write('Copy [and Rename] UFOs\n', prefix='### ')

    # Set the familyName
    if config['state'].get('familyname', None):
        familyName = config['state']['familyname']
    else:
        familyName = False

    # Copy UFO files from git repo to _out_src [renaming their filename and metadata]
    ufo_dirs = []
    for x in config['state'].get('process_files', []):
        if x.endswith('.ufo'):
            ufo_dirs.append(x)

    for _in_ufo in ufo_dirs:
        # Decide the incoming filepath
        _in_ufo_path = os.path.join(_in, _in_ufo)
        # Read the _in_ufo fontinfo.plist
        _in_ufoPlist = os.path.join(_in_ufo_path, 'fontinfo.plist')
        _in_ufoFontInfo = plistlib.readPlist(_in_ufoPlist)
        # Get the styleName
        styleName = _in_ufoFontInfo['styleName']
        # Always have a regular style
        if styleName == 'Normal':
            styleName = 'Regular'
        # Get the familyName, if its not set
        if not familyName:
            familyName = _in_ufoFontInfo.get('openTypeNamePreferredFamilyName', '') \
                or _in_ufoFontInfo.get('familyName', '')
            if not familyName:
                log.write('Please set familyName or openTypeNamePreferredFamilyName to next process', prefix='### ')
                raise Exception('Please set familyName or openTypeNamePreferredFamilyName to next process')

        # Remove whitespace from names
        styleNameNoWhitespace = re.sub(r'\s', '', styleName)
        familyNameNoWhitespace = re.sub(r'\s', '', familyName)<commit_msg>more descriptive exception for familyName #297<commit_after># coding: utf-8
# Copyright 2013 The Font Bakery Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# See AUTHORS.txt for the list of Authors and LICENSE.txt for the License.

from __future__ import print_function
import os
import sys
import glob
import subprocess
import codecs
from flask.ext.rq import job
import plistlib
from .utils import RedisFd
import re
import yaml
from fontTools import ttLib
from fontaine.ext.subsets import Extension as SubsetExtension


def run(command, cwd, log):
    """ Wrapper for subprocess.Popen with custom logging support.

        :param command: shell command to run, required
        :param cwd: - current working dir, required
        :param log: - logging object with .write() method, required

    """
    # print the command on the worker console
    print("[%s]:%s" % (cwd, command))
    # log the command
    log.write('\n$ %s\n' % command)
    # Start the command
    env = os.environ.copy()
    env.update({'PYTHONPATH': os.pathsep.join(sys.path)})
    p = subprocess.Popen(command, shell=True, cwd=cwd, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE, close_fds=True, env=env)
    while True:
        # Read output and errors
        stdout = p.stdout.readline()
        stderr = p.stderr.readline()
        # Log output
        log.write(stdout)
        # Log error
        if stderr:
            # print the error on the worker console
            print(stderr, end='')
            # log error
            log.write(stderr, prefix='Error: ')
        # If no output and process no longer running, stop
        if not stdout and not stderr and p.poll() is not None:
            break
    # if the command did not exit cleanly (with returncode 0)
    if p.returncode:
        msg = 'Fatal: Exited with return code %s \n' % p.returncode
        # Log the exit status
        log.write(msg)
        # Raise an error on the worker
        raise StandardError(msg)


def prun(command, cwd, log=None):
    """
    Wrapper for subprocess.Popen that capture output and return as result

        :param command: shell command to run
        :param cwd: current working dir
        :param log: loggin object with .write() method

    """
    # print the command on the worker console
    print("[%s]:%s" % (cwd, command))
    env = os.environ.copy()
    env.update({'PYTHONPATH': os.pathsep.join(sys.path)})
    p = subprocess.Popen(command, shell=True, cwd=cwd,
                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                         close_fds=True, env=env)
    stdout = p.communicate()[0]
    if log:
        log.write('$ %s' % command)
        log.write(stdout)
    return stdout


@job
def project_git_sync(project):
    """
    Sync _in git repo, or download it if it doesn't yet exist.

    :param project: :class:`~bakery.models.Project` instance
    :param log: :class:`~bakery.utils.RedisFd` as log
    """
    from .app import db, app
    project.is_ready = False
    db.session.add(project)
    db.session.commit()
    db.session.refresh(project)

    _in = os.path.join(app.config['DATA_ROOT'], '%(login)s/%(id)s.in/' % project)
    # Create the incoming repo directory (_in) if it doesn't exist
    if not os.path.exists(_in):
        # log.write('Creating Incoming Directory\n', prefix='### ')
        prun('mkdir -p %s' % _in, cwd=app.config['DATA_ROOT'])
    # Update _in if it already exists with a .git directory
    if os.path.exists(os.path.join(_in, '.git')):
        # log.write('Sync Git Repository\n', prefix='### ')
        # remove anything in the _in directory that isn't checked in
        prun('git reset --hard', cwd=_in)
        prun('git clean --force', cwd=_in)
        # pull from origin master branch
        prun('git pull origin master', cwd=_in)
    # Since it doesn't exist as a git repo, get the _in repo
    else:
        # clone the repository
        # log.write('Copying Git Repository\n', prefix='### ')
        try:
            # TODO in the future, use http://schacon.github.io/git/git-ls-remote.html to validate the URL string
            # http://stackoverflow.com/questions/9610131/how-to-check-the-validity-of-a-remote-git-repository-url
            prun('git clone --depth=100 --quiet --branch=master %(clone)s .' % project, cwd=_in)
        # if the clone action didn't work, just copy it
        except:
            # if this is a file URL, copy the files, and set up the _in directory as a git repo
            if project.clone[:7] == "file://":
                # cp recursively, keeping all attributes, not following symlinks, not deleting existing files, verbosely
                prun('cp -a %(clone)s .' % project, cwd=_in)
                #
                prun('git init .', cwd=_in)
                prun('git add *', cwd=_in)
                msg = "Initial commit made automatically by Font Bakery"
                prun('git commit -a -m "%s"' % msg, cwd=_in)
        # Now we have it, create an initial project state
        finally:
            config = project.config

    # set project state as ready after sync is done
    project.is_ready = True
    db.session.add(project)
    db.session.commit()


def copy_ufo_files(project, build, log):
    from .app import app
    config = project.config

    param = {'login': project.login, 'id': project.id,
             'revision': build.revision, 'build': build.id}

    _in = os.path.join(app.config['DATA_ROOT'], '%(login)s/%(id)s.in/' % param)
    _out = os.path.join(app.config['DATA_ROOT'], '%(login)s/%(id)s.out/%(build)s.%(revision)s/' % param)
    _out_src = os.path.join(app.config['DATA_ROOT'], '%(login)s/%(id)s.out/%(build)s.%(revision)s/sources/' % param)

    log.write('Copy [and Rename] UFOs\n', prefix='### ')

    # Set the familyName
    if config['state'].get('familyname', None):
        familyName = config['state']['familyname']
    else:
        familyName = False

    # Copy UFO files from git repo to _out_src [renaming their filename and metadata]
    ufo_dirs = []
    for x in config['state'].get('process_files', []):
        if x.endswith('.ufo'):
            ufo_dirs.append(x)

    for _in_ufo in ufo_dirs:
        # Decide the incoming filepath
        _in_ufo_path = os.path.join(_in, _in_ufo)
        # Read the _in_ufo fontinfo.plist
        _in_ufoPlist = os.path.join(_in_ufo_path, 'fontinfo.plist')
        _in_ufoFontInfo = plistlib.readPlist(_in_ufoPlist)
        # Get the styleName
        styleName = _in_ufoFontInfo['styleName']
        # Always have a regular style
        if styleName == 'Normal':
            styleName = 'Regular'
        # Get the familyName, if its not set
        if not familyName:
            familyName = _in_ufoFontInfo.get('openTypeNamePreferredFamilyName', '') \
                or _in_ufoFontInfo.get('familyName', '')
            if not familyName:
                log.write('Please set openTypeNamePreferredFamilyName or familyName in %s fontinfo.plist and run another bake process.' % _in_ufo, prefix='### ')
                raise Exception('Please set openTypeNamePreferredFamilyName or familyName in %s fontinfo.plist and run another bake process.' % _in_ufo)

        # Remove whitespace from names
        styleNameNoWhitespace = re.sub(r'\s', '', styleName)
        familyNameNoWhitespace = re.sub(r'\s', '', familyName)
{'fraction_non_alphanumeric': 0.11391920352287957, 'fraction_numerical': 0.0029995532580254004, 'mean_word_length': 2.5629831741700775, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4068686', 'n_tokens_mistral': 4822, 'n_tokens_neox': 4694, 'n_words': 1500}