<commit_before>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""Generic script objects.

script.py, along with config.py and log.py, represents the core of
mozharness.
"""

import codecs
import os
import platform
import pprint
import re
import shutil
import subprocess
import sys
import urllib2
import urlparse

try:
    import simplejson as json
except ImportError:
    import json

from mozharness.base.config import BaseConfig
from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
     LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL

# OSMixin {{{1
class OSMixin(object):
    """Filesystem commands and the like.

    Depends on LogMixin, ShellMixin, and a self.config of some sort.
    """
    def mkdir_p(self, path, error_level=ERROR):
        """
        Returns None for success, not None for failure
        """
        if not os.path.exists(path):
            self.info("mkdir: %s" % path)
            if not self.config.get('noop'):
                try:
                    os.makedirs(path)
                except OSError:
                    self.log("Can't create directory %s!" % path,
                             level=error_level)
                    return -1
        else:
            self.debug("mkdir_p: %s Already exists." % path)

    def rmtree(self, path, log_level=INFO, error_level=ERROR, exit_code=-1):
        """
        Returns None for success, not None for failure
        """
        self.log("rmtree: %s" % path, level=log_level)
        if os.path.exists(path):
            if not self.config.get('noop'):
                if os.path.isdir(path):
                    if self._is_windows():
                        # bug 789520: using rmdir /s instead of _rmdir_recursive
                        self.run_command(['rmdir', '/S', path])
                    else:
                        shutil.rmtree(path)
                else:
                    os.remove(path)
                if os.path.exists(path):
                    self.log('Unable to remove %s!' % path, level=error_level,
                             exit_code=exit_code)
                    return -1
        else:
            self.debug("%s doesn't exist." % path)

    def _is_windows(self):
        system = platform.system()
        if system in ("Windows", "Microsoft"):
            return True
        if system.startswith("CYGWIN"):
            return True
<commit_msg>backout fc115f1ac2b7. r=bustage<commit_after>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""Generic script objects.

script.py, along with config.py and log.py, represents the core of
mozharness.
"""

import codecs
import os
import platform
import pprint
import re
import shutil
import subprocess
import sys
import urllib2
import urlparse

try:
    import simplejson as json
except ImportError:
    import json

from mozharness.base.config import BaseConfig
from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
     LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL

# OSMixin {{{1
class OSMixin(object):
    """Filesystem commands and the like.

    Depends on LogMixin, ShellMixin, and a self.config of some sort.
    """
    def mkdir_p(self, path, error_level=ERROR):
        """
        Returns None for success, not None for failure
        """
        if not os.path.exists(path):
            self.info("mkdir: %s" % path)
            if not self.config.get('noop'):
                try:
                    os.makedirs(path)
                except OSError:
                    self.log("Can't create directory %s!" % path,
                             level=error_level)
                    return -1
        else:
            self.debug("mkdir_p: %s Already exists." % path)

    def rmtree(self, path, log_level=INFO, error_level=ERROR, exit_code=-1):
        """
        Returns None for success, not None for failure
        """
        self.log("rmtree: %s" % path, level=log_level)
        if os.path.exists(path):
            if not self.config.get('noop'):
                if os.path.isdir(path):
                    if self._is_windows():
                        self._rmdir_recursive(path)
                    else:
                        shutil.rmtree(path)
                else:
                    os.remove(path)
                if os.path.exists(path):
                    self.log('Unable to remove %s!' % path, level=error_level,
                             exit_code=exit_code)
                    return -1
        else:
            self.debug("%s doesn't exist." % path)

    def _is_windows(self):
        system = platform.system()
        if system in ("Windows", "Microsoft"):
            return True
        if system.startswith("CYGWIN"):
            return True

{'fraction_non_alphanumeric': 0.10325563475245618, 'fraction_numerical': 0.005779233288383741, 'mean_word_length': 1.8543155579989006, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5254308', 'n_tokens_mistral': 1528, 'n_tokens_neox': 1450, 'n_words': 454}