<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.


"""
OpenQuake database maintenance tool, performs schema upgrades and/or data
migration.

  -h | --help     : prints this help string
       --host H   : database host machine name [default: localhost]
  -d | --db D     : database to use [default: openquake]
  -n | --dryrun   : don't do anything just show what needs done
  -p | --path P   : path to schema upgrade files [default: db/schema/upgrades]
  -U | --user U   : database user to use [default: postgres]
"""

from distutils import version
import getopt
import logging
import subprocess
import re
import sys
import os


logging.basicConfig(level=logging.INFO)


def run_cmd(cmds, ignore_exit_code=False):
    """Run the given command and return the exit code, stdout and stderr.

    :param list cmds: the strings that comprise the command to run
    :param bool ignore_exit_code: if `True` no `Exception` will be raised for
        non-zero command exit code.
    :returns: an `(exit code, stdout, stderr)` triple
    :raises Exception: when the command terminates with a non-zero command
        exit code.
    """
    p = subprocess.Popen(
        cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    if p.returncode != 0 and not ignore_exit_code:
        error = ("%s terminated with exit code: %s\n%s"
                 % (cmds[0], p.returncode, err))
        logging.error(error)
        raise Exception(error)
    return (p.returncode, out, err)


def psql(config, script=None, cmd=None, ignore_dryrun=False, runner=run_cmd):
    """Runs the `psql` tool either with a command or SQL script.

    If the `dryrun` configuration flag is set the command will not be run but
    merely printed.
    Please note that `file` and `cmd` are mutually exclusive.

    :param dict config: the configuration to use: database, host, user, path.
    :param string script: the script to run, relative to the path in `config`.
    :param string cmd: the command to run.
    :param bool ignore_dryrun: if `True` the `dryrun` flag in the
        configuration will be disregarded.
    :param runner: function to use for running the `psql` tool.
    :returns: a triple (exit code, stdout, stderr) with psql execution outcome
    """
    if script and cmd:
        raise Exception("Please specify either an SQL script or a command.")

    if not script and not cmd:
        raise Exception("Neither SQL script nor command specified.")

    if config['host'] in ["localhost", "127.0.0.1"]:
        psql_cmd = "psql -d %(db)s -U %(user)s" % config
    else:
        psql_cmd = "psql -d %(db)s -U %(user)s -h %(host)s" % config

    cmds = psql_cmd.split()

    if cmd:
        cmds.extend(['-c', "%s" % cmd])
    else:
        cmds.extend(["-f", "%s/%s" % (config['path'], script)])

    if config['dryrun'] and not ignore_dryrun:
        if cmd:
            cmds[-1] = '"%s"' % cmds[-1]
        print " ".join(cmds)
        return (-1, "", "")
    else:
        return runner(cmds)


def find_scripts(path):
    """Find all SQL scripts at level 2 of the given `path`."""
    result = []
    cmd = "find %s -mindepth 2 -maxdepth 2 -type f" \
          "     ( -name *.sql -o -name *.py )" % path

    code, out, err = run_cmd(cmd.split(), ignore_exit_code=True)

    if code == 0:
        prefix_length = len(path) + 1
        for file in out.split('\n'):
            result.append(file[prefix_length:])
    return [r for r in result if r]


def version_key(string):
    # remove the trailing '-<release>' number if any
    string, _ = (string + '-').split('-', 1)

    return version.StrictVersion(string)


def script_sort_key(script):
    # return a sort key to order files first by revision, then by
    # step, then by file name
    revision, step, name = script.rsplit('/', 3)

    return version_key(revision), int(step), name

<commit_msg>Add docstring for new functions.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.


"""
OpenQuake database maintenance tool, performs schema upgrades and/or data
migration.

  -h | --help     : prints this help string
       --host H   : database host machine name [default: localhost]
  -d | --db D     : database to use [default: openquake]
  -n | --dryrun   : don't do anything just show what needs done
  -p | --path P   : path to schema upgrade files [default: db/schema/upgrades]
  -U | --user U   : database user to use [default: postgres]
"""

from distutils import version
import getopt
import logging
import subprocess
import re
import sys
import os


logging.basicConfig(level=logging.INFO)


def run_cmd(cmds, ignore_exit_code=False):
    """Run the given command and return the exit code, stdout and stderr.

    :param list cmds: the strings that comprise the command to run
    :param bool ignore_exit_code: if `True` no `Exception` will be raised for
        non-zero command exit code.
    :returns: an `(exit code, stdout, stderr)` triple
    :raises Exception: when the command terminates with a non-zero command
        exit code.
    """
    p = subprocess.Popen(
        cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    if p.returncode != 0 and not ignore_exit_code:
        error = ("%s terminated with exit code: %s\n%s"
                 % (cmds[0], p.returncode, err))
        logging.error(error)
        raise Exception(error)
    return (p.returncode, out, err)


def psql(config, script=None, cmd=None, ignore_dryrun=False, runner=run_cmd):
    """Runs the `psql` tool either with a command or SQL script.

    If the `dryrun` configuration flag is set the command will not be run but
    merely printed.
    Please note that `file` and `cmd` are mutually exclusive.

    :param dict config: the configuration to use: database, host, user, path.
    :param string script: the script to run, relative to the path in `config`.
    :param string cmd: the command to run.
    :param bool ignore_dryrun: if `True` the `dryrun` flag in the
        configuration will be disregarded.
    :param runner: function to use for running the `psql` tool.
    :returns: a triple (exit code, stdout, stderr) with psql execution outcome
    """
    if script and cmd:
        raise Exception("Please specify either an SQL script or a command.")

    if not script and not cmd:
        raise Exception("Neither SQL script nor command specified.")

    if config['host'] in ["localhost", "127.0.0.1"]:
        psql_cmd = "psql -d %(db)s -U %(user)s" % config
    else:
        psql_cmd = "psql -d %(db)s -U %(user)s -h %(host)s" % config

    cmds = psql_cmd.split()

    if cmd:
        cmds.extend(['-c', "%s" % cmd])
    else:
        cmds.extend(["-f", "%s/%s" % (config['path'], script)])

    if config['dryrun'] and not ignore_dryrun:
        if cmd:
            cmds[-1] = '"%s"' % cmds[-1]
        print " ".join(cmds)
        return (-1, "", "")
    else:
        return runner(cmds)


def find_scripts(path):
    """Find all SQL scripts at level 2 of the given `path`."""
    result = []
    cmd = "find %s -mindepth 2 -maxdepth 2 -type f" \
          "     ( -name *.sql -o -name *.py )" % path

    code, out, err = run_cmd(cmd.split(), ignore_exit_code=True)

    if code == 0:
        prefix_length = len(path) + 1
        for file in out.split('\n'):
            result.append(file[prefix_length:])
    return [r for r in result if r]


def version_key(string):
    """Returns a version representation useful for version comparison"""

    # remove the trailing '-<release>' number if any
    string, _ = (string + '-').split('-', 1)

    return version.StrictVersion(string)


def script_sort_key(script):
    """
    Return a sort key to order upgrade scripts first by revision, then
    by step, then by file name
    """

    revision, step, name = script.rsplit('/', 3)

    return version_key(revision), int(step), name


{'fraction_non_alphanumeric': 0.10997899159663865, 'fraction_numerical': 0.007563025210084034, 'mean_word_length': 2.916495269436446, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '717757', 'n_tokens_mistral': 2969, 'n_tokens_neox': 2851, 'n_words': 1165}