<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at http://mozilla.org/MPL/2.0/.

import MySQLdb
from optparse import make_option

from django.core.management.base import BaseCommand

from treeherder.model.models import Datasource
from django.conf import settings


class Command(BaseCommand):
    help = ("Runs an arbitrary sql statement or file"
            " on a number of databases.")

    option_list = BaseCommand.option_list + (

        make_option(
            '--datasources',
            action='store',
            dest='datasources',
            default='all',
            help='A comma separated list of datasources to execute the sql code on'),

        make_option(
            '--data-type',
            action='store',
            dest='data_type',
            default='jobs',
            choices=['jobs', 'objectstore'],
            help=('The target data-type of the sql code (jobs or objectstore, '
                  'default jobs)')),

        make_option(
            '-s', '--sql-statement',
            action='store',
            dest='sql_statement',
            help='Sql statement',
            default=''),

        make_option(
            '-f', '--file',
            dest='sql_file',
            help='Sql source file',
            metavar='FILE',
            default="")

    )

    def handle(self, *args, **options):

        sql_code = options["sql_statement"]

        if not sql_code:
            if options["sql_file"]:
                with open(options["sql_file"]) as sql_file:
                    sql_code = sql_file.read()
            else:
                self.stderr.write("Either a SQL statement or file must be specified! See --help.")
                return

        self.stdout.write("SQL command: {}".format(sql_code))

        datasources = Datasource.objects.filter(contenttype=options['data_type'])
        if options['datasources'] != 'all':
            if ',' in options['datasources']:
                datasources = datasources.filter(
                    project__in=options['datasources'].split(','))
            else:
                datasources = datasources.filter(
                    project=options['datasources'])

        self.stdout.write("{0} datasource found".format(
            len(datasources)
        ))

        for datasource in datasources:
            self.stdout.write("--------------------------")
            db = MySQLdb.connect(
                host=datasource.host,
                db=datasource.name,
                user=settings.TREEHERDER_DATABASE_USER,
                passwd=settings.TREE<PASSWORD>DER_DATABASE_PASSWORD)
            try:
                cursor = db.cursor()
                cursor.execute(sql_code)
                self.stdout.write("Sql code executed on {0}".format(datasource))
            except Exception as e:
                error_string = "!!! Sql code execution failed on {0} !!!"
                self.stderr.write(error_string.format(datasource))
                self.stderr.write("{0}".format(e))
            finally:
                if cursor:
                    cursor.close()
<commit_msg>Bug 1163996 - run_sql: Print the query response to stdout<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, you can obtain one at http://mozilla.org/MPL/2.0/.

import MySQLdb
from optparse import make_option

from django.core.management.base import BaseCommand

from treeherder.model.models import Datasource
from django.conf import settings


class Command(BaseCommand):
    help = ("Runs an arbitrary sql statement or file"
            " on a number of databases.")

    option_list = BaseCommand.option_list + (

        make_option(
            '--datasources',
            action='store',
            dest='datasources',
            default='all',
            help='A comma separated list of datasources to execute the sql code on'),

        make_option(
            '--data-type',
            action='store',
            dest='data_type',
            default='jobs',
            choices=['jobs', 'objectstore'],
            help=('The target data-type of the sql code (jobs or objectstore, '
                  'default jobs)')),

        make_option(
            '-s', '--sql-statement',
            action='store',
            dest='sql_statement',
            help='Sql statement',
            default=''),

        make_option(
            '-f', '--file',
            dest='sql_file',
            help='Sql source file',
            metavar='FILE',
            default="")

    )

    def handle(self, *args, **options):

        sql_code = options["sql_statement"]

        if not sql_code:
            if options["sql_file"]:
                with open(options["sql_file"]) as sql_file:
                    sql_code = sql_file.read()
            else:
                self.stderr.write("Either a SQL statement or file must be specified! See --help.")
                return

        self.stdout.write("SQL command: {}".format(sql_code))

        datasources = Datasource.objects.filter(contenttype=options['data_type'])
        if options['datasources'] != 'all':
            if ',' in options['datasources']:
                datasources = datasources.filter(
                    project__in=options['datasources'].split(','))
            else:
                datasources = datasources.filter(
                    project=options['datasources'])

        self.stdout.write("{0} datasource found".format(
            len(datasources)
        ))

        for datasource in datasources:
            self.stdout.write("--------------------------")
            db = MySQLdb.connect(
                host=datasource.host,
                db=datasource.name,
                user=settings.TREEHERDER_DATABASE_USER,
                passwd=<PASSWORD>.<PASSWORD>_PASSWORD)
            try:
                cursor = db.cursor()
                cursor.execute(sql_code)
                self.stdout.write("Sql code executed on {}:".format(datasource))
                for row in cursor:
                    self.stdout.write("  {}".format(row))
            except Exception as e:
                error_string = "!!! Sql code execution failed on {0} !!!"
                self.stderr.write(error_string.format(datasource))
                self.stderr.write("{0}".format(e))
            finally:
                if cursor:
                    cursor.close()

{'fraction_non_alphanumeric': 0.11820796125321628, 'fraction_numerical': 0.0033298017254427123, 'mean_word_length': 1.6982441812984892, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5555949', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1663, 'n_words': 438}