<commit_before>#!/usr/bin/env python
#  Copyright 2014 <NAME> <<EMAIL>>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys, os, argparse, logging
import simplejson
from .extractor import try_scan_directories


class VerboseErrorParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help(sys.stderr)
        sys.exit(2)


def extract(args):
    errors, docs = try_scan_directories(args.base_directory)
    simplejson.dump(docs, args.json_filename, indent = '  ')
    args.json_filename.write('\n')
    if errors:
        sys.stderr.write('%d CLI executables had fatal errors and were excluded from the output.\n' % (len(errors), ))

def index(args):
    # local imports, so that we can extract without elasticsearch being available
    # (and index without ctk_cli being available)
    import elasticsearch
    from ctk_cli import isCLIExecutable
    from .indexer import create_elasticsearch_index, update_elasticsearch_index

    if len(args.path) == 1 and not (os.path.isdir(args.path[0]) or isCLIExecutable(args.path[0])):
        with open(args.path[0]) as f:
            docs = simplejson.load(f)
    else:
        errors, docs = try_scan_directories(args.path)
        if errors:
            sys.stderr.write('%d CLI executables had fatal errors and will not be indexed.\n' % (len(errors), ))

    # TODO: at the moment, the commandline is limited to *one* host/port, and no SSL or URL prefix
    es = elasticsearch.Elasticsearch([dict(host = args.host, port = args.port)])

    create_elasticsearch_index(es)

    update_elasticsearch_index(es, docs, args.source_name)

def main():
    parser = VerboseErrorParser(description = 'index CLI modules in elasticsearch database')

    logging.basicConfig()
    
    commands = parser.add_subparsers()
    extractor_parser = commands.add_parser(
        'extract', help = 'only create JSON description from CLI modules (no DB access)')
    extractor_parser.add_argument(
        'base_directory', nargs = '+',
        help = 'directories (at least one) in which to search for CLI module executables, '
        'or direct paths to executables')
    extractor_parser.add_argument(
        '--json_filename', '-o', type = argparse.FileType('w'), default = sys.stdout)
    extractor_parser.set_defaults(action = extract)


    index_parser = commands.add_parser(
        'index', help = 'update elasticsearch index (given CLI modules or JSON description)')
    index_parser.add_argument('--host', default = 'localhost',
                              help = 'hostname elasticsearch is listening on (default: localhost)')
    index_parser.add_argument('--port', default = 9200,
                              help = 'port elasticsearch is listening on (default: 9200)')

    index_parser.add_argument(
        'source_name', help = "identifier for the source "
        "(e.g. 'slicer' or 'nifty-reg') of this set of CLI modules "
        "(will be used to remove old documents from this source "
        "from the Elasticsearch index if they are no longer present)")
    index_parser.add_argument(
        'path', nargs = '+',
        help = 'one or more directories in which to search for CLI module executables, '
        'paths to CLI executables, or (exactly one) JSON file as created by `extract` subcommand')

    index_parser.set_defaults(action = index)


    args = parser.parse_args()
    args.action(args)


if __name__ == '__main__':
    main()<commit_msg>catch usage error (no argument on commandline)<commit_after>#!/usr/bin/env python
#  Copyright 2014 <NAME> <<EMAIL>>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys, os, argparse, logging
import simplejson
from .extractor import try_scan_directories


class VerboseErrorParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help(sys.stderr)
        sys.exit(2)


def extract(args):
    errors, docs = try_scan_directories(args.base_directory)
    simplejson.dump(docs, args.json_filename, indent = '  ')
    args.json_filename.write('\n')
    if errors:
        sys.stderr.write('%d CLI executables had fatal errors and were excluded from the output.\n' % (len(errors), ))

def index(args):
    # local imports, so that we can extract without elasticsearch being available
    # (and index without ctk_cli being available)
    import elasticsearch
    from ctk_cli import isCLIExecutable
    from .indexer import create_elasticsearch_index, update_elasticsearch_index

    if len(args.path) == 1 and not (os.path.isdir(args.path[0]) or isCLIExecutable(args.path[0])):
        with open(args.path[0]) as f:
            docs = simplejson.load(f)
    else:
        errors, docs = try_scan_directories(args.path)
        if errors:
            sys.stderr.write('%d CLI executables had fatal errors and will not be indexed.\n' % (len(errors), ))

    # TODO: at the moment, the commandline is limited to *one* host/port, and no SSL or URL prefix
    es = elasticsearch.Elasticsearch([dict(host = args.host, port = args.port)])

    create_elasticsearch_index(es)

    update_elasticsearch_index(es, docs, args.source_name)

def main():
    parser = VerboseErrorParser(description = 'index CLI modules in elasticsearch database')

    logging.basicConfig()
    
    commands = parser.add_subparsers()
    extractor_parser = commands.add_parser(
        'extract', help = 'only create JSON description from CLI modules (no DB access)')
    extractor_parser.add_argument(
        'base_directory', nargs = '+',
        help = 'directories (at least one) in which to search for CLI module executables, '
        'or direct paths to executables')
    extractor_parser.add_argument(
        '--json_filename', '-o', type = argparse.FileType('w'), default = sys.stdout)
    extractor_parser.set_defaults(action = extract)


    index_parser = commands.add_parser(
        'index', help = 'update elasticsearch index (given CLI modules or JSON description)')
    index_parser.add_argument('--host', default = 'localhost',
                              help = 'hostname elasticsearch is listening on (default: localhost)')
    index_parser.add_argument('--port', default = 9200,
                              help = 'port elasticsearch is listening on (default: 9200)')

    index_parser.add_argument(
        'source_name', help = "identifier for the source "
        "(e.g. 'slicer' or 'nifty-reg') of this set of CLI modules "
        "(will be used to remove old documents from this source "
        "from the Elasticsearch index if they are no longer present)")
    index_parser.add_argument(
        'path', nargs = '+',
        help = 'one or more directories in which to search for CLI module executables, '
        'paths to CLI executables, or (exactly one) JSON file as created by `extract` subcommand')

    index_parser.set_defaults(action = index)


    args = parser.parse_args()
    if 'action' in args:
        args.action(args)
    else:
        parser.error('Specifiy action (extract or index)')


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.09812108559498957, 'fraction_numerical': 0.005157804249048262, 'mean_word_length': 3.277310924369748, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4517527', 'n_tokens_mistral': 2371, 'n_tokens_neox': 2297, 'n_words': 785}