<commit_before>#pylint: disable=wrong-import-position,wrong-import-order,superfluous-parens
import os
import argparse
import sys
import dxf
import dxf.exceptions

_choices = ['auth',
            'push-blob',
            'pull-blob',
            'blob-size',
            'del-blob',
            'set-alias',
            'get-alias',
            'del-alias',
            'list-aliases',
            'list-repos']

_parser = argparse.ArgumentParser()
_subparsers = _parser.add_subparsers(dest='op')
for c in _choices:
    sp = _subparsers.add_parser(c)
    if c != 'list-repos':
        sp.add_argument("repo")
        sp.add_argument('args', nargs='*')

def _flatten(l):
    return [item for sublist in l for item in sublist]

# pylint: disable=too-many-statements
def doit(args, environ):
    def _auth(dxf_obj, response):
        # pylint: disable=redefined-outer-name
        username = environ.get('DXF_USERNAME')
        password = environ.get('DXF_PASSWORD')
        if username and password:
            dxf_obj.auth_by_password(username, password, response=response)

    # pylint: disable=redefined-variable-type
    args = _parser.parse_args(args)
    if args.op != 'list-repos':
        dxf_obj = dxf.DXF(environ['DXF_HOST'],
                          args.repo,
                          _auth,
                          environ.get('DXF_INSECURE'))
    else:
        dxf_obj = dxf.DXFBase(environ['DXF_HOST'],
                              _auth,
                              environ.get('DXF_INSECURE'))

    def _doit():
        # pylint: disable=too-many-branches
        if args.op == "auth":
            token = dxf_obj.auth_by_password(environ['DXF_USERNAME'],
                                             environ['DXF_PASSWORD'],
                                             actions=args.args)
            if token:
                print(token)
            return

        token = environ.get('DXF_TOKEN')
        if token:
            dxf_obj.token = token

        if args.op == "push-blob":
            if len(args.args) < 1:
                _parser.error('too few arguments')
            if len(args.args) > 2:
                _parser.error('too many arguments')
            if len(args.args) == 2 and not args.args[1].startswith('@'):
                _parser.error('invalid alias')
            dgst = dxf_obj.push_blob(args.args[0])
            if len(args.args) == 2:
                dxf_obj.set_alias(args.args[1][1:], dgst)
            print(dgst)

        elif args.op == "pull-blob":
            if len(args.args) == 0:
                dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
            else:
                dgsts = _flatten([dxf_obj.get_alias(name[1:])
                                  if name.startswith('@') else [name]
                                  for name in args.args])
            for dgst in dgsts:
                for chunk in dxf_obj.pull_blob(dgst):
                    sys.stdout.write(chunk)

        elif args.op == 'blob-size':
            if len(args.args) == 0:
                sizes = [dxf_obj.get_alias(manifest=sys.stdin.read(),
                                           sizes=True)]
            else:
                sizes = [dxf_obj.get_alias(name[1:], sizes=True)
                         if name.startswith('@') else
                         [(name, dxf_obj.blob_size(name))]
                         for name in args.args]
            for tuples in sizes:
                print(sum([size for _, size in tuples]))

        elif args.op == 'del-blob':
            if len(args.args) == 0:
                dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
            else:
                dgsts = _flatten([dxf_obj.del_alias(name[1:])
                                  if name.startswith('@') else [name]
                                  for name in args.args])
            for dgst in dgsts:
                dxf_obj.del_blob(dgst)

        elif args.op == "set-alias":
            if len(args.args) < 2:
                _parser.error('too few arguments')
            dgsts = [dxf.hash_file(dgst) if os.sep in dgst else dgst
                     for dgst in args.args[1:]]
            sys.stdout.write(dxf_obj.set_alias(args.args[0], *dgsts))

        elif args.op == "get-alias":
            if len(args.args) == 0:
                dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
            else:
                dgsts = _flatten([dxf_obj.get_alias(name) for name in args.args])
            for dgst in dgsts:
                print(dgst)

        elif args.op == "del-alias":
            for name in args.args:
                for dgst in dxf_obj.del_alias(name):
                    print(dgst)

        elif args.op == 'list-aliases':
            if len(args.args) > 0:
                _parser.error('too many arguments')
            for name in dxf_obj.list_aliases():
                print(name)

        elif args.op == 'list-repos':
            for name in dxf_obj.list_repos():
                print(name)

    try:
        _doit()
        return 0
    except dxf.exceptions.DXFUnauthorizedError:
        import traceback
        traceback.print_exc()
        import errno
        return errno.EACCES

def main():
    exit(doit(sys.argv[1:], os.environ))
<commit_msg>Check DXF_INSECURE being 1 rather than just existing<commit_after>#pylint: disable=wrong-import-position,wrong-import-order,superfluous-parens
import os
import argparse
import sys
import dxf
import dxf.exceptions

_choices = ['auth',
            'push-blob',
            'pull-blob',
            'blob-size',
            'del-blob',
            'set-alias',
            'get-alias',
            'del-alias',
            'list-aliases',
            'list-repos']

_parser = argparse.ArgumentParser()
_subparsers = _parser.add_subparsers(dest='op')
for c in _choices:
    sp = _subparsers.add_parser(c)
    if c != 'list-repos':
        sp.add_argument("repo")
        sp.add_argument('args', nargs='*')

def _flatten(l):
    return [item for sublist in l for item in sublist]

# pylint: disable=too-many-statements
def doit(args, environ):
    def _auth(dxf_obj, response):
        # pylint: disable=redefined-outer-name
        username = environ.get('DXF_USERNAME')
        password = environ.get('DXF_PASSWORD')
        if username and password:
            dxf_obj.auth_by_password(username, password, response=response)

    # pylint: disable=redefined-variable-type
    args = _parser.parse_args(args)
    if args.op != 'list-repos':
        dxf_obj = dxf.DXF(environ['DXF_HOST'],
                          args.repo,
                          _auth,
                          environ.get('DXF_INSECURE') == '1')
    else:
        dxf_obj = dxf.DXFBase(environ['DXF_HOST'],
                              _auth,
                              environ.get('DXF_INSECURE') == '1')

    def _doit():
        # pylint: disable=too-many-branches
        if args.op == "auth":
            token = dxf_obj.auth_by_password(environ['DXF_USERNAME'],
                                             environ['DXF_PASSWORD'],
                                             actions=args.args)
            if token:
                print(token)
            return

        token = environ.get('DXF_TOKEN')
        if token:
            dxf_obj.token = token

        if args.op == "push-blob":
            if len(args.args) < 1:
                _parser.error('too few arguments')
            if len(args.args) > 2:
                _parser.error('too many arguments')
            if len(args.args) == 2 and not args.args[1].startswith('@'):
                _parser.error('invalid alias')
            dgst = dxf_obj.push_blob(args.args[0])
            if len(args.args) == 2:
                dxf_obj.set_alias(args.args[1][1:], dgst)
            print(dgst)

        elif args.op == "pull-blob":
            if len(args.args) == 0:
                dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
            else:
                dgsts = _flatten([dxf_obj.get_alias(name[1:])
                                  if name.startswith('@') else [name]
                                  for name in args.args])
            for dgst in dgsts:
                for chunk in dxf_obj.pull_blob(dgst):
                    sys.stdout.write(chunk)

        elif args.op == 'blob-size':
            if len(args.args) == 0:
                sizes = [dxf_obj.get_alias(manifest=sys.stdin.read(),
                                           sizes=True)]
            else:
                sizes = [dxf_obj.get_alias(name[1:], sizes=True)
                         if name.startswith('@') else
                         [(name, dxf_obj.blob_size(name))]
                         for name in args.args]
            for tuples in sizes:
                print(sum([size for _, size in tuples]))

        elif args.op == 'del-blob':
            if len(args.args) == 0:
                dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
            else:
                dgsts = _flatten([dxf_obj.del_alias(name[1:])
                                  if name.startswith('@') else [name]
                                  for name in args.args])
            for dgst in dgsts:
                dxf_obj.del_blob(dgst)

        elif args.op == "set-alias":
            if len(args.args) < 2:
                _parser.error('too few arguments')
            dgsts = [dxf.hash_file(dgst) if os.sep in dgst else dgst
                     for dgst in args.args[1:]]
            sys.stdout.write(dxf_obj.set_alias(args.args[0], *dgsts))

        elif args.op == "get-alias":
            if len(args.args) == 0:
                dgsts = dxf_obj.get_alias(manifest=sys.stdin.read())
            else:
                dgsts = _flatten([dxf_obj.get_alias(name) for name in args.args])
            for dgst in dgsts:
                print(dgst)

        elif args.op == "del-alias":
            for name in args.args:
                for dgst in dxf_obj.del_alias(name):
                    print(dgst)

        elif args.op == 'list-aliases':
            if len(args.args) > 0:
                _parser.error('too many arguments')
            for name in dxf_obj.list_aliases():
                print(name)

        elif args.op == 'list-repos':
            for name in dxf_obj.list_repos():
                print(name)

    try:
        _doit()
        return 0
    except dxf.exceptions.DXFUnauthorizedError:
        import traceback
        traceback.print_exc()
        import errno
        return errno.EACCES

def main():
    exit(doit(sys.argv[1:], os.environ))

{'fraction_non_alphanumeric': 0.11543192355000473, 'fraction_numerical': 0.004257734885041158, 'mean_word_length': 1.4615742897065673, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4684516', 'n_tokens_mistral': 3343, 'n_tokens_neox': 3166, 'n_words': 645}