<commit_before>from __future__ import division

import os
import sys
import traceback

import click

import pyximport; pyximport.install()
import cyhdfs3

CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])


def main():
    try:
        cli(obj={})
    except Exception as e:
        click.echo(traceback.format_exc(), err=True)
        sys.exit(1)


@click.group(context_settings=CONTEXT_SETTINGS)
@click.option('--namenode', '-n', default='localhost', required=False, help='Namenode host', show_default=True)
@click.option('--port', '-p', default=8020, required=False, help='Namenode port', show_default=True)
@click.pass_context
def cli(ctx, namenode, port):
    os.environ["LIBHDFS3_CONF"] = "/etc/hadoop/conf/hdfs-site.xml"

    ctx.obj = {}
    ctx.obj['client'] = cyhdfs3.HDFSClient()


@cli.command(short_help='Display the first (n) bytes of a file')
@click.argument('path', required=False, default='/')
@click.option('--bytes', '-b', 'nbytes', default=1*2**10, required=False, help='Number of bytes', show_default=True)
@click.pass_context
def head(ctx, path, nbytes):
    client = ctx.obj['client']
    try:
        with client.open(path, 'r') as f:
            click.echo(f.read(length=nbytes))
    except IOError:
        if client.path_info(path).kind == 'd':
            click.echo("head: error reading '{}': Is a directory".format(path), err=True)
            sys.exit(1)


@cli.command(short_help='Display the last (n) bytes of a file')
@click.argument('path', required=False, default='/')
@click.option('--bytes', '-b', 'nbytes', default=1*2**10, required=False, help='Number of bytes', show_default=True)
@click.pass_context
def tail(ctx, path, nbytes):
    client = ctx.obj['client']
    try:
        with client.open(path, 'r') as f:
            start = max(0, f.info.size - nbytes)
            f.seek(start)
            click.echo(f.read(length=nbytes))
    except IOError as e:
        if client.path_info(path).kind == 'd':
            click.echo("tail: error reading '{}': Is a directory".format(path), err=True)


@cli.command(short_help='Create directory and all non-existent parents')
@click.argument('path', required=False, default='/')
@click.pass_context
def mkdir(ctx, path):
    client = ctx.obj['client']
    client.create_dir(path)


@cli.command(short_help='List a path')
@click.argument('path', required=False, default='/')
@click.option('--recurse', '-R', is_flag=True, default=False, required=False, help='Recurse into subdirectories', show_default=True)
@click.pass_context
def ls(ctx, path, recurse):
    client = ctx.obj['client']
    files = client.list_dir(path, recurse=recurse)

    for f in files:
        row = []

        perm = octal_to_perm(f.permissions)
        t_perm = ('d' if f.kind == 'd' else '-') + perm
        row.append(t_perm)
        s = max([len(j.owner) for j in files]) + 3
        row.append(f.owner.ljust(s))
        s = max([len(j.group) for j in files]) + 3
        row.append(f.group.ljust(s))
        s = max([len(str(j.size)) for j in files]) + 3
        row.append(str(f.size).ljust(s))
        row.append(f.name)

        click.echo(" ".join(row))


def octal_to_perm(octal):
    import stat
    perms = list("-" * 9)
    if octal & stat.S_IRUSR:
        perms[0] = "r"
    if octal & stat.S_IWUSR:
        perms[1] = "w"
    if octal & stat.S_IXUSR:
        perms[2] = "x"
    if octal & stat.S_IRGRP:
        perms[3] = "r"
    if octal & stat.S_IWGRP:
        perms[4] = "w"
    if octal & stat.S_IXGRP:
        perms[5] = "x"
    if octal & stat.S_IROTH:
        perms[6] = "r"
    if octal & stat.S_IWOTH:
        perms[7] = "w"
    if octal & stat.S_IXOTH:
        perms[8] = "x"
    return "".join(perms)


@cli.command(short_help='Display fs stats')
@click.pass_context
def df(ctx):
    client = ctx.obj['client']

    fs = "hdfs://{}:{}".format(client.host, client.port)
    used = client.get_used()
    capacity = client.get_capacity()
    avalable = capacity - used
    block_size = client.get_default_block_size()
    use_p = "%.2f" % (used / capacity)

    headers = ["Filesystem", "Block Size", "Size", "Used", "Available", "Use%"]
    row1 = [fs, block_size, capacity, used, avalable, use_p]
    cols_lenghts = []

    for header, row in zip(headers, row1):
        cols_lenghts.append(max([len(str(_)) for _ in [header, row]]) + 1)
    for row in [headers, row1]:
        lrow = []
        for i, col in enumerate(row):
            if i == 0:
                lrow.append(str(col).ljust(cols_lenghts[i]))
            else:
                lrow.append(str(col).rjust(cols_lenghts[i]))
        click.echo(" ".join(lrow))


if __name__ == '__main__':
    main()
<commit_msg>Add file exists for ls<commit_after>from __future__ import division

import os
import sys
import traceback

import click

import pyximport; pyximport.install()
import cyhdfs3

CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])


def main():
    try:
        cli(obj={})
    except Exception as e:
        click.echo(traceback.format_exc(), err=True)
        sys.exit(1)


@click.group(context_settings=CONTEXT_SETTINGS)
@click.option('--namenode', '-n', default='localhost', required=False, help='Namenode host', show_default=True)
@click.option('--port', '-p', default=8020, required=False, help='Namenode port', show_default=True)
@click.pass_context
def cli(ctx, namenode, port):
    os.environ["LIBHDFS3_CONF"] = "/etc/hadoop/conf/hdfs-site.xml"

    ctx.obj = {}
    ctx.obj['client'] = cyhdfs3.HDFSClient()


@cli.command(short_help='Display the first (n) bytes of a file')
@click.argument('path', required=False, default='/')
@click.option('--bytes', '-b', 'nbytes', default=1*2**10, required=False, help='Number of bytes', show_default=True)
@click.pass_context
def head(ctx, path, nbytes):
    client = ctx.obj['client']
    try:
        with client.open(path, 'r') as f:
            click.echo(f.read(length=nbytes))
    except IOError:
        if client.path_info(path).kind == 'd':
            click.echo("head: error reading '{}': Is a directory".format(path), err=True)
            sys.exit(1)


@cli.command(short_help='Display the last (n) bytes of a file')
@click.argument('path', required=False, default='/')
@click.option('--bytes', '-b', 'nbytes', default=1*2**10, required=False, help='Number of bytes', show_default=True)
@click.pass_context
def tail(ctx, path, nbytes):
    client = ctx.obj['client']
    try:
        with client.open(path, 'r') as f:
            start = max(0, f.info.size - nbytes)
            f.seek(start)
            click.echo(f.read(length=nbytes))
    except IOError as e:
        if client.path_info(path).kind == 'd':
            click.echo("tail: error reading '{}': Is a directory".format(path), err=True)


@cli.command(short_help='Create directory and all non-existent parents')
@click.argument('path', required=False, default='/')
@click.pass_context
def mkdir(ctx, path):
    client = ctx.obj['client']
    client.create_dir(path)


@cli.command(short_help='List a path')
@click.argument('path', required=False, default='/')
@click.option('--recurse', '-R', is_flag=True, default=False, required=False, help='Recurse into subdirectories', show_default=True)
@click.pass_context
def ls(ctx, path, recurse):
    client = ctx.obj['client']

    if not client.exists(path):
        click.echo("ls: cannot access {}: No such file or directory".format(path))
        sys.exit(2)

    files = client.list_dir(path, recurse=recurse)
    for f in files:
        row = []
        perm = octal_to_perm(f.permissions)
        t_perm = ('d' if f.kind == 'd' else '-') + perm
        row.append(t_perm)
        
        s = max([len(j.owner) for j in files]) + 3
        row.append(f.owner.ljust(s))
        s = max([len(j.group) for j in files]) + 3
        row.append(f.group.ljust(s))
        s = max([len(str(j.size)) for j in files]) + 3
        row.append(str(f.size).ljust(s))
        row.append(f.name)
        click.echo(" ".join(row))


def octal_to_perm(octal):
    import stat
    perms = list("-" * 9)
    if octal & stat.S_IRUSR:
        perms[0] = "r"
    if octal & stat.S_IWUSR:
        perms[1] = "w"
    if octal & stat.S_IXUSR:
        perms[2] = "x"
    if octal & stat.S_IRGRP:
        perms[3] = "r"
    if octal & stat.S_IWGRP:
        perms[4] = "w"
    if octal & stat.S_IXGRP:
        perms[5] = "x"
    if octal & stat.S_IROTH:
        perms[6] = "r"
    if octal & stat.S_IWOTH:
        perms[7] = "w"
    if octal & stat.S_IXOTH:
        perms[8] = "x"
    return "".join(perms)


@cli.command(short_help='Display fs stats')
@click.pass_context
def df(ctx):
    client = ctx.obj['client']

    fs = "hdfs://{}:{}".format(client.host, client.port)
    used = client.get_used()
    capacity = client.get_capacity()
    avalable = capacity - used
    block_size = client.get_default_block_size()
    use_p = "%.2f" % (used / capacity)

    headers = ["Filesystem", "Block Size", "Size", "Used", "Available", "Use%"]
    row1 = [fs, block_size, capacity, used, avalable, use_p]
    cols_lenghts = []

    for header, row in zip(headers, row1):
        cols_lenghts.append(max([len(str(_)) for _ in [header, row]]) + 1)
    for row in [headers, row1]:
        lrow = []
        for i, col in enumerate(row):
            if i == 0:
                lrow.append(str(col).ljust(cols_lenghts[i]))
            else:
                lrow.append(str(col).rjust(cols_lenghts[i]))
        click.echo(" ".join(lrow))


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.1669827222924568, 'fraction_numerical': 0.007901390644753476, 'mean_word_length': 3.0088682432432434, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3340025', 'n_tokens_mistral': 3461, 'n_tokens_neox': 3382, 'n_words': 753}