# Path: hadoopy/_runner.py
# def _find_hstreaming():
#     """Finds the whole path to the hadoop streaming jar.
# 
#     If the environmental var HADOOP_HOME is specified, then start the search
#     from there.
# 
#     Returns:
#         Full path to the hadoop streaming jar if found, else return an empty
#         string.
#     """
#     global WARNED_HADOOP_HOME, HADOOP_STREAMING_PATH_CACHE
#     if HADOOP_STREAMING_PATH_CACHE:
#         return HADOOP_STREAMING_PATH_CACHE
#     try:
#         search_root = os.environ['HADOOP_HOME']
#     except KeyError:
#         search_root = '/'
#     cmd = 'find %s -name hadoop*streaming*.jar' % (search_root)
#     p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE,
#                          stderr=subprocess.PIPE)
#     HADOOP_STREAMING_PATH_CACHE = p.communicate()[0].split('\n')[0]
#     if search_root == '/' and not WARNED_HADOOP_HOME:
#         WARNED_HADOOP_HOME = True
#         hadoop_home = HADOOP_STREAMING_PATH_CACHE[:HADOOP_STREAMING_PATH_CACHE.rfind('/contrib/')]
#         logging.warn('Set the HADOOP_HOME environmental variable to your hadoop path to improve performance. Put the following [export HADOOP_HOME="%s"] in ~/.bashrc' % hadoop_home)
#     return HADOOP_STREAMING_PATH_CACHE


import subprocess
import re
import os
import hadoopy
    import select
from hadoopy._runner import _find_hstreaming
    out = []
    part_num = 0

    def _flush(out, part_num):
        hadoopy.writetb('%s/part-%.5d' % (path, part_num), out, **kw)
        return [], part_num + 1
    for kv in kvs:
        out.append(kv)
        if len(out) >= num_per_file:
            out, part_num = _flush(out, part_num)
    if out:
        out, part_num = _flush(out, part_num)


def readtb(paths, num_procs=1, java_mem_mb=256, ignore_logs=True):
    """Read typedbytes sequence files on HDFS (with optional compression).

    By default, ignores files who's names start with an underscore '_' as they
    are log files.  This allows you to cat a directory that may be a variety of
    outputs from hadoop (e.g., _SUCCESS, _logs).  This works on directories and
    files.  The KV pairs may be interleaved between files
    (they are read in parallel).

    :param paths: HDFS path (str) or paths (iterator)
    :param num_procs: Number of reading procs to open (default 1)
    :param java_mem_mb: Integer of java heap size in MB (default 256)
    :param ignore_logs: If True, ignore all files who's name starts with an underscore.  Defaults to True.
    :returns: An iterator of key, value pairs.
    :raises: IOError: An error occurred reading the directory (e.g., not available).
    """

# Path: hadoopy/_runner.py
# def _find_hstreaming():
#     """Finds the whole path to the hadoop streaming jar.
# 
#     If the environmental var HADOOP_HOME is specified, then start the search
#     from there.
# 
#     Returns:
#         Full path to the hadoop streaming jar if found, else return an empty
#         string.
#     """
#     global WARNED_HADOOP_HOME, HADOOP_STREAMING_PATH_CACHE
#     if HADOOP_STREAMING_PATH_CACHE:
#         return HADOOP_STREAMING_PATH_CACHE
#     try:
#         search_root = os.environ['HADOOP_HOME']
#     except KeyError:
#         search_root = '/'
#     cmd = 'find %s -name hadoop*streaming*.jar' % (search_root)
#     p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE,
#                          stderr=subprocess.PIPE)
#     HADOOP_STREAMING_PATH_CACHE = p.communicate()[0].split('\n')[0]
#     if search_root == '/' and not WARNED_HADOOP_HOME:
#         WARNED_HADOOP_HOME = True
#         hadoop_home = HADOOP_STREAMING_PATH_CACHE[:HADOOP_STREAMING_PATH_CACHE.rfind('/contrib/')]
#         logging.warn('Set the HADOOP_HOME environmental variable to your hadoop path to improve performance. Put the following [export HADOOP_HOME="%s"] in ~/.bashrc' % hadoop_home)
#     return HADOOP_STREAMING_PATH_CACHE


# Path: hadoopy/_hdfs.py
import subprocess
import re
import os
import hadoopy
    import select
from hadoopy._runner import _find_hstreaming

    out = []
    part_num = 0

    def _flush(out, part_num):
        hadoopy.writetb('%s/part-%.5d' % (path, part_num), out, **kw)
        return [], part_num + 1
    for kv in kvs:
        out.append(kv)
        if len(out) >= num_per_file:
            out, part_num = _flush(out, part_num)
    if out:
        out, part_num = _flush(out, part_num)


def readtb(paths, num_procs=1, java_mem_mb=256, ignore_logs=True):
    """Read typedbytes sequence files on HDFS (with optional compression).

    By default, ignores files who's names start with an underscore '_' as they
    are log files.  This allows you to cat a directory that may be a variety of
    outputs from hadoop (e.g., _SUCCESS, _logs).  This works on directories and
    files.  The KV pairs may be interleaved between files
    (they are read in parallel).

    :param paths: HDFS path (str) or paths (iterator)
    :param num_procs: Number of reading procs to open (default 1)
    :param java_mem_mb: Integer of java heap size in MB (default 256)
    :param ignore_logs: If True, ignore all files who's name starts with an underscore.  Defaults to True.
    :returns: An iterator of key, value pairs.
    :raises: IOError: An error occurred reading the directory (e.g., not available).
    """

    hstreaming = _find_hstreaming()