<commit_before># -*- coding: utf-8 -*-
from nose.tools import *
from relshell.recorddef import RecordDef
from relshell.record import Record
from relshell.batch import Batch
from relshell.shelloperator import ShellOperator


def _create_batch():
    rdef = RecordDef([{'name': 'text', 'type': 'STRING'}])
    return Batch((
        Record(rdef, 'test1'),
        Record(rdef, 'test2'),
        Record(rdef, 'test3'),
        Record(rdef, 'test4'),
    ))


def test_output_batch_as_is():
    op = ShellOperator(
        'cat < IN_BATCH0 > OUT_BATCH',
        out_record_def = RecordDef([{'name': 'text', 'type': 'STRING'}]),
    )
    in_batch  = _create_batch()
    out_batch = op.run(in_batches=(in_batch, ))
    eq_(out_batch, in_batch)


# def test_output_2in_1out():
#     batch0 = _create_batch()
#     rdef   = RecordDef([{'name': 'text', 'type': 'STRING'}])
#     batch1 = Batch((
#         Record(rdef, 'test1'),
#         Record(rdef, 'test2xx'),
#         Record(rdef, 'test3'),
#         Record(rdef, 'test4'),
#     ))

#     op = ShellOperator(
#         'diff IN_BATCH0 IN_BATCH1 > OUT_BATCH',
#         out_record_def=RecordDef([
#             {'name': 'position',
#              'type': 'STRING',
#             },
#             {'name': 'diff',
#              'type': 'STRING',
#             },
#         ]),
#         # terminatorはいらない．diffは2バッチ比較終わったらそのままexit(0)するから．
#         # daemonize=Falseなので暗黙のうちにop.kill()が呼ばれるが，
#         # これは「もうプロセスが死んでいたら何もしない」という動作にする．

#         col_patterns = {   # defaultでは r'^.*$\n' にするって感じ
#             'position': re.compile(r'''
#                 # diff position
#                 (
#                     \d (,\d)? [acd] \d (,\d)?
#                 )
#             ''', re.VERBOSE | re.MULTILINE),

#             'diff': re.compile(r'''
#                 # diff content
#                 (
#                     (^[<>] .*?$\n)+
#                     (
#                         ^---$\n
#                         (^> .*?$\n)*
#                     )?
#                 )
#             ''', re.VERBOSE | re.MULTILINE)
#         }
#     )
#     out_batch = op.run(in_batches=(batch0, ))
#     batch2 = op.run(in_batches=(batch1, ))
#     eq_(batch2, batch0)


# def test_daemonized_process():
#     op = ShellOperator(
#         'cat < IN_BATCH0 > OUT_BATCH',
#         out_record_def = RecordDef([{'name': 'text', 'type': 'STRING'}]),
#         terminator     = Terminator.EOF  # Terminator.SIGKILL, ...
#         daemonize      = True,  # op.kill() を明示的に呼び出すまでterminatorの実行がなされない
#     )
#     for i in xrange(10):
#         in_batch  = _create_batch()
#         out_batch = op.run(in_batches=(in_batch, ))   # daemonize=Trueなので，シェルプロセスが走り続けていても，
#                                                 # 1個のin_batchについての処理が終わればrun呼び出しは終わる
#                                                 # (1個のin_batchについて同期呼び出し)
#                                                 # ただし，「どこまでが1つのoutput_batchか」を判定する必要はある．
#                                                 # 「空行が1つ出たら」とか，「何ms新しい出力がなかったら」とか?
#         eq_(out_batch, in_batch)
#     op.kill()  # daemonize=Trueのときのkillし忘れを何とかしたい．subprocessがゾンビになっちゃうし．
#                # デストラクタで警告出しつつkill -9送る?
#                # というか，kill()みたいに「呼び出さないといけない」インターフェースよりwith構文使うべきな感じがある<commit_msg>cascade test passes Signed-off-by: <NAME> <<EMAIL>><commit_after># -*- coding: utf-8 -*-
from nose.tools import *
from relshell.recorddef import RecordDef
from relshell.record import Record
from relshell.batch import Batch
from relshell.shelloperator import ShellOperator


def _create_batch():
    rdef = RecordDef([{'name': 'text', 'type': 'STRING'}])
    return Batch((
        Record(rdef, 'test1'),
        Record(rdef, 'test2'),
        Record(rdef, 'test3'),
        Record(rdef, 'test4'),
    ))


def test_output_batch_as_is():
    op = ShellOperator(
        'cat < IN_BATCH0 > OUT_BATCH',
        out_record_def = RecordDef([{'name': 'text', 'type': 'STRING'}]),
    )
    in_batch  = _create_batch()
    out_batch = op.run(in_batches=(in_batch, ))
    eq_(out_batch, in_batch)


def test_output_batch_cascade():
    op = ShellOperator(
        'cat < IN_BATCH0 > OUT_BATCH',
        out_record_def = RecordDef([{'name': 'text', 'type': 'STRING'}]),
    )
    batch_a = _create_batch()
    batch_b = op.run(in_batches=(batch_a, ))
    batch_c = op.run(in_batches=(batch_b, ))
    eq_(batch_c, batch_a)
{'fraction_non_alphanumeric': 0.15209471766848817, 'fraction_numerical': 0.008652094717668488, 'mean_word_length': 1.9965893587994543, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1869360', 'n_tokens_mistral': 1662, 'n_tokens_neox': 1623, 'n_words': 244}