<commit_before># -*- coding: utf-8 -*-
from nose.tools import *
import os
from os.path import join
from tempfile import gettempdir
from relshell.record import Record
from relshell.recorddef import RecordDef
from relshell.batch import Batch
from shellstreaming.outputstream.localfile import LocalFile


TEST_FILE = join(gettempdir(), 'shellstreaming_test_localfile.txt')


def teardown():
    pass
    # os.remove(TEST_FILE)


def test_localfile_usage():
    stream  = _create_batches()
    ostream = LocalFile(TEST_FILE)
    for batch in stream:
        ostream.write(batch)    # [fix] - Batch's output format has to be customized by user

    # check contents
    with open(TEST_FILE) as f:
        eq_(f.read(),
'''(
    ("col0": "111", )
    ("col0": "222", )
)
(
    ("col0": "333", )
)
'''
        )


def _create_batches():
    rdef = RecordDef([{'name': 'col0', 'type': 'INT'}])
    return (<commit_msg>removes file after test<commit_after># -*- coding: utf-8 -*-
from nose.tools import *
import os
from os.path import join
from tempfile import gettempdir
from relshell.record import Record
from relshell.recorddef import RecordDef
from relshell.batch import Batch
from shellstreaming.outputstream.localfile import LocalFile


TEST_FILE = join(gettempdir(), 'shellstreaming_test_localfile.txt')


def teardown():
     os.remove(TEST_FILE)


def test_localfile_usage():
    stream  = _create_batches()
    ostream = LocalFile(TEST_FILE)
    for batch in stream:
        ostream.write(batch)    # [fix] - Batch's output format has to be customized by user

    # check contents
    with open(TEST_FILE) as f:
        eq_(f.read(),
'''(
    ("col0": "111", )
    ("col0": "222", )
)
(
    ("col0": "333", )
)
'''
        )


def _create_batches():
    rdef = RecordDef([{'name': 'col0', 'type': 'INT'}])
    return (
{'fraction_non_alphanumeric': 0.13131868131868132, 'fraction_numerical': 0.015384615384615385, 'mean_word_length': 3.4962962962962965, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4734085', 'n_tokens_mistral': 621, 'n_tokens_neox': 595, 'n_words': 164}