<commit_before>import os
import pyexcel as pe
from _compact import BytesIO, StringIO
from base import create_sample_file1
from nose.tools import raises


def do_read_stringio(file_name):
    create_sample_file1(file_name)
    file_type = file_name.split('.')[-1]
    with open(file_name, "rb") as f:
        content = f.read()
        r = pe.get_sheet(file_type=file_type, file_content=content)
        result=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 1.1, 1]
        actual = pe.utils.to_array(r.enumerate())
        assert result == actual
    if os.path.exists(file_name):
        os.unlink(file_name)


def do_book_read_stringio(file_name):
    create_sample_file1(file_name)
    file_type = file_name.split('.')[-1]
    with open(file_name, "rb") as f:
        content = f.read()
        b = pe.get_book(file_type=file_type, file_content=content)
        result=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 1.1, 1]
        actual = pe.utils.to_array(b[0].enumerate())
        assert result == actual
    if os.path.exists(file_name):
        os.unlink(file_name)


def do_write_stringio(file_type):
    data = [
        [1, 2, 3],
        [4, 5, 6]
    ]
    io = pe.save_as(dest_file_type=file_type, array=data)
    r = pe.get_sheet(file_type=file_type, file_content=io.getvalue())
    result=[1, 2, 3, 4, 5, 6]
    actual = pe.utils.to_array(r.enumerate())
    assert actual == result


def do_write_stringio2(file_type):
    data = [
        [1, 2, 3],
        [4, 5, 6]
    ]
    r = pe.Sheet(data)
    io = StringIO()
    r.save_to_memory(file_type, io)
    r = pe.load_from_memory(file_type, io.getvalue())
    result=[1, 2, 3, 4, 5, 6]
    actual = pe.utils.to_array(r.enumerate())
    assert actual == result


class TestIO:
    @raises(IOError)
    def test_wrong_io_input(self):
        pe.Reader(1000)

    def test_csv_stringio(self):
        do_read_stringio("cute.csv")

    def test_xls_stringio(self):
        do_read_stringio("cute.xls")

    def test_book_stringio(self):
        do_book_read_stringio("cute-book.xls")

    def test_csv_output_stringio(self):
        do_write_stringio('csv')

    def test_csv_output_stringio2(self):
        do_write_stringio2('csv')
    
    def test_csvz_stringio(self):
        do_read_stringio('cute.csvz')

    def test_csvz_output_stringio(self):
        do_write_stringio('csvz')

    def test_csvz_output_stringio2(self):
        do_write_stringio2('csvz')

    def test_tsvz_stringio(self):
        do_read_stringio('tsvz')

    def test_tsvz_output_stringio(self):
        do_write_stringio('tsvz')

    def test_tsvz_output_stringio2(self):
        do_write_stringio2('tsvz')

    def test_xls_output_stringio(self):
        do_write_stringio2('xls')

    def test_xlsm_output_stringio(self):
        do_write_stringio2('xlsm')

    def test_book_output_stringio(self):
        data = {
            "Sheet 1": [
                [1, 2, 3],
                [4, 5, 6]
            ]
        }
        io = pe.save_book_as(dest_file_type="xlsm",bookdict=data)
        b = pe.load_book_from_memory("xlsm", io.getvalue())
        result=[1, 2, 3, 4, 5, 6]
        actual = pe.utils.to_array(b[0].enumerate())
        assert result == actual

    def test_book_save_to_stringio(self):
        data = {
            "Sheet 1": [
                [1, 2, 3],
                [4, 5, 6]
            ]
        }
        book = pe.Book(data)
        io = BytesIO()
        book.save_to_memory("xlsm", io)
        b = pe.load_book_from_memory("xlsm", io.getvalue())
        result=[1, 2, 3, 4, 5, 6]
        actual = pe.utils.to_array(b[0].enumerate())
        assert result == actual
<commit_msg>fix the build<commit_after>import os
import pyexcel as pe
from _compact import BytesIO, StringIO
from base import create_sample_file1
from nose.tools import raises


def do_read_stringio(file_name):
    create_sample_file1(file_name)
    file_type = file_name.split('.')[-1]
    open_flag = 'rb'
    if file_type in ['csv', 'tsv']:
        open_flag = 'r'
    with open(file_name, open_flag) as f:
        content = f.read()
        r = pe.get_sheet(file_type=file_type, file_content=content)
        result=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 1.1, 1]
        actual = pe.utils.to_array(r.enumerate())
        assert result == actual
    if os.path.exists(file_name):
        os.unlink(file_name)


def do_book_read_stringio(file_name):
    create_sample_file1(file_name)
    file_type = file_name.split('.')[-1]
    open_flag = 'rb'
    if file_type in ['csv', 'tsv']:
        open_flag = 'r'
    with open(file_name, open_flag) as f:
        content = f.read()
        b = pe.get_book(file_type=file_type, file_content=content)
        result=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 1.1, 1]
        actual = pe.utils.to_array(b[0].enumerate())
        assert result == actual
    if os.path.exists(file_name):
        os.unlink(file_name)


def do_write_stringio(file_type):
    data = [
        [1, 2, 3],
        [4, 5, 6]
    ]
    io = pe.save_as(dest_file_type=file_type, array=data)
    r = pe.get_sheet(file_type=file_type, file_content=io.getvalue())
    result=[1, 2, 3, 4, 5, 6]
    actual = pe.utils.to_array(r.enumerate())
    assert actual == result


def do_write_stringio2(file_type):
    data = [
        [1, 2, 3],
        [4, 5, 6]
    ]
    r = pe.Sheet(data)
    io = StringIO()
    r.save_to_memory(file_type, io)
    r = pe.load_from_memory(file_type, io.getvalue())
    result=[1, 2, 3, 4, 5, 6]
    actual = pe.utils.to_array(r.enumerate())
    assert actual == result


class TestIO:
    @raises(IOError)
    def test_wrong_io_input(self):
        pe.Reader(1000)

    def test_csv_stringio(self):
        do_read_stringio("cute.csv")

    def test_xls_stringio(self):
        do_read_stringio("cute.xls")

    def test_book_stringio(self):
        do_book_read_stringio("cute-book.xls")

    def test_csv_output_stringio(self):
        do_write_stringio('csv')

    def test_csv_output_stringio2(self):
        do_write_stringio2('csv')
    
    def test_csvz_stringio(self):
        do_read_stringio('cute.csvz')

    def test_csvz_output_stringio(self):
        do_write_stringio('csvz')

    def test_csvz_output_stringio2(self):
        do_write_stringio2('csvz')

    def test_tsvz_stringio(self):
        do_read_stringio('tsvz')

    def test_tsvz_output_stringio(self):
        do_write_stringio('tsvz')

    def test_tsvz_output_stringio2(self):
        do_write_stringio2('tsvz')

    def test_xls_output_stringio(self):
        do_write_stringio2('xls')

    def test_xlsm_output_stringio(self):
        do_write_stringio2('xlsm')

    def test_book_output_stringio(self):
        data = {
            "Sheet 1": [
                [1, 2, 3],
                [4, 5, 6]
            ]
        }
        io = pe.save_book_as(dest_file_type="xlsm",bookdict=data)
        b = pe.load_book_from_memory("xlsm", io.getvalue())
        result=[1, 2, 3, 4, 5, 6]
        actual = pe.utils.to_array(b[0].enumerate())
        assert result == actual

    def test_book_save_to_stringio(self):
        data = {
            "Sheet 1": [
                [1, 2, 3],
                [4, 5, 6]
            ]
        }
        book = pe.Book(data)
        io = BytesIO()
        book.save_to_memory("xlsm", io)
        b = pe.load_book_from_memory("xlsm", io.getvalue())
        result=[1, 2, 3, 4, 5, 6]
        actual = pe.utils.to_array(b[0].enumerate())
        assert result == actual

{'fraction_non_alphanumeric': 0.13843691651106962, 'fraction_numerical': 0.020538810349426513, 'mean_word_length': 2.572653644592663, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5073661', 'n_tokens_mistral': 2913, 'n_tokens_neox': 2788, 'n_words': 514}