<commit_before>from __future__ import annotations

from typing import Optional, TYPE_CHECKING
import csv
import io

import numpy as np

from .util import EmptyTableError

if TYPE_CHECKING:
    from .ast import PanTableOption


def load_csv(
    data: str,
    options: PanTableOption,
    encoding: Optional[str] = None,
):
    include = options.include
    with (io.StringIO(data) if include is None else io.open(str(include), encoding=encoding)) as f:
        table_list = list(csv.reader(f, **options.csv_kwargs))

    if not table_list:
        raise EmptyTableError

    return table_list


def dump_csv(
    data: np.ndarray[str],
    options: PanTableOption,
) -> str:
    # TODO: if options.include...
    with io.StringIO() as file:
        writer = csv.writer(file, **options.csv_kwargs)
        writer.writerows(data)
        return file.getvalue()
<commit_msg>io.py: add include and its error-handling<commit_after>from __future__ import annotations

from pathlib import Path

from typing import TYPE_CHECKING
import csv
import io
import sys

import numpy as np

from .util import EmptyTableError

if TYPE_CHECKING:
    from typing import Optional, Iterator

    from .ast import PanTableOption


def load_csv(
    data: str,
    options: PanTableOption,
    encoding: Optional[str] = None,
):
    '''loading CSV table

    Note that this can emit EmptyTableError, FileNotFoundError
    '''
    include = options.include
    with (
        open(include, encoding=encoding, newline='')
    ) if include else (
        io.StringIO(data, newline='')
    ) as f:
        table_list = list(csv.reader(f, **options.csv_kwargs))
    if table_list:
        for row in table_list:
            if row:
                for i in row:
                    if i.strip():
                        return table_list
    raise EmptyTableError


def dump_csv_str(
    data: np.ndarray[str],
    options: PanTableOption,
) -> str:
    '''dump data as CSV string
    '''
    with io.StringIO(newline='') as f:
        writer = csv.writer(f, **options.csv_kwargs)
        writer.writerows(data)
        return f.getvalue()


def dump_csv(
    data: np.ndarray[str],
    options: PanTableOption,
) -> Optional[str]:
    '''dump data as CSV

    it will mutate options.include if it is an invalid write path.
    '''
    _include = options.include

    text = dump_csv_str(data, options)

    if _include:
        try:
            include = Path(_include)
            include.parent.mkdir(parents=True, exist_ok=True)
            with open(include, 'x', newline='') as f:
                f.write(text)
            return None
        except (PermissionError, FileExistsError):
            print(f'Data cannot be written to file {options.include}, Overriding include path to empty...', file=sys.stderr)
            options.include = ''
    return text

{'fraction_non_alphanumeric': 0.09302325581395349, 'fraction_numerical': 0.0, 'mean_word_length': 2.575566750629723, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5850336', 'n_tokens_mistral': 857, 'n_tokens_neox': 824, 'n_words': 259}