# Path: swe_pipeline.py
# def run_tokenization(options, filename, non_capitalized=None):
#     with (gzip.open(filename, "rt", encoding="utf-8")
#             if filename.endswith(".gz")
#             else open(filename, "r", encoding="utf-8")) as input_file:
#         data = input_file.read()
# 
#         if options.skip_tokenization:
#             sentences = [
#                 sentence.split('\n')
#                 for sentence in data.split('\n\n')
#                 if sentence.strip()
#             ]
#         elif options.skip_segmentation:
#             sentences = [
#                     build_sentences(line, segment=False)
#                     for line in data.split('\n')
#                     if line.strip()
#             ]
#         else:
#             if non_capitalized is None:
#                 n_capitalized = len(re.findall(r'[\.!?] +[A-ZÅÄÖ]', data))
#                 n_non_capitalized = len(re.findall(r'[\.!?] +[a-zåäö]', data))
#                 non_capitalized = n_non_capitalized > 5*n_capitalized
#             sentences = build_sentences(data, non_capitalized=non_capitalized)
# 
#     sentences = list(filter(bool,
#         [[token for token in sentence if len(token) <= MAX_TOKEN]
#             for sentence in sentences]))
#     return sentences


from swe_pipeline import run_tokenization
from textwrap import dedent
from unittest.mock import patch, MagicMock, mock_open
import unittest
            Hej mitt namn är Slim Shady
        """).strip()

        open_mock = mock_open(read_data=text)
        with patch("swe_pipeline.open", open_mock, create=True):
            self.assertEqual(run_tokenization(options, "file.txt"), [
                ["Hej", "mitt", "namn", "är"],
                ["Hej", "mitt", "namn", "är", "Slim", "Shady"],
            ])

    @patch("swe_pipeline.build_sentences")
    def test_sentences_without_tokenization(self, build_sentences_mock):
        options = self._default_options(skip_tokenization=True)
        text = dedent("""
            Hej
            mitt
            namn
            är

            Hej
            mitt
            namn
            är
            Slim
            Shady

        """).strip()

        open_mock = mock_open(read_data=text)
        with patch("swe_pipeline.open", open_mock, create=True):

# Path: swe_pipeline.py
# def run_tokenization(options, filename, non_capitalized=None):
#     with (gzip.open(filename, "rt", encoding="utf-8")
#             if filename.endswith(".gz")
#             else open(filename, "r", encoding="utf-8")) as input_file:
#         data = input_file.read()
# 
#         if options.skip_tokenization:
#             sentences = [
#                 sentence.split('\n')
#                 for sentence in data.split('\n\n')
#                 if sentence.strip()
#             ]
#         elif options.skip_segmentation:
#             sentences = [
#                     build_sentences(line, segment=False)
#                     for line in data.split('\n')
#                     if line.strip()
#             ]
#         else:
#             if non_capitalized is None:
#                 n_capitalized = len(re.findall(r'[\.!?] +[A-ZÅÄÖ]', data))
#                 n_non_capitalized = len(re.findall(r'[\.!?] +[a-zåäö]', data))
#                 non_capitalized = n_non_capitalized > 5*n_capitalized
#             sentences = build_sentences(data, non_capitalized=non_capitalized)
# 
#     sentences = list(filter(bool,
#         [[token for token in sentence if len(token) <= MAX_TOKEN]
#             for sentence in sentences]))
#     return sentences


# Path: tests/test_pipeline_run_tokenization.py
from swe_pipeline import run_tokenization
from textwrap import dedent
from unittest.mock import patch, MagicMock, mock_open
import unittest

            Hej mitt namn är Slim Shady
        """).strip()

        open_mock = mock_open(read_data=text)
        with patch("swe_pipeline.open", open_mock, create=True):
            self.assertEqual(run_tokenization(options, "file.txt"), [
                ["Hej", "mitt", "namn", "är"],
                ["Hej", "mitt", "namn", "är", "Slim", "Shady"],
            ])

    @patch("swe_pipeline.build_sentences")
    def test_sentences_without_tokenization(self, build_sentences_mock):
        options = self._default_options(skip_tokenization=True)
        text = dedent("""
            Hej
            mitt
            namn
            är

            Hej
            mitt
            namn
            är
            Slim
            Shady

        """).strip()

        open_mock = mock_open(read_data=text)
        with patch("swe_pipeline.open", open_mock, create=True):

            self.assertEqual(run_tokenization(options, "file.txt"), [