<commit_before>from six import assertRaisesRegex
from unittest import TestCase
from tempfile import mkstemp
from os import close, unlink, write
from contextlib import contextmanager
from io import StringIO
from json import dumps, loads
import numpy as np

from pysam import CHARD_CLIP, CMATCH

from dark.reads import Read, ReadFilter
from dark.sam import (
    PaddedSAM, SAMFilter, UnequalReferenceLengthError, UnknownReference,
    InvalidSAM, samReferencesToStr, _hardClip, DistanceMatrix)


# These tests actually use the filesystem to read files. That's due to the API
# to pysam and the fact that it calls a C function to open files, so we can't
# mock Python's 'open' method. Hence the following context manager.
@contextmanager
def dataFile(data):
    """
    Create a context manager to store data in a temporary file and
    later remove it.
    """
    fd, filename = mkstemp()
    write(fd, data.encode('utf-8'))<commit_msg>Added comment to the start of test_sam.py<commit_after>from six import assertRaisesRegex
from unittest import TestCase
from tempfile import mkstemp
from os import close, unlink, write
from contextlib import contextmanager
from io import StringIO
from json import dumps, loads
import numpy as np

from pysam import CHARD_CLIP, CMATCH

from dark.reads import Read, ReadFilter
from dark.sam import (
    PaddedSAM, SAMFilter, UnequalReferenceLengthError, UnknownReference,
    InvalidSAM, samReferencesToStr, _hardClip, DistanceMatrix)

# Note: getReferenceInfo in dark/sam.py is tested by the calls to
# consensusFromBAM in test/test_consensus.py


# Some tests below actually use the filesystem to read files. That's due to
# the API to pysam and the fact that it calls a C function to open files,
# so we can't mock Python's 'open' method. Hence the following context
# manager.
@contextmanager
def dataFile(data):
    """
    Create a context manager to store data in a temporary file and
    later remove it.
    """
    fd, filename = mkstemp()
    write(fd, data.encode('utf-8'))
{'fraction_non_alphanumeric': 0.05894006934125805, 'fraction_numerical': 0.0009905894006934125, 'mean_word_length': 4.489130434782608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6226829', 'n_tokens_mistral': 579, 'n_tokens_neox': 554, 'n_words': 257}