<commit_before>from copy import deepcopy
from warnings import warn
import types

from scipy.spatial.distance import cdist

import numpy as np
from nibabel.affines import apply_affine
from nibabel.streamlines import ArraySequence as Streamlines
from dipy.tracking.streamlinespeed import set_number_of_points, length
from dipy.tracking.distances import bundles_distances_mdf
import dipy.tracking.utils as ut
from dipy.core.geometry import dist_to_corner
import dipy.align.vector_fields as vfu


def unlist_streamlines(streamlines):
    """ Return the streamlines not as a list but as an array and an offset

    Parameters
    ----------
    streamlines: sequence

    Returns
    -------
    points : array
    offsets : array

    """

    points = np.concatenate(streamlines, axis=0)
    offsets = np.zeros(len(streamlines), dtype='i8')

    curr_pos = 0<commit_msg>put compress_streamlines back<commit_after>from copy import deepcopy
from warnings import warn
import types

from scipy.spatial.distance import cdist

import numpy as np
from nibabel.affines import apply_affine
from nibabel.streamlines import ArraySequence as Streamlines
from dipy.tracking.streamlinespeed import (set_number_of_points, length,
                                           compress_streamlines)
from dipy.tracking.distances import bundles_distances_mdf
import dipy.tracking.utils as ut
from dipy.core.geometry import dist_to_corner
import dipy.align.vector_fields as vfu


def unlist_streamlines(streamlines):
    """ Return the streamlines not as a list but as an array and an offset

    Parameters
    ----------
    streamlines: sequence

    Returns
    -------
    points : array
    offsets : array

    """

    points = np.concatenate(streamlines, axis=0)
    offsets = np.zeros(len(streamlines), dtype='i8')

    curr_pos = 0
{'fraction_non_alphanumeric': 0.0720968629609246, 'fraction_numerical': 0.003302146395156852, 'mean_word_length': 3.488888888888889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4709655', 'n_tokens_mistral': 536, 'n_tokens_neox': 507, 'n_words': 164}