<commit_before>from itertools import izip
import vtk
from vtk.util import numpy_support as ns
import numpy as np


def read_vtkPolyData(filename):
    if filename.endswith('xml') or filename.endswith('vtp'):
        polydata_reader = vtk.vtkXMLPolyDataReader()
    else:
        polydata_reader = vtk.vtkPolyDataReader()

    polydata_reader.SetFileName(filename)
    polydata_reader.Update()

    polydata = polydata_reader.GetOutput()

    result = {}
    result['lines'] = ns.vtk_to_numpy(polydata.GetLines().GetData())
    result['points'] = ns.vtk_to_numpy(polydata.GetPoints().GetData())
    result['numberOfLines'] = polydata.GetNumberOfLines()

    data = {}
    if polydata.GetPointData().GetScalars():
        data['ActiveScalars'] =  polydata.GetPointData.GetScalars().GetName()
    if polydata.GetPointData().GetVectors():
        data['ActiveVectors'] =  polydata.GetPointData.GetVectors().GetName()
    if polydata.GetPointData().GetTensors():
        data['ActiveTensors'] =  polydata.GetPointData.GetTensors().GetName()

    for i in xrange(polydata.GetPointData().GetNumberOfArrays()):
        array = polydata.GetPointData().GetArray(i)
        data[polydata.GetPointData().GetArrayName(i)] = ns.vtk_to_numpy(array)

    result['pointData'] = data

    return result

def vtkPolyData_to_lines(polydata):
    lines_ids = ns.vtk_to_numpy(polydata.GetLines().GetData())
    points = ns.vtk_to_numpy(polydata.GetPoints().GetData())

    lines = []
    lines_indices = []
    actual_line_index = 0
    for i in xrange(polydata.GetNumberOfLines()):
        next_line_index =actual_line_index + lines_ids[actual_line_index] + 1

        lines_indices.append(lines_ids[actual_line_index + 1: next_line_index])
        lines.append(points[lines_indices[-1]])

        actual_line_index = next_line_index

    point_data = {}
    for i in xrange( polydata.GetPointData().GetNumberOfArrays() ):
        vtk_array = polydata.GetPointData().GetArray(i)
        array_data = ns.vtk_to_numpy(vtk_array)
        if array_data.ndim == 1:
            data = [
                ns.numpy.ascontiguousarray(array_data[line_indices][:, None])
                for line_indices in lines_indices
            ]
        else:
            data = [
                ns.numpy.ascontiguousarray(array_data[line_indices])
                for line_indices in lines_indices
            ]

        point_data[ vtk_array.GetName() ] = data

    scalars = polydata.GetPointData().GetScalars()
    if scalars is not None:
        point_data['ActiveScalars'] = scalars.GetName()

    vectors = polydata.GetPointData().GetVectors()
    if vectors is not None:
        point_data['ActiveVectors'] = vectors.GetName()

    tensors = polydata.GetPointData().GetTensors()
    if tensors is not None:
        point_data['ActiveTensors'] = tensors.GetName()

    return lines, lines_indices, point_data


def lines_to_vtkPolyData(lines, lines_indices=None, point_data={}):
    lengths = [len(p) for p in lines]
    line_starts = ns.numpy.r_[0, ns.numpy.cumsum(lengths)]
    if lines_indices is None:
        lines_indices = [
            ns.numpy.arange(length) + line_start
            for length, line_start in izip(lengths, line_starts)
        ]


    ids = ns.numpy.hstack([
        ns.numpy.r_[c[0], c[1]]
        for c in izip(lengths,lines_indices)
    ])
    vtk_ids = ns.numpy_to_vtkIdTypeArray(ids, deep=True)

    cell_array = vtk.vtkCellArray()
    cell_array.SetCells(len(lines), vtk_ids)
    points = ns.numpy.vstack(lines).astype(
        ns.get_vtk_to_numpy_typemap()[vtk.VTK_DOUBLE]
    )
    points_array = ns.numpy_to_vtk(points, deep=True)

    poly_data = vtk.vtkPolyData()
    vtk_points = vtk.vtkPoints()
    vtk_points.SetData(points_array)
    poly_data.SetPoints(vtk_points)
    poly_data.SetLines(cell_array)


    saved_keys = set()
    for key, value in point_data.items():
        if key in saved_keys:
            continue
        if key.startswith('Active'):
            saved_keys.add(value)
            name = value
            value = point_data[value]
        else:
            name = key

        if len(value) == len(lines):
            value_ = ns.numpy.vstack(value)
        elif len(value) == len(points):
            value_ = value
        else:
            raise ValueError("Data in %s does not have the correct number of items")

        vtk_value = ns.numpy_to_vtk(np.ascontiguousarray(value_), deep=True)
        vtk_value.SetName(name)
        if key == 'ActiveScalars':
            poly_data.GetPointData().SetScalars(vtk_value)
        if key == 'ActiveVectors':
            poly_data.GetPointData().SetVectors(vtk_value)
        if key == 'ActiveTensors':
            poly_data.GetPointData().SetTensors(vtk_value)
        else:
            poly_data.GetPointData().AddArray(vtk_value)

    poly_data.BuildCells()

    return poly_data


def writeLinesToVtkPolyData(filename, lines, point_data={}):
    poly_data = lines_to_vtkPolyData(lines, point_data=point_data)

    if filename.endswith('.xml') or filename.endswith('.vtp'):
        writer = vtk.vtkXMLPolyDataWriter()
        writer.SetDataModeToBinary()
    else:
        writer = vtk.vtkPolyDataWriter()
        writer.SetFileTypeToBinary()<commit_msg>BUG: fixed bug when writing a scalar datatype<commit_after>from itertools import izip
import vtk
from vtk.util import numpy_support as ns
import numpy as np


def read_vtkPolyData(filename):
    if filename.endswith('xml') or filename.endswith('vtp'):
        polydata_reader = vtk.vtkXMLPolyDataReader()
    else:
        polydata_reader = vtk.vtkPolyDataReader()

    polydata_reader.SetFileName(filename)
    polydata_reader.Update()

    polydata = polydata_reader.GetOutput()

    result = {}
    result['lines'] = ns.vtk_to_numpy(polydata.GetLines().GetData())
    result['points'] = ns.vtk_to_numpy(polydata.GetPoints().GetData())
    result['numberOfLines'] = polydata.GetNumberOfLines()

    data = {}
    if polydata.GetPointData().GetScalars():
        data['ActiveScalars'] =  polydata.GetPointData.GetScalars().GetName()
    if polydata.GetPointData().GetVectors():
        data['ActiveVectors'] =  polydata.GetPointData.GetVectors().GetName()
    if polydata.GetPointData().GetTensors():
        data['ActiveTensors'] =  polydata.GetPointData.GetTensors().GetName()

    for i in xrange(polydata.GetPointData().GetNumberOfArrays()):
        array = polydata.GetPointData().GetArray(i)
        data[polydata.GetPointData().GetArrayName(i)] = ns.vtk_to_numpy(array)

    result['pointData'] = data

    return result

def vtkPolyData_to_lines(polydata):
    lines_ids = ns.vtk_to_numpy(polydata.GetLines().GetData())
    points = ns.vtk_to_numpy(polydata.GetPoints().GetData())

    lines = []
    lines_indices = []
    actual_line_index = 0
    for i in xrange(polydata.GetNumberOfLines()):
        next_line_index =actual_line_index + lines_ids[actual_line_index] + 1

        lines_indices.append(lines_ids[actual_line_index + 1: next_line_index])
        lines.append(points[lines_indices[-1]])

        actual_line_index = next_line_index

    point_data = {}
    for i in xrange( polydata.GetPointData().GetNumberOfArrays() ):
        vtk_array = polydata.GetPointData().GetArray(i)
        array_data = ns.vtk_to_numpy(vtk_array)
        if array_data.ndim == 1:
            data = [
                ns.numpy.ascontiguousarray(array_data[line_indices][:, None])
                for line_indices in lines_indices
            ]
        else:
            data = [
                ns.numpy.ascontiguousarray(array_data[line_indices])
                for line_indices in lines_indices
            ]

        point_data[ vtk_array.GetName() ] = data

    scalars = polydata.GetPointData().GetScalars()
    if scalars is not None:
        point_data['ActiveScalars'] = scalars.GetName()

    vectors = polydata.GetPointData().GetVectors()
    if vectors is not None:
        point_data['ActiveVectors'] = vectors.GetName()

    tensors = polydata.GetPointData().GetTensors()
    if tensors is not None:
        point_data['ActiveTensors'] = tensors.GetName()

    return lines, lines_indices, point_data


def lines_to_vtkPolyData(lines, lines_indices=None, point_data={}):
    lengths = [len(p) for p in lines]
    line_starts = ns.numpy.r_[0, ns.numpy.cumsum(lengths)]
    if lines_indices is None:
        lines_indices = [
            ns.numpy.arange(length) + line_start
            for length, line_start in izip(lengths, line_starts)
        ]


    ids = ns.numpy.hstack([
        ns.numpy.r_[c[0], c[1]]
        for c in izip(lengths,lines_indices)
    ])
    vtk_ids = ns.numpy_to_vtkIdTypeArray(ids, deep=True)

    cell_array = vtk.vtkCellArray()
    cell_array.SetCells(len(lines), vtk_ids)
    points = ns.numpy.vstack(lines).astype(
        ns.get_vtk_to_numpy_typemap()[vtk.VTK_DOUBLE]
    )
    points_array = ns.numpy_to_vtk(points, deep=True)

    poly_data = vtk.vtkPolyData()
    vtk_points = vtk.vtkPoints()
    vtk_points.SetData(points_array)
    poly_data.SetPoints(vtk_points)
    poly_data.SetLines(cell_array)


    saved_keys = set()
    for key, value in point_data.items():
        if key in saved_keys:
            continue
        if key.startswith('Active'):
            saved_keys.add(value)
            name = value
            value = point_data[value]
        else:
            name = key

        if len(value) == len(lines):
            if value[0].ndim == 1:
                value_ = ns.numpy.hstack(value)[:, None]
            else:
                value_ = ns.numpy.vstack(value)
        elif len(value) == len(points):
            value_ = value
        else:
            raise ValueError("Data in %s does not have the correct number of items")

        vtk_value = ns.numpy_to_vtk(np.ascontiguousarray(value_), deep=True)
        vtk_value.SetName(name)
        if key == 'ActiveScalars':
            poly_data.GetPointData().SetScalars(vtk_value)
        if key == 'ActiveVectors':
            poly_data.GetPointData().SetVectors(vtk_value)
        if key == 'ActiveTensors':
            poly_data.GetPointData().SetTensors(vtk_value)
        else:
            poly_data.GetPointData().AddArray(vtk_value)

    poly_data.BuildCells()

    return poly_data


def writeLinesToVtkPolyData(filename, lines, point_data={}):
    poly_data = lines_to_vtkPolyData(lines, point_data=point_data)

    if filename.endswith('.xml') or filename.endswith('.vtp'):
        writer = vtk.vtkXMLPolyDataWriter()
        writer.SetDataModeToBinary()
    else:
        writer = vtk.vtkPolyDataWriter()
        writer.SetFileTypeToBinary()
{'fraction_non_alphanumeric': 0.1122737556561086, 'fraction_numerical': 0.0016968325791855204, 'mean_word_length': 2.891782831988261, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4218309', 'n_tokens_mistral': 3553, 'n_tokens_neox': 3468, 'n_words': 585}