<commit_before>"""
I/O for the PLY format, cf.
<https://en.wikipedia.org/wiki/PLY_(file_format)>.
<https://web.archive.org/web/20161221115231/http://www.cs.virginia.edu/~gfx/Courses/2001/Advanced.spring.01/plylib/Ply.txt>.
"""
import re
import warnings

import numpy

from ._mesh import Mesh

# Reference dtypes
ply_to_numpy_dtype = {
    "short": numpy.int16,
    "ushort": numpy.uint16,
    "int": numpy.int32,
    "int32": numpy.int32,
    "uint": numpy.uint32,
    "uint8": numpy.uint8,
    "float": numpy.float32,
    "float32": numpy.float32,
    "double": numpy.float64,
}
numpy_to_ply_dtype = {numpy.dtype(v): k for k, v in ply_to_numpy_dtype.items()}


def read(filename):
    with open(filename, "rb") as f:
        mesh = read_buffer(f)
    return mesh


def _fast_forward(f):
    # fast forward to the next significant line
    while True:
        line = f.readline().decode("utf-8").strip()
        if line and line[:7] != "comment":
            break
    return line


def read_buffer(f):
    # assert that the first line reads `ply`
    line = f.readline().decode("utf-8")
    assert line.strip() == "ply"

    line = _fast_forward(f)
    assert line == "format ascii 1.0"

    line = _fast_forward(f)
    m = re.match("element vertex (\\d+)", line)
    num_verts = int(m.groups()[0])

    # fast forward to the next significant line
    formats = []
    point_data_names = []
    line = _fast_forward(f)
    while line[:8] == "property":
        m = re.match("property (.+) (.+)", line)
        formats.append(m.groups()[0])
        point_data_names.append(m.groups()[1])
        line = _fast_forward(f)
    # assert that all formats are the same
    assert all(formats[0] == fmt for fmt in formats)
    dtype_verts = ply_to_numpy_dtype[formats[0]]

    m = re.match("element face (\\d+)", line)
    num_faces = int(m.groups()[0])

    # read property lists
    line = _fast_forward(f)
    cell_data_names = []
    dtypes = []
    while line != "end_header":
        m = re.match("property list (.+) (.+) (.+)", line)
        types = m.groups()[:2]
        name = m.groups()[2]
        cell_data_names.append(name)
        dtypes.append(ply_to_numpy_dtype[types[1]])
        line = _fast_forward(f)

    assert line == "end_header"

    # Now read the data
    point_data = numpy.fromfile(
        f, dtype=dtype_verts, count=len(point_data_names) * num_verts, sep=" "
    ).reshape(num_verts, len(point_data_names))
    assert point_data_names[0] == "x"
    assert point_data_names[1] == "y"
    k = 3 if point_data_names[2] == "z" else 2
    verts = point_data[:, :k]
    point_data = {
        name: data for name, data in zip(point_data_names[k:], point_data.T[k:])
    }

    # the faces must be read line-by-line
    for k in range(num_faces):
        line = f.readline().decode("utf-8").strip()
        data = line.split()
        if k == 0:
            # initialize the cell data arrays
            n = []
            i = 0
            cell_data = {}
            for name, dtype in zip(cell_data_names, dtypes):
                n = int(data[i])
                cell_data[name] = numpy.empty((num_faces, n), dtype=dtype)
                i += n + 1

        i = 0<commit_msg>some progress in ply binary reads<commit_after>"""
I/O for the PLY format, cf.
<https://en.wikipedia.org/wiki/PLY_(file_format)>.
<https://web.archive.org/web/20161221115231/http://www.cs.virginia.edu/~gfx/Courses/2001/Advanced.spring.01/plylib/Ply.txt>.
"""
import re
import warnings

import numpy

from ._mesh import Mesh

# Reference dtypes
ply_to_numpy_dtype = {
    "short": numpy.int16,
    "ushort": numpy.uint16,
    "int": numpy.int32,
    "int32": numpy.int32,
    "uint": numpy.uint32,
    "uint8": numpy.uint8,
    "float": numpy.float32,
    "float32": numpy.float32,
    "double": numpy.float64,
}
numpy_to_ply_dtype = {numpy.dtype(v): k for k, v in ply_to_numpy_dtype.items()}


def read(filename):
    with open(filename, "rb") as f:
        mesh = read_buffer(f)
    return mesh


def _fast_forward(f):
    # fast forward to the next significant line
    while True:
        line = f.readline().decode("utf-8").strip()
        if line and line[:7] != "comment":
            break
    return line


def read_buffer(f):
    # assert that the first line reads `ply`
    line = f.readline().decode("utf-8").strip()
    assert line == "ply"

    line = _fast_forward(f)
    if line == "format ascii 1.0":
        is_binary = False
    elif line == "format binary_big_endian 1.0":
        is_binary = True
        endianness = ">"
    else:
        assert line == "format binary_little_endian 1.0"
        is_binary = True
        endianness = "<"

    line = _fast_forward(f)
    m = re.match("element vertex (\\d+)", line)
    num_verts = int(m.groups()[0])

    # fast forward to the next significant line
    formats = []
    point_data_names = []
    line = _fast_forward(f)
    while line[:8] == "property":
        m = re.match("property (.+) (.+)", line)
        formats.append(m.groups()[0])
        point_data_names.append(m.groups()[1])
        line = _fast_forward(f)

    m = re.match("element face (\\d+)", line)
    num_faces = int(m.groups()[0])

    # read property lists
    line = _fast_forward(f)
    cell_data_names = []
    dtypes = []
    while line != "end_header":
        m = re.match("property list (.+) (.+) (.+)", line)
        types = m.groups()[:2]
        name = m.groups()[2]
        cell_data_names.append(name)
        dtypes.append(ply_to_numpy_dtype[types[1]])
        line = _fast_forward(f)

    assert line == "end_header"

    ply_to_numpy_dtype_string = {
        "uchar": "i1",
        "uint8": "u1",
        "float": "f4",
        "float32": "f4",
        "double": "f8",
    }

    if is_binary:
        dtype = [
            (name, endianness + ply_to_numpy_dtype_string[fmt])
            for name, fmt in zip(point_data_names, formats)
        ]
        point_data = numpy.fromfile(f, count=num_verts, dtype=dtype)
        verts = numpy.column_stack([point_data["x"], point_data["y"], point_data["z"]])
        point_data = {
            name: point_data[name]
            for name in point_data_names
            if name not in ["x", "y", "z"]
        }
    else:
        assert point_data_names[0] == "x"
        assert point_data_names[1] == "y"
        k = 3 if point_data_names[2] == "z" else 2
        # assert that all formats are the same
        assert all(formats[0] == fmt for fmt in formats)
        dtype_verts = ply_to_numpy_dtype[formats[0]]
        # Now read the data
        point_data = numpy.fromfile(
            f, dtype=dtype_verts, count=len(point_data_names) * num_verts, sep=" "
        ).reshape(num_verts, len(point_data_names))

        assert point_data_names[0] == "x"
        assert point_data_names[1] == "y"
        k = 3 if point_data_names[2] == "z" else 2
        verts = point_data[:, :k]

    # the faces must be read line-by-line
    for k in range(num_faces):
        line = f.readline().decode("utf-8").strip()
        data = line.split()
        if k == 0:
            # initialize the cell data arrays
            n = []
            i = 0
            cell_data = {}
            for name, dtype in zip(cell_data_names, dtypes):
                n = int(data[i])
                cell_data[name] = numpy.empty((num_faces, n), dtype=dtype)
                i += n + 1

        i = 0
{'fraction_non_alphanumeric': 0.14454237288135593, 'fraction_numerical': 0.02047457627118644, 'mean_word_length': 2.3711151736745886, 'pattern_counts': {'":': 31, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4389542', 'n_tokens_mistral': 2623, 'n_tokens_neox': 2528, 'n_words': 593}