<commit_before>#!/usr/bin/env python3

# Simple .ply file loader to avoid adding an unnecessary dependency.

import pathlib
from pathlib import Path

import numpy

def load_ply(filename, enableCaching=True):
    """ Load an ascii based .ply file.
        Inputs:
            filename -- string or path
            enableCaching -- bool, defaults to True
        Outputs:
            data -- row-major AOS numpy array.
            columnnames -- list of string
            columntypes -- list of string 
    """
    # Parse the header
    filename = str(filename)
    file = open(filename,'r')

    header_lines = 0

    assert file.readline().strip() == 'ply'
    header_lines += 1

    assert file.readline().strip() == 'format ascii 1.0'
    header_lines += 1

    nextline = file.readline().strip()
    while(nextline.split(' ')[0] == 'comment'):
        header_lines += 1
        nextline = file.readline().strip()

    assert nextline.split(' ')[0] == 'element'
    assert nextline.split(' ')[1] == 'vertex'
    expected_vertices = int(nextline.split(' ')[2])
    header_lines += 1

    columntypes = []
    columnnames = []

    while(1):
        nextline = file.readline().strip().split(' ')
        if nextline[0] == 'property':
            columntypes.append(nextline[1])
            columnnames.append(nextline[2])
            header_lines += 1<commit_msg>Add ability to use external library for ply loading<commit_after>#!/usr/bin/env python3

# Simple .ply file loader to avoid adding an unnecessary dependency.

import pathlib
from pathlib import Path

import numpy

def load_ply_using_library(filename):
    """ Load a plyfile using the plyfile library.
        Doing it this way softens the dependency on the external library. """
    from plyfile import PlyData, PlyElement
    plydata = PlyData.read(filename)

    data = plydata['vertex'].data
    properties = plydata.elements[0].properties
    columnnames = [p.name for p in properties]

    # The library represents data in a list of tuples.
    # I use a flat numpy array. plyfile uses structured arrays
    columns = len(data.dtype)
    rows = len(data)
    outdata = numpy.zeros((rows,columns),dtype=numpy.float32)
    i = 0
    for columnname in columnnames:
        outdata[:,i] = data[columnname]
        i += 1

    columntypes = [str(data.dtype[i]) for i in range(len(data.dtype))] # numpy typestrings
    return outdata, columnnames, columntypes


def load_ply(filename, enableCaching=True):
    """ Load an ascii based .ply file.
        Inputs:
            filename -- string or path
            enableCaching -- bool, defaults to True
        Outputs:
            data -- row-major AOS numpy array.
            columnnames -- list of string
            columntypes -- list of string 
    """
    # Parse the header
    filename = str(filename)
    file = open(filename,'r')

    header_lines = 0

    try:
        assert file.readline().strip() == 'ply'
    except:
        print("Trouble running readline on the file! Will try loading the file using the plyfile library!")
        return load_ply_using_library(filename)
    header_lines += 1

    nextline = file.readline().strip()
    if 'ascii' not in nextline:
        assert 'binary' in nextline, 'Cannot even detect if ply file is ascii or binary!'
        return load_ply_using_library(filename)
    assert nextline == 'format ascii 1.0'
    header_lines += 1

    nextline = file.readline().strip()
    while(nextline.split(' ')[0] == 'comment'):
        header_lines += 1
        nextline = file.readline().strip()

    assert nextline.split(' ')[0] == 'element'
    assert nextline.split(' ')[1] == 'vertex'
    expected_vertices = int(nextline.split(' ')[2])
    header_lines += 1

    columntypes = []
    columnnames = []

    while(1):
        nextline = file.readline().strip().split(' ')
        if nextline[0] == 'property':
            columntypes.append(nextline[1])
            columnnames.append(nextline[2])
            header_lines += 1
{'fraction_non_alphanumeric': 0.1149511645379414, 'fraction_numerical': 0.009767092411720512, 'mean_word_length': 2.5565449688334816, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7502606', 'n_tokens_mistral': 1213, 'n_tokens_neox': 1142, 'n_words': 350}