<commit_before>import imvu
import md5
import logging
logger = logging.getLogger("imvu." + __name__)

import ctypes

if __debug__:
    cal3d_dll  = ctypes.cdll.cal3d_debug
else:
    cal3d_dll = ctypes.cdll.cal3d

cal3d_dll.CalError_GetLastErrorText.restype = ctypes.c_char_p
cal3d_dll.CalCoreMaterial_GetMapFilename.restype = ctypes.c_char_p
cal3d_dll.CalCoreMaterial_GetMapType.restype = ctypes.c_char_p

gLoader = cal3d_dll.CalLoader_New()

gMaterialInfoCache = {}
gMeshInfoCache = {}

class Cal3dException(Exception): pass

def getMaterialInfo(materialBuffer):
    global gMaterialInfoCache
    cachekey = md5.md5(materialBuffer).hexdigest()
    cached = gMaterialInfoCache.get(cachekey, None)
    if cached: return cached

    global gLoader
    material = cal3d_dll.CalLoader_LoadCoreMaterialFromBuffer(gLoader, materialBuffer, len(materialBuffer))
    if not material:
        raise Cal3dException, cal3d_dll.CalError_GetLastErrorText()
    mapCount = cal3d_dll.CalCoreMaterial_GetMapCount(material)
    materialInfo = {}
    materialInfo['maps'] = {}
    for i in range(0, mapCount):
        mapType = cal3d_dll.CalCoreMaterial_GetMapType(material, i)
        mapName = cal3d_dll.CalCoreMaterial_GetMapFilename(material, i)
        materialInfo['maps'][mapType] = mapName
    gMaterialInfoCache[cachekey] = materialInfo
    return materialInfo

def getMeshInfo(meshBuffer):
    global gMeshInfoCache
    cachekey = md5.md5(meshBuffer).hexdigest()
    cached = gMeshInfoCache.get(cachekey, None)
    if cached:
        return cached

    global gLoader
    mesh = cal3d_dll.CalLoader_LoadCoreMeshFromBuffer(gLoader, meshBuffer, len(meshBuffer))
    if not mesh:
        raise Cal3dException, cal3d_dll.CalError_GetLastErrorText()
    submeshCount = cal3d_dll.CalCoreMesh_GetCoreSubmeshCount(mesh)
    result = {}
    for i in range(0, submeshCount):
        result[i] = {}
        submesh = cal3d_dll.CalCoreMesh_GetCoreSubmesh(mesh, i)
        result[i]['materialId'] = cal3d_dll.CalCoreSubmesh_GetCoreMaterialThreadId(submesh)
        result[i]['faceCount'] = cal3d_dll.CalCoreSubmesh_GetFaceCount(submesh)
        result[i]['lodCount'] = cal3d_dll.CalCoreSubmesh_GetLodCount(submesh)
        result[i]['vertexCount'] = cal3d_dll.CalCoreSubmesh_GetVertexCount(submesh)
    gMeshInfoCache[cachekey] = result
    return result<commit_msg>In python 2.5, md5 has moved to the hashlib module.  In python 2.6, the md5 module is deprecated. (force is fixing buildbot)<commit_after>import imvu
import hashlib
import logging
logger = logging.getLogger("imvu." + __name__)

import ctypes

if __debug__:
    cal3d_dll  = ctypes.cdll.cal3d_debug
else:
    cal3d_dll = ctypes.cdll.cal3d

cal3d_dll.CalError_GetLastErrorText.restype = ctypes.c_char_p
cal3d_dll.CalCoreMaterial_GetMapFilename.restype = ctypes.c_char_p
cal3d_dll.CalCoreMaterial_GetMapType.restype = ctypes.c_char_p

gLoader = cal3d_dll.CalLoader_New()

gMaterialInfoCache = {}
gMeshInfoCache = {}

class Cal3dException(Exception): pass

def getMaterialInfo(materialBuffer):
    global gMaterialInfoCache
    cachekey = hashlib.md5(materialBuffer).hexdigest()
    cached = gMaterialInfoCache.get(cachekey, None)
    if cached: return cached

    global gLoader
    material = cal3d_dll.CalLoader_LoadCoreMaterialFromBuffer(gLoader, materialBuffer, len(materialBuffer))
    if not material:
        raise Cal3dException, cal3d_dll.CalError_GetLastErrorText()
    mapCount = cal3d_dll.CalCoreMaterial_GetMapCount(material)
    materialInfo = {}
    materialInfo['maps'] = {}
    for i in range(0, mapCount):
        mapType = cal3d_dll.CalCoreMaterial_GetMapType(material, i)
        mapName = cal3d_dll.CalCoreMaterial_GetMapFilename(material, i)
        materialInfo['maps'][mapType] = mapName
    gMaterialInfoCache[cachekey] = materialInfo
    return materialInfo

def getMeshInfo(meshBuffer):
    global gMeshInfoCache
    cachekey = hashlib.md5(meshBuffer).hexdigest()
    cached = gMeshInfoCache.get(cachekey, None)
    if cached:
        return cached

    global gLoader
    mesh = cal3d_dll.CalLoader_LoadCoreMeshFromBuffer(gLoader, meshBuffer, len(meshBuffer))
    if not mesh:
        raise Cal3dException, cal3d_dll.CalError_GetLastErrorText()
    submeshCount = cal3d_dll.CalCoreMesh_GetCoreSubmeshCount(mesh)
    result = {}
    for i in range(0, submeshCount):
        result[i] = {}
        submesh = cal3d_dll.CalCoreMesh_GetCoreSubmesh(mesh, i)
        result[i]['materialId'] = cal3d_dll.CalCoreSubmesh_GetCoreMaterialThreadId(submesh)
        result[i]['faceCount'] = cal3d_dll.CalCoreSubmesh_GetFaceCount(submesh)
        result[i]['lodCount'] = cal3d_dll.CalCoreSubmesh_GetLodCount(submesh)
        result[i]['vertexCount'] = cal3d_dll.CalCoreSubmesh_GetVertexCount(submesh)
    gMeshInfoCache[cachekey] = result
    return result
{'fraction_non_alphanumeric': 0.08650447078394677, 'fraction_numerical': 0.013516323559991682, 'mean_word_length': 4.6389214536928485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2588665', 'n_tokens_mistral': 1648, 'n_tokens_neox': 1593, 'n_words': 272}