<commit_before>#!/usr/bin/env python
import os
import sys
import unitypack
import yaml
from argparse import ArgumentParser


def handle_asset(asset):
	for id, obj in asset.objects.items():
		d = obj.read()

		print(yaml.dump(d))


def asset_representer(dumper, data):
	return dumper.represent_scalar("!asset", data.name)
yaml.add_representer(unitypack.Asset, asset_representer)


def objectpointer_representer(dumper, data):
	return dumper.represent_sequence("!PPtr", [data.file_id, data.path_id])
yaml.add_representer(unitypack.ObjectPointer, objectpointer_representer)


def shader_representer(dumper, data):
	obj = data._obj.copy()
	obj["m_Script"] = "<stripped>"
	return dumper.represent_mapping("!Shader", obj)


def textasset_representer(dumper, data):
	obj = data._obj.copy()
	obj["m_Script"] = "<stripped>"
	return dumper.represent_mapping("!TextAsset", obj)


def texture2d_representer(dumper, data):
	obj = data._obj.copy()
	obj["image data"] = "<stripped>"
	return dumper.represent_mapping("!Texture2D", obj)


def mesh_representer(dumper, data):
	return dumper.represent_mapping("!Mesh", {data.name: "<stripped>"})


def movietexture_representer(dumper, data):
	obj = data._obj.copy()
	obj["m_MovieData"] = "<stripped>"
	return dumper.represent_mapping("!MovieTexture", obj)


def main():
	p = ArgumentParser()
	p.add_argument("files", nargs="+")
	p.add_argument("-s", "--strip", action="store_true", help="Strip extractable data")
	args = p.parse_args(sys.argv[1:])

	if args.strip:
		yaml.add_representer(unitypack.engine.mesh.Mesh, mesh_representer)
		yaml.add_representer(unitypack.engine.movie.MovieTexture, movietexture_representer)
		yaml.add_representer(unitypack.engine.text.Shader, shader_representer)
		yaml.add_representer(unitypack.engine.text.TextAsset, textasset_representer)
		yaml.add_representer(unitypack.engine.texture.Texture2D, texture2d_representer)

	for file in args.files:
		if file.endswith(".assets"):<commit_msg>Add a generic Unity Object representation in yaml<commit_after>#!/usr/bin/env python
import os
import sys
import unitypack
import yaml
from argparse import ArgumentParser


def handle_asset(asset):
	for id, obj in asset.objects.items():
		d = obj.read()

		print(yaml.dump(d))


def asset_representer(dumper, data):
	return dumper.represent_scalar("!asset", data.name)
yaml.add_representer(unitypack.Asset, asset_representer)


def objectpointer_representer(dumper, data):
	return dumper.represent_sequence("!PPtr", [data.file_id, data.path_id])
yaml.add_representer(unitypack.ObjectPointer, objectpointer_representer)


def unityobj_representer(dumper, data):
	return dumper.represent_mapping("!unitypack:%s" % (data.__class__.__name__), data._obj)


def shader_representer(dumper, data):
	obj = data._obj.copy()
	obj["m_Script"] = "<stripped>"
	return dumper.represent_mapping("!Shader", obj)


def textasset_representer(dumper, data):
	obj = data._obj.copy()
	obj["m_Script"] = "<stripped>"
	return dumper.represent_mapping("!TextAsset", obj)


def texture2d_representer(dumper, data):
	obj = data._obj.copy()
	obj["image data"] = "<stripped>"
	return dumper.represent_mapping("!Texture2D", obj)


def mesh_representer(dumper, data):
	return dumper.represent_mapping("!Mesh", {data.name: "<stripped>"})


def movietexture_representer(dumper, data):
	obj = data._obj.copy()
	obj["m_MovieData"] = "<stripped>"
	return dumper.represent_mapping("!MovieTexture", obj)


def main():
	p = ArgumentParser()
	p.add_argument("files", nargs="+")
	p.add_argument("-s", "--strip", action="store_true", help="Strip extractable data")
	args = p.parse_args(sys.argv[1:])

	for k, v in unitypack.engine.__dict__.items():
		if isinstance(v, type) and issubclass(v, unitypack.engine.object.Object):
			yaml.add_representer(v, unityobj_representer)

	if args.strip:
		yaml.add_representer(unitypack.engine.mesh.Mesh, mesh_representer)
		yaml.add_representer(unitypack.engine.movie.MovieTexture, movietexture_representer)
		yaml.add_representer(unitypack.engine.text.Shader, shader_representer)
		yaml.add_representer(unitypack.engine.text.TextAsset, textasset_representer)
		yaml.add_representer(unitypack.engine.texture.Texture2D, texture2d_representer)

	for file in args.files:
		if file.endswith(".assets"):
{'fraction_non_alphanumeric': 0.13750882976218506, 'fraction_numerical': 0.0023546032493524842, 'mean_word_length': 6.695652173913044, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '302257', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1427, 'n_words': 268}