<commit_before>﻿using System;
using System.Collections.Generic;
using System.Text;
using NekoKun.RubyBindings;

namespace NekoKun.RPGMaker
{
    public class ScriptListFile : NekoKun.ScriptListFile
    {
        protected string scriptDir;
        public ScriptListFile(string filename)
            : base(filename)
        {
            Program.Logger.Log("加载脚本索引文件：{0}", filename);

            using (System.IO.FileStream scriptFile = System.IO.File.Open(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read))
            {
                List<object> scripts = RubyBindings.RubyMarshal.Load(scriptFile, true) as List<object>;

                foreach (List<object> item in scripts)
                {
                    string title;
                    byte[] bytes;

                    var ran = new System.Random();
                    int id = (int)item[0];
                    while (containsID(id))
                        id = ran.Next(1, 100000000);

                    if (item[1] is RubyBindings.RubyExpendObject)
                        title = UnicodeStringFromUTF8Bytes((byte[])((RubyBindings.RubyExpendObject)item[1]).BaseObject);
                    else
                        title = UnicodeStringFromUTF8Bytes((byte[])item[1]);

                    if (item[2] is RubyBindings.RubyExpendObject)
                        bytes = (byte[])((RubyBindings.RubyExpendObject)item[2]).BaseObject;
                    else
                        bytes = (byte[])item[2];

                    byte[] inflated = Ionic.Zlib.ZlibStream.UncompressBuffer(bytes);
                    string code = "";

                    if (inflated.Length > 0) code = System.Text.Encoding.UTF8.GetString(inflated);

                    this.scripts.Add(new ScriptFile(this, code, title, id));
                }

                scriptFile.Close();
            }
        }

        public ScriptListFile(Dictionary<string, object> node)
            : this(
                System.IO.Path.Combine(
                    ProjectManager.ProjectDir,
                    node["FileName"].ToString()
                )<commit_msg>Dead R?DATA? ScriptList File Fix<commit_after>﻿using System;
using System.Collections.Generic;
using System.Text;
using NekoKun.RubyBindings;

namespace NekoKun.RPGMaker
{
    public class ScriptListFile : NekoKun.ScriptListFile
    {
        protected string scriptDir;
        public ScriptListFile(string filename)
            : base(filename)
        {
            Program.Logger.Log("加载脚本索引文件：{0}", filename);

            using (System.IO.FileStream scriptFile = System.IO.File.Open(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read))
            {
                List<object> scripts = RubyBindings.RubyMarshal.Load(scriptFile, true) as List<object>;

                foreach (List<object> item in scripts)
                {
                    string title;
                    byte[] bytes;

                    var ran = new System.Random();
					int id;
					
					if (item[0] is int)
						id = (int)item[0];
					else
					{
						id = ran.Next(1, 100000000);
						this.MakeDirty();
						Program.Logger.Log("唯一 ID 遭到破坏。{0}", item[0].ToString());
		                Program.Logger.Log("因为脚本文件中存在错误，现已经被修复，因此文件脏了。");
					}
						
                    while (containsID(id))
					{
                        id = ran.Next(1, 100000000);
						this.MakeDirty();
					}

                    if (item[1] is RubyBindings.RubyExpendObject)
                        title = UnicodeStringFromUTF8Bytes((byte[])((RubyBindings.RubyExpendObject)item[1]).BaseObject);
                    else
                        title = UnicodeStringFromUTF8Bytes((byte[])item[1]);

                    if (item[2] is RubyBindings.RubyExpendObject)
                        bytes = (byte[])((RubyBindings.RubyExpendObject)item[2]).BaseObject;
                    else
                        bytes = (byte[])item[2];

                    byte[] inflated = Ionic.Zlib.ZlibStream.UncompressBuffer(bytes);
                    string code = "";

                    if (inflated.Length > 0) code = System.Text.Encoding.UTF8.GetString(inflated);

                    this.scripts.Add(new ScriptFile(this, code, title, id));
                }

                scriptFile.Close();
            }
        }

        public ScriptListFile(Dictionary<string, object> node)
            : this(
                System.IO.Path.Combine(
                    ProjectManager.ProjectDir,
                    node["FileName"].ToString()
                )
{'fraction_non_alphanumeric': 0.10603732162458837, 'fraction_numerical': 0.012513721185510428, 'mean_word_length': 1.5711060948081264, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5924259', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1204, 'n_words': 216}