<commit_before>/*
 * Copyright (c) Contributors, http://opensimulator.org/
 * See CONTRIBUTORS.TXT for a full list of copyright holders.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the OpenSimulator Project nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

using System;
using System.IO;
using System.Runtime.Remoting;
using System.Runtime.Remoting.Lifetime;
using System.Threading;
using System.Collections;
using System.Collections.Generic;
using System.Security.Policy;
using System.Reflection;
using System.Globalization;
using System.Xml;
using OpenMetaverse;
using log4net;
using Nini.Config;
using Amib.Threading;
using OpenSim.Framework;
using OpenSim.Region.CoreModules;
using OpenSim.Region.Framework.Scenes;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.ScriptEngine.Shared;
using OpenSim.Region.ScriptEngine.Shared.Api;
using OpenSim.Region.ScriptEngine.Shared.Api.Runtime;
using OpenSim.Region.ScriptEngine.Shared.ScriptBase;
using OpenSim.Region.ScriptEngine.Shared.CodeTools;
using OpenSim.Region.ScriptEngine.Interfaces;

namespace OpenSim.Region.ScriptEngine.Shared.Instance
{
    public class ScriptInstance : MarshalByRefObject, IScriptInstance
    {
        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

        /// <summary>
        /// The current work item if an event for this script is running or waiting to run,
        /// </summary>
        /// <remarks>
        /// Null if there is no running or waiting to run event.  Must be changed only under an EventQueue lock.
        /// </remarks>
        private IScriptWorkItem m_CurrentWorkItem;

        private IScript m_Script;
        private DetectParams[] m_DetectParams;
        private bool m_TimerQueued;
        private DateTime m_EventStart;
        private bool m_InEvent;
        private string m_Assembly;
        private string m_CurrentEvent = String.Empty;
        private bool m_InSelfDelete;
        private int m_MaxScriptQueue;
        private bool m_SaveState = true;
        private int m_ControlEventsInQueue;
        private int m_LastControlLevel;
        private bool m_CollisionInQueue;

        // The following is for setting a minimum delay between events
        private double m_minEventDelay;
        
        private long m_eventDelayTicks;
        private long m_nextEventTimeTicks;
        private bool m_startOnInit = true;
        private UUID m_AttachedAvatar;
        private StateSource m_stateSource;
        private bool m_postOnRez;
        private bool m_startedFromSavedState;
        private UUID m_CurrentStateHash;
        private UUID m_RegionID;

        public Dictionary<KeyValuePair<int, int>, KeyValuePair<int, int>> LineMap { get; set; }

        private Dictionary<string,IScriptApi> m_Apis = new Dictionary<string,IScriptApi>();

        public Object[] PluginData = new Object[0];

        /// <summary>
        /// Used by llMinEventDelay to suppress events happening any faster than this speed.
        /// This currently restricts all events in one go. Not sure if each event type has
        /// its own check so take the simple route first.
        /// </summary>
        public double MinEventDelay
        {
            get { return m_minEventDelay; }
            set
            {
                if (value > 0.001)
                    m_minEventDelay = value;
                else 
                    m_minEventDelay = 0.0;

                m_eventDelayTicks = (long)(m_minEventDelay * 10000000L);
                m_nextEventTimeTicks = DateTime.Now.Ticks;
            }
        }

        public bool Running { get; set; }

        public bool Suspended
        {
            get { return m_Suspended; }

            set
            {
                // Need to do this inside a lock in order to avoid races with EventProcessor()
                lock (m_Script)
                {
                    bool wasSuspended = m_Suspended;
                    m_Suspended = value;
    
                    if (wasSuspended && !m_Suspended)
                    {
                        lock (EventQueue)
                        {
                            // Need to place ourselves back in a work item if there are events to process
                            if (EventQueue.Count > 0 && Running && !ShuttingDown)
                                m_CurrentWorkItem = Engine.QueueEventHandler(this);
                        }
                    }
                }
            }
        }
        private bool m_Suspended;

        public bool ShuttingDown { get; set; }

        public string State { get; set; }

        public IScriptEngine Engine { get; private set; }

        public UUID AppDomain { get; set; }

        public string PrimName { get; private set; }

        public string ScriptName { get; private set; }

        public UUID ItemID { get; private set; }

        public UUID ObjectID { get; private set; }

        public uint LocalID { get; private set; }

        public UUID RootObjectID { get; private set; }

        public uint RootLocalID { get; private set; }

        public UUID AssetID { get; private set; }

        public Queue EventQueue { get; private set; }

        public int StartParam { get; set; }

        public TaskInventoryItem ScriptTask { get; private set; }

        public DateTime TimeStarted { get; private set; }

        public long MeasurementPeriodTickStart { get; private set; }

        public long MeasurementPeriodExecutionTime { get; private set; }

        public static readonly long MaxMeasurementPeriod = 30 * TimeSpan.TicksPerMinute;

        public void ClearQueue()
        {
            m_TimerQueued = false;
            EventQueue.Clear();
        }

        public ScriptInstance(IScriptEngine engine, SceneObjectPart part,
                UUID itemID, UUID assetID, string assembly,
                AppDomain dom, string primName, string scriptName,
                int startParam, bool postOnRez, StateSource stateSource,
                int maxScriptQueue)
        {
            State = "default";
            EventQueue = new Queue(32);

            Engine = engine;
            LocalID = part.LocalId;
            ObjectID = part.UUID;
            RootLocalID = part.ParentGroup.LocalId;
            RootObjectID = part.ParentGroup.UUID;
            ItemID = itemID;
            AssetID = assetID;
            PrimName = primName;
            ScriptName = scriptName;
            m_Assembly = assembly;
            StartParam = startParam;
            m_MaxScriptQueue = maxScriptQueue;
            m_stateSource = stateSource;
            m_postOnRez = postOnRez;
            m_AttachedAvatar = part.ParentGroup.AttachedAvatar;
            m_RegionID = part.ParentGroup.Scene.RegionInfo.RegionID;

            if (part != null)
            {
                lock (part.TaskInventory)
                {
                    if (part.TaskInventory.ContainsKey(ItemID))
                    {
                        ScriptTask = part.TaskInventory[ItemID];
                    }
                }
            }

            ApiManager am = new ApiManager();

            foreach (string api in am.GetApis())
            {
                m_Apis[api] = am.CreateApi(api);
                m_Apis[api].Initialize(engine, part, ScriptTask);
            }
    
            try
            {
                if (dom != System.AppDomain.CurrentDomain)
                    m_Script = (IScript)dom.CreateInstanceAndUnwrap(
                            Path.GetFileNameWithoutExtension(assembly),
                            "SecondLife.Script");
                else
                    m_Script = (IScript)Assembly.Load(
                            Path.GetFileNameWithoutExtension(assembly)).CreateInstance(
                            "SecondLife.Script");

                //ILease lease = (ILease)RemotingServices.GetLifetimeService(m_Script as ScriptBaseClass);
                //RemotingServices.GetLifetimeService(m_Script as ScriptBaseClass);
//                lease.Register(this);
            }
            catch (Exception e)
            {
                m_log.ErrorFormat(
                    "[SCRIPT INSTANCE]: Error loading assembly {0}.  Exception {1}{2}",
                    assembly, e.Message, e.StackTrace);
            }

            try
            {
                foreach (KeyValuePair<string,IScriptApi> kv in m_Apis)
                {
                    m_Script.InitApi(kv.Key, kv.Value);
                }

//                // m_log.Debug("[Script] Script instance created");

                part.SetScriptEvents(ItemID,
                                     (int)m_Script.GetStateEventFlags(State));
            }
            catch (Exception e)
            {
                m_log.ErrorFormat(
                    "[SCRIPT INSTANCE]: Error loading script instance from assembly {0}.  Exception {1}{2}",
                    assembly, e.Message, e.StackTrace);

                return;
            }

            m_SaveState = true;

            string savedState = Path.Combine(Path.GetDirectoryName(assembly),
                    ItemID.ToString() + ".state");
            if (File.Exists(savedState))
            {
                string xml = String.Empty;

                try
                {
                    FileInfo fi = new FileInfo(savedState);
                    int size = (int)fi.Length;
                    if (size < 512000)
                    {
                        using (FileStream fs = File.Open(savedState,
                                                         FileMode.Open, FileAccess.Read, FileShare.None))
                        {
                            System.Text.UTF8Encoding enc =
                                new System.Text.UTF8Encoding();

                            Byte[] data = new Byte[size];
                            fs.Read(data, 0, size);

                            xml = enc.GetString(data);

                            ScriptSerializer.Deserialize(xml, this);

                            AsyncCommandManager.CreateFromData(Engine,
                                LocalID, ItemID, ObjectID,
                                PluginData);

//                            m_log.DebugFormat("[Script] Successfully retrieved state for script {0}.{1}", PrimName, m_ScriptName);

                            part.SetScriptEvents(ItemID,
                                    (int)m_Script.GetStateEventFlags(State));

                            Running = false;

                            if (ShuttingDown)
                                m_startOnInit = false;

                            // we get new rez events on sim restart, too
                            // but if there is state, then we fire the change
                            // event

                            // We loaded state, don't force a re-save
                            m_SaveState = false;
                            m_startedFromSavedState = true;
                        }
                    }
                    else
                    {
                        m_log.ErrorFormat(
                            "[SCRIPT INSTANCE]: Unable to load script state file {0} from assembly {1}: Memory limit exceeded",
                            savedState, assembly);
                    }
                }
                catch (Exception e)
                {
                     m_log.ErrorFormat(
                         "[SCRIPT INSTANCE]: Unable to load script state file {0} from assembly {1}.  XML is {2}.  Exception {3}{4}",
                         savedState, assembly, xml, e.Message, e.StackTrace);
                }
            }
//            else
//            {
//                ScenePresence presence = Engine.World.GetScenePresence(part.OwnerID);

//                if (presence != null && (!postOnRez))
//                    presence.ControllingClient.SendAgentAlertMessage("Compile successful", false);

//            }
        }

        public void Init()
        {
            if (!m_startOnInit)
                return;

            if (m_startedFromSavedState) 
            {
                Start();
                if (m_postOnRez) 
                {
                    PostEvent(new EventParams("on_rez",
                        new Object[] {new LSL_Types.LSLInteger(StartParam)}, new DetectParams[0]));
                }

                if (m_stateSource == StateSource.AttachedRez)
                {
                    PostEvent(new EventParams("attach",
                        new object[] { new LSL_Types.LSLString(m_AttachedAvatar.ToString()) }, new DetectParams[0]));
                }<commit_msg>Add item id, name, prim name and id to log message when state exists but loading fails.<commit_after>/*
 * Copyright (c) Contributors, http://opensimulator.org/
 * See CONTRIBUTORS.TXT for a full list of copyright holders.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the OpenSimulator Project nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

using System;
using System.IO;
using System.Runtime.Remoting;
using System.Runtime.Remoting.Lifetime;
using System.Threading;
using System.Collections;
using System.Collections.Generic;
using System.Security.Policy;
using System.Reflection;
using System.Globalization;
using System.Xml;
using OpenMetaverse;
using log4net;
using Nini.Config;
using Amib.Threading;
using OpenSim.Framework;
using OpenSim.Region.CoreModules;
using OpenSim.Region.Framework.Scenes;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.ScriptEngine.Shared;
using OpenSim.Region.ScriptEngine.Shared.Api;
using OpenSim.Region.ScriptEngine.Shared.Api.Runtime;
using OpenSim.Region.ScriptEngine.Shared.ScriptBase;
using OpenSim.Region.ScriptEngine.Shared.CodeTools;
using OpenSim.Region.ScriptEngine.Interfaces;

namespace OpenSim.Region.ScriptEngine.Shared.Instance
{
    public class ScriptInstance : MarshalByRefObject, IScriptInstance
    {
        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

        /// <summary>
        /// The current work item if an event for this script is running or waiting to run,
        /// </summary>
        /// <remarks>
        /// Null if there is no running or waiting to run event.  Must be changed only under an EventQueue lock.
        /// </remarks>
        private IScriptWorkItem m_CurrentWorkItem;

        private IScript m_Script;
        private DetectParams[] m_DetectParams;
        private bool m_TimerQueued;
        private DateTime m_EventStart;
        private bool m_InEvent;
        private string m_Assembly;
        private string m_CurrentEvent = String.Empty;
        private bool m_InSelfDelete;
        private int m_MaxScriptQueue;
        private bool m_SaveState = true;
        private int m_ControlEventsInQueue;
        private int m_LastControlLevel;
        private bool m_CollisionInQueue;

        // The following is for setting a minimum delay between events
        private double m_minEventDelay;
        
        private long m_eventDelayTicks;
        private long m_nextEventTimeTicks;
        private bool m_startOnInit = true;
        private UUID m_AttachedAvatar;
        private StateSource m_stateSource;
        private bool m_postOnRez;
        private bool m_startedFromSavedState;
        private UUID m_CurrentStateHash;
        private UUID m_RegionID;

        public Dictionary<KeyValuePair<int, int>, KeyValuePair<int, int>> LineMap { get; set; }

        private Dictionary<string,IScriptApi> m_Apis = new Dictionary<string,IScriptApi>();

        public Object[] PluginData = new Object[0];

        /// <summary>
        /// Used by llMinEventDelay to suppress events happening any faster than this speed.
        /// This currently restricts all events in one go. Not sure if each event type has
        /// its own check so take the simple route first.
        /// </summary>
        public double MinEventDelay
        {
            get { return m_minEventDelay; }
            set
            {
                if (value > 0.001)
                    m_minEventDelay = value;
                else 
                    m_minEventDelay = 0.0;

                m_eventDelayTicks = (long)(m_minEventDelay * 10000000L);
                m_nextEventTimeTicks = DateTime.Now.Ticks;
            }
        }

        public bool Running { get; set; }

        public bool Suspended
        {
            get { return m_Suspended; }

            set
            {
                // Need to do this inside a lock in order to avoid races with EventProcessor()
                lock (m_Script)
                {
                    bool wasSuspended = m_Suspended;
                    m_Suspended = value;
    
                    if (wasSuspended && !m_Suspended)
                    {
                        lock (EventQueue)
                        {
                            // Need to place ourselves back in a work item if there are events to process
                            if (EventQueue.Count > 0 && Running && !ShuttingDown)
                                m_CurrentWorkItem = Engine.QueueEventHandler(this);
                        }
                    }
                }
            }
        }
        private bool m_Suspended;

        public bool ShuttingDown { get; set; }

        public string State { get; set; }

        public IScriptEngine Engine { get; private set; }

        public UUID AppDomain { get; set; }

        public string PrimName { get; private set; }

        public string ScriptName { get; private set; }

        public UUID ItemID { get; private set; }

        public UUID ObjectID { get; private set; }

        public uint LocalID { get; private set; }

        public UUID RootObjectID { get; private set; }

        public uint RootLocalID { get; private set; }

        public UUID AssetID { get; private set; }

        public Queue EventQueue { get; private set; }

        public int StartParam { get; set; }

        public TaskInventoryItem ScriptTask { get; private set; }

        public DateTime TimeStarted { get; private set; }

        public long MeasurementPeriodTickStart { get; private set; }

        public long MeasurementPeriodExecutionTime { get; private set; }

        public static readonly long MaxMeasurementPeriod = 30 * TimeSpan.TicksPerMinute;

        public void ClearQueue()
        {
            m_TimerQueued = false;
            EventQueue.Clear();
        }

        public ScriptInstance(IScriptEngine engine, SceneObjectPart part,
                UUID itemID, UUID assetID, string assembly,
                AppDomain dom, string primName, string scriptName,
                int startParam, bool postOnRez, StateSource stateSource,
                int maxScriptQueue)
        {
            State = "default";
            EventQueue = new Queue(32);

            Engine = engine;
            LocalID = part.LocalId;
            ObjectID = part.UUID;
            RootLocalID = part.ParentGroup.LocalId;
            RootObjectID = part.ParentGroup.UUID;
            ItemID = itemID;
            AssetID = assetID;
            PrimName = primName;
            ScriptName = scriptName;
            m_Assembly = assembly;
            StartParam = startParam;
            m_MaxScriptQueue = maxScriptQueue;
            m_stateSource = stateSource;
            m_postOnRez = postOnRez;
            m_AttachedAvatar = part.ParentGroup.AttachedAvatar;
            m_RegionID = part.ParentGroup.Scene.RegionInfo.RegionID;

            if (part != null)
            {
                lock (part.TaskInventory)
                {
                    if (part.TaskInventory.ContainsKey(ItemID))
                    {
                        ScriptTask = part.TaskInventory[ItemID];
                    }
                }
            }

            ApiManager am = new ApiManager();

            foreach (string api in am.GetApis())
            {
                m_Apis[api] = am.CreateApi(api);
                m_Apis[api].Initialize(engine, part, ScriptTask);
            }
    
            try
            {
                if (dom != System.AppDomain.CurrentDomain)
                    m_Script = (IScript)dom.CreateInstanceAndUnwrap(
                            Path.GetFileNameWithoutExtension(assembly),
                            "SecondLife.Script");
                else
                    m_Script = (IScript)Assembly.Load(
                            Path.GetFileNameWithoutExtension(assembly)).CreateInstance(
                            "SecondLife.Script");

                //ILease lease = (ILease)RemotingServices.GetLifetimeService(m_Script as ScriptBaseClass);
                //RemotingServices.GetLifetimeService(m_Script as ScriptBaseClass);
//                lease.Register(this);
            }
            catch (Exception e)
            {
                m_log.ErrorFormat(
                    "[SCRIPT INSTANCE]: Error loading assembly {0}.  Exception {1}{2}",
                    assembly, e.Message, e.StackTrace);
            }

            try
            {
                foreach (KeyValuePair<string,IScriptApi> kv in m_Apis)
                {
                    m_Script.InitApi(kv.Key, kv.Value);
                }

//                // m_log.Debug("[Script] Script instance created");

                part.SetScriptEvents(ItemID,
                                     (int)m_Script.GetStateEventFlags(State));
            }
            catch (Exception e)
            {
                m_log.ErrorFormat(
                    "[SCRIPT INSTANCE]: Error loading script instance from assembly {0}.  Exception {1}{2}",
                    assembly, e.Message, e.StackTrace);

                return;
            }

            m_SaveState = true;

            string savedState = Path.Combine(Path.GetDirectoryName(assembly),
                    ItemID.ToString() + ".state");
            if (File.Exists(savedState))
            {
                string xml = String.Empty;

                try
                {
                    FileInfo fi = new FileInfo(savedState);
                    int size = (int)fi.Length;
                    if (size < 512000)
                    {
                        using (FileStream fs = File.Open(savedState,
                                                         FileMode.Open, FileAccess.Read, FileShare.None))
                        {
                            System.Text.UTF8Encoding enc =
                                new System.Text.UTF8Encoding();

                            Byte[] data = new Byte[size];
                            fs.Read(data, 0, size);

                            xml = enc.GetString(data);

                            ScriptSerializer.Deserialize(xml, this);

                            AsyncCommandManager.CreateFromData(Engine,
                                LocalID, ItemID, ObjectID,
                                PluginData);

//                            m_log.DebugFormat("[Script] Successfully retrieved state for script {0}.{1}", PrimName, m_ScriptName);

                            part.SetScriptEvents(ItemID,
                                    (int)m_Script.GetStateEventFlags(State));

                            Running = false;

                            if (ShuttingDown)
                                m_startOnInit = false;

                            // we get new rez events on sim restart, too
                            // but if there is state, then we fire the change
                            // event

                            // We loaded state, don't force a re-save
                            m_SaveState = false;
                            m_startedFromSavedState = true;
                        }
                    }
                    else
                    {
                        m_log.WarnFormat(
                            "[SCRIPT INSTANCE]: Unable to load script state file {0} for script {1} {2} in {3} {4} (assembly {5}).  Memory limit exceeded",
                            savedState, ScriptName, ItemID, PrimName, ObjectID, assembly);
                    }
                }
                catch (Exception e)
                {
                     m_log.ErrorFormat(
                         "[SCRIPT INSTANCE]: Unable to load script state file {0} for script {1} {2} in {3} {4} (assembly {5}).  XML is {6}.  Exception {7}{8}",
                         savedState, ScriptName, ItemID, PrimName, ObjectID, assembly, xml, e.Message, e.StackTrace);
                }
            }
//            else
//            {
//                ScenePresence presence = Engine.World.GetScenePresence(part.OwnerID);

//                if (presence != null && (!postOnRez))
//                    presence.ControllingClient.SendAgentAlertMessage("Compile successful", false);

//            }
        }

        public void Init()
        {
            if (!m_startOnInit)
                return;

            if (m_startedFromSavedState) 
            {
                Start();
                if (m_postOnRez) 
                {
                    PostEvent(new EventParams("on_rez",
                        new Object[] {new LSL_Types.LSLInteger(StartParam)}, new DetectParams[0]));
                }

                if (m_stateSource == StateSource.AttachedRez)
                {
                    PostEvent(new EventParams("attach",
                        new object[] { new LSL_Types.LSLString(m_AttachedAvatar.ToString()) }, new DetectParams[0]));
                }
{'fraction_non_alphanumeric': 0.07252662012552007, 'fraction_numerical': 0.0035963613285381847, 'mean_word_length': 1.5566071750495762, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6270560', 'n_tokens_mistral': 7162, 'n_tokens_neox': 6673, 'n_words': 2042}