<commit_before>﻿// Nu Game Engine.
// Copyright (C) <NAME>, 2013-2020.

namespace Nu
open System
open System.ComponentModel
open System.Numerics
open Prime
open Nu

/// Masks for Transform flags.
module TransformMasks =

    // OPTIMIZATION: Transform flag bit-masks for performance.
    let [<Literal>] ActiveMask =                    0b000000000000000000001u
    let [<Literal>] DirtyMask =                     0b000000000000000000010u
    let [<Literal>] InvalidatedMask =               0b000000000000000000100u
    let [<Literal>] OmnipresentMask =               0b000000000000000001000u
    let [<Literal>] AbsoluteMask =                  0b000000000000000010000u
    let [<Literal>] ImperativeMask =                0b000000000000000100000u
    let [<Literal>] PublishChangeBindingsMask =     0b000000000000001000000u
    let [<Literal>] PublishChangeEventsMask =       0b000000000000010000000u
    let [<Literal>] EnabledMask =                   0b000000000000100000000u
    let [<Literal>] VisibleMask =                   0b000000000001000000000u
    let [<Literal>] AlwaysUpdateMask =              0b000000000010000000000u
    let [<Literal>] PublishUpdatesMask =            0b000000000100000000000u
    let [<Literal>] PublishPostUpdatesMask =        0b000000001000000000000u
    let [<Literal>] PersistentMask =                0b000000010000000000000u
    let [<Literal>] IgnorePropertyBindingsMask =    0b000000100000000000000u
    let [<Literal>] MountedMask =                   0b000001000000000000000u
    let [<Literal>] EnabledLocalMask =              0b000010000000000000000u
    let [<Literal>] VisibleLocalMask =              0b000100000000000000000u
    let [<Literal>] RotationMatrixDirtyMask =       0b001000000000000000000u
    let [<Literal>] AffineMatrixDirtyMask =         0b010000000000000000000u
    let [<Literal>] Is2dMask =                      0b100000000000000000000u
    let [<Literal>] DefaultFlags =                  0b000110010001100100001u

// NOTE: opening this in order to make the Transform property implementations reasonably succinct.
open TransformMasks

/// Carries transformation data specific to an Entity.
type [<NoEquality; NoComparison>] Transform =
    struct
        // cache line 1
        val mutable private Flags_ : uint
        val mutable private Position_ : Vector3
        val mutable private Rotation_ : Quaternion
        // cache line 2
        val mutable private Scale_ : Vector3
        val mutable private Offset_ : Vector3
        val mutable private RotationMatrixOpt_ : Matrix4x4 ref
        val mutable private AffineMatrixOpt_ : Matrix4x4 ref
        // cache line 3
        //val mutable private AABBOpt_ : Box3 ref
        val mutable private Angles_ : Vector3
        val mutable private Size_ : Vector3
        val mutable private Elevation_ : single
        end

    member this.Active with get () = this.Flags_ &&& ActiveMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| ActiveMask else this.Flags_ &&& ~~~ActiveMask
    member this.Dirty with get () = this.Flags_ &&& DirtyMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| DirtyMask else this.Flags_ &&& ~~~DirtyMask
    member this.Invalidated with get () = this.Flags_ &&& InvalidatedMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| InvalidatedMask else this.Flags_ &&& ~~~InvalidatedMask
    member this.Omnipresent with get () = this.Flags_ &&& OmnipresentMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| OmnipresentMask else this.Flags_ &&& ~~~OmnipresentMask
    member this.Absolute with get () = this.Flags_ &&& AbsoluteMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| AbsoluteMask else this.Flags_ &&& ~~~AbsoluteMask
    member this.Imperative with get () = this.Flags_ &&& ImperativeMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| ImperativeMask else this.Flags_ &&& ~~~ImperativeMask
    member this.PublishChangeBindings with get () = this.Flags_ &&& PublishChangeBindingsMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PublishChangeBindingsMask else this.Flags_ &&& ~~~PublishChangeBindingsMask
    member this.PublishChangeEvents with get () = this.Flags_ &&& PublishChangeEventsMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PublishChangeEventsMask else this.Flags_ &&& ~~~PublishChangeEventsMask
    member this.Enabled with get () = this.Flags_ &&& EnabledMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| EnabledMask else this.Flags_ &&& ~~~EnabledMask
    member this.Visible with get () = this.Flags_ &&& VisibleMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| VisibleMask else this.Flags_ &&& ~~~VisibleMask
    member this.AlwaysUpdate with get () = this.Flags_ &&& AlwaysUpdateMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| AlwaysUpdateMask else this.Flags_ &&& ~~~AlwaysUpdateMask
    member this.PublishUpdates with get () = this.Flags_ &&& PublishUpdatesMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PublishUpdatesMask else this.Flags_ &&& ~~~PublishUpdatesMask
    member this.PublishPostUpdates with get () = this.Flags_ &&& PublishPostUpdatesMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PublishPostUpdatesMask else this.Flags_ &&& ~~~PublishPostUpdatesMask
    member this.Persistent with get () = this.Flags_ &&& PersistentMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PersistentMask else this.Flags_ &&& ~~~PersistentMask
    member this.IgnorePropertyBindings with get () = this.Flags_ &&& IgnorePropertyBindingsMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| IgnorePropertyBindingsMask else this.Flags_ &&& ~~~IgnorePropertyBindingsMask
    member this.Mounted with get () = this.Flags_ &&& MountedMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| MountedMask else this.Flags_ &&& ~~~MountedMask
    member this.EnabledLocal with get () = this.Flags_ &&& EnabledLocalMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| EnabledLocalMask else this.Flags_ &&& ~~~EnabledLocalMask
    member this.VisibleLocal with get () = this.Flags_ &&& VisibleLocalMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| VisibleLocalMask else this.Flags_ &&& ~~~VisibleLocalMask
    member this.RotationMatrixDirty with get () = this.Flags_ &&& RotationMatrixDirtyMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| RotationMatrixDirtyMask else this.Flags_ &&& ~~~RotationMatrixDirtyMask
    member this.AffineMatrixDirty with get () = this.Flags_ &&& AffineMatrixDirtyMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| AffineMatrixDirtyMask else this.Flags_ &&& ~~~AffineMatrixDirtyMask
    member this.Is2d with get () = this.Flags_ &&& Is2dMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| Is2dMask else this.Flags_ &&& ~~~Is2dMask
    member this.Optimized with get () = this.Imperative && this.Omnipresent && not this.PublishChangeBindings && not this.PublishChangeEvents // TODO: see if I can remove all conditionals from here.

    member this.Position with get () = this.Position_ and set value = this.Position_ <- value; this.AffineMatrixDirty <- true
    member this.Scale with get () = this.Scale_ and set value = this.Scale_ <- value; this.AffineMatrixDirty <- true
    member this.Offset with get () = this.Offset_ and set value = this.Offset_ <- value; this.AffineMatrixDirty <- true
    member this.Size with get () = this.Size_ and set value = this.Size_ <- value; this.AffineMatrixDirty <- true
    member this.Elevation with get () = this.Elevation_ and set value = this.Elevation_ <- value; this.AffineMatrixDirty <- true

    member this.Rotation
        with get () = this.Rotation_
        and set value =
            this.Rotation_ <- value
            let pitchYawRoll = value.PitchYawRoll ()
            this.Angles_.X <- pitchYawRoll.X
            this.Angles_.Y <- pitchYawRoll.Y
            this.Angles_.Z <- pitchYawRoll.Z
            this.RotationMatrixDirty <- true
            this.AffineMatrixDirty <- true

    member this.Angles
        with get () = this.Angles_
        and set value =
            this.Angles_ <- value
            this.Rotation_ <- Quaternion.CreateFromYawPitchRoll (value.Y, value.X, value.Z)
            this.RotationMatrixDirty <- true
            this.AffineMatrixDirty <- true

    member this.RotationMatrix =
        if notNull (this.RotationMatrixOpt_ :> obj) then
            if this.RotationMatrixDirty then this.RotationMatrixOpt_ <- ref (Matrix4x4.CreateFromQuaternion this.Rotation_)
            this.RotationMatrixOpt_.Value
        else Matrix4x4.Identity

    member this.AffineMatrix =
        if notNull (this.AffineMatrixOpt_ :> obj) then
            if this.AffineMatrixDirty then
                // TODO: P1: optimize this hella!
                let positionMatrix = Matrix4x4.CreateTranslation this.Position_
                let rotationMatrix = this.RotationMatrix
                let scaleMatrix = Matrix4x4.CreateScale this.Scale_
                this.AffineMatrixOpt_ <- ref (positionMatrix * rotationMatrix * scaleMatrix)
            this.AffineMatrixOpt_.Value
        else Matrix4x4.Identity

    member this.Extent = this.Size_ * 0.5f
    member this.Right = Vector3 (this.RotationMatrix.M11, this.RotationMatrix.M12, this.RotationMatrix.M13) // TODO: implement Row properties.
    member this.Up = Vector3 (this.RotationMatrix.M21, this.RotationMatrix.M22, this.RotationMatrix.M23)
    member this.Forward = -Vector3 (this.RotationMatrix.M31, this.RotationMatrix.M32, this.RotationMatrix.M33)
    member this.Left = -this.Right
    member this.Down = -this.Up
    member this.Backward = -this.Forward

    member this.Dimensions =
        let size = this.Size_
        let extent = size * 0.5f
        let position = this.Position_ - extent
        let offset =
            if not this.Rotation_.IsIdentity
            then Vector3.Transform (this.Offset_ * size, this.Rotation_)
            else this.Offset_ * size
        Box3 (position + offset, size)

    member this.PositionScaled = this.Position_ * this.Scale_
    member this.SizeScaled = this.Size_ * this.Scale_
    member this.ExtentScaled = this.Extent * this.Scale_
    member this.DimensionsScaled =
        let scale = this.Scale_
        let sizeScaled = this.Size_ * scale
        let extentScaled = sizeScaled * 0.5f
        let positionScaled = this.Position_ - extentScaled
        let offsetScaled =
            if not this.Rotation_.IsIdentity
            then Vector3.Transform (this.Offset_ * sizeScaled, this.Rotation_) * scale
            else this.Offset_ * sizeScaled
        Box3 (positionScaled + offsetScaled, sizeScaled)

    member this.AABB =
        let dimensions = this.DimensionsScaled
        dimensions.Orient this.Rotation_

    member this.InvalidateFast () =
        this.Flags_ <- this.Flags_ ||| TransformMasks.InvalidatedMask

    /// Test transforms for equality.
    static member equalsByRef (left : Transform inref, right : Transform inref) =
        left.Flags_ = right.Flags_ &&
        left.Position_.Equals right.Position_ &&
        left.Rotation_.Equals right.Rotation_ &&
        left.Scale_.Equals right.Scale_ &&
        left.Offset_.Equals right.Offset_ &&
        left.Angles_.Equals right.Angles_ &&
        left.Size_.Equals right.Size_ &&<commit_msg>Working on new bounds pipeline.<commit_after>﻿// Nu Game Engine.
// Copyright (C) <NAME>, 2013-2020.

namespace Nu
open System
open System.ComponentModel
open System.Numerics
open Prime
open Nu

/// Masks for Transform flags.
module TransformMasks =

    // OPTIMIZATION: Transform flag bit-masks for performance.
    let [<Literal>] ActiveMask =                    0b000000000000000000001u
    let [<Literal>] DirtyMask =                     0b000000000000000000010u
    let [<Literal>] InvalidatedMask =               0b000000000000000000100u
    let [<Literal>] OmnipresentMask =               0b000000000000000001000u
    let [<Literal>] AbsoluteMask =                  0b000000000000000010000u
    let [<Literal>] ImperativeMask =                0b000000000000000100000u
    let [<Literal>] PublishChangeBindingsMask =     0b000000000000001000000u
    let [<Literal>] PublishChangeEventsMask =       0b000000000000010000000u
    let [<Literal>] EnabledMask =                   0b000000000000100000000u
    let [<Literal>] VisibleMask =                   0b000000000001000000000u
    let [<Literal>] AlwaysUpdateMask =              0b000000000010000000000u
    let [<Literal>] PublishUpdatesMask =            0b000000000100000000000u
    let [<Literal>] PublishPostUpdatesMask =        0b000000001000000000000u
    let [<Literal>] PersistentMask =                0b000000010000000000000u
    let [<Literal>] IgnorePropertyBindingsMask =    0b000000100000000000000u
    let [<Literal>] MountedMask =                   0b000001000000000000000u
    let [<Literal>] EnabledLocalMask =              0b000010000000000000000u
    let [<Literal>] VisibleLocalMask =              0b000100000000000000000u
    let [<Literal>] RotationMatrixDirtyMask =       0b001000000000000000000u
    let [<Literal>] AffineMatrixDirtyMask =         0b010000000000000000000u
    let [<Literal>] Is2dMask =                      0b100000000000000000000u
    let [<Literal>] DefaultFlags =                  0b000110010001100100001u

// NOTE: opening this in order to make the Transform property implementations reasonably succinct.
open TransformMasks

/// Carries transformation data specific to an Entity.
type [<NoEquality; NoComparison>] Transform =
    struct
        // cache line 1
        val mutable private Flags_ : uint
        val mutable private Position_ : Vector3
        val mutable private Rotation_ : Quaternion
        // cache line 2
        val mutable private Scale_ : Vector3
        val mutable private Offset_ : Vector3
        val mutable private RotationMatrixOpt_ : Matrix4x4 ref
        val mutable private AffineMatrixOpt_ : Matrix4x4 ref
        // cache line 3
        //val mutable private AABBOpt_ : Box3 ref
        val mutable private Angles_ : Vector3
        val mutable private Size_ : Vector3
        val mutable private Elevation_ : single
        end

    member this.Active with get () = this.Flags_ &&& ActiveMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| ActiveMask else this.Flags_ &&& ~~~ActiveMask
    member this.Dirty with get () = this.Flags_ &&& DirtyMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| DirtyMask else this.Flags_ &&& ~~~DirtyMask
    member this.Invalidated with get () = this.Flags_ &&& InvalidatedMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| InvalidatedMask else this.Flags_ &&& ~~~InvalidatedMask
    member this.Omnipresent with get () = this.Flags_ &&& OmnipresentMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| OmnipresentMask else this.Flags_ &&& ~~~OmnipresentMask
    member this.Absolute with get () = this.Flags_ &&& AbsoluteMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| AbsoluteMask else this.Flags_ &&& ~~~AbsoluteMask
    member this.Imperative with get () = this.Flags_ &&& ImperativeMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| ImperativeMask else this.Flags_ &&& ~~~ImperativeMask
    member this.PublishChangeBindings with get () = this.Flags_ &&& PublishChangeBindingsMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PublishChangeBindingsMask else this.Flags_ &&& ~~~PublishChangeBindingsMask
    member this.PublishChangeEvents with get () = this.Flags_ &&& PublishChangeEventsMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PublishChangeEventsMask else this.Flags_ &&& ~~~PublishChangeEventsMask
    member this.Enabled with get () = this.Flags_ &&& EnabledMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| EnabledMask else this.Flags_ &&& ~~~EnabledMask
    member this.Visible with get () = this.Flags_ &&& VisibleMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| VisibleMask else this.Flags_ &&& ~~~VisibleMask
    member this.AlwaysUpdate with get () = this.Flags_ &&& AlwaysUpdateMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| AlwaysUpdateMask else this.Flags_ &&& ~~~AlwaysUpdateMask
    member this.PublishUpdates with get () = this.Flags_ &&& PublishUpdatesMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PublishUpdatesMask else this.Flags_ &&& ~~~PublishUpdatesMask
    member this.PublishPostUpdates with get () = this.Flags_ &&& PublishPostUpdatesMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PublishPostUpdatesMask else this.Flags_ &&& ~~~PublishPostUpdatesMask
    member this.Persistent with get () = this.Flags_ &&& PersistentMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| PersistentMask else this.Flags_ &&& ~~~PersistentMask
    member this.IgnorePropertyBindings with get () = this.Flags_ &&& IgnorePropertyBindingsMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| IgnorePropertyBindingsMask else this.Flags_ &&& ~~~IgnorePropertyBindingsMask
    member this.Mounted with get () = this.Flags_ &&& MountedMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| MountedMask else this.Flags_ &&& ~~~MountedMask
    member this.EnabledLocal with get () = this.Flags_ &&& EnabledLocalMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| EnabledLocalMask else this.Flags_ &&& ~~~EnabledLocalMask
    member this.VisibleLocal with get () = this.Flags_ &&& VisibleLocalMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| VisibleLocalMask else this.Flags_ &&& ~~~VisibleLocalMask
    member this.RotationMatrixDirty with get () = this.Flags_ &&& RotationMatrixDirtyMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| RotationMatrixDirtyMask else this.Flags_ &&& ~~~RotationMatrixDirtyMask
    member this.AffineMatrixDirty with get () = this.Flags_ &&& AffineMatrixDirtyMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| AffineMatrixDirtyMask else this.Flags_ &&& ~~~AffineMatrixDirtyMask
    member this.Is2d with get () = this.Flags_ &&& Is2dMask <> 0u and set value = this.Flags_ <- if value then this.Flags_ ||| Is2dMask else this.Flags_ &&& ~~~Is2dMask
    member this.Optimized with get () = this.Imperative && this.Omnipresent && not this.PublishChangeBindings && not this.PublishChangeEvents // TODO: see if I can remove all conditionals from here.

    member this.Position with get () = this.Position_ and set value = this.Position_ <- value; this.AffineMatrixDirty <- true
    member this.Scale with get () = this.Scale_ and set value = this.Scale_ <- value; this.AffineMatrixDirty <- true
    member this.Offset with get () = this.Offset_ and set value = this.Offset_ <- value; this.AffineMatrixDirty <- true
    member this.Size with get () = this.Size_ and set value = this.Size_ <- value; this.AffineMatrixDirty <- true
    member this.Elevation with get () = this.Elevation_ and set value = this.Elevation_ <- value; this.AffineMatrixDirty <- true

    member this.Rotation
        with get () = this.Rotation_
        and set value =
            this.Rotation_ <- value
            let pitchYawRoll = value.PitchYawRoll ()
            this.Angles_.X <- pitchYawRoll.X
            this.Angles_.Y <- pitchYawRoll.Y
            this.Angles_.Z <- pitchYawRoll.Z
            this.RotationMatrixDirty <- true
            this.AffineMatrixDirty <- true

    member this.Angles
        with get () = this.Angles_
        and set value =
            this.Angles_ <- value
            this.Rotation_ <- Quaternion.CreateFromYawPitchRoll (value.Y, value.X, value.Z)
            this.RotationMatrixDirty <- true
            this.AffineMatrixDirty <- true

    member this.RotationMatrix =
        if notNull (this.RotationMatrixOpt_ :> obj) then
            if this.RotationMatrixDirty then this.RotationMatrixOpt_ <- ref (Matrix4x4.CreateFromQuaternion this.Rotation_)
            this.RotationMatrixOpt_.Value
        else Matrix4x4.Identity

    member this.AffineMatrix =
        if notNull (this.AffineMatrixOpt_ :> obj) then
            if this.AffineMatrixDirty then
                // TODO: P1: optimize this hella!
                let positionMatrix = Matrix4x4.CreateTranslation this.Position_
                let rotationMatrix = this.RotationMatrix
                let scaleMatrix = Matrix4x4.CreateScale this.Scale_
                this.AffineMatrixOpt_ <- ref (positionMatrix * rotationMatrix * scaleMatrix)
            this.AffineMatrixOpt_.Value
        else Matrix4x4.Identity

    member this.Extent = this.Size_ * 0.5f
    member this.Right = Vector3 (this.RotationMatrix.M11, this.RotationMatrix.M12, this.RotationMatrix.M13) // TODO: implement Row properties.
    member this.Up = Vector3 (this.RotationMatrix.M21, this.RotationMatrix.M22, this.RotationMatrix.M23)
    member this.Forward = -Vector3 (this.RotationMatrix.M31, this.RotationMatrix.M32, this.RotationMatrix.M33)
    member this.Left = -this.Right
    member this.Down = -this.Up
    member this.Backward = -this.Forward

    member this.Dimensions
        with get () =
            let size = this.Size_
            let extent = size * 0.5f
            let position = this.Position_ - extent
            let offset = this.Offset_ * size
            Box3 (position + offset, size)
        and set (value : Box3) =
            let size = value.Size
            let extent = size * 0.5f
            let offset = this.Offset_ * size
            let position = value.Position + extent - offset
            this.Position_ <- position
            this.Size <- size

    member this.PositionScaled = this.Position_ * this.Scale_
    member this.OffsetScaled = this.Offset_ * this.Scale_
    member this.SizeScaled = this.Size_ * this.Scale_
    member this.ExtentScaled = this.Extent * this.Scale_

    member this.DimensionsScaled =
        let scale = this.Scale_
        let sizeScaled = this.Size_ * scale
        let extentScaled = sizeScaled * 0.5f
        let positionScaled = this.Position_ - extentScaled
        let offsetScaled = this.Offset_ * sizeScaled
        Box3 (positionScaled + offsetScaled, sizeScaled)

    member this.DimensionsOriented =
        let dimensions = this.DimensionsScaled
        dimensions.Orient this.Rotation_

    member inline this.DimensionsOverflowed =
        this.DimensionsOriented // no overflow value yet, so just stick with orienteds

    member this.InvalidateFast () =
        this.Flags_ <- this.Flags_ ||| TransformMasks.InvalidatedMask

    /// Test transforms for equality.
    static member equalsByRef (left : Transform inref, right : Transform inref) =
        left.Flags_ = right.Flags_ &&
        left.Position_.Equals right.Position_ &&
        left.Rotation_.Equals right.Rotation_ &&
        left.Scale_.Equals right.Scale_ &&
        left.Offset_.Equals right.Offset_ &&
        left.Angles_.Equals right.Angles_ &&
        left.Size_.Equals right.Size_ &&
{'fraction_non_alphanumeric': 0.0929825310724811, 'fraction_numerical': 0.04898987741852817, 'mean_word_length': 2.9684745762711864, 'pattern_counts': {'":': 0, '<': 183, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 56, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2428544', 'n_tokens_mistral': 7638, 'n_tokens_neox': 6476, 'n_words': 1904}