<commit_before>﻿namespace Metrics
open System
open System.Collections.Generic
open System.Numerics
open Prime
open Nu
open Nu.Declarative

#if ECS_HYBRID
type [<NoEquality; NoComparison; Struct>] StaticSpriteComponent =
    { mutable Active : bool
      mutable Entity : Entity
      mutable Sprite : Image AssetTag }
    interface StaticSpriteComponent Component with
        member this.TypeName = nameof StaticSpriteComponent
        member this.Active with get () = this.Active and set value = this.Active <- value
#endif

#if ECS
type [<NoEquality; NoComparison; Struct>] Position =
    { mutable Active : bool
      mutable Position : Vector2 }
    interface Position Component with
        member this.TypeName = nameof Position
        member this.Active with get () = this.Active and set value = this.Active <- value

type [<NoEquality; NoComparison; Struct>] Velocity =
    { mutable Active : bool
      mutable Velocity : Vector2 }
    interface Velocity Component with
        member this.TypeName = nameof Velocity
        member this.Active with get () = this.Active and set value = this.Active <- value

type [<NoEquality; NoComparison; Struct>] Shake =
    { mutable Active : bool
      mutable Origin : Vector2
      mutable Offset : Vector2 }
    interface Shake Component with
        member this.TypeName = nameof Shake
        member this.Active with get () = this.Active and set value = this.Active <- value
#endif

#if FACETED
type MetricsEntityDispatcher () =
    inherit EntityDispatcher ()

  #if !ECS_HYBRID && !ECS
    static member Facets =
        [typeof<StaticSpriteFacet>]

    override this.Update (entity, world) =
        entity.SetRotation (entity.GetRotation world + 0.03f) world
  #endif

  #if ECS_HYBRID
    override this.Register (entity, world) =
        let ecs = entity.Parent.Parent.GetEcs world
        let _ : Guid = ecs.RegisterCorrelated<StaticSpriteComponent> { Active = false; Entity = entity; Sprite = Assets.Default.Image4 } (entity.GetId world)
        world

    override this.Unregister (entity, world) =
        let ecs = entity.Parent.Parent.GetEcs world
        let _ : bool = ecs.UnregisterCorrelated<StaticSpriteComponent> (entity.GetId world)
        world
  #endif
#else
type MetricsEntityDispatcher () =
  #if ECS_HYBRID
    inherit EntityDispatcher ()
  #else
    inherit EntityDispatcher<Image AssetTag, unit, unit> (Assets.Default.Image)
  #endif

  #if !ECS_HYBRID && !ECS
    override this.Update (entity, world) =
        entity.SetRotation (entity.GetRotation world + 0.03f) world

    override this.View (staticImage, entity, world) =
        let transform = entity.GetTransform world
        View.Render
            (transform.Elevation,
             transform.Position.Y,
             AssetTag.generalize staticImage,
             SpriteDescriptor
                { Transform = transform
                  Offset = Vector2.Zero
                  Absolute = false
                  InsetOpt = None
                  Image = staticImage
                  Color = colWhite
                  Blend = Transparent
                  Glow = colZero
                  Flip = FlipNone })
  #endif

  #if ECS_HYBRID
    override this.Register (entity, world) =
        let ecs = entity.Parent.Parent.GetEcs world
        let _ : Guid = ecs.RegisterCorrelated<StaticSpriteComponent> { Active = false; Entity = entity; Sprite = Assets.Default.Image4 } (entity.GetId world)
        world

    override this.Unregister (entity, world) =
        let ecs = entity.Parent.Parent.GetEcs world
        let _ : bool = ecs.UnregisterCorrelated<StaticSpriteComponent> (entity.GetId world)
        world
  #endif
#endif

type MyGameDispatcher () =
    inherit GameDispatcher<unit, unit, unit> (())

    let Fps = Simulants.DefaultGroup / "Fps"

    override this.Register (game, world) =
        let world = base.Register (game, world)
        let (screen, world) = World.createScreen (Some Simulants.DefaultScreen.Name) world
#if ECS_HYBRID
        // grab ecs from current screen
        let ecs = screen.GetEcs world

        // create static sprite store
        ecs.RegisterStore (StoreCorrelated<StaticSpriteComponent, World> ecs)
#endif
#if ECS
        // get ecs
        let ecs = screen.GetEcs world

        // create component stores
        let _ = ecs.RegisterStore (CorrelatedStore<EntityId, World> ecs)
        let _ = ecs.RegisterStore (CorrelatedStore<Position, World> ecs)
        let _ = ecs.RegisterStore (CorrelatedStore<Velocity, World> ecs)
        let _ = ecs.RegisterStore (CorrelatedStore<Shake, World> ecs)

        // create movers system
        let movers = ecs.RegisterSystem (System<Position, Velocity, World> ecs)

        // create shakers system
        let shakers = ecs.RegisterSystem (System<EntityId, Position, Shake, World> ecs)

        // create 3M movers (goal: 60FPS, current: 60FPS)
        let world =
            Seq.fold (fun world _ ->
                movers.Allocate { Active = true; Position = v2Zero } { Active = true; Velocity = v2One } world |> snd')
                world (Seq.init 3000000 id)

        // create 3000 shakers
        let world =
            Seq.fold (fun world _ ->
                let struct (entityId, world) = movers.NextEntityId world
                shakers.Allocate { Active = true; EntityId = entityId } { Active = true; Position = v2Zero } { Active = true; Origin = v2Zero; Offset = v2One } world |> snd')
                world (Seq.init 3000 id)

        // define update for movers
        ecs.Subscribe EcsEvents.Update $ fun _ _ _ ->
            movers.Iterate $
                new Statement<_, _, _> (fun position velocity world ->
                    position.Position.X <- position.Position.X + velocity.Velocity.X
                    position.Position.Y <- position.Position.Y + velocity.Velocity.Y
                    world)

        // define update for shakers
        ecs.Subscribe EcsEvents.Update $ fun _ _ _ ->
            shakers.Iterate $
                new Statement<_, _, _, _> (fun _ position shake world ->
                    position.Position.X <- shake.Origin.X + Gen.randomf1 shake.Offset.X
                    position.Position.Y <- shake.Origin.Y + Gen.randomf1 shake.Offset.Y
                    world)

        // [| mutable P : Vector2; mutable V : Vector2 |]       8M
        //
        // { mutable P : Vector2; mutable V : Vector2 }         5M / 1.25M when placement randomized
        //
        // [| [| componentRef P |] [| componentRef V |] |]      4M / 3M #if !SYSTEM_ITERATION
        //
        // [| [| ref P |] [| ref V |] |]                        2.5M
        //
        // { mutable P : P; mutable V : V }                     2M / 250K when placement randomized
        //
        // out-of-place entities * ops                          50K * 5 = 250K
        //
        // flat array -> 8M, partitioned array -> 2M, partitioned array tree -> ???, randomized cyclic graph -> 250K

        // raw, manually managed array of structs               inits fast, runs fast. optimal but inflexible.
        //
        // uncorrelated components                              inits medium fast (int HashSet op may be required), runs fast. slight flexibility increase.
        //
        // correlated components                                inits slow, runs medium fast. suboptimal, but very flexible.

        // elmish                                               1,000's
        //
        // classic                                              10,000's
        //
        // ecs                                                  100,000's
        //
        // compute shaders                                      1,000,000's

#endif
        let world = World.createGroup (Some Simulants.DefaultGroup.Name) Simulants.DefaultScreen world |> snd
        let world = World.createEntity<FpsDispatcher> (Some Fps.Name) DefaultOverlay Simulants.DefaultGroup world |> snd
        let world = Fps.SetPosition (v2 200.0f -250.0f) world
#if !ECS
        let positions = // 19,663 entity positions (goal: 60FPS, current: 60FPS)
            seq {
                for i in 0 .. 52 do
                    for j in 0 .. 52 do
                        for k in 0 .. 6 do
                            yield v2 (single i * 12.0f + single k) (single j * 12.0f + single k) }
        let world =
            Seq.fold (fun world position ->
                let (entity, world) = World.createEntity<MetricsEntityDispatcher> None NoOverlay Simulants.DefaultGroup world
                let world = entity.SetOmnipresent true world
                let world = entity.SetPosition (position + v2 -450.0f -265.0f) world
                let world = entity.SetSize (v2One * 8.0f) world
                world)
                world positions
#endif
        let world = World.selectScreen IdlingState Simulants.DefaultScreen world
#if ECS_HYBRID
        // define update for static sprites
        ecs.Subscribe EcsEvents.Update $ fun _ _ _ ->
            for components in ecs.GetComponentArrays<StaticSpriteComponent> () do
                for i in 0 .. components.Length - 1 do
                    let comp = &components.[i]
                    if comp.Active then
                        let state = comp.Entity.State world
                        state.Rotation <- state.Rotation + 0.03f

        // define actualize for static sprites
        ecs.SubscribePlus Gen.id EcsEvents.Actualize $ fun _ _ _ world ->
            let messages = List ()
            for components in ecs.GetComponentArrays<StaticSpriteComponent> () do
                for i in 0 .. components.Length - 1 do
                    let comp = &components.[i]
                    if comp.Active then
                        let state = comp.Entity.State world
                        if state.Visible then
                            let spriteDescriptor = SpriteDescriptor { Transform = state.Transform; Absolute = state.Absolute; Offset = Vector2.Zero; InsetOpt = None; Image = comp.Sprite; Color = Color.White; Blend = Transparent; Glow = Color.Zero; Flip = FlipNone }
                            let layeredMessage = { Elevation = state.Elevation; PositionY = state.Position.Y; AssetTag = AssetTag.generalize comp.Sprite; RenderDescriptor = spriteDescriptor }
                            messages.Add layeredMessage
            World.enqueueRenderLayeredMessages messages world
#else
        ignore screen
#endif
        world

#if ELMISH
type ElmishEntityDispatcher () =
    inherit EntityDispatcher<Image AssetTag, unit, unit> (Assets.Default.Image)

    override this.View (staticImage, entity, world) =
        let transform = entity.GetTransform world
        View.Render
            (transform.Elevation,
             transform.Position.Y,
             AssetTag.generalize staticImage,
             SpriteDescriptor
                { Transform = transform
                  Offset = Vector2.Zero
                  Absolute = false
                  InsetOpt = None
                  Image = staticImage
                  Color = colWhite
                  Blend = Transparent
                  Glow = colZero
                  Flip = FlipNone })

#if PROPERTIES
type ElmishGameDispatcher () =
    inherit GameDispatcher<int, int, unit> (0)

    override this.Channel (_, game) =
        [game.UpdateEvent => msg 0]

    override this.Message (int, message, _, _) =
        match message with
        | 0 -> just (inc int)
        | _ -> just int

    override this.Content (int, _) =
        [Content.screen Gen.name Vanilla []
            [Content.group Gen.name []
                [Content.entity<ElmishEntityDispatcher> Gen.name
                    (seq {
                        yield Entity.Omnipresent == true
                        for index in 0 .. 100000 do yield Entity.Size <== int --> fun int -> v2 (single (int % 12)) (single (index % 12)) } |> // 100,000 property bindings (goal: 60FPS, current: 40FPS)
                        Seq.toList)]
             Content.group Gen.name []
                [Content.fps "Fps" [Entity.Position == v2 200.0f -250.0f]]]]
#else
type [<ReferenceEquality>] Ints =
    { Ints : Map<int, int> }
    static member init n =
        { Ints = Seq.init n (fun a -> (a, a)) |> Map.ofSeq }
    static member inc ints =
        { Ints = ints.Ints |> Seq.map (fun kvp -> (kvp.Key, inc kvp.Value)) |> Map.ofSeq }

type [<ReferenceEquality>] Intss =
    { Intss : Map<int, Ints> }
    static member init n =
        { Intss = Seq.init n (fun a -> (a, Ints.init n)) |> Map.ofSeq }
    static member inc intss =
        { Intss = intss.Intss |> Seq.map (fun kvp -> (kvp.Key, Ints.inc kvp.Value)) |> Map.ofSeq }
        //{ Intss = intss.Intss |> Seq.map (fun kvp -> (kvp.Key, if kvp.Key = 0 then Ints.inc kvp.Value else kvp.Value)) |> Map.ofSeq }

type ElmishGameDispatcher () =
    inherit GameDispatcher<Intss, int, unit> (Intss.init 100) // 10,000 ints (goal: 60FPS, current: 40FPS)

    override this.Channel (_, game) =
        [game.UpdateEvent => msg 0]
<commit_msg>Documented speed increase when Elmish metrics not rendering.<commit_after>﻿namespace Metrics
open System
open System.Collections.Generic
open System.Numerics
open Prime
open Nu
open Nu.Declarative

#if ECS_HYBRID
type [<NoEquality; NoComparison; Struct>] StaticSpriteComponent =
    { mutable Active : bool
      mutable Entity : Entity
      mutable Sprite : Image AssetTag }
    interface StaticSpriteComponent Component with
        member this.TypeName = nameof StaticSpriteComponent
        member this.Active with get () = this.Active and set value = this.Active <- value
#endif

#if ECS
type [<NoEquality; NoComparison; Struct>] Position =
    { mutable Active : bool
      mutable Position : Vector2 }
    interface Position Component with
        member this.TypeName = nameof Position
        member this.Active with get () = this.Active and set value = this.Active <- value

type [<NoEquality; NoComparison; Struct>] Velocity =
    { mutable Active : bool
      mutable Velocity : Vector2 }
    interface Velocity Component with
        member this.TypeName = nameof Velocity
        member this.Active with get () = this.Active and set value = this.Active <- value

type [<NoEquality; NoComparison; Struct>] Shake =
    { mutable Active : bool
      mutable Origin : Vector2
      mutable Offset : Vector2 }
    interface Shake Component with
        member this.TypeName = nameof Shake
        member this.Active with get () = this.Active and set value = this.Active <- value
#endif

#if FACETED
type MetricsEntityDispatcher () =
    inherit EntityDispatcher ()

  #if !ECS_HYBRID && !ECS
    static member Facets =
        [typeof<StaticSpriteFacet>]

    override this.Update (entity, world) =
        entity.SetRotation (entity.GetRotation world + 0.03f) world
  #endif

  #if ECS_HYBRID
    override this.Register (entity, world) =
        let ecs = entity.Parent.Parent.GetEcs world
        let _ : Guid = ecs.RegisterCorrelated<StaticSpriteComponent> { Active = false; Entity = entity; Sprite = Assets.Default.Image4 } (entity.GetId world)
        world

    override this.Unregister (entity, world) =
        let ecs = entity.Parent.Parent.GetEcs world
        let _ : bool = ecs.UnregisterCorrelated<StaticSpriteComponent> (entity.GetId world)
        world
  #endif
#else
type MetricsEntityDispatcher () =
  #if ECS_HYBRID
    inherit EntityDispatcher ()
  #else
    inherit EntityDispatcher<Image AssetTag, unit, unit> (Assets.Default.Image)
  #endif

  #if !ECS_HYBRID && !ECS
    override this.Update (entity, world) =
        entity.SetRotation (entity.GetRotation world + 0.03f) world

    override this.View (staticImage, entity, world) =
        let transform = entity.GetTransform world
        View.Render
            (transform.Elevation,
             transform.Position.Y,
             AssetTag.generalize staticImage,
             SpriteDescriptor
                { Transform = transform
                  Offset = Vector2.Zero
                  Absolute = false
                  InsetOpt = None
                  Image = staticImage
                  Color = colWhite
                  Blend = Transparent
                  Glow = colZero
                  Flip = FlipNone })
  #endif

  #if ECS_HYBRID
    override this.Register (entity, world) =
        let ecs = entity.Parent.Parent.GetEcs world
        let _ : Guid = ecs.RegisterCorrelated<StaticSpriteComponent> { Active = false; Entity = entity; Sprite = Assets.Default.Image4 } (entity.GetId world)
        world

    override this.Unregister (entity, world) =
        let ecs = entity.Parent.Parent.GetEcs world
        let _ : bool = ecs.UnregisterCorrelated<StaticSpriteComponent> (entity.GetId world)
        world
  #endif
#endif

type MyGameDispatcher () =
    inherit GameDispatcher<unit, unit, unit> (())

    let Fps = Simulants.DefaultGroup / "Fps"

    override this.Register (game, world) =
        let world = base.Register (game, world)
        let (screen, world) = World.createScreen (Some Simulants.DefaultScreen.Name) world
#if ECS_HYBRID
        // grab ecs from current screen
        let ecs = screen.GetEcs world

        // create static sprite store
        ecs.RegisterStore (StoreCorrelated<StaticSpriteComponent, World> ecs)
#endif
#if ECS
        // get ecs
        let ecs = screen.GetEcs world

        // create component stores
        let _ = ecs.RegisterStore (CorrelatedStore<EntityId, World> ecs)
        let _ = ecs.RegisterStore (CorrelatedStore<Position, World> ecs)
        let _ = ecs.RegisterStore (CorrelatedStore<Velocity, World> ecs)
        let _ = ecs.RegisterStore (CorrelatedStore<Shake, World> ecs)

        // create movers system
        let movers = ecs.RegisterSystem (System<Position, Velocity, World> ecs)

        // create shakers system
        let shakers = ecs.RegisterSystem (System<EntityId, Position, Shake, World> ecs)

        // create 3M movers (goal: 60FPS, current: 60FPS)
        let world =
            Seq.fold (fun world _ ->
                movers.Allocate { Active = true; Position = v2Zero } { Active = true; Velocity = v2One } world |> snd')
                world (Seq.init 3000000 id)

        // create 3000 shakers
        let world =
            Seq.fold (fun world _ ->
                let struct (entityId, world) = movers.NextEntityId world
                shakers.Allocate { Active = true; EntityId = entityId } { Active = true; Position = v2Zero } { Active = true; Origin = v2Zero; Offset = v2One } world |> snd')
                world (Seq.init 3000 id)

        // define update for movers
        ecs.Subscribe EcsEvents.Update $ fun _ _ _ ->
            movers.Iterate $
                new Statement<_, _, _> (fun position velocity world ->
                    position.Position.X <- position.Position.X + velocity.Velocity.X
                    position.Position.Y <- position.Position.Y + velocity.Velocity.Y
                    world)

        // define update for shakers
        ecs.Subscribe EcsEvents.Update $ fun _ _ _ ->
            shakers.Iterate $
                new Statement<_, _, _, _> (fun _ position shake world ->
                    position.Position.X <- shake.Origin.X + Gen.randomf1 shake.Offset.X
                    position.Position.Y <- shake.Origin.Y + Gen.randomf1 shake.Offset.Y
                    world)

        // [| mutable P : Vector2; mutable V : Vector2 |]       8M
        //
        // { mutable P : Vector2; mutable V : Vector2 }         5M / 1.25M when placement randomized
        //
        // [| [| componentRef P |] [| componentRef V |] |]      4M / 3M #if !SYSTEM_ITERATION
        //
        // [| [| ref P |] [| ref V |] |]                        2.5M
        //
        // { mutable P : P; mutable V : V }                     2M / 250K when placement randomized
        //
        // out-of-place entities * ops                          50K * 5 = 250K
        //
        // flat array -> 8M, partitioned array -> 2M, partitioned array tree -> ???, randomized cyclic graph -> 250K

        // raw, manually managed array of structs               inits fast, runs fast. optimal but inflexible.
        //
        // uncorrelated components                              inits medium fast (int HashSet op may be required), runs fast. slight flexibility increase.
        //
        // correlated components                                inits slow, runs medium fast. suboptimal, but very flexible.

        // elmish                                               1,000's
        //
        // classic                                              10,000's
        //
        // ecs                                                  100,000's
        //
        // compute shaders                                      1,000,000's

#endif
        let world = World.createGroup (Some Simulants.DefaultGroup.Name) Simulants.DefaultScreen world |> snd
        let world = World.createEntity<FpsDispatcher> (Some Fps.Name) DefaultOverlay Simulants.DefaultGroup world |> snd
        let world = Fps.SetPosition (v2 200.0f -250.0f) world
#if !ECS
        let positions = // 19,663 entity positions (goal: 60FPS, current: 60FPS)
            seq {
                for i in 0 .. 52 do
                    for j in 0 .. 52 do
                        for k in 0 .. 6 do
                            yield v2 (single i * 12.0f + single k) (single j * 12.0f + single k) }
        let world =
            Seq.fold (fun world position ->
                let (entity, world) = World.createEntity<MetricsEntityDispatcher> None NoOverlay Simulants.DefaultGroup world
                let world = entity.SetOmnipresent true world
                let world = entity.SetPosition (position + v2 -450.0f -265.0f) world
                let world = entity.SetSize (v2One * 8.0f) world
                world)
                world positions
#endif
        let world = World.selectScreen IdlingState Simulants.DefaultScreen world
#if ECS_HYBRID
        // define update for static sprites
        ecs.Subscribe EcsEvents.Update $ fun _ _ _ ->
            for components in ecs.GetComponentArrays<StaticSpriteComponent> () do
                for i in 0 .. components.Length - 1 do
                    let comp = &components.[i]
                    if comp.Active then
                        let state = comp.Entity.State world
                        state.Rotation <- state.Rotation + 0.03f

        // define actualize for static sprites
        ecs.SubscribePlus Gen.id EcsEvents.Actualize $ fun _ _ _ world ->
            let messages = List ()
            for components in ecs.GetComponentArrays<StaticSpriteComponent> () do
                for i in 0 .. components.Length - 1 do
                    let comp = &components.[i]
                    if comp.Active then
                        let state = comp.Entity.State world
                        if state.Visible then
                            let spriteDescriptor = SpriteDescriptor { Transform = state.Transform; Absolute = state.Absolute; Offset = Vector2.Zero; InsetOpt = None; Image = comp.Sprite; Color = Color.White; Blend = Transparent; Glow = Color.Zero; Flip = FlipNone }
                            let layeredMessage = { Elevation = state.Elevation; PositionY = state.Position.Y; AssetTag = AssetTag.generalize comp.Sprite; RenderDescriptor = spriteDescriptor }
                            messages.Add layeredMessage
            World.enqueueRenderLayeredMessages messages world
#else
        ignore screen
#endif
        world

#if ELMISH
type ElmishEntityDispatcher () =
    inherit EntityDispatcher<Image AssetTag, unit, unit> (Assets.Default.Image)

    override this.View (staticImage, entity, world) =
        let transform = entity.GetTransform world
        View.Render
            (transform.Elevation,
             transform.Position.Y,
             AssetTag.generalize staticImage,
             SpriteDescriptor
                { Transform = transform
                  Offset = Vector2.Zero
                  Absolute = false
                  InsetOpt = None
                  Image = staticImage
                  Color = colWhite
                  Blend = Transparent
                  Glow = colZero
                  Flip = FlipNone })

#if PROPERTIES
type ElmishGameDispatcher () =
    inherit GameDispatcher<int, int, unit> (0)

    override this.Channel (_, game) =
        [game.UpdateEvent => msg 0]

    override this.Message (int, message, _, _) =
        match message with
        | 0 -> just (inc int)
        | _ -> just int

    override this.Content (int, _) =
        [Content.screen Gen.name Vanilla []
            [Content.group Gen.name []
                [Content.entity<ElmishEntityDispatcher> Gen.name
                    (seq {
                        yield Entity.Omnipresent == true
                        for index in 0 .. 100000 do yield Entity.Size <== int --> fun int -> v2 (single (int % 12)) (single (index % 12)) } |> // 100,000 property bindings (goal: 60FPS, current: 40FPS)
                        Seq.toList)]
             Content.group Gen.name []
                [Content.fps "Fps" [Entity.Position == v2 200.0f -250.0f]]]]
#else
type [<ReferenceEquality>] Ints =
    { Ints : Map<int, int> }
    static member init n =
        { Ints = Seq.init n (fun a -> (a, a)) |> Map.ofSeq }
    static member inc ints =
        { Ints = ints.Ints |> Seq.map (fun kvp -> (kvp.Key, inc kvp.Value)) |> Map.ofSeq }

type [<ReferenceEquality>] Intss =
    { Intss : Map<int, Ints> }
    static member init n =
        { Intss = Seq.init n (fun a -> (a, Ints.init n)) |> Map.ofSeq }
    static member inc intss =
        { Intss = intss.Intss |> Seq.map (fun kvp -> (kvp.Key, Ints.inc kvp.Value)) |> Map.ofSeq }
        //{ Intss = intss.Intss |> Seq.map (fun kvp -> (kvp.Key, if kvp.Key = 0 then Ints.inc kvp.Value else kvp.Value)) |> Map.ofSeq }

type ElmishGameDispatcher () =
    inherit GameDispatcher<Intss, int, unit> (Intss.init 100) // 10,000 ints (goal: 60FPS, current: 40FPS [50FPS w/o rendering])

    override this.Channel (_, game) =
        [game.UpdateEvent => msg 0]

{'fraction_non_alphanumeric': 0.08978529603122967, 'fraction_numerical': 0.01477285774426882, 'mean_word_length': 1.8221190193325414, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 141, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3920762', 'n_tokens_mistral': 7624, 'n_tokens_neox': 7026, 'n_words': 2128}