<commit_before>/*
Copyright (c) 2014, <NAME>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met: 

1. Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer. 
2. Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution. 

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those
of the authors and should not be interpreted as representing official policies, 
either expressed or implied, of the FreeBSD Project.
*/

//#define AA_TIPS

using System;
using MatterHackers.Agg;
using MatterHackers.Agg.Image;
using MatterHackers.Agg.Transform;
using MatterHackers.Agg.VertexSource;
using MatterHackers.RenderOpenGl.OpenGl;
using MatterHackers.VectorMath;
using Tesselate;

namespace MatterHackers.RenderOpenGl
{
    public abstract class VertexTesselatorAbstract : Tesselator
    {
        public abstract void AddVertex(double x, double y);
    }

    public class Graphics2DOpenGL : Graphics2D
    {
        // We can have a single static instance because all gl rendering is required to happen on the ui thread so there can
        // be no runtime contention for this object (no thread contention).
        static AARenderToGLTesselator triangleEddgeInfo = new AARenderToGLTesselator();
        static int AATextureHandle = -1;

        public bool DoEdgeAntiAliasing = true;
        static RenderToGLTesselator renderNowTesselator = new RenderToGLTesselator();

        int width;
        int height;
		RectangleDouble cachedClipRect;

		public Graphics2DOpenGL(int width, int height)
        {
            this.width = width;
            this.height = height;
			cachedClipRect = new RectangleDouble(0, 0, width, height);
        }

        public override RectangleDouble GetClippingRect()
        {
            return cachedClipRect;
        }

        public override void SetClippingRect(RectangleDouble clippingRect)
        {
			cachedClipRect = clippingRect;
            GL.Scissor((int)Math.Floor(Math.Max(clippingRect.Left, 0)), (int)Math.Floor(Math.Max(clippingRect.Bottom, 0)),
                (int)Math.Ceiling(Math.Max(clippingRect.Width, 0)), (int)Math.Ceiling(Math.Max(clippingRect.Height, 0)));
			GL.Enable(EnableCap.ScissorTest);
        }

        public override IScanlineCache ScanlineCache
        {
            get { return null; }
            set { throw new Exception("There is no scanline cache on a GL surface."); }
        }

        public void PushOrthoProjection()
        {
			GL.Disable(EnableCap.CullFace);

            GL.MatrixMode(MatrixMode.Projection);
            GL.PushMatrix();
            GL.LoadIdentity();
            GL.Ortho(0, width, 0, height, 0, 1);

            GL.MatrixMode(MatrixMode.Modelview);
            GL.PushMatrix();
            GL.LoadIdentity();
        }

        public void PopOrthoProjection()
        {
			GL.MatrixMode(MatrixMode.Projection);
            GL.PopMatrix();
            GL.MatrixMode(MatrixMode.Modelview);
            GL.PopMatrix();
        }

        public static void SendShapeToTesselator(VertexTesselatorAbstract tesselator, IVertexSource vertexSource)
        {
            
#if !DEBUG
            try
#endif
            {
                tesselator.BeginPolygon();

                ShapePath.FlagsAndCommand PathAndFlags = 0;
                double x, y;
                bool haveBegunContour = false;
                while (!ShapePath.is_stop(PathAndFlags = vertexSource.vertex(out x, out y)))
                {
                    if (ShapePath.is_close(PathAndFlags)
                        || (haveBegunContour && ShapePath.is_move_to(PathAndFlags)))
                    {
                        tesselator.EndContour();
                        haveBegunContour = false;
                    }

                    if (!ShapePath.is_close(PathAndFlags))
                    {
                        if (!haveBegunContour)
                        {
                            tesselator.BeginContour();
                            haveBegunContour = true;
                        }

                        tesselator.AddVertex(x, y);
                    }
                }

                if (haveBegunContour)
                {
                    tesselator.EndContour();
                }

                tesselator.EndPolygon();
            }
#if !DEBUG
            catch
            {
            }
#endif
        }

		static byte[] CreateBufferForAATexture()
		{
			byte[] hardwarePixelBuffer = new byte[1024 * 4 * 4];
			for (int y = 0; y < 4; y++)
			{
				byte alpha = 0;
				for (int x = 0; x < 1024; x++)
				{
					hardwarePixelBuffer[(y * 1024 + x) * 4 + 0] = 255;
					hardwarePixelBuffer[(y * 1024 + x) * 4 + 1] = 255;
					hardwarePixelBuffer[(y * 1024 + x) * 4 + 2] = 255;
					hardwarePixelBuffer[(y * 1024 + x) * 4 + 3] = alpha;
					alpha = 255;
				}
			}
			return hardwarePixelBuffer;
		}

        void CheckLineImageCache()
        {
            if (AATextureHandle == -1)
            {
                // Create the texture handle and display list handle
                GL.GenTextures(1, out AATextureHandle);

                // Set up some texture parameters for openGL
                GL.BindTexture(TextureTarget.Texture2D, AATextureHandle);
                GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
                GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);

                GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.ClampToEdge);
                GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.ClampToEdge);

                byte[] hardwarePixelBuffer = CreateBufferForAATexture();

                // Create the texture
                GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgba, 1024, 4,
                    0, PixelFormat.Rgba, PixelType.UnsignedByte, hardwarePixelBuffer);
            }
        }

        void DrawAAShape(IVertexSource vertexSource)
        {
			CheckLineImageCache();
            GL.Enable(EnableCap.Texture2D);
            GL.BindTexture(TextureTarget.Texture2D, AATextureHandle);

            triangleEddgeInfo.Clear();
            Graphics2DOpenGL.SendShapeToTesselator(triangleEddgeInfo, vertexSource);

            // now render it
            triangleEddgeInfo.RenderLastToGL();
        }

        public override void Render(IVertexSource vertexSource, int pathIndexToRender, RGBA_Bytes colorBytes)
        {
			PushOrthoProjection();

            GL.BlendFunc(BlendingFactorSrc.SrcAlpha, BlendingFactorDest.OneMinusSrcAlpha);
            GL.Enable(EnableCap.Blend);

            vertexSource.rewind(pathIndexToRender);

            GL.Color4(colorBytes.red, colorBytes.green, colorBytes.blue, colorBytes.alpha);

            Affine transform = GetTransform();
            if (!transform.is_identity())
            {
                vertexSource = new VertexSourceApplyTransform(vertexSource, transform);
            }

            if (DoEdgeAntiAliasing)
            {
                DrawAAShape(vertexSource);
            }
            else
            {
                renderNowTesselator.Clear();
                Graphics2DOpenGL.SendShapeToTesselator(renderNowTesselator, vertexSource);
            }

            PopOrthoProjection();
        }

        public override void Render(IImageByte source,
            double x, double y,
            double angleRadians,
            double scaleX, double scaleY)
        {
#if true
            Affine transform = GetTransform();
            if (!transform.is_identity())
            {
                if (scaleX != 1 || scaleY != 1)// || angleDegrees != 0)
                {
                    throw new NotImplementedException();
                }
                // TODO: <BUG> make this do rotation and scalling
                transform.transform(ref x, ref y);
                scaleX *= transform.sx;
                scaleY *= transform.sy;
            }
#endif

#if true
            // TODO: <BUG> make this do rotation and scalling
            RectangleInt sourceBounds = source.GetBounds();
            sourceBounds.Offset((int)x, (int)y);
            RectangleInt destBounds = new RectangleInt((int)cachedClipRect.Left, (int)cachedClipRect.Bottom, (int)cachedClipRect.Right, (int)cachedClipRect.Top);

            if (!RectangleInt.DoIntersect(sourceBounds, destBounds))
            {
                if (scaleX != 1 || scaleY != 1)// || angleDegrees != 0)
                {
                    //throw new NotImplementedException();
                }
                //return;
            }
#endif

            ImageBuffer sourceAsImageBuffer = (ImageBuffer)source;
            ImageGlPlugin glPlugin = ImageGlPlugin.GetImageGlPlugin(sourceAsImageBuffer, false);

            // Prepare openGL for rendering
            PushOrthoProjection();
			GL.Disable(EnableCap.Lighting);
            GL.Enable(EnableCap.Texture2D);
            GL.Disable(EnableCap.DepthTest);
            
            GL.Enable(EnableCap.Blend);
            GL.BlendFunc(BlendingFactorSrc.SrcAlpha, BlendingFactorDest.OneMinusSrcAlpha);
            
            GL.Translate(x, y, 0);
            GL.Rotate(MathHelper.RadiansToDegrees(angleRadians), 0, 0, 1);
            GL.Scale(scaleX, scaleY, 1);

            RGBA_Bytes color = RGBA_Bytes.White;
            GL.Color4((byte)color.Red0To255, (byte)color.Green0To255, (byte)color.Blue0To255, (byte)color.Alpha0To255);

            glPlugin.DrawToGL();

            //Restore openGL state
            PopOrthoProjection();
        }

        public override void Render(IImageFloat imageSource,
            double x, double y,
            double angleDegrees,
            double scaleX, double ScaleY)
        {
            throw new NotImplementedException();
        }

        public override void FillRectangle(double left, double bottom, double right, double top, IColorType fillColor)
        {
            RoundedRect rect = new RoundedRect(left, bottom, right, top, 0);
            Render(rect, fillColor.GetAsRGBA_Bytes());
        }
        
        public override void Clear(IColorType color)
        {
            Affine transform = GetTransform();

            RoundedRect clearRect = new RoundedRect(new RectangleDouble(
                0 - transform.tx, width - transform.ty,
                0 - transform.tx, height - transform.ty), 0);
            Render(clearRect, color.GetAsRGBA_Bytes());
        }<commit_msg>Made a fast FillRectangle for open gl when the rectangle is pixel aligned (can write the other later). This helps a lot with clears (that happen a lot)<commit_after>/*
Copyright (c) 2014, <NAME>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met: 

1. Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer. 
2. Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution. 

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those
of the authors and should not be interpreted as representing official policies, 
either expressed or implied, of the FreeBSD Project.
*/

//#define AA_TIPS

using System;
using MatterHackers.Agg;
using MatterHackers.Agg.Image;
using MatterHackers.Agg.Transform;
using MatterHackers.Agg.VertexSource;
using MatterHackers.RenderOpenGl.OpenGl;
using MatterHackers.VectorMath;
using Tesselate;

namespace MatterHackers.RenderOpenGl
{
    public abstract class VertexTesselatorAbstract : Tesselator
    {
        public abstract void AddVertex(double x, double y);
    }

    public class Graphics2DOpenGL : Graphics2D
    {
        // We can have a single static instance because all gl rendering is required to happen on the ui thread so there can
        // be no runtime contention for this object (no thread contention).
        static AARenderToGLTesselator triangleEddgeInfo = new AARenderToGLTesselator();
        static int AATextureHandle = -1;

        public bool DoEdgeAntiAliasing = true;
        static RenderToGLTesselator renderNowTesselator = new RenderToGLTesselator();

        int width;
        int height;
		RectangleDouble cachedClipRect;

		public Graphics2DOpenGL(int width, int height)
        {
            this.width = width;
            this.height = height;
			cachedClipRect = new RectangleDouble(0, 0, width, height);
        }

        public override RectangleDouble GetClippingRect()
        {
            return cachedClipRect;
        }

        public override void SetClippingRect(RectangleDouble clippingRect)
        {
			cachedClipRect = clippingRect;
            GL.Scissor((int)Math.Floor(Math.Max(clippingRect.Left, 0)), (int)Math.Floor(Math.Max(clippingRect.Bottom, 0)),
                (int)Math.Ceiling(Math.Max(clippingRect.Width, 0)), (int)Math.Ceiling(Math.Max(clippingRect.Height, 0)));
			GL.Enable(EnableCap.ScissorTest);
        }

        public override IScanlineCache ScanlineCache
        {
            get { return null; }
            set { throw new Exception("There is no scanline cache on a GL surface."); }
        }

        public void PushOrthoProjection()
        {
			GL.Disable(EnableCap.CullFace);

            GL.MatrixMode(MatrixMode.Projection);
            GL.PushMatrix();
            GL.LoadIdentity();
            GL.Ortho(0, width, 0, height, 0, 1);

            GL.MatrixMode(MatrixMode.Modelview);
            GL.PushMatrix();
            GL.LoadIdentity();
        }

        public void PopOrthoProjection()
        {
			GL.MatrixMode(MatrixMode.Projection);
            GL.PopMatrix();
            GL.MatrixMode(MatrixMode.Modelview);
            GL.PopMatrix();
        }

        public static void SendShapeToTesselator(VertexTesselatorAbstract tesselator, IVertexSource vertexSource)
        {
            
#if !DEBUG
            try
#endif
            {
                tesselator.BeginPolygon();

                ShapePath.FlagsAndCommand PathAndFlags = 0;
                double x, y;
                bool haveBegunContour = false;
                while (!ShapePath.is_stop(PathAndFlags = vertexSource.vertex(out x, out y)))
                {
                    if (ShapePath.is_close(PathAndFlags)
                        || (haveBegunContour && ShapePath.is_move_to(PathAndFlags)))
                    {
                        tesselator.EndContour();
                        haveBegunContour = false;
                    }

                    if (!ShapePath.is_close(PathAndFlags))
                    {
                        if (!haveBegunContour)
                        {
                            tesselator.BeginContour();
                            haveBegunContour = true;
                        }

                        tesselator.AddVertex(x, y);
                    }
                }

                if (haveBegunContour)
                {
                    tesselator.EndContour();
                }

                tesselator.EndPolygon();
            }
#if !DEBUG
            catch
            {
            }
#endif
        }

		static byte[] CreateBufferForAATexture()
		{
			byte[] hardwarePixelBuffer = new byte[1024 * 4 * 4];
			for (int y = 0; y < 4; y++)
			{
				byte alpha = 0;
				for (int x = 0; x < 1024; x++)
				{
					hardwarePixelBuffer[(y * 1024 + x) * 4 + 0] = 255;
					hardwarePixelBuffer[(y * 1024 + x) * 4 + 1] = 255;
					hardwarePixelBuffer[(y * 1024 + x) * 4 + 2] = 255;
					hardwarePixelBuffer[(y * 1024 + x) * 4 + 3] = alpha;
					alpha = 255;
				}
			}
			return hardwarePixelBuffer;
		}

        void CheckLineImageCache()
        {
            if (AATextureHandle == -1)
            {
                // Create the texture handle and display list handle
                GL.GenTextures(1, out AATextureHandle);

                // Set up some texture parameters for openGL
                GL.BindTexture(TextureTarget.Texture2D, AATextureHandle);
                GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
                GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);

                GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapS, (int)TextureWrapMode.ClampToEdge);
                GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureWrapT, (int)TextureWrapMode.ClampToEdge);

                byte[] hardwarePixelBuffer = CreateBufferForAATexture();

                // Create the texture
                GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Rgba, 1024, 4,
                    0, PixelFormat.Rgba, PixelType.UnsignedByte, hardwarePixelBuffer);
            }
        }

        void DrawAAShape(IVertexSource vertexSource)
        {
			CheckLineImageCache();
            GL.Enable(EnableCap.Texture2D);
            GL.BindTexture(TextureTarget.Texture2D, AATextureHandle);

            triangleEddgeInfo.Clear();
            Graphics2DOpenGL.SendShapeToTesselator(triangleEddgeInfo, vertexSource);

            // now render it
            triangleEddgeInfo.RenderLastToGL();
        }

        public override void Render(IVertexSource vertexSource, int pathIndexToRender, RGBA_Bytes colorBytes)
        {
			PushOrthoProjection();

            GL.BlendFunc(BlendingFactorSrc.SrcAlpha, BlendingFactorDest.OneMinusSrcAlpha);
            GL.Enable(EnableCap.Blend);

            vertexSource.rewind(pathIndexToRender);

            GL.Color4(colorBytes.red, colorBytes.green, colorBytes.blue, colorBytes.alpha);

            Affine transform = GetTransform();
            if (!transform.is_identity())
            {
                vertexSource = new VertexSourceApplyTransform(vertexSource, transform);
            }

            if (DoEdgeAntiAliasing)
            {
                DrawAAShape(vertexSource);
            }
            else
            {
                renderNowTesselator.Clear();
                Graphics2DOpenGL.SendShapeToTesselator(renderNowTesselator, vertexSource);
            }

            PopOrthoProjection();
        }

        public override void Render(IImageByte source,
            double x, double y,
            double angleRadians,
            double scaleX, double scaleY)
        {
#if true
            Affine transform = GetTransform();
            if (!transform.is_identity())
            {
                if (scaleX != 1 || scaleY != 1)// || angleDegrees != 0)
                {
                    throw new NotImplementedException();
                }
                // TODO: <BUG> make this do rotation and scalling
                transform.transform(ref x, ref y);
                scaleX *= transform.sx;
                scaleY *= transform.sy;
            }
#endif

#if true
            // TODO: <BUG> make this do rotation and scalling
            RectangleInt sourceBounds = source.GetBounds();
            sourceBounds.Offset((int)x, (int)y);
            RectangleInt destBounds = new RectangleInt((int)cachedClipRect.Left, (int)cachedClipRect.Bottom, (int)cachedClipRect.Right, (int)cachedClipRect.Top);

            if (!RectangleInt.DoIntersect(sourceBounds, destBounds))
            {
                if (scaleX != 1 || scaleY != 1)// || angleDegrees != 0)
                {
                    //throw new NotImplementedException();
                }
                //return;
            }
#endif

            ImageBuffer sourceAsImageBuffer = (ImageBuffer)source;
            ImageGlPlugin glPlugin = ImageGlPlugin.GetImageGlPlugin(sourceAsImageBuffer, false);

            // Prepare openGL for rendering
            PushOrthoProjection();
			GL.Disable(EnableCap.Lighting);
            GL.Enable(EnableCap.Texture2D);
            GL.Disable(EnableCap.DepthTest);
            
            GL.Enable(EnableCap.Blend);
            GL.BlendFunc(BlendingFactorSrc.SrcAlpha, BlendingFactorDest.OneMinusSrcAlpha);
            
            GL.Translate(x, y, 0);
            GL.Rotate(MathHelper.RadiansToDegrees(angleRadians), 0, 0, 1);
            GL.Scale(scaleX, scaleY, 1);

            RGBA_Bytes color = RGBA_Bytes.White;
            GL.Color4((byte)color.Red0To255, (byte)color.Green0To255, (byte)color.Blue0To255, (byte)color.Alpha0To255);

            glPlugin.DrawToGL();

            //Restore openGL state
            PopOrthoProjection();
        }

        public override void Render(IImageFloat imageSource,
            double x, double y,
            double angleDegrees,
            double scaleX, double ScaleY)
        {
            throw new NotImplementedException();
        }

        public override void FillRectangle(double left, double bottom, double right, double top, IColorType fillColor)
        {
            // This only works for translation. If we have a rotation or scale in the transform this will have some problems.
            Affine transform = GetTransform();
            transform.transform(ref left, ref bottom);
            transform.transform(ref right, ref top);

            if (left == (int)left
                && bottom == (int)bottom
                && right == (int)right
                && top == (int)top)
            {
                PushOrthoProjection();

                GL.Disable(EnableCap.Texture2D);
                GL.BlendFunc(BlendingFactorSrc.SrcAlpha, BlendingFactorDest.OneMinusSrcAlpha);
                GL.Enable(EnableCap.Blend);

                GL.Color4(fillColor.Red0To255, fillColor.Green0To255, fillColor.Blue0To255, fillColor.Alpha0To255);

                GL.Begin(BeginMode.Triangles);
                // triangel 1
                {
                    GL.Vertex2(left, bottom);
                    GL.Vertex2(right, bottom);
                    GL.Vertex2(right, top);
                }

                // triangel 2
                {
                    GL.Vertex2(left, bottom);
                    GL.Vertex2(right, top);
                    GL.Vertex2(left, top);
                }
                GL.End();

                PopOrthoProjection();
            }
            else
            {
                RoundedRect rect = new RoundedRect(left, bottom, right, top, 0);
                Render(rect, fillColor.GetAsRGBA_Bytes());
            }
        }
        
        public override void Clear(IColorType color)
        {
            Affine transform = GetTransform();

            RoundedRect clearRect = new RoundedRect(new RectangleDouble(
                0 - transform.tx, width - transform.ty,
                0 - transform.tx, height - transform.ty), 0);
            Render(clearRect, color.GetAsRGBA_Bytes());
        }
{'fraction_non_alphanumeric': 0.08113329040566646, 'fraction_numerical': 0.011107533805537669, 'mean_word_length': 1.9702366722448004, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '396884', 'n_tokens_mistral': 7158, 'n_tokens_neox': 6408, 'n_words': 1682}