<commit_before><!DOCTYPE HTML>
<html>
<head>
    <meta charset="utf-8">
    <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">

    <link href="../css/bootstrap.min.css" rel="stylesheet">
    <link href="../css/cornerstone.min.css" rel="stylesheet">
</head>
<body>
<div class="container">
    <div class="page-header">
        <h1>
            Display DICOM Segmentation IOD
        </h1>
        <p>
            This example demonstrates how to display a DICOM Segmentation object.
        </p>
        <a href="../index.html">Go back to the Examples page</a>
    </div>

    <div class="row">
        <button id="sampleData">Get Sample Data</button><p id="statusLine"></p>
        <div class="col-xs-9">
            <div style="width:512px;height:512px;position:relative;display:inline-block;"
                 oncontextmenu="return false"
                 class='cornerstone-enabled-image'
                 unselectable='on'
                 onselectstart='return false;'
                 onmousedown='return false;'>
                <div id="dicomImage"
                     style="width:512px;height:512px;top:0px;left:0px; position:absolute;">
                </div>
            </div>
            <p>Sample data is from the dicom4qi connectathon event to be held at RSNA 2017.  This is <a href='https://qiicr.gitbooks.io/dicom4qi/content/instructions/seg.html#test-dataset-2'>test dataset #2</a>
        </div>
        <div class="col-xs-3">
            <label for="layers">Select active layer</label>
            <select name="layers" id="layers" size="2" style="width: 100%; min-width: 150px;"></select>
            <div id="properties" style="width: 100%; min-width: 150px; margin-top: 15px;">
                <label>Layer Properties</label>
                <div style="width: 100%; padding: 5px 5px 5px 10px;">
                    <div style="margin-bottom: 15px;">
                        <label for="visible"> Visible</label><br>
                        <input name="visible" type="radio" value="1"> Yes &nbsp;&nbsp;
                        <input name="visible" type="radio" value="0"> No
                    </div>
                    <div style="margin-bottom: 15px;">
                        <label for="imageOpacity"> Opacity</label>
                        <input id="imageOpacity" type="range" class="range" min=0 max=1 step=0.1 value=0>
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>
</body>

<script src="../js/jquery.min.js"></script>
<script src="../js/jqueryFileDrop.js"></script>

<script src="../js/cornerstone.min.js"></script>
<script src="../js/cornerstoneMath.min.js"></script>
<script src="../js/cornerstoneTools.min.js"></script>
<script src="../js/dicomParser.min.js"></script>
<script src="../js/cornerstoneWADOImageLoader.min.js"></script>
<script src="../js/initWebWorkers.js"></script>

<script src="../../build/dcmjs.js"></script>

<script>
  let element;
  let metaData = {};
  $(document).ready(function() {
    element = document.getElementById('dicomImage');

    cornerstone.enable(element);

    //
    // associate per-frame position with cornerstone imageID
    // so that correct overlay is selected when scrolling
    // through the stack
    //
    function metaDataProvider(type, imageId) {
      if (!metaData[imageId]) {
        return;
      }

      return metaData[imageId][type];
    }

    function addMetaData(type, imageId, data) {
      metaData[imageId] = metaData[imageId] || {};
      metaData[imageId][type] = data;
    }

    cornerstone.metaData.addProvider(metaDataProvider);

    //
    // uses cornerstone caching to access a bytearray of the
    // part10 dicom, then uses dcmjs to parse this
    // into javascript object and populates the
    // metadata for the per-frame imageIDs.
    //
    function loadMultiFrameAndPopulateMetadata(baseImageId) {
      var promise = new Promise(function (resolve, reject) {
        var multiframe;
        cornerstone.loadAndCacheImage(baseImageId).then(function(image) {
          var arrayBuffer = image.data.byteArray.buffer;
          dicomData = DCMJS.data.DicomMessage.readFile(arrayBuffer);
          let dataset = DCMJS.data.DicomMetaDictionary.naturalizeDataset(dicomData.dict);
          dataset._meta = DCMJS.data.DicomMetaDictionary.namifyDataset(dicomData.meta);

          multiframe = DCMJS.normalizers.Normalizer.normalizeToDataset([dataset]);

          const numFrames = Number(multiframe.NumberOfFrames);
          for (let i=0; i < numFrames; i++) {
            const imageId = baseImageId + '?frame=' + i;

            var functionalGroup = multiframe.PerFrameFunctionalGroupsSequence[i];
            var imagePositionArray = functionalGroup.PlanePositionSequence.ImagePositionPatient;

            addMetaData('imagePlane', imageId, {
              imagePositionPatient: {
                x: imagePositionArray[0],
                y: imagePositionArray[1],
                z: imagePositionArray[2],
              }
            });
          }

          resolve(multiframe);
        });
      });
      return promise;
    }

    //
    // creates an array of per-frame imageIds in the form needed for cornerstone processing.
    // If segmentNumber is provided, then only create imageIds for the frames where the 
    // segment is defined.
    //
    function getImageIds(multiframe, baseImageId, segmentNumber) {
      const imageIds = [];
      const numFrames = Number(multiframe.NumberOfFrames);
      for (let i=0; i < numFrames; i++) {
        let segNum;
        if (multiframe.PerFrameFunctionalGroupsSequence[i].SegmentIdentificationSequence) {
          segNum = multiframe.PerFrameFunctionalGroupsSequence[i].SegmentIdentificationSequence.ReferencedSegmentNumber;
        }

        if ((segmentNumber && segNum && segNum === segmentNumber) ||
            segmentNumber === undefined) {
          const imageId = baseImageId + '?frame=' + i;
          imageIds.push(imageId);
        }
      }
      return imageIds;
    }

    function downloadAndDisplaySampleData() {
      cornerstone.disable(element);
      cornerstone.enable(element);
      cornerstoneTools.clearToolState(element, 'stack');

      const urlRoot = 'https://s3.amazonaws.com/IsomicsPublic/SampleData/rsna2017/seg/task2/'

      const baseImageId = 'dicomweb:' + urlRoot + 'PT-multiframe.dcm';
      var dataPromise = loadMultiFrameAndPopulateMetadata(baseImageId);

      const segBaseImageId = 'dicomweb:' + urlRoot + 'SEG/tumor_User1_Manual_Trial1.dcm';
      var segPromise = loadMultiFrameAndPopulateMetadata(segBaseImageId);

      //
      // once the images have been loaded
      //
      Promise.all([dataPromise, segPromise]).then(values => {
        const multiframe = values[0];
        const segMultiframe = values[1];

        // make the base image layer
        var baseImageIds = getImageIds(multiframe, baseImageId);
        const baseStack = {
          imageIds: baseImageIds,
          currentImageIdIndex: 0,
          options: {
            name: 'Referenced Image'
          }
        };

        //
        // make the set of layers associated with the segmentations
        //
        const segStacks = [];
        segMultiframe.SegmentSequence.forEach(segment => {
          // first, map the dicom color into a cornerstone colormap
          const cielab = segment.RecommendedDisplayCIELabValue;
          let rgba = DCMJS.data.Colors.dicomlab2RGB(cielab).map(x => x * 255);
          rgba.push(255);
          const colormapId = 'Colormap_' + segment.SegmentNumber;
          var colormap = cornerstone.colors.getColormap(colormapId);
          colormap.setNumberOfColors(2);
          colormap.insertColor(0, [0, 0, 255, 0]);
          colormap.insertColor(1, rgba);
          // define a stack based on the image ids, color, and segment name
          const segImageIds = getImageIds(segMultiframe, segBaseImageId, segment.SegmentNumber);
          const segStack = {
            imageIds: segImageIds,
            currentImageIdIndex: 0,
            options: {
              opacity: 0.7,
              visible: true,
              colormap: colormapId,
              name: segment.SegmentLabel,
              viewport: {
                pixelReplication: true
              }
            }
          };
          segStacks.push(segStack);
        });

        //
        // create a FusionRenderer and provide it with a function
        // to find the closest overlay image based on z position
        //
        var renderer = new cornerstoneTools.stackRenderers.FusionRenderer();
        renderer.findImageFn = function(imageIds, targetImageId) {
          var minDistance = 1;
          var targetImagePlane = cornerstone.metaData.get('imagePlane', targetImageId);
          var imagePositionZ = targetImagePlane.imagePositionPatient.z;
          var closest;
          imageIds.forEach(function(imageId) {
            var imagePlane = cornerstone.metaData.get('imagePlane', imageId);
            var imgPosZ = imagePlane.imagePositionPatient.z;
            var distance = Math.abs(imgPosZ - imagePositionZ);
            if (distance < minDistance) {
              minDistance = distance;
              closest = imageId;
            }
          });
          return closest;
        };

        //
        // now setup the cornerstone element with out base image
        // and enable the tools we want to have work on it
        //
        cornerstone.loadAndCacheImage(baseStack.imageIds[0]).then(function(image) {
          cornerstoneTools.addStackStateManager(element, ['stack']);
          cornerstoneTools.addToolState(element, 'stackRenderer', renderer);
          cornerstoneTools.addToolState(element, 'stack', baseStack);

          segStacks.forEach(segStack => {
            cornerstoneTools.addToolState(element, 'stack', segStack);
          });

          cornerstone.displayImage(element, image);

          cornerstoneTools.mouseInput.enable(element);
          cornerstoneTools.mouseWheelInput.enable(element);
          cornerstoneTools.keyboardInput.enable(element);

          cornerstoneTools.wwwc.activate(element, 1);
          cornerstoneTools.pan.activate(element, 2);
          cornerstoneTools.zoom.activate(element, 4);
          cornerstoneTools.stackScrollWheel.activate(element);
          cornerstoneTools.stackScrollKeyboard.activate(element);

          // Update dropdown size to make all layers name visible
          $('#layers').prop('size', layers.length);

          //cornerstoneTools.scrollToIndex(element, 51);

          // Listen to `change` event to set the selected layer as active
          $("#layers").change(function(event) {
            var layerId = event.currentTarget.value;
            if (layerId === "") {
              return;
            }

            cornerstone.setActiveLayer(element, layerId);
          });
        });

        $('p#statusLine').text("");

      });

    }

    // Select the right layer in the dropdown
    function updateSelectedLayer(layerId) {
      var $layers = $('#layers');
      var currentLayerId = $layers.val();

      if(currentLayerId !== layerId) {
        $layers.val(layerId);
        $layers.trigger('change');
      }
    }

    // Listen to `change` event to update the opacity of the active layer
    $("#imageOpacity").on('input',function(event) {
      var layer = cornerstone.getActiveLayer(element);
      layer.options.opacity = parseFloat(event.currentTarget.value);
      cornerstone.updateImage(element);
    });

    // Listen to `change` event to update the visibility of the active layer
    $('input[name=visible]').change(function(event) {
      var layer = cornerstone.getActiveLayer(element);
      layer.options.visible = (event.currentTarget.value === "1");
      cornerstone.updateImage(element);
    });

    // This event will be called every time a layer is added through cornerstone.addLayer
    // The layer is added to the dropdown to make it possible to select and interact with it
    $(element).on('CornerstoneLayerAdded', function(e, eventData) {
      var layer = cornerstone.getLayer(eventData.element, eventData.layerId);
      var $layers = $('#layers');
      var $layerOption = $('<option></option>')
        .val(layer.layerId)
        .text(layer.options.name);

      // Set the layer as selected in case its the the first layer to be added
      if($layers.children().length === 0) {
        $layerOption.prop('selected', true);
      }

      $layers.append($layerOption);
    });

    // This event will be called every time cornerstone.setActiveLayer is called
    // We need to load the layer properties and update the selected layer in the dropdown
    $(element).on('CornerstoneActiveLayerChanged', function(e, eventData) {
      var layer = cornerstone.getActiveLayer(element);
      var opacity = layer.options.opacity == null ? 1 : layer.options.opacity;
      var visible = layer.options.visible !== false ? 1 : 0;

      // Restore all properties for the active layer
      $('#imageOpacity').val(opacity);
      $("input[name=visible][value=" + visible + "]").prop('checked', true);

      updateSelectedLayer(eventData.layerId);
    });

    $('#sampleData').click(function(event) {
      $('p#statusLine').text("Downloading...");
      downloadAndDisplaySampleData();
    });


    window.addEventListener('resize', function() {
      cornerstone.resize(element, true);
    });

    window.dispatchEvent(new Event('resize'));
  });
</script>
</html>
<commit_msg>Invert the PET image to match Andrey's display preference<commit_after><!DOCTYPE HTML>
<html>
<head>
    <meta charset="utf-8">
    <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">

    <link href="../css/bootstrap.min.css" rel="stylesheet">
    <link href="../css/cornerstone.min.css" rel="stylesheet">
</head>
<body>
<div class="container">
    <div class="page-header">
        <h1>
            Display DICOM Segmentation IOD
        </h1>
        <p>
            This example demonstrates how to display a DICOM Segmentation object.
        </p>
        <a href="../index.html">Go back to the Examples page</a>
    </div>

    <div class="row">
        <button id="sampleData">Get Sample Data</button><p id="statusLine"></p>
        <div class="col-xs-9">
            <div style="width:512px;height:512px;position:relative;display:inline-block;"
                 oncontextmenu="return false"
                 class='cornerstone-enabled-image'
                 unselectable='on'
                 onselectstart='return false;'
                 onmousedown='return false;'>
                <div id="dicomImage"
                     style="width:512px;height:512px;top:0px;left:0px; position:absolute;">
                </div>
            </div>
            <p>Sample data is from the dicom4qi connectathon event to be held at RSNA 2017.  This is <a href='https://qiicr.gitbooks.io/dicom4qi/content/instructions/seg.html#test-dataset-2'>test dataset #2</a>
        </div>
        <div class="col-xs-3">
            <label for="layers">Select active layer</label>
            <select name="layers" id="layers" size="2" style="width: 100%; min-width: 150px;"></select>
            <div id="properties" style="width: 100%; min-width: 150px; margin-top: 15px;">
                <label>Layer Properties</label>
                <div style="width: 100%; padding: 5px 5px 5px 10px;">
                    <div style="margin-bottom: 15px;">
                        <label for="visible"> Visible</label><br>
                        <input name="visible" type="radio" value="1"> Yes &nbsp;&nbsp;
                        <input name="visible" type="radio" value="0"> No
                    </div>
                    <div style="margin-bottom: 15px;">
                        <label for="imageOpacity"> Opacity</label>
                        <input id="imageOpacity" type="range" class="range" min=0 max=1 step=0.1 value=0>
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>
</body>

<script src="../js/jquery.min.js"></script>
<script src="../js/jqueryFileDrop.js"></script>

<script src="../js/cornerstone.min.js"></script>
<script src="../js/cornerstoneMath.min.js"></script>
<script src="../js/cornerstoneTools.min.js"></script>
<script src="../js/dicomParser.min.js"></script>
<script src="../js/cornerstoneWADOImageLoader.min.js"></script>
<script src="../js/initWebWorkers.js"></script>

<script src="../../build/dcmjs.js"></script>

<script>
  let element;
  let metaData = {};
  $(document).ready(function() {
    element = document.getElementById('dicomImage');

    cornerstone.enable(element);

    //
    // associate per-frame position with cornerstone imageID
    // so that correct overlay is selected when scrolling
    // through the stack
    //
    function metaDataProvider(type, imageId) {
      if (!metaData[imageId]) {
        return;
      }

      return metaData[imageId][type];
    }

    function addMetaData(type, imageId, data) {
      metaData[imageId] = metaData[imageId] || {};
      metaData[imageId][type] = data;
    }

    cornerstone.metaData.addProvider(metaDataProvider);

    //
    // uses cornerstone caching to access a bytearray of the
    // part10 dicom, then uses dcmjs to parse this
    // into javascript object and populates the
    // metadata for the per-frame imageIDs.
    //
    function loadMultiFrameAndPopulateMetadata(baseImageId) {
      var promise = new Promise(function (resolve, reject) {
        var multiframe;
        cornerstone.loadAndCacheImage(baseImageId).then(function(image) {
          var arrayBuffer = image.data.byteArray.buffer;
          dicomData = DCMJS.data.DicomMessage.readFile(arrayBuffer);
          let dataset = DCMJS.data.DicomMetaDictionary.naturalizeDataset(dicomData.dict);
          dataset._meta = DCMJS.data.DicomMetaDictionary.namifyDataset(dicomData.meta);

          multiframe = DCMJS.normalizers.Normalizer.normalizeToDataset([dataset]);

          const numFrames = Number(multiframe.NumberOfFrames);
          for (let i=0; i < numFrames; i++) {
            const imageId = baseImageId + '?frame=' + i;

            var functionalGroup = multiframe.PerFrameFunctionalGroupsSequence[i];
            var imagePositionArray = functionalGroup.PlanePositionSequence.ImagePositionPatient;

            addMetaData('imagePlane', imageId, {
              imagePositionPatient: {
                x: imagePositionArray[0],
                y: imagePositionArray[1],
                z: imagePositionArray[2],
              }
            });
          }

          resolve(multiframe);
        });
      });
      return promise;
    }

    //
    // creates an array of per-frame imageIds in the form needed for cornerstone processing.
    // If segmentNumber is provided, then only create imageIds for the frames where the 
    // segment is defined.
    //
    function getImageIds(multiframe, baseImageId, segmentNumber) {
      const imageIds = [];
      const numFrames = Number(multiframe.NumberOfFrames);
      for (let i=0; i < numFrames; i++) {
        let segNum;
        if (multiframe.PerFrameFunctionalGroupsSequence[i].SegmentIdentificationSequence) {
          segNum = multiframe.PerFrameFunctionalGroupsSequence[i].SegmentIdentificationSequence.ReferencedSegmentNumber;
        }

        if ((segmentNumber && segNum && segNum === segmentNumber) ||
            segmentNumber === undefined) {
          const imageId = baseImageId + '?frame=' + i;
          imageIds.push(imageId);
        }
      }
      return imageIds;
    }

    function downloadAndDisplaySampleData() {
      cornerstone.disable(element);
      cornerstone.enable(element);
      cornerstoneTools.clearToolState(element, 'stack');

      const urlRoot = 'https://s3.amazonaws.com/IsomicsPublic/SampleData/rsna2017/seg/task2/'

      const baseImageId = 'dicomweb:' + urlRoot + 'PT-multiframe.dcm';
      var dataPromise = loadMultiFrameAndPopulateMetadata(baseImageId);

      const segBaseImageId = 'dicomweb:' + urlRoot + 'SEG/tumor_User1_Manual_Trial1.dcm';
      var segPromise = loadMultiFrameAndPopulateMetadata(segBaseImageId);

      //
      // once the images have been loaded
      //
      Promise.all([dataPromise, segPromise]).then(values => {
        const multiframe = values[0];
        const segMultiframe = values[1];

        // make the base image layer
        var baseImageIds = getImageIds(multiframe, baseImageId);
        const baseStack = {
          imageIds: baseImageIds,
          currentImageIdIndex: 0,
          options: {
            name: 'Referenced Image',
            viewport: {
              invert: true
            }
          },
        };

        //
        // make the set of layers associated with the segmentations
        //
        const segStacks = [];
        segMultiframe.SegmentSequence.forEach(segment => {
          // first, map the dicom color into a cornerstone colormap
          const cielab = segment.RecommendedDisplayCIELabValue;
          let rgba = DCMJS.data.Colors.dicomlab2RGB(cielab).map(x => x * 255);
          rgba.push(255);
          const colormapId = 'Colormap_' + segment.SegmentNumber;
          var colormap = cornerstone.colors.getColormap(colormapId);
          colormap.setNumberOfColors(2);
          colormap.insertColor(0, [0, 0, 255, 0]);
          colormap.insertColor(1, rgba);
          // define a stack based on the image ids, color, and segment name
          const segImageIds = getImageIds(segMultiframe, segBaseImageId, segment.SegmentNumber);
          const segStack = {
            imageIds: segImageIds,
            currentImageIdIndex: 0,
            options: {
              opacity: 0.7,
              visible: true,
              colormap: colormapId,
              name: segment.SegmentLabel,
              viewport: {
                pixelReplication: true
              }
            }
          };
          segStacks.push(segStack);
        });

        //
        // create a FusionRenderer and provide it with a function
        // to find the closest overlay image based on z position
        //
        var renderer = new cornerstoneTools.stackRenderers.FusionRenderer();
        renderer.findImageFn = function(imageIds, targetImageId) {
          var minDistance = 1;
          var targetImagePlane = cornerstone.metaData.get('imagePlane', targetImageId);
          var imagePositionZ = targetImagePlane.imagePositionPatient.z;
          var closest;
          imageIds.forEach(function(imageId) {
            var imagePlane = cornerstone.metaData.get('imagePlane', imageId);
            var imgPosZ = imagePlane.imagePositionPatient.z;
            var distance = Math.abs(imgPosZ - imagePositionZ);
            if (distance < minDistance) {
              minDistance = distance;
              closest = imageId;
            }
          });
          return closest;
        };

        //
        // now setup the cornerstone element with out base image
        // and enable the tools we want to have work on it
        //
        cornerstone.loadAndCacheImage(baseStack.imageIds[0]).then(function(image) {
          cornerstoneTools.addStackStateManager(element, ['stack']);
          cornerstoneTools.addToolState(element, 'stackRenderer', renderer);
          cornerstoneTools.addToolState(element, 'stack', baseStack);

          segStacks.forEach(segStack => {
            cornerstoneTools.addToolState(element, 'stack', segStack);
          });

          cornerstone.displayImage(element, image);

          cornerstoneTools.mouseInput.enable(element);
          cornerstoneTools.mouseWheelInput.enable(element);
          cornerstoneTools.keyboardInput.enable(element);

          cornerstoneTools.wwwc.activate(element, 1);
          cornerstoneTools.pan.activate(element, 2);
          cornerstoneTools.zoom.activate(element, 4);
          cornerstoneTools.stackScrollWheel.activate(element);
          cornerstoneTools.stackScrollKeyboard.activate(element);

          // Update dropdown size to make all layers name visible
          $('#layers').prop('size', layers.length);

          //cornerstoneTools.scrollToIndex(element, 51);

          // Listen to `change` event to set the selected layer as active
          $("#layers").change(function(event) {
            var layerId = event.currentTarget.value;
            if (layerId === "") {
              return;
            }

            cornerstone.setActiveLayer(element, layerId);
          });
        });

        $('p#statusLine').text("");

      });

    }

    // Select the right layer in the dropdown
    function updateSelectedLayer(layerId) {
      var $layers = $('#layers');
      var currentLayerId = $layers.val();

      if(currentLayerId !== layerId) {
        $layers.val(layerId);
        $layers.trigger('change');
      }
    }

    // Listen to `change` event to update the opacity of the active layer
    $("#imageOpacity").on('input',function(event) {
      var layer = cornerstone.getActiveLayer(element);
      layer.options.opacity = parseFloat(event.currentTarget.value);
      cornerstone.updateImage(element);
    });

    // Listen to `change` event to update the visibility of the active layer
    $('input[name=visible]').change(function(event) {
      var layer = cornerstone.getActiveLayer(element);
      layer.options.visible = (event.currentTarget.value === "1");
      cornerstone.updateImage(element);
    });

    // This event will be called every time a layer is added through cornerstone.addLayer
    // The layer is added to the dropdown to make it possible to select and interact with it
    $(element).on('CornerstoneLayerAdded', function(e, eventData) {
      var layer = cornerstone.getLayer(eventData.element, eventData.layerId);
      var $layers = $('#layers');
      var $layerOption = $('<option></option>')
        .val(layer.layerId)
        .text(layer.options.name);

      // Set the layer as selected in case its the the first layer to be added
      if($layers.children().length === 0) {
        $layerOption.prop('selected', true);
      }

      $layers.append($layerOption);
    });

    // This event will be called every time cornerstone.setActiveLayer is called
    // We need to load the layer properties and update the selected layer in the dropdown
    $(element).on('CornerstoneActiveLayerChanged', function(e, eventData) {
      var layer = cornerstone.getActiveLayer(element);
      var opacity = layer.options.opacity == null ? 1 : layer.options.opacity;
      var visible = layer.options.visible !== false ? 1 : 0;

      // Restore all properties for the active layer
      $('#imageOpacity').val(opacity);
      $("input[name=visible][value=" + visible + "]").prop('checked', true);

      updateSelectedLayer(eventData.layerId);
    });

    $('#sampleData').click(function(event) {
      $('p#statusLine').text("Downloading...");
      downloadAndDisplaySampleData();
    });


    window.addEventListener('resize', function() {
      cornerstone.resize(element, true);
    });

    window.dispatchEvent(new Event('resize'));
  });
</script>
</html>

{'fraction_non_alphanumeric': 0.12174199934015177, 'fraction_numerical': 0.008211444701052091, 'mean_word_length': 2.3728981206726014, 'pattern_counts': {'":': 0, '<': 173, '<?xml version=': 0, '>': 175, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4655107', 'n_tokens_mistral': 7775, 'n_tokens_neox': 7398, 'n_words': 1699}