<commit_before>/*
 * #%L
 * OME Bio-Formats package for reading and converting biological file formats.
 * %%
 * Copyright (C) 2005 - 2013 Open Microscopy Environment:
 *   - Board of Regents of the University of Wisconsin-Madison
 *   - Glencoe Software, Inc.
 *   - University of Dundee
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 2 of the 
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public 
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/gpl-2.0.html>.
 * #L%
 */

package loci.formats.in;

import java.io.IOException;
import java.util.Vector;

import loci.common.DataTools;
import loci.common.Location;
import loci.common.RandomAccessInputStream;
import loci.common.services.DependencyException;
import loci.common.services.ServiceException;
import loci.common.services.ServiceFactory;
import loci.formats.CoreMetadata;
import loci.formats.FormatException;
import loci.formats.FormatReader;
import loci.formats.FormatTools;
import loci.formats.MetadataTools;
import loci.formats.MissingLibraryException;
import loci.formats.meta.MetadataStore;
import loci.formats.services.NetCDFService;
import loci.formats.services.NetCDFServiceImpl;
import ome.xml.model.primitives.Color;
import ome.xml.model.primitives.PositiveFloat;

/**
 * Reader for Bitplane Imaris 5.5 (HDF) files.
 *
 * <dl><dt><b>Source code:</b></dt>
 * <dd><a href="http://trac.openmicroscopy.org.uk/ome/browser/bioformats.git/components/bio-formats/src/loci/formats/in/ImarisHDFReader.java">Trac</a>,
 * <a href="http://git.openmicroscopy.org/?p=bioformats.git;a=blob;f=components/bio-formats/src/loci/formats/in/ImarisHDFReader.java;hb=HEAD">Gitweb</a></dd></dl>
 */
public class ImarisHDFReader extends FormatReader {

  // -- Constants --

  public static final String HDF_MAGIC_STRING = "HDF";

  private static final String[] DELIMITERS = {" ", "-", "."};

  // -- Fields --

  private double pixelSizeX, pixelSizeY, pixelSizeZ;
  private double minX, minY, minZ, maxX, maxY, maxZ;
  private int seriesCount;
  private NetCDFService netcdf;

  // channel parameters
  private Vector<String> emWave, exWave, channelMin, channelMax;
  private Vector<String> gain, pinhole, channelName, microscopyMode;
  private Vector<double[]> colors;
  private int lastChannel = 0;

  // -- Constructor --

  /** Constructs a new Imaris HDF reader. */
  public ImarisHDFReader() {
    super("Bitplane Imaris 5.5 (HDF)", "ims");
    suffixSufficient = false;
    domains = new String[] {FormatTools.UNKNOWN_DOMAIN};
  }

  // -- IFormatReader API methods --

  /* @see loci.formats.IFormatReader#getOptimalTileWidth() */
  public int getOptimalTileWidth() {
    return core.get(core.size() - 1).sizeX;
  }

  /* @see loci.formats.IFormatReader#getOptimalTileHeight() */
  public int getOptimalTileHeight() {
    return core.get(core.size() - 1).sizeY;
  }

  /* @see loci.formats.IFormatReader#isThisType(RandomAccessInputStream) */
  public boolean isThisType(RandomAccessInputStream stream) throws IOException {
    final int blockLen = 8;
    if (!FormatTools.validStream(stream, blockLen, false)) return false;
    return stream.readString(blockLen).indexOf(HDF_MAGIC_STRING) >= 0;
  }

  /* @see loci.formats.IFormatReader#get8BitLookupTable() */
  public byte[][] get8BitLookupTable() {
    FormatTools.assertId(currentId, true, 1);
    if (getPixelType() != FormatTools.UINT8 || !isIndexed()) return null;

    if (lastChannel < 0 || lastChannel >= colors.size()) {
      return null;
    }

    double[] color = colors.get(lastChannel);

    byte[][] lut = new byte[3][256];
    for (int c=0; c<lut.length; c++) {
      double max = color[c] * 255;
      for (int p=0; p<lut[c].length; p++) {
        lut[c][p] = (byte) ((p / 255.0) * max);
      }
    }
    return lut;
  }

  /* @see loci.formats.IFormatReaderget16BitLookupTable() */
  public short[][] get16BitLookupTable() {
    FormatTools.assertId(currentId, true, 1);
    if (getPixelType() != FormatTools.UINT16 || !isIndexed()) return null;

    if (lastChannel < 0 || lastChannel >= colors.size()) {
      return null;
    }

    double[] color = colors.get(lastChannel);

    short[][] lut = new short[3][65536];
    for (int c=0; c<lut.length; c++) {
      double max = color[c] * 65535;
      for (int p=0; p<lut[c].length; p++) {
        lut[c][p] = (short) ((p / 65535.0) * max);
      }
    }
    return lut;
  }

  /**
   * @see loci.formats.IFormatReader#openBytes(int, byte[], int, int, int, int)
   */
  public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h)
    throws FormatException, IOException
  {
    FormatTools.checkPlaneParameters(this, no, buf.length, x, y, w, h);

    lastChannel = getZCTCoords(no)[1];

    // pixel data is stored in XYZ blocks

    Object image = getImageData(no, x, y, w, h);

    boolean big = !isLittleEndian();
    int bpp = FormatTools.getBytesPerPixel(getPixelType());
    for (int row=0; row<h; row++) {
      int base = row * w * bpp;
      if (image instanceof byte[][]) {
        byte[][] data = (byte[][]) image;
        byte[] rowData = data[row];
        System.arraycopy(rowData, 0, buf, row*w, w);
      }
      else if (image instanceof short[][]) {
        short[][] data = (short[][]) image;
        short[] rowData = data[row];
        for (int i=0; i<w; i++) {
          DataTools.unpackBytes(rowData[i], buf, base + 2*i, 2, big);
        }
      }
      else if (image instanceof int[][]) {
        int[][] data = (int[][]) image;
        int[] rowData = data[row];
        for (int i=0; i<w; i++) {
          DataTools.unpackBytes(rowData[i], buf, base + i*4, 4, big);
        }
      }
      else if (image instanceof float[][]) {
        float[][] data = (float[][]) image;
        float[] rowData = data[row];
        for (int i=0; i<w; i++) {
          int v = Float.floatToIntBits(rowData[i]);
          DataTools.unpackBytes(v, buf, base + i*4, 4, big);
        }
      }
      else if (image instanceof double[][]) {
        double[][] data = (double[][]) image;
        double[] rowData = data[row];
        for (int i=0; i<w; i++) {
          long v = Double.doubleToLongBits(rowData[i]);
          DataTools.unpackBytes(v, buf, base + i * 8, 8, big);
        }
      }
    }

    return buf;
  }

  /* @see loci.formats.IFormatReader#close(boolean) */
  public void close(boolean fileOnly) throws IOException {
    super.close(fileOnly);
    if (!fileOnly) {
      seriesCount = 0;
      pixelSizeX = pixelSizeY = pixelSizeZ = 0;
      minX = minY = minZ = maxX = maxY = maxZ = 0;

      if (netcdf != null) netcdf.close();
      netcdf = null;

      emWave = exWave = channelMin = channelMax = null;
      gain = pinhole = channelName = microscopyMode = null;
      colors = null;
      lastChannel = 0;
    }
  }

  // -- Internal FormatReader API methods --

  /* @see loci.formats.FormatReader#initFile(String) */
  protected void initFile(String id) throws FormatException, IOException {
    super.initFile(id);

    try {
      ServiceFactory factory = new ServiceFactory();
      netcdf = factory.getInstance(NetCDFService.class);
      netcdf.setFile(id);
    }
    catch (DependencyException e) {
      throw new MissingLibraryException(NetCDFServiceImpl.NO_NETCDF_MSG, e);
    }

    pixelSizeX = pixelSizeY = pixelSizeZ = 1;

    emWave = new Vector<String>();
    exWave = new Vector<String>();
    channelMin = new Vector<String>();
    channelMax = new Vector<String>();
    gain = new Vector<String>();
    pinhole = new Vector<String>();
    channelName = new Vector<String>();
    microscopyMode = new Vector<String>();
    colors = new Vector<double[]>();

    seriesCount = 0;

    // read all of the metadata key/value pairs

    parseAttributes();

    CoreMetadata ms0 = core.get(0);

    if (seriesCount > 1) {
      for (int i=1; i<seriesCount; i++) {
        core.add(new CoreMetadata());
      }

      ms0.resolutionCount = seriesCount;

      for (int i=1; i<seriesCount; i++) {
        CoreMetadata ms = core.get(i);
        String groupPath =
          "/DataSet/ResolutionLevel_" + i + "/TimePoint_0/Channel_0";
        ms.sizeX =
          Integer.parseInt(netcdf.getAttributeValue(groupPath + "/ImageSizeX"));
        ms.sizeY =
          Integer.parseInt(netcdf.getAttributeValue(groupPath + "/ImageSizeY"));
        ms.sizeZ =
          Integer.parseInt(netcdf.getAttributeValue(groupPath + "/ImageSizeZ"));
        ms.imageCount = ms.sizeZ * getSizeC() * getSizeT();
        ms.sizeC = getSizeC();
        ms.sizeT = getSizeT();
        ms.thumbnail = true;
      }
    }
    ms0.imageCount = getSizeZ() * getSizeC() * getSizeT();
    ms0.thumbnail = false;
    ms0.dimensionOrder = "XYZCT";

    // determine pixel type - this isn't stored in the metadata, so we need
    // to check the pixels themselves

    int type = -1;

    Object pix = getImageData(0, 0, 0, 1, 1);
    if (pix instanceof byte[][]) type = FormatTools.UINT8;
    else if (pix instanceof short[][]) type = FormatTools.UINT16;
    else if (pix instanceof int[][]) type = FormatTools.UINT32;
    else if (pix instanceof float[][]) type = FormatTools.FLOAT;
    else if (pix instanceof double[][]) type = FormatTools.DOUBLE;
    else {
      throw new FormatException("Unknown pixel type: " + pix);
    }

    for (int i=0; i<core.size(); i++) {
      CoreMetadata ms = core.get(i);
      ms.pixelType = type;
      ms.dimensionOrder = "XYZCT";
      ms.rgb = false;
      ms.thumbSizeX = 128;
      ms.thumbSizeY = 128;
      ms.orderCertain = true;
      ms.littleEndian = true;
      ms.interleaved = false;
      ms.indexed = colors.size() >= getSizeC();
    }

    MetadataStore store = makeFilterMetadata();
    MetadataTools.populatePixels(store, this);

    String imageName = new Location(getCurrentFile()).getName();
    for (int s=0; s<getSeriesCount(); s++) {
      store.setImageName(imageName + " Resolution Level " + (s + 1), s);
    }

    if (getMetadataOptions().getMetadataLevel() == MetadataLevel.MINIMUM) {
      return;
    }

    int cIndex = 0;
    for (int s=0; s<getSeriesCount(); s++) {
      setSeries(s);
      double px = pixelSizeX, py = pixelSizeY, pz = pixelSizeZ;
      if (px == 1) px = (maxX - minX) / getSizeX();
      if (py == 1) py = (maxY - minY) / getSizeY();
      if (pz == 1) pz = (maxZ - minZ) / getSizeZ();

      if (px > 0) {
        store.setPixelsPhysicalSizeX(new PositiveFloat(px), s);
      }
      else {
        LOGGER.warn("Expected positive value for PhysicalSizeX; got {}", px);
      }
      if (py > 0) {
        store.setPixelsPhysicalSizeY(new PositiveFloat(py), s);
      }
      else {
        LOGGER.warn("Expected positive value for PhysicalSizeY; got {}", py);
      }
      if (pz > 0) {
        store.setPixelsPhysicalSizeZ(new PositiveFloat(pz), s);
      }
      else {
        LOGGER.warn("Expected positive value for PhysicalSizeZ; got {}", pz);
      }

      for (int i=0; i<getSizeC(); i++, cIndex++) {
        Float gainValue = null;
        Integer pinholeValue = null, emWaveValue = null, exWaveValue;

        if (cIndex < gain.size()) {
          try {
            gainValue = new Float(gain.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }
        if (cIndex < pinhole.size()) {
          try {
            pinholeValue = new Integer(pinhole.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }
        if (cIndex < emWave.size()) {
          try {
            emWaveValue = new Integer(emWave.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }
        if (cIndex < exWave.size()) {
          try {
            exWaveValue = new Integer(exWave.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }

        // CHECK
        /*
        store.setLogicalChannelName((String) channelName.get(cIndex), s, i);
        store.setDetectorSettingsGain(gainValue, s, i);
        store.setLogicalChannelPinholeSize(pinholeValue, s, i);
        store.setLogicalChannelMode((String) microscopyMode.get(cIndex), s, i);
        store.setLogicalChannelEmWave(emWaveValue, s, i);
        store.setLogicalChannelExWave(exWaveValue, s, i);
        */

        Double minValue = null, maxValue = null;

        if (cIndex < channelMin.size()) {
          try {
            minValue = new Double(channelMin.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }
        if (cIndex < channelMax.size()) {
          try {
            maxValue = new Double(channelMax.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }

        if (i < colors.size()) {
          double[] color = colors.get(i);
          Color realColor = new Color(
            (int) (color[0] * 255), (int) (color[1] * 255),
            (int) (color[2] * 255), 255);
          store.setChannelColor(realColor, s, i);
        }
      }
    }
    setSeries(0);
  }

  // -- Helper methods --

  private Object getImageData(int no, int x, int y, int width, int height)
    throws FormatException
  {
    int[] zct = getZCTCoords(no);
    String path = "/DataSet/ResolutionLevel_" + getCoreIndex() + "/TimePoint_" +
      zct[2] + "/Channel_" + zct[1] + "/Data";
    Object image = null;

    // the width and height cannot be 1, because then netCDF will give us a
    // singleton instead of an array
    if (height == 1) {
      height++;
    }
    if (width == 1) {
      width++;
    }
<commit_msg>Imaris HDF: fix retrieval of tiles from lower right quadrant<commit_after>/*
 * #%L
 * OME Bio-Formats package for reading and converting biological file formats.
 * %%
 * Copyright (C) 2005 - 2013 Open Microscopy Environment:
 *   - Board of Regents of the University of Wisconsin-Madison
 *   - Glencoe Software, Inc.
 *   - University of Dundee
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 2 of the 
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public 
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/gpl-2.0.html>.
 * #L%
 */

package loci.formats.in;

import java.io.IOException;
import java.util.Vector;

import loci.common.DataTools;
import loci.common.Location;
import loci.common.RandomAccessInputStream;
import loci.common.services.DependencyException;
import loci.common.services.ServiceException;
import loci.common.services.ServiceFactory;
import loci.formats.CoreMetadata;
import loci.formats.FormatException;
import loci.formats.FormatReader;
import loci.formats.FormatTools;
import loci.formats.MetadataTools;
import loci.formats.MissingLibraryException;
import loci.formats.meta.MetadataStore;
import loci.formats.services.NetCDFService;
import loci.formats.services.NetCDFServiceImpl;
import ome.xml.model.primitives.Color;
import ome.xml.model.primitives.PositiveFloat;

/**
 * Reader for Bitplane Imaris 5.5 (HDF) files.
 *
 * <dl><dt><b>Source code:</b></dt>
 * <dd><a href="http://trac.openmicroscopy.org.uk/ome/browser/bioformats.git/components/bio-formats/src/loci/formats/in/ImarisHDFReader.java">Trac</a>,
 * <a href="http://git.openmicroscopy.org/?p=bioformats.git;a=blob;f=components/bio-formats/src/loci/formats/in/ImarisHDFReader.java;hb=HEAD">Gitweb</a></dd></dl>
 */
public class ImarisHDFReader extends FormatReader {

  // -- Constants --

  public static final String HDF_MAGIC_STRING = "HDF";

  private static final String[] DELIMITERS = {" ", "-", "."};

  // -- Fields --

  private double pixelSizeX, pixelSizeY, pixelSizeZ;
  private double minX, minY, minZ, maxX, maxY, maxZ;
  private int seriesCount;
  private NetCDFService netcdf;

  // channel parameters
  private Vector<String> emWave, exWave, channelMin, channelMax;
  private Vector<String> gain, pinhole, channelName, microscopyMode;
  private Vector<double[]> colors;
  private int lastChannel = 0;

  // -- Constructor --

  /** Constructs a new Imaris HDF reader. */
  public ImarisHDFReader() {
    super("Bitplane Imaris 5.5 (HDF)", "ims");
    suffixSufficient = false;
    domains = new String[] {FormatTools.UNKNOWN_DOMAIN};
  }

  // -- IFormatReader API methods --

  /* @see loci.formats.IFormatReader#getOptimalTileWidth() */
  public int getOptimalTileWidth() {
    return core.get(core.size() - 1).sizeX;
  }

  /* @see loci.formats.IFormatReader#getOptimalTileHeight() */
  public int getOptimalTileHeight() {
    return core.get(core.size() - 1).sizeY;
  }

  /* @see loci.formats.IFormatReader#isThisType(RandomAccessInputStream) */
  public boolean isThisType(RandomAccessInputStream stream) throws IOException {
    final int blockLen = 8;
    if (!FormatTools.validStream(stream, blockLen, false)) return false;
    return stream.readString(blockLen).indexOf(HDF_MAGIC_STRING) >= 0;
  }

  /* @see loci.formats.IFormatReader#get8BitLookupTable() */
  public byte[][] get8BitLookupTable() {
    FormatTools.assertId(currentId, true, 1);
    if (getPixelType() != FormatTools.UINT8 || !isIndexed()) return null;

    if (lastChannel < 0 || lastChannel >= colors.size()) {
      return null;
    }

    double[] color = colors.get(lastChannel);

    byte[][] lut = new byte[3][256];
    for (int c=0; c<lut.length; c++) {
      double max = color[c] * 255;
      for (int p=0; p<lut[c].length; p++) {
        lut[c][p] = (byte) ((p / 255.0) * max);
      }
    }
    return lut;
  }

  /* @see loci.formats.IFormatReaderget16BitLookupTable() */
  public short[][] get16BitLookupTable() {
    FormatTools.assertId(currentId, true, 1);
    if (getPixelType() != FormatTools.UINT16 || !isIndexed()) return null;

    if (lastChannel < 0 || lastChannel >= colors.size()) {
      return null;
    }

    double[] color = colors.get(lastChannel);

    short[][] lut = new short[3][65536];
    for (int c=0; c<lut.length; c++) {
      double max = color[c] * 65535;
      for (int p=0; p<lut[c].length; p++) {
        lut[c][p] = (short) ((p / 65535.0) * max);
      }
    }
    return lut;
  }

  /**
   * @see loci.formats.IFormatReader#openBytes(int, byte[], int, int, int, int)
   */
  public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h)
    throws FormatException, IOException
  {
    FormatTools.checkPlaneParameters(this, no, buf.length, x, y, w, h);

    lastChannel = getZCTCoords(no)[1];

    // pixel data is stored in XYZ blocks

    Object image = getImageData(no, x, y, w, h);

    boolean big = !isLittleEndian();
    int bpp = FormatTools.getBytesPerPixel(getPixelType());
    for (int row=0; row<h; row++) {
      int rowlen = w * bpp;
      int base = row * rowlen;
      if (image instanceof byte[][]) {
        byte[][] data = (byte[][]) image;
        byte[] rowData = data[row + data.length - h];
        System.arraycopy(rowData, rowData.length - w, buf, base, w);
      }
      else if (image instanceof short[][]) {
        short[][] data = (short[][]) image;
        short[] rowData = data[row + data.length - h];
        int index = rowData.length - w;
        for (int i=0; i<w; i++) {
          DataTools.unpackBytes(rowData[i + index], buf, base + 2*i, 2, big);
        }
      }
      else if (image instanceof int[][]) {
        int[][] data = (int[][]) image;
        int[] rowData = data[row + data.length - h];
        int index = rowData.length - w;
        for (int i=0; i<w; i++) {
          DataTools.unpackBytes(rowData[i + index], buf, base + i*4, 4, big);
        }
      }
      else if (image instanceof float[][]) {
        float[][] data = (float[][]) image;
        float[] rowData = data[row + data.length - h];
        int index = rowData.length - w;
        for (int i=0; i<w; i++) {
          int v = Float.floatToIntBits(rowData[i + index]);
          DataTools.unpackBytes(v, buf, base + i*4, 4, big);
        }
      }
      else if (image instanceof double[][]) {
        double[][] data = (double[][]) image;
        double[] rowData = data[row + data.length - h];
        int index = rowData.length - w;
        for (int i=0; i<w; i++) {
          long v = Double.doubleToLongBits(rowData[i + index]);
          DataTools.unpackBytes(v, buf, base + i * 8, 8, big);
        }
      }
    }

    return buf;
  }

  /* @see loci.formats.IFormatReader#close(boolean) */
  public void close(boolean fileOnly) throws IOException {
    super.close(fileOnly);
    if (!fileOnly) {
      seriesCount = 0;
      pixelSizeX = pixelSizeY = pixelSizeZ = 0;
      minX = minY = minZ = maxX = maxY = maxZ = 0;

      if (netcdf != null) netcdf.close();
      netcdf = null;

      emWave = exWave = channelMin = channelMax = null;
      gain = pinhole = channelName = microscopyMode = null;
      colors = null;
      lastChannel = 0;
    }
  }

  // -- Internal FormatReader API methods --

  /* @see loci.formats.FormatReader#initFile(String) */
  protected void initFile(String id) throws FormatException, IOException {
    super.initFile(id);

    try {
      ServiceFactory factory = new ServiceFactory();
      netcdf = factory.getInstance(NetCDFService.class);
      netcdf.setFile(id);
    }
    catch (DependencyException e) {
      throw new MissingLibraryException(NetCDFServiceImpl.NO_NETCDF_MSG, e);
    }

    pixelSizeX = pixelSizeY = pixelSizeZ = 1;

    emWave = new Vector<String>();
    exWave = new Vector<String>();
    channelMin = new Vector<String>();
    channelMax = new Vector<String>();
    gain = new Vector<String>();
    pinhole = new Vector<String>();
    channelName = new Vector<String>();
    microscopyMode = new Vector<String>();
    colors = new Vector<double[]>();

    seriesCount = 0;

    // read all of the metadata key/value pairs

    parseAttributes();

    CoreMetadata ms0 = core.get(0);

    if (seriesCount > 1) {
      for (int i=1; i<seriesCount; i++) {
        core.add(new CoreMetadata());
      }

      ms0.resolutionCount = seriesCount;

      for (int i=1; i<seriesCount; i++) {
        CoreMetadata ms = core.get(i);
        String groupPath =
          "/DataSet/ResolutionLevel_" + i + "/TimePoint_0/Channel_0";
        ms.sizeX =
          Integer.parseInt(netcdf.getAttributeValue(groupPath + "/ImageSizeX"));
        ms.sizeY =
          Integer.parseInt(netcdf.getAttributeValue(groupPath + "/ImageSizeY"));
        ms.sizeZ =
          Integer.parseInt(netcdf.getAttributeValue(groupPath + "/ImageSizeZ"));
        ms.imageCount = ms.sizeZ * getSizeC() * getSizeT();
        ms.sizeC = getSizeC();
        ms.sizeT = getSizeT();
        ms.thumbnail = true;
      }
    }
    ms0.imageCount = getSizeZ() * getSizeC() * getSizeT();
    ms0.thumbnail = false;
    ms0.dimensionOrder = "XYZCT";

    // determine pixel type - this isn't stored in the metadata, so we need
    // to check the pixels themselves

    int type = -1;

    Object pix = getImageData(0, 0, 0, 1, 1);
    if (pix instanceof byte[][]) type = FormatTools.UINT8;
    else if (pix instanceof short[][]) type = FormatTools.UINT16;
    else if (pix instanceof int[][]) type = FormatTools.UINT32;
    else if (pix instanceof float[][]) type = FormatTools.FLOAT;
    else if (pix instanceof double[][]) type = FormatTools.DOUBLE;
    else {
      throw new FormatException("Unknown pixel type: " + pix);
    }

    for (int i=0; i<core.size(); i++) {
      CoreMetadata ms = core.get(i);
      ms.pixelType = type;
      ms.dimensionOrder = "XYZCT";
      ms.rgb = false;
      ms.thumbSizeX = 128;
      ms.thumbSizeY = 128;
      ms.orderCertain = true;
      ms.littleEndian = true;
      ms.interleaved = false;
      ms.indexed = colors.size() >= getSizeC();
    }

    MetadataStore store = makeFilterMetadata();
    MetadataTools.populatePixels(store, this);

    String imageName = new Location(getCurrentFile()).getName();
    for (int s=0; s<getSeriesCount(); s++) {
      store.setImageName(imageName + " Resolution Level " + (s + 1), s);
    }

    if (getMetadataOptions().getMetadataLevel() == MetadataLevel.MINIMUM) {
      return;
    }

    int cIndex = 0;
    for (int s=0; s<getSeriesCount(); s++) {
      setSeries(s);
      double px = pixelSizeX, py = pixelSizeY, pz = pixelSizeZ;
      if (px == 1) px = (maxX - minX) / getSizeX();
      if (py == 1) py = (maxY - minY) / getSizeY();
      if (pz == 1) pz = (maxZ - minZ) / getSizeZ();

      if (px > 0) {
        store.setPixelsPhysicalSizeX(new PositiveFloat(px), s);
      }
      else {
        LOGGER.warn("Expected positive value for PhysicalSizeX; got {}", px);
      }
      if (py > 0) {
        store.setPixelsPhysicalSizeY(new PositiveFloat(py), s);
      }
      else {
        LOGGER.warn("Expected positive value for PhysicalSizeY; got {}", py);
      }
      if (pz > 0) {
        store.setPixelsPhysicalSizeZ(new PositiveFloat(pz), s);
      }
      else {
        LOGGER.warn("Expected positive value for PhysicalSizeZ; got {}", pz);
      }

      for (int i=0; i<getSizeC(); i++, cIndex++) {
        Float gainValue = null;
        Integer pinholeValue = null, emWaveValue = null, exWaveValue;

        if (cIndex < gain.size()) {
          try {
            gainValue = new Float(gain.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }
        if (cIndex < pinhole.size()) {
          try {
            pinholeValue = new Integer(pinhole.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }
        if (cIndex < emWave.size()) {
          try {
            emWaveValue = new Integer(emWave.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }
        if (cIndex < exWave.size()) {
          try {
            exWaveValue = new Integer(exWave.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }

        // CHECK
        /*
        store.setLogicalChannelName((String) channelName.get(cIndex), s, i);
        store.setDetectorSettingsGain(gainValue, s, i);
        store.setLogicalChannelPinholeSize(pinholeValue, s, i);
        store.setLogicalChannelMode((String) microscopyMode.get(cIndex), s, i);
        store.setLogicalChannelEmWave(emWaveValue, s, i);
        store.setLogicalChannelExWave(exWaveValue, s, i);
        */

        Double minValue = null, maxValue = null;

        if (cIndex < channelMin.size()) {
          try {
            minValue = new Double(channelMin.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }
        if (cIndex < channelMax.size()) {
          try {
            maxValue = new Double(channelMax.get(cIndex));
          }
          catch (NumberFormatException e) { }
        }

        if (i < colors.size()) {
          double[] color = colors.get(i);
          Color realColor = new Color(
            (int) (color[0] * 255), (int) (color[1] * 255),
            (int) (color[2] * 255), 255);
          store.setChannelColor(realColor, s, i);
        }
      }
    }
    setSeries(0);
  }

  // -- Helper methods --

  private Object getImageData(int no, int x, int y, int width, int height)
    throws FormatException
  {
    int[] zct = getZCTCoords(no);
    String path = "/DataSet/ResolutionLevel_" + getCoreIndex() + "/TimePoint_" +
      zct[2] + "/Channel_" + zct[1] + "/Data";
    Object image = null;

    // the width and height cannot be 1, because then netCDF will give us a
    // singleton instead of an array
    if (height == 1) {
      height++;
    }
    if (width == 1) {
      width++;
    }
    if (x >= getSizeX() / 2 && y >= getSizeY() / 2) {
      width += x - (getSizeX() / 2) + 1;
      x = (getSizeX() / 2) - 1;
    }

{'fraction_non_alphanumeric': 0.13844526705446852, 'fraction_numerical': 0.010541159880133969, 'mean_word_length': 2.6144240570846073, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3314687', 'n_tokens_mistral': 9396, 'n_tokens_neox': 8958, 'n_words': 2325}