<commit_before>/*
 * Copyright 1997-2008 Unidata Program Center/University Corporation for
 * Atmospheric Research, P.O. Box 3000, Boulder, CO 80307,
 * <EMAIL>.
 *
 * This library is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or (at
 * your option) any later version.
 *
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
 * General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library; if not, write to the Free Software Foundation,
 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package ucar.nc2.iosp.misc;

import ucar.unidata.io.RandomAccessFile;
import ucar.nc2.units.DateFormatter;
import ucar.nc2.iosp.AbstractIOServiceProvider;
import ucar.nc2.*;
import ucar.nc2.constants._Coordinate;
import ucar.nc2.constants.AxisType;
import ucar.nc2.util.CancelTask;
import ucar.ma2.Array;
import ucar.ma2.Section;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.DataType;

import java.io.IOException;
import java.io.EOFException;
import java.util.*;

/**
 * NMC Office Note 29
 *
 * @author caron
 * @since Feb 22, 2008
 */
public class NmcObsLegacy extends AbstractIOServiceProvider {
  private RandomAccessFile raf;
  private Map<String, List<Report>> map = new HashMap<String, List<Report>>();

  private int nobs = 0, nstations = 0;
  private Calendar cal = null;
  private DateFormatter dateFormatter = new DateFormatter();
  private Date refDate; // from the header
  private String refString; // debug

  private boolean showObs = false, showSkip = false, showOverflow = false, showData = false,
      showHeader = false, showTime = false;
  private boolean readData = false, summarizeData = false, showTimes = false;
  private boolean checkType = false, checkSort = true, checkPositions = false;

  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    byte[] h = raf.readBytes(60);

    short hour = Short.parseShort(new String(h, 0, 2));
    short minute = Short.parseShort(new String(h, 2, 2));
    short year = Short.parseShort(new String(h, 4, 2));
    short month = Short.parseShort(new String(h, 6, 2));
    short day = Short.parseShort(new String(h, 8, 2));

    return true;
  }

  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    this.raf = raf;
    init();
    make(ncfile);
  }

  private void make(NetcdfFile ncfile) {

    ncfile.addDimension(null, new Dimension("station", nstations));

    Structure top = new Structure(ncfile, null, null, "stationProfiles");
    top.setDimensions("station");
    ncfile.addVariable(null, top);

    try {
      Variable v = top.addMemberVariable(new Variable(ncfile, null, top, "stationName", DataType.STRING, ""));
      v.addAttribute(new Attribute("long_name", "name of station"));

      v = top.addMemberVariable(new Variable(ncfile, null, top, "lat", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "degrees_north"));
      v.addAttribute(new Attribute("long_name", "geographic latitude"));
      v.addAttribute(new Attribute("accuracy", "degree/100"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString()));

      v = top.addMemberVariable(new Variable(ncfile, null, top, "lon", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "degrees_east"));
      v.addAttribute(new Attribute("long_name", "geographic longitude"));
      v.addAttribute(new Attribute("accuracy", "degree/100"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lon.toString()));

      v = top.addMemberVariable(new Variable(ncfile, null, top, "elev", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "meters"));
      v.addAttribute(new Attribute("long_name", "station elevation above MSL"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Height.toString()));

      Sequence obs = new Sequence(ncfile, null, top, "report");
      top.addMemberVariable(obs);

      v = obs.addMemberVariable(new Variable(ncfile, null, top, "time", DataType.INT, ""));
      v.addAttribute(new Attribute("units", "secs since 1970-01-01 00:00"));
      v.addAttribute(new Attribute("long_name", "observation time"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Time.toString()));

      v = obs.addMemberVariable(new Variable(ncfile, null, top, "reportType", DataType.SHORT, ""));
      v.addAttribute(new Attribute("long_name", "report type from Table R.1"));

      // only for ON29
      v = obs.addMemberVariable(new Variable(ncfile, null, top, "instType", DataType.SHORT, ""));
      v.addAttribute(new Attribute("long_name", "instrument type from Table R.2"));

      v = obs.addMemberVariable(new Variable(ncfile, null, top, "reserved", DataType.BYTE, ""));
      v.setDimensionsAnonymous(new int[]{7});
      v.addAttribute(new Attribute("long_name", "reserved characters"));

      Sequence mandl = new Sequence(ncfile, null, top, "mandatoryLevels");
      obs.addMemberVariable(mandl);

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "pressure", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "mbars"));
      v.addAttribute(new Attribute("long_name", "pressure level"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Pressure.toString()));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "geopotential", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "meter"));
      v.addAttribute(new Attribute("long_name", "geopotential"));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "temperature", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "celsius"));
      v.addAttribute(new Attribute("long_name", "temperature"));
      v.addAttribute(new Attribute("accuracy", "celsius/10"));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "dewpoint", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "celsius"));
      v.addAttribute(new Attribute("long_name", "dewpoint depression"));
      v.addAttribute(new Attribute("accuracy", "celsius/10"));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "windDir", DataType.SHORT, ""));
      v.addAttribute(new Attribute("units", "degrees"));
      v.addAttribute(new Attribute("long_name", "wind direction"));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "windSpeed", DataType.SHORT, ""));
      v.addAttribute(new Attribute("units", "knots"));
      v.addAttribute(new Attribute("long_name", "wind speed"));

    } catch (InvalidRangeException e) {
      e.printStackTrace();
    }
  }

  public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {
    return null;  //To change body of implemented methods use File | Settings | File Templates.
  }

  public void close() throws IOException {
    raf.close();
  }

  private void init() throws IOException {
    int badPos = 0;
    int badType = 0;
    short firstType = -1;

    readHeader(raf);
<commit_msg>remove nestedRead for member variables<commit_after>/*
 * Copyright 1997-2008 Unidata Program Center/University Corporation for
 * Atmospheric Research, P.O. Box 3000, Boulder, CO 80307,
 * <EMAIL>.
 *
 * This library is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or (at
 * your option) any later version.
 *
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
 * General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library; if not, write to the Free Software Foundation,
 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package ucar.nc2.iosp.misc;

import ucar.unidata.io.RandomAccessFile;
import ucar.nc2.units.DateFormatter;
import ucar.nc2.iosp.AbstractIOServiceProvider;
import ucar.nc2.*;
import ucar.nc2.constants._Coordinate;
import ucar.nc2.constants.AxisType;
import ucar.nc2.util.CancelTask;
import ucar.ma2.*;

import java.io.IOException;
import java.io.EOFException;
import java.util.*;

/**
 * NMC Office Note 29
 *
 * @author caron
 * @since Feb 22, 2008
 */
public class NmcObsLegacy extends AbstractIOServiceProvider {
  private RandomAccessFile raf;
  private Map<String, List<Report>> map = new HashMap<String, List<Report>>();

  private int nobs = 0, nstations = 0;
  private Calendar cal = null;
  private DateFormatter dateFormatter = new DateFormatter();
  private Date refDate; // from the header
  private String refString; // debug

  private boolean showObs = false, showSkip = false, showOverflow = false, showData = false,
      showHeader = false, showTime = false;
  private boolean readData = false, summarizeData = false, showTimes = false;
  private boolean checkType = false, checkSort = true, checkPositions = false;

  @Override
  public boolean isValidFile(RandomAccessFile raf) throws IOException {
    byte[] h = raf.readBytes(60);

    short hour = Short.parseShort(new String(h, 0, 2));
    short minute = Short.parseShort(new String(h, 2, 2));
    short year = Short.parseShort(new String(h, 4, 2));
    short month = Short.parseShort(new String(h, 6, 2));
    short day = Short.parseShort(new String(h, 8, 2));

    return true;
  }

  @Override
  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
    this.raf = raf;
    init();
    makeNetcdfFile(ncfile);
  }

  private void makeNetcdfFile(NetcdfFile ncfile) {

    ncfile.addDimension(null, new Dimension("station", nstations));

    Structure top = new Structure(ncfile, null, null, "stationProfiles");
    top.setDimensions("station");
    ncfile.addVariable(null, top);

    try {
      Variable v = top.addMemberVariable(new Variable(ncfile, null, top, "stationName", DataType.STRING, ""));
      v.addAttribute(new Attribute("long_name", "name of station"));

      v = top.addMemberVariable(new Variable(ncfile, null, top, "lat", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "degrees_north"));
      v.addAttribute(new Attribute("long_name", "geographic latitude"));
      v.addAttribute(new Attribute("accuracy", "degree/100"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString()));

      v = top.addMemberVariable(new Variable(ncfile, null, top, "lon", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "degrees_east"));
      v.addAttribute(new Attribute("long_name", "geographic longitude"));
      v.addAttribute(new Attribute("accuracy", "degree/100"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lon.toString()));

      v = top.addMemberVariable(new Variable(ncfile, null, top, "elev", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "meters"));
      v.addAttribute(new Attribute("long_name", "station elevation above MSL"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Height.toString()));

      Sequence obs = new Sequence(ncfile, null, top, "report");
      top.addMemberVariable(obs);

      v = obs.addMemberVariable(new Variable(ncfile, null, top, "time", DataType.INT, ""));
      v.addAttribute(new Attribute("units", "secs since 1970-01-01 00:00"));
      v.addAttribute(new Attribute("long_name", "observation time"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Time.toString()));

      v = obs.addMemberVariable(new Variable(ncfile, null, top, "reportType", DataType.SHORT, ""));
      v.addAttribute(new Attribute("long_name", "report type from Table R.1"));

      // only for ON29
      v = obs.addMemberVariable(new Variable(ncfile, null, top, "instType", DataType.SHORT, ""));
      v.addAttribute(new Attribute("long_name", "instrument type from Table R.2"));

      v = obs.addMemberVariable(new Variable(ncfile, null, top, "reserved", DataType.BYTE, ""));
      v.setDimensionsAnonymous(new int[]{7});
      v.addAttribute(new Attribute("long_name", "reserved characters"));

      Sequence mandl = new Sequence(ncfile, null, top, "mandatoryLevels");
      obs.addMemberVariable(mandl);

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "pressure", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "mbars"));
      v.addAttribute(new Attribute("long_name", "pressure level"));
      v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Pressure.toString()));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "geopotential", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "meter"));
      v.addAttribute(new Attribute("long_name", "geopotential"));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "temperature", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "celsius"));
      v.addAttribute(new Attribute("long_name", "temperature"));
      v.addAttribute(new Attribute("accuracy", "celsius/10"));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "dewpoint", DataType.FLOAT, ""));
      v.addAttribute(new Attribute("units", "celsius"));
      v.addAttribute(new Attribute("long_name", "dewpoint depression"));
      v.addAttribute(new Attribute("accuracy", "celsius/10"));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "windDir", DataType.SHORT, ""));
      v.addAttribute(new Attribute("units", "degrees"));
      v.addAttribute(new Attribute("long_name", "wind direction"));

      v = mandl.addMemberVariable(new Variable(ncfile, null, top, "windSpeed", DataType.SHORT, ""));
      v.addAttribute(new Attribute("units", "knots"));
      v.addAttribute(new Attribute("long_name", "wind speed"));

    } catch (InvalidRangeException e) {
      e.printStackTrace();
    }
  }

  @Override
  public Array readData(Variable v, Section section) throws IOException, InvalidRangeException {
    if (v.getName().equals("stationProfiles"))
      throw new IllegalArgumentException();

    Structure s = (Structure) v;

    StructureMembers members = s.makeStructureMembers();
    StructureDataW sdata = new StructureDataW(members);

    for (Variable v2 : s.getVariables()) {
      Array dataArray = readData(v2, dataPos + vinfo.dataPos, v2.getShapeAsSection());
      sdata.setMemberData(v2.getShortName(), dataArray);
    }

    ArrayStructureW result = new ArrayStructureW(members, new int[] {1});
    result.setStructureData(sdata, 0);
    return result;
  }

  @Override
  public void close() throws IOException {
    raf.close();
  }

  private void init() throws IOException {
    int badPos = 0;
    int badType = 0;
    short firstType = -1;

    readHeader(raf);

{'fraction_non_alphanumeric': 0.13165119623480193, 'fraction_numerical': 0.012812132304876454, 'mean_word_length': 4.302946273830156, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5212957', 'n_tokens_mistral': 4753, 'n_tokens_neox': 4452, 'n_words': 1242}