ucar.nc2.NetcdfFile Java Examples

/**
 * Get number of coordinates in this Dataset.
 * If not already set, open the file and get it from the aggregation dimension.
 *
 * @param cancelTask allow cancellation
 * @return number of coordinates in this Dataset.
 * @throws IOException if io error
 */
public int getNcoords(CancelTask cancelTask) throws IOException {
  if (ncoord <= 0) {
    try (NetcdfFile ncd = acquireFile(cancelTask)) {
      if ((cancelTask != null) && cancelTask.isCancel())
        return 0;

      Dimension d = ncd.findDimension(aggregationOuter.dimName); // long name of dimension
      if (d != null)
        ncoord = d.getLength();
      else
        throw new IllegalArgumentException("Dimension not found= " + aggregationOuter.dimName);
    }
  }
  return ncoord;
}
 

public void testDimensions(NetcdfFile ncfile) {
  Dimension latDim = ncfile.findDimension("latitude");
  assert null != latDim;
  assert latDim.getShortName().equals("latitude");
  assert latDim.getLength() == 630;
  assert !latDim.isUnlimited();

  Dimension lonDim = ncfile.findDimension("longitude");
  assert null != lonDim;
  assert lonDim.getShortName().equals("longitude");
  assert lonDim.getLength() == 630;
  assert !lonDim.isUnlimited();

  Dimension timeDim = ncfile.findDimension("time");
  assert null != timeDim;
  assert timeDim.getShortName().equals("time");
  assert timeDim.getLength() == 6;
}
 

protected Array read(Variable mainv, CancelTask cancelTask) throws IOException {
  NetcdfFile ncd = null;
  try {
    ncd = acquireFile(cancelTask);
    if ((cancelTask != null) && cancelTask.isCancel())
      return null;

    Variable v = findVariable(ncd, mainv);
    if (debugRead)
      System.out.printf("Agg.read %s from %s in %s%n", mainv.getNameAndDimensions(), v.getNameAndDimensions(),
          getLocation());
    return v.read();

  } finally {
    close(ncd);
  }
}
 

public static boolean isValidFile(NetcdfFile ds) {
  if (!ds.findAttValueIgnoreCase(null, "cdm_data_type", "").equalsIgnoreCase(FeatureType.POINT.toString())
      && !ds.findAttValueIgnoreCase(null, "cdm_datatype", "").equalsIgnoreCase(FeatureType.POINT.toString()))
    return false;

  String conv = ds.findAttValueIgnoreCase(null, "Conventions", null);
  if (conv == null)
    return false;

  StringTokenizer stoke = new StringTokenizer(conv, ",");
  while (stoke.hasMoreTokens()) {
    String toke = stoke.nextToken().trim();
    if (toke.equalsIgnoreCase("Unidata Observation Dataset v1.0"))
      return true;
  }

  return false;
}
 

/**
 * Read a section of the local Variable.
 *
 * @param mainv aggregated Variable
 * @param cancelTask let user cancel
 * @param section reletive to the local Variable
 * @return the complete Array for mainv
 * @throws IOException on I/O error
 * @throws InvalidRangeException on section error
 */
protected Array read(Variable mainv, CancelTask cancelTask, List<Range> section)
    throws IOException, InvalidRangeException {
  NetcdfFile ncd = null;
  try {
    ncd = acquireFile(cancelTask);
    if ((cancelTask != null) && cancelTask.isCancel())
      return null;

    Variable v = findVariable(ncd, mainv);
    if (debugRead) {
      Section want = new Section(section);
      System.out.printf("Agg.read(%s) %s from %s in %s%n", want, mainv.getNameAndDimensions(),
          v.getNameAndDimensions(), getLocation());
    }

    return v.read(section);

  } finally {
    close(ncd);
  }
}
 

@Test(expected = IOException.class)
public void testNcmlDatasetNoProtocolInFilenameOrNcmlAbsPath() throws IOException, InvalidRangeException {
  // if using an absolute path in the NcML file location attr of the element netcdf, then
  // you must prepend file:
  // this should fail with an IOException
  String filename = "./" + TestNcMLRead.topDir + "exclude/aggExisting6.xml";

  NetcdfFile ncfile = NetcdfDataset.openDataset(filename, true, null);
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

public void testReadSlice(NetcdfFile ncfile, int[] origin, int[] shape) throws IOException, InvalidRangeException {

    Variable v = ncfile.findVariable("T");

    Array data = v.read(origin, shape);
    assert data.getRank() == 3;
    assert data.getSize() == shape[0] * shape[1] * shape[2];
    assert data.getShape()[0] == shape[0] : data.getShape()[0] + " " + shape[0];
    assert data.getShape()[1] == shape[1];
    assert data.getShape()[2] == shape[2];
    assert data.getElementType() == double.class;

    Index tIndex = data.getIndex();
    for (int i = 0; i < shape[0]; i++)
      for (int j = 0; j < shape[1]; j++)
        for (int k = 0; k < shape[2]; k++) {
          double val = data.getDouble(tIndex.set(i, j, k));
          Assert2.assertNearlyEquals(val, 100 * (i + origin[0]) + 10 * j + k);
        }

  }
 

@Test
public void testReadSubsection() throws IOException, InvalidRangeException {
  String location = canonjoin(TestDir.cdmTestDataDir, "thredds/public/testdata/nc_test_cdf5.nc");
  try (RandomAccessFile raf = RandomAccessFile.acquire(location)) {
    // Verify that this is a netcdf-5 file
    int format = NCheader.checkFileType(raf);
    Assert.assertTrue("Fail: file format is not CDF-5", format == NCheader.NC_FORMAT_64BIT_DATA);
  }
  try (NetcdfFile jni = openJni(location)) {
    jni.setLocation(location + " (jni)");
    Array data = read(jni, "f4", "0:2");
    if (prop_visual) {
      String dump = Ncdump.printArray(data);
      logger.debug(dump);
      String testresult = dump.replace('r', ' ').replace('\n', ' ').trim();
      visual("CDF Read", testresult);
    }
    Assert.assertTrue(String.format("***Fail: data mismatch"), MAMath.nearlyEquals(data, BASELINE));
    System.err.println("***Pass");
  }
}
 

@Test
public void testStandaloneNoEnhanceDataset() throws IOException {
  try (NetcdfFile ncfile = NetcdfDatasets.openDataset(dataDir + "testStandaloneNoEnhance.ncml", false, null)) {
    Variable unvar = ncfile.findVariable("unvar");
    assertThat((Object) unvar).isNotNull();
    assertThat(unvar.getDataType()).isEqualTo(DataType.SHORT);
    assertThat(unvar.attributes().hasAttribute("_Unsigned")).isTrue();
    assertThat(unvar.attributes().findAttributeString("_Unsigned", "")).isEqualTo("true");
    assertThat(unvar.readScalarShort()).isEqualTo(-9981);

    Variable scaledvar = ncfile.findVariable("scaledvar");
    assertThat((Object) scaledvar).isNotNull();
    assertThat(scaledvar.getDataType()).isEqualTo(DataType.SHORT);
    assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isTrue();
    assertThat(scaledvar.attributes().findAttributeDouble("scale_factor", 1.0)).isEqualTo(2.0);
    assertThat(scaledvar.readScalarShort()).isEqualTo(1);
  }
}
 

public void testCoordVar(NetcdfFile ncfile) throws IOException {

    Variable lat = ncfile.findVariable("latitude");
    assert lat.getDataType() == DataType.FLOAT;
    assert lat.getDimension(0).equals(ncfile.findDimension("latitude"));

    Attribute att = lat.findAttribute("units");
    assert null != att;
    assert !att.isArray();
    assert att.isString();
    assert att.getDataType() == DataType.STRING;
    assert att.getStringValue().equals("degree_N");

    Array data = lat.read();
    assert data.getRank() == 1;
    assert data.getSize() == 630;
    assert data.getShape()[0] == 630;
    assert data.getElementType() == float.class;

    IndexIterator dataI = data.getIndexIterator();
    Assert2.assertNearlyEquals(dataI.getFloatNext(), 43.0f);
    Assert2.assertNearlyEquals(dataI.getFloatNext(), 43.01045f);
    Assert2.assertNearlyEquals(dataI.getFloatNext(), 43.020893f);
  }
 

public void testWriteNcml() throws IOException, InvalidRangeException {
  final Average fileAvg = new Average();
  final NcMLWriter writer = new NcMLWriter();

  testAllInDir(new File("C:/data/grib/"), new MClosure() {
    public void run(String filename) throws IOException, InvalidRangeException {
      if (!filename.endsWith("grib1"))
        return;
      NetcdfFile ncfile = NetcdfDataset.openFile(filename, null);
      File fileout = new File(filename + ".ncml");
      if (fileout.exists())
        fileout.delete();
      ncfile.writeNcML(new FileOutputStream(fileout), filename);
      System.out.println(" wrote ncml file  =" + fileout);
    }
  });
}
 

protected static String ncdumpmetadata(NetcdfFile ncfile, String datasetname) throws Exception {
  StringWriter sw = new StringWriter();
  StringBuilder args = new StringBuilder("-strict");
  if (datasetname != null) {
    args.append(" -datasetname ");
    args.append(datasetname);
  }
  // Print the meta-databuffer using these args to NcdumpW
  try {
    if (!ucar.nc2.NCdumpW.print(ncfile, args.toString(), sw, null))
      throw new Exception("NcdumpW failed");
  } catch (IOException ioe) {
    throw new Exception("NcdumpW failed", ioe);
  }
  sw.close();
  return sw.toString();
}
 

protected static String ncdumpdata(NetcdfFile ncfile, String datasetname) throws Exception {
  StringBuilder args = new StringBuilder("-strict -vall");
  if (datasetname != null) {
    args.append(" -datasetname ");
    args.append(datasetname);
  }
  // Dump the databuffer
  StringWriter sw = new StringWriter();
  try {
    if (!ucar.nc2.NCdumpW.print(ncfile, args.toString(), sw, null))
      throw new Exception("NCdumpW failed");
  } catch (IOException ioe) {
    ioe.printStackTrace();
    throw new Exception("NCdumpW failed", ioe);
  }
  sw.close();
  return sw.toString();
}
 

static protected String ncdumpdata(NetcdfFile ncfile, String datasetname) throws Exception {
  StringBuilder args = new StringBuilder("-strict -vall");
  if (datasetname != null) {
    args.append(" -datasetname ");
    args.append(datasetname);
  }
  // Dump the databuffer
  StringWriter sw = new StringWriter();
  try {
    Ncdump.ncdump(ncfile, args.toString(), sw, null);
  } catch (IOException ioe) {
    ioe.printStackTrace();
    throw new Exception("NCdumpW failed", ioe);
  }
  sw.close();
  return sw.toString();
}
 

private void testAggCoordVar2(NetcdfFile ncfile) throws IOException {

    Variable time = ncfile.findVariable("time");
    assert null != time;
    assert time.getShortName().equals("time");
    assert time.getRank() == 1 : time.getRank();
    assert time.getShape()[0] == 3;
    assert time.getDataType() == DataType.INT;

    assert time.getDimension(0) == ncfile.findDimension("time");

    Array data = time.read();

    assert (data instanceof ArrayInt);
    IndexIterator dataI = data.getIndexIterator();
    assert dataI.getIntNext() == 0 : dataI.getIntCurrent();
    assert dataI.getIntNext() == 1 : dataI.getIntCurrent();
    assert dataI.getIntNext() == 2 : dataI.getIntCurrent();
  }
 

@Deprecated
@Override
public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
  super.open(raf, ncfile, cancelTask);

  String location = raf.getLocation();
  if (!location.startsWith("http:")) {
    File file = new File(location);
    if (file.exists())
      lastModified = file.lastModified();
  }

  raf.order(RandomAccessFile.BIG_ENDIAN);
  header = createHeader();

  Group.Builder rootGroup = Group.builder().setName("").setNcfile(ncfile);
  header.read(raf, rootGroup, null);
  ncfile.setRootGroup(rootGroup.build());
  ncfile.finish();
}
 

static private void stnDuplicates(String filename, Set<Integer> stns, boolean wantDups) throws IOException {
  System.out.printf("%s%n", filename);
  int count = 0;
  int countDups = 0;
  NetcdfFile ncfile = open(filename);
  Sequence seq = (Sequence) ncfile.findVariable(STNS);
  try (StructureDataIterator iter = seq.getStructureIterator(-1)) {
    while (iter.hasNext()) {
      count++;
      StructureData sdata = iter.next();
      StructureMembers.Member m = sdata.findMember(STNID);
      int stnid = sdata.getScalarInt(m);
      if (stns.contains(stnid)) {
        countDups++;
        if (!wantDups)
          System.out.printf("  dup %d%n", stnid);
      } else {
        stns.add(stnid);
        if (wantDups)
          System.out.printf("  dup %d%n", stnid);
      }
    }
  }
  System.out.printf(" counts=%d dups=%d%n", count, countDups);
}
 

private void testDimensions(NetcdfFile ncfile) {
  logger.debug("ncfile = {}", ncfile);

  Dimension latDim = ncfile.findDimension("lat");
  assert null != latDim;
  assert latDim.getShortName().equals("lat");
  assert latDim.getLength() == 3;
  assert !latDim.isUnlimited();

  Dimension lonDim = ncfile.findDimension("lon");
  assert null != lonDim;
  assert lonDim.getShortName().equals("lon");
  assert lonDim.getLength() == 4;
  assert !lonDim.isUnlimited();

  Dimension timeDim = ncfile.findDimension("time");
  assert null != timeDim;
  assert timeDim.getShortName().equals("time");
  assert timeDim.getLength() == 3 : timeDim.getLength();
}
 

private NetcdfFile proto2nc(NcStreamProto.Header proto, NetcdfFile ncfile) {
  if (ncfile == null)
    ncfile = new NetcdfFileSubclass(); // not used i think
  ncfile.setLocation(proto.getLocation());
  if (!proto.getId().isEmpty())
    ncfile.setId(proto.getId());
  if (!proto.getTitle().isEmpty())
    ncfile.setTitle(proto.getTitle());

  NcStreamProto.Group root = proto.getRoot();
  Group.Builder rootBuilder = Group.builder().setNcfile(ncfile).setName("");
  NcStream.readGroup(root, rootBuilder);
  ncfile.setRootGroup(rootBuilder.build());
  ncfile.finish();
  return ncfile;
}
 

public NetcdfFile acquireFile(CancelTask cancelTask) throws IOException {
  if (debugOpenFile)
    System.out.println(" try to acquire " + cacheLocation);
  long start = System.currentTimeMillis();

  if (durl == null)
    durl = DatasetUrl.findDatasetUrl(cacheLocation); // cache the ServiceType so we dont have to keep figuring it
  // out
  NetcdfFile ncfile = NetcdfDatasets.acquireFile(reader, null, durl, -1, cancelTask, spiObject);
  if (ncmlElem == null && (enhance.isEmpty()))
    return ncfile;

  NetcdfDataset.Builder builder = NcMLReaderNew.mergeNcML(ncfile, ncmlElem); // create new dataset
  builder.setEnhanceMode(enhance);

  if (debugOpenFile)
    System.out.println(" acquire (enhance) " + cacheLocation + " took " + (System.currentTimeMillis() - start));
  return builder.build();
}
 

protected Array read(AggDatasetOuter dset, NetcdfFile ncfile) throws IOException {
  invocation++;

  Array data = getData(dset.getId());
  if (data != null)
    return data;
  if (type == Type.joinNew)
    return null;

  Variable v = ncfile.findVariable(varName);
  data = v.read();
  putData(dset.getId(), data);
  if (debugCache)
    System.out.println("caching " + varName + " complete data");
  return data;
}
 

/**
 * Main program.
 * <p>
 * <strong>ucar.nc2.NCdumpW filename [-cdl | -ncml] [-c | -vall] [-v varName1;varName2;..] [-v varName(0:1,:,12)]
 * </strong>
 * <p>
 * where:
 * <ul>
 * <li>filename : path of any CDM readable file
 * <li>cdl or ncml: output format is CDL or NcML
 * <li>-vall : dump all variable data
 * <li>-c : dump coordinate variable data
 * <li>-v varName1;varName2; : dump specified variable(s)
 * <li>-v varName(0:1,:,12) : dump specified variable section
 * </ul>
 * Default is to dump the header info only.
 *
 * @param args arguments
 */
public static void main(String[] args) {
  if (args.length == 0) {
    System.out.println(usage);
    return;
  }

  // pull out the filename from the command
  String filename = args[0];
  try (Writer writer = new BufferedWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8));
      NetcdfFile nc = NetcdfDatasets.openFile(filename, null)) {
    // the rest of the command
    StringBuilder command = new StringBuilder();
    for (int i = 1; i < args.length; i++) {
      command.append(args[i]);
      command.append(" ");
    }
    ncdump(nc, command.toString(), writer, null);
  } catch (IOException ioe) {
    ioe.printStackTrace();
  }
}
 

@Test(expected = IOException.class)
public void testNcmlDatasetNoProtocolInNcmlAbsPath() throws IOException, InvalidRangeException {
  // if using an absolute path in the NcML file location attr of the element netcdf, then
  // you must prepend file:
  // this should fail with an IOException
  String filename = "file:./" + TestNcMLRead.topDir + "exclude/aggExisting6.xml";

  NetcdfFile ncfile = NetcdfDatasets.openDataset(filename, true, null);
  logger.debug(" TestNcmlAggExisting.open {}", filename);

  testDimensions(ncfile);
  testCoordVar(ncfile);
  testAggCoordVar(ncfile);
  testReadData(ncfile);
  testReadSlice(ncfile);
  ncfile.close();
}
 

private Array read(TimeInventory.Instance timeInstance, String fullNameEsc, List<Range> innerSection,
    HashMap<String, NetcdfDataset> openFilesRead) throws IOException, InvalidRangeException {
  NetcdfFile ncfile = open(timeInstance.getDatasetLocation(), openFilesRead);
  if (ncfile == null)
    return null; // file might be deleted ??

  Variable v = ncfile.findVariable(fullNameEsc);
  if (v == null)
    return null; // v could be missing, return missing data i think

  // assume time is first dimension LOOK: out of-order; ensemble; section different ??
  Range timeRange = new Range(timeInstance.getDatasetIndex(), timeInstance.getDatasetIndex());
  Section.Builder sb = Section.builder().appendRanges(innerSection);
  sb.insertRange(0, timeRange);
  return v.read(sb.build());
}
 

@Override
public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
  super.open(raf, ncfile, cancelTask);
  Group.Builder rootGroup = Group.builder().setName("").setNcfile(ncfile);
  header = new H5headerNew(raf, rootGroup, this);
  header.read(null);
  ncfile.setRootGroup(rootGroup.build());

  // check if its an HDF5-EOS file
  if (useHdfEos) {
    rootGroup.findGroupLocal(HdfEos.HDF5_GROUP).ifPresent(eosGroup -> {
      try {
        isEos = HdfEos.amendFromODL(raf.getLocation(), header, eosGroup);
      } catch (IOException e) {
        log.warn(" HdfEos.amendFromODL failed");
      }
    });
  }

  ncfile.finish();
}
 

@Override
public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
  super.open(raf, ncfile, cancelTask);

  scanner = new MessageScanner(raf);
  protoMessage = scanner.getFirstDataMessage();
  if (protoMessage == null)
    throw new IOException("No data messages in the file= " + ncfile.getLocation());
  if (!protoMessage.isTablesComplete())
    throw new IllegalStateException("BUFR file has incomplete tables");

  // just get the fields
  config = BufrConfig.openFromMessage(raf, protoMessage, iospParam);

  // this fills the netcdf object
  Construct2 construct = new Construct2(protoMessage, config, ncfile);
  obsStructure = construct.getObsStructure();
  ncfile.finish();
  isSingle = false;
}
 

public <T extends NetcdfFile> void testFullReadGoes16S3(T nc) throws IOException {
  Dimension x = nc.findDimension("x");
  Dimension y = nc.findDimension("y");
  assertThat(x).isNotNull();
  assertThat(y).isNotNull();

  if (nc instanceof NetcdfDataset) {
    String partialConventionValue = "CF-1.";
    // read conventions string
    String conventions = nc.getRootGroup().attributes().findAttributeString(CF.CONVENTIONS, "");

    // check that the file was read the CF convention builder
    assertThat(conventions).startsWith(partialConventionValue);
    assertThat(((NetcdfDataset) nc).getConventionUsed()).startsWith(partialConventionValue);
  }

  testG16RadVar(nc);
}
 

private void writeExtraVariables() throws IOException {
  if (extra == null)
    return;

  for (Variable v : extra) {
    NetcdfFile ncfile = writer.getOutputFile();
    Variable mv = ncfile.findVariable(v.getFullName());
    if (mv == null)
      continue; // may be removed
    try {
      writer.write(mv, v.read());
    } catch (InvalidRangeException e) {
      e.printStackTrace(); // cant happen haha
    }
  }
}
 

@RequestMapping(value = "/**", method = RequestMethod.GET, params = "req=capabilities")
public ModelAndView handleCapabilitiesRequest(HttpServletRequest request, HttpServletResponse response)
    throws IOException {

  if (!allowedServices.isAllowed(StandardService.cdmRemote))
    throw new ServiceNotAllowed(StandardService.cdmRemote.toString());

  String datasetPath = TdsPathUtils.extractPath(request, "/cdmremote");
  String absPath = getAbsolutePath(request);

  try (NetcdfFile ncfile = TdsRequestedDataset.getNetcdfFile(request, response, datasetPath)) {
    if (ncfile == null)
      return null;

    Element rootElem = new Element("cdmRemoteCapabilities");
    Document doc = new Document(rootElem);
    rootElem.setAttribute("location", absPath);

    Element elem = new Element("featureDataset");
    FeatureType ftFromMetadata = FeatureDatasetFactoryManager.findFeatureType(ncfile); // LOOK BAD - must figure out
                                                                                       // what is the featureType and
                                                                                       // save it
    if (ftFromMetadata != null)
      elem.setAttribute("type", ftFromMetadata.toString());
    elem.setAttribute("url", absPath);
    rootElem.addContent(elem);

    return new ModelAndView("threddsXmlView", "Document", doc);
  }
}
 

public NetcdfFile open(DatasetUrl cacheName, int buffer_size, CancelTask cancelTask, Object spiObject)
    throws IOException {
  if (debugAggDetail) {
    System.out.println(" NcmlElementReader open nested dataset " + cacheName);
  }
  NetcdfFile.Builder result = readNcML(ncmlLocation, location, netcdfElem, cancelTask);
  result.setLocation(ncmlLocation + "#" + location);
  return result.build();
}