ucar.nc2.dataset.NetcdfDataset Java Examples

public boolean isMine(FeatureType wantFeatureType, NetcdfDataset ds) {
  if ((wantFeatureType != FeatureType.ANY_POINT) && (wantFeatureType != FeatureType.STATION)
      && (wantFeatureType != FeatureType.POINT))
    return false;

  FeatureType ft = FeatureDatasetFactoryManager.findFeatureType(ds);
  if (((ft != FeatureType.STATION) && (ft != FeatureType.POINT)))
    return false;

  String conv = ds.getRootGroup().findAttributeString(CDM.CONVENTIONS, null);
  if (conv == null)
    return false;

  StringTokenizer stoke = new StringTokenizer(conv, ",");
  while (stoke.hasMoreTokens()) {
    String toke = stoke.nextToken().trim();
    if (toke.equalsIgnoreCase("Unidata Observation Dataset v1.0"))
      return true;
  }

  return false;
}
 

private CFRadial2Variable(NetcdfDataset nds, Variable v0) {
  super(v0.getShortName(), v0);

  sweeps = new ArrayList<>();

  int[] shape = v0.getShape();
  int ngates = shape[v0.getRank() - 1];
  flattened = v0.findDimensionIndex("n_points") == 0;

  for (int i = 0; i < nsweeps; i++) {
    // For flattened (1D stored data) find max number of gates
    if (flattened) {
      ngates = ray_n_gates[rayStartIdx[i]];
      for (int ray = rayStartIdx[i]; ray <= rayEndIdx[i]; ++ray)
        ngates = ray_n_gates[ray] > ngates ? ray_n_gates[ray] : ngates;
    }
    sweeps.add(new CFRadial2Sweep(v0, i, ngates, rayStartIdx[i], rayEndIdx[i]));
  }
}
 

public void testNoCoords() throws IOException {
  String filename = "file:./" + TestNcMLRead.topDir + "exclude/aggExistingNoCoords.xml";
  logger.debug("{}", filename);
  NetcdfDataset ncd = null;

  try {
    ncd = NetcdfDatasets.openDataset(filename, true, null);
    Variable time = ncd.getRootGroup().findVariableLocal("time");
    Array data = time.read();
    // all missing
    // assert data.getInt(0) ==
  } finally {
    if (ncd != null)
      ncd.close();
  }
  // logger.debug("{}", ncd);
  // assert false;
}
 

/**
 * Open a ucar.nc2.ft.PointFeatureCollection, write out in CF point format.
 *
 * @param fileIn open through TypedDatasetFactory.open(FeatureType.POINT, ..)
 * @param fileOut write to this netcdf-3 file
 * @param inMemory if true, read file into memory for efficiency
 * @return true on success
 * @throws IOException on read/write error
 */
public static boolean rewritePointFeatureDataset(String fileIn, String fileOut, boolean inMemory) throws IOException {
  System.out.println("Rewrite2 .nc files from " + fileIn + " to " + fileOut + " inMemory= " + inMemory);

  long start = System.currentTimeMillis();

  // do it in memory for speed
  NetcdfFile ncfile = inMemory ? NetcdfFile.openInMemory(fileIn) : NetcdfFile.open(fileIn);
  NetcdfDataset ncd = new NetcdfDataset(ncfile);

  Formatter errlog = new Formatter();
  FeatureDataset fd = FeatureDatasetFactoryManager.wrap(FeatureType.ANY_POINT, ncd, null, errlog);
  if (fd == null)
    return false;

  if (fd instanceof FeatureDatasetPoint) {
    writePointFeatureCollection((FeatureDatasetPoint) fd, fileOut);
    fd.close();
    long took = System.currentTimeMillis() - start;
    System.out.println(" that took " + (took - start) + " msecs");
    return true;
  }

  return false;

}
 

private static boolean readAccess(Access access, DataFactory fac, PrintWriter pw) {
  Formatter log = new Formatter();
  try (NetcdfDataset ncd = fac.openDataset(access, false, null, log)) {
    if (ncd == null) {
      pw.format("  Dataset opendap fatalError=%s%n", log);
      return false;

    } else {
      pw.format("  Dataset '%s' opened as %s%n", access.getDataset().getName(), access.getService());
      return readRandom(ncd, pw);
    }
  } catch (InvalidRangeException | IOException e) {
    e.printStackTrace(pw);
    return false;
  }
}
 

static void testRead(String filename, Stat stat, boolean readData) throws IOException, InvalidRangeException {
  long bytes = 0;
  long start = System.nanoTime();
  if (show)
    System.out.printf("%n------Reading filename %s (%s)%n", filename, stat.getName());

  try (NetcdfFile ncfile = NetcdfDataset.openFile(filename, null)) {
    if (readData)
      bytes = readAllData(ncfile);
    long end = System.nanoTime();
    double took = ((double) (end - start)) / 1000 / 1000 / 1000; // secs
    double rate = 0;
    ncfile.close();
    if (stat != null && bytes > 0) {
      rate = bytes / took / 1000 / 1000;
      stat.sample(rate); // Mb/sec
    }
    if (show)
      System.out.printf(" bytes = %d took =%f secs rate=%f MB/sec%n", bytes, took, rate);
  }
}
 

public OldUnidataStationObsDataset(NetcdfDataset ds) throws IOException {
  super(ds);
  this.dataset = ds;

  String ncmlURL = null;

  String kind = ds.findAttValueIgnoreCase(null, "title", null);
  if ("METAR definition".equals(kind)) {
    Variable v = ds.findVariable("station");
    if (v != null)
      ncmlURL = CoordSysBuilder.resourcesDir + "metar2ncMetar.ncml";
    else // older form
      ncmlURL = CoordSysBuilder.resourcesDir + "metar2ncMetar2.ncml";

  } else if ("SYNOPTIC definition".equals(kind)) {
    ncmlURL = CoordSysBuilder.resourcesDir + "metar2ncSynoptic.ncml";

  } else if ("BUOY definition".equals(kind)) {
    ncmlURL = CoordSysBuilder.resourcesDir + "metar2ncBuoy.ncml";
  }

  if (ncmlURL == null)
    throw new IOException("unknown StationObsDataset type " + ds.getLocation());

  init(ncmlURL);
}
 

private ProjectionCT makeLCProjection(NetcdfDataset ds, String name) throws NoSuchElementException {
  double centralLat = findAttributeDouble(ds, "centralLat");
  double centralLon = findAttributeDouble(ds, "centralLon");
  double rotation = findAttributeDouble(ds, "rotation");

  // we have to project in order to find the origin
  LambertConformal lc = new LambertConformal(rotation, centralLon, centralLat, centralLat);
  double lat0 = findAttributeDouble(ds, "lat00");
  double lon0 = findAttributeDouble(ds, "lon00");
  ProjectionPointImpl start = (ProjectionPointImpl) lc.latLonToProj(LatLonPoint.create(lat0, lon0));
  if (debugProj)
    parseInfo.format("getLCProjection start at proj coord %s%n", start);
  startx = start.getX();
  starty = start.getY();

  return new ProjectionCT(name, "FGDC", lc);
}
 

private TableConfig makeSingle(NetcdfDataset ds, Dimension obsDim, Formatter errlog) {

    Table.Type obsTableType = Table.Type.Structure;
    TableConfig obsTable = new TableConfig(obsTableType, "single");
    obsTable.dimName = obsDim.getShortName();

    obsTable.lat = matchAxisTypeAndDimension(ds, AxisType.Lat, obsDim);
    obsTable.lon = matchAxisTypeAndDimension(ds, AxisType.Lon, obsDim);
    obsTable.elev = matchAxisTypeAndDimension(ds, AxisType.Height, obsDim);
    if (obsTable.elev == null)
      obsTable.elev = matchAxisTypeAndDimension(ds, AxisType.Pressure, obsDim);
    if (obsTable.elev == null)
      obsTable.elev = matchAxisTypeAndDimension(ds, AxisType.GeoZ, obsDim);
    obsTable.time = matchAxisTypeAndDimension(ds, AxisType.Time, obsDim);

    makeStructureInfo(obsTable, ds, null, obsDim);
    return obsTable;
  }
 

@Test
public void testUnsignedWrap() throws IOException {
  String ncml = "<?xml version='1.0' encoding='UTF-8'?>\n"
      + "<netcdf xmlns='http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2' location='"
      + TestDir.cdmLocalTestDataDir + "testWrite.nc'>\n" + "  <variable name='bvar' shape='lat' type='byte'>\n"
      + "    <attribute name='_Unsigned' value='true' />\n"
      + "    <attribute name='scale_factor' type='float' value='2.0' />\n" + "   </variable>\n" + "</netcdf>";

  try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), null);
      NetcdfFile ncd = NetcdfDataset.wrap(ncfile, NetcdfDataset.getEnhanceAll())) {

    Variable v = ncd.findVariable("bvar");
    Assert.assertNotNull(v);
    Assert.assertEquals(DataType.FLOAT, v.getDataType());

    boolean hasSigned = false;
    Array data = v.read();
    while (data.hasNext()) {
      float b = data.nextFloat();
      if (b < 0)
        hasSigned = true;
    }
    Assert.assertTrue(!hasSigned);
  }
}
 

private CoordinateSystem findReplacementCs(CoordinateSystem protoCs, String timeDim, NetcdfDataset result) {
  CoordinateSystem replace = result.findCoordinateSystem(protoCs.getName());
  if (replace != null)
    return replace;

  List<CoordinateAxis> axes = new ArrayList<>();
  for (CoordinateAxis axis : protoCs.getCoordinateAxes()) {
    CoordinateAxis ra = result.findCoordinateAxis(axis.getFullNameEscaped());
    axes.add(ra);
  }

  // coord transforms are immutable and can be shared
  CoordinateSystem cs = new CoordinateSystem(result, axes, protoCs.getCoordinateTransforms());
  result.addCoordinateSystem(cs);
  return cs;
}
 

/**
 * Get the dataset named by the path.
 * called by DatasetHandler.getNetcdfFile()
 *
 * @param matchPath remaining path from match
 * @return requested dataset
 * @throws IOException if read error
 */
public NetcdfDataset getNetcdfDataset(String matchPath) throws IOException {
  int pos = matchPath.indexOf('/');
  String type = (pos > -1) ? matchPath.substring(0, pos) : matchPath;
  String name = (pos > -1) ? matchPath.substring(pos + 1) : "";

  // this assumes that these are files. also might be remote datasets from a catalog
  if (type.equalsIgnoreCase(FILES)) {
    if (topDirectory == null)
      return null;

    String filename =
        new StringBuilder(topDirectory).append(topDirectory.endsWith("/") ? "" : "/").append(name).toString();
    DatasetUrl durl = DatasetUrl.create(null, filename);
    if (useNetcdfJavaBuilders || DatasetManager.isLocationObjectStore(durl.getTrueurl())) {
      return NetcdfDatasets.acquireDataset(null, durl, null, -1, null, null); // no enhancement
    } else {
      return NetcdfDataset.acquireDataset(null, durl, null, -1, null, null); // no enhancement
    }
  }

  GridDataset gds = getGridDataset(matchPath); // LOOK cant be right
  return (gds == null) ? null : (NetcdfDataset) gds.getNetcdfFile();
}
 

String dumpmetadata(NetcdfDataset ncfile, String datasetname) throws Exception {
  StringWriter sw = new StringWriter();
  StringBuilder args = new StringBuilder("-strict");
  if (datasetname != null) {
    args.append(" -datasetname ");
    args.append(datasetname);
  }
  // Print the meta-databuffer using these args to NcdumpW
  try {
    if (!ucar.nc2.NCdumpW.print(ncfile, args.toString(), sw, null))
      throw new Exception("NcdumpW failed");
  } catch (IOException ioe) {
    throw new Exception("NcdumpW failed", ioe);
  }
  sw.close();
  return sw.toString();
}
 

private CoordinateAxis makeLatCoordAxis(NetcdfDataset ds, int n, String xname) {
  double min = findAttributeDouble(ds, "yMin");
  double max = findAttributeDouble(ds, "yMax");
  double d = findAttributeDouble(ds, "dy");
  if (Double.isNaN(min) || Double.isNaN(max) || Double.isNaN(d))
    return null;

  CoordinateAxis v = new CoordinateAxis1D(ds, null, xname, DataType.DOUBLE, xname, CDM.LAT_UNITS, "latitude");
  v.setValues(n, min, d);
  v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString()));

  double maxCalc = min + d * n;
  parseInfo.format("Created Lat Coordinate Axis (max calc= %f should be = %f)%n", maxCalc, max);
  v.getNameAndDimensions(parseInfo, true, false);
  parseInfo.format("%n");

  return v;
}
 

private DtCoverageDataset getDtCoverageDataset(NetcdfFile ncf) throws IOException {
  DtCoverageDataset dtc;
  if (TestDir.cdmUseBuilders) {
    dtc = new DtCoverageDataset(NetcdfDatasets.enhance(ncf, NetcdfDataset.getDefaultEnhanceMode(), null));
  } else {
    dtc = new DtCoverageDataset(new NetcdfDataset(ncf), null);
  }
  return dtc;
}
 

@Override
public ucar.nc2.dataset.NetcdfDataset getNetcdfDataset(Dataset ds, GroupGC group, String filename,
    FeatureCollectionConfig config, Formatter errlog, org.slf4j.Logger logger) throws IOException {

  ucar.nc2.grib.collection.Grib2Iosp iosp = new ucar.nc2.grib.collection.Grib2Iosp(group, ds.getType());
  NetcdfFile ncfile = new NetcdfFileSubclass(iosp, null, getLocation(), null);
  return new NetcdfDataset(ncfile);
}
 

protected String matchAxisTypeAndDimension(NetcdfDataset ds, AxisType type, Dimension outer, Dimension inner) {
  Variable var = CoordSysEvaluator.findCoordByType(ds, type,
      axis -> ((axis.getRank() == 2) && outer.equals(axis.getDimension(0)) && inner.equals(axis.getDimension(1))));
  if (var == null)
    return null;
  return var.getShortName();
}
 

static void testNcd() throws IOException {
  String filename = "C:/data/test2.nc";
  NetcdfDataset ncfile = NetcdfDataset.openDataset(filename);
  measureSize("C:/data/test2.nc", ncfile, null, true);

  NetcdfDatasetInfo info = new NetcdfDatasetInfo(filename);
  measureSize("info", info, null, true);
  String pifo = info.getParseInfo();
  System.out.println("info= " + pifo);
  ncfile.close();
}
 

String ncdumpdata(NetcdfDataset ncfile, String datasetname) {
  boolean ok = false;
  String dump = "";
  StringBuilder args = new StringBuilder("-strict -vall");
  if (datasetname != null) {
    args.append(" -datasetname ");
    args.append(datasetname);
  }

  // Dump the databuffer

  ok = false;
  try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
      OutputStreamWriter outputStreamWriter = new OutputStreamWriter(byteArrayOutputStream, StandardCharsets.UTF_8)) {
    ok = ucar.nc2.NCdumpW.print(ncfile, args.toString(), outputStreamWriter, null);
    dump = byteArrayOutputStream.toString(StandardCharsets.UTF_8.name());
  } catch (IOException ioe) {
    ioe.printStackTrace();
    ok = false;
  }

  if (!ok) {
    System.err.println("NcdumpW failed");
    System.exit(1);
  }
  // return shortenFileName(sw.toString(), ncfile.getLocation());
  return dump;
}
 

public void augmentDataset(NetcdfDataset ds, CancelTask cancelTask) throws IOException {
  if (null != ds.findVariable("xCoord"))
    return; // check if its already been done - aggregating enhanced datasets.

  parseInfo.format("IFPS augmentDataset %n");

  // Figure out projection info. Assume the same for all variables
  VariableDS lonVar = (VariableDS) ds.findVariable("longitude");
  lonVar.setUnitsString(CDM.LON_UNITS);
  lonVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lon.toString()));
  VariableDS latVar = (VariableDS) ds.findVariable("latitude");
  latVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString()));
  latVar.setUnitsString(CDM.LAT_UNITS);

  projVar = latVar;
  String projName = ds.findAttValueIgnoreCase(projVar, "projectionType", null);
  if ("LAMBERT_CONFORMAL".equals(projName)) {
    Projection proj = makeLCProjection(ds);
    makeXYcoords(ds, proj, latVar, lonVar);
  }

  // figure out the time coordinate for each data variable
  // LOOK : always seperate; could try to discover if they are the same
  List<Variable> vars = ds.getVariables();
  for (Variable ncvar : vars) {
    // variables that are used but not displayable or have no data have DIM_0, also don't want history, since those
    // are just how the person edited the grids
    if ((!ncvar.getDimension(0).getShortName().equals("DIM_0")) && !ncvar.getShortName().endsWith("History")
        && (ncvar.getRank() > 2) && !ncvar.getShortName().startsWith("Tool")) {
      createTimeCoordinate(ds, ncvar);
    } else if (ncvar.getShortName().equals("Topo")) {
      // Deal with Topography variable
      ncvar.addAttribute(new Attribute(CDM.LONG_NAME, "Topography"));
      ncvar.addAttribute(new Attribute(CDM.UNITS, "ft"));
    }
  }

  ds.finish();
}
 

private void testEnhanceEquals(String ncmlLocation) throws IOException {
  System.out.println("testEnhanceEquals");
  try (NetcdfDataset ncml = NcMLReader.readNcML(ncmlLocation, null);
      NetcdfDataset ncmlEnhanced = new NetcdfDataset(ncml, true)) {

    String locref = ncml.getReferencedFile().getLocation();
    NetcdfDataset ncdrefEnhanced = NetcdfDataset.openDataset(locref, true, null);

    Assert
        .assertTrue(CompareNetcdf2.compareFiles(ncmlEnhanced, ncdrefEnhanced, new Formatter(), false, false, false));
  }
}
 

/**
 * Wrap a NetcdfDataset as a FeatureDataset.
 *
 * @param wantFeatureType open this kind of FeatureDataset; may be null, which means search all factories.
 *        If datatype is not null, only return FeatureDataset with objects of that type
 * @param ncd the NetcdfDataset to wrap as a FeatureDataset
 * @param task user may cancel
 * @param errlog place errors here, may not be null
 * @return a subclass of FeatureDataset, or null if no suitable factory was found
 * @throws java.io.IOException on io error
 */
public static FeatureDataset wrap(FeatureType wantFeatureType, NetcdfDataset ncd, ucar.nc2.util.CancelTask task,
    Formatter errlog) throws IOException {
  if (debug)
    System.out.println("wrap " + ncd.getLocation() + " want = " + wantFeatureType);

  // the case where we dont know what type it is
  if ((wantFeatureType == null) || (wantFeatureType == FeatureType.ANY)) {
    return wrapUnknown(ncd, task, errlog);
  }

  // find a Factory that claims this dataset by passing back an "analysis result" object
  Object analysis = null;
  FeatureDatasetFactory useFactory = null;
  for (Factory fac : factoryList) {
    if (!featureTypeOk(wantFeatureType, fac.featureType))
      continue;
    if (debug)
      System.out.println(" wrap try factory " + fac.factory.getClass().getName());

    analysis = fac.factory.isMine(wantFeatureType, ncd, errlog);
    if (analysis != null) {
      useFactory = fac.factory;
      break;
    }
  }

  if (null == useFactory) {
    errlog.format("**Failed to find FeatureDatasetFactory for= %s datatype=%s%n", ncd.getLocation(), wantFeatureType);
    return null;
  }

  // this call must be thread safe - done by implementation
  return useFactory.open(wantFeatureType, ncd, analysis, task, errlog);
}
 

/**
 * Constructor
 *
 * @param name name of the dataset, at bottom of DDS
 * @param ncfile create DDS from this
 */
public NcDDS(String name, NetcdfFile ncfile) {
  super((name));

  if (ncfile instanceof NetcdfDataset) {
    NetcdfDataset ncd = (NetcdfDataset) ncfile;
    if (ncd.getEnhanceMode().contains(NetcdfDataset.Enhance.CoordSystems)) {
      createFromDataset((NetcdfDataset) ncfile);
      return;
    }
  }
  createFromFile(ncfile);
}
 

@Test
public void test_ordered_sequence_units() throws IOException {
  // Make sure we return the udunits string of "count"
  String filename = "tds/ncep/WW3_Coastal_Alaska_20140804_0000.grib2";
  try (NetcdfDataset ds = NetcdfDatasets.openDataset(TestDir.cdmUnitTestDir + filename)) {
    Variable var = ds.getRootGroup().findVariableLocal("ordered_sequence_of_data");
    Attribute att = var.findAttribute("units");
    Assert.assertNotNull(att);
    Assert.assertEquals("count", att.getStringValue());
  }
}
 

public static void timeChannelWriter(String location, String fileOut) throws IOException, InvalidRangeException {
  long startTime = System.currentTimeMillis();

  NetcdfFile ncfile = NetcdfDataset.openFile(location, null);

  FileOutputStream fout = new FileOutputStream(fileOut);
  N3channelWriter.writeToChannel(ncfile, fout.getChannel());
  long diff = System.currentTimeMillis() - startTime; // msecs

  File result = new File(fileOut);
  double size = .001 * result.length(); // km
  double rate = size / diff; // Mb/sec
  System.out
      .println("N3channelWriter.writeToFile " + location + "  took " + diff + " msecs rate= " + rate + " Mb/sec");
}
 

public CoordinatesHelper build(NetcdfDataset ncd) {
  Preconditions.checkNotNull(ncd);
  if (built)
    throw new IllegalStateException("already built");
  built = true;
  return new CoordinatesHelper(this, ncd);
}
 

/**
 * search for Dimension used by axis of given by Type.
 * 
 * @param ds search in this dataset's "Best" coordinate system.
 * @param atype search for this type of CoordinateAxis. takes the first one it finds.
 * @return the found CoordinateAxis' first Dimension, or null if none or scalar
 */
public static Dimension findDimensionByType(NetcdfDataset ds, AxisType atype) {
  CoordinateAxis axis = findCoordByType(ds, atype);
  if (axis == null)
    return null;
  if (axis.isScalar())
    return null;
  return axis.getDimension(0);
}
 

private CoordinatesHelper(Builder builder, NetcdfDataset ncd) {
  List<CoordinateAxis> axes = new ArrayList<>();
  addAxes(ncd.getRootGroup(), axes);
  this.coordAxes = ImmutableList.copyOf(axes);

  coordTransforms =
      builder.coordTransforms.stream().map(ct -> ct.build(ncd)).filter(Objects::nonNull).collect(Collectors.toList());

  coordTransforms.addAll(builder.verticalCTBuilders.stream().map(ct -> ct.makeVerticalCT(ncd))
      .filter(Objects::nonNull).collect(Collectors.toList()));

  this.coordSystems = builder.coordSys.stream().map(s -> s.build(ncd, this.coordAxes, this.coordTransforms))
      .collect(Collectors.toList());
}
 

public TableConfig getConfig(FeatureType wantFeatureType, NetcdfDataset ds, Formatter errlog) throws IOException {
  String title = ds.getRootGroup().findAttributeString("version", null);
  assert title != null;
  boolean v4 = title.startsWith("Forecast Systems Lab 1.4");
  String xml = v4 ? "resources/nj22/pointConfig/FslRaob14.xml" : "resources/nj22/pointConfig/FslRaob13.xml";

  PointConfigXML reader = new PointConfigXML();
  TableConfig tc = reader.readConfigXMLfromResource(xml, wantFeatureType, ds, errlog);

  for (TableConfig inner : tc.children.get(0).children)
    makeMultidimInner(ds, tc, inner, inner.outerName, inner.innerName);
  return tc;
}
 

/**
 * Make the 2D dataset
 *
 * @param result use this empty NetcdfDataset, may be null (used by NcML)
 * @return 2D dataset
 * @throws IOException on read error
 */
GridDataset getNetcdfDataset2D(NetcdfDataset result) throws IOException {
  State localState;
  synchronized (lock) {
    localState = state;
  }
  return buildDataset2D(result, localState.proto, localState.lite);
}