ucar.nc2.ft.FeatureDataset Java Examples

/**
 * Open a ucar.nc2.ft.PointFeatureCollection, write out in CF point format.
 *
 * @param fileIn open through TypedDatasetFactory.open(FeatureType.POINT, ..)
 * @param fileOut write to this netcdf-3 file
 * @param inMemory if true, read file into memory for efficiency
 * @return true on success
 * @throws IOException on read/write error
 */
public static boolean rewritePointFeatureDataset(String fileIn, String fileOut, boolean inMemory) throws IOException {
  System.out.println("Rewrite2 .nc files from " + fileIn + " to " + fileOut + " inMemory= " + inMemory);

  long start = System.currentTimeMillis();

  // do it in memory for speed
  NetcdfFile ncfile = inMemory ? NetcdfFile.openInMemory(fileIn) : NetcdfFile.open(fileIn);
  NetcdfDataset ncd = new NetcdfDataset(ncfile);

  Formatter errlog = new Formatter();
  FeatureDataset fd = FeatureDatasetFactoryManager.wrap(FeatureType.ANY_POINT, ncd, null, errlog);
  if (fd == null)
    return false;

  if (fd instanceof FeatureDatasetPoint) {
    writePointFeatureCollection((FeatureDatasetPoint) fd, fileOut);
    fd.close();
    long took = System.currentTimeMillis() - start;
    System.out.println(" that took " + (took - start) + " msecs");
    return true;
  }

  return false;

}
 

@Test
public void getFDForDatasetScanStationDataset() throws IOException {
  MockHttpServletRequest req = new MockHttpServletRequest();
  MockHttpServletResponse res = new MockHttpServletResponse();

  String reqPath = "testStationScan/Surface_METAR_20130826_0000.nc";
  FeatureDataset fd = TdsRequestedDataset.getPointDataset(req, res, reqPath);
  if (fd == null) {
    DataRootManager.DataRootMatch match = matcher.findDataRootMatch(reqPath);
    if (match == null) {
      Formatter f = new Formatter();
      matcher.showRoots(f);
      System.out.printf("DataRoots%n%s%n", f);
    }

  }
  assertNotNull(fd);
  assertEquals(FeatureType.STATION, fd.getFeatureType());
}
 

/**
 *
 */
public void setDataset(FeatureDataset fd) {
  if (fd == null) {
    return;
  }
  if (!(fd instanceof FeatureDatasetCoverage)) {
    return;
  }

  try {
    closeOpenFiles();
  } catch (IOException ioe) {
    logger.warn("close failed");
  }

  dsTable.setCollection((FeatureDatasetCoverage) fd);
  setSelectedItem(fd.getLocation());
}
 

public void setDataset(FeatureDataset dataset) {
  this.sds = (StationRadialDataset) dataset;

  if (debugStationDatsets)
    System.out.println("PointObsViewer open type " + dataset.getClass().getName());
  CalendarDate startDate = dataset.getCalendarDateStart();
  CalendarDate endDate = dataset.getCalendarDateEnd();
  if ((startDate != null) && (endDate != null))
    chooser.setDateRange(new DateRange(startDate.toDate(), endDate.toDate()));

  List<StationBean> stationBeans = new ArrayList<>();
  List<Station> stations = sds.getStations();
  if (stations == null)
    return;

  for (Station station : stations)
    stationBeans.add(new StationBean(station));

  stnTable.setBeans(stationBeans);
  chooser.setStations(stationBeans);
  rdTable.clear();
}
 

/**
 *
 */
public boolean setStationRadialDataset(FeatureDataset dataset) {
  if (dataset == null) {
    return false;
  }

  try {
    if (radarCollectionDataset != null) {
      radarCollectionDataset.close();
    }
  } catch (IOException ioe) {
    logger.warn("close failed");
  }

  radarCollectionDataset = dataset;
  radialViewer.setDataset(radarCollectionDataset);
  setSelectedItem(radarCollectionDataset.getLocation());
  return true;
}
 

private ThreddsDataFactory.Result openFeatureDataset(FeatureType wantFeatureType, InvAccess access,
    ucar.nc2.util.CancelTask task, Result result) throws IOException {
  result.featureType = wantFeatureType;
  result.accessUsed = access;

  // special handling for IMAGE
  if (result.featureType == FeatureType.IMAGE) {
    result.imageURL = access.getStandardUrlName();
    result.location = result.imageURL;
    return result;
  }

  if (access.getService().getServiceType() == ServiceType.CdmrFeature) {
    Optional<FeatureDataset> opt = CdmrFeatureDataset.factory(wantFeatureType, access.getStandardUrlName());
    if (opt.isPresent())
      result.featureDataset = opt.get();
    else
      result.errLog.format("%s", opt.getErrorMessage());

  } else {

    // all other datatypes
    NetcdfDataset ncd = openDataset(access, true, task, result);
    if (null != ncd) {
      result.featureDataset = FeatureDatasetFactoryManager.wrap(result.featureType, ncd, task, result.errLog);
    }
  }

  if (null == result.featureDataset)
    result.fatalError = true;
  else {
    result.location = result.featureDataset.getLocation();
    result.featureType = result.featureDataset.getFeatureType();
  }

  return result;
}
 

private void checkGridAsPointNetcdfNew(byte[] content, String varName) throws JDOMException, IOException {
  // Open the binary response in memory
  Formatter errlog = new Formatter();
  try (NetcdfFile nf = NetcdfFiles.openInMemory("checkGridAsPointNetcdf.nc", content)) {
    FeatureDataset fd = FeatureDatasetFactoryManager.wrap(FeatureType.STATION,
        NetcdfDatasets.enhance(nf, NetcdfDataset.getDefaultEnhanceMode(), null), null, errlog);
    assertNotNull(errlog.toString(), fd);
    VariableSimpleIF v = fd.getDataVariable(varName);
    assertNotNull(varName, v);
  }
}
 

private void checkGridAsPointNetcdfOld(byte[] content, String varName) throws JDOMException, IOException {
  // Open the binary response in memory
  Formatter errlog = new Formatter();
  try (NetcdfFile nf = NetcdfFile.openInMemory("checkGridAsPointNetcdf.nc", content)) {
    FeatureDataset fd = FeatureDatasetFactoryManager.wrap(FeatureType.STATION, new NetcdfDataset(nf), null, errlog);
    assertNotNull(errlog.toString(), fd);
    VariableSimpleIF v = fd.getDataVariable(varName);
    assertNotNull(varName, v);
  }
}
 

@Test
public void writeGridAsPointNetcdf() throws JDOMException, IOException {
  String endpoint = TestOnLocalServer.withHttpPath(ds + "?var=" + varName + query + "&accept=netcdf");
  File tempFile = tempFolder.newFile();
  logger.debug(" write {} to {}", endpoint, tempFile.getAbsolutePath());

  try (HTTPSession session = HTTPFactory.newSession(endpoint)) {
    HTTPMethod method = HTTPFactory.Get(session);
    int statusCode = method.execute();
    if (statusCode != 200) {
      logger.debug("statusCode = {} '{}'", statusCode, method.getResponseAsString());
      return;
    }

    IO.appendToFile(method.getResponseAsStream(), tempFile.getAbsolutePath());
  }
  logger.debug(" file length {} bytes exists={} {}", tempFile.length(), tempFile.exists(),
      tempFile.getAbsolutePath());

  // Open the result file as Station feature dataset
  Formatter errlog = new Formatter();
  try (FeatureDataset fd =
      FeatureDatasetFactoryManager.open(FeatureType.STATION, tempFile.getAbsolutePath(), null, errlog)) {
    assertNotNull(errlog.toString(), fd);
    VariableSimpleIF v = fd.getDataVariable(varName);
    assertNotNull(varName, v);
  }
}
 

@Test
public void getFDForFeatureCollectionStationDataset() throws IOException {
  MockHttpServletRequest req = new MockHttpServletRequest();
  MockHttpServletResponse res = new MockHttpServletResponse();

  FeatureDataset fd =
      TdsRequestedDataset.getPointDataset(req, res, "testStationFeatureCollection/Metar_Station_Data_fc.cdmr");
  assertNotNull(fd);
  assertEquals(FeatureType.STATION, fd.getFeatureType());
}
 

@Test
public void getFDForFeatureCollectionGridDataset() throws IOException {
  MockHttpServletRequest req = new MockHttpServletRequest();
  MockHttpServletResponse res = new MockHttpServletResponse();

  FeatureDataset fd = TdsRequestedDataset.getGridDataset(req, res, "testGFSfmrc/GFS_CONUS_80km_nc_best.ncd");
  assertNotNull(fd);
  assertEquals(FeatureType.GRID, fd.getFeatureType());
}
 

@Test
public void getFDForDatasetScanGridDataset() throws IOException {
  MockHttpServletRequest req = new MockHttpServletRequest();
  MockHttpServletResponse res = new MockHttpServletResponse();
  FeatureDataset fd = TdsRequestedDataset.getGridDataset(req, res, "testGridScan/GFS_CONUS_80km_20120227_0000.grib1");
  assertNotNull(fd);
  assertEquals(FeatureType.GRID, fd.getFeatureType());
}
 

@Test
public void getFDForSingleFileStationDataset() throws IOException {
  MockHttpServletRequest req = new MockHttpServletRequest();
  MockHttpServletResponse res = new MockHttpServletResponse();
  FeatureDataset fd = TdsRequestedDataset.getPointDataset(req, res,
      "cdmUnitTest/ncss/point_features/metar/Surface_METAR_20130826_0000.nc");
  assertNotNull(fd);
  assertEquals(FeatureType.STATION, fd.getFeatureType());
}
 

@Test
public void getFDForSingleFileGridDataset() throws IOException {
  MockHttpServletRequest req = new MockHttpServletRequest();
  MockHttpServletResponse res = new MockHttpServletResponse();
  FeatureDataset fd =
      TdsRequestedDataset.getGridDataset(req, res, "cdmUnitTest/ncss/CONUS_80km_nc/GFS_CONUS_80km_20120419_0000.nc");
  assertNotNull(fd);
  assertEquals(FeatureType.GRID, fd.getFeatureType());
}
 

public static SupportedOperation getSupportedOperation(FeatureDataset fd) {
  switch (fd.getFeatureType()) {
    case POINT:
      return SupportedOperation.POINT_REQUEST;
    case STATION:
      return SupportedOperation.STATION_REQUEST;
    default:
      throw new UnsupportedOperationException(
          String.format("'%s' format not currently supported for DSG subset writing.", fd.getFeatureType()));
  }
}
 

@Test
public void testFeatureDatasets() throws IOException {
  for (File f : getAllFilesInDirectoryStandardFilter(dir)) {
    logger.debug("Open FeatureDataset {}", f.getPath());
    try (FeatureDataset fd = FeatureDatasetFactoryManager.open(type, f.getPath(), null, new Formatter())) {
      Assert.assertNotNull(f.getPath(), fd);
      if (type == FeatureType.GRID)
        Assert.assertTrue(f.getPath(), fd.getFeatureType().isCoverageFeatureType());
      else if (type == FeatureType.POINT)
        Assert.assertTrue(f.getPath(), fd.getFeatureType().isPointFeatureType());
    }
  }
}
 

@Test
public void testOpen() throws IOException, InvalidRangeException {
  System.out.printf("FeatureDatasetFactoryManager.open %s%n", ds);
  Formatter errlog = new Formatter();
  try (FeatureDataset fd = FeatureDatasetFactoryManager.open(what, ds, null, errlog)) {
    Assert.assertNotNull(errlog.toString() + " " + ds, fd);
    if (fd.getFeatureType().isCoverageFeatureType())
      testCoverage(ds);
  }
}
 

@Override
public FeatureDataset open(FeatureType ftype, NetcdfDataset ncd, Object analysis, CancelTask task, Formatter errlog)
    throws IOException {

  // must have an index file
  File indexFile = BufrCdmIndex.calcIndexFile(ncd.getLocation());
  if (indexFile == null)
    return null;

  BufrCdmIndex index = BufrCdmIndex.readIndex(indexFile.getPath());
  return new BufrStationDataset(ncd, index);
}
 

@Override
public FeatureDataset open(FeatureType wantFeatureType, NetcdfDataset ncd, Object analyser,
    ucar.nc2.util.CancelTask task, Formatter errlog) throws IOException {
  if (analyser == null)
    analyser = TableAnalyzer.factory(null, wantFeatureType, ncd);

  return new PointDatasetStandard(wantFeatureType, (TableAnalyzer) analyser, ncd, errlog);
}
 

private DataFactory.Result openFeatureDataset(FeatureType wantFeatureType, Access access,
    ucar.nc2.util.CancelTask task, Result result) throws IOException {
  result.featureType = wantFeatureType;
  result.accessUsed = access;

  // special handling for IMAGE
  if (result.featureType == FeatureType.IMAGE) {
    result.imageURL = access.getStandardUrlName();
    result.location = result.imageURL;
    return result;
  }

  if (access.getService().getType() == ServiceType.CdmrFeature) {
    Optional<FeatureDataset> opt = CdmrFeatureDataset.factory(wantFeatureType, access.getStandardUrlName());
    if (opt.isPresent())
      result.featureDataset = opt.get();
    else
      result.errLog.format("%s", opt.getErrorMessage());

  } else {

    // all other datatypes
    NetcdfDataset ncd = openDataset(access, true, task, result);
    if (null != ncd) {
      result.featureDataset = FeatureDatasetFactoryManager.wrap(result.featureType, ncd, task, result.errLog);
    }
  }

  if (null == result.featureDataset)
    result.fatalError = true;
  else {
    result.location = result.featureDataset.getLocation();
    result.featureType = result.featureDataset.getFeatureType();
  }

  return result;
}
 

public void testOpenFileHandles() throws IOException {

    if (checkRafCache) {
      if (RandomAccessFile.getGlobalFileCache() != null) {
        RandomAccessFile.getGlobalFileCache().enable();
      } else {
        RandomAccessFile.enableDefaultGlobalFileCache();
      }
    } else {
      RandomAccessFile.getGlobalFileCache().clearCache(true);
      RandomAccessFile.shutdown();
      // RandomAccessFile.getGlobalFileCache().disable();
    }


    if (checkNetcdfFileCache) {
      // FeatureCollections still use NetcdfDataset, so work with that cache.
      NetcdfDataset.getNetcdfFileCache().enable();
    } else {
      NetcdfDataset.getNetcdfFileCache().clearCache(true);
      NetcdfDataset.getNetcdfFileCache().disable();
    }

    // open the collection
    String collection = ucar.nc2.ft.point.collection.CompositeDatasetFactory.SCHEME + TestDir.cdmUnitTestDir
        + "/ft/station/gempak/collection_with_missing_station_features/#yyMMdd#.sf$";
    FeatureDataset fds = FeatureDatasetFactoryManager.open(FeatureType.STATION, collection, null, null);
    assertThat(fds instanceof FeatureDatasetPoint);
    FeatureDatasetPoint fdp = (FeatureDatasetPoint) fds;

    // the collection dataset should have one feature collection, and it should
    // be a CompositeStationCollection
    assertThat(fdp.getPointFeatureCollectionList()).hasSize(1);
    DsgFeatureCollection dfc = fdp.getPointFeatureCollectionList().get(0);
    assertThat(dfc instanceof CompositeStationCollection);
    CompositeStationCollection csc = (CompositeStationCollection) dfc;

    // now it gets tricky. We have to tickle the PointFeatureIterator for each station trigger
    // the bug, but the iterator is hidden within the station collection because the collection
    // is made up of multiple files. Here, we use the StationHelper to get at the PointFeatureIterator
    // at the individual file level of the collection.
    StationHelper helper = csc.createStationHelper();
    List<Station> stations = helper.getStations();

    // Ensure the RandomAccessFile cache does not contain any locked files
    if (checkRafCache) {
      testRafCache();
    }

    if (checkNetcdfFileCache) {
      testNetcdfFileCache();
    }

    // Now, iterate over the stations. Each station cycles through one or more files of the collection,
    // and the bug is that the file isn't closed or released if it does not contain the given station.
    for (int station = 0; station < 2; station++) {
      // Get the PointFeatureIterator for the given station
      PointFeatureIterator pointFeatureIterator =
          ((StationTimeSeriesFeature) stations.get(station)).getPointFeatureIterator();
      // all we have to do is call .hasNext(), and if not, the underlying code will cycle through the
      // datasets of the collection but not close them
      pointFeatureIterator.hasNext();
      pointFeatureIterator.close();
    }

    if (checkRafCache) {
      testRafCache();
    }

    if (checkNetcdfFileCache) {
      testNetcdfFileCache();
    }
  }
 

public static int checkPointFeatureDataset(FeatureDataset fdataset, boolean show) throws IOException {
  long start = System.currentTimeMillis();
  int count = 0;

  CalendarDate d1 = fdataset.getCalendarDateStart();
  CalendarDate d2 = fdataset.getCalendarDateEnd();
  if ((d1 != null) && (d2 != null))
    Assert.assertTrue("calendar date min <= max", d1.isBefore(d2) || d1.equals(d2));

  List<VariableSimpleIF> dataVars = fdataset.getDataVariables();
  Assert.assertNotNull("fdataset.getDataVariables()", dataVars);
  for (VariableSimpleIF v : dataVars) {
    Assert.assertNotNull(v.getShortName(), fdataset.getDataVariable(v.getShortName()));
  }

  // FeatureDatasetPoint
  Assert.assertTrue("fdataset instanceof FeatureDatasetPoint", fdataset instanceof FeatureDatasetPoint);
  FeatureDatasetPoint fdpoint = (FeatureDatasetPoint) fdataset;

  for (DsgFeatureCollection fc : fdpoint.getPointFeatureCollectionList()) {
    checkDsgFeatureCollection(fc);

    if (fc instanceof PointFeatureCollection) {
      PointFeatureCollection pfc = (PointFeatureCollection) fc;
      count = checkPointFeatureCollection(pfc, show);
      Assert.assertEquals("PointFeatureCollection getData count = size", count, pfc.size());

    } else if (fc instanceof StationTimeSeriesFeatureCollection) {
      count = checkStationFeatureCollection((StationTimeSeriesFeatureCollection) fc);
      // testNestedPointFeatureCollection((StationTimeSeriesFeatureCollection) fc, show);

    } else if (fc instanceof StationProfileFeatureCollection) {
      count = checkStationProfileFeatureCollection((StationProfileFeatureCollection) fc, show);
      if (showStructureData)
        showStructureData((StationProfileFeatureCollection) fc);

    } else if (fc instanceof TrajectoryProfileFeatureCollection) {
      count = checkSectionFeatureCollection((TrajectoryProfileFeatureCollection) fc, show);

    } else if (fc instanceof ProfileFeatureCollection) {
      count = checkProfileFeatureCollection((ProfileFeatureCollection) fc, show);

    } else {
      count = checkOther(fc, show);
    }
    checkInfo(fc);
  }

  long took = System.currentTimeMillis() - start;
  System.out.printf(" nobs=%d took= %d msec%n", count, took);
  return count;
}
 

public void respond(HttpServletResponse res, FeatureDataset ft, String requestPathInfo, SubsetParams queryParams,
    SupportedFormat format) throws Exception {
  write();
}
 

public static FeatureDataset factory(String location, FeatureType wantFeatureType, MFileCollectionManager dcm,
    Formatter errlog) throws IOException {

  TimedCollection collection = new TimedCollection(dcm, errlog);
  if (collection.getDatasets().isEmpty()) {
    throw new FileNotFoundException("Collection is empty; spec=" + dcm);
  }

  DsgFeatureCollection first;
  TimedCollection.Dataset d = collection.getPrototype();
  try (FeatureDatasetPoint proto =
      (FeatureDatasetPoint) FeatureDatasetFactoryManager.open(wantFeatureType, d.getLocation(), null, errlog)) {
    if (proto == null) {
      throw new FileNotFoundException("Collection dataset is not a FeatureDatasetPoint; spec=" + dcm);
    }
    if (wantFeatureType == FeatureType.ANY_POINT)
      wantFeatureType = proto.getFeatureType();

    List<DsgFeatureCollection> fcList = proto.getPointFeatureCollectionList();
    if (fcList.isEmpty()) {
      throw new FileNotFoundException("FeatureCollectionList is empty; spec=" + dcm);
    }
    first = fcList.get(0);

    // LatLonRect bb = null;
    DsgFeatureCollection fc;
    switch (wantFeatureType) {
      case POINT:
        PointFeatureCollection firstPc = (PointFeatureCollection) first;
        CompositePointCollection pfc = new CompositePointCollection(dcm.getCollectionName(), firstPc.getTimeUnit(),
            firstPc.getAltUnits(), collection);
        // bb = pfc.getBoundingBox();
        fc = pfc;
        break;
      case STATION:
        PointFeatureCC firstNpc = (PointFeatureCC) first;
        CompositeStationCollection sfc = new CompositeStationCollection(dcm.getCollectionName(),
            firstNpc.getTimeUnit(), firstNpc.getAltUnits(), collection);
        // bb = sfc.getBoundingBox();
        fc = sfc;
        break;
      default:
        return null;
    }

    return new CompositePointDataset(location, wantFeatureType, fc, collection, null);
  }
}
 

public FeatureDataset open(FeatureType ftype, NetcdfDataset ncd, Object analysis, CancelTask task, Formatter errlog)
    throws IOException {
  FeatureDatasetFactory fac = (FeatureDatasetFactory) analysis;
  return fac.open(FeatureType.RADIAL, ncd, null, task, errlog);
}
 

public FeatureDataset open(FeatureType ftype, NetcdfDataset ncd, Object analysis, ucar.nc2.util.CancelTask task,
    Formatter errlog) {
  return new NsslRadialAdapter(ncd);
}
 

public FeatureDataset open(FeatureType ftype, NetcdfDataset ncd, Object analysis, ucar.nc2.util.CancelTask task,
    Formatter errlog) {
  return new Dorade2RadialAdapter(ncd);
}
 

public FeatureDataset open(FeatureType ftype, NetcdfDataset ncd, Object analysis, ucar.nc2.util.CancelTask task,
    Formatter errlog) {
  return new Nexrad2RadialAdapter(ncd);
}
 

public FeatureDataset open(FeatureType ftype, NetcdfDataset ncd, Object analysis, ucar.nc2.util.CancelTask task,
    Formatter errlog) {
  return new NidsRadialAdapter(ncd);
}
 

public FeatureDataset open(FeatureType ftype, NetcdfDataset ncd, Object analysis, ucar.nc2.util.CancelTask task,
    Formatter errlog) {
  return new CFRadialAdapter(ncd);
}