Java Code Examples for ucar.nc2.NetcdfFile#open()

/**
 * Open a ucar.nc2.ft.PointFeatureCollection, write out in CF point format.
 *
 * @param fileIn open through TypedDatasetFactory.open(FeatureType.POINT, ..)
 * @param fileOut write to this netcdf-3 file
 * @param inMemory if true, read file into memory for efficiency
 * @return true on success
 * @throws IOException on read/write error
 */
public static boolean rewritePointFeatureDataset(String fileIn, String fileOut, boolean inMemory) throws IOException {
  System.out.println("Rewrite2 .nc files from " + fileIn + " to " + fileOut + " inMemory= " + inMemory);

  long start = System.currentTimeMillis();

  // do it in memory for speed
  NetcdfFile ncfile = inMemory ? NetcdfFile.openInMemory(fileIn) : NetcdfFile.open(fileIn);
  NetcdfDataset ncd = new NetcdfDataset(ncfile);

  Formatter errlog = new Formatter();
  FeatureDataset fd = FeatureDatasetFactoryManager.wrap(FeatureType.ANY_POINT, ncd, null, errlog);
  if (fd == null)
    return false;

  if (fd instanceof FeatureDatasetPoint) {
    writePointFeatureCollection((FeatureDatasetPoint) fd, fileOut);
    fd.close();
    long took = System.currentTimeMillis() - start;
    System.out.println(" that took " + (took - start) + " msecs");
    return true;
  }

  return false;

}
 

/**
 * Open a ucar.nc2.dt.PointObsDataset, write out in CF point format.
 *
 * @param fileIn open through TypedDatasetFactory.open(FeatureType.POINT, ..)
 * @param fileOut write to this netcdf-3 file
 * @param inMemory if true, read file into memory for efficiency
 * @return true on success
 * @throws IOException on read/write error
 */
public static boolean rewritePointObsDataset(String fileIn, String fileOut, boolean inMemory) throws IOException {
  System.out.println("Rewrite2 .nc files from " + fileIn + " to " + fileOut + " inMemory= " + inMemory);

  long start = System.currentTimeMillis();

  // do it in memory for speed
  NetcdfFile ncfile = inMemory ? NetcdfFile.openInMemory(fileIn) : NetcdfFile.open(fileIn);
  NetcdfDataset ncd = new NetcdfDataset(ncfile);

  StringBuilder errlog = new StringBuilder();
  PointObsDataset pobsDataset = (PointObsDataset) TypedDatasetFactory.open(FeatureType.POINT, ncd, null, errlog);
  if (pobsDataset == null)
    return false;

  writePointObsDataset(pobsDataset, fileOut);
  pobsDataset.close();

  long took = System.currentTimeMillis() - start;
  System.out.println(" that took " + (took - start) + " msecs");
  return true;
}
 

private void checkConcatenatedValues(File firstFile, File targetFile, File secondFile, int split) throws IOException {
	NetcdfFile firstNetcdf = null;
	NetcdfFile secondNetcdf = null;
	NetcdfFile concatenatedNetcdf = null;
	try {
		firstNetcdf = NetcdfFile.open(firstFile.toString());
		secondNetcdf = NetcdfFile.open(secondFile.toString());
		concatenatedNetcdf = NetcdfFile.open(targetFile.toString());
		double[] firstValues = (double[]) firstNetcdf.findVariable("Runoff").read().get1DJavaArray(Double.TYPE);
		double[] secondValues = (double[]) secondNetcdf.findVariable("Runoff").read().get1DJavaArray(Double.TYPE);
		double[] concatenatedValues = (double[]) concatenatedNetcdf.findVariable("Runoff").read().get1DJavaArray(Double.TYPE);
		assertEquals(firstValues.length + secondValues.length - 1, concatenatedValues.length);
		for (int i = 0; i < split; i++) {
			assertEquals(firstValues[i], concatenatedValues[i]);
		}
		for (int i = split; i < concatenatedValues.length; i++) {
			assertEquals(secondValues[i - 6], concatenatedValues[i]);
		}
	} finally {
		if (firstNetcdf != null) firstNetcdf.close();
		if (secondNetcdf != null) secondNetcdf.close();
		if (concatenatedNetcdf != null) concatenatedNetcdf.close();
	}
}
 

private void compareWithBuilder(String filename) throws Exception {
  logger.info("TestBuilders on {}%n", filename);
  SPFactory.setServiceProvider("ucar.nc2.iosp.netcdf3.N3raf");
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    SPFactory.setServiceProvider("ucar.nc2.internal.iosp.netcdf3.N3iospNew");
    try (NetcdfFile withBuilder = NetcdfFile.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      if (!compare.compare(org, withBuilder)) {
        System.out.printf("Compare %s%n%s%n", filename, f);
        fail();
      }
    }
  } finally {
    SPFactory.setServiceProvider("ucar.nc2.iosp.netcdf3.N3raf");
  }
}
 

@Test
public void compareWithBuilder()
    throws IOException, InstantiationException, IllegalAccessException, ClassNotFoundException {
  logger.info("TestBuilders on {}%n", filename);
  SPFactory.setServiceProvider("ucar.nc2.iosp.netcdf3.N3raf");
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    SPFactory.setServiceProvider("ucar.nc2.internal.iosp.netcdf3.N3iospNew");
    try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      if (!compare.compare(org, withBuilder)) {
        System.out.printf("Compare %s%n%s%n", filename, f);
        fail();
      }
    }
  } finally {
    SPFactory.setServiceProvider("ucar.nc2.iosp.netcdf3.N3raf");
  }
}
 

/**
 * read data from netcdf file and write data to table store.
 * @param meta
 * @param ncFileName
 * @throws Exception
 */
public void importFromNcFile(GridDataSetMeta meta, String ncFileName) throws Exception {
    GridDataWriter writer = tableStoreGrid.getDataWriter(meta);
    NetcdfFile ncFile = NetcdfFile.open(ncFileName);
    List<Variable> variables = ncFile.getVariables();
    for (Variable variable : variables) {
        if (meta.getVariables().contains(variable.getShortName())) {
            for (int t = 0; t < meta.gettSize(); t++) {
                for (int z = 0; z < meta.getzSize(); z++) {
                    Array array = variable.read(new int[]{t, z, 0, 0}, new int[]{1, 1, meta.getxSize(), meta.getySize()});
                    Grid2D grid2D = new Grid2D(array.getDataAsByteBuffer(), variable.getDataType(),
                            new int[] {0, 0}, new int[] {meta.getxSize(), meta.getySize()});
                    writer.writeGrid2D(variable.getShortName(), t, z, grid2D);
                }
            }
        }
    }
}
 

private void compareBuilder() {
  if (ds == null)
    return;
  String fileLocation = ds.getLocation();
  try (NetcdfFile org = NetcdfFile.open(fileLocation)) {
    try (NetcdfFile withBuilder = NetcdfFiles.open(fileLocation)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      boolean ok = compare.compare(org, withBuilder, new CoordsObjFilter());
      infoTA.setText(f.toString());
      infoTA.gotoTop();
      infoWindow.setTitle("Compare Old (file1) with Builder (file 2)");
      infoWindow.show();
    }
  } catch (Throwable ioe) {
    StringWriter sw = new StringWriter(10000);
    ioe.printStackTrace(new PrintWriter(sw));
    infoTA.setText(sw.toString());
    infoTA.gotoTop();
    infoWindow.show();
  }
}
 

public static NetcdfFile open(String filename) throws IOException {
  logger.debug("**** Open {}", filename);
  NetcdfFile ncfile;
  if (cdmUseBuilders || isLocationObjectStore(filename)) {
    ncfile = NetcdfFiles.open(filename, null);
  } else {
    ncfile = NetcdfFile.open(filename, null);
  }
  logger.debug("open {}", ncfile);

  return ncfile;
}
 

public static NetcdfFile open(String filename) {
  try {
    System.out.println("**** Open " + filename);
    NetcdfFile ncfile = NetcdfFile.open(filename);
    if (TestH5.dumpFile)
      System.out.println("open " + ncfile);
    return ncfile;

  } catch (java.io.IOException e) {
    System.out.println(" fail = " + e);
    e.printStackTrace();
    assert (false);
    return null;
  }
}
 

double copyChannelFromNetcdf(File f) throws IOException, InvalidRangeException {
  InetSocketAddress sadd = new InetSocketAddress(host, port);
  SocketChannel sc = java.nio.channels.SocketChannel.open(sadd);

  int done = 0;
  NetcdfFile ncfile = NetcdfFile.open(f.getPath());
  String want = "T,u,v,RH,Z,omega,absvor,T_fhg,u_fhg,v_fhg,RH_fhg";
  StringTokenizer stoke = new StringTokenizer(want, ",");
  while (stoke.hasMoreTokens()) {
    ParsedSectionSpec cer = ParsedSectionSpec.parseVariableSection(ncfile, stoke.nextToken());
    done += cer.v.readToByteChannel(cer.section, sc);
  }
  sc.close();
  return ((double) done) / (1000 * 1000);
}
 

double streamOutputWriter(File f) throws IOException, InvalidRangeException {
  Socket s = new Socket(host, port);
  DataOutputStream stream = new DataOutputStream(new BufferedOutputStream(s.getOutputStream(), bufferSize));

  NetcdfFile ncfile = NetcdfFile.open(f.getPath());
  N3outputStreamWriter writer = new N3outputStreamWriter(ncfile);

  writer.writeHeader(stream, -1);
  writer.writeDataAll(stream);
  stream.close();

  stream.close();

  return ((double) f.length()) / (1000 * 1000);
}
 

private void showOrg(String filename) throws IOException {

    try (NetcdfFile org = NetcdfFile.open(filename)) {
      // Variable v = org.findVariable("catchments_part_node_count");
      // Array data = v.read();
      System.out.printf("org = %s%n", org);
    }
  }
 

private void compareWithBuilder(String filename) throws IOException {
  logger.info("TestBuilders on {}%n", filename);
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      boolean ok = compare.compare(org, withBuilder, null);
      System.out.printf("%s %s%n", ok ? "OK" : "NOT OK", f);
      assertThat(ok).isTrue();
    }
  }
}
 

@Test
public void compareWithBuilder() throws IOException {
  System.out.printf("TestBuilders on %s%n", filename);
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      if (!compare.compare(org, withBuilder, null)) {
        System.out.printf("Compare %s%n%s%n", filename, f);
        fail();
      }
    }
  }
}
 

@Test
public void testForStructMetadata() throws IOException {
  System.out.printf("TestH4eosReadAll %s%n", filename);
  try (NetcdfFile ncfile = NetcdfFile.open(filename)) {
    Group root = ncfile.getRootGroup();
    Group g = root.findGroupLocal("HDFEOS INFORMATION");
    if (g == null)
      g = ncfile.getRootGroup();

    Variable dset = g.findVariableLocal("StructMetadata.0");
    assert (dset != null);
  }
}
 

private void showOrg(String filename) throws IOException {

    try (NetcdfFile org = NetcdfFile.open(filename)) {
      // Variable v = org.findVariable("catchments_part_node_count");
      // Array data = v.read();
      System.out.printf("org = %s%n", org);
    }
  }
 

private void compareWithBuilder(String filename) throws IOException {
  logger.info("TestBuilders on {}%n", filename);
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      if (!compare.compare(org, withBuilder, new DimensionsFilter())) {
        System.out.printf("Compare %s%n%s%n", filename, f);
        fail();
      }
    }
  }
}
 

@Test
public void compareWithBuilder() throws IOException {
  System.out.printf("TestBuilders on %s%n", filename);
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f);
      if (!compare.compare(org, withBuilder, new DimensionsFilter())) {
        System.out.printf("Compare %s%n%s%n", filename, f);
        fail();
      }
    }
  }
}
 

private boolean compareWithBuilder(String filename) throws IOException {
  logger.info("TestBuilders on {}%n", filename);
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      if (!compare.compare(org, withBuilder, null)) {
        System.out.printf("Compare %s%n%s%n", filename, f);
        fail();
        return false;
      }
    }
  }
  return true;
}
 

@Override
public Iterator<LCMSSpectrum> getIterator(String inputFile)
    throws ParserConfigurationException, IOException, XMLStreamException {

  final NetcdfFile netcdfFile = NetcdfFile.open(inputFile);

  // Assumption: all referenced Variables will always exist in the NetcdfFfile.

  // Assumption: these arrays will have the same length.
  final Array mzValues = netcdfFile.findVariable(MASS_VALUES).read();
  final Array intensityValues = netcdfFile.findVariable(INTENSITY_VALUES).read();
  assert(mzValues.getSize() == intensityValues.getSize());
  // Assumption: the mz/intensity values are always floats.
  assert(mzValues.getDataType() == DataType.FLOAT &&
      intensityValues.getDataType() == DataType.FLOAT);

  // Assumption: all of these variables' arrays will have the same lengths.
  final Array scanTimeArray = netcdfFile.findVariable(SCAN_TIME).read();
  final Array scanPointsStartArray = netcdfFile.findVariable(SCAN_POINTS_START).read();
  final Array scanPointsCountArray = netcdfFile.findVariable(SCAN_POINTS_COUNT).read();
  final Array totalIntensityArray = netcdfFile.findVariable(TOTAL_INTENSITY).read();
  assert(scanTimeArray.getSize() == scanPointsStartArray.getSize() &&
      scanPointsStartArray.getSize() == scanPointsCountArray.getSize() &&
      scanPointsCountArray.getSize() == totalIntensityArray.getSize());
  // Assumption: the following four columns always have these types.
  assert(scanTimeArray.getDataType() == DataType.DOUBLE &&
      scanPointsStartArray.getDataType() == DataType.INT &&
      scanPointsCountArray.getDataType() == DataType.INT &&
      totalIntensityArray.getDataType() == DataType.DOUBLE);

  final long size = scanTimeArray.getSize();

  return new Iterator<LCMSSpectrum>() {
    private int i = 0;
    @Override
    public boolean hasNext() {
      return this.i < size;
    }

    @Override
    public LCMSSpectrum next() {
      int pointCount = scanPointsCountArray.getInt(i);
      List<Pair<Double, Double>> mzIntPairs = new ArrayList<>(pointCount);

      int pointsStart = scanPointsStartArray.getInt(i);
      int pointsEnd = pointsStart + pointCount;
      for (int p = pointsStart; p < pointsEnd; p++) {
        Double mz = Float.valueOf(mzValues.getFloat(p)).doubleValue();
        Double intensity = Float.valueOf(intensityValues.getFloat(p)).doubleValue();
        mzIntPairs.add(Pair.of(mz, intensity));
      }

      LCMSSpectrum s = new LCMSSpectrum(i, scanTimeArray.getDouble(i), "s", mzIntPairs,
          null, null, null, i, totalIntensityArray.getDouble(i));

      // Don't forget to advance the counter!
      this.i++;

      // Close the file if we're done with all the array contents.
      if (i >= size) {
        try {
          netcdfFile.close();
        } catch (IOException e) {
          throw new RuntimeException(e); // TODO: can we do better?
        }
      }

      return s;
    }
  };
}