ucar.unidata.io.RandomAccessFile Java Examples

/**
 * Open an existing file (read only), specifying which IOSP is to be used.
 *
 * @param location location of file
 * @param iospClassName fully qualified class name of the IOSP class to handle this file
 * @param bufferSize RandomAccessFile buffer size, if <= 0, use default size
 * @param cancelTask allow task to be cancelled; may be null.
 * @param iospMessage special iosp tweaking (sent before open is called), may be null
 * @return NetcdfFile object, or null if cant find IOServiceProver
 * @throws IOException if read error
 * @throws ClassNotFoundException cannat find iospClassName in thye class path
 * @throws InstantiationException if class cannot be instantiated
 * @throws IllegalAccessException if class is not accessible
 * @deprecated use NetcdfFiles.open
 */
@Deprecated
public static NetcdfFile open(String location, String iospClassName, int bufferSize, CancelTask cancelTask,
    Object iospMessage) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException {

  Class iospClass = NetcdfFile.class.getClassLoader().loadClass(iospClassName);
  IOServiceProvider spi = (IOServiceProvider) iospClass.newInstance(); // fail fast

  // send iospMessage before iosp is opened
  if (iospMessage != null)
    spi.sendIospMessage(iospMessage);

  if (bufferSize <= 0)
    bufferSize = default_buffersize;

  RandomAccessFile raf = RandomAccessFile.acquire(canonicalizeUriString(location), bufferSize);

  NetcdfFile result = new NetcdfFile(spi, raf, location, cancelTask);

  // send after iosp is opened
  if (iospMessage != null)
    spi.sendIospMessage(iospMessage);

  return result;
}
 

@Override
protected void show(RandomAccessFile rafData, long pos) throws IOException {
  Grib2Record gr = Grib2RecordScanner.findRecordByDrspos(rafData, pos);
  if (gr != null) {
    Formatter f = new Formatter();
    f.format("File=%s%n", rafData.getLocation());
    f.format("  Parameter=%s%n", cust.getVariableName(gr));
    f.format("  ReferenceDate=%s%n", gr.getReferenceDate());
    f.format("  ForecastDate=%s%n", cust.getForecastDate(gr));
    TimeCoordIntvDateValue tinv = cust.getForecastTimeInterval(gr);
    if (tinv != null)
      f.format("  TimeInterval=%s%n", tinv);
    f.format("  ");
    gr.getPDS().show(f);
    System.out.printf("%nGrib2Record.readData at drsPos %d = %s%n", pos, f.toString());
  }
}
 

private void doDrsSummary(ucar.nc2.grib.grib2.Grib2Record gr, RandomAccessFile raf, boolean extra, Counters counters)
    throws IOException {
  Grib2SectionDataRepresentation drss = gr.getDataRepresentationSection();
  int template = drss.getDataTemplate();
  counters.count("DRS_template", template);

  // Grib2SectionBitMap bms = gr.getBitmapSection();
  counters.count("BMS indicator", gr.repeat);

  GribData.Info info = gr.getBinaryDataInfo(raf);
  counters.count("Number_of_Bits", info.numberOfBits);

  if (extra && template == 40) { // expensive
    Grib2Drs.Type40 drs40 = gr.readDataTest(raf);
    if (drs40 != null) {
      if (drs40.hasSignedProblem())
        counters.count("DRS template 40 signed problem", 1);
      else
        counters.count("DRS template 40 signed problem", 0);
    }
  }
}
 

/**
 * Read a zero terminated String at current position; advance file to a multiple of 8.
 *
 * @param raf from this file
 * @return String (dont include zero terminator)
 * @throws java.io.IOException on io error
 */
private String readString8(RandomAccessFile raf) throws IOException {
  long filePos = raf.getFilePointer();

  int count = 0;
  while (raf.readByte() != 0)
    count++;

  raf.seek(filePos);
  byte[] s = new byte[count];
  raf.readFully(s);

  // skip to 8 byte boundary, note zero byte is skipped
  count++;
  count += padding(count, 8);
  raf.seek(filePos + count);

  return new String(s, StandardCharsets.UTF_8); // all Strings are UTF-8 unicode
}
 

public void execute(String filename) throws IOException {
  try (RandomAccessFile mraf = new RandomAccessFile(filename, "r")) {
    MessageScanner scanner = new MessageScanner(mraf);

    while (scanner.hasNext()) {
      Message m = scanner.next();
      if (m == null)
        continue;
      total_msgs++;
      if (m.getNumberDatasets() == 0)
        continue;

      // LOOK check on tables complete etc ??

      m.setRawBytes(scanner.getMessageBytes(m));

      // decide what to do with the message
      dispatcher.dispatch(m);
    }

    dispatcher.resetBufrTableMessages();
  }
}
 

@Override
public boolean readMFiles(Path indexFile, List<MFile> result) throws IOException {
  logger.debug("GribCdmIndex.readMFiles {}", indexFile);
  try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile.toString())) {
    // GribCollectionType type = getType(raf);
    // if (type == GribCollectionType.GRIB1 || type == GribCollectionType.GRIB2) {
    if (openIndex(raf, logger)) {
      File protoDir = new File(gribCollectionIndex.getTopDir());
      int n = gribCollectionIndex.getMfilesCount();
      for (int i = 0; i < n; i++) {
        GribCollectionProto.MFile mfilep = gribCollectionIndex.getMfiles(i);
        result.add(new GcMFile(protoDir, mfilep.getFilename(), mfilep.getLastModified(), mfilep.getLength(),
            mfilep.getIndex()));
      }
    }
    return true;
    // }
  }
  // return false;
}
 

@Override
public void create(String filename, ucar.nc2.NetcdfFile ncfile, int extra, long preallocateSize, boolean largeFile)
    throws IOException {
  this.ncfile = ncfile;

  // finish any structures
  ncfile.finish();

  raf = new ucar.unidata.io.RandomAccessFile(filename, "rw");
  raf.order(RandomAccessFile.BIG_ENDIAN);

  if (preallocateSize > 0) {
    java.io.RandomAccessFile myRaf = raf.getRandomAccessFile();
    myRaf.setLength(preallocateSize);
  }

  N3headerWriter headerw = new N3headerWriter(this, raf, ncfile);
  headerw.create(extra, largeFile, null);
  this.header = headerw;

  if (fill)
    fillNonRecordVariables();
  // else
  // raf.setMinLength(recStart); // make sure file length is long enough, even if not written to.
}
 

public void open(RandomAccessFile raf, NetcdfFile ncfile, Message single) throws IOException {
  this.raf = raf;

  protoMessage = single;
  protoMessage.getRootDataDescriptor(); // construct the data descriptors, check for complete tables
  if (!protoMessage.isTablesComplete())
    throw new IllegalStateException("BUFR file has incomplete tables");

  BufrConfig config = BufrConfig.openFromMessage(raf, protoMessage, null);

  // this fills the netcdf object
  Construct2 construct = new Construct2(protoMessage, config, ncfile);
  obsStructure = construct.getObsStructure();
  isSingle = true;

  ncfile.finish();
  this.ncfile = ncfile;
}
 

public void readData1(RandomAccessFile raf, int datatype, Range gateRange, IndexIterator ii) throws IOException {
  long offset = message_offset;
  offset += MESSAGE_HEADER_SIZE; // offset is from "start of digital radar data message header"
  offset += getDataOffset(datatype);
  raf.seek(offset);
  if (logger.isDebugEnabled()) {
    logger.debug("  read recno " + recno + " at offset " + offset + " count= " + getGateCount(datatype));
    logger.debug(
        "   offset: reflect= " + reflect_offset + " velocity= " + velocity_offset + " spWidth= " + spectWidth_offset);
  }

  int dataCount = getGateCount(datatype);
  short[] data = new short[dataCount];
  raf.readShort(data, 0, dataCount);

  for (int idx : gateRange) {
    if (idx >= dataCount)
      ii.setShortNext((short) -32768);
    else
      ii.setShortNext(data[idx]);
  }

}
 

/**
 * Create a grib collection / partition collection from an existing ncx2 file.
 * PartionCollection.partition.getGribCollection().
 *
 * @param indexRaf the ncx2 file already open
 * @param config special configuration
 * @return the resulting GribCollection, or null on failure
 * @throws IOException on io error
 */
@Nullable
public static GribCollectionImmutable openGribCollectionFromIndexFile(RandomAccessFile indexRaf,
    FeatureCollectionConfig config, org.slf4j.Logger logger) throws IOException {

  GribCollectionType type = getType(indexRaf);

  String location = indexRaf.getLocation();
  File f = new File(location);
  int pos = f.getName().lastIndexOf(".");
  String name = (pos > 0) ? f.getName().substring(0, pos) : f.getName(); // remove ".ncx2"

  switch (type) {
    case Partition1:
      return Grib1PartitionBuilderFromIndex.createTimePartitionFromIndex(name, indexRaf, config, logger);
    case GRIB1:
      return Grib1CollectionBuilderFromIndex.readFromIndex(name, indexRaf, config, logger);
    case Partition2:
      return Grib2PartitionBuilderFromIndex.createTimePartitionFromIndex(name, indexRaf, config, logger);
    case GRIB2:
      return Grib2CollectionBuilderFromIndex.readFromIndex(name, indexRaf, config, logger);
  }

  return null;
}
 

static private void readDataRegexp(String filename) throws IOException {
  int balony = 0;
  long start = System.currentTimeMillis();
  System.out.printf("regexp %s%n", filename);
  try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) {
    String line;
    while (true) {
      line = raf.readLine();
      if (line == null)
        break;
      if (line.startsWith("#"))
        continue;
      if (line.trim().length() == 0)
        continue;
      balony += parseLine(line);
    }
  }

  long took = System.currentTimeMillis() - start;
  System.out.printf("DONE %d == %d msecs%n", balony, took);
}
 

/**
 * Read the grid
 *
 * @param gr the grid record
 * @return the data
 */
public float[] readGrid(McIDASGridRecord gr) throws IOException {

  float[] data;
  // try {
  int te = (gr.getOffsetToHeader() + 64) * 4;
  int rows = gr.getRows();
  int cols = gr.getColumns();
  rf.seek(te);

  float scale = (float) gr.getParamScale();

  data = new float[rows * cols];
  rf.order(needToSwap ? RandomAccessFile.LITTLE_ENDIAN : RandomAccessFile.BIG_ENDIAN);
  // int n = 0;
  // store such that 0,0 is in lower left corner...
  for (int nc = 0; nc < cols; nc++) {
    for (int nr = 0; nr < rows; nr++) {
      int temp = rf.readInt(); // check for missing value
      data[(rows - nr - 1) * cols + nc] = (temp == McIDASUtil.MCMISSING) ? Float.NaN : ((float) temp) / scale;
    }
  }
  rf.order(RandomAccessFile.BIG_ENDIAN);
  return data;
}
 

@Override
public boolean isValidFile(RandomAccessFile raf) throws IOException {
  if (raf instanceof HTTPRandomAccessFile) { // only do remote if memory resident
    if (raf.length() > raf.getBufferSize())
      return false;

  } else { // wont accept remote index
    GribCdmIndex.GribCollectionType type = GribCdmIndex.getType(raf);
    if (type == GribCdmIndex.GribCollectionType.GRIB2)
      return true;
    if (type == GribCdmIndex.GribCollectionType.Partition2)
      return true;
  }

  // check for GRIB2 data file
  return Grib2RecordScanner.isValidFile(raf);
}
 

/**
 * Read data from this ray.
 *
 * @param raf read from this file
 * @param abbrev which data type we want
 * @param gateRange handles the possible subset of data to return
 * @param ii put the data here
 */
public void readData(RandomAccessFile raf, String abbrev, Range gateRange, IndexIterator ii) throws IOException {
  long offset = rayOffset;
  offset += (getDataOffset(abbrev) * 2 - 2);
  raf.seek(offset);
  byte[] b2 = new byte[2];
  int dataCount = getGateCount(abbrev);
  byte[] data = new byte[dataCount * 2];
  raf.readFully(data);

  for (int gateIdx : gateRange) {
    if (gateIdx >= dataCount)
      ii.setShortNext(uf_header2.missing);
    else {
      b2[0] = data[gateIdx * 2];
      b2[1] = data[gateIdx * 2 + 1];
      short value = getShort(b2, 0);

      ii.setShortNext(value);
    }
  }

}
 

void setHdf4File(RandomAccessFile raf) throws IOException {
  closeOpenFiles();

  this.location = raf.getLocation();
  List<TagBean> beanList = new ArrayList<>();

  iosp = new H4iosp();
  NetcdfFile ncfile = new NetcdfFileSubclass(iosp, location);

  try {
    iosp.open(raf, ncfile, null);
  } catch (Throwable t) {
    StringWriter sw = new StringWriter(20000);
    t.printStackTrace(new PrintWriter(sw));
    dumpTA.setText(sw.toString());
  }

  header = (H4header) iosp.sendIospMessage("header");
  for (H4header.Tag tag : header.getTags()) {
    beanList.add(new TagBean(tag));
  }

  tagTable.setBeans(beanList);
}
 

/**
 * Open the service provider for reading.
 *
 * @param raf file to read from
 * @param ncfile netCDF file we are writing to (memory)
 * @param cancelTask task for cancelling
 * @throws IOException problem reading file
 */
public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException {
  // debugProj = true;
  super.open(raf, ncfile, cancelTask);
  long start = System.currentTimeMillis();
  if (mcGridReader == null) {
    mcGridReader = new McIDASGridReader();
  }
  mcGridReader.init(raf);
  GridIndex index = mcGridReader.getGridIndex();
  open(index, cancelTask);
  if (debugOpen) {
    System.out.println(
        " GridServiceProvider.open " + ncfile.getLocation() + " took " + (System.currentTimeMillis() - start));
  }
}
 

private StructureDataRegexp.Vinfo setVinfo(RandomAccessFile raff, NetcdfFile ncfile, Pattern p, String seqName) {
  Sequence seq = (Sequence) ncfile.findVariable(seqName);
  StructureMembers sm = seq.makeStructureMembers();
  StructureDataRegexp.Vinfo result = new StructureDataRegexp.Vinfo(raff, sm, p);
  seq.setSPobject(result);

  int fldno = 1;
  for (StructureMembers.Member m : sm.getMembers()) {
    StructureDataRegexp.VinfoField vf = new StructureDataRegexp.VinfoField(fldno++);
    Variable v = seq.findVariable(m.getName());
    Attribute att = v.findAttribute("iosp_scale");
    if (att != null) {
      vf.hasScale = true;
      vf.scale = att.getNumericValue().floatValue();
      // v.remove(att);
    }
    m.setDataObject(vf);
  }

  return result;
}
 

public void setHdf5File(RandomAccessFile raf) throws IOException {
  closeOpenFiles();

  this.location = raf.getLocation();
  List<VarBean> beanList = new ArrayList<>();

  iosp = new H5iosp();
  NetcdfFile ncfile = new NetcdfFileSubclass(iosp, location);
  try {
    iosp.open(raf, ncfile, null);
  } catch (Throwable t) {
    StringWriter sw = new StringWriter(20000);
    t.printStackTrace(new PrintWriter(sw));
    infoTA.setText(sw.toString());
  }

  for (Variable v : ncfile.getVariables()) {
    beanList.add(new VarBean(v));
  }

  objectTable.setBeans(beanList);
}
 

public GribData.Info getBinaryDataInfo(RandomAccessFile raf) throws IOException {
  GribData.Info info = dataSection.getBinaryDataInfo(raf);
  info.decimalScaleFactor = pdss.getDecimalScale();
  info.bitmapLength = (bitmap == null) ? 0 : bitmap.getLength(raf);
  info.nPoints = getGDS().getNpts();
  info.msgLength = is.getMessageLength();

  if (bitmap == null) {
    info.ndataPoints = info.nPoints;
  } else {
    byte[] bm = bitmap.getBitmap(raf);
    if (bm == null) {
      info.ndataPoints = info.nPoints;
    } else { // have to count the bits to see how many data values are stored
      info.ndataPoints = GribNumbers.countBits(bm);
    }
  }

  return info;
}
 

public boolean isValidFile(RandomAccessFile raf) throws IOException {
  raf.seek(0);
  int n = MAGIC.length();
  byte[] b = new byte[n];
  raf.read(b);
  String got = new String(b);
  return got.equals(MAGIC);
}
 

/**
 * Read Grib2SectionLocalUse from raf.
 *
 * @param raf RandomAccessFile, with pointer at start od section
 */
public Grib2SectionLocalUse(RandomAccessFile raf) throws IOException {

  // octets 1-4 (Length of GDS)
  int length = GribNumbers.int4(raf);
  int section = raf.read(); // This is section 2

  if (section != 2) { // no local use section
    raf.skipBytes(-5);
    rawData = null;
  } else {
    rawData = new byte[length - 5];
    raf.readFully(rawData);
  }
}
 

/**
 * Read a zero terminated String. Leave file positioned after zero terminator byte.
 *
 * @param raf from this file
 * @return String (dont include zero terminator)
 * @throws java.io.IOException on io error
 */
private String readString(RandomAccessFile raf) throws IOException {
  long filePos = raf.getFilePointer();

  int count = 0;
  while (raf.readByte() != 0)
    count++;

  raf.seek(filePos);
  String result = raf.readString(count);
  raf.readByte(); // skip the zero byte! nn
  return result;
}
 

/**
 * Read StationName strings
 */
public java.util.Map<String, String> readStnNames(ucar.unidata.io.RandomAccessFile raf) {
  java.util.Map<String, String> hdrNames = new java.util.HashMap<>();
  try {
    raf.seek(6288);
    String stnName = raf.readString(16);
    raf.seek(6306);
    String stnName_util = raf.readString(16);
    hdrNames.put("StationName", stnName.trim());
    hdrNames.put("StationName_SetupUtility", stnName_util.trim());
  } catch (Exception e) {
    logger.warn("readStnNames", e);
  }
  return hdrNames;
}
 

@Nullable
Object getFirstGrib1Bean(MFile mf, RandomAccessFile raf) throws IOException {
  Grib1Record first = null;
  Grib1RecordScanner reader = new Grib1RecordScanner(raf);
  while (reader.hasNext()) {
    first = reader.next();
    break;
  }

  if (first == null) {
    return null;
  } else {
    return new Grib1Bean(mf, first);
  }
}
 

private Grib2Index createIndex(MFile mf, Formatter f) throws IOException {
  String path = mf.getPath();
  Grib2Index index = new Grib2Index();
  if (!index.readIndex(path, mf.getLastModified())) {
    // make sure its a grib2 file
    try (RandomAccessFile raf = new RandomAccessFile(path, "r")) {
      if (!Grib2RecordScanner.isValidFile(raf))
        return null;
      index.makeIndex(path, raf);
    }
  }
  return index;
}
 

@AfterClass
static public void after() {
  Grib.setDebugFlags(new DebugFlagsImpl());
  Formatter out = new Formatter(System.out);

  FileCacheIF cache = GribCdmIndex.gribCollectionCache;
  if (cache != null) {
    cache.showTracking(out);
    cache.showCache(out);
    cache.clearCache(false);
  }

  FileCacheIF rafCache = RandomAccessFile.getGlobalFileCache();
  if (rafCache != null) {
    rafCache.showCache(out);
  }

  System.out.printf("            countGC=%7d%n", GribCollectionImmutable.countGC);
  System.out.printf("            countPC=%7d%n", PartitionCollectionImmutable.countPC);
  System.out.printf("    countDataAccess=%7d%n", GribIosp.debugIndexOnlyCount);
  System.out.printf(" total files needed=%7d%n",
      GribCollectionImmutable.countGC + PartitionCollectionImmutable.countPC + GribIosp.debugIndexOnlyCount);

  FileCache.shutdown();
  RandomAccessFile.setGlobalFileCache(null);
  TestDir.checkLeaks();
  RandomAccessFile.setDebugLeaks(false);
}
 

/**
 * This can only be used for netcdf-3 files served over HTTP
 *
 * @param url HTTP URL location
 * @throws IOException if error
 * @deprecated use NetcdfFiles.open( http:location) or NetcdfDatasets.openFile( http:location)
 */
@Deprecated
public NetcdfFile(URL url) throws IOException {
  this.location = url.toString();
  RandomAccessFile raf = new HTTPRandomAccessFile(location);
  this.iosp = SPFactory.getServiceProvider();
  iosp.open(raf, this, null);
  finish();
}
 

public static int writeVInt(RandomAccessFile out, int value) throws IOException {
  int count = 0;

  while (true) {
    if ((value & ~0x7F) == 0) {
      out.write((byte) value);
      break;
    } else {
      out.write((byte) ((value & 0x7F) | 0x80));
      value >>>= 7;
    }
  }

  return count + 1;
}
 

@Override
public boolean isPartition(Path indexFile) throws IOException {
  logger.debug("GribCdmIndex.isPartition {}", indexFile);
  try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile.toString())) {
    GribCollectionType type = getType(raf);
    return (type == GribCollectionType.Partition1) || (type == GribCollectionType.Partition2);
  }
}
 

@AfterClass
static public void after() {
  Grib.setDebugFlags(new DebugFlagsImpl());
  Formatter out = new Formatter(System.out);

  FileCacheIF cache = GribCdmIndex.gribCollectionCache;
  if (show && cache != null) {
    cache.showTracking(out);
    cache.showCache(out);
    cache.clearCache(false);
  }

  FileCacheIF rafCache = RandomAccessFile.getGlobalFileCache();
  if (show && rafCache != null) {
    rafCache.showCache(out);
  }

  System.out.printf("            countGC=%7d%n", GribCollectionImmutable.countGC);
  System.out.printf("            countPC=%7d%n", PartitionCollectionImmutable.countPC);
  System.out.printf("    countDataAccess=%7d%n", GribIosp.debugIndexOnlyCount);
  System.out.printf(" total files needed=%7d%n",
      GribCollectionImmutable.countGC + PartitionCollectionImmutable.countPC + GribIosp.debugIndexOnlyCount);

  FileCache.shutdown();
  RandomAccessFile.setGlobalFileCache(null);
  TestDir.checkLeaks();
  RandomAccessFile.setDebugLeaks(false);
}