org.apache.hadoop.hdfs.inotify.EventBatchList Java Examples

@Override
public EventBatchList getEditsFromTxid(long txid) throws IOException {
  GetEditsFromTxidRequestProto req = GetEditsFromTxidRequestProto.newBuilder()
      .setTxid(txid).build();
  try {
    return PBHelper.convert(rpcProxy.getEditsFromTxid(null, req));
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

@Override
public EventBatchList getEditsFromTxid(long txid) throws IOException {
  GetEditsFromTxidRequestProto req = GetEditsFromTxidRequestProto.newBuilder()
      .setTxid(txid).build();
  try {
    return PBHelper.convert(rpcProxy.getEditsFromTxid(null, req));
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

/**
 * Returns the next batch of events in the stream or null if no new
 * batches are currently available.
 *
 * @throws IOException because of network error or edit log
 * corruption. Also possible if JournalNodes are unresponsive in the
 * QJM setting (even one unresponsive JournalNode is enough in rare cases),
 * so catching this exception and retrying at least a few times is
 * recommended.
 * @throws MissingEventsException if we cannot return the next batch in the
 * stream because the data for the events (and possibly some subsequent
 * events) has been deleted (generally because this stream is a very large
 * number of transactions behind the current state of the NameNode). It is
 * safe to continue reading from the stream after this exception is thrown
 * The next available batch of events will be returned.
 */
public EventBatch poll() throws IOException, MissingEventsException {
  TraceScope scope =
      Trace.startSpan("inotifyPoll", traceSampler);
  try {
    // need to keep retrying until the NN sends us the latest committed txid
    if (lastReadTxid == -1) {
      LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
      lastReadTxid = namenode.getCurrentEditLogTxid();
      return null;
    }
    if (!it.hasNext()) {
      EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
      if (el.getLastTxid() != -1) {
        // we only want to set syncTxid when we were actually able to read some
        // edits on the NN -- otherwise it will seem like edits are being
        // generated faster than we can read them when the problem is really
        // that we are temporarily unable to read edits
        syncTxid = el.getSyncTxid();
        it = el.getBatches().iterator();
        long formerLastReadTxid = lastReadTxid;
        lastReadTxid = el.getLastTxid();
        if (el.getFirstTxid() != formerLastReadTxid + 1) {
          throw new MissingEventsException(formerLastReadTxid + 1,
              el.getFirstTxid());
        }
      } else {
        LOG.debug("poll(): read no edits from the NN when requesting edits " +
          "after txid {}", lastReadTxid);
        return null;
      }
    }

    if (it.hasNext()) { // can be empty if el.getLastTxid != -1 but none of the
      // newly seen edit log ops actually got converted to events
      return it.next();
    } else {
      return null;
    }
  } finally {
    scope.close();
  }
}
 

@Override // ClientProtocol
public EventBatchList getEditsFromTxid(long txid) throws IOException {
  checkNNStartup();
  namesystem.checkOperation(OperationCategory.READ); // only active
  namesystem.checkSuperuserPrivilege();
  int maxEventsPerRPC = nn.conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY,
      DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_DEFAULT);
  FSEditLog log = namesystem.getFSImage().getEditLog();
  long syncTxid = log.getSyncTxId();
  // If we haven't synced anything yet, we can only read finalized
  // segments since we can't reliably determine which txns in in-progress
  // segments have actually been committed (e.g. written to a quorum of JNs).
  // If we have synced txns, we can definitely read up to syncTxid since
  // syncTxid is only updated after a transaction is committed to all
  // journals. (In-progress segments written by old writers are already
  // discarded for us, so if we read any in-progress segments they are
  // guaranteed to have been written by this NameNode.)
  boolean readInProgress = syncTxid > 0;

  List<EventBatch> batches = Lists.newArrayList();
  int totalEvents = 0;
  long maxSeenTxid = -1;
  long firstSeenTxid = -1;

  if (syncTxid > 0 && txid > syncTxid) {
    // we can't read past syncTxid, so there's no point in going any further
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
  }

  Collection<EditLogInputStream> streams = null;
  try {
    streams = log.selectInputStreams(txid, 0, null, readInProgress);
  } catch (IllegalStateException e) { // can happen if we have
    // transitioned out of active and haven't yet transitioned to standby
    // and are using QJM -- the edit log will be closed and this exception
    // will result
    LOG.info("NN is transitioning from active to standby and FSEditLog " +
    "is closed -- could not read edits");
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
  }

  boolean breakOuter = false;
  for (EditLogInputStream elis : streams) {
    // our assumption in this code is the EditLogInputStreams are ordered by
    // starting txid
    try {
      FSEditLogOp op = null;
      while ((op = readOp(elis)) != null) {
        // break out of here in the unlikely event that syncTxid is so
        // out of date that its segment has already been deleted, so the first
        // txid we get is greater than syncTxid
        if (syncTxid > 0 && op.getTransactionId() > syncTxid) {
          breakOuter = true;
          break;
        }

        EventBatch eventBatch = InotifyFSEditLogOpTranslator.translate(op);
        if (eventBatch != null) {
          batches.add(eventBatch);
          totalEvents += eventBatch.getEvents().length;
        }
        if (op.getTransactionId() > maxSeenTxid) {
          maxSeenTxid = op.getTransactionId();
        }
        if (firstSeenTxid == -1) {
          firstSeenTxid = op.getTransactionId();
        }
        if (totalEvents >= maxEventsPerRPC || (syncTxid > 0 &&
            op.getTransactionId() == syncTxid)) {
          // we're done
          breakOuter = true;
          break;
        }
      }
    } finally {
      elis.close();
    }
    if (breakOuter) {
      break;
    }
  }

  return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}
 

public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
  IOException {
  final InotifyProtos.EventsListProto list = resp.getEventsList();
  final long firstTxid = list.getFirstTxid();
  final long lastTxid = list.getLastTxid();

  List<EventBatch> batches = Lists.newArrayList();
  if (list.getEventsList().size() > 0) {
    throw new IOException("Can't handle old inotify server response.");
  }
  for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
    long txid = bp.getTxid();
    if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
      throw new IOException("Error converting TxidResponseProto: got a " +
          "transaction id " + txid + " that was outside the range of [" +
          firstTxid + ", " + lastTxid + "].");
    }
    List<Event> events = Lists.newArrayList();
    for (InotifyProtos.EventProto p : bp.getEventsList()) {
      switch (p.getType()) {
        case EVENT_CLOSE:
          InotifyProtos.CloseEventProto close =
              InotifyProtos.CloseEventProto.parseFrom(p.getContents());
          events.add(new Event.CloseEvent(close.getPath(),
              close.getFileSize(), close.getTimestamp()));
          break;
        case EVENT_CREATE:
          InotifyProtos.CreateEventProto create =
              InotifyProtos.CreateEventProto.parseFrom(p.getContents());
          events.add(new Event.CreateEvent.Builder()
              .iNodeType(createTypeConvert(create.getType()))
              .path(create.getPath())
              .ctime(create.getCtime())
              .ownerName(create.getOwnerName())
              .groupName(create.getGroupName())
              .perms(convert(create.getPerms()))
              .replication(create.getReplication())
              .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
                  create.getSymlinkTarget())
              .defaultBlockSize(create.getDefaultBlockSize())
              .overwrite(create.getOverwrite()).build());
          break;
        case EVENT_METADATA:
          InotifyProtos.MetadataUpdateEventProto meta =
              InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
          events.add(new Event.MetadataUpdateEvent.Builder()
              .path(meta.getPath())
              .metadataType(metadataUpdateTypeConvert(meta.getType()))
              .mtime(meta.getMtime())
              .atime(meta.getAtime())
              .replication(meta.getReplication())
              .ownerName(
                  meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
              .groupName(
                  meta.getGroupName().isEmpty() ? null : meta.getGroupName())
              .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
              .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
                  meta.getAclsList()))
              .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
                  meta.getXAttrsList()))
              .xAttrsRemoved(meta.getXAttrsRemoved())
              .build());
          break;
        case EVENT_RENAME:
          InotifyProtos.RenameEventProto rename =
              InotifyProtos.RenameEventProto.parseFrom(p.getContents());
          events.add(new Event.RenameEvent.Builder()
                .srcPath(rename.getSrcPath())
                .dstPath(rename.getDestPath())
                .timestamp(rename.getTimestamp())
                .build());
          break;
        case EVENT_APPEND:
          InotifyProtos.AppendEventProto append =
              InotifyProtos.AppendEventProto.parseFrom(p.getContents());
          events.add(new Event.AppendEvent.Builder().path(append.getPath())
              .newBlock(append.hasNewBlock() && append.getNewBlock())
              .build());
          break;
        case EVENT_UNLINK:
          InotifyProtos.UnlinkEventProto unlink =
              InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
          events.add(new Event.UnlinkEvent.Builder()
                .path(unlink.getPath())
                .timestamp(unlink.getTimestamp())
                .build());
          break;
        default:
          throw new RuntimeException("Unexpected inotify event type: " +
              p.getType());
      }
    }
    batches.add(new EventBatch(txid, events.toArray(new Event[0])));
  }
  return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
      resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}
 

@Override
public EventBatchList getEditsFromTxid(long txid) throws IOException {
    throw new IOException("Invalid operation, do not use proxy");
}
 

/**
 * Returns the next batch of events in the stream or null if no new
 * batches are currently available.
 *
 * @throws IOException because of network error or edit log
 * corruption. Also possible if JournalNodes are unresponsive in the
 * QJM setting (even one unresponsive JournalNode is enough in rare cases),
 * so catching this exception and retrying at least a few times is
 * recommended.
 * @throws MissingEventsException if we cannot return the next batch in the
 * stream because the data for the events (and possibly some subsequent
 * events) has been deleted (generally because this stream is a very large
 * number of transactions behind the current state of the NameNode). It is
 * safe to continue reading from the stream after this exception is thrown
 * The next available batch of events will be returned.
 */
public EventBatch poll() throws IOException, MissingEventsException {
  TraceScope scope =
      Trace.startSpan("inotifyPoll", traceSampler);
  try {
    // need to keep retrying until the NN sends us the latest committed txid
    if (lastReadTxid == -1) {
      LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
      lastReadTxid = namenode.getCurrentEditLogTxid();
      return null;
    }
    if (!it.hasNext()) {
      EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
      if (el.getLastTxid() != -1) {
        // we only want to set syncTxid when we were actually able to read some
        // edits on the NN -- otherwise it will seem like edits are being
        // generated faster than we can read them when the problem is really
        // that we are temporarily unable to read edits
        syncTxid = el.getSyncTxid();
        it = el.getBatches().iterator();
        long formerLastReadTxid = lastReadTxid;
        lastReadTxid = el.getLastTxid();
        if (el.getFirstTxid() != formerLastReadTxid + 1) {
          throw new MissingEventsException(formerLastReadTxid + 1,
              el.getFirstTxid());
        }
      } else {
        LOG.debug("poll(): read no edits from the NN when requesting edits " +
          "after txid {}", lastReadTxid);
        return null;
      }
    }

    if (it.hasNext()) { // can be empty if el.getLastTxid != -1 but none of the
      // newly seen edit log ops actually got converted to events
      return it.next();
    } else {
      return null;
    }
  } finally {
    scope.close();
  }
}
 

@Override // ClientProtocol
public EventBatchList getEditsFromTxid(long txid) throws IOException {
  checkNNStartup();
  namesystem.checkOperation(OperationCategory.READ); // only active
  namesystem.checkSuperuserPrivilege();
  int maxEventsPerRPC = nn.conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY,
      DFSConfigKeys.DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_DEFAULT);
  FSEditLog log = namesystem.getFSImage().getEditLog();
  long syncTxid = log.getSyncTxId();
  // If we haven't synced anything yet, we can only read finalized
  // segments since we can't reliably determine which txns in in-progress
  // segments have actually been committed (e.g. written to a quorum of JNs).
  // If we have synced txns, we can definitely read up to syncTxid since
  // syncTxid is only updated after a transaction is committed to all
  // journals. (In-progress segments written by old writers are already
  // discarded for us, so if we read any in-progress segments they are
  // guaranteed to have been written by this NameNode.)
  boolean readInProgress = syncTxid > 0;

  List<EventBatch> batches = Lists.newArrayList();
  int totalEvents = 0;
  long maxSeenTxid = -1;
  long firstSeenTxid = -1;

  if (syncTxid > 0 && txid > syncTxid) {
    // we can't read past syncTxid, so there's no point in going any further
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
  }

  Collection<EditLogInputStream> streams = null;
  try {
    streams = log.selectInputStreams(txid, 0, null, readInProgress);
  } catch (IllegalStateException e) { // can happen if we have
    // transitioned out of active and haven't yet transitioned to standby
    // and are using QJM -- the edit log will be closed and this exception
    // will result
    LOG.info("NN is transitioning from active to standby and FSEditLog " +
    "is closed -- could not read edits");
    return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
  }

  boolean breakOuter = false;
  for (EditLogInputStream elis : streams) {
    // our assumption in this code is the EditLogInputStreams are ordered by
    // starting txid
    try {
      FSEditLogOp op = null;
      while ((op = readOp(elis)) != null) {
        // break out of here in the unlikely event that syncTxid is so
        // out of date that its segment has already been deleted, so the first
        // txid we get is greater than syncTxid
        if (syncTxid > 0 && op.getTransactionId() > syncTxid) {
          breakOuter = true;
          break;
        }

        EventBatch eventBatch = InotifyFSEditLogOpTranslator.translate(op);
        if (eventBatch != null) {
          batches.add(eventBatch);
          totalEvents += eventBatch.getEvents().length;
        }
        if (op.getTransactionId() > maxSeenTxid) {
          maxSeenTxid = op.getTransactionId();
        }
        if (firstSeenTxid == -1) {
          firstSeenTxid = op.getTransactionId();
        }
        if (totalEvents >= maxEventsPerRPC || (syncTxid > 0 &&
            op.getTransactionId() == syncTxid)) {
          // we're done
          breakOuter = true;
          break;
        }
      }
    } finally {
      elis.close();
    }
    if (breakOuter) {
      break;
    }
  }

  return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}
 

public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
  IOException {
  final InotifyProtos.EventsListProto list = resp.getEventsList();
  final long firstTxid = list.getFirstTxid();
  final long lastTxid = list.getLastTxid();

  List<EventBatch> batches = Lists.newArrayList();
  if (list.getEventsList().size() > 0) {
    throw new IOException("Can't handle old inotify server response.");
  }
  for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
    long txid = bp.getTxid();
    if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
      throw new IOException("Error converting TxidResponseProto: got a " +
          "transaction id " + txid + " that was outside the range of [" +
          firstTxid + ", " + lastTxid + "].");
    }
    List<Event> events = Lists.newArrayList();
    for (InotifyProtos.EventProto p : bp.getEventsList()) {
      switch (p.getType()) {
        case EVENT_CLOSE:
          InotifyProtos.CloseEventProto close =
              InotifyProtos.CloseEventProto.parseFrom(p.getContents());
          events.add(new Event.CloseEvent(close.getPath(),
              close.getFileSize(), close.getTimestamp()));
          break;
        case EVENT_CREATE:
          InotifyProtos.CreateEventProto create =
              InotifyProtos.CreateEventProto.parseFrom(p.getContents());
          events.add(new Event.CreateEvent.Builder()
              .iNodeType(createTypeConvert(create.getType()))
              .path(create.getPath())
              .ctime(create.getCtime())
              .ownerName(create.getOwnerName())
              .groupName(create.getGroupName())
              .perms(convert(create.getPerms()))
              .replication(create.getReplication())
              .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
                  create.getSymlinkTarget())
              .defaultBlockSize(create.getDefaultBlockSize())
              .overwrite(create.getOverwrite()).build());
          break;
        case EVENT_METADATA:
          InotifyProtos.MetadataUpdateEventProto meta =
              InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
          events.add(new Event.MetadataUpdateEvent.Builder()
              .path(meta.getPath())
              .metadataType(metadataUpdateTypeConvert(meta.getType()))
              .mtime(meta.getMtime())
              .atime(meta.getAtime())
              .replication(meta.getReplication())
              .ownerName(
                  meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
              .groupName(
                  meta.getGroupName().isEmpty() ? null : meta.getGroupName())
              .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
              .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
                  meta.getAclsList()))
              .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
                  meta.getXAttrsList()))
              .xAttrsRemoved(meta.getXAttrsRemoved())
              .build());
          break;
        case EVENT_RENAME:
          InotifyProtos.RenameEventProto rename =
              InotifyProtos.RenameEventProto.parseFrom(p.getContents());
          events.add(new Event.RenameEvent.Builder()
                .srcPath(rename.getSrcPath())
                .dstPath(rename.getDestPath())
                .timestamp(rename.getTimestamp())
                .build());
          break;
        case EVENT_APPEND:
          InotifyProtos.AppendEventProto append =
              InotifyProtos.AppendEventProto.parseFrom(p.getContents());
          events.add(new Event.AppendEvent.Builder().path(append.getPath())
              .newBlock(append.hasNewBlock() && append.getNewBlock())
              .build());
          break;
        case EVENT_UNLINK:
          InotifyProtos.UnlinkEventProto unlink =
              InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
          events.add(new Event.UnlinkEvent.Builder()
                .path(unlink.getPath())
                .timestamp(unlink.getTimestamp())
                .build());
          break;
        default:
          throw new RuntimeException("Unexpected inotify event type: " +
              p.getType());
      }
    }
    batches.add(new EventBatch(txid, events.toArray(new Event[0])));
  }
  return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
      resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}
 

/**
 * Get an ordered list of batches of events corresponding to the edit log
 * transactions for txids equal to or greater than txid.
 */
@Idempotent
public EventBatchList getEditsFromTxid(long txid) throws IOException;
 

/**
 * Get an ordered list of batches of events corresponding to the edit log
 * transactions for txids equal to or greater than txid.
 */
@Idempotent
public EventBatchList getEditsFromTxid(long txid) throws IOException;