Java Code Examples for javax.net.ssl.SSLEngineResult#getStatus()

/** Returns last {@link HandshakeStatus} of the loop */
private void wrapLoop(ByteBufferSet source) throws SSLException {
  while (true) {
    SSLEngineResult result = callEngineWrap(source);
    switch (result.getStatus()) {
      case OK:
      case CLOSED:
        return;
      case BUFFER_OVERFLOW:
        Util.assertTrue(result.bytesConsumed() == 0);
        outEncrypted.enlarge();
        break;
      case BUFFER_UNDERFLOW:
        throw new IllegalStateException();
    }
  }
}
 

/**
 * Check status and retry the negotiation process if needed.
 *
 * @param res Result.
 * @throws GridNioException If exception occurred during handshake.
 * @throws SSLException If failed to process SSL data
 */
private void renegotiateIfNeeded(SSLEngineResult res) throws IgniteCheckedException, SSLException {
    if (res.getStatus() != CLOSED && res.getStatus() != BUFFER_UNDERFLOW
        && res.getHandshakeStatus() != NOT_HANDSHAKING) {
        // Renegotiation required.
        handshakeStatus = res.getHandshakeStatus();

        if (log.isDebugEnabled())
            log.debug("Renegotiation requested [status=" + res.getStatus() + ", handshakeStatus = " +
                handshakeStatus + "ses=" + ses + ']');

        handshakeFinished = false;

        handshake();
    }
}
 

/**
 * Performs the WRAP function
 * @param doWrite boolean
 * @return SSLEngineResult
 * @throws IOException
 */
private SSLEngineResult handshakeWrap(boolean doWrite) throws IOException {
  logger.trace("SSLHandshake handshakeWrap {}", getConnectionId());
  if (netWriteBuffer.hasRemaining()) {
    throw new IllegalStateException("handshakeWrap called with netWriteBuffer not empty");
  }
  //this should never be called with a network buffer that contains data
  //so we can clear it here.
  netWriteBuffer.clear();
  SSLEngineResult result = sslEngine.wrap(emptyBuf, netWriteBuffer);
  //prepare the results to be written
  netWriteBuffer.flip();
  handshakeStatus = result.getHandshakeStatus();
  if (result.getStatus() == Status.OK && result.getHandshakeStatus() == HandshakeStatus.NEED_TASK) {
    handshakeStatus = runDelegatedTasks();
  }
  if (doWrite) {
    flush(netWriteBuffer);
  }
  return result;
}
 

/**
 * Start SSL shutdown process.
 * 
 * @return <tt>true</tt> if shutdown process is started. <tt>false</tt> if
 *         shutdown process is already finished.
 * @throws SSLException
 *             on errors
 */
/* no qualifier */boolean closeOutbound() throws SSLException {
    if (sslEngine == null || sslEngine.isOutboundDone()) {
        return false;
    }

    sslEngine.closeOutbound();

    createOutNetBuffer(0);
    SSLEngineResult result;
    for (;;) {
        result = sslEngine.wrap(emptyBuffer.buf(), outNetBuffer.buf());
        if (result.getStatus() == SSLEngineResult.Status.BUFFER_OVERFLOW) {
            outNetBuffer.capacity(outNetBuffer.capacity() << 1);
            outNetBuffer.limit(outNetBuffer.capacity());
        } else {
            break;
        }
    }

    if (result.getStatus() != SSLEngineResult.Status.CLOSED) {
        throw new SSLException("Improper close state: " + result);
    }
    outNetBuffer.flip();
    return true;
}
 

private void checkStatus(SSLEngineResult res) throws SSLException {
    SSLEngineResult.Status status = res.getStatus();

    /*
     * The status may be:
     * OK          - Normal operation
     * OVERFLOW    - Should never happen since the application buffer is sized to hold the maximum packet size.
     * UNDERFLOW   - Need to read more data from the socket. It's normal.
     * CLOSED      - The other peer closed the socket. Also normal.
     */
    if (status == SSLEngineResult.Status.BUFFER_OVERFLOW) {
        throw new SSLException("SSLEngine error during decrypt: " + status + " inNetBuffer: " + inNetBuffer + "appBuffer: " + appBuffer);
    }
}
 

/**
 * Perform handshake unwrap
 * @param doread boolean
 * @return the result
 * @throws IOException An IO error occurred
 */
protected SSLEngineResult handshakeUnwrap(boolean doread) throws IOException {

    if (netInBuffer.position() == netInBuffer.limit()) {
        //clear the buffer if we have emptied it out on data
        netInBuffer.clear();
    }
    if (doread)  {
        //if we have data to read, read it
        int read = sc.read(netInBuffer);
        if (read == -1) {
            throw new IOException(sm.getString("channel.nio.ssl.eofDuringHandshake"));
        }
    }
    SSLEngineResult result;
    boolean cont = false;
    //loop while we can perform pure SSLEngine data
    do {
        //prepare the buffer with the incoming data
        netInBuffer.flip();
        //call unwrap
        getBufHandler().configureReadBufferForWrite();
        result = sslEngine.unwrap(netInBuffer, getBufHandler().getReadBuffer());
        //compact the buffer, this is an optional method, wonder what would happen if we didn't
        netInBuffer.compact();
        //read in the status
        handshakeStatus = result.getHandshakeStatus();
        if ( result.getStatus() == SSLEngineResult.Status.OK &&
             result.getHandshakeStatus() == HandshakeStatus.NEED_TASK ) {
            //execute tasks if we need to
            handshakeStatus = tasks();
        }
        //perform another unwrap?
        cont = result.getStatus() == SSLEngineResult.Status.OK &&
               handshakeStatus == HandshakeStatus.NEED_UNWRAP;
    }while ( cont );
    return result;
}
 

/**
 * @param res SSL engine result.
 * @throws SSLException If status is not acceptable.
 */
private void checkStatus(SSLEngineResult res)
    throws SSLException {

    SSLEngineResult.Status status = res.getStatus();

    if (status != Status.OK && status != CLOSED && status != BUFFER_UNDERFLOW)
        throw new SSLException("Failed to unwrap incoming data (SSL engine error) [ses" + ses + ", status=" +
            status + ']');
}
 

/**
 * Checks that status of result {@code r} is {@code wantedStatus}.
 *
 * @param r            - Result.
 * @param wantedStatus - Wanted status of the result.
 * @throws AssertionError - if status or {@code r} is not
 *                        {@code wantedStatus}.
 */
public static void checkResult(SSLEngineResult r,
                               SSLEngineResult.Status wantedStatus) {
    SSLEngineResult.Status rs = r.getStatus();
    if (!rs.equals(wantedStatus)) {
        throw new AssertionError("Unexpected status " + rs.name()
                + ", should be " + wantedStatus.name());
    }
}
 

/**
 * Encrypt provided buffer. Encrypted data returned by getOutNetBuffer().
 *
 * @param src data to encrypt
 * @throws SSLException on errors
 */
void encrypt(ByteBuffer src) throws SSLException {
	if (!handshakeComplete)
		throw new IllegalStateException();

	if (!src.hasRemaining()) {
		if (outNetBuffer == null)
			outNetBuffer = IoBuffer.allocate(0);
		return;
	}

	createOutNetBuffer(src.remaining());

	// Loop until there is no more data in src
	while (src.hasRemaining()) {
		SSLEngineResult result = sslEngine.wrap(src, outNetBuffer.buf());

		if (result.getStatus() == Status.OK) {
			if (result.getHandshakeStatus() == HandshakeStatus.NEED_TASK)
				doTasks();
		} else if (result.getStatus() == Status.BUFFER_OVERFLOW) {
			outNetBuffer = IoBuffer.reallocate(outNetBuffer, outNetBuffer.capacity() << 1);
			outNetBuffer.limit(outNetBuffer.capacity());
		} else
			throw new SSLException("SSLEngine error during encrypt: " + result.getStatus() + " src: " + src + "outNetBuffer: " + outNetBuffer);
	}

	outNetBuffer.flip();
}
 

private void tryCloseOutbound() {
	if (!engine.isOutboundDone()) {
		engine.closeOutbound();
		try {
			while (!engine.isOutboundDone()) {
				SSLEngineResult result = tryToWrap();
				if (result.getStatus() == CLOSED) {
					break;
				}
			}
		} catch (SSLException ignored) {
		}
	}
}
 

/**
 * Perform handshake unwrap
 * @param doread boolean
 * @return SSLEngineResult
 * @throws IOException
 */
protected SSLEngineResult handshakeUnwrap(boolean doread) throws IOException {

    if (netInBuffer.position() == netInBuffer.limit()) {
        //clear the buffer if we have emptied it out on data
        netInBuffer.clear();
    }
    if ( doread )  {
        //if we have data to read, read it
        int read = sc.read(netInBuffer);
        if (read == -1) throw new IOException("EOF encountered during handshake.");
    }
    SSLEngineResult result;
    boolean cont = false;
    //loop while we can perform pure SSLEngine data
    do {
        //prepare the buffer with the incoming data
        netInBuffer.flip();
        //call unwrap
        result = sslEngine.unwrap(netInBuffer, bufHandler.getReadBuffer());
        //compact the buffer, this is an optional method, wonder what would happen if we didn't
        netInBuffer.compact();
        //read in the status
        handshakeStatus = result.getHandshakeStatus();
        if ( result.getStatus() == SSLEngineResult.Status.OK &&
             result.getHandshakeStatus() == HandshakeStatus.NEED_TASK ) {
            //execute tasks if we need to
            handshakeStatus = tasks();
        }
        //perform another unwrap?
        cont = result.getStatus() == SSLEngineResult.Status.OK &&
               handshakeStatus == HandshakeStatus.NEED_UNWRAP;
    }while ( cont );
    return result;
}
 

/**
 * Attempts to decrypt the given buffer of data, writing the result into {@link #destinationBuffer}. If successful, will return <code>true</code>.
 * If more data is needed in order to perform the decryption, will return <code>false</code>.
 *
 * @param encrypted the ByteBuffer containing the data to decrypt
 * @return <code>true</code> if decryption was successful, <code>false</code> otherwise
 * @throws IOException if the Peer closed the connection or if unable to decrypt the message
 */
private boolean decrypt(final ByteBuffer encrypted) throws IOException {
    if (sslEngine == null) {
        throw new SSLException("Unable to decrypt message because no SSLEngine has been configured");
    }

    destinationBuffer.clear();

    while (true) {
        final SSLEngineResult result = sslEngine.unwrap(encrypted, destinationBuffer);

        switch (result.getStatus()) {
            case OK:
                destinationBuffer.flip();
                return true;
            case CLOSED:
                throw new IOException("Failed to decrypt data from Peer " + peerDescription + " because Peer unexpectedly closed connection");
            case BUFFER_OVERFLOW:
                // ecnryptedBuffer is not large enough. Need to increase the size.
                final ByteBuffer tempBuffer = ByteBuffer.allocate(encrypted.position() + sslEngine.getSession().getApplicationBufferSize());
                destinationBuffer.flip();
                tempBuffer.put(destinationBuffer);
                destinationBuffer = tempBuffer;

                break;
            case BUFFER_UNDERFLOW:
                // Not enough data to decrypt. Must read more from the channel.
                return false;
        }
    }
}
 

private void performHandshake() throws IOException {
    // Generate handshake message
    final byte[] emptyMessage = new byte[0];
    handshaking = true;
    logger.debug("{} Performing Handshake", this);

    try {
        while (true) {
            switch (engine.getHandshakeStatus()) {
                case FINISHED:
                    return;
                case NEED_WRAP: {
                    final ByteBuffer appDataOut = ByteBuffer.wrap(emptyMessage);

                    final ByteBuffer outboundBuffer = streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());

                    final SSLEngineResult wrapHelloResult = engine.wrap(appDataOut, outboundBuffer);
                    if (wrapHelloResult.getStatus() == Status.BUFFER_OVERFLOW) {
                        streamOutManager.prepareForWrite(engine.getSession().getApplicationBufferSize());
                        continue;
                    }

                    if (wrapHelloResult.getStatus() != Status.OK) {
                        throw new SSLHandshakeException("Could not generate SSL Handshake information: SSLEngineResult: "
                                + wrapHelloResult.toString());
                    }

                    logger.trace("{} Handshake response after wrapping: {}", this, wrapHelloResult);

                    final ByteBuffer readableStreamOut = streamOutManager.prepareForRead(1);
                    final int bytesToSend = readableStreamOut.remaining();
                    writeFully(readableStreamOut);
                    logger.trace("{} Sent {} bytes of wrapped data for handshake", this, bytesToSend);

                    streamOutManager.clear();
                }
                continue;
                case NEED_UNWRAP: {
                    final ByteBuffer readableDataIn = streamInManager.prepareForRead(0);
                    final ByteBuffer appData = appDataManager.prepareForWrite(engine.getSession().getApplicationBufferSize());

                    // Read handshake response from other side
                    logger.trace("{} Unwrapping: {} to {}", this, readableDataIn, appData);
                    SSLEngineResult handshakeResponseResult = engine.unwrap(readableDataIn, appData);
                    logger.trace("{} Handshake response after unwrapping: {}", this, handshakeResponseResult);

                    if (handshakeResponseResult.getStatus() == Status.BUFFER_UNDERFLOW) {
                        final ByteBuffer writableDataIn = streamInManager.prepareForWrite(engine.getSession().getPacketBufferSize());
                        final int bytesRead = readData(writableDataIn);
                        if (bytesRead > 0) {
                            logger.trace("{} Read {} bytes for handshake", this, bytesRead);
                        }

                        if (bytesRead < 0) {
                            throw new SSLHandshakeException("Reached End-of-File marker while performing handshake");
                        }
                    } else if (handshakeResponseResult.getStatus() == Status.CLOSED) {
                        throw new IOException("Channel was closed by peer during handshake");
                    } else {
                        streamInManager.compact();
                        appDataManager.clear();
                    }
                }
                break;
                case NEED_TASK:
                    performTasks();
                    continue;
                case NOT_HANDSHAKING:
                    return;
            }
        }
    } finally {
        handshaking = false;
    }
}
 

private SSLEngineResult.Status unwrapHandshake() throws SSLException {
	// Prepare the net data for reading.
	if (inNetBuffer != null) {
		inNetBuffer.flip();
	}

	if (inNetBuffer == null || !inNetBuffer.hasRemaining()) {
		// Need more data.
		return SSLEngineResult.Status.BUFFER_UNDERFLOW;
	}

	SSLEngineResult res = unwrap();
	handshakeStatus = res.getHandshakeStatus();

	checkStatus(res);

	// If handshake finished, no data was produced, and the status is still ok, try to unwrap more
	if (handshakeStatus == SSLEngineResult.HandshakeStatus.FINISHED
			&& res.getStatus() == SSLEngineResult.Status.OK
			&& inNetBuffer.hasRemaining()) {
		res = unwrap();

		// prepare to be written again
		if (inNetBuffer.hasRemaining()) {
			inNetBuffer.compact();
		} else {
			inNetBuffer = null;
		}

		renegotiateIfNeeded(res);
	} else {
		// prepare to be written again
		if (inNetBuffer.hasRemaining()) {
			inNetBuffer.compact();
		} else {
			inNetBuffer = null;
		}
	}

	return res.getStatus();
}
 

private SSLEngineResult wrap(ByteBufAllocator alloc, SSLEngine engine, ByteBuf in, ByteBuf out)
        throws SSLException {
    ByteBuf newDirectIn = null;
    try {
        int readerIndex = in.readerIndex();
        int readableBytes = in.readableBytes();

        // We will call SslEngine.wrap(ByteBuffer[], ByteBuffer) to allow efficient handling of
        // CompositeByteBuf without force an extra memory copy when CompositeByteBuffer.nioBuffer() is called.
        final ByteBuffer[] in0;
        if (in.isDirect() || !wantsDirectBuffer) {
            // As CompositeByteBuf.nioBufferCount() can be expensive (as it needs to check all composed ByteBuf
            // to calculate the count) we will just assume a CompositeByteBuf contains more then 1 ByteBuf.
            // The worst that can happen is that we allocate an extra ByteBuffer[] in CompositeByteBuf.nioBuffers()
            // which is better then walking the composed ByteBuf in most cases.
            if (!(in instanceof CompositeByteBuf) && in.nioBufferCount() == 1) {
                in0 = singleBuffer;
                // We know its only backed by 1 ByteBuffer so use internalNioBuffer to keep object allocation
                // to a minimum.
                in0[0] = in.internalNioBuffer(readerIndex, readableBytes);
            } else {
                in0 = in.nioBuffers();
            }
        } else {
            // We could even go further here and check if its a CompositeByteBuf and if so try to decompose it and
            // only replace the ByteBuffer that are not direct. At the moment we just will replace the whole
            // CompositeByteBuf to keep the complexity to a minimum
            newDirectIn = alloc.directBuffer(readableBytes);
            newDirectIn.writeBytes(in, readerIndex, readableBytes);
            in0 = singleBuffer;
            in0[0] = newDirectIn.internalNioBuffer(0, readableBytes);
        }

        for (;;) {
            ByteBuffer out0 = out.nioBuffer(out.writerIndex(), out.writableBytes());
            SSLEngineResult result = engine.wrap(in0, out0);
            in.skipBytes(result.bytesConsumed());
            out.writerIndex(out.writerIndex() + result.bytesProduced());

            switch (result.getStatus()) {
            case BUFFER_OVERFLOW:
                out.ensureWritable(maxPacketBufferSize);
                break;
            default:
                return result;
            }
        }
    } finally {
        // Null out to allow GC of ByteBuffer
        singleBuffer[0] = null;

        if (newDirectIn != null) {
            newDirectIn.release();
        }
    }
}
 

private void doUnWrap() {
    try {
        ByteBuffer netBuffer = netReadBuffer.buffer();
        ByteBuffer appBuffer = appReadBuffer.buffer();
        netBuffer.flip();
        SSLEngineResult result = sslEngine.unwrap(netBuffer, appBuffer);
        boolean closed = false;
        while (!closed && result.getStatus() != SSLEngineResult.Status.OK) {
            switch (result.getStatus()) {
                case BUFFER_OVERFLOW:
                    logger.warn("BUFFER_OVERFLOW error");
                    break;
                case BUFFER_UNDERFLOW:
                    if (netBuffer.limit() == netBuffer.capacity()) {
                        logger.warn("BUFFER_UNDERFLOW error");
                    } else {
                        if (logger.isDebugEnabled()) {
                            logger.debug("BUFFER_UNDERFLOW,continue read:" + netBuffer);
                        }
                        if (netBuffer.position() > 0) {
                            netBuffer.compact();
                        } else {
                            netBuffer.position(netBuffer.limit());
                            netBuffer.limit(netBuffer.capacity());
                        }
                    }
                    return;
                case CLOSED:
                    logger.warn("doUnWrap Result:" + result.getStatus());
                    closed = true;
                    break;
                default:
                    logger.warn("doUnWrap Result:" + result.getStatus());
            }
            result = sslEngine.unwrap(netBuffer, appBuffer);
        }
        netBuffer.compact();
    } catch (SSLException e) {
        throw new RuntimeException(e);
    }
}
 

private SSLEngineResult sslEngineWrap(ByteBuffer src) throws IOException {
    SSLEngineResult result = sslEngine.wrap(src, wrapBuffer.getWritable(sessionPacketBufferSize));
    if (result.getStatus() == Status.CLOSED) throw new ClosedChannelException();
    return result;
}
 

private SSLEngineResult.Status unwrapHandshake(NextFilter nextFilter) throws SSLException {
    // Prepare the net data for reading.
    if (inNetBuffer != null) {
        inNetBuffer.flip();
    }

    if (inNetBuffer == null || !inNetBuffer.hasRemaining()) {
        // Need more data.
        return SSLEngineResult.Status.BUFFER_UNDERFLOW;
    }

    SSLEngineResult res = unwrap();
    handshakeStatus = res.getHandshakeStatus();

    checkStatus(res);

    // If handshake finished, no data was produced, and the status is still
    // ok, try to unwrap more
    if (handshakeStatus == SSLEngineResult.HandshakeStatus.FINISHED && res.getStatus() == SSLEngineResult.Status.OK
            && inNetBuffer.hasRemaining()) {
        res = unwrap();

        // prepare to be written again
        if (inNetBuffer.hasRemaining()) {
            inNetBuffer.compact();
        } else {
            inNetBuffer = null;
        }

        renegotiateIfNeeded(nextFilter, res);
    } else {
        // prepare to be written again
        if (inNetBuffer.hasRemaining()) {
            inNetBuffer.compact();
        } else {
            inNetBuffer = null;
        }
    }

    return res.getStatus();
}
 

private void handshake0() throws SSLException {
	for (;;) {
		switch (handshakeStatus) {
		case FINISHED:
		case NOT_HANDSHAKING:
			handshakeComplete = true;
			return;
		case NEED_TASK:
			handshakeStatus = doTasks();
               break;
		case NEED_UNWRAP:
			SSLEngineResult.Status status = unwrapHandshake();
			if (status == SSLEngineResult.Status.BUFFER_UNDERFLOW && handshakeStatus != SSLEngineResult.HandshakeStatus.FINISHED || isInboundDone()) {
				// Need more data
				return;
			}
			break;
		case NEED_WRAP:
			// First make sure that the out buffer is completely empty.Since we cannot call wrap with data left on the buffer
               if (outNetBuffer != null && outNetBuffer.hasRemaining()) {
               	return;
               }
               SSLEngineResult result;
               createOutNetBuffer(0);

               for (;;) {
                   result = sslEngine.wrap(emptyBuffer.buf(), outNetBuffer.buf());
                   if (result.getStatus() == SSLEngineResult.Status.BUFFER_OVERFLOW) {
                       outNetBuffer.capacity(outNetBuffer.capacity() << 1);
                       outNetBuffer.limit(outNetBuffer.capacity());
                   } else {
                       break;
                   }
               }

               outNetBuffer.flip();
               handshakeStatus = result.getHandshakeStatus();
               writeNetBuffer();
               break;
		default:
			String msg = "Invalid handshaking state" + handshakeStatus + " while processing the Handshake for session.";
			throw new IllegalStateException(msg);
		}
	}
}
 

SSLEngineResult unwrap(ByteBuffer encryptedData) throws SSLException
{
  ByteBuffer allEncryptedData = _buffers.prependCached(encryptedData);
  _buffers.prepareForUnwrap(allEncryptedData);
  SSLEngineResult result = doUnwrap();
  debug("unwrap: doUnwrap result: " + result);
  allEncryptedData.position(result.bytesConsumed());
  ByteBuffer unprocessedEncryptedData = BufferUtils.slice(allEncryptedData);

  emitPlainData(result);

  switch (result.getStatus()) {
    case BUFFER_UNDERFLOW:
      _buffers.cache(unprocessedEncryptedData);
      break;
    case BUFFER_OVERFLOW:
      _buffers.grow(BufferType.IN_PLAIN);
      if (unprocessedEncryptedData == null) {
        throw new RuntimeException("Worker.unwrap had "
                + "buffer_overflow but all data was consumed!!");
      } else {
        unwrap(unprocessedEncryptedData);
      }
      break;
    case OK:
      if (unprocessedEncryptedData == null) {
        _buffers.clearCache();
      } else {
        _buffers.cache(unprocessedEncryptedData);
      }
      break;
    case CLOSED:

      break;
  }
  if (_buffers.isCacheEmpty() == false 
          && result.getStatus() == SSLEngineResult.Status.OK
          && result.bytesConsumed() > 0) {
    debug("Still data in cahce");
    result = unwrap(ByteBuffer.allocate(0));
  }
  return result;
}