Java Code Examples for javax.net.ssl.SSLEngine#wrap()

private void checkBufferOverflowOnUnWrap(SSLEngine wrappingEngine,
        SSLEngine unwrappingEngine)
        throws SSLException {
    String wrapperMode = wrappingEngine.getUseClientMode() ? "client"
            : "server";
    String unwrapperMode = unwrappingEngine.getUseClientMode() ? "client"
            : "server";
    if (wrapperMode.equals(unwrapperMode)) {
        throw new Error("Test error: both engines are in the same mode!");
    }
    System.out.println("================================================="
            + "===========");
    System.out.println("Testing SSLEngine buffer overflow"
            + " on unwrap by " + unwrapperMode);
    ByteBuffer app = ByteBuffer.wrap(MESSAGE.getBytes());
    ByteBuffer net = ByteBuffer
            .allocate(wrappingEngine.getSession().getPacketBufferSize());
    SSLEngineResult r = wrappingEngine.wrap(app, net);
    checkResult(r, SSLEngineResult.Status.OK);
    //Making app buffer size less than required by 1 byte.
    app = ByteBuffer.allocate(MESSAGE.length() - 1);
    net.flip();
    r = unwrappingEngine.unwrap(net, app);
    checkResult(r, SSLEngineResult.Status.BUFFER_OVERFLOW);
    System.out.println("Passed");
}
 

/**
 * Wraps data with the specified engine.
 *
 * @param engine        - SSLEngine that wraps data.
 * @param wrapper       - Set wrapper id, e.g. "server" of "client".
 *                        Used for logging only.
 * @param maxPacketSize - Max packet size to check that MFLN extension
 *                        works or zero for no check.
 * @param app           - Buffer with data to wrap.
 * @param wantedStatus  - Specifies expected result status of wrapping.
 * @param result        - Array which first element will be used to output
 *                        wrap result object.
 * @return - Buffer with wrapped data.
 * @throws SSLException - thrown on engine errors.
 */
public static ByteBuffer doWrap(SSLEngine engine, String wrapper,
                                int maxPacketSize, ByteBuffer app,
                                SSLEngineResult.Status wantedStatus,
                                SSLEngineResult[] result)
        throws SSLException {
    ByteBuffer net = ByteBuffer.allocate(engine.getSession()
            .getPacketBufferSize());
    SSLEngineResult r = engine.wrap(app, net);
    net.flip();
    int length = net.remaining();
    System.out.println(wrapper + " wrapped " + length + " bytes.");
    System.out.println(wrapper + " handshake status is "
            + engine.getHandshakeStatus());
    if (maxPacketSize < length && maxPacketSize != 0) {
        throw new AssertionError("Handshake wrapped net buffer length "
                + length + " exceeds maximum packet size "
                + maxPacketSize);
    }
    checkResult(r, wantedStatus);
    if (result != null && result.length > 0) {
        result[0] = r;
    }
    return net;
}
 

@Test
public void testNeededDstCapacityIsCorrectlyCalculated() throws Exception {
    clientSslCtx = SslContextBuilder.forClient()
            .trustManager(InsecureTrustManagerFactory.INSTANCE)
            .sslProvider(sslClientProvider())
            .build();
    SelfSignedCertificate ssc = new SelfSignedCertificate();
    serverSslCtx = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey())
            .sslProvider(sslServerProvider())
            .build();
    SSLEngine clientEngine = null;
    SSLEngine serverEngine = null;
    try {
        clientEngine = clientSslCtx.newEngine(UnpooledByteBufAllocator.DEFAULT);
        serverEngine = serverSslCtx.newEngine(UnpooledByteBufAllocator.DEFAULT);
        handshake(clientEngine, serverEngine);

        ByteBuffer src = allocateBuffer(1024);
        ByteBuffer src2 = src.duplicate();

        ByteBuffer dst = allocateBuffer(src.capacity()
                + ((ReferenceCountedOpenSslEngine) clientEngine).maxWrapOverhead());

        SSLEngineResult result = clientEngine.wrap(new ByteBuffer[] { src, src2 }, dst);
        assertEquals(SSLEngineResult.Status.BUFFER_OVERFLOW, result.getStatus());
        assertEquals(0, src.position());
        assertEquals(0, src2.position());
        assertEquals(0, dst.position());
        assertEquals(0, result.bytesConsumed());
        assertEquals(0, result.bytesProduced());
    } finally {
        cleanupClientSslEngine(clientEngine);
        cleanupServerSslEngine(serverEngine);
    }
}
 

/**
 * Makes sure the specified {@link SslContext} is configured properly. If configured as client context or
 * key store password is not given to key store when {@link SslContext} was created using
 * {@link KeyManagerFactory}, the validation will fail and an {@link IllegalStateException} will be raised.
 */
private static SslContext validateSslContext(SslContext sslContext) {
    if (!sslContext.isServer()) {
        throw new IllegalArgumentException("sslContext: " + sslContext + " (expected: server context)");
    }

    SSLEngine serverEngine = null;
    SSLEngine clientEngine = null;

    try {
        serverEngine = sslContext.newEngine(ByteBufAllocator.DEFAULT);
        serverEngine.setUseClientMode(false);
        serverEngine.setNeedClientAuth(false);

        final SslContext sslContextClient =
                buildSslContext(SslContextBuilder::forClient, ImmutableList.of());
        clientEngine = sslContextClient.newEngine(ByteBufAllocator.DEFAULT);
        clientEngine.setUseClientMode(true);

        final ByteBuffer appBuf = ByteBuffer.allocate(clientEngine.getSession().getApplicationBufferSize());
        final ByteBuffer packetBuf = ByteBuffer.allocate(clientEngine.getSession().getPacketBufferSize());

        clientEngine.wrap(appBuf, packetBuf);
        appBuf.clear();
        packetBuf.flip();
        serverEngine.unwrap(packetBuf, appBuf);
    } catch (SSLException e) {
        throw new IllegalStateException("failed to validate SSL/TLS configuration: " + e.getMessage(), e);
    } finally {
        ReferenceCountUtil.release(serverEngine);
        ReferenceCountUtil.release(clientEngine);
    }

    return sslContext;
}
 

@Test
public void testSSLEngineUnwrapNoSslRecord() throws Exception {
    clientSslCtx = SslContextBuilder
            .forClient()
            .sslProvider(sslClientProvider())
            .build();
    SSLEngine client = clientSslCtx.newEngine(UnpooledByteBufAllocator.DEFAULT);

    try {
        ByteBuffer src = allocateBuffer(client.getSession().getApplicationBufferSize());
        ByteBuffer dst = allocateBuffer(client.getSession().getPacketBufferSize());
        ByteBuffer empty = allocateBuffer(0);

        SSLEngineResult clientResult = client.wrap(empty, dst);
        assertEquals(SSLEngineResult.Status.OK, clientResult.getStatus());
        assertEquals(SSLEngineResult.HandshakeStatus.NEED_UNWRAP, clientResult.getHandshakeStatus());

        try {
            client.unwrap(src, dst);
            fail();
        } catch (SSLException expected) {
            // expected
        }
    } finally {
        cleanupClientSslEngine(client);
    }
}
 

/**
 * DTLS encrypt byte buffer
 */
boolean diameterDTLSEncryptBuffer(SSLEngine engine, ByteBuffer source, ByteBuffer appNet) throws Exception {

    //printHex("Received application data for Encrypt", source);
    
    List<DatagramPacket> packets = new ArrayList<>();
    SSLEngineResult r = engine.wrap(source, appNet);
    appNet.flip();

    SSLEngineResult.Status rs = r.getStatus();
    if (rs == SSLEngineResult.Status.BUFFER_OVERFLOW) {
        // the client maximum fragment size config does not work?
        logger.warn("Buffer overflow: " + "incorrect server maximum fragment size");
        return false;
    } else if (rs == SSLEngineResult.Status.BUFFER_UNDERFLOW) {
        // unlikely
        logger.warn("Buffer underflow during wraping");
        return false;
    } else if (rs == SSLEngineResult.Status.CLOSED) {
        logger.warn("SSLEngine has closed");
        return false;
    } else if (rs == SSLEngineResult.Status.OK) {
        // OK
    } else {
        logger.warn("Can't reach here, result is " + rs);
        return false;
    }

    // SSLEngineResult.Status.OK:
    // printHex("Produced application data by Encrypt", appNet);
    return true;
}
 

protected void handshake(SSLEngine clientEngine, SSLEngine serverEngine) throws SSLException {
    ByteBuffer cTOs = allocateBuffer(clientEngine.getSession().getPacketBufferSize());
    ByteBuffer sTOc = allocateBuffer(serverEngine.getSession().getPacketBufferSize());

    ByteBuffer serverAppReadBuffer = allocateBuffer(
            serverEngine.getSession().getApplicationBufferSize());
    ByteBuffer clientAppReadBuffer = allocateBuffer(
            clientEngine.getSession().getApplicationBufferSize());

    clientEngine.beginHandshake();
    serverEngine.beginHandshake();

    ByteBuffer empty = allocateBuffer(0);

    SSLEngineResult clientResult;
    SSLEngineResult serverResult;

    boolean clientHandshakeFinished = false;
    boolean serverHandshakeFinished = false;

    do {
        int cTOsPos = cTOs.position();
        int sTOcPos = sTOc.position();

        if (!clientHandshakeFinished) {
            clientResult = clientEngine.wrap(empty, cTOs);
            runDelegatedTasks(clientResult, clientEngine);
            assertEquals(empty.remaining(), clientResult.bytesConsumed());
            assertEquals(cTOs.position() - cTOsPos,  clientResult.bytesProduced());

            if (isHandshakeFinished(clientResult)) {
                clientHandshakeFinished = true;
            }
        }

        if (!serverHandshakeFinished) {
            serverResult = serverEngine.wrap(empty, sTOc);
            runDelegatedTasks(serverResult, serverEngine);
            assertEquals(empty.remaining(), serverResult.bytesConsumed());
            assertEquals(sTOc.position() - sTOcPos, serverResult.bytesProduced());

            if (isHandshakeFinished(serverResult)) {
                serverHandshakeFinished = true;
            }
        }

        cTOs.flip();
        sTOc.flip();

        cTOsPos = cTOs.position();
        sTOcPos = sTOc.position();

        if (!clientHandshakeFinished) {
            int clientAppReadBufferPos = clientAppReadBuffer.position();
            clientResult = clientEngine.unwrap(sTOc, clientAppReadBuffer);

            runDelegatedTasks(clientResult, clientEngine);
            assertEquals(sTOc.position() - sTOcPos, clientResult.bytesConsumed());
            assertEquals(clientAppReadBuffer.position() - clientAppReadBufferPos, clientResult.bytesProduced());

            if (isHandshakeFinished(clientResult)) {
                clientHandshakeFinished = true;
            }
        } else {
            assertFalse(sTOc.hasRemaining());
        }

        if (!serverHandshakeFinished) {
            int serverAppReadBufferPos = serverAppReadBuffer.position();
            serverResult = serverEngine.unwrap(cTOs, serverAppReadBuffer);
            runDelegatedTasks(serverResult, serverEngine);
            assertEquals(cTOs.position() - cTOsPos, serverResult.bytesConsumed());
            assertEquals(serverAppReadBuffer.position() - serverAppReadBufferPos, serverResult.bytesProduced());

            if (isHandshakeFinished(serverResult)) {
                serverHandshakeFinished = true;
            }
        } else {
            assertFalse(cTOs.hasRemaining());
        }

        sTOc.compact();
        cTOs.compact();
    } while (!clientHandshakeFinished || !serverHandshakeFinished);
}
 

public static void run() throws Exception {
    SSLEngine[][] enginesToTest = getSSLEnginesToTest();

    for (SSLEngine[] engineToTest : enginesToTest) {

        SSLEngine clientSSLEngine = engineToTest[0];
        SSLEngine serverSSLEngine = engineToTest[1];

        // SSLEngine code based on RedhandshakeFinished.java

        boolean dataDone = false;

        ByteBuffer clientOut = null;
        ByteBuffer clientIn = null;
        ByteBuffer serverOut = null;
        ByteBuffer serverIn = null;
        ByteBuffer cTOs;
        ByteBuffer sTOc;

        SSLSession session = clientSSLEngine.getSession();
        int appBufferMax = session.getApplicationBufferSize();
        int netBufferMax = session.getPacketBufferSize();

        clientIn = ByteBuffer.allocate(appBufferMax + 50);
        serverIn = ByteBuffer.allocate(appBufferMax + 50);

        cTOs = ByteBuffer.allocateDirect(netBufferMax);
        sTOc = ByteBuffer.allocateDirect(netBufferMax);

        clientOut = ByteBuffer.wrap(
                "Hi Server, I'm Client".getBytes());
        serverOut = ByteBuffer.wrap(
                "Hello Client, I'm Server".getBytes());

        SSLEngineResult clientResult;
        SSLEngineResult serverResult;

        while (!dataDone) {
            clientResult = clientSSLEngine.wrap(clientOut, cTOs);
            runDelegatedTasks(clientResult, clientSSLEngine);
            serverResult = serverSSLEngine.wrap(serverOut, sTOc);
            runDelegatedTasks(serverResult, serverSSLEngine);
            cTOs.flip();
            sTOc.flip();

            if (enableDebug) {
                System.out.println("Client -> Network");
                printTlsNetworkPacket("", cTOs);
                System.out.println("");
                System.out.println("Server -> Network");
                printTlsNetworkPacket("", sTOc);
                System.out.println("");
            }

            clientResult = clientSSLEngine.unwrap(sTOc, clientIn);
            runDelegatedTasks(clientResult, clientSSLEngine);
            serverResult = serverSSLEngine.unwrap(cTOs, serverIn);
            runDelegatedTasks(serverResult, serverSSLEngine);

            cTOs.compact();
            sTOc.compact();

            if (!dataDone &&
                    (clientOut.limit() == serverIn.position()) &&
                    (serverOut.limit() == clientIn.position())) {
                checkTransfer(serverOut, clientIn);
                checkTransfer(clientOut, serverIn);
                dataDone = true;
            }
        }
    }
}
 

@Test
public void testBufferUnderFlow() throws Exception {
    SelfSignedCertificate cert = new SelfSignedCertificate();

    clientSslCtx = SslContextBuilder
            .forClient()
            .trustManager(cert.cert())
            .sslProvider(sslClientProvider())
            .build();
    SSLEngine client = clientSslCtx.newEngine(UnpooledByteBufAllocator.DEFAULT);

    serverSslCtx = SslContextBuilder
            .forServer(cert.certificate(), cert.privateKey())
            .sslProvider(sslServerProvider())
            .build();
    SSLEngine server = serverSslCtx.newEngine(UnpooledByteBufAllocator.DEFAULT);

    try {
        ByteBuffer plainClient = allocateBuffer(1024);
        plainClient.limit(plainClient.capacity());

        ByteBuffer encClientToServer = allocateBuffer(client.getSession().getPacketBufferSize());
        ByteBuffer plainServer = allocateBuffer(server.getSession().getApplicationBufferSize());

        handshake(client, server);

        SSLEngineResult result = client.wrap(plainClient, encClientToServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(result.bytesConsumed(), plainClient.capacity());

        // Flip so we can read it.
        encClientToServer.flip();
        int remaining = encClientToServer.remaining();

        // We limit the buffer so we have less then the header to read, this should result in an BUFFER_UNDERFLOW.
        encClientToServer.limit(SSL_RECORD_HEADER_LENGTH - 1);
        result = server.unwrap(encClientToServer, plainServer);
        assertResultIsBufferUnderflow(result);

        // We limit the buffer so we can read the header but not the rest, this should result in an
        // BUFFER_UNDERFLOW.
        encClientToServer.limit(SSL_RECORD_HEADER_LENGTH);
        result = server.unwrap(encClientToServer, plainServer);
        assertResultIsBufferUnderflow(result);

        // We limit the buffer so we can read the header and partly the rest, this should result in an
        // BUFFER_UNDERFLOW.
        encClientToServer.limit(SSL_RECORD_HEADER_LENGTH  + remaining - 1 - SSL_RECORD_HEADER_LENGTH);
        result = server.unwrap(encClientToServer, plainServer);
        assertResultIsBufferUnderflow(result);

        // Reset limit so we can read the full record.
        encClientToServer.limit(remaining);

        result = server.unwrap(encClientToServer, plainServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(result.bytesConsumed(), remaining);
        assertTrue(result.bytesProduced() > 0);
    } finally {
        cert.delete();
        cleanupClientSslEngine(client);
        cleanupServerSslEngine(server);
    }
}
 

public static void run() throws Exception {
    SSLEngine[][] enginesToTest = getSSLEnginesToTest();

    for (SSLEngine[] engineToTest : enginesToTest) {

        SSLEngine clientSSLEngine = engineToTest[0];
        SSLEngine serverSSLEngine = engineToTest[1];

        // SSLEngine code based on RedhandshakeFinished.java

        boolean dataDone = false;

        ByteBuffer clientOut = null;
        ByteBuffer clientIn = null;
        ByteBuffer serverOut = null;
        ByteBuffer serverIn = null;
        ByteBuffer cTOs;
        ByteBuffer sTOc;

        SSLSession session = clientSSLEngine.getSession();
        int appBufferMax = session.getApplicationBufferSize();
        int netBufferMax = session.getPacketBufferSize();

        clientIn = ByteBuffer.allocate(appBufferMax + 50);
        serverIn = ByteBuffer.allocate(appBufferMax + 50);

        cTOs = ByteBuffer.allocateDirect(netBufferMax);
        sTOc = ByteBuffer.allocateDirect(netBufferMax);

        clientOut = ByteBuffer.wrap(
                "Hi Server, I'm Client".getBytes());
        serverOut = ByteBuffer.wrap(
                "Hello Client, I'm Server".getBytes());

        SSLEngineResult clientResult;
        SSLEngineResult serverResult;

        while (!dataDone) {
            clientResult = clientSSLEngine.wrap(clientOut, cTOs);
            runDelegatedTasks(clientResult, clientSSLEngine);
            serverResult = serverSSLEngine.wrap(serverOut, sTOc);
            runDelegatedTasks(serverResult, serverSSLEngine);
            cTOs.flip();
            sTOc.flip();

            if (enableDebug) {
                System.out.println("Client -> Network");
                printTlsNetworkPacket("", cTOs);
                System.out.println("");
                System.out.println("Server -> Network");
                printTlsNetworkPacket("", sTOc);
                System.out.println("");
            }

            clientResult = clientSSLEngine.unwrap(sTOc, clientIn);
            runDelegatedTasks(clientResult, clientSSLEngine);
            serverResult = serverSSLEngine.unwrap(cTOs, serverIn);
            runDelegatedTasks(serverResult, serverSSLEngine);

            cTOs.compact();
            sTOc.compact();

            if (!dataDone &&
                    (clientOut.limit() == serverIn.position()) &&
                    (serverOut.limit() == clientIn.position())) {
                checkTransfer(serverOut, clientIn);
                checkTransfer(clientOut, serverIn);
                dataDone = true;
            }
        }
    }
}
 

public static void run() throws Exception {
    SSLEngine[][] enginesToTest = getSSLEnginesToTest();

    for (SSLEngine[] engineToTest : enginesToTest) {

        SSLEngine clientSSLEngine = engineToTest[0];
        SSLEngine serverSSLEngine = engineToTest[1];

        // SSLEngine code based on RedhandshakeFinished.java

        boolean dataDone = false;

        ByteBuffer clientOut = null;
        ByteBuffer clientIn = null;
        ByteBuffer serverOut = null;
        ByteBuffer serverIn = null;
        ByteBuffer cTOs;
        ByteBuffer sTOc;

        SSLSession session = clientSSLEngine.getSession();
        int appBufferMax = session.getApplicationBufferSize();
        int netBufferMax = session.getPacketBufferSize();

        clientIn = ByteBuffer.allocate(appBufferMax + 50);
        serverIn = ByteBuffer.allocate(appBufferMax + 50);

        cTOs = ByteBuffer.allocateDirect(netBufferMax);
        sTOc = ByteBuffer.allocateDirect(netBufferMax);

        clientOut = ByteBuffer.wrap(
                "Hi Server, I'm Client".getBytes());
        serverOut = ByteBuffer.wrap(
                "Hello Client, I'm Server".getBytes());

        SSLEngineResult clientResult;
        SSLEngineResult serverResult;

        while (!dataDone) {
            clientResult = clientSSLEngine.wrap(clientOut, cTOs);
            runDelegatedTasks(clientResult, clientSSLEngine);
            serverResult = serverSSLEngine.wrap(serverOut, sTOc);
            runDelegatedTasks(serverResult, serverSSLEngine);
            cTOs.flip();
            sTOc.flip();

            if (enableDebug) {
                System.out.println("Client -> Network");
                printTlsNetworkPacket("", cTOs);
                System.out.println("");
                System.out.println("Server -> Network");
                printTlsNetworkPacket("", sTOc);
                System.out.println("");
            }

            clientResult = clientSSLEngine.unwrap(sTOc, clientIn);
            runDelegatedTasks(clientResult, clientSSLEngine);
            serverResult = serverSSLEngine.unwrap(cTOs, serverIn);
            runDelegatedTasks(serverResult, serverSSLEngine);

            cTOs.compact();
            sTOc.compact();

            if (!dataDone &&
                    (clientOut.limit() == serverIn.position()) &&
                    (serverOut.limit() == clientIn.position())) {
                checkTransfer(serverOut, clientIn);
                checkTransfer(clientOut, serverIn);
                dataDone = true;
            }
        }
    }
}
 

@Test
public void testUnwrapBehavior() throws Exception {
    SelfSignedCertificate cert = new SelfSignedCertificate();

    clientSslCtx = SslContextBuilder
            .forClient()
            .trustManager(cert.cert())
            .sslProvider(sslClientProvider())
            .build();
    SSLEngine client = clientSslCtx.newEngine(UnpooledByteBufAllocator.DEFAULT);

    serverSslCtx = SslContextBuilder
            .forServer(cert.certificate(), cert.privateKey())
            .sslProvider(sslServerProvider())
            .build();
    SSLEngine server = serverSslCtx.newEngine(UnpooledByteBufAllocator.DEFAULT);

    byte[] bytes = "Hello World".getBytes(CharsetUtil.US_ASCII);

    try {
        ByteBuffer plainClientOut = allocateBuffer(client.getSession().getApplicationBufferSize());
        ByteBuffer encryptedClientToServer = allocateBuffer(server.getSession().getPacketBufferSize() * 2);
        ByteBuffer plainServerIn = allocateBuffer(server.getSession().getApplicationBufferSize());

        handshake(client, server);

        // create two TLS frames

        // first frame
        plainClientOut.put(bytes, 0, 5);
        plainClientOut.flip();

        SSLEngineResult result = client.wrap(plainClientOut, encryptedClientToServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(5, result.bytesConsumed());
        assertTrue(result.bytesProduced() > 0);

        assertFalse(plainClientOut.hasRemaining());

        // second frame
        plainClientOut.clear();
        plainClientOut.put(bytes, 5, 6);
        plainClientOut.flip();

        result = client.wrap(plainClientOut, encryptedClientToServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(6, result.bytesConsumed());
        assertTrue(result.bytesProduced() > 0);

        // send over to server
        encryptedClientToServer.flip();

        // try with too small output buffer first (to check BUFFER_OVERFLOW case)
        int remaining = encryptedClientToServer.remaining();
        ByteBuffer small = allocateBuffer(3);
        result = server.unwrap(encryptedClientToServer, small);
        assertEquals(SSLEngineResult.Status.BUFFER_OVERFLOW, result.getStatus());
        assertEquals(remaining, encryptedClientToServer.remaining());

        // now with big enough buffer
        result = server.unwrap(encryptedClientToServer, plainServerIn);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());

        assertEquals(5, result.bytesProduced());
        assertTrue(encryptedClientToServer.hasRemaining());

        result = server.unwrap(encryptedClientToServer, plainServerIn);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(6, result.bytesProduced());
        assertFalse(encryptedClientToServer.hasRemaining());

        plainServerIn.flip();

        assertEquals(ByteBuffer.wrap(bytes), plainServerIn);
    } finally {
        cleanupClientSslEngine(client);
        cleanupServerSslEngine(server);
        cert.delete();
    }
}
 

public static void run() throws Exception {
    SSLEngine[][] enginesToTest = getSSLEnginesToTest();

    for (SSLEngine[] engineToTest : enginesToTest) {

        SSLEngine clientSSLEngine = engineToTest[0];
        SSLEngine serverSSLEngine = engineToTest[1];

        // SSLEngine code based on RedhandshakeFinished.java

        boolean dataDone = false;

        ByteBuffer clientOut = null;
        ByteBuffer clientIn = null;
        ByteBuffer serverOut = null;
        ByteBuffer serverIn = null;
        ByteBuffer cTOs;
        ByteBuffer sTOc;

        SSLSession session = clientSSLEngine.getSession();
        int appBufferMax = session.getApplicationBufferSize();
        int netBufferMax = session.getPacketBufferSize();

        clientIn = ByteBuffer.allocate(appBufferMax + 50);
        serverIn = ByteBuffer.allocate(appBufferMax + 50);

        cTOs = ByteBuffer.allocateDirect(netBufferMax);
        sTOc = ByteBuffer.allocateDirect(netBufferMax);

        clientOut = ByteBuffer.wrap(
                "Hi Server, I'm Client".getBytes());
        serverOut = ByteBuffer.wrap(
                "Hello Client, I'm Server".getBytes());

        SSLEngineResult clientResult;
        SSLEngineResult serverResult;

        while (!dataDone) {
            clientResult = clientSSLEngine.wrap(clientOut, cTOs);
            runDelegatedTasks(clientResult, clientSSLEngine);
            serverResult = serverSSLEngine.wrap(serverOut, sTOc);
            runDelegatedTasks(serverResult, serverSSLEngine);
            cTOs.flip();
            sTOc.flip();

            if (enableDebug) {
                System.out.println("Client -> Network");
                printTlsNetworkPacket("", cTOs);
                System.out.println("");
                System.out.println("Server -> Network");
                printTlsNetworkPacket("", sTOc);
                System.out.println("");
            }

            clientResult = clientSSLEngine.unwrap(sTOc, clientIn);
            runDelegatedTasks(clientResult, clientSSLEngine);
            serverResult = serverSSLEngine.unwrap(cTOs, serverIn);
            runDelegatedTasks(serverResult, serverSSLEngine);

            cTOs.compact();
            sTOc.compact();

            if (!dataDone &&
                    (clientOut.limit() == serverIn.position()) &&
                    (serverOut.limit() == clientIn.position())) {
                checkTransfer(serverOut, clientIn);
                checkTransfer(clientOut, serverIn);
                dataDone = true;
            }
        }
    }
}
 

@Test
public void testBufferUnderFlowAvoidedIfJDKCompatabilityModeOff() throws Exception {
    SelfSignedCertificate cert = new SelfSignedCertificate();

    clientSslCtx = SslContextBuilder
            .forClient()
            .trustManager(cert.cert())
            .sslProvider(sslClientProvider())
            .build();
    SSLEngine client = clientSslCtx.newHandler(UnpooledByteBufAllocator.DEFAULT).engine();

    serverSslCtx = SslContextBuilder
            .forServer(cert.certificate(), cert.privateKey())
            .sslProvider(sslServerProvider())
            .build();
    SSLEngine server = serverSslCtx.newHandler(UnpooledByteBufAllocator.DEFAULT).engine();

    try {
        ByteBuffer plainClient = allocateBuffer(1024);
        plainClient.limit(plainClient.capacity());

        ByteBuffer encClientToServer = allocateBuffer(client.getSession().getPacketBufferSize());
        ByteBuffer plainServer = allocateBuffer(server.getSession().getApplicationBufferSize());

        handshake(client, server);

        SSLEngineResult result = client.wrap(plainClient, encClientToServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(result.bytesConsumed(), plainClient.capacity());

        // Flip so we can read it.
        encClientToServer.flip();
        int remaining = encClientToServer.remaining();

        // We limit the buffer so we have less then the header to read, this should result in an BUFFER_UNDERFLOW.
        encClientToServer.limit(SslUtils.SSL_RECORD_HEADER_LENGTH - 1);
        result = server.unwrap(encClientToServer, plainServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(SslUtils.SSL_RECORD_HEADER_LENGTH - 1, result.bytesConsumed());
        assertEquals(0, result.bytesProduced());
        remaining -= result.bytesConsumed();

        // We limit the buffer so we can read the header but not the rest, this should result in an
        // BUFFER_UNDERFLOW.
        encClientToServer.limit(SslUtils.SSL_RECORD_HEADER_LENGTH);
        result = server.unwrap(encClientToServer, plainServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(1, result.bytesConsumed());
        assertEquals(0, result.bytesProduced());
        remaining -= result.bytesConsumed();

        // We limit the buffer so we can read the header and partly the rest, this should result in an
        // BUFFER_UNDERFLOW.
        encClientToServer.limit(
                SslUtils.SSL_RECORD_HEADER_LENGTH  + remaining - 1 - SslUtils.SSL_RECORD_HEADER_LENGTH);
        result = server.unwrap(encClientToServer, plainServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(encClientToServer.limit() - SslUtils.SSL_RECORD_HEADER_LENGTH, result.bytesConsumed());
        assertEquals(0, result.bytesProduced());
        remaining -= result.bytesConsumed();

        // Reset limit so we can read the full record.
        encClientToServer.limit(remaining);
        assertEquals(0, encClientToServer.remaining());
        result = server.unwrap(encClientToServer, plainServer);
        assertEquals(SSLEngineResult.Status.BUFFER_UNDERFLOW, result.getStatus());
        assertEquals(0, result.bytesConsumed());
        assertEquals(0, result.bytesProduced());

        encClientToServer.position(0);
        result = server.unwrap(encClientToServer, plainServer);
        assertEquals(SSLEngineResult.Status.OK, result.getStatus());
        assertEquals(remaining, result.bytesConsumed());
        assertEquals(0, result.bytesProduced());
    } finally {
        cert.delete();
        cleanupClientSslEngine(client);
        cleanupServerSslEngine(server);
    }
}
 

private void doHandshake(SocketChannel socketChannel, SSLEngine sslEngine, ByteBuffer myNetData, ByteBuffer peerNetData)
        throws IOException {

    // Create byte buffers to use for holding application data
    int appBufferSize = sslEngine.getSession().getApplicationBufferSize();
    ByteBuffer myAppData = ByteBuffer.allocate(appBufferSize);
    ByteBuffer peerAppData = ByteBuffer.allocate(appBufferSize);

    SSLEngineResult.HandshakeStatus hs = sslEngine.getHandshakeStatus();

    // Process handshaking message
    while (hs != SSLEngineResult.HandshakeStatus.FINISHED && hs != SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) {

        switch (hs) {

            case NEED_UNWRAP:
                // Receive handshaking data from peer
                if (socketChannel.read(peerNetData) < 0) {
                    throw new SocketException("Channel closed");
                }

                // Process incoming handshaking data
                peerNetData.flip();
                SSLEngineResult res = sslEngine.unwrap(peerNetData, peerAppData);
                peerNetData.compact();
                hs = res.getHandshakeStatus();

                // Check status
                switch (res.getStatus()) {
                    case OK:
                        // Handle OK status
                        break;
                    case BUFFER_OVERFLOW:
                        peerAppData = enlargeBuffer(peerNetData, peerAppData);
                        break;

                    case BUFFER_UNDERFLOW:

                        break;
                }
                break;

            case NEED_WRAP:
                // Empty the local network packet buffer.
                myNetData.clear();

                // Generate handshaking data
                res = sslEngine.wrap(myAppData, myNetData);
                hs = res.getHandshakeStatus();

                // Check status
                switch (res.getStatus()) {
                    case OK:
                        myNetData.flip();

                        // Send the handshaking data to peer
                        while (myNetData.hasRemaining()) {
                            if (socketChannel.write(myNetData) < 0) {
                                // Handle closed channel
                            }
                        }
                        break;
                    case BUFFER_OVERFLOW:
                        myNetData = enlargeBuffer(myAppData, myNetData);
                        break;

                    case BUFFER_UNDERFLOW:
                        break;
                    case CLOSED:

                        if (sslEngine.isOutboundDone()) {
                            return;
                        } else {
                            sslEngine.closeOutbound();
                            hs = sslEngine.getHandshakeStatus();
                            break;
                        }

                    default:
                        throw new IOException("Cannot wrap data: " + res.getStatus());

                }
                break;

            case NEED_TASK:
                Runnable task;
                while ((task = sslEngine.getDelegatedTask()) != null) {
                    task.run();
                }
                hs = sslEngine.getHandshakeStatus();
                break;

            case FINISHED:
                return;
        }
    }
}
 

/**
 * Perform the handshaking step of the TLS connection. We use the `sslEngine' along with the `channel' to exchange messages with the server to setup an
 * encrypted channel.
 * 
 * @param sslEngine
 *            {@link SSLEngine}
 * @param channel
 *            {@link AsynchronousSocketChannel}
 * @throws SSLException
 *             in case of handshake error
 */
private static void performTlsHandshake(SSLEngine sslEngine, AsynchronousSocketChannel channel) throws SSLException {
    sslEngine.beginHandshake();
    HandshakeStatus handshakeStatus = sslEngine.getHandshakeStatus();

    // Create byte buffers to use for holding application data
    int packetBufferSize = sslEngine.getSession().getPacketBufferSize();
    ByteBuffer myNetData = ByteBuffer.allocate(packetBufferSize);
    ByteBuffer peerNetData = ByteBuffer.allocate(packetBufferSize);
    int appBufferSize = sslEngine.getSession().getApplicationBufferSize();
    ByteBuffer myAppData = ByteBuffer.allocate(appBufferSize);
    ByteBuffer peerAppData = ByteBuffer.allocate(appBufferSize);

    SSLEngineResult res = null;

    while (handshakeStatus != HandshakeStatus.FINISHED && handshakeStatus != HandshakeStatus.NOT_HANDSHAKING) {
        switch (handshakeStatus) {
            case NEED_WRAP:
                myNetData.clear();
                res = sslEngine.wrap(myAppData, myNetData);
                handshakeStatus = res.getHandshakeStatus();
                switch (res.getStatus()) {
                    case OK:
                        myNetData.flip();
                        write(channel, myNetData);
                        break;
                    case BUFFER_OVERFLOW:
                    case BUFFER_UNDERFLOW:
                    case CLOSED:
                        throw new CJCommunicationsException("Unacceptable SSLEngine result: " + res);
                }
                break;
            case NEED_UNWRAP:
                peerNetData.flip(); // Process incoming handshaking data
                res = sslEngine.unwrap(peerNetData, peerAppData);
                handshakeStatus = res.getHandshakeStatus();
                switch (res.getStatus()) {
                    case OK:
                        peerNetData.compact();
                        break;
                    case BUFFER_OVERFLOW:
                        // Check if we need to enlarge the peer application data buffer.
                        final int newPeerAppDataSize = sslEngine.getSession().getApplicationBufferSize();
                        if (newPeerAppDataSize > peerAppData.capacity()) {
                            // enlarge the peer application data buffer
                            ByteBuffer newPeerAppData = ByteBuffer.allocate(newPeerAppDataSize);
                            newPeerAppData.put(peerAppData);
                            newPeerAppData.flip();
                            peerAppData = newPeerAppData;
                        } else {
                            peerAppData.compact();
                        }
                        break;
                    case BUFFER_UNDERFLOW:
                        // Check if we need to enlarge the peer network packet buffer
                        final int newPeerNetDataSize = sslEngine.getSession().getPacketBufferSize();
                        if (newPeerNetDataSize > peerNetData.capacity()) {
                            // enlarge the peer network packet buffer
                            ByteBuffer newPeerNetData = ByteBuffer.allocate(newPeerNetDataSize);
                            newPeerNetData.put(peerNetData);
                            newPeerNetData.flip();
                            peerNetData = newPeerNetData;
                        } else {
                            peerNetData.compact();
                        }
                        // obtain more inbound network data and then retry the operation
                        if (read(channel, peerNetData) < 0) {
                            throw new CJCommunicationsException("Server does not provide enough data to proceed with SSL handshake.");
                        }
                        break;
                    case CLOSED:
                        throw new CJCommunicationsException("Unacceptable SSLEngine result: " + res);
                }
                break;

            case NEED_TASK:
                sslEngine.getDelegatedTask().run();
                handshakeStatus = sslEngine.getHandshakeStatus();
                break;
            case FINISHED:
            case NOT_HANDSHAKING:
                break;
        }
    }
}
 

@Test
public void testMultipleRecordsInOneBufferWithNonZeroPositionJDKCompatabilityModeOff() throws Exception {
    SelfSignedCertificate cert = new SelfSignedCertificate();

    clientSslCtx = SslContextBuilder
            .forClient()
            .trustManager(cert.cert())
            .sslProvider(sslClientProvider())
            .build();
    SSLEngine client = clientSslCtx.newHandler(UnpooledByteBufAllocator.DEFAULT).engine();

    serverSslCtx = SslContextBuilder
            .forServer(cert.certificate(), cert.privateKey())
            .sslProvider(sslServerProvider())
            .build();
    SSLEngine server = serverSslCtx.newHandler(UnpooledByteBufAllocator.DEFAULT).engine();

    try {
        // Choose buffer size small enough that we can put multiple buffers into one buffer and pass it into the
        // unwrap call without exceed MAX_ENCRYPTED_PACKET_LENGTH.
        final int plainClientOutLen = 1024;
        ByteBuffer plainClientOut = allocateBuffer(plainClientOutLen);
        ByteBuffer plainServerOut = allocateBuffer(server.getSession().getApplicationBufferSize());

        ByteBuffer encClientToServer = allocateBuffer(client.getSession().getPacketBufferSize());

        int positionOffset = 1;
        // We need to be able to hold 2 records + positionOffset
        ByteBuffer combinedEncClientToServer = allocateBuffer(
                encClientToServer.capacity() * 2 + positionOffset);
        combinedEncClientToServer.position(positionOffset);

        handshake(client, server);

        plainClientOut.limit(plainClientOut.capacity());
        SSLEngineResult result = client.wrap(plainClientOut, encClientToServer);
        assertEquals(plainClientOut.capacity(), result.bytesConsumed());
        assertTrue(result.bytesProduced() > 0);

        encClientToServer.flip();

        // Copy the first record into the combined buffer
        combinedEncClientToServer.put(encClientToServer);

        plainClientOut.clear();
        encClientToServer.clear();

        result = client.wrap(plainClientOut, encClientToServer);
        assertEquals(plainClientOut.capacity(), result.bytesConsumed());
        assertTrue(result.bytesProduced() > 0);

        encClientToServer.flip();

        // Copy the first record into the combined buffer
        combinedEncClientToServer.put(encClientToServer);

        encClientToServer.clear();

        combinedEncClientToServer.flip();
        combinedEncClientToServer.position(positionOffset);

        // Make sure the limit takes positionOffset into account to the content we are looking at is correct.
        combinedEncClientToServer.limit(
                combinedEncClientToServer.limit() - positionOffset);
        final int combinedEncClientToServerLen = combinedEncClientToServer.remaining();

        result = server.unwrap(combinedEncClientToServer, plainServerOut);
        assertEquals(0, combinedEncClientToServer.remaining());
        assertEquals(combinedEncClientToServerLen, result.bytesConsumed());
        assertEquals(plainClientOutLen, result.bytesProduced());
    } finally {
        cert.delete();
        cleanupClientSslEngine(client);
        cleanupServerSslEngine(server);
    }
}
 

private SSLEngineResult wrap(ByteBufAllocator alloc, SSLEngine engine, ByteBuf in, ByteBuf out)
        throws SSLException {
    ByteBuf newDirectIn = null;
    try {
        int readerIndex = in.readerIndex();
        int readableBytes = in.readableBytes();

        // We will call SslEngine.wrap(ByteBuffer[], ByteBuffer) to allow efficient handling of
        // CompositeByteBuf without force an extra memory copy when CompositeByteBuffer.nioBuffer() is called.
        final ByteBuffer[] in0;
        if (in.isDirect() || !wantsDirectBuffer) {
            // As CompositeByteBuf.nioBufferCount() can be expensive (as it needs to check all composed ByteBuf
            // to calculate the count) we will just assume a CompositeByteBuf contains more then 1 ByteBuf.
            // The worst that can happen is that we allocate an extra ByteBuffer[] in CompositeByteBuf.nioBuffers()
            // which is better then walking the composed ByteBuf in most cases.
            if (!(in instanceof CompositeByteBuf) && in.nioBufferCount() == 1) {
                in0 = singleBuffer;
                // We know its only backed by 1 ByteBuffer so use internalNioBuffer to keep object allocation
                // to a minimum.
                in0[0] = in.internalNioBuffer(readerIndex, readableBytes);
            } else {
                in0 = in.nioBuffers();
            }
        } else {
            // We could even go further here and check if its a CompositeByteBuf and if so try to decompose it and
            // only replace the ByteBuffer that are not direct. At the moment we just will replace the whole
            // CompositeByteBuf to keep the complexity to a minimum
            newDirectIn = alloc.directBuffer(readableBytes);
            newDirectIn.writeBytes(in, readerIndex, readableBytes);
            in0 = singleBuffer;
            in0[0] = newDirectIn.internalNioBuffer(0, readableBytes);
        }

        for (;;) {
            ByteBuffer out0 = out.nioBuffer(out.writerIndex(), out.writableBytes());
            SSLEngineResult result = engine.wrap(in0, out0);
            in.skipBytes(result.bytesConsumed());
            out.writerIndex(out.writerIndex() + result.bytesProduced());

            switch (result.getStatus()) {
            case BUFFER_OVERFLOW:
                out.ensureWritable(maxPacketBufferSize);
                break;
            default:
                return result;
            }
        }
    } finally {
        // Null out to allow GC of ByteBuffer
        singleBuffer[0] = null;

        if (newDirectIn != null) {
            newDirectIn.release();
        }
    }
}
 

public static SSLEngineResult.HandshakeStatus doHandshake(SSLEngine tlsEngine,
                                                          ByteBuffer netDataBuf,
                                                          OutputStream out,
                                                          InputStream in){
    try {
        ByteBuffer empty;
        /*Apparently on Android 4.4 (API_19) SSLEngine whines about BUFFER_OVERFLOW for this
        buffer even though nothing ever gets written to it*/
        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT_WATCH){
            empty = ByteBuffer.allocate(0);
        } else {
            empty = ByteBuffer.allocate(tlsEngine.getSession().getApplicationBufferSize());
        }

        // ClientHello -> netDataBuf
        tlsEngine.wrap(empty, netDataBuf);
        netDataBuf.flip();
        byte[] clientHello = new byte[netDataBuf.limit()];
        netDataBuf.get(clientHello);
        out.write(ConnectionHelper.intToByteArray(clientHello.length));
        out.write(clientHello);

        // netDataBuf <- ServerHello..ServerHelloDone
        int serverHelloSize = ByteBuffer.wrap(ConnectionHelper.readAll(4, in)).getInt();
        byte[] serverHello = ConnectionHelper.readAll(serverHelloSize, in);
        netDataBuf.clear();
        netDataBuf.put(serverHello);
        netDataBuf.flip();
        SSLEngineResult result = tlsEngine.unwrap(netDataBuf, empty);
        while (result.getHandshakeStatus() == SSLEngineResult.HandshakeStatus.NEED_UNWRAP){
            result = tlsEngine.unwrap(netDataBuf, empty);
        }
        Runnable task = tlsEngine.getDelegatedTask();
        while (task != null){
            task.run();
            task = tlsEngine.getDelegatedTask();
        }

        // [client]Certificate*..ClientKeyExchange..Finished -> netDataBuf
        netDataBuf.clear();
        result = tlsEngine.wrap(empty, netDataBuf);
        while (result.getHandshakeStatus() == SSLEngineResult.HandshakeStatus.NEED_WRAP){
            result = tlsEngine.wrap(empty, netDataBuf);
        }
        netDataBuf.flip();
        byte[] clientKeyExchange = new byte[netDataBuf.limit()];
        netDataBuf.get(clientKeyExchange);
        out.write(ConnectionHelper.intToByteArray(clientKeyExchange.length));
        out.write(clientKeyExchange);

        // netDataBuf <- ChangeCipherSpec..Finished
        int serverChangeCipherSpecSize = ByteBuffer.wrap(ConnectionHelper.readAll(4, in)).getInt();
        byte[] serverChangeCipherSpec = ConnectionHelper.readAll(serverChangeCipherSpecSize, in);
        netDataBuf.clear();
        netDataBuf.put(serverChangeCipherSpec);
        netDataBuf.flip();
        result = tlsEngine.unwrap(netDataBuf, empty);
        while (result.getHandshakeStatus() == SSLEngineResult.HandshakeStatus.NEED_UNWRAP){
            result = tlsEngine.unwrap(netDataBuf, empty);
        }

        /*On Android 8.1 (LineageOS 15.1) on a Xiaomi device (not sure about others) the
        SSL_ENGINE may return NEED_WRAP here, so we do that even though no data gets written
        by the SSL_ENGINE ???*/
        if (result.getHandshakeStatus() == SSLEngineResult.HandshakeStatus.NEED_WRAP) {
            netDataBuf.clear();
            result = tlsEngine.wrap(empty, netDataBuf);
            // netDataBuf still empty here...
        }

        /*Apparently on Android 4.4 (API_19) with SSLEngine the latest call tlsEngine.unwrap(..)
        that finishes the handshake returns NOT_HANDSHAKING instead of FINISHED as the result*/
        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT_WATCH){
            return result.getHandshakeStatus();
        } else {
            if (result.getHandshakeStatus() == SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING){
                return SSLEngineResult.HandshakeStatus.FINISHED;
            } else if (result.getHandshakeStatus() == SSLEngineResult.HandshakeStatus.FINISHED) {
                // just in case
                return result.getHandshakeStatus();
            } else {
                return null;
            }
        }
    } catch (IOException e){
        return null;
    }
}
 

private SSLEngineResult wrap(ByteBufAllocator alloc, SSLEngine engine, ByteBuf in, ByteBuf out)
        throws SSLException {
    ByteBuf newDirectIn = null;
    try {
        int readerIndex = in.readerIndex();
        int readableBytes = in.readableBytes();

        // We will call SslEngine.wrap(ByteBuffer[], ByteBuffer) to allow efficient handling of
        // CompositeByteBuf without force an extra memory copy when CompositeByteBuffer.nioBuffer() is called.
        final ByteBuffer[] in0;
        if (in.isDirect() || !engineType.wantsDirectBuffer) {
            // As CompositeByteBuf.nioBufferCount() can be expensive (as it needs to check all composed ByteBuf
            // to calculate the count) we will just assume a CompositeByteBuf contains more then 1 ByteBuf.
            // The worst that can happen is that we allocate an extra ByteBuffer[] in CompositeByteBuf.nioBuffers()
            // which is better then walking the composed ByteBuf in most cases.
            if (!(in instanceof CompositeByteBuf) && in.nioBufferCount() == 1) {
                in0 = singleBuffer;
                // We know its only backed by 1 ByteBuffer so use internalNioBuffer to keep object allocation
                // to a minimum.
                in0[0] = in.internalNioBuffer(readerIndex, readableBytes);
            } else {
                in0 = in.nioBuffers();
            }
        } else {
            // We could even go further here and check if its a CompositeByteBuf and if so try to decompose it and
            // only replace the ByteBuffer that are not direct. At the moment we just will replace the whole
            // CompositeByteBuf to keep the complexity to a minimum
            newDirectIn = alloc.directBuffer(readableBytes);
            newDirectIn.writeBytes(in, readerIndex, readableBytes);
            in0 = singleBuffer;
            in0[0] = newDirectIn.internalNioBuffer(newDirectIn.readerIndex(), readableBytes);
        }

        for (;;) {
            ByteBuffer out0 = out.nioBuffer(out.writerIndex(), out.writableBytes());
            SSLEngineResult result = engine.wrap(in0, out0);
            in.skipBytes(result.bytesConsumed());
            out.writerIndex(out.writerIndex() + result.bytesProduced());

            switch (result.getStatus()) {
            case BUFFER_OVERFLOW:
                out.ensureWritable(engine.getSession().getPacketBufferSize());
                break;
            default:
                return result;
            }
        }
    } finally {
        // Null out to allow GC of ByteBuffer
        singleBuffer[0] = null;

        if (newDirectIn != null) {
            newDirectIn.release();
        }
    }
}