Java Code Examples for org.apache.brooklyn.core.test.entity.TestApplication#start()

@Test
public void testWarmStandby() throws Exception {
    HaMgmtNode n1 = newNode();
    n1.ha.start(HighAvailabilityMode.AUTO);
    assertEquals(n1.ha.getNodeState(), ManagementNodeState.MASTER);
    
    TestApplication app = TestApplication.Factory.newManagedInstanceForTests(n1.mgmt);
    app.start(MutableList.<Location>of());
    
    n1.mgmt.getRebindManager().forcePersistNow(false, null);

    HaMgmtNode n2 = newNode();
    n2.ha.start(HighAvailabilityMode.STANDBY);
    assertEquals(n2.ha.getNodeState(), ManagementNodeState.STANDBY);

    assertEquals(n2.mgmt.getApplications().size(), 0);
}
 

private TestApplication newVanillaSoftwareProcessApp(int suffix) {
    Location loc = mgmt.getLocationManager().createLocation(LocationSpec.create(FixedListMachineProvisioningLocation.class)
            .configure(FixedListMachineProvisioningLocation.MACHINE_SPECS, ImmutableList.of(
                    LocationSpec.create(SshMachineLocation.class)
                            .configure("address", "1.2.3.4")
                            .configure("sshToolClass", RecordingSshTool.class.getName()))));

    TestApplication app = mgmt.getEntityManager().createEntity(EntitySpec.create(EntitySpec.create(TestApplication.class)
            .displayName("app-"+suffix)
            .child(EntitySpec.create(VanillaSoftwareProcess.class)
                    .configure(VanillaSoftwareProcess.INSTALL_COMMAND, "myInstall")
                    .configure(VanillaSoftwareProcess.LAUNCH_COMMAND, "myLaunch")
                    .configure(VanillaSoftwareProcess.CHECK_RUNNING_COMMAND, "myCheckRunning")
                    .configure(VanillaSoftwareProcess.STOP_COMMAND, "myStop")
                    .configure(DynamicCluster.MEMBER_SPEC, EntitySpec.create(BasicStartable.class)))));
    app.start(ImmutableList.of(loc));
    return app;
}
 

/**
 * Configures rubyrep to connect to the two databases and starts the app
 */
public static void startInLocation(TestApplication tapp, DatastoreCommon db1, String dbName1, DatastoreCommon db2, String dbName2, Location... locations) throws Exception {
    tapp.createAndManageChild(EntitySpec.create(RubyRepNode.class)
            .configure("startupTimeout", 300)
            .configure("leftDatabase", db1)
            .configure("rightDatabase", db2)
            .configure("leftUsername", "sqluser")
            .configure("rightUsername", "sqluser")
            .configure("rightPassword", "sqluserpw")
            .configure("leftPassword", "sqluserpw")
            .configure("leftDatabaseName", dbName1)
            .configure("rightDatabaseName", dbName2)
            .configure("replicationInterval", 1)
    );

    tapp.start(Arrays.asList(locations));
}
 

@Test
public void testAddExistingMachineFromSpec() {
    TestApplication app = createAppWithChildren(getInitialPoolSize());
    app.start(ImmutableList.of(pool.getDynamicLocation()));
    assertAvailableCountEventuallyEquals(0);

    Collection<Entity> added = pool.addExistingMachinesFromSpec("byon:(hosts=\"localhost,localhost\")");
    assertEquals(added.size(), 2, "Added: " + Joiner.on(", ").join(added));
    Iterator<Entity> it = added.iterator();
    assertFalse(it.next().getConfig(ServerPoolImpl.REMOVABLE));
    assertFalse(it.next().getConfig(ServerPoolImpl.REMOVABLE));
    assertAvailableCountEventuallyEquals(2);

    TestApplication app2 = createAppWithChildren(2);
    app2.start(ImmutableList.of(pool.getDynamicLocation()));
    assertAvailableCountEventuallyEquals(0);
}
 

@Test
public void testDeployReleaseDeploy() {
    TestApplication app = createAppWithChildren(getInitialPoolSize());
    TestApplication app2 = createAppWithChildren(1);

    app.start(ImmutableList.of(pool.getDynamicLocation()));
    EntityAsserts.assertAttributeEqualsEventually(app, Attributes.SERVICE_UP, true);
    assertAvailableCountEventuallyEquals(0);
    assertNoMachinesAvailableForApp(app2);

    app.stop();
    assertFalse(app.getAttribute(Attributes.SERVICE_UP));
    assertAvailableCountEventuallyEquals(getInitialPoolSize());

    app2.start(ImmutableList.of(pool.getDynamicLocation()));
    EntityAsserts.assertAttributeEqualsEventually(app2, Attributes.SERVICE_UP, true);
    
    assertAvailableCountEventuallyEquals(getInitialPoolSize() - 1);
    assertClaimedCountEventuallyEquals(1);
}
 

@Test
public void testListAndGetMachineUsage() throws Exception {
    Location location = getManagementContext().getLocationManager().createLocation(LocationSpec.create(DynamicLocalhostMachineProvisioningLocation.class));
    TestApplication app = getManagementContext().getEntityManager().createEntity(EntitySpec.create(TestApplication.class));
    SoftwareProcessEntityTest.MyService entity = app.createAndManageChild(org.apache.brooklyn.api.entity.EntitySpec.create(SoftwareProcessEntityTest.MyService.class));
    
    Calendar preStart = new GregorianCalendar();
    app.start(ImmutableList.of(location));
    Calendar postStart = new GregorianCalendar();
    Location machine = Iterables.getOnlyElement(entity.getLocations());

    // All machines
    Response response = client().path("/usage/machines").get();
    assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
    Iterable<UsageStatistics> usages = response.readEntity(new GenericType<List<UsageStatistics>>() {});
    UsageStatistics usage = Iterables.getOnlyElement(usages);
    assertMachineUsage(usage, app.getId(), machine.getId(), ImmutableList.of(Status.ACCEPTED), roundDown(preStart), postStart);

    // Specific machine
    response = client().path("/usage/machines/"+machine.getId()).get();
    assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
    assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
    usage = response.readEntity(new GenericType<UsageStatistics>() {});
    assertMachineUsage(usage, app.getId(), machine.getId(), ImmutableList.of(Status.ACCEPTED), roundDown(preStart), postStart);
}
 

public static void testBindStartsAndUpdates(TestApplication app, Location testLocation) throws Exception {
    DynamicCluster cluster;
    BindDnsServer dns;

    SameServerEntity sse = app.createAndManageChild(EntitySpec.create(SameServerEntity.class));
    EntitySpec<EmptySoftwareProcess> memberSpec = EntitySpec.create(EmptySoftwareProcess.class)
            .enricher(EnricherSpec.create(PrefixAndIdEnricher.class)
                    .configure(PrefixAndIdEnricher.PREFIX, "dns-live-test-")
                    .configure(PrefixAndIdEnricher.MONITOR, Attributes.HOSTNAME));
    cluster = sse.addChild(EntitySpec.create(DynamicCluster.class)
            .configure(DynamicCluster.MEMBER_SPEC, memberSpec)
            .configure(DynamicCluster.INITIAL_SIZE, 1));
    dns = sse.addChild((EntitySpec.create(BindDnsServer.class)
            .configure(BindDnsServer.ENTITY_FILTER, Predicates.instanceOf(EmptySoftwareProcess.class))
            .configure(BindDnsServer.HOSTNAME_SENSOR, PrefixAndIdEnricher.SENSOR)));

    app.start(ImmutableList.of(testLocation));
    EntityAsserts.assertAttributeEqualsEventually(dns, Attributes.SERVICE_UP, true);

    logDnsMappings(dns);
    assertEquals(dns.getAttribute(BindDnsServer.ADDRESS_MAPPINGS).entries().size(), 1);
    assertEquals(dns.getAttribute(BindDnsServer.A_RECORDS).size(), 1);
    assertEquals(dns.getAttribute(BindDnsServer.CNAME_RECORDS).size(), 0);
    // Harder to make assertions on PTR because the entity servers might not be in the right CIDR

    cluster.resize(2);
    waitForNumberOfAddressMappings(dns, 2);
    logDnsMappings(dns);
    assertEquals(dns.getAttribute(BindDnsServer.ADDRESS_MAPPINGS).entries().size(), 2);
    assertEquals(dns.getAttribute(BindDnsServer.A_RECORDS).size(), 1);
    assertEquals(dns.getAttribute(BindDnsServer.CNAME_RECORDS).size(), 1);

    cluster.resize(1);
    waitForNumberOfAddressMappings(dns, 1);
    logDnsMappings(dns);
    assertEquals(dns.getAttribute(BindDnsServer.ADDRESS_MAPPINGS).entries().size(), 1);
    assertEquals(dns.getAttribute(BindDnsServer.A_RECORDS).size(), 1);
    assertEquals(dns.getAttribute(BindDnsServer.CNAME_RECORDS).size(), 0);
}
 

private void startAppAssertingFailure(TestApplication app, Location loc) {
    try {
        app.start(ImmutableList.of(loc));
        Asserts.shouldHaveFailedPreviously();
    } catch (Throwable t) {
        Asserts.expectedFailureContains(t, "Test failed on group member");
    }
}
 

@Test
public void testStandbyRebind() throws Exception {
    useSharedTime();
    HaMgmtNode n1 = newNode();
    HaMgmtNode n2 = newNode();

    // first auto should become master
    n1.ha.start(HighAvailabilityMode.AUTO);
    n2.ha.start(HighAvailabilityMode.AUTO);

    TestApplication app = n1.mgmt.getEntityManager().createEntity(
            EntitySpec.create(TestApplication.class).impl(TestEntityFailingRebind.class));
    app.start(ImmutableList.<Location>of());

    n1.mgmt.getRebindManager().forcePersistNow(false, null);

    //don't publish state for a while (i.e. long store delays, failures)
    sharedTickerAdvance(Duration.ONE_MINUTE);

    try {
        n2.ha.publishAndCheck(false);
        fail("n2 rebind failure expected");
    } catch (Exception e) {
        assertNestedRebindException(e);
    }

    TestEntityFailingRebind.setThrowOnRebind(false);
    n1.ha.publishAndCheck(false);

    ManagementPlaneSyncRecord memento = n1.ha.loadManagementPlaneSyncRecord(true);
    assertEquals(memento.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.MASTER);
    assertEquals(memento.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.FAILED);
}
 

@Test(groups = "Live")
public void testAppCanBeDeployedToPool() {
    TestApplication app = createAppWithChildren(1);
    app.start(ImmutableList.of(pool.getDynamicLocation()));
    assertTrue(app.getAttribute(Attributes.SERVICE_UP));
    for (Entity child : app.getChildren()) {
        assertTrue(child.getAttribute(Attributes.SERVICE_UP));
    }
    TestApplication app2 = createAppWithChildren(1);
    assertNoMachinesAvailableForApp(app2);
}
 

@Test
public void testSubscribeToJmxNotificationAndEmitCorrespondingNotificationSensor() throws Exception {
    final TestApplication app2 = mgmt.getEntityManager().createEntity(EntitySpec.create(TestApplication.class));
    final EntityWithEmitter entity = app2.addChild(EntitySpec.create(EntityWithEmitter.class));
    
    try {
        app2.start(ImmutableList.of(new SimulatedLocation()));
        
        final List<SensorEvent<String>> received = Lists.newArrayList();
        app2.subscriptions().subscribe(null, EntityWithEmitter.MY_NOTIF, new SensorEventListener<String>() {
            @Override
            public void onEvent(SensorEvent<String> event) {
                received.add(event);
            }});

        final StandardEmitterMBean mbean = jmxService.registerMBean(ImmutableList.of("one"), objectName);
        final AtomicInteger sequence = new AtomicInteger(0);
        
        jmxHelper.connect(TIMEOUT_MS);
        jmxHelper.addNotificationListener(jmxObjectName, new NotificationListener() {
                @Override
                public void handleNotification(Notification notif, Object callback) {
                    if (notif.getType().equals("one")) {
                        entity.sensors().emit(EntityWithEmitter.MY_NOTIF, (String) notif.getUserData());
                    }
                }});
        

        Asserts.succeedsEventually(ImmutableMap.of("timeout", TIMEOUT_MS), new Runnable() {
            @Override
            public void run() {
                sendNotification(mbean, "one", sequence.getAndIncrement(), "abc");
                assertTrue(received.size() > 0, "received size should be bigger than 0");
                assertEquals(received.get(0).getValue(), "abc");
            }});
    } finally {
        Entities.destroyAll(app2.getManagementContext());
    }
}
 

@Test
public void testCanAddExistingMachinesToPool() {
    TestApplication app = createAppWithChildren(getInitialPoolSize());
    app.start(ImmutableList.of(pool.getDynamicLocation()));
    assertAvailableCountEventuallyEquals(0);

    LocalhostMachine loc = mgmt.getLocationManager().createLocation(LocationSpec.create(LocalhostMachine.class));
    Entity added = pool.addExistingMachine(loc);
    assertFalse(added.getConfig(ServerPoolImpl.REMOVABLE));
    assertAvailableCountEventuallyEquals(1);

    TestApplication app2 = createAppWithChildren(1);
    app2.start(ImmutableList.of(pool.getDynamicLocation()));
    assertAvailableCountEventuallyEquals(0);
}
 

@Test
public void testAppCanBeDeployedToServerPool() {
    TestApplication app = createAppWithChildren(1);
    app.start(ImmutableList.of(pool.getDynamicLocation()));
    assertTrue(app.getAttribute(Attributes.SERVICE_UP));
    for (Entity child : app.getChildren()) {
        assertTrue(child.getAttribute(Attributes.SERVICE_UP));
    }
}
 

@Test
@SuppressWarnings("deprecation")
public void testUsesInitialLocationsFromAppSpec() throws Exception {
    TestApplication app2 = mgmt.getEntityManager().createEntity(EntitySpec.create(TestApplication.class)
        .configure(BrooklynConfigKeys.SKIP_ON_BOX_BASE_DIR_RESOLUTION, true)
        .child(EntitySpec.create(DynamicRegionsFabric.class)
            .configure("memberSpec", EntitySpec.create(TestEntity.class)))
        .locations(ImmutableList.of(loc1, loc2)));
    DynamicRegionsFabric fabric2 = (DynamicRegionsFabric) Iterables.getOnlyElement(app2.getChildren());
    app2.start(ImmutableList.<Location>of());
    
    assertFabricChildren(fabric2, 2, ImmutableList.of(loc1, loc2));
}
 

@Test
public void testListMachinesUsageForApp() throws Exception {
    Location location = getManagementContext().getLocationManager().createLocation(LocationSpec.create(DynamicLocalhostMachineProvisioningLocation.class));
    TestApplication app = getManagementContext().getEntityManager().createEntity(EntitySpec.create(TestApplication.class));
    SoftwareProcessEntityTest.MyService entity = app.createAndManageChild(org.apache.brooklyn.api.entity.EntitySpec.create(SoftwareProcessEntityTest.MyService.class));
    String appId = app.getId();
    
    Calendar preStart = new GregorianCalendar();
    app.start(ImmutableList.of(location));
    Calendar postStart = new GregorianCalendar();
    Location machine = Iterables.getOnlyElement(entity.getLocations());

    // For running machine
    Response response = client().path("/usage/machines").query("application", appId).get();
    assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
    Iterable<UsageStatistics> usages = response.readEntity(new GenericType<List<UsageStatistics>>() {});
    UsageStatistics usage = Iterables.getOnlyElement(usages);
    assertMachineUsage(usage, app.getId(), machine.getId(), ImmutableList.of(Status.ACCEPTED), roundDown(preStart), postStart);
    
    // Stop the machine
    Calendar preStop = new GregorianCalendar();
    app.stop();
    Calendar postStop = new GregorianCalendar();
    
    // Deleted machine still returned, if in time range
    response = client().path("/usage/machines").query("application", appId).get();
    assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
    usages = response.readEntity(new GenericType<List<UsageStatistics>>() {});
    usage = Iterables.getOnlyElement(usages);
    assertMachineUsage(usage, app.getId(), machine.getId(), ImmutableList.of(Status.ACCEPTED, Status.DESTROYED), roundDown(preStart), postStop);
    assertMachineUsage(ImmutableList.copyOf(usage.getStatistics()).subList(1,2), appId, machine.getId(), ImmutableList.of(Status.DESTROYED), roundDown(preStop), postStop);

    // Terminated machines ignored if terminated since start-time
    long futureTime = postStop.getTime().getTime()+1;
    waitForFuture(futureTime);
    response = client().path("/usage/applications").query("start", futureTime).get();
    assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
    usages = response.readEntity(new GenericType<List<UsageStatistics>>() {});
    assertTrue(Iterables.isEmpty(usages), "usages="+usages);
}
 

public static void testHazelcastCluster(TestApplication app, Location loc) {
    HazelcastCluster cluster = app.createAndManageChild(EntitySpec.create(HazelcastCluster.class)
            .configure(HazelcastCluster.INITIAL_SIZE, 3)
            .configure(HazelcastCluster.MEMBER_SPEC, EntitySpec.create(HazelcastNode.class)));
    app.start(ImmutableList.of(loc));

    EntityAsserts.assertAttributeEqualsEventually(cluster, HazelcastNode.SERVICE_UP, true);

    HazelcastNode first = (HazelcastNode) Iterables.get(cluster.getMembers(), 0);
    HazelcastNode second = (HazelcastNode) Iterables.get(cluster.getMembers(), 1);

    assertNodesUpAndInCluster(first, second);

    EntityAsserts.assertAttributeEqualsEventually(cluster, Attributes.SERVICE_UP, true);
}
 

@Test
public void testStopAppThenShutdownAndStopAppsWaitsForFirstStop() throws InterruptedException {
    ReflectiveEntityDriverFactory f = ((BasicEntityDriverManager)getManagementContext().getEntityDriverManager()).getReflectiveDriverFactory();
    f.addClassFullNameMapping("org.apache.brooklyn.entity.software.base.EmptySoftwareProcessDriver", "org.apache.brooklyn.rest.resources.ServerResourceTest$EmptySoftwareProcessTestDriver");

    // Second stop on SoftwareProcess could return early, while the first stop is still in progress
    // This causes the app to shutdown prematurely, leaking machines.
    EntityManager emgr = getManagementContext().getEntityManager();
    EntitySpec<TestApplication> appSpec = EntitySpec.create(TestApplication.class);
    TestApplication app = emgr.createEntity(appSpec);
    EntitySpec<StopLatchEntity> latchEntitySpec = EntitySpec.create(StopLatchEntity.class);
    final StopLatchEntity entity = app.createAndManageChild(latchEntitySpec);
    app.start(ImmutableSet.of(app.newLocalhostProvisioningLocation()));
    EntityAsserts.assertAttributeEquals(entity, Attributes.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING);

    try {
        final Task<Void> firstStop = app.invoke(Startable.STOP, ImmutableMap.<String, Object>of());
        Asserts.succeedsEventually(new Runnable() {
            @Override
            public void run() {
                assertTrue(entity.isBlocked());
            }
        });

        final AtomicReference<Exception> shutdownError = new AtomicReference<>();
        // Can't use ExecutionContext as it will be stopped on shutdown
        Thread shutdownThread = new Thread() {
            @Override
            public void run() {
                try {
                    MultivaluedMap<String, String> formData = new MultivaluedHashMap<>();
                    formData.add("stopAppsFirst", "true");
                    formData.add("shutdownTimeout", "0");
                    formData.add("requestTimeout", "0");
                    formData.add("delayForHttpReturn", "0");
                    client().path("/server/shutdown").type(MediaType.APPLICATION_FORM_URLENCODED).post(formData);
                } catch (Exception e) {
                    log.error("Shutdown request error", e);
                    shutdownError.set(e);
                    throw Exceptions.propagate(e);
                }
            }
        };
        shutdownThread.start();

        //shutdown must wait until the first stop completes (or time out)
        Asserts.succeedsContinually(new Runnable() {
            @Override
            public void run() {
                assertFalse(firstStop.isDone());
                assertEquals(getManagementContext().getApplications().size(), 1);
                assertFalse(shutdownListener.isRequested());
            }
        });

        // NOTE test is not fully deterministic. Depending on thread scheduling this will
        // execute before or after ServerResource.shutdown does the app stop loop. This
        // means that the shutdown code might not see the app at all. In any case though
        // the test must succeed.
        entity.unblock();

        Asserts.succeedsEventually(new Runnable() {
            @Override
            public void run() {
                assertTrue(firstStop.isDone());
                assertTrue(shutdownListener.isRequested());
                assertFalse(getManagementContext().isRunning());
            }
        });

        shutdownThread.join();
        assertNull(shutdownError.get(), "Shutdown request error, logged above");
    } finally {
        // Be sure we always unblock entity stop even in the case of an exception.
        // In the success path the entity is already unblocked above.
        entity.unblock();
    }
}
 

@Test
public void testIfNodeStopsBeingAbleToWrite() throws Exception {
    useSharedTime();
    log.info("time at start "+sharedTickerCurrentMillis());
    
    HaMgmtNode n1 = newNode();
    HaMgmtNode n2 = newNode();
    
    // first auto should become master
    n1.ha.start(HighAvailabilityMode.AUTO);
    ManagementPlaneSyncRecord memento1 = n1.ha.loadManagementPlaneSyncRecord(true);
    
    log.info(n1+" HA: "+memento1);
    assertEquals(memento1.getMasterNodeId(), n1.ownNodeId);
    Long time0 = sharedTickerCurrentMillis();
    assertEquals(memento1.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(), time0);
    assertEquals(memento1.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.MASTER);

    // second - make explicit hot; that's a strictly more complex case than cold standby, so provides pretty good coverage
    n2.ha.start(HighAvailabilityMode.HOT_STANDBY);
    ManagementPlaneSyncRecord memento2 = n2.ha.loadManagementPlaneSyncRecord(true);
    
    log.info(n2+" HA: "+memento2);
    assertEquals(memento2.getMasterNodeId(), n1.ownNodeId);
    assertEquals(memento2.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.MASTER);
    assertEquals(memento2.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.HOT_STANDBY);
    assertEquals(memento2.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(), time0);
    assertEquals(memento2.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(), time0);
    
    // and no entities at either
    assertEquals(n1.mgmt.getApplications().size(), 0);
    assertEquals(n2.mgmt.getApplications().size(), 0);

    // create
    TestApplication app = n1.mgmt.getEntityManager().createEntity(EntitySpec.create(TestApplication.class));
    app.start(ImmutableList.<Location>of());
    app.sensors().set(TestApplication.MY_ATTRIBUTE, "hello");
    
    assertEquals(n1.mgmt.getApplications().size(), 1);
    assertEquals(n2.mgmt.getApplications().size(), 0);
    log.info("persisting "+n1.ownNodeId);
    n1.mgmt.getRebindManager().forcePersistNow(false, null);
    
    n1.objectStore.setWritesFailSilently(true);
    log.info(n1+" writes off");
    sharedTickerAdvance(Duration.ONE_MINUTE);
    log.info("time now "+sharedTickerCurrentMillis());
    Long time1 = sharedTickerCurrentMillis();
    
    log.info("publish "+n2.ownNodeId);
    n2.ha.publishAndCheck(false);
    ManagementPlaneSyncRecord memento2b = n2.ha.loadManagementPlaneSyncRecord(true);
    log.info(n2+" HA now: "+memento2b);
    
    // n2 infers n1 as failed 
    assertEquals(memento2b.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.FAILED);
    assertEquals(memento2b.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.MASTER);
    assertEquals(memento2b.getMasterNodeId(), n2.ownNodeId);
    assertEquals(memento2b.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(), time0);
    assertEquals(memento2b.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(), time1);
    
    assertEquals(n1.mgmt.getApplications().size(), 1);
    assertEquals(n2.mgmt.getApplications().size(), 1);
    assertEquals(n1.mgmt.getApplications().iterator().next().getAttribute(TestApplication.MY_ATTRIBUTE), "hello");
    
    n1.objectStore.setWritesFailSilently(false);
    log.info(n1+" writes on");
    
    sharedTickerAdvance(Duration.ONE_SECOND);
    log.info("time now "+sharedTickerCurrentMillis());
    Long time2 = sharedTickerCurrentMillis();
    
    log.info("publish "+n1.ownNodeId);
    n1.ha.publishAndCheck(false);
    ManagementPlaneSyncRecord memento1b = n1.ha.loadManagementPlaneSyncRecord(true);
    log.info(n1+" HA now: "+memento1b);
    
    ManagementNodeState expectedStateAfterDemotion = BrooklynFeatureEnablement.isEnabled(BrooklynFeatureEnablement.FEATURE_DEFAULT_STANDBY_IS_HOT_PROPERTY) ?
        ManagementNodeState.HOT_STANDBY : ManagementNodeState.STANDBY;
    
    // n1 comes back and demotes himself 
    assertEquals(memento1b.getManagementNodes().get(n1.ownNodeId).getStatus(), expectedStateAfterDemotion);
    assertEquals(memento1b.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.MASTER);
    assertEquals(memento1b.getMasterNodeId(), n2.ownNodeId);
    assertEquals(memento1b.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(), time2);
    assertEquals(memento1b.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(), time1);
    
    // n2 now sees itself as master, with n1 in standby again
    ManagementPlaneSyncRecord memento2c = n2.ha.loadManagementPlaneSyncRecord(true);
    log.info(n2+" HA now: "+memento2c);
    assertEquals(memento2c.getManagementNodes().get(n1.ownNodeId).getStatus(), expectedStateAfterDemotion);
    assertEquals(memento2c.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.MASTER);
    assertEquals(memento2c.getMasterNodeId(), n2.ownNodeId);
    assertEquals(memento2c.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(), time2);
    assertEquals(memento2c.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(), time2);

    // right number of entities at n2; n1 may or may not depending whether hot standby is default
    assertEquals(n2.mgmt.getApplications().size(), 1);
    assertEquals(n1.mgmt.getApplications().size(), BrooklynFeatureEnablement.isEnabled(BrooklynFeatureEnablement.FEATURE_DEFAULT_STANDBY_IS_HOT_PROPERTY) ? 1 : 0);
}
 

public static MySqlCluster initCluster(TestApplication app, Location location, EntitySpec<MySqlCluster> spec) {
    MySqlCluster mysql = app.createAndManageChild(spec);
    app.start(ImmutableList.of(location));
    log.info("MySQL started");
    return mysql;
}
 

private TestApplication newEmptyApp(int suffix) {
    TestApplication app = mgmt.getEntityManager().createEntity(EntitySpec.create(EntitySpec.create(TestApplication.class)
            .displayName("app-"+suffix)));
    app.start(ImmutableList.of(app.newLocalhostProvisioningLocation()));
    return app;
}