Java Code Examples for com.bigdata.journal.Journal#destroy()

public void test_metaAlloc() {

			Journal store = (Journal) getStore();
            try {

			final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

			final RWStore rw = bs.getStore();
			long realAddr = 0;
				for (int i = 0; i < 100000; i++) {
					int allocAddr = rw.metaAlloc();

					realAddr = rw.metaBit2Addr(allocAddr);
				}
				if(log.isInfoEnabled())log.info("metaAlloc lastAddr: " + realAddr);
			} finally {
				store.destroy();
			}
		}
 

/**
 * Test low level RWStore add/removeAddress methods as used in HA
 * WriteCache replication to ensure Allocation consistency
 * 
 * @throws IOException
 */
public void testStressReplication() throws IOException {

	// Create a couple of stores with temp files
	final Journal store1 = (Journal) getStore();
	final Journal store2 = (Journal) getStore();
	try {		
		final RWStore rw1 = ((RWStrategy) store1.getBufferStrategy()).getStore();
		final RWStore rw2 = ((RWStrategy) store2.getBufferStrategy()).getStore();
	
		assertTrue(rw1 != rw2);
		
		final Random r = new Random();
		
		for (int i = 0; i < 100000; i++) {
			final int sze = 1 + r.nextInt(2000);
			final int addr = rw1.alloc(sze, null);
			rw2.addAddress(addr, sze);
		}
	} finally {
		store1.destroy();
		store2.destroy();
	}
}
 

public void test_multiVoidCommit() {

			final Journal store = (Journal) getStore();
            try {

			final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

			final RWStore rw = bs.getStore();
			
			final boolean initRequire = bs.requiresCommit(store.getRootBlockView());
			assertTrue(initRequire);
			for (int n = 0; n < 20; n++) {
				store.commit();
				final IRootBlockView rbv = store.getRootBlockView();
				assertTrue(1 == rbv.getCommitCounter());
				assertFalse(bs.requiresCommit(store.getRootBlockView()));
				assertFalse(rw.requiresCommit());
			}
			} finally {
				store.destroy();
			}
		}
 

/**
 * State3
 * 
 * Allocate - Commit - Free - Commit
 * 
 * Tracks writeCache state through allocation
 */
public void test_allocCommitFreeCommitWriteCache() {
	Journal store = (Journal) getStore();
          try {
          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	// Has just been written so must be in cache
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	// since data is committed, should be accessible from any new
          	// readCommitted transaction
          	assertTrue(bs.inWriteCache(addr));

          	store.commit();
          	
          	// If no transactions are around (which they are not) then must
          	//	be released as may be re-allocated
          	assertFalse(bs.inWriteCache(addr));
          	
          } finally {
          	store.destroy();
          }
}
 

/**
 * State4
 * 
 * Allocate - Commit - Free - Commit
 * 
 * ..but with session protection using a RawTx
 */
public void test_allocCommitFreeCommitSessionWriteCache() {
	final Journal store = (Journal) getStore();
          try {
          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	// Has just been written so must be in cache
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.commit();
          	
          	IRawTx tx = bs.newTx();
          	
          	bs.delete(addr);
          	
          	// since data is committed, should be accessible from any new
          	// readCommitted transaction
          	assertTrue(bs.inWriteCache(addr));

          	store.commit();
          	
          	// Since transactions are around the cache will not have been
          	//	cleared (and cannot be re-allocated)
          	assertTrue(bs.inWriteCache(addr));
          	
          	tx.close(); // Release Session
          	
          	// Cache must have been cleared
          	assertFalse(bs.inWriteCache(addr));
          	
          } finally {
          	store.destroy();
          }
}
 

/**
 * Tests semantics of a simple reset
 * 
 * Commit some data
 * Delete committed and allocate new data
 * Reset
 * Test that deletion and new allocation are void
 */
public void test_simpleReset() {
	final Journal store = (Journal) getStore();
       try {
       	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
       	
       	final long addr = bs.write(randomData(78));
       	
       	// Has just been written so must be in cache
       	assertTrue(bs.inWriteCache(addr));
       	
       	store.commit();
       	
       	bs.delete(addr);
       	
       	final long addr2 = bs.write(randomData(78));
          	assertTrue(bs.inWriteCache(addr2));
       	
       	bs.abort();
       	
       	assertTrue(bs.inWriteCache(addr)); // not removed in reset
          	assertFalse(bs.inWriteCache(addr2));
       	try {
       		bs.read(addr2);
       		fail("Exception expected");
       	} catch (IllegalArgumentException e) {
       		// expected
       	}
       	store.commit();

       	assertTrue(bs.isCommitted(addr));
       } finally {
       	store.destroy();
       }
	
}
 

/**
       * Verify that we correctly restore the RWStore commit state if
       * {@link RWStore#commit()} is followed by {@link RWStore#reset()}
       * rather than {@link RWStore#postCommit()}.
       * 
       * @see <a href="http://trac.blazegraph.com/ticket/973" >RWStore commit is
       *      not robust to internal failure.</a>
       */
public void test_commitState() {
	Journal store = (Journal) getStore();
          try {

          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final RWStore rws = bs.getStore();
          	
          	final long addr = bs.write(randomData(78));

          	// do 1st half of the RWStore commit protocol.
          	rws.commit();
          	            	
          	// then discard write set.
          	store.abort();
          	
          	assertFalse(bs.isCommitted(addr)); // rolled back
          	
          	// now cycle standard commit to confirm correct reset
          	for (int c = 0; c < 50; c++) {
          		bs.write(randomData(78));
          		store.commit();
          	}
         	
          	
          } finally {
          	store.destroy();
          }
}
 

/**
 * Reallocates the same object several times, then commits and tests
 * read back.
 * 
 * Has been amended to exercise different cache read paths.
 */
public void test_reallocate() {
	final Journal store = (Journal) getStore();

	try {

		final byte[] buf = new byte[1024]; // 2Mb buffer of random data
		r.nextBytes(buf);

		final ByteBuffer bb = ByteBuffer.wrap(buf);

		final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bs.getStore();

		long faddr1 = bs.write(bb);
		bb.position(0);
		// bs.delete(faddr);

		long faddr2 = bs.write(bb);
		bb.position(0);

		// test read from writeCache
		ByteBuffer inbb1 = bs.read(faddr1);
		ByteBuffer inbb2 = bs.read(faddr2);

		assertEquals(bb, inbb1);
		assertEquals(bb, inbb2);
		
		store.commit();

		// test direct read from readCache / postCommit
		inbb1 = bs.read(faddr1);
		inbb2 = bs.read(faddr2);

		assertEquals(bb, inbb1);
		assertEquals(bb, inbb2);
		
		rw.reset();

		// test read from store (via readcache)
		inbb1 = bs.read(faddr1);
		inbb2 = bs.read(faddr2);

		assertEquals(bb, inbb1);
		assertEquals(bb, inbb2);

		// test read direct from readCache
		inbb1 = bs.read(faddr1);
		inbb2 = bs.read(faddr2);

		assertEquals(bb, inbb1);
		assertEquals(bb, inbb2);

	} finally {
		store.destroy();
	}

}
 

/**
       * At scale the small slot handling can lead to large amounts of store
       * waste, tending to the small slot allocation thresholds of 50%,
       * dependent on use case.
       * <p>
       * To mitigate this problem, when more than some minimum number of
       * allocators are in use the storage stats are used to check on overall
       * usage. If the waste is above some specified amount, then an attempt
       * is made to locate a "reasonable" candidate allocator to be used.
       * <p>
       * To test below sets thresholds to quickly trigger this behaviour, with
       * a low minimum number of allocators, and low "candidate" re-use
       * threshold.
       * 
       * @see BLZG-1278 (Small slot optimization to minimize waste).
       */
public void test_smallSlotWasteRecylcing() {
	// Note that the Waste parameter effectively controls the recycling in high waste scenarios
	final Journal store = (Journal) getSmallSlotStore(1024, 10 /*min allocators to check*/, 20.0f/* % waste of store total slot waste before checking for candidates*/);
	try {

	    final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bufferStrategy.getStore();
		final int cSlotSize = 128;
		final int cAllocSize = 99;
		
		int breaks = 0;
		int contiguous = 0;
		
		ArrayList<Integer> recycle = new ArrayList<Integer>();
		
		long pap = rw.physicalAddress(rw.alloc(cAllocSize, null));
		for (int i = 0; i < 500000; i++) {
			final int a = rw.alloc(cSlotSize, null);
			final long pa = rw.physicalAddress(a);
			
			if (r.nextInt(7) < 3) { // less than 50% recycle, so will not be added automatically to free list
				recycle.add(a);
			}
			
			if (pa == (pap+cSlotSize)) {
				contiguous++;
			} else {
				breaks++;
			}
			
			if ((i+1) % 20000 == 0) {
				// add intermediate commit to stress correct recycling
				store.commit();
			}
			
			pap = pa;
			
			if (recycle.size() > 5000) {
				log.warn("Transient Frees for immediate recyling");
				for (int e : recycle) {
					rw.free(e, cAllocSize);
				}
				recycle.clear();
			}
		}
		
		store.commit();
		
		final StringBuilder sb = new StringBuilder();
		rw.showAllocators(sb);
		
		log.warn("Contiguous: " + contiguous + ", breaks: " + breaks + "\n" + sb.toString());

	} finally {
		store.destroy();
	}
}
 

/**
 * Unit test verifies that {@link Options#CREATE} may be used to initialize
 * a journal on a newly created empty file.
 * 
 * @throws IOException
 */
public void test_create_emptyFile() throws IOException {

	final File file = File.createTempFile(getName(), Options.JNL);

	final Properties properties = new Properties();

	properties.setProperty(Options.BUFFER_MODE, BufferMode.DiskRW.toString());

	properties.setProperty(Options.FILE, file.toString());

	properties.setProperty(Options.WRITE_CACHE_ENABLED, "" + writeCacheEnabled);

	final Journal journal = new Journal(properties);

	try {

		assertEquals(file, journal.getFile());

	} finally {

		journal.destroy();

	}

}
 

public void test_allocCommitFreeWithHistory() {
	final Properties properties = new Properties(getProperties());

	properties.setProperty(
			AbstractTransactionService.Options.MIN_RELEASE_AGE, "100");

	final Journal store = getJournal(properties);
	try {

		MemStrategy bs = (MemStrategy) store.getBufferStrategy();

		final long addr = bs.write(randomData(78));

		store.commit();

		bs.delete(addr);

		assertTrue(bs.isCommitted(addr));
	} finally {
		store.destroy();
	}
}
 

void doBlobDeferredFrees(final int cAddrs) {

			final Properties properties = new Properties(getProperties());

            properties.setProperty(
                    AbstractTransactionService.Options.MIN_RELEASE_AGE, "4000");

            properties.setProperty(RWStore.Options.ALLOCATION_SIZES,
                    "1,2,3,5,8,12,16"); // 1K

			final Journal store = (Journal) getStore(properties);
            try {

            	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
            	
            	final ArrayList<Long> addrs = new ArrayList<Long>();
            	for (int i = 0; i < cAddrs; i++) {
            		addrs.add(bs.write(randomData(45)));
            	}
            	store.commit();

            	for (long addr : addrs) {
            		bs.delete(addr);
            	}
                for (int i = 0; i < cAddrs; i++) {
                    if(!bs.isCommitted(addrs.get(i))) {
                        fail("i="+i+", addr="+addrs.get(i));
                    }
                }

               	store.commit();
               	
            	// Age the history (of the deletes!)
            	Thread.currentThread().sleep(6000);
            	
            	// modify store but do not allocate similar size block
            	// as that we want to see has been removed
               	final long addr2 = bs.write(randomData(220)); // modify store
            	
            	store.commit();
            	bs.delete(addr2); // modify store
               	store.commit();
            	
               	// delete is actioned
            	for (int i = 0; i < 4000; i++) {
                   	assertFalse(bs.isCommitted(addrs.get(i)));
            	}
              } catch (InterruptedException e) {
			} finally {
            	store.destroy();
            }
		}
 

/**
 * Repeats the BlobBlobHeader of deferred frees, but also with blob data.
 */
public void test_stressBlobBlobHeaderBlobDataDeferredFrees() {

          final Properties properties = new Properties(getProperties());

          properties.setProperty(
                  AbstractTransactionService.Options.MIN_RELEASE_AGE, "4000");

          final int maxFixed = 3; // 192 bytes (3 * 64)
          properties.setProperty(RWStore.Options.ALLOCATION_SIZES,
          		"1,2," + maxFixed);
          
          final int maxAlloc = maxFixed * 64;
          final int leafEntries = maxAlloc / 8;
          final int headerEntries = maxAlloc / 4;
          final int threshold = headerEntries * leafEntries;
          
          final int nallocs = threshold << 3; // 8 times greater than allocation threshold

	Journal store = (Journal) getStore(properties);
          try {

          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	ArrayList<Long> addrs = new ArrayList<Long>();
          	for (int i = 0; i < nallocs; i++) {
          		addrs.add(bs.write(randomData(1024))); // mostly blob data
          	}
          	store.commit();

          	for (long addr : addrs) {
          		bs.delete(addr);
          	}
              for (int i = 0; i < nallocs; i++) {
                  if(!bs.isCommitted(addrs.get(i))) {
                      fail("i="+i+", addr="+addrs.get(i));
                  }
              }

             	store.commit();
             	
          	// Age the history (of the deletes!)
          	Thread.currentThread().sleep(6000);
          	
          	// modify store but do not allocate similar size block
          	// as that we want to see has been removed
             	final long addr2 = bs.write(randomData(1024)); // modify store
          	
          	store.commit();
          	bs.delete(addr2); // modify store
             	store.commit();
          	
             	// delete is actioned
          	for (int i = 0; i < nallocs; i++) {
                 	assertFalse(bs.isCommitted(addrs.get(i)));
          	}
            } catch (InterruptedException e) {
	} finally {
          	store.destroy();
          }
}
 

/**
 * State2
 * 
 * Allocate - Commit - Free - Commit
 * 
 * assert that allocation is no longer committed
 */
public void test_allocCommitFreeCommit() {
	Journal store = (Journal) getStore();
          try {

          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	assertTrue(bs.isCommitted(addr));

          	store.commit();

          	assertFalse(bs.isCommitted(addr));
          } finally {
          	store.destroy();
          }
}
 

public void testAllocationContexts() {
	
	final Journal store = (Journal) getStore();

	try {

		final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bufferStrategy.getStore();
		
		final IAllocationContext cntxt1 = rw.newAllocationContext(true/*isolated*/);
		
		final IAllocationContext cntxt2 = rw.newAllocationContext(true/*isolated*/);
		
		// allocate a global address
		final int gaddr = rw.alloc(412, null);
		
		store.commit();
		
		// allocate a context address grabbing previous global allocation
		int c1addr = rw.alloc(412, cntxt1);
		// imagine we are re-allocating the original address
		rw.free(gaddr, 412, cntxt1);				
		// now abort context
		rw.abortContext(cntxt1);
		
		store.commit();

		final long paddr = rw.physicalAddress(gaddr);
		
		assertTrue("Global allocation must be protected", paddr != 0);

	} finally {
		store.destroy();
	}
}
 

/**
 * In order to see deferred recycling we need to make two
 * commits (with modifications) after the retention period
 * has expired.
 * 
 * To be able to check for the address release, we must ensure that
 * the same address cannot be re-allocated, so a different size
 * allocation is requested.
 */
public void test_allocCommitFreeCommitWithHistory() {
	Journal store = (Journal) getStore(4);
          try {

          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	assertTrue(bs.isCommitted(addr));

             	store.commit();

          	// delete is deferred
          	assertTrue(bs.isCommitted(addr));
          	
          	Thread.currentThread().sleep(5000);
          	
          	// modify store but do not allocate similar size block
          	// as that we want to see has been removed
             	final long addr2 = bs.write(randomData(220)); // modify store
          	
          	store.commit();
          	bs.delete(addr2); // modify store
             	store.commit();
          	
             	// delete is actioned
          	assertFalse(bs.isCommitted(addr));
          	
          } catch (InterruptedException e) {

	} finally {
          	store.destroy();
          }
}
 

public static void main(String[] args) throws InterruptedException,
        ExecutionException {

    final Properties properties = new Properties();

    properties.setProperty(Journal.Options.FILE, "testJournal.jnl");
    
    final Journal store = new Journal(properties);

    try {

        /*
         * Register the index. There are a lot of options for the B+Tree,
         * but you only need to specify the index name and the UUID for the
         * index. Each store can hold multiple named indices.
         */
        {
            
            final IndexMetadata indexMetadata = new IndexMetadata(
                    "testIndex", UUID.randomUUID());

            store.registerIndex(indexMetadata);
            
            // commit the store so the index is on record.
            store.commit();
            
        }

        /*
         * Demonstrate that the each view has read/write concurrency against
         * the mutable B+Tree.
         */
        final List<Callable<Void>> tasks = new LinkedList<Callable<Void>>();

        final UnisolatedReadWriteIndex btree = new UnisolatedReadWriteIndex(
                store.getIndex("testIndex"));

        final int nops = 50000;

        tasks.add(new ReadWriteTask(btree, nops));

        tasks.add(new ReadWriteTask(btree, nops));

        tasks.add(new ReadWriteTask(btree, nops));

        // run tasks on the journal's executor service.
        final List<Future<Void>> futures = store.getExecutorService()
                .invokeAll(tasks);

        for(final Future<Void> future : futures) {
            
            // check for errors.
            future.get();
            
        }
        
        // commit the store.
        store.commit();
        
        // show #of operations executed and #of tuples in the B+Tree.
        System.out.println("nops=" + (nops * tasks.size())
                + ", rangeCount=" + btree.rangeCount());

        System.out.println(new Date().toString());

    } finally {

        // destroy the backing store.
        store.destroy();

    }

}
 

/**
 * Ensures the allocation of unique addresses by mapping allocated
 * address with uniqueness assertion against physical address.
 */
public void test_addressingContiguous() {

	final Journal store = (Journal) getStore();

	try {

	    final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bufferStrategy.getStore();
		final int cSlotSize = 128;
		final int cAllocSize = 99;
		
		long pap = rw.physicalAddress(rw.alloc(cAllocSize, null));
		for (int i = 0; i < 500000; i++) {
			final int a = rw.alloc(cAllocSize, null);
			final long pa = rw.physicalAddress(a);
			
			if (pa != (pap+cSlotSize)) {
				// for debug
				rw.physicalAddress(a);
				fail("Non-Contiguous slots: " + i + ", " + pa + "!=" + (pap+cSlotSize));
			}
			
			pap = pa;
			
		}
		
		store.commit();
		
		final StringBuilder sb = new StringBuilder();
		rw.showAllocators(sb);
		
		log.warn(sb.toString());

	} finally {

		store.destroy();

	}

}
 

/**
     * Test correct detection of a write-write conflict. An index is registered
     * and the journal is committed. Two transactions (tx1, tx2) are then
     * started. Both transactions write a value under the same key. tx1 prepares
     * and commits.  tx2 attempts to prepare, and the test verifies that a
     * {@link ValidationError} is reported.
     */
    public void test_writeWriteConflict_correctDetection() {

        final Journal journal = new Journal(getProperties());

        try {
        
        String name = "abc";

        final byte[] k1 = new byte[] { 1 };

        final byte[] v1a = new byte[] { 1 };
        final byte[] v1b = new byte[] { 2 };

        {

            /*
             * register an index and commit the journal.
             */
            
            IndexMetadata metadata = new IndexMetadata(name, UUID.randomUUID());
            
            metadata.setIsolatable(true);
            
            // Note: No conflict resolver.
            
            journal.registerIndex(name, BTree.create(journal,metadata) );

            journal.commit();

        }

        /*
         * Create two transactions.
         */
        
        final long tx1 = journal.newTx(ITx.UNISOLATED);
        
        final long tx2 = journal.newTx(ITx.UNISOLATED);
        
        /*
         * Write a value under the same key on the same index in both
         * transactions.
         */
        
        journal.getIndex(name,tx1).insert(k1, v1a);

        journal.getIndex(name,tx2).insert(k1, v1b);

        journal.commit(tx1);
        
        /*
         * verify that the value from tx1 is found under the key on the
         * unisolated index.
         */
        assertEquals(v1a,(byte[])journal.getIndex(name).lookup(k1));

//        final ITx tmp = journal.getTx(tx2);

        try {
            journal.commit(tx2);
            fail("Expecting: "+ValidationError.class);
        } catch(ValidationError ex) {
        	if(log.isInfoEnabled())
            	log.info("Ignoring expected exception: "+ex);
//            assertTrue(tmp.isAborted());
        }
        
        } finally {

            journal.destroy();
            
        }

    }
 

/**
 * Test of blob allocation and read-back, firstly from cache and then
 * from disk.
 * @throws InterruptedException 
 */
public void test_blob_readBack() {

	final Journal store = (Journal) getStore();

	try {
		final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bs.getStore();

		final byte[] buf = new byte[2 * 1024 * 1024]; // 5Mb buffer of random
												// data
		r.nextBytes(buf);

		final ByteBuffer bb = ByteBuffer.wrap(buf);

		final long faddr = bs.write(bb); // rw.alloc(buf, buf.length);

		if(log.isInfoEnabled())log.info("Blob Allocation at " + rw.convertFromAddr(faddr));

		bb.position(0);

		ByteBuffer rdBuf = bs.read(faddr);

		assertEquals(bb, rdBuf);

		if(log.isInfoEnabled())log.info("Now commit to disk");

		store.commit();

		// Now reset - clears writeCache and reinits from disk
		rw.reset();
		
		// RWStore.reset() no longer resets the write cache, we need to do that explicitly!
		rw.getWriteCacheService().resetAndClear();

		rdBuf = bs.read(faddr);
		assertEquals(bb, rdBuf);

	} catch (InterruptedException e) {
		throw new RuntimeException(e);
	} finally {

		store.destroy();

	}

}