Java Code Examples for com.bigdata.journal.Journal#getBufferStrategy()

public void test_multiVoidCommit() {

			final Journal store = (Journal) getStore();
            try {

			final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

			final RWStore rw = bs.getStore();
			
			final boolean initRequire = bs.requiresCommit(store.getRootBlockView());
			assertTrue(initRequire);
			for (int n = 0; n < 20; n++) {
				store.commit();
				final IRootBlockView rbv = store.getRootBlockView();
				assertTrue(1 == rbv.getCommitCounter());
				assertFalse(bs.requiresCommit(store.getRootBlockView()));
				assertFalse(rw.requiresCommit());
			}
			} finally {
				store.destroy();
			}
		}
 

public void test_metaAlloc() {

			Journal store = (Journal) getStore();
            try {

			final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

			final RWStore rw = bs.getStore();
			long realAddr = 0;
				for (int i = 0; i < 100000; i++) {
					int allocAddr = rw.metaAlloc();

					realAddr = rw.metaBit2Addr(allocAddr);
				}
				if(log.isInfoEnabled())log.info("metaAlloc lastAddr: " + realAddr);
			} finally {
				store.destroy();
			}
		}
 

private long pureAllocBatch(final Journal store, final int tsts, int min, int sze, int grp) {

		    final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

			final RWStore rw = bs.getStore();
			final int freeAddr[] = new int[512];
			int freeCurs = 0;
			for (int i = 0; i < grp; i++) {
				final int alloc = min + r.nextInt(sze - min);
				final int addr = rw.alloc(alloc, null);

				if (i % 3 != 0) { // make avail 2 out of 3 for realloc
					freeAddr[freeCurs++] = addr;
					if (freeCurs == freeAddr.length) {
						for (int f = 0; f < freeAddr.length; f++) {
							rw.free(freeAddr[f], 0);
						}
						freeCurs = 0;
					}
				}
			}

			return 0L;
		}
 

/**
 * State3
 * 
 * Allocate - Commit - Free - Commit
 * 
 * Tracks writeCache state through allocation
 */
public void test_allocCommitFreeCommitWriteCache() {
	Journal store = (Journal) getStore();
          try {
          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	// Has just been written so must be in cache
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	// since data is committed, should be accessible from any new
          	// readCommitted transaction
          	assertTrue(bs.inWriteCache(addr));

          	store.commit();
          	
          	// If no transactions are around (which they are not) then must
          	//	be released as may be re-allocated
          	assertFalse(bs.inWriteCache(addr));
          	
          } finally {
          	store.destroy();
          }
}
 

/**
 * Can be tested by removing RWStore call to journal.removeCommitRecordEntries
 * in freeDeferrals.
 * 
 * final int commitPointsRemoved = journal.removeCommitRecordEntries(fromKey, toKey);
 * 
 * replaced with
 * 
 * final int commitPointsRemoved = commitPointsRecycled;
 * 
 */
public void testVerifyCommitRecordIndex() {
          final Properties properties = new Properties(getProperties());

          properties.setProperty(
                  AbstractTransactionService.Options.MIN_RELEASE_AGE, "100");

	final Journal store = (Journal) getStore(properties);
          try {

          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	      	
       	for (int r = 0; r < 10; r++) {
        	ArrayList<Long> addrs = new ArrayList<Long>();
        	for (int i = 0; i < 100; i++) {
        		addrs.add(bs.write(randomData(45)));
        	}
        	store.commit();

        	for (long addr : addrs) {
        		bs.delete(addr);
        	}

           	store.commit();
           	
        	// Age the history (of the deletes!)
        	Thread.currentThread().sleep(200);
       	}
       	
		final String fname = bs.getStore().getStoreFile().getAbsolutePath();
		
       	store.close();
       	
       	VerifyCommitRecordIndex.main(new String[]{fname});
      	
          } catch (InterruptedException e) {
	} finally {
          	store.destroy();
          }
}
 

public void test_allocCommitFreeCommitWriteCacheWithHistory() {
	final Journal store = (Journal) getStore(5);
          try {
          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	// Has just been written so must be in cache
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	// since data is committed, should be accessible from any new
          	// readCommitted transaction
          	assertTrue(bs.inWriteCache(addr));

          	store.commit();
          	
          	// Since state is retained, the delete is deferred
          	assertTrue(bs.inWriteCache(addr));
          	
          } finally {
          	store.destroy();
          }
}
 

/**
 * State4
 * 
 * Allocate - Commit - Free - Commit
 * 
 * ..but with session protection using a RawTx
 */
public void test_allocCommitFreeCommitSessionWriteCache() {
	final Journal store = (Journal) getStore();
          try {
          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	// Has just been written so must be in cache
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.commit();
          	
          	IRawTx tx = bs.newTx();
          	
          	bs.delete(addr);
          	
          	// since data is committed, should be accessible from any new
          	// readCommitted transaction
          	assertTrue(bs.inWriteCache(addr));

          	store.commit();
          	
          	// Since transactions are around the cache will not have been
          	//	cleared (and cannot be re-allocated)
          	assertTrue(bs.inWriteCache(addr));
          	
          	tx.close(); // Release Session
          	
          	// Cache must have been cleared
          	assertFalse(bs.inWriteCache(addr));
          	
          } finally {
          	store.destroy();
          }
}
 

/**
       * Verify that a double-commit causes an illegal state exception.
       * Further verify that an {@link RWStore#reset()} allwos us to then
       * apply and commit new write sets.
       * 
       * @see <a href="http://trac.blazegraph.com/ticket/973" >RWStore commit is
       *      not robust to internal failure.</a>
       */
public void test_commitStateIllegal() {
	final Journal store = (Journal) getStore();
          try {

          final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final RWStore rws = bs.getStore();
          	
          	bs.write(randomData(78));
          	
          	rws.commit();
          	
          	try {
          		store.commit();
          		
          		fail("Expected failure");
          	} catch (Exception ise) {
          		if (InnerCause.isInnerCause(ise, IllegalStateException.class)) {
           		store.abort();
           		
           		store.commit();
          		} else {
          			fail("Unexpected Exception");
          		}
          	}
         	
          	
          } finally {
          	store.destroy();
          }
}
 

private long allocBatch(Journal store, int tsts, int min, int sze, int grp) {

			final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

			final byte[] buf = new byte[sze + 4]; // extra for checksum
			r.nextBytes(buf);

			for (int i = 0; i < grp; i++) {
				int alloc = min + r.nextInt(sze - min);
				ByteBuffer bb = ByteBuffer.wrap(buf, 0, alloc);
				bs.write(bb);
			}

			return 0L;
		}
 

/**
 * State2
 * 
 * Allocate - Commit - Free - Commit
 * 
 * assert that allocation is no longer committed
 */
public void test_allocCommitFreeCommit() {
	Journal store = (Journal) getStore();
          try {

          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	assertTrue(bs.isCommitted(addr));

          	store.commit();

          	assertFalse(bs.isCommitted(addr));
          } finally {
          	store.destroy();
          }
}
 

/**
 * Tests the recycling of small slot alloctors and outputs statistics related
 * to contiguous allocations indicative of reduced IOPS.
 */
public void test_smallSlotRecycling() {

	final Journal store = (Journal) getSmallSlotStore(1024);

	try {

	    final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bufferStrategy.getStore();
		final int cSlotSize = 128;
		final int cAllocSize = 99;
		
		int breaks = 0;
		int contiguous = 0;
		
		ArrayList<Integer> recycle = new ArrayList<Integer>();
		
		long pap = rw.physicalAddress(rw.alloc(cAllocSize, null));
		for (int i = 0; i < 500000; i++) {
			final int a = rw.alloc(cSlotSize, null);
			final long pa = rw.physicalAddress(a);
			
			if (r.nextInt(7) < 5) { // more than 50% recycle
				recycle.add(a);
			}
			
			if (pa == (pap+cSlotSize)) {
				contiguous++;
			} else {
				breaks++;
			}
			
			pap = pa;
			
			if (recycle.size() > 5000) {
				log.warn("Transient Frees for immediate recyling");
				for (int e : recycle) {
					rw.free(e, cAllocSize);
				}
				recycle.clear();
			}
		}
		
		store.commit();
		
		final StringBuilder sb = new StringBuilder();
		rw.showAllocators(sb);
		
		log.warn("Contiguous: " + contiguous + ", breaks: " + breaks + "\n" + sb.toString());

	} finally {

		store.destroy();

	}

}
 

/**
 * Repeats the BlobBlobHeader of deferred frees, but also with blob data.
 */
public void test_stressBlobBlobHeaderBlobDataDeferredFrees() {

          final Properties properties = new Properties(getProperties());

          properties.setProperty(
                  AbstractTransactionService.Options.MIN_RELEASE_AGE, "4000");

          final int maxFixed = 3; // 192 bytes (3 * 64)
          properties.setProperty(RWStore.Options.ALLOCATION_SIZES,
          		"1,2," + maxFixed);
          
          final int maxAlloc = maxFixed * 64;
          final int leafEntries = maxAlloc / 8;
          final int headerEntries = maxAlloc / 4;
          final int threshold = headerEntries * leafEntries;
          
          final int nallocs = threshold << 3; // 8 times greater than allocation threshold

	Journal store = (Journal) getStore(properties);
          try {

          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	ArrayList<Long> addrs = new ArrayList<Long>();
          	for (int i = 0; i < nallocs; i++) {
          		addrs.add(bs.write(randomData(1024))); // mostly blob data
          	}
          	store.commit();

          	for (long addr : addrs) {
          		bs.delete(addr);
          	}
              for (int i = 0; i < nallocs; i++) {
                  if(!bs.isCommitted(addrs.get(i))) {
                      fail("i="+i+", addr="+addrs.get(i));
                  }
              }

             	store.commit();
             	
          	// Age the history (of the deletes!)
          	Thread.currentThread().sleep(6000);
          	
          	// modify store but do not allocate similar size block
          	// as that we want to see has been removed
             	final long addr2 = bs.write(randomData(1024)); // modify store
          	
          	store.commit();
          	bs.delete(addr2); // modify store
             	store.commit();
          	
             	// delete is actioned
          	for (int i = 0; i < nallocs; i++) {
                 	assertFalse(bs.isCommitted(addrs.get(i)));
          	}
            } catch (InterruptedException e) {
	} finally {
          	store.destroy();
          }
}
 

public void testSimpleUnisolatedAllocationContextRecycling() {
	
	final Journal store = (Journal) getStore(0);

	try {

		final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bufferStrategy.getStore();
		
		final IAllocationContext cntxt = rw.newAllocationContext(false /*Isolated*/); // Unisolated context
		
		// allocate three global addresses of different sizes
		final int sze1 = 48; // 64 bytes allocator
		final int sze2 = 72; // 128	
		final int sze3 = 135; // 256
		
		final int addr1 = rw.alloc(sze1, cntxt);
		final int addr2 = rw.alloc(sze2, cntxt);
		final int addr3 = rw.alloc(sze3, cntxt);
		
		// Commit the addresses
		store.commit();
		
		showAddress(rw, addr1);
		showAddress(rw, addr2);
		showAddress(rw, addr3);
		
		// Now create some allocation contexts
		final IAllocationContext iso_cntxt = rw.newAllocationContext(true /*Isolated*/); // Isolated context
		
		// now allocate a new unisolated address
		rw.alloc(sze1, cntxt);
		// free the originall
		rw.free(addr1,  sze1, cntxt);
		// and ensure that allocator is 'owned' by an iso
		rw.alloc(sze1, iso_cntxt);
		
		// now grab a pristine allocator
		rw.alloc(sze2,  iso_cntxt);
		// and free the original address (with the unisolated context)
		rw.free(addr2, sze2, cntxt);				
		
		if (log.isInfoEnabled()) {
			final StringBuilder str = new StringBuilder();
			rw.showAllocators(str);
			
			log.info(str);
		}	
		
		store.abort();
		
		log.info("ABORT");
		showAddress(rw, addr1);
		showAddress(rw, addr2);
		showAddress(rw, addr3);
		
		// This is the point that the AllocationContext ends up setting the free bits, iff we have committed after the abort.				
		rw.detachContext(iso_cntxt);
		
		log.info("DETACH");
		showAddress(rw, addr1);
		showAddress(rw, addr2);
		showAddress(rw, addr3);
		
		store.commit();
		
		log.info("COMMIT");
		showAddress(rw, addr1);
		showAddress(rw, addr2);
		showAddress(rw, addr3);
		
	} finally {
		store.destroy();
	}
}
 

void doBlobDeferredFrees(final int cAddrs) {

			final Properties properties = new Properties(getProperties());

            properties.setProperty(
                    AbstractTransactionService.Options.MIN_RELEASE_AGE, "4000");

            properties.setProperty(RWStore.Options.ALLOCATION_SIZES,
                    "1,2,3,5,8,12,16"); // 1K

			final Journal store = (Journal) getStore(properties);
            try {

            	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
            	
            	final ArrayList<Long> addrs = new ArrayList<Long>();
            	for (int i = 0; i < cAddrs; i++) {
            		addrs.add(bs.write(randomData(45)));
            	}
            	store.commit();

            	for (long addr : addrs) {
            		bs.delete(addr);
            	}
                for (int i = 0; i < cAddrs; i++) {
                    if(!bs.isCommitted(addrs.get(i))) {
                        fail("i="+i+", addr="+addrs.get(i));
                    }
                }

               	store.commit();
               	
            	// Age the history (of the deletes!)
            	Thread.currentThread().sleep(6000);
            	
            	// modify store but do not allocate similar size block
            	// as that we want to see has been removed
               	final long addr2 = bs.write(randomData(220)); // modify store
            	
            	store.commit();
            	bs.delete(addr2); // modify store
               	store.commit();
            	
               	// delete is actioned
            	for (int i = 0; i < 4000; i++) {
                   	assertFalse(bs.isCommitted(addrs.get(i)));
            	}
              } catch (InterruptedException e) {
			} finally {
            	store.destroy();
            }
		}
 

/**
 * From a RWStore, creates multiple AllocationContexts to isolate
 * updates, re-allocate storage and protect against by concurrent
 * Contexts. This is the core functionality required to support
 * Transactions.
 * 
 * If an allocation is made for an AllocationContext then this will
 * result in a ContextAllocation object being created in the RWStore
 * within which "shadow" allocations can be made. If such a shadow
 * allocation is deleted, within the AllocationContext, then this can be
 * removed immediately.
 * 
 * @throws IOException
 */
public void test_allocationContexts() throws IOException {
	final Journal store = (Journal) getStore();
	try {
	    final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bs.getStore();

		// JournalShadow shadow = new JournalShadow(store);

		// Create a couple of contexts
		final IAllocationContext allocContext1 = rw.newAllocationContext(true);
		final IAllocationContext allocContext2 = rw.newAllocationContext(true);

		final int sze = 650;
		final byte[] buf = new byte[sze + 4]; // extra for checksum
		r.nextBytes(buf);

		final long addr1a = bs.write(ByteBuffer.wrap(buf), allocContext1);
		final long addr1b = bs.write(ByteBuffer.wrap(buf), allocContext1);
		rw.detachContext(allocContext1);

		final long addr2a = bs.write(ByteBuffer.wrap(buf), allocContext2);
		final long addr2b = bs.write(ByteBuffer.wrap(buf), allocContext2);
		rw.detachContext(allocContext2);

		// Attempt to re-establish context
		try {
			bs.write(ByteBuffer.wrap(buf), allocContext1);
			fail("Should have failed to re-use detached context");
		} catch (IllegalStateException ise) {
			// Expected
		}
		
		final IAllocationContext allocContext3 = rw.newAllocationContext(true);
		final long addr1c = bs.write(ByteBuffer.wrap(buf), allocContext3);

		// By detaching contexts we end up using the same allocator
		assertTrue("allocator re-use", bs.getPhysicalAddress(addr1c) > bs.getPhysicalAddress(addr2b));

		// Now, prior to commit, try deleting an uncommitted allocation
		bs.delete(addr1c, allocContext3);
		// and re-allocating it from the same context
		final long addr1d = bs.write(ByteBuffer.wrap(buf), allocContext3);

		assertTrue("re-allocation", addr1c == addr1d);

		rw.detachContext(allocContext3);

		// Now commit
		store.commit();

		// now try deleting and re-allocating again, but in a global
		// context
		bs.delete(addr1d); // this should call deferFree
		final long addr1e = bs.write(ByteBuffer.wrap(buf));

		assertTrue("deferred-delete", addr1e != addr1d);

		// Now commit
		store.commit();
	} finally {
		store.destroy();
	}

}
 

/**
 * Verify normal operation and basic assumptions when creating a new journal
 * using {@link BufferMode#DiskRW}.
 * 
 * @throws IOException
 */
public void test_create_disk01() throws IOException {

	File file = null;

	final Properties properties = getProperties();

	final Journal journal = new Journal(properties);

	try {

		final RWStrategy bufferStrategy = (RWStrategy) journal.getBufferStrategy();

		assertTrue("isStable", bufferStrategy.isStable());
		assertFalse("isFullyBuffered", bufferStrategy.isFullyBuffered());
		// assertEquals(Options.FILE, properties.getProperty(Options.FILE),
		// bufferStrategy.file.toString());
		assertEquals(Options.INITIAL_EXTENT, Long.parseLong(Options.DEFAULT_INITIAL_EXTENT), bufferStrategy
				.getInitialExtent());
		assertEquals(Options.MAXIMUM_EXTENT, 0L/* soft limit for disk mode */, bufferStrategy.getMaximumExtent());
		// assertNotNull("raf", bufferStrategy.getRandomAccessFile());
		assertEquals(Options.BUFFER_MODE, BufferMode.DiskRW, bufferStrategy.getBufferMode());

		file = journal.getFile();

	} finally {

		journal.destroy();

	}

	if (file != null && file.exists())
		fail("Did not delete the backing file: " + file);

}
 

public void testAllocationContexts() {
	
	final Journal store = (Journal) getStore();

	try {

		final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bufferStrategy.getStore();
		
		final IAllocationContext cntxt1 = rw.newAllocationContext(true/*isolated*/);
		
		final IAllocationContext cntxt2 = rw.newAllocationContext(true/*isolated*/);
		
		// allocate a global address
		final int gaddr = rw.alloc(412, null);
		
		store.commit();
		
		// allocate a context address grabbing previous global allocation
		int c1addr = rw.alloc(412, cntxt1);
		// imagine we are re-allocating the original address
		rw.free(gaddr, 412, cntxt1);				
		// now abort context
		rw.abortContext(cntxt1);
		
		store.commit();

		final long paddr = rw.physicalAddress(gaddr);
		
		assertTrue("Global allocation must be protected", paddr != 0);

	} finally {
		store.destroy();
	}
}
 

/**
 * Tests whether tasks are able to access and modify data safely by
 * emulating transactions by calling activateTx and deactivateTx
 * directly.
 */
public void test_sessionProtection() {
	// Sequential logic

	final Journal store = (Journal) getStore();
	try {
	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
	final RWStore rw = bs.getStore();

	final byte[] buf = new byte[300]; // Just some data
	r.nextBytes(buf);

	final ByteBuffer bb = ByteBuffer.wrap(buf);

	long faddr = bs.write(bb); // rw.alloc(buf, buf.length);

	IRawTx tx = rw.newTx();

	bs.delete(faddr); // delettion protected by session

	bb.position(0);

	final ByteBuffer rdBuf = bs.read(faddr);

	// should be able to successfully read from freed address
	assertEquals(bb, rdBuf);

	tx.close();

	store.commit();
	} finally {
	    store.destroy();
	}
	
}
 

/**
 * Ttest write() + flush() + update() - for this case the data have been
 * flushed from the write cache so the update will be a random write on
 * the file rather than being buffered by the write cache.
 */
public void test_write_flush_update() {

	final Journal store = (Journal) getStore();

	try {

		RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final int nbytes = 60;

		// random data.
		byte[] a = new byte[nbytes];
		r.nextBytes(a);

		// write a new record.
		final long addr = bufferStrategy.write(ByteBuffer.wrap(a));

		assertEquals(nbytes, store.getByteCount(addr));

		// Note: This will result flush the write cache.
		store.commit();

		/*
		 * Read back the record and verify the update is visible.
		 */
		{

			final ByteBuffer b = bufferStrategy.read(addr);

			assertNotNull(b);

			for (int i = 20; i < 40; i++) {

				assertEquals("data differs at offset=" + i, a[i], b.get(i));

			}

		}

	} finally {

		store.destroy();

	}

}
 

/**
 * Tests whether tasks are able to access and modify data safely by
 * emulating transactions by calling activateTx and deactivateTx
 * directly.
 */
public void test_sessionProtection() {

       final Properties p = getProperties();

       // Note: No longer the default. Must be explicitly set.
       p.setProperty(AbstractTransactionService.Options.MIN_RELEASE_AGE, "0");

	final Journal store = getJournal(p);
	try {
		final IRWStrategy bs = (IRWStrategy) store.getBufferStrategy();
	
		final byte[] buf = new byte[300]; // Just some data
		r.nextBytes(buf);
	
		final ByteBuffer bb = ByteBuffer.wrap(buf);
	
		final long faddr = store.write(bb); // rw.alloc(buf, buf.length);
	
		final IRawTx tx = bs.newTx();
	
		ByteBuffer rdBuf = store.read(faddr);
	
		bb.position(0);
	
		assertEquals(bb, rdBuf);
	
		store.delete(faddr); // deletion protected by session
	
		bb.position(0);
	
		rdBuf = store.read(faddr);
	
		// should be able to successfully read from freed address
		assertEquals(bb, rdBuf);
	
		tx.close();
	
		store.commit();
	} finally {
	    store.destroy();
	}
	
}