Java Code Examples for com.bigdata.journal.Journal#commit()

/**
 * Create and populate relation in the {@link #namespace}.
 */
private void loadData(final Journal store) {

    // create the relation.
    final R rel = new R(store, namespace, ITx.UNISOLATED, new Properties());
    rel.create();

    // data to insert.
    final E[] a = {//
            new E("John", "Mary"),// 
            new E("Mary", "Paul"),// 
            new E("Paul", "Leon"),// 
            new E("Leon", "Paul"),// 
            new E("Mary", "John"),// 
    };

    // insert data (the records are not pre-sorted).
    rel.insert(new ChunkedArrayIterator<E>(a.length, a, null/* keyOrder */));

    // Do commit since not scale-out.
    store.commit();

}
 

/**
 * Create and populate relation in the {@link #namespace}.
 */
private void loadData(final Journal store) {

    // create the relation.
    final R rel = new R(store, namespace, ITx.UNISOLATED, new Properties());
    rel.create();

    // data to insert.
    final E[] a = {//
            new E("John", "Mary"),// 
            new E("Mary", "Paul"),// 
            new E("Paul", "Leon"),// 
            new E("Leon", "Paul"),// 
            new E("Mary", "John"),// 
    };

    // insert data (the records are not pre-sorted).
    rel.insert(new ChunkedArrayIterator<E>(a.length, a, null/* keyOrder */));

    // Do commit since not scale-out.
    store.commit();

}
 

public void test_multiVoidCommit() {

			final Journal store = (Journal) getStore();
            try {

			final RWStrategy bs = (RWStrategy) store.getBufferStrategy();

			final RWStore rw = bs.getStore();
			
			final boolean initRequire = bs.requiresCommit(store.getRootBlockView());
			assertTrue(initRequire);
			for (int n = 0; n < 20; n++) {
				store.commit();
				final IRootBlockView rbv = store.getRootBlockView();
				assertTrue(1 == rbv.getCommitCounter());
				assertFalse(bs.requiresCommit(store.getRootBlockView()));
				assertFalse(rw.requiresCommit());
			}
			} finally {
				store.destroy();
			}
		}
 

/**
* Create and populate relation in the {@link #namespace}.
* 
* FIXME The {@link DistinctTermAdvancer} is IV specific code. We need to
* use a R(elation) and E(lement) type that use IVs to write this test.
*/
  private void loadData(final Journal store) {

      // create the relation.
      final R rel = new R(store, namespace, ITx.UNISOLATED, new Properties());
      rel.create();

      // data to insert.
      final E[] a = {//
              new E("John", "Mary"),// 
              new E("Mary", "Paul"),// 
              new E("Paul", "Leon"),// 
              new E("Leon", "Paul"),// 
              new E("Mary", "John"),// 
      };

      // insert data (the records are not pre-sorted).
      rel.insert(new ChunkedArrayIterator<E>(a.length, a, null/* keyOrder */));

      // Do commit since not scale-out.
      store.commit();

  }
 

public void test_allocCommitFreeWithHistory() {
	final Journal store = (Journal) getStore(4);
          try {

          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	assertTrue(bs.isCommitted(addr));
          } finally {
          	store.destroy();
          }
}
 

/**
       * Verify that we correctly restore the RWStore commit state if
       * {@link RWStore#commit()} is followed by {@link RWStore#reset()}
       * rather than {@link RWStore#postCommit()}.
       * 
       * @see <a href="http://trac.blazegraph.com/ticket/973" >RWStore commit is
       *      not robust to internal failure.</a>
       */
public void test_commitState() {
	Journal store = (Journal) getStore();
          try {

          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final RWStore rws = bs.getStore();
          	
          	final long addr = bs.write(randomData(78));

          	// do 1st half of the RWStore commit protocol.
          	rws.commit();
          	            	
          	// then discard write set.
          	store.abort();
          	
          	assertFalse(bs.isCommitted(addr)); // rolled back
          	
          	// now cycle standard commit to confirm correct reset
          	for (int c = 0; c < 50; c++) {
          		bs.write(randomData(78));
          		store.commit();
          	}
         	
          	
          } finally {
          	store.destroy();
          }
}
 

/**
 * State4
 * 
 * Allocate - Commit - Free - Commit
 * 
 * ..but with session protection using a RawTx
 */
public void test_allocCommitFreeCommitSessionWriteCache() {
	final Journal store = (Journal) getStore();
          try {
          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	// Has just been written so must be in cache
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.commit();
          	
          	IRawTx tx = bs.newTx();
          	
          	bs.delete(addr);
          	
          	// since data is committed, should be accessible from any new
          	// readCommitted transaction
          	assertTrue(bs.inWriteCache(addr));

          	store.commit();
          	
          	// Since transactions are around the cache will not have been
          	//	cleared (and cannot be re-allocated)
          	assertTrue(bs.inWriteCache(addr));
          	
          	tx.close(); // Release Session
          	
          	// Cache must have been cleared
          	assertFalse(bs.inWriteCache(addr));
          	
          } finally {
          	store.destroy();
          }
}
 

public void testResetHARootBlock() {
          final Properties properties = new Properties(getProperties());

		final Journal store = (Journal) getStore(properties);
          try {

          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	final RWStore rw = bs.getStore();
          	
        	for (int r = 0; r < 10; r++) {
        	ArrayList<Long> addrs = new ArrayList<Long>();
        	for (int i = 0; i < 1000; i++) {
        		addrs.add(bs.write(randomData(2048)));
        	}
        	store.commit();
        	
        	final StorageStats stats1 = rw.getStorageStats();
        	
        	rw.resetFromHARootBlock(store.getRootBlockView());
        	
        	final StorageStats stats2 = rw.getStorageStats();

        	// Now check that we can read all allocations after reset
        	for (long addr : addrs) {
        		store.read(addr);
        	}

       	}
       	
		final String fname = bs.getStore().getStoreFile().getAbsolutePath();
		
       	store.close();
       	
		} finally {
          	store.destroy();
          }
}
 

public void test_allocCommitFreeCommitWriteCacheWithHistory() {
	final Journal store = (Journal) getStore(5);
          try {
          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	// Has just been written so must be in cache
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	// since data is committed, should be accessible from any new
          	// readCommitted transaction
          	assertTrue(bs.inWriteCache(addr));

          	store.commit();
          	
          	// Since state is retained, the delete is deferred
          	assertTrue(bs.inWriteCache(addr));
          	
          } finally {
          	store.destroy();
          }
}
 

/**
 * Tests the recycling of small slot alloctors and outputs statistics related
 * to contiguous allocations indicative of reduced IOPS.
 */
public void test_smallSlotRecycling() {

	final Journal store = (Journal) getSmallSlotStore(1024);

	try {

	    final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bufferStrategy.getStore();
		final int cSlotSize = 128;
		final int cAllocSize = 99;
		
		int breaks = 0;
		int contiguous = 0;
		
		ArrayList<Integer> recycle = new ArrayList<Integer>();
		
		long pap = rw.physicalAddress(rw.alloc(cAllocSize, null));
		for (int i = 0; i < 500000; i++) {
			final int a = rw.alloc(cSlotSize, null);
			final long pa = rw.physicalAddress(a);
			
			if (r.nextInt(7) < 5) { // more than 50% recycle
				recycle.add(a);
			}
			
			if (pa == (pap+cSlotSize)) {
				contiguous++;
			} else {
				breaks++;
			}
			
			pap = pa;
			
			if (recycle.size() > 5000) {
				log.warn("Transient Frees for immediate recyling");
				for (int e : recycle) {
					rw.free(e, cAllocSize);
				}
				recycle.clear();
			}
		}
		
		store.commit();
		
		final StringBuilder sb = new StringBuilder();
		rw.showAllocators(sb);
		
		log.warn("Contiguous: " + contiguous + ", breaks: " + breaks + "\n" + sb.toString());

	} finally {

		store.destroy();

	}

}
 

void doBlobDeferredFrees(final int cAddrs) {

			final Properties properties = new Properties(getProperties());

            properties.setProperty(
                    AbstractTransactionService.Options.MIN_RELEASE_AGE, "4000");

            properties.setProperty(RWStore.Options.ALLOCATION_SIZES,
                    "1,2,3,5,8,12,16"); // 1K

			final Journal store = (Journal) getStore(properties);
            try {

            	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
            	
            	final ArrayList<Long> addrs = new ArrayList<Long>();
            	for (int i = 0; i < cAddrs; i++) {
            		addrs.add(bs.write(randomData(45)));
            	}
            	store.commit();

            	for (long addr : addrs) {
            		bs.delete(addr);
            	}
                for (int i = 0; i < cAddrs; i++) {
                    if(!bs.isCommitted(addrs.get(i))) {
                        fail("i="+i+", addr="+addrs.get(i));
                    }
                }

               	store.commit();
               	
            	// Age the history (of the deletes!)
            	Thread.currentThread().sleep(6000);
            	
            	// modify store but do not allocate similar size block
            	// as that we want to see has been removed
               	final long addr2 = bs.write(randomData(220)); // modify store
            	
            	store.commit();
            	bs.delete(addr2); // modify store
               	store.commit();
            	
               	// delete is actioned
            	for (int i = 0; i < 4000; i++) {
                   	assertFalse(bs.isCommitted(addrs.get(i)));
            	}
              } catch (InterruptedException e) {
			} finally {
            	store.destroy();
            }
		}
 

/**
 * State2
 * 
 * Allocate - Commit - Free - Commit
 * 
 * assert that allocation is no longer committed
 */
public void test_allocCommitFreeCommit() {
	Journal store = (Journal) getStore();
          try {

          	RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	store.commit();
          	
          	bs.delete(addr);
          	
          	assertTrue(bs.isCommitted(addr));

          	store.commit();

          	assertFalse(bs.isCommitted(addr));
          } finally {
          	store.destroy();
          }
}
 

/**
 * Tests whether tasks are able to access and modify data safely by
 * emulating transactions by calling activateTx and deactivateTx
 * directly.
 */
public void test_sessionProtection() {
	// Sequential logic

	final Journal store = (Journal) getStore();
	try {
	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
	final RWStore rw = bs.getStore();

	final byte[] buf = new byte[300]; // Just some data
	r.nextBytes(buf);

	final ByteBuffer bb = ByteBuffer.wrap(buf);

	long faddr = bs.write(bb); // rw.alloc(buf, buf.length);

	IRawTx tx = rw.newTx();

	bs.delete(faddr); // delettion protected by session

	bb.position(0);

	final ByteBuffer rdBuf = bs.read(faddr);

	// should be able to successfully read from freed address
	assertEquals(bb, rdBuf);

	tx.close();

	store.commit();
	} finally {
	    store.destroy();
	}
	
}
 

public static void main(String[] args) throws InterruptedException,
        ExecutionException {

    final Properties properties = new Properties();

    properties.setProperty(Journal.Options.FILE, "testJournal.jnl");
    
    final Journal store = new Journal(properties);

    try {

        /*
         * Register the index. There are a lot of options for the B+Tree,
         * but you only need to specify the index name and the UUID for the
         * index. Each store can hold multiple named indices.
         */
        {
            
            final IndexMetadata indexMetadata = new IndexMetadata(
                    "testIndex", UUID.randomUUID());

            store.registerIndex(indexMetadata);
            
            // commit the store so the index is on record.
            store.commit();
            
        }

        /*
         * Demonstrate that the each view has read/write concurrency against
         * the mutable B+Tree.
         */
        final List<Callable<Void>> tasks = new LinkedList<Callable<Void>>();

        final UnisolatedReadWriteIndex btree = new UnisolatedReadWriteIndex(
                store.getIndex("testIndex"));

        final int nops = 50000;

        tasks.add(new ReadWriteTask(btree, nops));

        tasks.add(new ReadWriteTask(btree, nops));

        tasks.add(new ReadWriteTask(btree, nops));

        // run tasks on the journal's executor service.
        final List<Future<Void>> futures = store.getExecutorService()
                .invokeAll(tasks);

        for(final Future<Void> future : futures) {
            
            // check for errors.
            future.get();
            
        }
        
        // commit the store.
        store.commit();
        
        // show #of operations executed and #of tuples in the B+Tree.
        System.out.println("nops=" + (nops * tasks.size())
                + ", rangeCount=" + btree.rangeCount());

        System.out.println(new Date().toString());

    } finally {

        // destroy the backing store.
        store.destroy();

    }

}
 

/**
 * Ensures the allocation of unique addresses by mapping allocated
 * address with uniqueness assertion against physical address.
 */
public void test_addressingContiguous() {

	final Journal store = (Journal) getStore();

	try {

	    final RWStrategy bufferStrategy = (RWStrategy) store.getBufferStrategy();

		final RWStore rw = bufferStrategy.getStore();
		final int cSlotSize = 128;
		final int cAllocSize = 99;
		
		long pap = rw.physicalAddress(rw.alloc(cAllocSize, null));
		for (int i = 0; i < 500000; i++) {
			final int a = rw.alloc(cAllocSize, null);
			final long pa = rw.physicalAddress(a);
			
			if (pa != (pap+cSlotSize)) {
				// for debug
				rw.physicalAddress(a);
				fail("Non-Contiguous slots: " + i + ", " + pa + "!=" + (pap+cSlotSize));
			}
			
			pap = pa;
			
		}
		
		store.commit();
		
		final StringBuilder sb = new StringBuilder();
		rw.showAllocators(sb);
		
		log.warn(sb.toString());

	} finally {

		store.destroy();

	}

}
 

/**
 * Reads counters from XML files into a Journal.
 * 
 * @param args
 * @throws SAXException
 * @throws ParserConfigurationException
 * @throws IOException
 */
static public void main(String[] args) throws IOException,
        ParserConfigurationException, SAXException {

    final Properties properties = new Properties();

    // @todo config
    properties.setProperty(Journal.Options.FILE,"counters.jnl");
    
    // (re-)open the store.
    final Journal store = new Journal(properties);

    // (re-)open/create the counter set B+Tree on the store.
    CounterSetBTree btree = (CounterSetBTree) store.getIndex("counters");

    if (btree == null) {

        // not registered, so create and register it now.
        btree = CounterSetBTree.create(store);

        store.registerIndex("counters", btree);

        // commit the registered index.
        store.commit();
        
    }
    
    /*
     * Process each file, loading data into the counter set B+Tree. 
     */
    for (String s : args) {

        final File file = new File(s);

        loadFile(btree, file);
        
    } // next source file.

    System.err.println("There are " + btree.rangeCount()
            + " counter values covering "
            + new Date(btree.getFirstTimestamp()) + " to "
            + new Date(btree.getLastTimestamp()));

}
 

/**
 * State5
 * 
 * Allocate - Commit - Free - Commit
 * 
 * Tracks writeCache state through allocation
 */
public void test_allocCommitFreeCommitAllocSessionWriteCache() {
	final Journal store = (Journal) getStore();
          try {
          	final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
          	
          	final long addr = bs.write(randomData(78));
          	
          	// Has just been written so must be in cache
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.commit();
          	
          	IRawTx tx = bs.newTx();
          	
          	bs.delete(addr);
          	
          	// since data is committed, should be accessible from any new
          	// readCommitted transaction
          	assertTrue(bs.inWriteCache(addr));

          	store.commit();
          	
          	// Since transactions are around the cache will not have been
          	//	cleared (and cannot be re-allocated)
          	assertTrue(bs.inWriteCache(addr));
          	
          	tx.close(); // Release Session
          	
          	// Cache must have been cleared
          	assertFalse(bs.inWriteCache(addr));
          	
          	final long addr2 = bs.write(randomData(78));
          	
          	assertTrue(addr2 == addr);
          	
          	assertTrue(bs.inWriteCache(addr));
          	
          	store.abort(); // removes uncommitted data
          	
          	assertFalse(bs.inWriteCache(addr));

          } finally {
          	store.destroy();
          }
}
 

/**
     * Test correct detection of a write-write conflict. An index is registered
     * and the journal is committed. Two transactions (tx1, tx2) are then
     * started. Both transactions write a value under the same key. tx1 prepares
     * and commits.  tx2 attempts to prepare, and the test verifies that a
     * {@link ValidationError} is reported.
     */
    public void test_writeWriteConflict_correctDetection() {

        final Journal journal = new Journal(getProperties());

        try {
        
        String name = "abc";

        final byte[] k1 = new byte[] { 1 };

        final byte[] v1a = new byte[] { 1 };
        final byte[] v1b = new byte[] { 2 };

        {

            /*
             * register an index and commit the journal.
             */
            
            IndexMetadata metadata = new IndexMetadata(name, UUID.randomUUID());
            
            metadata.setIsolatable(true);
            
            // Note: No conflict resolver.
            
            journal.registerIndex(name, BTree.create(journal,metadata) );

            journal.commit();

        }

        /*
         * Create two transactions.
         */
        
        final long tx1 = journal.newTx(ITx.UNISOLATED);
        
        final long tx2 = journal.newTx(ITx.UNISOLATED);
        
        /*
         * Write a value under the same key on the same index in both
         * transactions.
         */
        
        journal.getIndex(name,tx1).insert(k1, v1a);

        journal.getIndex(name,tx2).insert(k1, v1b);

        journal.commit(tx1);
        
        /*
         * verify that the value from tx1 is found under the key on the
         * unisolated index.
         */
        assertEquals(v1a,(byte[])journal.getIndex(name).lookup(k1));

//        final ITx tmp = journal.getTx(tx2);

        try {
            journal.commit(tx2);
            fail("Expecting: "+ValidationError.class);
        } catch(ValidationError ex) {
        	if(log.isInfoEnabled())
            	log.info("Ignoring expected exception: "+ex);
//            assertTrue(tmp.isAborted());
        }
        
        } finally {

            journal.destroy();
            
        }

    }
 

/**
 * Tests whether tasks are able to access and modify data safely by
 * emulating transactions by calling activateTx and deactivateTx
 * directly.
 */
public void test_sessionProtection() {

       final Properties p = getProperties();

       // Note: No longer the default. Must be explicitly set.
       p.setProperty(AbstractTransactionService.Options.MIN_RELEASE_AGE, "0");

	final Journal store = getJournal(p);
	try {
		final IRWStrategy bs = (IRWStrategy) store.getBufferStrategy();
	
		final byte[] buf = new byte[300]; // Just some data
		r.nextBytes(buf);
	
		final ByteBuffer bb = ByteBuffer.wrap(buf);
	
		final long faddr = store.write(bb); // rw.alloc(buf, buf.length);
	
		final IRawTx tx = bs.newTx();
	
		ByteBuffer rdBuf = store.read(faddr);
	
		bb.position(0);
	
		assertEquals(bb, rdBuf);
	
		store.delete(faddr); // deletion protected by session
	
		bb.position(0);
	
		rdBuf = store.read(faddr);
	
		// should be able to successfully read from freed address
		assertEquals(bb, rdBuf);
	
		tx.close();
	
		store.commit();
	} finally {
	    store.destroy();
	}
	
}
 

/**
 * Extends the basic behavior to force a commit of the {@link Journal}.
 * This makes the {@link Journal} appear to have "auto-commit" semantics
 * from the perspective of the unit tests that are written to the assumption
 * that the {@link IIndexManager} is an {@link IBigdataFederation}.
 * Otherwise those unit tests tend not to force a commit and hence
 * restart-safe tests tend to fail for one reason or another.
 */
protected Journal reopenStore(final Journal store) {

    store.commit();
    
    return super.reopenStore(store);
    
}