it.unimi.dsi.fastutil.io.FastByteArrayOutputStream Java Examples

public void testSkipSelfDelimUTF8() throws IOException {
	final FastByteArrayOutputStream fastByteArrayOutputStream = new FastByteArrayOutputStream();
	new MutableString( "a" ).writeSelfDelimUTF8( fastByteArrayOutputStream );
	new MutableString( "b" ).writeSelfDelimUTF8( fastByteArrayOutputStream );
	new MutableString( "\u221E" ).writeSelfDelimUTF8( fastByteArrayOutputStream );
	new MutableString( "c" ).writeSelfDelimUTF8( fastByteArrayOutputStream );
	fastByteArrayOutputStream.flush();
	final FastByteArrayInputStream fastByteArrayInputStream = new FastByteArrayInputStream( fastByteArrayOutputStream.array );
	assertEquals( "a", new MutableString().readSelfDelimUTF8( fastByteArrayInputStream ).toString() );
	assertEquals( "b", new MutableString().readSelfDelimUTF8( fastByteArrayInputStream ).toString() );
	assertEquals( 1, MutableString.skipSelfDelimUTF8( fastByteArrayInputStream ) );
	assertEquals( "c", new MutableString().readSelfDelimUTF8( fastByteArrayInputStream ).toString() );
	fastByteArrayInputStream.position( 0 );
	assertEquals( "a", new MutableString().readSelfDelimUTF8( fastByteArrayInputStream ).toString() );
	assertEquals( 1, MutableString.skipSelfDelimUTF8( fastByteArrayInputStream ) );
	assertEquals( "\uu221E", new MutableString().readSelfDelimUTF8( fastByteArrayInputStream ).toString() );
	assertEquals( "c", new MutableString().readSelfDelimUTF8( fastByteArrayInputStream ).toString() );
}
 

protected void checkPrefixCodec( PrefixCodec codec, Random r ) throws IOException {
	int[] symbol = new int[ 100 ];
	BooleanArrayList bits = new BooleanArrayList();
	for( int i = 0; i < symbol.length; i++ ) symbol[ i ] = r.nextInt( codec.size() ); 
	for( int i = 0; i < symbol.length; i++ ) {
		BitVector word = codec.codeWords()[ symbol[ i ] ];
		for( int j = 0; j < word.size(); j++ ) bits.add( word.get( j ) );
	}

	BooleanIterator booleanIterator = bits.iterator();
	Decoder decoder = codec.decoder();
	for( int i = 0; i < symbol.length; i++ ) {
		assertEquals( decoder.decode( booleanIterator ), symbol[ i ] );
	}
	
	FastByteArrayOutputStream fbaos = new FastByteArrayOutputStream();
	OutputBitStream obs = new OutputBitStream( fbaos, 0 );
	obs.write( bits.iterator() );
	obs.flush();
	InputBitStream ibs = new InputBitStream( fbaos.array );
	
	for( int i = 0; i < symbol.length; i++ ) {
		assertEquals( decoder.decode( ibs ), symbol[ i ] );
	}
}
 

/** Creates a Warc parallel output stream.
 *
 * @param outputStream the final output stream.
 * @param compress whether to write compressed records.
 * @param numberOfBuffers the number of buffers.
 */
public ParallelBufferedWarcWriter(final OutputStream outputStream, final boolean compress, final int numberOfBuffers) {
	this.outputStream = outputStream;
	emptyPairs = new ArrayBlockingQueue<>(numberOfBuffers);
	filledPairs = new ArrayBlockingQueue<>(numberOfBuffers);
	for(int i = numberOfBuffers; i-- != 0;) {
		final FastByteArrayOutputStream stream = new FastByteArrayOutputStream();
		emptyPairs.add(new WriterPair(compress ? new CompressedWarcWriter(stream) : new UncompressedWarcWriter(stream), stream));
	}
	(flushingThread = new FlushingThread()).start();
	flushingThread.setName(ParallelBufferedWarcWriter.class.getSimpleName());
}
 

@Test
public void testINTEGER() throws IOException {
	Random r = new Random(0);
	for (int i = 0; i < 100; i++) {
		int x = r.nextInt();
		FastByteArrayOutputStream fbaos = new FastByteArrayOutputStream();
		ByteSerializerDeserializer.INTEGER.toStream(Integer.valueOf(x), fbaos);
		Integer result = ByteSerializerDeserializer.INTEGER.fromStream(new FastByteArrayInputStream(fbaos.array));
		assertEquals(x, result.intValue());
	}
}
 

protected ObjectDiskQueue(ByteDiskQueue byteDiskQueue) {
	this.byteDiskQueue = byteDiskQueue;
	this.fbaos = new FastByteArrayOutputStream(1024);
}
 

private WriterPair(final WarcWriter writer, final FastByteArrayOutputStream stream) {
	this.writer = writer;
	this.stream = stream;
}
 

@Test
public void testRecord() throws IOException, InterruptedException, URISyntaxException {

	for(boolean gzip: new boolean[] { true, false }) {
		final FastByteArrayOutputStream out = new FastByteArrayOutputStream();
		final ParallelBufferedWarcWriter warcParallelOutputStream = new ParallelBufferedWarcWriter(out, gzip);
		final Thread thread[] = new Thread[NUM_THREADS];

		final URI fakeUri = new URI("http://this.is/a/fake");
		final RandomTestMocks.HttpResponse[] response = new RandomTestMocks.HttpResponse[NUM_RECORDS];
		for(int i = 0; i < NUM_THREADS; i++)
			(thread[i] = new Thread(Integer.toString(i)) {
				@Override
				public void run() {
					final int index = Integer.parseInt(getName());
					for (int i = index * (NUM_RECORDS / NUM_THREADS); i < (index + 1) * (NUM_RECORDS / NUM_THREADS); i++) {
						try {
							response[i] = new RandomTestMocks.HttpResponse(MAX_NUMBER_OF_HEADERS, MAX_LENGTH_OF_HEADER, MAX_LENGTH_OF_BODY, i);
							HttpResponseWarcRecord record = new HttpResponseWarcRecord(fakeUri, response[i]);
							warcParallelOutputStream.write(record);
							LOGGER.info("Thread " + index + " wrote record " + i);
						} catch(Exception e) { throw new RuntimeException(e); }
					}
				}
			}).start();


		for(Thread t: thread) t.join();
		warcParallelOutputStream.close();
		out.close();

		final FastBufferedInputStream in = new FastBufferedInputStream(new FastByteArrayInputStream(out.array, 0, out.length));
		WarcReader reader = gzip ? new CompressedWarcReader(in) : new UncompressedWarcReader(in);

		final boolean found[] = new boolean[NUM_RECORDS];
		for (int i = 0; i < NUM_RECORDS; i++) {
			final HttpResponseWarcRecord r = (HttpResponseWarcRecord) reader.read();
			final int pos = Integer.parseInt(r.getFirstHeader("Position").getValue());
			found[pos] = true;
			assertArrayEquals(ByteStreams.toByteArray(response[pos].getEntity().getContent()), ByteStreams.toByteArray(r.getEntity().getContent()));
		}
		in.close();

		for(int i = NUM_RECORDS; i-- != 0;) assertTrue(Integer.toString(i), found[i]);
	}
}
 

/**
 * Verify we can regenerate the {@link Fast64CodeWordCoder} from the code
 * word[]. This is tested by coding and decoding random symbol sequences.
 * For this test we need to reconstruct the {@link Fast64CodeWordCoder}. To
 * do that, we need to use the codeWord[] and create a long[] having the
 * same values as the codeWords, but expressed as 64-bit integers.
 * 
 * @param frequency
 *            The frequency[] should include a reasonable proportion of
 *            symbols with a zero frequency in order to replicate the
 *            expected conditions when coding non-random data such as are
 *            found in the keys of a B+Tree.
 * 
 * @throws IOException
 */
public void doRecoderRoundTripTest(final int frequency[]) throws IOException {
    
    final DecoderInputs decoderInputs = new DecoderInputs();
    
    final HuffmanCodec codec = new HuffmanCodec(frequency, decoderInputs);

    final PrefixCoder expected = codec.coder();
    
    final PrefixCoder actual = new Fast64CodeWordCoder(codec.codeWords());
    
    if (log.isDebugEnabled())
        log.debug(printCodeBook(codec.codeWords()));

    /*
     * First verify that both coders produce the same coded values for a
     * symbol sequence of random length drawn from the full set of symbols
     * of random length [1:nsymbols].
     */
    final int[] value = new int[r.nextInt(frequency.length) + 1];
    for(int i=0; i<value.length; i++) {
        // any of the symbols in [0:nsymbols-1].
        value[i] = r.nextInt(frequency.length);
    }

    /*
     * Now code the symbol sequence using both coders and then compare the
     * coded values. They should be the same.
     */
    final byte[] codedValue;
    {
        final FastByteArrayOutputStream ebaos = new FastByteArrayOutputStream();
        final FastByteArrayOutputStream abaos = new FastByteArrayOutputStream();
        final OutputBitStream eobs = new OutputBitStream(ebaos);
        final OutputBitStream aobs = new OutputBitStream(abaos);
        for (int i = 0; i < value.length; i++) {
            final int symbol = value[i];
            expected.encode(symbol, eobs);
            actual.encode(symbol, aobs);
        }
        eobs.flush();
        aobs.flush();
        assertEquals(0, BytesUtil.compareBytesWithLenAndOffset(0/* aoff */,
                ebaos.length, ebaos.array, 0/* boff */, abaos.length,
                abaos.array));
        codedValue = new byte[abaos.length];
        System.arraycopy(abaos.array/*src*/, 0/*srcPos*/, codedValue/*dest*/, 0/*destPos*/, abaos.length/*len*/);
    }

    /*
     * Now verify that the coded sequence decodes to the original symbol
     * sequence using a Decoder which is reconstructed from the bit length
     * and symbol arrays of the codec.
     */
    final CanonicalFast64CodeWordDecoder actualDecoder = new CanonicalFast64CodeWordDecoder(
            decoderInputs.getLengths(), decoderInputs.getSymbols());

    {

        final InputBitStream ibs = new InputBitStream(codedValue);
        
        for (int i = 0; i < value.length; i++) {

            assertEquals(value[i]/* symbol */, actualDecoder.decode(ibs));

        }
        
    }

}