com.esotericsoftware.kryo.io.Output Java Examples

private void serialize(String filename, Object obj) {
    synchronized (kryoLock) {
        File file = fileCache.create(filename);
        Output output = null;
        try {
            output = createOutput(new FileOutputStream(file));
            kryo.writeObject(output, obj);
        } catch (Exception e) {
            Logger.e(TAG, e);
        } catch (OutOfMemoryError oom) {
            MainApplication.freeMemory();
            Logger.e(TAG, oom);
        } finally {
            IOUtils.closeQuietly(output);
        }
        fileCache.put(file);
    }
}
 

@Test
public void testDataFileKryoSerialization() throws Exception {
  File data = temp.newFile();
  Assert.assertTrue(data.delete());
  Kryo kryo = new KryoSerializer(new SparkConf()).newKryo();

  try (Output out = new Output(new FileOutputStream(data))) {
    kryo.writeClassAndObject(out, DATA_FILE);
    kryo.writeClassAndObject(out, DATA_FILE.copy());
  }

  try (Input in = new Input(new FileInputStream(data))) {
    for (int i = 0; i < 2; i += 1) {
      Object obj = kryo.readClassAndObject(in);
      Assert.assertTrue("Should be a DataFile", obj instanceof DataFile);
      checkDataFile(DATA_FILE, (DataFile) obj);
    }
  }
}
 

@Override
protected void writeObject(Kryo kryo, Output output, Feature feature) throws Exception {
	output.writeString(feature.getName());
	output.writeLong(feature.getDuration());
	kryo.writeObject(output, feature.getSparseDescriptors());
	float[][][] m = feature.getDenseDescriptors();
	output.writeInt(m.length); // write x
	if(m.length == 0) return;
	
	output.writeInt(m[0].length); // write y
	output.writeInt(m[0][0].length); // write z
	for(int x=0; x<m.length; x++){
		for(int y=0; y<m[0].length; y++){
			for(int z=0; z<m[0][0].length; z++){
				output.writeFloat(m[x][y][z]);
			}
		}
	}
}
 

@Test(groups = "sv")
public void testSerializationAndHashCode() {
    final AlignmentInterval one = fromSAMRecordString("asm002362:tig00002\t16\tchr2\t1422222\t60\t75M56I139M\t*\t0\t0\tATGCTGGGGAATTTGTGTGCTCCTTGGGTGGGGACGAGCATGGAAGGCGCGTGGGACTGAAGCCTTGAAGACCCCGCAGGCGCCTCTCCTGGACAGACCTCGTGCAGGCGCCTCTCCTGGACCGACCTCGTGCAGGCGCCTCTCCTGGACAGACCTCGTGCAGGCGCCTCTCCTGGACCGACCTCGTGCAGGCGCCGCGCTGGACCGACCTCGTGCAGGCGCCGCGCTGGGCCATGGGGAGAGCGAGAGCCTGGTGTGCCCCTCAGGGAC\t*\tSA:Z:chr2_KI270774v1_alt,105288,-,114M1I27M1I127M,56,13;\tMD:Z:214\tRG:Z:GATKSVContigAlignments\tNM:i:56\tAS:i:142\tXS:i:0\n",
            true);
    final AlignmentInterval two = fromSAMRecordString("asm002362:tig00002\t2064\tchr2_KI270774v1_alt\t105288\t56\t114M1I27M1I127M\t*\t0\t0\tATGCTGGGGAATTTGTGTGCTCCTTGGGTGGGGACGAGCATGGAAGGCGCGTGGGACTGAAGCCTTGAAGACCCCGCAGGCGCCTCTCCTGGACAGACCTCGTGCAGGCGCCTCTCCTGGACCGACCTCGTGCAGGCGCCTCTCCTGGACAGACCTCGTGCAGGCGCCTCTCCTGGACCGACCTCGTGCAGGCGCCGCGCTGGACCGACCTCGTGCAGGCGCCGCGCTGGGCCATGGGGAGAGCGAGAGCCTGGTGTGCCCCTCAGGGAC\t*\tSA:Z:chr2,1422222,-,75M56I139M,60,56;\tMD:Z:94C17G1G6T13T3G1G34A3T9T68T8\tRG:Z:GATKSVContigAlignments\tNM:i:13\tAS:i:179\tXS:i:142",
            true);
    final AlignedContig sourceTig = new AlignedContig("asm002362:tig00002", "GTCCCTGAGGGGCACACCAGGCTCTCGCTCTCCCCATGGCCCAGCGCGGCGCCTGCACGAGGTCGGTCCAGCGCGGCGCCTGCACGAGGTCGGTCCAGGAGAGGCGCCTGCACGAGGTCTGTCCAGGAGAGGCGCCTGCACGAGGTCGGTCCAGGAGAGGCGCCTGCACGAGGTCTGTCCAGGAGAGGCGCCTGCGGGGTCTTCAAGGCTTCAGTCCCACGCGCCTTCCATGCTCGTCCCCACCCAAGGAGCACACAAATTCCCCAGCAT".getBytes(),
            Arrays.asList(one, two));
    final List<AssemblyContigAlignmentsConfigPicker.GoodAndBadMappings> config = AssemblyContigAlignmentsConfigPicker.pickBestConfigurations(sourceTig, new HashSet<>(Collections.singletonList("chr2")), 0.);
    final AssemblyContigWithFineTunedAlignments tig = AssemblyContigAlignmentsConfigPicker.reConstructContigFromPickedConfiguration(new Tuple2<>(new Tuple2<>(sourceTig.getContigName(), sourceTig.getContigSequence()),
            config)).next();

    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    final Output out = new Output(bos);
    final Kryo kryo = new Kryo();
    kryo.writeClassAndObject(out, tig);
    out.flush();

    final ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    final Input in = new Input(bis);
    @SuppressWarnings("unchecked")
    final AssemblyContigWithFineTunedAlignments roundTrip = (AssemblyContigWithFineTunedAlignments) kryo.readClassAndObject(in);
    Assert.assertEquals(tig, roundTrip);
    Assert.assertEquals(tig.hashCode(), roundTrip.hashCode());
}
 

/**
 * Serialize an object.
 * Only supports BIN and BIN_GZ SerializationMode.
 * 
 * @param outputStream
 * @param o
 * @throws IOException
 */
public static void serialize(OutputStream outStream, Object o, SerializationMode mode) {
  try {
    if (mode == SerializationMode.BIN || mode == SerializationMode.BIN_GZ) {
      Kryo kryo = new Kryo();
      kryo.setReferences(false);
      Output output = mode == SerializationMode.BIN_GZ ? new Output(new GZIPOutputStream(
          outStream)) : new Output(outStream);
      kryo.writeObject(output, o);
      output.close();

    } else {
      logger.warn("Unsupported serialization mode: {} file: {}", mode);
    }
  
  } catch (KryoException | IOException e) {
    logger.error("Serialization exception", e);
    throw new RuntimeException(e);
  }
}
 

@Override
public void serialize(T object, Output output)
{
  Class type = object.getClass();
  Serde serde = null;
  if (clazz == type) {
    serde = getDefaultSerde(type);
  }
  if (serde != null) {
    serde.serialize(object, output);
    return;
  }

  //delegate to kryo
  if (clazz == null) {
    kryo.writeClassAndObject(output, object);
  } else {
    kryo.writeObject(output, object);
  }
}
 

public Kryo5Codec(ClassLoader classLoader) {

        this.kryoPool = new Pool<Kryo>(true, false, 1024) {
            @Override
            protected Kryo create() {
                return createKryo(classLoader);
            }
        };

        this.inputPool = new Pool<Input>(true, false, 512) {
            @Override
            protected Input create() {
                return new Input(8192);
            }
        };

        this.outputPool = new Pool<Output>(true, false, 512) {
            @Override
            protected Output create() {
                return new Output(8192, -1);
            }
        };
    }
 

public byte[] serialize(Object object) {
    byte[] result;

    if (object == null) {
        result = new byte[0];
        return result;
    }

    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    try {
        Output output = new Output(stream);
        kryo.writeObject(output, object);
        output.close();
    } catch (Exception e) {
        result = new byte[0];
    }
    result = stream.toByteArray();

    return result;
}
 

public void write(Kryo kryo, Output output, ArrayMetaData<float[]> object) {
    try {
        float[] arrData = arrayMetaData.getArrData();
        arrayMetaData.send(output);
        int arrSegNum = arrayMetaData.getSegNum();
        for (int i = 0; i < arrSegNum; i++) {
            int from = arrayMetaData.getFrom(i);
            int to = arrayMetaData.getTo(i);
            for (int j = from; j < to; j++) {
                output.writeFloat(arrData[j]);
            }
        }
    } catch (IOException e) {
        LOG.error("double array write exception", e);
        System.exit(1);
    }
}
 

@Override
public void write(Kryo kryo, Output output, EventFields eventFields) {
    // Write out the number of entries;
    intSerializer.write(kryo, output, size);
    for (Entry<String, Set<FieldValue>> entry : map.entrySet()) {
        for(FieldValue fieldValue : entry.getValue()) {
            // Write the fields in the value
            stringSerializer.write(kryo, output, entry.getKey());
            valueSerializer.write(kryo, output, fieldValue.getVisibility().getExpression().length > 0 ? fieldValue.getVisibility().flatten() : fieldValue.getVisibility().getExpression());
            valueSerializer.write(kryo, output, fieldValue.getValue());
            valueSerializer.write(kryo, output, fieldValue.getMetadata());
        }
    }

    output.flush();

}
 

@Test
public void testSerializeDeserialize() {

    final Map<String,Integer> accessionToTaxMap = new HashMap<>();
    accessionToTaxMap.put("A",1);
    accessionToTaxMap.put("B",1);
    accessionToTaxMap.put("C",2);
    accessionToTaxMap.put("D",3);

    final PSTree tree = new PSTree(1);
    tree.addNode(2, "node2", 1, 0, "genus");
    tree.addNode(3, "node3", 1, 0, "genus");
    tree.addNode(4, "node4", 2, 200, "species");
    tree.addNode(5, "node5", 2, 300, "species");
    tree.addNode(6, "node6", 3, 100, "species");

    final PSTaxonomyDatabase taxonomyDatabase = new PSTaxonomyDatabase(tree, accessionToTaxMap);

    final Kryo kryo = new Kryo();
    final Output output = new Output(new ByteArrayOutputStream());
    kryo.writeObject(output, taxonomyDatabase);

    final Input input = new Input(new ByteArrayInputStream(output.getBuffer()));
    final PSTaxonomyDatabase taxonomyDatabaseTest = kryo.readObject(input, PSTaxonomyDatabase.class);

    Assert.assertEquals(taxonomyDatabaseTest.tree, taxonomyDatabase.tree);
    Assert.assertEquals(taxonomyDatabaseTest.accessionToTaxId, taxonomyDatabase.accessionToTaxId);
}
 

@Override
public void write(Kryo kryo, Output output) {
  int numVariableMappings = variableMappings.isEmpty()?0:variableMappings.size();
  output.writeShort(numVariableMappings);
  for(Map m : variableMappings) {
    byte[] b = m == null?new byte[]{}:SerDeUtils.toBytes(m);
    output.writeInt(b.length);
    if(b.length > 0) {
      output.writeBytes(b);
    }
  }
}
 

@Override
public byte[] serialize(Object object) {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    Output output = new Output(baos);
    kryo.writeClassAndObject(output, object);
    output.flush();
    output.close();
    return baos.toByteArray();
}
 

public void write(Kryo k, Output o, PriorityQueue<?> q) {
	k.writeClassAndObject(o, getComparator(q));
	o.writeInt(q.size(), true);
	for (Object a : q) {
		k.writeClassAndObject(o, a);
		o.flush();
	}
}
 

private void serializeStrings(List<Object> values, Output out) {
	out.writeInt(values.size(), true);
	for (Object o : values) {
		String str = (String) o;
		out.writeString(str);
	}
}
 

public static byte[] serialize(Object obj) {
	if (obj == null) {
		return null;
	}
	
	Output output = new Output(1024, -1);
	kryos.get().writeClassAndObject(output, obj);
	
	return output.getBuffer();
}
 

protected void serialize(final Kryo kryo, final Output output) {
    super.serialize(kryo, output);
    output.writeString(dupSeqRepeatUnitRefSpan.getContig());
    output.writeInt(dupSeqRepeatUnitRefSpan.getStart());
    output.writeInt(dupSeqRepeatUnitRefSpan.getEnd());
    output.writeInt(dupSeqRepeatNumOnRef);
    output.writeInt(dupSeqRepeatNumOnCtg);
    dupSeqStrandOnRef.forEach(s -> output.writeInt(s.ordinal()));
    dupSeqStrandOnCtg.forEach(s -> output.writeInt(s.ordinal()));
}
 

final void write(Recoverable op)
{
  if (replayMode.get()) {
    throw new IllegalStateException("Request to write while journal is replaying operations");
  }
  Integer classId = RecoverableOperation.getId(op.getClass());
  if (classId == null) {
    throw new IllegalArgumentException("Class not registered " + op.getClass());
  }
  while (true) {
    final Output out = output.get();
    if (out != null) {
      // need to atomically write id, operation and flush the output stream
      synchronized (out) {
        try {
          LOG.debug("WAL write {}", RecoverableOperation.get(classId));
          out.writeInt(classId);
          op.write(out);
          out.flush();
          break;
        } catch (KryoException e) {
          // check that no other threads sneaked between get() and synchronized block and set output stream to a new
          // stream or null leading to the current stream being closed
          if (output.get() == out) {
            throw e;
          }
        }
      }
    } else {
      LOG.warn("Journal output stream is null. Skipping write to the WAL.");
      break;
    }
  }
}
 

@Override
public void write(Kryo kryo, Output output,
        IpPrefix object) {
    byte[] octs = object.address().toOctets();
    output.writeInt(octs.length);
    output.writeBytes(octs);
    output.writeInt(object.prefixLength());
}
 

@Override
public void send(Output output) throws IOException {
    output.writeInt(srcRank);
    output.writeInt(destRank);
    output.writeInt(step);
    output.writeInt(sum);
    output.writeInt(segNum);
    for (int i = 0; i < segNum; i++) {
        output.writeInt(ranks.get(i));
        output.writeInt(segFroms.get(i));
        output.writeInt(segTos.get(i));
    }
    output.flush();
}
 

/**
 * Serializes given object to byte array using Kryo instance in pool.
 *
 * @param obj Object to serialize
 * @param bufferSize maximum size of serialized bytes
 * @return serialized bytes
 */
public byte[] serialize(final Object obj, final int bufferSize) {
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream(bufferSize);
    Output out = new Output(outputStream);
    return pool.run(kryo -> {
        kryo.writeClassAndObject(out, obj);
        out.flush();
        return outputStream.toByteArray();
    });
}
 

@Test(dataProvider="AlignedAssemblySerializationTest", groups = "sv")
public void testAlignedAssemblySerialization(final Integer assemblyID, final AlignedAssembly expectedAssembly) {

    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    final Output out = new Output(bos);
    final Kryo kryo = new Kryo();
    kryo.writeClassAndObject(out, expectedAssembly);
    out.flush();

    final ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    final Input in = new Input(bis);
    @SuppressWarnings("unchecked")
    final AlignedAssembly roundTrip = (AlignedAssembly) kryo.readClassAndObject(in);
    Assert.assertEquals(roundTrip, expectedAssembly);
}
 

@Override
public void write(final DataOutput output) throws IOException {
    Kryo kryo = new Kryo();
    kryo.register(HBaseVertex.class, new HBaseVertexSerializer());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    Output out = new Output(baos);
    kryo.writeObject(out, this.vertex);
    out.close();
    final byte[] serialized = baos.toByteArray();
    WritableUtils.writeCompressedByteArray(output, serialized);
    Writable writable = value != null ? value : NullWritable.get();
    Text.writeString(output, writable.getClass().getName());
    writable.write(output);
}
 

public static byte[] serialize(Object object) {
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    Output output = new Output(stream);

    Utilities.runtimeSerializationKryo.get().writeObject(output, object);

    output.close(); // close() also calls flush()
    return stream.toByteArray();
}
 

@Test
public void serDeTest() throws IOException {
  final Kryo kryo = new Kryo();
  File destination = File.createTempFile("partition_", ".gz");
  final Output output = new Output(new GZIPOutputStream(new FileOutputStream(destination)));
  int numColumns = 6;
  int numValues = 6;
  OffHeapPartition p = createPartition();
  List<DataRow> data = TestUtils.genMultiColumnData("f1", numColumns, numValues);
  data.addAll(TestUtils.genMultiColumnData("f2", numColumns, numValues));
  p.insertData(data);
  log.info("Writing partition with {} records to {}", data.size(), destination.getAbsolutePath());
  long t0 = System.nanoTime();
  kryo.writeObject(output, p);
  output.close();
  long t1 = System.nanoTime();
  log.info("Took {} ms to write {} records", (t1 - t0) / 1000000, data.size());

  // Now reading the file
  Input input = new Input(new GZIPInputStream(new FileInputStream(destination)));
  t0 = System.nanoTime();
  OffHeapPartition newP = kryo.readObject(input, OffHeapPartition.class);
  t1 = System.nanoTime();
  assertEquals(p.getNumRecords(), newP.getNumRecords());
  long cP =
      TestUtils.checkMatchMultiFilter(
          p, "c", "f1_0", "f1_0_0", "f1_0", "f1_0_1", "f1_0", "f1_0_2"); // ,
  long cNp =
      TestUtils.checkMatchMultiFilter(
          newP, "c", "f1_0", "f1_0_0", "f1_0", "f1_0_1", "f1_0", "f1_0_2"); // ,
  assertEquals(cP, cNp);
  log.info("Took {} ms to read {} records", (t1 - t0) / 1000000, data.size());
  log.info("{}", newP.getStats());
  destination.deleteOnExit();
}
 

@Override
public CachedData encode(final Object obj) {
    final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
    final Output output = new Output(byteStream);
    kryo.writeClassAndObject(output, obj);
    output.flush();
    IOUtils.closeQuietly(output);
    final byte[] bytes = byteStream.toByteArray();
    return new CachedData(0, bytes, bytes.length);
}
 

@Test
public void treeNode() {
    final Result<DecisionTreeRuleSet> result = (new CommisionRuleSetSupplier()).get();
    EhSupport.ensure(result.isSuccess(), "Could not create decision tree");
    final DecisionTreeRuleSet ruleSet = result.getData();

    final TreeNode node = DecisionTreeFactory.constructDecisionTree(ruleSet, DecisionTreeType.SINGLE);

    final Kryo kryo = new Kryo();
    // no default no-arg constructors
    kryo.setInstantiatorStrategy(new StdInstantiatorStrategy());

    final InstantiatorStrategy defaultInstantiatorStrategy = new Kryo.DefaultInstantiatorStrategy();
    kryo.getRegistration(ArrayList.class)
            .setInstantiator(defaultInstantiatorStrategy.newInstantiatorOf(ArrayList.class));
    kryo.getRegistration(HashSet.class)
            .setInstantiator(defaultInstantiatorStrategy.newInstantiatorOf(HashSet.class));
    UnmodifiableCollectionsSerializer.registerSerializers(kryo);

    final ByteArrayOutputStream out = new ByteArrayOutputStream();
    final Output output = new Output(out);
    kryo.writeObject(output, node);
    output.flush();
    output.close();

    final ByteArrayInputStream inputStream = new ByteArrayInputStream(out.toByteArray());
    final Input kryoInput = new Input(inputStream);
    final TreeNode tree = kryo.readObject(kryoInput, BaseTreeNode.class);

    final SingleDecisionTreeFactoryTest test = new SingleDecisionTreeFactoryTest();
    test.checkTreeNode(tree, ruleSet);

    assertEquals(node, tree);
}
 

@Override
public void write(Kryo kryo, Output output, LocalDate localDate) {
	output.writeInt(localDate.getYear());
	output.writeInt(localDate.getMonthOfYear());
	output.writeInt(localDate.getDayOfMonth());

	final Chronology chronology = localDate.getChronology();
	if (chronology != null && chronology != ISOChronology.getInstanceUTC()) {
		throw new RuntimeException("Unsupported chronology: " + chronology);
	}
}
 

@Override
public void write(final Kryo kryo, final Output output, final ImmutableList<Object> object) {
  output.writeInt(object.size(), true);
  final UnmodifiableIterator iterator = object.iterator();

  while (iterator.hasNext()) {
    final Object value = iterator.next();
    kryo.writeClassAndObject(output, value);
  }
}
 

@Override
public void write(Kryo kryo, Output output, ImmutableList<?> object) {
    output.writeInt(object.size());
    for (Object e : object) {
        kryo.writeClassAndObject(output, e);
    }
}