org.apache.beam.sdk.coders.CoderException Java Examples

@Override public KettleRow decode( InputStream inStream ) throws CoderException, IOException {

    ObjectInputStream in = new ObjectInputStream( inStream );

    Object[] row = null;
    int length = in.readInt();
    if ( length < 0 ) {
      return new KettleRow( row );
    }
    row = new Object[ length ];
    for ( int i = 0; i < length; i++ ) {
      // Null?
      boolean isNull = in.readBoolean();
      if ( !isNull ) {
        int objectType = in.readInt();
        Object object = read( in, objectType );
        row[i] = object;
      }
    }

    return new KettleRow(row);
  }
 

@Override
public KV<K, Iterable<WindowedValue<V>>> map(Tuple<byte[], Iterator<byte[]>> input) {
  K key = null;
  Iterable<WindowedValue<V>> value = null;
  try {
    key = CoderUtils.decodeFromByteArray(keyCoder, input.getKey());

    value = StreamSupport.stream(
        Spliterators.spliteratorUnknownSize(input.getValue(), Spliterator.ORDERED), false)
        .map(bytes -> TranslationUtils.fromByteArray(bytes, wvCoder))
        .collect(Collectors.toList());
  } catch (CoderException e) {
    e.printStackTrace();
  }
  return KV.of(key, value);
}
 

private void verifyUnmodifiedThrowingCheckedExceptions() throws CoderException {
  // Since there is no guarantee that cloning an object via the coder will
  // return the exact same type as value, We are cloning the possiblyModifiedObject
  // before getting it's structural value. This way we are guaranteed to compare the same
  // types.
  T possiblyModifiedClonedValue = CoderUtils.clone(coder, possiblyModifiedObject);
  Object newStructuralValue = coder.structuralValue(possiblyModifiedClonedValue);
  if (originalStructuralValue.equals(newStructuralValue)) {
    return;
  } else if (Objects.deepEquals(
      encodedOriginalObject, CoderUtils.encodeToByteArray(coder, possiblyModifiedObject))) {
    LOG.warn(
        "{} of type {} has a #structuralValue method which does not return true when the "
            + "encoding of the elements is equal. Element {}",
        Coder.class.getSimpleName(),
        coder.getClass(),
        possiblyModifiedObject);
    return;
  }
  illegalMutation(clonedOriginalObject, possiblyModifiedClonedValue);
}
 

@Override
public Tuple<byte[], byte[]> map(WindowedValue<KV<K, V>> input) {
  Tuple<byte[], byte[]> element = null;

  WindowedValue<KV<K, WindowedValue<V>>> temp =
      WindowedValue.of(
          KV.of(
              input.getValue().getKey(),
              WindowedValue.of(
                  input.getValue().getValue(),
                  input.getTimestamp(),
                  input.getWindows(),
                  input.getPane())),
          input.getTimestamp(),
          input.getWindows(),
          input.getPane());
  try {
    element =
        new Tuple<>(
            CoderUtils.encodeToByteArray(keyCoder, temp.getValue().getKey()),
            CoderUtils.encodeToByteArray(wvCoder, temp.getValue().getValue()));
  } catch (CoderException e) {
    LOG.info(e.getMessage());
  }
  return element;
}
 

@Override
public KV<K, Iterable<WindowedValue<V>>> map(Tuple<byte[], Iterator<byte[]>> input) {
  K key = null;
  Iterable<WindowedValue<V>> value = null;
  try {
    key = CoderUtils.decodeFromByteArray(keyCoder, input.getKey());
    // TODO need to replace this with a custom iterator
    value =
        StreamSupport.stream(
                Spliterators.spliteratorUnknownSize(input.getValue(), Spliterator.ORDERED), false)
            .map(bytes -> TranslationUtils.fromByteArray(bytes, wvCoder))
            .collect(Collectors.toList());
  } catch (CoderException e) {
    LOG.info(e.getMessage());
  }
  return KV.of(key, value);
}
 

@Override
public PCollectionView<ActualT> expand(PBegin input) {
  final Coder<T> coder = actual.getCoder();
  return actual
      .apply("FilterActuals", rewindowActuals.prepareActuals())
      .apply("GatherPanes", GatherAllPanes.globally())
      .apply("ExtractPane", MapElements.via(extractPane))
      .setCoder(IterableCoder.of(actual.getCoder()))
      .apply(Flatten.iterables())
      .apply("RewindowActuals", rewindowActuals.windowActuals())
      .apply(
          ParDo.of(
              new DoFn<T, T>() {
                @ProcessElement
                public void processElement(ProcessContext context) throws CoderException {
                  context.output(CoderUtils.clone(coder, context.element()));
                }
              }))
      .apply(actualView);
}
 

@VisibleForTesting
static <T> T decode(Coder<T> coder, Coder.Context context, byte[] bytes)
    throws CoderException, IOException {
  @SuppressWarnings("unchecked")
  Coder<T> deserializedCoder = SerializableUtils.clone(coder);

  byte[] buffer;
  if (Objects.equals(context, Coder.Context.NESTED)) {
    buffer = new byte[bytes.length + 1];
    System.arraycopy(bytes, 0, buffer, 0, bytes.length);
    buffer[bytes.length] = 1;
  } else {
    buffer = bytes;
  }

  CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(buffer));
  T value = deserializedCoder.decode(new UnownedInputStream(cis), context);
  assertThat(
      "consumed bytes equal to encoded bytes", cis.getCount(), equalTo((long) bytes.length));
  return value;
}
 

/**
 * Creates a new {@code MutationDetector} for the provided {@code value} that uses the provided
 * {@link Coder} to perform deep copies and comparisons by serializing and deserializing values.
 *
 * <p>It is permissible for {@code value} to be {@code null}. Since {@code null} is immutable, the
 * mutation check will always succeed.
 */
public static <T> MutationDetector forValueWithCoder(T value, Coder<T> coder)
    throws CoderException {
  if (value == null) {
    return noopMutationDetector();
  } else {
    return new CodedValueMutationDetector<>(value, coder);
  }
}
 

@Override
public Person decode(InputStream inStream) throws CoderException, IOException {
  long id = LONG_CODER.decode(inStream);
  String name = STRING_CODER.decode(inStream);
  String emailAddress = STRING_CODER.decode(inStream);
  String creditCard = STRING_CODER.decode(inStream);
  String city = STRING_CODER.decode(inStream);
  String state = STRING_CODER.decode(inStream);
  Instant dateTime = INSTANT_CODER.decode(inStream);
  String extra = STRING_CODER.decode(inStream);
  return new Person(id, name, emailAddress, creditCard, city, state, dateTime, extra);
}
 

@Test
public void keyedBundleEncodeFailsAddFails() {
  PCollection<Record> pc = p.apply(Create.empty(new RecordNoEncodeCoder()));
  UncommittedBundle<Record> bundle =
      factory.createKeyedBundle(StructuralKey.of("foo", StringUtf8Coder.of()), pc);

  thrown.expect(UserCodeException.class);
  thrown.expectCause(isA(CoderException.class));
  thrown.expectMessage("Encode not allowed");
  bundle.add(WindowedValue.valueInGlobalWindow(new Record()));
}
 

/**
 * Encodes {@code value} to the given {@code stream}, which should be a stream that never throws
 * {@code IOException}, such as {@code ByteArrayOutputStream} or {@link
 * ExposedByteArrayOutputStream}.
 */
private static <T> void encodeToSafeStream(
    Coder<T> coder, T value, OutputStream stream, Coder.Context context) throws CoderException {
  try {
    coder.encode(value, new UnownedOutputStream(stream), context);
  } catch (IOException exn) {
    Throwables.propagateIfPossible(exn, CoderException.class);
    throw new IllegalArgumentException("Forbidden IOException when writing to OutputStream", exn);
  }
}
 

@Override
public void encode(CategoryPrice value, OutputStream outStream)
    throws CoderException, IOException {
  LONG_CODER.encode(value.category, outStream);
  LONG_CODER.encode(value.price, outStream);
  INT_CODER.encode(value.isLast ? 1 : 0, outStream);
}
 

@Override
public CategoryPrice decode(InputStream inStream) throws CoderException, IOException {
  long category = LONG_CODER.decode(inStream);
  long price = LONG_CODER.decode(inStream);
  boolean isLast = INT_CODER.decode(inStream) != 0;
  return new CategoryPrice(category, price, isLast);
}
 

@Override
public WindowedAccumulator<InputT, ValueT, AccumT, ?> decode(InputStream inStream)
    throws CoderException, IOException {
  if (type.isMapBased()) {
    return WindowedAccumulator.create(toValue, type, wrap.decode(inStream), windowComparator);
  }
  boolean empty = inStream.read() == 0;
  if (empty) {
    return WindowedAccumulator.create(toValue, type, windowComparator);
  }
  return WindowedAccumulator.create(
      toValue, type, Arrays.asList(accumCoder.decode(inStream)), windowComparator);
}
 

@Override
public TimerData decode(InputStream inStream) throws CoderException, IOException {
  String timerId = STRING_CODER.decode(inStream);
  StateNamespace namespace =
      StateNamespaces.fromString(STRING_CODER.decode(inStream), windowCoder);
  Instant timestamp = INSTANT_CODER.decode(inStream);
  TimeDomain domain = TimeDomain.valueOf(STRING_CODER.decode(inStream));
  return TimerData.of(timerId, namespace, timestamp, timestamp, domain);
}
 

private T decode(String encodedElementString) throws CoderException {
  // TODO: Replace with the real encoding used by the
  // front end, when we know what it is.
  byte[] encodedElement = StringUtils.jsonStringToByteArray(encodedElementString);
  notifyElementRead(encodedElement.length);
  return CoderUtils.decodeFromByteArray(coder, encodedElement);
}
 

@Override
public CountSum decode(InputStream inStream) throws CoderException, IOException {
  DataInputStream dataStream = new DataInputStream(inStream);
  long count = dataStream.readLong();
  double sum = dataStream.readDouble();
  return new CountSum(count, sum);
}
 

@Override
public Record decode(InputStream inStream) throws CoderException, IOException {
  return new Record() {
    @Override
    public String toString() {
      return "DecodedRecord";
    }
  };
}
 

@Override
@SuppressWarnings("unchecked")
public void encode(CoGbkResult value, OutputStream outStream)
    throws CoderException, IOException {
  if (!schema.equals(value.getSchema())) {
    throw new CoderException("input schema does not match coder schema");
  }
  if (schema.size() == 0) {
    return;
  }
  for (int unionTag = 0; unionTag < schema.size(); unionTag++) {
    tagListCoder(unionTag).encode(value.valueMap.get(unionTag), outStream);
  }
}
 

/**
 * Encodes the given value of type {@code T} onto the given output stream using provided {@link
 * ThriftCoder#protocolFactory}.
 *
 * @param value {@link org.apache.thrift.TBase} to encode.
 * @param outStream stream to output encoded value to.
 * @throws IOException if writing to the {@code OutputStream} fails for some reason
 * @throws CoderException if the value could not be encoded for some reason
 */
@Override
public void encode(T value, OutputStream outStream) throws CoderException, IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(baos));
  try {
    TBase<?, ?> tBase = (TBase<?, ?>) value;
    tBase.write(protocol);
  } catch (Exception te) {
    throw new CoderException("Could not write value. Error: " + te.getMessage());
  }
  outStream.write(baos.toByteArray());
}
 

@Override
public T[] get() {
  if (value == null) {
    try {
      @SuppressWarnings("unchecked")
      T[] decoded = (T[]) CoderUtils.decodeFromByteArray(coder, encodedValue).toArray();
      value = decoded;
    } catch (CoderException exc) {
      throw new RuntimeException("Error deserializing via Coder", exc);
    }
  }
  return value;
}
 

@Test
public void encodeNullThrowsCoderException() throws Exception {
  thrown.expect(CoderException.class);
  thrown.expectMessage("cannot encode a null MessageA");

  CoderUtils.encodeToBase64(ProtoCoder.of(MessageA.class), null);
}
 

@Override
public void encode(TimerData timer, OutputStream outStream) throws CoderException, IOException {
  STRING_CODER.encode(timer.getTimerId(), outStream);
  STRING_CODER.encode(timer.getNamespace().stringKey(), outStream);
  INSTANT_CODER.encode(timer.getTimestamp(), outStream);
  STRING_CODER.encode(timer.getDomain().name(), outStream);
}
 

@Override public void encode( HopRow value, OutputStream outStream ) throws CoderException, IOException {

    Object[] row = value.getRow();
    ObjectOutputStream out = new ObjectOutputStream( outStream );

    // Length
    //
    if ( row == null ) {
      out.writeInt( -1 );
      return; // all done
    } else {
      out.writeInt( row.length );
    }

    // The values
    //
    for ( int i = 0; i < row.length; i++ ) {
      Object object = row[ i ];
      // Null?
      //
      out.writeBoolean( object == null );

      if ( object != null ) {
        // Type?
        //
        int objectType = getObjectType( object );
        out.writeInt( objectType );

        // The object itself
        //
        write( out, objectType, object );
      }
    }
    out.flush();
  }
 

@Override
public void encode(QuantileState<T, ComparatorT> state, OutputStream outStream)
    throws CoderException, IOException {
  intCoder.encode(state.numQuantiles, outStream);
  intCoder.encode(state.bufferSize, outStream);
  elementCoder.encode(state.min, outStream);
  elementCoder.encode(state.max, outStream);
  elementListCoder.encode(state.unbufferedElements, outStream);
  BigEndianIntegerCoder.of().encode(state.buffers.size(), outStream);
  for (QuantileBuffer<T> buffer : state.buffers) {
    encodeBuffer(buffer, outStream);
  }
}
 

@Test(expected = CoderException.class)
public void testWrongRegistrarDecoding() throws IOException {
  final KryoRegistrar registrarCoding = k -> k.register(ClassToBeEncoded.class);
  final KryoRegistrar registrarDecoding =
      k -> {
        // No-op
      };
  final KryoCoder<ClassToBeEncoded> coderToEncode = KryoCoder.of(OPTIONS, registrarCoding);
  final KryoCoder<ClassToBeEncoded> coderToDecode = KryoCoder.of(OPTIONS, registrarDecoding);
  assertEncoding(coderToEncode, coderToDecode);
}
 

@Override
public void encode(AttributeValue value, OutputStream outStream) throws IOException {

  if (value.getS() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.s.toString(), outStream);
    StringUtf8Coder.of().encode(value.getS(), outStream);
  } else if (value.getN() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.n.toString(), outStream);
    StringUtf8Coder.of().encode(value.getN(), outStream);
  } else if (value.getBOOL() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.bOOL.toString(), outStream);
    BooleanCoder.of().encode(value.getBOOL(), outStream);
  } else if (value.getB() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.b.toString(), outStream);
    ByteArrayCoder.of().encode(convertToByteArray(value.getB()), outStream);
  } else if (value.getSS() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.sS.toString(), outStream);
    LIST_STRING_CODER.encode(value.getSS(), outStream);
  } else if (value.getNS() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.nS.toString(), outStream);
    LIST_STRING_CODER.encode(value.getNS(), outStream);
  } else if (value.getBS() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.bS.toString(), outStream);
    LIST_BYTE_CODER.encode(convertToListByteArray(value.getBS()), outStream);
  } else if (value.getL() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.l.toString(), outStream);
    LIST_ATTRIBUTE_CODER.encode(value.getL(), outStream);
  } else if (value.getM() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.m.toString(), outStream);
    MAP_ATTRIBUTE_CODER.encode(value.getM(), outStream);
  } else if (value.getNULL() != null) {
    StringUtf8Coder.of().encode(AttributeValueType.nULLValue.toString(), outStream);
    BooleanCoder.of().encode(value.getNULL(), outStream);
  } else {
    throw new CoderException("Unknown Type");
  }
}
 

@Override
protected long getEncodedElementByteSize(HyperLogLogPlus value) throws IOException {
  if (value == null) {
    throw new CoderException("cannot encode a null HyperLogLogPlus sketch");
  }
  return value.sizeof();
}
 

@Override
public void verifyUnmodified() {
  try {
    verifyUnmodifiedThrowingCheckedExceptions();
  } catch (CoderException exn) {
    throw new RuntimeException(exn);
  }
}
 

@Test
public void testCoderExceptionPropagation() throws Exception {
  @SuppressWarnings("unchecked")
  Coder<String> crashingCoder = mock(Coder.class);
  doThrow(new CoderException("testing exception"))
      .when(crashingCoder)
      .encode(anyString(), any(OutputStream.class), any(Coder.Context.class));

  expectedException.expect(CoderException.class);
  expectedException.expectMessage("testing exception");

  CoderUtils.encodeToByteArray(crashingCoder, "hello");
}