Java Code Examples for com.couchbase.client.java.document.RawJsonDocument#create()

/**
 * Test that a single Json document can be written successfully
 * @throws IOException
 * @throws DataConversionException
 * @throws ExecutionException
 * @throws InterruptedException
 */
@Test(groups={"timeout"})
public void testJsonDocumentWrite()
    throws IOException, DataConversionException, ExecutionException, InterruptedException {
  CouchbaseWriter writer = new CouchbaseWriter(_couchbaseEnvironment, ConfigFactory.empty());
  try {

    String key = "hello";
    String testContent = "hello world";
    HashMap<String, String> contentMap = new HashMap<>();
    contentMap.put("value", testContent);
    Gson gson = new Gson();
    String jsonString = gson.toJson(contentMap);
    RawJsonDocument jsonDocument = RawJsonDocument.create(key, jsonString);
    writer.write(jsonDocument, null).get();
    RawJsonDocument returnDoc = writer.getBucket().get(key, RawJsonDocument.class);

    Map<String, String> returnedMap = gson.fromJson(returnDoc.content(), Map.class);
    Assert.assertEquals(testContent, returnedMap.get("value"));
  } finally {
    writer.close();
  }
}
 

@Override
public Iterable<RawJsonDocument> convertRecord(String outputSchema, Object inputRecord, WorkUnitState workUnit)
    throws DataConversionException {

  JsonElement jsonElement = GSON.toJsonTree(inputRecord);
  if (!jsonElement.isJsonObject())
  {
    throw new DataConversionException("Expecting json element " + jsonElement.toString()
        + " to be of type JsonObject.");
  }

  JsonObject jsonObject = jsonElement.getAsJsonObject();

  if (!jsonObject.has(keyField))
  {
    throw new DataConversionException("Could not find key field " + keyField
        + " in json object " + jsonObject.toString());
  }

  JsonElement keyValueElement = jsonObject.get(keyField);
  String keyString;
  try {
    keyString = keyValueElement.getAsString();
  }
  catch (Exception e)
  {
    throw new DataConversionException("Could not get the key " + keyValueElement.toString() + " as a string", e);
  }
  String valueString = GSON.toJson(jsonElement);

  RawJsonDocument jsonDocument = RawJsonDocument.create(keyString, valueString);
  return new SingleRecordIterable<>(jsonDocument);
}
 

/**
 * Test that a single Json document can be written successfully with TTL
 * @throws IOException
 * @throws DataConversionException
 * @throws ExecutionException
 * @throws InterruptedException
 */
@Test(groups={"timeout"})
public void testJsonDocumentWriteTTL()
    throws IOException, DataConversionException, ExecutionException, InterruptedException {
  int ttl = 1000;
  int expiry = Math.toIntExact(System.currentTimeMillis() / 1000) + ttl;
  Config config = getConfig("default", Optional.of(ttl), Optional.empty(), Optional.empty());
  CouchbaseWriter writer = new CouchbaseWriter(_couchbaseEnvironment, config);
  try {

    String key = "hello";
    String testContent = "hello world";
    HashMap<String, String> contentMap = new HashMap<>();
    contentMap.put("value", testContent);
    Gson gson = new Gson();
    String jsonString = gson.toJson(contentMap);
    RawJsonDocument jsonDocument = RawJsonDocument.create(key, jsonString);
    AbstractDocument storedDoc = ((GenericWriteResponse<AbstractDocument>) writer.write(jsonDocument, null).get()).getRawResponse();
    RawJsonDocument returnDoc = writer.getBucket().get(key, RawJsonDocument.class);

    Map<String, String> returnedMap = gson.fromJson(returnDoc.content(), Map.class);
    Assert.assertEquals(testContent, returnedMap.get("value"));
    Assert.assertEquals(storedDoc.expiry(), expiry, 50);
  } finally {
    writer.close();
  }
}
 

/**
 * Test that a single Json document can be written successfully with TTL and timeunits
 * @throws IOException
 * @throws DataConversionException
 * @throws ExecutionException
 * @throws InterruptedException
 */
@Test(groups={"timeout"})
public void testJsonDocumentWriteTTLWithTimeUnits()
    throws IOException, DataConversionException, ExecutionException, InterruptedException {
  int ttl = 1;
  TimeUnit timeUnit = TimeUnit.DAYS;
  int expiry = Math.toIntExact(System.currentTimeMillis() / 1000) + (int) TimeUnit.SECONDS.convert(ttl, timeUnit);
  Config config = getConfig("default", Optional.of(ttl), Optional.of(timeUnit), Optional.empty());
  CouchbaseWriter writer = new CouchbaseWriter(_couchbaseEnvironment, config);
  try {

    String key = "hello";
    String testContent = "hello world";
    HashMap<String, String> contentMap = new HashMap<>();
    contentMap.put("value", testContent);
    Gson gson = new Gson();
    String jsonString = gson.toJson(contentMap);
    RawJsonDocument jsonDocument = RawJsonDocument.create(key, jsonString);
    AbstractDocument storedDoc = ((GenericWriteResponse<AbstractDocument>) writer.write(jsonDocument, null).get()).getRawResponse();
    RawJsonDocument returnDoc = writer.getBucket().get(key, RawJsonDocument.class);

    Map<String, String> returnedMap = gson.fromJson(returnDoc.content(), Map.class);
    Assert.assertEquals(testContent, returnedMap.get("value"));
    Assert.assertEquals(storedDoc.expiry() - expiry, 0, 50);
  } finally {
    writer.close();
  }
}
 

/**
 * Test that a single Json document can be written successfully with TTL and timeunits
 * @throws IOException
 * @throws DataConversionException
 * @throws ExecutionException
 * @throws InterruptedException
 */
@Test(groups={"timeout"})
public void testJsonDocumentWriteTtlWithField()
    throws ExecutionException, InterruptedException {
  int ttl = 30;
  int originDiffFromNow = 5;
  TimeUnit timeUnit = TimeUnit.DAYS;
  String ttlOriginField = "time";
  long now = System.currentTimeMillis();
  long originDelta =  TimeUnit.MILLISECONDS.convert(originDiffFromNow, TimeUnit.DAYS);
  long origin = now - originDelta;
  long expiry = TimeUnit.SECONDS.convert(now, TimeUnit.MILLISECONDS) + TimeUnit.SECONDS.convert(ttl, timeUnit) - TimeUnit.SECONDS.convert(originDiffFromNow, timeUnit) ;

  Config config = getConfig("default", Optional.of(ttl), Optional.of(timeUnit), Optional.of(ttlOriginField));
  CouchbaseWriter writer = new CouchbaseWriter(_couchbaseEnvironment, config);
  try {

    String key = "hello";
    String testContent = "hello world";
    HashMap<String, String> contentMap = new HashMap<>();
    contentMap.put("value", testContent);
    contentMap.put(ttlOriginField, "" + origin);
    Gson gson = new Gson();
    String jsonString = gson.toJson(contentMap);
    RawJsonDocument jsonDocument = RawJsonDocument.create(key, jsonString);
    AbstractDocument storedDoc = ((GenericWriteResponse<AbstractDocument>) writer.write(jsonDocument, null).get()).getRawResponse();
    RawJsonDocument returnDoc = writer.getBucket().get(key, RawJsonDocument.class);

    Map<String, String> returnedMap = gson.fromJson(returnDoc.content(), Map.class);
    Assert.assertEquals(testContent, returnedMap.get("value"));
    Assert.assertEquals(storedDoc.expiry() , expiry, 50);
  } finally {
    writer.close();
  }
}
 

@Test(groups={"timeout"})
public void testJsonDocumentWriteTtlWithNestedField()
    throws ExecutionException, InterruptedException {
  int ttl = 30;
  int originDiffFromNow = 5;
  TimeUnit timeUnit = TimeUnit.DAYS;
  String ttlOriginField = "a.b.time";
  long now = System.currentTimeMillis();
  long originDelta =  TimeUnit.MILLISECONDS.convert(originDiffFromNow, timeUnit);
  long origin = now - originDelta;
  long expiry = TimeUnit.SECONDS.convert(now, TimeUnit.MILLISECONDS) + TimeUnit.SECONDS.convert(ttl, timeUnit) - TimeUnit.SECONDS.convert(originDiffFromNow, timeUnit) ;

  Config config = getConfig("default", Optional.of(ttl), Optional.of(timeUnit), Optional.of(ttlOriginField));
  CouchbaseWriter writer = new CouchbaseWriter(_couchbaseEnvironment, config);
  try {
    JsonObject jsonRoot = new JsonObject();
    String key = "keyValue";
    String testContent = "hello world";
    String valueKey = "value";
    jsonRoot.addProperty(valueKey, testContent);
    jsonRoot.addProperty(ttlOriginField, origin);

    RawJsonDocument jsonDocument = RawJsonDocument.create(key, jsonRoot.toString());
    AbstractDocument storedDoc = ((GenericWriteResponse<AbstractDocument>) writer.write(jsonDocument, null).get()).getRawResponse();
    RawJsonDocument returnDoc = writer.getBucket().get(key, RawJsonDocument.class);

    Map<String, String> returnedMap = new Gson().fromJson(returnDoc.content(), Map.class);
    Assert.assertEquals(returnedMap.get(valueKey), testContent);
    Assert.assertEquals(storedDoc.expiry() , expiry, 50);
  } finally {
    writer.close();
  }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final byte[] content = new byte[(int) flowFile.getSize()];
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, content, true);
        }
    });

    String docId = flowFile.getAttribute(CoreAttributes.UUID.key());
    if (!StringUtils.isEmpty(context.getProperty(DOC_ID).getValue())) {
        docId = context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
    }

    try {
        Document<?> doc = null;
        final DocumentType documentType = DocumentType.valueOf(context.getProperty(DOCUMENT_TYPE).getValue());
        switch (documentType) {
            case Json: {
                doc = RawJsonDocument.create(docId, new String(content, StandardCharsets.UTF_8));
                break;
            }
            case Binary: {
                final ByteBuf buf = Unpooled.copiedBuffer(content);
                doc = BinaryDocument.create(docId, buf);
                break;
            }
        }

        final PersistTo persistTo = PersistTo.valueOf(context.getProperty(PERSIST_TO).getValue());
        final ReplicateTo replicateTo = ReplicateTo.valueOf(context.getProperty(REPLICATE_TO).getValue());
        doc = openBucket(context).upsert(doc, persistTo, replicateTo);

        final Map<String, String> updatedAttrs = new HashMap<>();
        updatedAttrs.put(CouchbaseAttributes.Cluster.key(), context.getProperty(COUCHBASE_CLUSTER_SERVICE).getValue());
        updatedAttrs.put(CouchbaseAttributes.Bucket.key(), context.getProperty(BUCKET_NAME).getValue());
        updatedAttrs.put(CouchbaseAttributes.DocId.key(), docId);
        updatedAttrs.put(CouchbaseAttributes.Cas.key(), String.valueOf(doc.cas()));
        updatedAttrs.put(CouchbaseAttributes.Expiry.key(), String.valueOf(doc.expiry()));

        flowFile = session.putAllAttributes(flowFile, updatedAttrs);
        session.getProvenanceReporter().send(flowFile, getTransitUrl(context, docId));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final CouchbaseException e) {
        String errMsg = String.format("Writing document %s to Couchbase Server using %s failed due to %s", docId, flowFile, e);
        handleCouchbaseException(context, session, logger, flowFile, e, errMsg);
    }
}
 

/**
 * Returns a new document with 32 bit (int) timestamp expiration date for the document. Note this is a current limitation in couchbase.
 * This approach should work for documents that do not expire until 2038. This should be enough headroom for couchbase
 * to reimplement the design.
 * Source: https://forums.couchbase.com/t/document-expiry-in-seconds-or-a-timestamp/6519/6
 * @param record
 * @return
 */
private D setDocumentTTL(D record) throws DataRecordException {
  boolean recordIsTupleDocument = record instanceof TupleDocument;
  boolean recordIsJsonDocument = record instanceof RawJsonDocument;

  long ttlSpanSec = TimeUnit.SECONDS.convert(_documentTTL, _documentTTLTimeUnits);
  long eventOriginSec = 0;
  String dataJson = null;
  if (_documentTTL == 0) {
    return record;
  } else if (_documentTTLOriginField != null && !_documentTTLOriginField.isEmpty()) {

    if (recordIsTupleDocument) {
      ByteBuf dataByteBuffer = ((Tuple2<ByteBuf, Integer>) record.content()).value1();
      dataJson = new String(dataByteBuffer.array(), StandardCharsets.UTF_8);
    } else {
      dataJson = (String) record.content();
    }
    JsonElement jsonDataRootElement = new JsonParser().parse(dataJson);
    if (!jsonDataRootElement.isJsonObject()) {
      throw new DataRecordException(
          String.format("Document TTL Field is set but the record's value is not a valid json object.: '%s'",
              jsonDataRootElement.toString()));
    }
    JsonObject jsonDataRoot = jsonDataRootElement.getAsJsonObject();
    long documentTTLOrigin = jsonDataRoot.get(_documentTTLOriginField).getAsLong();
    eventOriginSec = TimeUnit.SECONDS.convert(documentTTLOrigin, _documentTTLOriginUnits);
  } else {
    eventOriginSec = System.currentTimeMillis() / 1000;
  }
  try {
    int expiration = Math.toIntExact(ttlSpanSec + eventOriginSec);
    if (recordIsTupleDocument) {
      return (D) _tupleDocumentTranscoder.newDocument(record.id(), expiration,
          (Tuple2<ByteBuf, Integer>) record.content(), record.cas(), record.mutationToken());
    } else if (recordIsJsonDocument) {
      return (D) RawJsonDocument.create(record.id(), expiration, (String) record.content(), record.cas(),
          record.mutationToken());
    } else {
      throw new RuntimeException(" Only TupleDocument and RawJsonDocument documents are supported");
    }
  } catch (ArithmeticException e) {
    throw new RuntimeException(
        "There was an overflow calculating the expiry timestamp. couchbase currently only supports expiry until January 19, 2038 03:14:07 GMT",
        e);
  }
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final byte[] content = new byte[(int) flowFile.getSize()];
    session.read(flowFile, new InputStreamCallback() {
        @Override
        public void process(final InputStream in) throws IOException {
            StreamUtils.fillBuffer(in, content, true);
        }
    });

    String docId = flowFile.getAttribute(CoreAttributes.UUID.key());
    if (context.getProperty(DOC_ID).isSet()) {
        docId = context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
    }

    try {
        Document<?> doc = null;
        final DocumentType documentType = DocumentType.valueOf(context.getProperty(DOCUMENT_TYPE).getValue());
        switch (documentType) {
            case Json: {
                doc = RawJsonDocument.create(docId, new String(content, StandardCharsets.UTF_8));
                break;
            }
            case Binary: {
                doc = ByteArrayDocument.create(docId, content);
                break;
            }
        }

        final PersistTo persistTo = PersistTo.valueOf(context.getProperty(PERSIST_TO).getValue());
        final ReplicateTo replicateTo = ReplicateTo.valueOf(context.getProperty(REPLICATE_TO).getValue());
        final Bucket bucket = openBucket(context);
        doc = bucket.upsert(doc, persistTo, replicateTo);

        final Map<String, String> updatedAttrs = new HashMap<>();
        updatedAttrs.put(CouchbaseAttributes.Cluster.key(), context.getProperty(COUCHBASE_CLUSTER_SERVICE).getValue());
        updatedAttrs.put(CouchbaseAttributes.Bucket.key(), bucket.name());
        updatedAttrs.put(CouchbaseAttributes.DocId.key(), docId);
        updatedAttrs.put(CouchbaseAttributes.Cas.key(), String.valueOf(doc.cas()));
        updatedAttrs.put(CouchbaseAttributes.Expiry.key(), String.valueOf(doc.expiry()));

        flowFile = session.putAllAttributes(flowFile, updatedAttrs);
        session.getProvenanceReporter().send(flowFile, getTransitUrl(bucket, docId));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final CouchbaseException e) {
        String errMsg = String.format("Writing document %s to Couchbase Server using %s failed due to %s", docId, flowFile, e);
        handleCouchbaseException(context, session, logger, flowFile, e, errMsg);
    }
}