Java Code Examples for org.apache.avro.generic.IndexedRecord#put()

/**
 * Checks {@link JiraUpdateWriter#write(Object)} updates issues on Jira server
 * 
 * @throws IOException
 */
public void testUpdateIssues() throws IOException {
    IndexedRecord updateIssueRecord1 = new GenericData.Record(UPDATE_SCHEMA);
    String updateIssue1 = "{\"fields\":{\"summary\":\"Updated test issue 1\"}}";
    updateIssueRecord1.put(0, "ITP-1");
    updateIssueRecord1.put(1, updateIssue1);

    IndexedRecord updateIssueRecord2 = new GenericData.Record(UPDATE_SCHEMA);
    String updateIssue2 = "{\"fields\":{\"summary\":\"Updated test issue 2\"}}";
    updateIssueRecord2.put(0, "ITP-2");
    updateIssueRecord2.put(1, updateIssue2);
    
    JiraWriter updateIssueWriter = JiraTestsHelper.createWriter(HOST_PORT, USER, PASS, Resource.ISSUE, Action.UPDATE);
    
    updateIssueWriter.open("updIss");
    try {
        updateIssueWriter.write(updateIssueRecord1);
        updateIssueWriter.write(updateIssueRecord2);
        updateIssueWriter.close();
    } catch (DataRejectException e) {
        String rejectError = e.getRejectInfo().get("error").toString();
        LOG.error(rejectError);
        collector.addError(new Throwable(rejectError));
    }
}
 

public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == mapMapValueSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(mapMapValueSchema0);
    }
    int unionIndex0 = (decoder.readIndex());
    if (unionIndex0 == 0) {
        decoder.readNull();
    }
    if (unionIndex0 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 

public IndexedRecord deserializerecordName0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord recordName;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == readerSchema)) {
        recordName = ((IndexedRecord)(reuse));
    } else {
        recordName = new org.apache.avro.generic.GenericData.Record(readerSchema);
    }
    if (recordName.get(0) instanceof Utf8) {
        recordName.put(0, (decoder).readString(((Utf8) recordName.get(0))));
    } else {
        recordName.put(0, (decoder).readString(null));
    }
    int unionIndex0 = (decoder.readIndex());
    if (unionIndex0 == 0) {
        decoder.readNull();
    }
    if (unionIndex0 == 1) {
        recordName.put(1, deserializerecordName0(recordName.get(1), (decoder)));
    }
    return recordName;
}
 

@Override
protected boolean doProcess(Record inputRecord) {
  Record outputRecord = inputRecord.copy();
  AbstractParser.removeAttachments(outputRecord);
  Map<String, Collection<Object>> map = inputRecord.getFields().asMap();
  map = new HashMap<String, Collection<Object>>(map); // make it mutable
  Field field = schema.getFields().get(0);
  Object avroResult = AvroConversions.toAvro(map, field.schema());
  if (avroResult == AvroConversions.ERROR) {
    LOG.debug("Cannot convert record: {} to schema: {}", inputRecord, schema);
    return false;          
  }
  
  IndexedRecord avroRecord = new GenericData.Record(schema); 
  avroRecord.put(field.pos(), avroResult);
  outputRecord.put(Fields.ATTACHMENT_BODY, avroRecord);
  
  // pass record to next command in chain:
  return super.doProcess(outputRecord);
}
 

/**
 * Checks {@link JiraInsertWriter#write(Object)} inserts issues on Jira server
 * 
 * @throws IOException
 */
public void testInsertIssues() throws IOException {
    IndexedRecord insertIssueRecord1 = new GenericData.Record(INSERT_SCHEMA);
    String insertIssue1 = "{\"fields\":{\"project\":{\"key\":\"ITP\"},\"summary\":\"Integration test issue 1\",\"issuetype\":{\"id\":\"10000\"}}}";
    insertIssueRecord1.put(0, insertIssue1);

    IndexedRecord insertIssueRecord2 = new GenericData.Record(INSERT_SCHEMA);
    String insertIssue2 = "{\"fields\":{\"project\":{\"key\":\"ITP\"},\"summary\":\"Integration test issue 2\",\"issuetype\":{\"id\":\"10000\"}}}";
    insertIssueRecord2.put(0, insertIssue2);
    
    JiraWriter insertIssueWriter = JiraTestsHelper.createWriter(HOST_PORT, USER, PASS, Resource.ISSUE, Action.INSERT);
    
    insertIssueWriter.open("insIss");
    try {
        insertIssueWriter.write(insertIssueRecord1);
        insertIssueWriter.write(insertIssueRecord2);
        insertIssueWriter.close();

    } catch (DataRejectException e) {
        String rejectError = e.getRejectInfo().get("error").toString();
        LOG.error(rejectError);
        collector.addError(new Throwable(rejectError));
    }
}
 

public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == arrayElemOptionSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(arrayElemOptionSchema0);
    }
    int unionIndex1 = (decoder.readIndex());
    if (unionIndex1 == 0) {
        decoder.readNull();
    }
    if (unionIndex1 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 

public IndexedRecord deserializerecordName0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord recordName;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == readerSchema)) {
        recordName = ((IndexedRecord)(reuse));
    } else {
        recordName = new org.apache.avro.generic.GenericData.Record(readerSchema);
    }
    if (recordName.get(0) instanceof Utf8) {
        recordName.put(0, (decoder).readString(((Utf8) recordName.get(0))));
    } else {
        recordName.put(0, (decoder).readString(null));
    }
    int unionIndex0 = (decoder.readIndex());
    if (unionIndex0 == 0) {
        decoder.readNull();
    }
    if (unionIndex0 == 1) {
        recordName.put(1, deserializerecordName0(recordName.get(1), (decoder)));
    }
    return recordName;
}
 

private void handleReject(IndexedRecord input, SQLException e) throws IOException {
    rejectCount++;

    IndexedRecord reject = new GenericData.Record(rejectSchema);

    for (Schema.Field outField : reject.getSchema().getFields()) {
        Object outValue = null;
        Schema.Field inField = input.getSchema().getField(outField.name());

        if (inField != null) {
            outValue = input.get(inField.pos());
        } else if ("errorCode".equals(outField.name())) {
            outValue = e.getSQLState();
        } else if ("errorMessage".equals(outField.name())) {
            outValue = e.getMessage() + " - Line: " + result.totalCount;
        }

        reject.put(outField.pos(), outValue);
    }

    rejectedWrites.add(reject);
}
 

public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == arrayArrayElemSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(arrayArrayElemSchema0);
    }
    int unionIndex0 = (decoder.readIndex());
    if (unionIndex0 == 0) {
        decoder.readNull();
    }
    if (unionIndex0 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 

@Test
public void testSyncMultipleLeads() throws Exception {
    doReturn(getSyncMultipleLeadsResult()).when(port).syncMultipleLeads(any(ParamsSyncMultipleLeads.class),
            any(AuthenticationHeader.class));
    oprops.afterOutputOperation();
    oprops.beforeMappingInput();
    mktoSR = client.syncLead(oprops, null);
    assertNotNull(mktoSR);
    assertFalse(mktoSR.isSuccess());
    IndexedRecord record = new Record(MarketoConstants.getSOAPOutputSchemaForSyncLead());
    record.put(0, 12345);
    record.put(1, "[email protected]");
    mktoSR = client.syncMultipleLeads(oprops, Arrays.asList(record));
    assertNotNull(mktoSR);
    assertTrue(mktoSR.isSuccess());
    //
    doThrow(new RuntimeException("error")).when(port).syncMultipleLeads(any(ParamsSyncMultipleLeads.class),
            any(AuthenticationHeader.class));
    mktoSR = client.syncMultipleLeads(oprops, Arrays.asList(record));
    assertNotNull(mktoSR);
    assertFalse(mktoSR.isSuccess());
}
 

@Test(expected = IndexedRecordConverter.UnmodifiableAdapterException.class)
public void testIndexedRecordUnmodifiable() throws IOException {
    converter.setSchema(SCHEMA);

    BulkResult result = new BulkResult();
    result.setValue("Id", "12345");
    result.setValue("Name", "Qwerty");
    result.setValue("FieldX", "42");
    result.setValue("FieldY", "true");

    IndexedRecord indexedRecord = converter.convertToAvro(result);
    indexedRecord.put(1, "Asdfgh");
}
 

@Test(expected = IndexedRecordConverter.UnmodifiableAdapterException.class)
public void testIndexedRecordUnmodifiable() throws IOException {
    converter.setSchema(SCHEMA);

    BulkResult result = new BulkResult();
    result.setValue("Id", "12345");
    result.setValue("Name", "Qwerty");
    result.setValue("FieldX", "42");
    result.setValue("FieldY", "true");

    IndexedRecord indexedRecord = converter.convertToAvro(result);
    indexedRecord.put(1, "Asdfgh");
}
 

@SuppressWarnings("rawtypes")
@Test
public void testReplace() throws Throwable {
    currentTable = tbl_test + "Replace";
    insertTestValues(currentTable);
    properties.schema.schema.setValue(getSimpleTestSchema());
    properties.actionOnData.setValue(ActionOnData.Replace);
    properties.schemaListener.afterSchema();
    Writer<?> writer = createWriter(properties);
    writer.open("test-uid");
    for (String p : partitions) {
        for (String r : rows) {
            // IndexedRecord entity = new GenericData.Record(getWriteSchema());
            IndexedRecord entity = new GenericData.Record(getSimpleTestSchema());
            entity.put(0, p);
            entity.put(1, r);
            entity.put(2, "NewValue");
            writer.write(entity);
        }
    }
    writer.close();
    // check results...
    BoundedReader reader = createReader(currentTable, filter, false);
    int counted = 0;
    assertTrue(reader.start());
    do {
        counted++;
        IndexedRecord current = (IndexedRecord) reader.getCurrent();
        assertEquals(current.get(current.getSchema().getField("StringValue").pos()), "NewValue");
        assertEquals(4, current.getSchema().getFields().size());
    } while (reader.advance());
    reader.close();
    // we should have read 9 rows...
    assertEquals(9, counted);
}
 

private void fillIndexed(IndexedRecord record, String[] data) {
  for (int i = 0; i < indexes.length; i += 1) {
    int index = indexes[i];
    record.put(i,
        makeValue(index < data.length ? data[index] : null, fields[i]));
  }
}
 

@Override
public void write(Object object) throws IOException {
    if (object == null) {
        return;
    }
    IndexedRecord input = (IndexedRecord) object;
    Object data = input.get(0);
    LOG.debug("data [{}] {}.", data.getClass().getCanonicalName(), data.toString());
    byte[] bytes = null;
    if (data instanceof byte[]) {
        bytes = (byte[]) data;
    } else {
        bytes = data.toString().getBytes();
    }
    //
    String destinationFolderId = properties.destinationFolderAccessMethod.getValue().equals(AccessMethod.Id)
            ? properties.destinationFolder.getValue()
            : utils.getFolderId(properties.destinationFolder.getValue(), false);
    GoogleDrivePutParameters p = new GoogleDrivePutParameters(destinationFolderId, properties.fileName.getValue(),
            properties.overwrite.getValue(), bytes);
    sentFile = utils.putResource(p);
    //
    IndexedRecord record = new Record(properties.schemaMain.schema.getValue());
    record.put(0, bytes);
    // TODO should return this values in outOfBandRecord
    record.put(1, sentFile.getParents().get(0));
    record.put(2, sentFile.getId());
    cleanWrites();
    result.successCount++;
    result.totalCount++;
    successfulWrites.add(record);
}
 

@Test
public void testSyncLead() throws Exception {
    MarketoSource source = new MarketoSource();
    outProperties.outputOperation.setValue(OutputOperation.syncLead);
    outProperties.updateSchemaRelated();
    source.initialize(null, listProperties);
    MarketoClientService client = source.getClientService(null);
    // test attributes
    List<Field> fields = new ArrayList<>();
    Field field = new Schema.Field("FirstName", Schema.create(Schema.Type.STRING), null, (Object) null);
    fields.add(field);
    field = new Schema.Field("LastName", Schema.create(Schema.Type.STRING), null, (Object) null);
    fields.add(field);
    field = new Schema.Field("AccountType", Schema.create(Schema.Type.STRING), null, (Object) null);
    fields.add(field);
    Schema s = MarketoUtils.newSchema(outProperties.schemaInput.schema.getValue(), "leadAttribute", fields);
    IndexedRecord record = new GenericData.Record(s);
    record.put(0, null);
    record.put(1, "[email protected]");
    record.put(2, "ForeignPersonSysId");
    record.put(3, "SFDC");// CUSTOM, SFDC, NETSUITE;
    record.put(4, "My firstName");
    record.put(5, "My lastName");
    record.put(6, "Conservative customer");
    outProperties.schemaInput.schema.setValue(s);
    outProperties.beforeMappingInput();
    //
    MarketoSyncResult result = client.syncLead(outProperties, record);
    assertEquals("UPDATED", result.getRecords().get(0).getStatus());
}
 

@Test
public void testUpdate() throws Exception {
    TJDBCOutputDefinition definition = new TJDBCOutputDefinition();
    TJDBCOutputProperties properties = DBTestUtils.createCommonJDBCOutputProperties(allSetting, definition);

    Schema schema = DBTestUtils.createTestSchema2(tablename);
    properties.main.schema.setValue(schema);
    properties.updateOutputSchemas();

    properties.tableSelection.tablename.setValue(tablename);
    properties.dataAction.setValue(DataAction.UPDATE);
    properties.dieOnError.setValue(true);

    String randomInfo = randomBatchAndCommit(properties);

    JDBCOutputWriter writer = DBTestUtils.createCommonJDBCOutputWriter(definition, properties);

    try {
        writer.open("wid");

        IndexedRecord r1 = new GenericData.Record(properties.main.schema.getValue());
        r1.put(0, 1);
        r1.put(1, "wangwei1");
        writer.write(r1);

        DBTestUtils.assertSuccessRecord(writer, r1);

        IndexedRecord r2 = new GenericData.Record(properties.main.schema.getValue());
        r2.put(0, 2);
        r2.put(1, "gaoyan1");
        writer.write(r2);

        DBTestUtils.assertSuccessRecord(writer, r2);

        writer.close();
    } finally {
        writer.close();
    }

    TJDBCInputDefinition definition1 = new TJDBCInputDefinition();
    TJDBCInputProperties properties1 = DBTestUtils.createCommonJDBCInputProperties(allSetting, definition1);
    List<IndexedRecord> records = DBTestUtils.fetchDataByReaderFromTable(tablename, schema, definition1, properties1);

    assertThat(records, hasSize(3));
    Assert.assertEquals(new Integer(1), records.get(0).get(0));
    Assert.assertEquals("wangwei1", records.get(0).get(1));
    Assert.assertEquals(new Integer(2), records.get(1).get(0));
    Assert.assertEquals(randomInfo, "gaoyan1", records.get(1).get(1));
    Assert.assertEquals(new Integer(3), records.get(2).get(0));
    Assert.assertEquals("dabao", records.get(2).get(1));
}
 

@SuppressWarnings("rawtypes")
@Test
public void testCommit() throws IOException, ClassNotFoundException, SQLException {
    // connection part
    TJDBCConnectionDefinition connectionDefinition = new TJDBCConnectionDefinition();
    TJDBCConnectionProperties connectionProperties = DBTestUtils.createCommonJDBCConnectionProperties(allSetting,
            connectionDefinition);

    JDBCSourceOrSink sourceOrSink = new JDBCSourceOrSink();
    sourceOrSink.initialize(null, connectionProperties);

    ValidationResult result = sourceOrSink.validate(container);
    assertTrue(result.getStatus() == ValidationResult.Result.OK);

    // output part
    TJDBCOutputDefinition outputDefinition = new TJDBCOutputDefinition();
    TJDBCOutputProperties outputProperties = (TJDBCOutputProperties) outputDefinition.createRuntimeProperties();

    outputProperties.main.schema.setValue(DBTestUtils.createTestSchema(tablename));
    outputProperties.updateOutputSchemas();

    outputProperties.tableSelection.tablename.setValue(tablename);

    outputProperties.dataAction.setValue(DataAction.INSERT);

    outputProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
    outputProperties.referencedComponent.setReference(connectionProperties);

    JDBCSink sink = new JDBCSink();
    sink.initialize(container, outputProperties);

    WriteOperation writerOperation = sink.createWriteOperation();
    writerOperation.initialize(container);
    JDBCOutputInsertWriter writer = (JDBCOutputInsertWriter) writerOperation.createWriter(container);

    try {
        writer.open("wid");

        IndexedRecord r1 = new GenericData.Record(outputProperties.main.schema.getValue());
        r1.put(0, 4);
        r1.put(1, "xiaoming");
        writer.write(r1);

        DBTestUtils.assertSuccessRecord(writer, r1);

        IndexedRecord r2 = new GenericData.Record(outputProperties.main.schema.getValue());
        r2.put(0, 5);
        r2.put(1, "xiaobai");
        writer.write(r2);

        DBTestUtils.assertSuccessRecord(writer, r2);

        writer.close();
    } finally {
        writer.close();
    }

    // commit part
    TJDBCCommitDefinition commitDefinition = new TJDBCCommitDefinition();
    TJDBCCommitProperties commitProperties = (TJDBCCommitProperties) commitDefinition.createRuntimeProperties();

    commitProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
    commitProperties.closeConnection.setValue(false);

    JDBCCommitSourceOrSink commitSourceOrSink = new JDBCCommitSourceOrSink();
    commitSourceOrSink.initialize(container, commitProperties);
    commitSourceOrSink.validate(container);

    int count = -1;

    // create another session and check if the data is inserted
    try (Connection conn = JdbcRuntimeUtils.createConnection(allSetting);
            Statement statement = conn.createStatement();
            ResultSet resultset = statement.executeQuery("select count(*) from " + tablename)) {
        if (resultset.next()) {
            count = resultset.getInt(1);
        }
    }

    Assert.assertEquals(5, count);

    try (java.sql.Connection refConnection = (java.sql.Connection) container
            .getComponentData(ComponentConstants.CONNECTION_KEY, refComponentId)) {
        assertTrue(refConnection != null);
        Assert.assertTrue(!refConnection.isClosed());
    }
}
 

@Override
public IndexedRecord getCurrent() throws NoSuchElementException {
    if (!startable || (advanceable != null && !advanceable)) {
        throw new NoSuchElementException();
    }

    IndexedRecord record = new GenericData.Record(schema);
    try {
        for (Field f : schema.getFields()) {
            switch (f.name()) {
            case TAzureStorageQueueInputProperties.FIELD_MESSAGE_ID:
                record.put(f.pos(), current.getMessageId());
                break;
            case TAzureStorageQueueInputProperties.FIELD_MESSAGE_CONTENT:
                record.put(f.pos(), current.getMessageContentAsString());
                break;
            case TAzureStorageQueueInputProperties.FIELD_INSERTION_TIME:
                record.put(f.pos(), current.getInsertionTime());
                break;
            case TAzureStorageQueueInputProperties.FIELD_EXPIRATION_TIME:
                record.put(f.pos(), current.getExpirationTime());
                break;
            case TAzureStorageQueueInputProperties.FIELD_DEQUEUE_COUNT:
                record.put(f.pos(), current.getDequeueCount());
                break;
            case TAzureStorageQueueInputProperties.FIELD_POP_RECEIPT:
                record.put(f.pos(), current.getPopReceipt());
                break;
            case TAzureStorageQueueInputProperties.FIELD_NEXT_VISIBLE_TIME:
                record.put(f.pos(), current.getNextVisibleTime());
                break;
            default:
                LOGGER.warn(i18nMessages.getMessage("warn.UnknowField", f));
            }
        }
    } catch (StorageException e) {
        LOGGER.error(e.getLocalizedMessage());
        if (dieOnError)
            throw new ComponentException(e);
    }

    return record;
}
 

@Override
protected Object createComposite(Object[] children) {

  IndexedRecord record = (IndexedRecord) avroData.newRecord(null, getDataType());

  for (int i = 0; i < children.length; ++i) {

    record.put(i, children[i]);
  }

  return record;
}