org.apache.avro.generic.IndexedRecord Java Examples

@Test
public void testSyncCustomObjects() throws Exception {
    oprops.customObjectSyncAction.setValue(CustomObjectSyncAction.createOrUpdate);
    oprops.customObjectDedupeBy.setValue("marketoGUID");
    //
    doThrow(new MarketoException("REST", "error")).when(client).executePostRequest(eq(SyncResult.class),
            any(JsonObject.class));
    List<IndexedRecord> records = new ArrayList<>();
    IndexedRecord record = new Record(MarketoConstants.getCustomObjectRecordSchema());
    record.put(0, "mkto-123456");
    records.add(record);
    mktoSR = client.syncCustomObjects(oprops, records);
    assertFalse(mktoSR.isSuccess());
    assertFalse(mktoSR.getErrorsString().isEmpty());
    //
    doReturn(new SyncResult()).when(client).executePostRequest(eq(SyncResult.class), any(JsonObject.class));
    mktoSR = client.syncCustomObjects(oprops, records);
    assertFalse(mktoSR.isSuccess());
    //
    doReturn(getListOperationResult(true, "deleted")).when(client).executePostRequest(eq(SyncResult.class),
            any(JsonObject.class));
    mktoSR = client.syncCustomObjects(oprops, records);
    assertTrue(mktoSR.isSuccess());
    assertTrue(mktoSR.getErrorsString().isEmpty());
}
 

@Test
public void testGetCustomObjectWithCompoundKey() throws Exception {
    irProps.customObjectAction.setValue(CustomObjectAction.get);
    irProps.customObjectName.setValue(TEST_CO_NAME_CAR);
    irProps.validateFetchCustomObjectSchema();
    irProps.useCompoundKey.setValue(true);
    // "searchableFields": "[[\"customerId\",\"VIN\"],[\"marketoGUID\"],[\"customerId\"]]"
    irProps.compoundKey.keyName.setValue(Arrays.asList("customerId", "VIN"));
    irProps.compoundKey.keyValue.setValue(Arrays.asList("4137181", "WBA4R7C30HK896061"));// WBA4R7C55HK895912
    MarketoSource source = new MarketoSource();
    source.initialize(null, irProps);
    MarketoRESTClient client = (MarketoRESTClient) source.getClientService(null);
    MarketoRecordResult result = client.getCustomObjects(irProps, null);
    LOG.debug("result = {}.", result);
    assertNotNull(result.getRecords());
    assertEquals(1, result.getRecords().size());
    IndexedRecord record = result.getRecords().get(0);
    Schema s = record.getSchema();
    assertEquals(4137181, record.get(s.getField("customerId").pos()));
    assertEquals("WBA4R7C30HK896061", record.get(s.getField("VIN").pos()));
    assertEquals("FIT", record.get(s.getField("brand").pos()));
}
 

/**
 * Translate NetSuite data model object to {@code IndexedRecord}.
 *
 * @param data NetSuite data object
 * @return indexed record
 */
public IndexedRecord read(Object data) {
    prepare();

    Map<String, FieldDesc> fieldMap = typeDesc.getFieldMap();
    Map<String, Object> mapView = getMapView(data, runtimeSchema, typeDesc);

    GenericRecord indexedRecord = new GenericData.Record(runtimeSchema);

    for (Schema.Field field : runtimeSchema.getFields()) {
        String nsFieldName = NetSuiteDatasetRuntimeImpl.getNsFieldName(field);

        FieldDesc fieldDesc = fieldMap.get(nsFieldName);
        if (fieldDesc == null) {
            continue;
        }

        Object value = readField(mapView, fieldDesc);

        indexedRecord.put(field.name(), value);
    }

    return indexedRecord;
}
 

@Override
public RecordReader<Void, IndexedRecord> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException {
  String encoding = context.getConfiguration().get(TALEND_ENCODING);
  String sheet = context.getConfiguration().get(TALEND_EXCEL_SHEET_NAME);
  long header = context.getConfiguration().getLong(TALEND_HEADER, 0l);
  long footer = context.getConfiguration().getLong(TALEND_FOOTER, 0l);
  String excelFormat = context.getConfiguration().get(TALEND_EXCEL_FORMAT, "EXCEL2007");
  long limit = context.getConfiguration().getLong(TALEND_EXCEL_LIMIT, -1);
  
  if("EXCEL2007".equals(excelFormat)) {
    return new Excel2007FileRecordReader(sheet, header, footer, limit);
  } else if("EXCEL97".equals(excelFormat)) {
    return new Excel97FileRecordReader(sheet, header, footer, limit);
  } else if("HTML".equals(excelFormat)) {
    return new ExcelHTMLFileRecordReader(encoding, header, footer, limit);
  }
  
  throw new IOException("not a valid excel format");
}
 

@Test
public void testGetLeadsChanges() throws Exception {
    inputProperties.inputOperation.setValue(getLeadChanges);
    inputProperties.afterInputOperation();
    inputProperties.beforeMappingInput();
    inputProperties.batchSize.setValue(1000);
    //
    inputProperties.oldestCreateDate.setValue(DATE_OLDEST_CREATE);
    inputProperties.latestCreateDate.setValue(DATE_LATEST_CREATE);
    MarketoSource source = new MarketoSource();
    source.initialize(null, inputProperties);
    MarketoClientService client = source.getClientService(null);
    //
    MarketoRecordResult result = client.getLeadChanges(inputProperties, null);
    List<IndexedRecord> changes = result.getRecords();
    assertTrue(changes.size() > 0);
    assertTrue(result.getRemainCount() > 0);
    List<IndexedRecord> records = result.getRecords();
    assertTrue(records.size() > 0);
    for (IndexedRecord r : records) {
        assertNotNull(r.get(0));
        assertTrue(r.get(0) instanceof Long);
    }
}
 

@SuppressWarnings("unchecked")
public void serializeRecord0(IndexedRecord data, Encoder encoder)
    throws IOException
{
    CharSequence field0 = ((CharSequence) data.get(0));
    if (field0 == null) {
        (encoder).writeIndex(0);
        (encoder).writeNull();
    } else {
        if (field0 instanceof CharSequence) {
            (encoder).writeIndex(1);
            if (field0 instanceof Utf8) {
                (encoder).writeString(((Utf8) field0));
            } else {
                (encoder).writeString(field0 .toString());
            }
        }
    }
}
 

private void checkSamples(int limit, String checkFieldExists) throws Exception {
    runtime.initialize(null, dataset);
    final List<IndexedRecord> samples = new ArrayList<>();
    runtime.getSample(limit, new Consumer<IndexedRecord>() {

        @Override
        public void accept(IndexedRecord indexedRecord) {
            samples.add(indexedRecord);
        }
    });
    assertThat(samples.size(), Matchers.greaterThan(0));
    assertThat(samples.size(), Matchers.lessThan(limit + 1));
    for (IndexedRecord r : samples) {
        assertNotNull(r.get(r.getSchema().getField(checkFieldExists).pos()));
    }
}
 

@Override
public TableRow convertToDatum(IndexedRecord indexedRecord) {
    // When BigQueryOutput do not specify schema, so read it from the incoming data
    if (schema == null) {
        schema = indexedRecord.getSchema();
        initFieldConverters();
    }

    TableRow row = new TableRow();
    for (Schema.Field field : schema.getFields()) {
        Object v = indexedRecord.get(field.pos());
        if (v != null) {
            row.set(field.name(), fieldConverters.get(field.name()).convertToDatum(v));
        }
    }
    return row;
}
 

/**
 * From the input: {"a": "a", "b": "b", "c": "c"}
 * 
 * no extracted element.
 * 
 * The result should be:
 * 
 * key: empty
 * 
 * value: {"a": "a", "b": "b", "c": "c"}
 */
@Test
public void test_EverythingIsAValue() throws Exception {
    GenericRecord inputRecord = new GenericRecordBuilder(inputSimpleSchema) //
            .set("a", "a") //
            .set("b", "b") //
            .set("c", "c") //
            .build();

    String transformedIndexedRecord = ("{'key': {}, " + "'value': {'a': 'a', 'b': 'b', 'c': 'c'}}").replaceAll("\\'", "\"");
    IndexedRecord outputRecord = KeyValueUtils.transformToKV(inputRecord,
            SchemaGeneratorUtils.extractKeyValues(inputRecord.getSchema(), new ArrayList<String>()));
    assertEquals(transformedIndexedRecord, outputRecord.toString());

    Schema kvSchema = SchemaGeneratorUtils.mergeKeyValues(outputRecord.getSchema());
    String mergedRecord = ("{'a': 'a', 'b': 'b', 'c': 'c'}").replaceAll("\\'", "\"");
    assertEquals(mergedRecord, KeyValueUtils.transformFromKV(outputRecord, kvSchema).toString());
}
 

public void serialize(Map<CharSequence, IndexedRecord> data, Encoder encoder)
    throws IOException
{
    (encoder).writeMapStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        for (CharSequence key0 : ((Map<CharSequence, IndexedRecord> ) data).keySet()) {
            (encoder).startItem();
            (encoder).writeString(key0);
            IndexedRecord union0 = null;
            union0 = ((Map<CharSequence, IndexedRecord> ) data).get(key0);
            if (union0 == null) {
                (encoder).writeIndex(0);
                (encoder).writeNull();
            } else {
                if ((union0 instanceof IndexedRecord)&&"com.adpilot.utils.generated.avro.record".equals(((IndexedRecord) union0).getSchema().getFullName())) {
                    (encoder).writeIndex(1);
                    serializeRecord0(((IndexedRecord) union0), (encoder));
                }
            }
        }
    }
    (encoder).writeMapEnd();
}
 

public void serialize(Map<CharSequence, IndexedRecord> data, Encoder encoder)
    throws IOException
{
    (encoder).writeMapStart();
    if ((data == null)||data.isEmpty()) {
        (encoder).setItemCount(0);
    } else {
        (encoder).setItemCount(data.size());
        for (CharSequence key0 : ((Map<CharSequence, IndexedRecord> ) data).keySet()) {
            (encoder).startItem();
            (encoder).writeString(key0);
            IndexedRecord record0 = null;
            record0 = ((Map<CharSequence, IndexedRecord> ) data).get(key0);
            serializeRecord0(record0, (encoder));
        }
    }
    (encoder).writeMapEnd();
}
 

public static Object unionResolver(Object o) {
  if (o instanceof org.apache.avro.util.Utf8) {
    return o.toString();
  } else if (o instanceof IndexedRecord) {
    return new AvroTupleWrapper<IndexedRecord>((IndexedRecord) o);
  } else if (o instanceof GenericArray) {
    return new AvroBagWrapper<GenericData.Record>(
        (GenericArray<GenericData.Record>) o);
  } else if (o instanceof Map) {
    return new AvroMapWrapper((Map<CharSequence, Object>) o);
  } else if (o instanceof GenericData.Fixed) {
    return new DataByteArray(((GenericData.Fixed) o).bytes());
  } else if (o instanceof ByteBuffer) {
    return new DataByteArray(((ByteBuffer) o).array());
  } else if (o instanceof GenericEnumSymbol) {
    return o.toString();
  } else {
    return o;
  }
}
 

@Test
public void testGetLeadFail() throws Exception {
    inputProperties.inputOperation.setValue(getLead);
    inputProperties.leadKeyTypeSOAP.setValue(EMAIL);
    inputProperties.afterInputOperation();
    //
    inputProperties.leadKeyValue.setValue(EMAIL_INEXISTANT);
    MarketoSource source = new MarketoSource();
    source.initialize(null, inputProperties);
    MarketoClientService client = source.getClientService(null);
    //
    MarketoRecordResult result = client.getLead(inputProperties, null);
    List<IndexedRecord> records = result.getRecords();
    assertEquals(emptyList(), records);
    LOG.debug("record = " + records);
}
 

public void getLeadsSample(int limit, Consumer<IndexedRecord> consumer) {
    final List<String> idList = new ArrayList<>();
    Consumer<IndexedRecord> changes = new Consumer<IndexedRecord>() {

        @Override
        public void accept(IndexedRecord r) {
            idList.add(String.valueOf(r.get(r.getSchema().getField("leadId").pos())));
        }
    };
    dataset.operation.setValue(Operation.getLeadChanges);
    dataset.afterOperation();
    getLeadChangesSample();
    ReaderDataProvider<IndexedRecord> readerDataProvider = new ReaderDataProvider<>(source.createReader(container), limit,
            changes);
    readerDataProvider.retrieveData();
    dataset.operation.setValue(Operation.getLeads);
    dataset.afterOperation();
    properties.leadKeyType.setValue("id");
    properties.leadKeyValue.setValue(idList.stream().collect(Collectors.joining(",")));
    source.initialize(container, properties);
}
 

public IndexedRecord deserializerecord0(Object reuse, Decoder decoder)
    throws IOException
{
    IndexedRecord record;
    if ((((reuse)!= null)&&((reuse) instanceof IndexedRecord))&&(((IndexedRecord)(reuse)).getSchema() == arrayArrayElemSchema0)) {
        record = ((IndexedRecord)(reuse));
    } else {
        record = new org.apache.avro.generic.GenericData.Record(arrayArrayElemSchema0);
    }
    int unionIndex0 = (decoder.readIndex());
    if (unionIndex0 == 0) {
        decoder.readNull();
    }
    if (unionIndex0 == 1) {
        if (record.get(0) instanceof Utf8) {
            record.put(0, (decoder).readString(((Utf8) record.get(0))));
        } else {
            record.put(0, (decoder).readString(null));
        }
    }
    return record;
}
 

public static <T> List<IndexedRecord> makeIndexedRecords(NetSuiteClientService<?> clientService, Schema schema,
        ObjectComposer<T> objectComposer, int count) throws Exception {

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(clientService, schema, schema.getName());

    List<IndexedRecord> recordList = new ArrayList<>();

    while (count > 0) {
        T nsRecord = objectComposer.composeObject();

        IndexedRecord convertedRecord = transducer.read(nsRecord);
        Schema recordSchema = convertedRecord.getSchema();

        GenericRecord record = new GenericData.Record(recordSchema);
        for (Schema.Field field : schema.getFields()) {
            Object value = convertedRecord.get(field.pos());
            record.put(field.pos(), value);
        }

        recordList.add(record);

        count--;
    }

    return recordList;
}
 

/**
 * Checks {@link JiraUpdateWriter#write()} throws {@link IOException} which message contains
 * "Reason: record update failed"
 * in case server responses with 400 Bad Request status code
 * 
 * @throws IOException
 */
@Test
public void testWriteBadRequest() throws IOException {
    IndexedRecord badJsonRecord = new GenericData.Record(UPDATE_SCHEMA);
    String badProject = "{\"name\":\"Updated Integration Test Project\"\"assigneeType\":\"PROJECT_LEAD\"}";
    badJsonRecord.put(0, "TP");
    badJsonRecord.put(1, badProject);

    thrown.expect(IOException.class);
    thrown.expectMessage("Reason: record update failed");
    thrown.expectMessage("Record: " + badProject);
    thrown.expectMessage("Error: ");
    thrown.expectMessage("{\"errorMessages\":[\"Unexpected character (\'\\\"\' (code 34)):");

    JiraWriter updateProjectWriter = JiraTestsHelper.createWriter(HOST_PORT, USER, PASS, Resource.PROJECT, Action.UPDATE);

    updateProjectWriter.open("upd");
    try {
        updateProjectWriter.write(badJsonRecord);
    } finally {
        updateProjectWriter.close();
    }
}
 

@Test
public void testGetSampleExcelHtml() throws Exception {
    String fileSpec = sourceFilePrepare("sales-force.html");

    // Configure the component.
    SimpleFileIODatasetProperties props = createDatasetProperties();
    props.path.setValue(fileSpec);
    props.format.setValue(SimpleFileIOFormat.EXCEL);
    props.excelFormat.setValue(ExcelFormat.HTML);
    props.setHeaderLine.setValue(true);
    props.headerLine.setValue(1);

    final List<IndexedRecord> actual = getSample(props,100);

    assertThat(actual, hasSize(100));
    List<Field> fields = actual.get(0).getSchema().getFields();
    assertThat(fields, hasSize(7));
    assertThat("UID", equalTo(fields.get(0).name()));
    assertThat("Hire_Date", equalTo(fields.get(6).name()));
    
    assertThat("000001", equalTo(actual.get(0).get(0)));
    assertThat("France", equalTo(actual.get(0).get(5)));
}
 

private void checkRecord(TypeDescriptor<?> type, Schema schema) {
  // For a record, we want to make sure that all the fields are deterministic.
  Class<?> clazz = type.getRawType();
  for (Schema.Field fieldSchema : schema.getFields()) {
    Field field = getField(clazz, fieldSchema.name());
    String fieldContext = field.getDeclaringClass().getName() + "#" + field.getName();

    if (field.isAnnotationPresent(AvroEncode.class)) {
      reportError(
          fieldContext, "Custom encoders may be non-deterministic -- remove @AvroEncode");
      continue;
    }

    if (!IndexedRecord.class.isAssignableFrom(field.getType())
        && field.isAnnotationPresent(AvroSchema.class)) {
      // TODO: We should be able to support custom schemas on POJO fields, but we shouldn't
      // need to, so we just allow it in the case of IndexedRecords.
      reportError(
          fieldContext, "Custom schemas are only supported for subtypes of IndexedRecord.");
      continue;
    }

    TypeDescriptor<?> fieldType = type.resolveType(field.getGenericType());
    recurse(fieldContext, fieldType, fieldSchema.schema());
  }
}
 

/**
 * Converts the nested avro object in SamzaMessage to relational message corresponding to
 * the tableName with relational schema.
 */
@Override
public SamzaSqlRelMessage convertToRelMessage(KV<Object, Object> samzaMessage) {
  List<String> payloadFieldNames = new ArrayList<>();
  List<Object> payloadFieldValues = new ArrayList<>();
  Object value = samzaMessage.getValue();
  if (value instanceof IndexedRecord) {
    fetchFieldNamesAndValuesFromIndexedRecord((IndexedRecord) value, payloadFieldNames, payloadFieldValues,
        payloadSchema);
  } else if (value == null) {
    // If the payload is null, set each record value as null
    payloadFieldNames.addAll(payloadSchema.getFields().stream().map(Schema.Field::name).collect(Collectors.toList()));
    IntStream.range(0, payloadFieldNames.size()).forEach(x -> payloadFieldValues.add(null));
  } else {
    String msg = "Avro message converter doesn't support messages of type " + value.getClass();
    LOG.error(msg);
    throw new SamzaException(msg);
  }

  return new SamzaSqlRelMessage(samzaMessage.getKey(), payloadFieldNames, payloadFieldValues,
      new SamzaSqlRelMsgMetadata(0L, 0L));
}
 

/**
 * Utility method to generate a function for the given schema and check that it is appropriately deterministic.
 */
public static GeneratorFunction<IndexedRecord> generatorOfRecord(Schema schema) {
    // Create a function generator and context.
    GeneratorFunction<IndexedRecord> fn = GeneratorFunctions.ofRecord(schema);
    assertThat(fn, notNullValue());

    // Create a record.
    GeneratorContext ctx = generatorContextOf(0, 0L);
    IndexedRecord r1 = fn.apply(ctx);

    // Verify that reusing the function generator with the same context returns the same record.
    ctx.setRowId(0);
    assertThat(fn.apply(ctx).toString(), equalTo(r1.toString()));

    // Verify that generating a new function generator with the same schema returns the same record.
    assertThat(GeneratorFunctions.ofRecord(schema).apply(generatorContextOf(0, 0L)).toString(), equalTo(r1.toString()));

    // Return the generator for further testing.
    return fn;
}
 

@Test
public void testGetFileReader() throws IOException {
  // parquet file format.
  final Configuration hadoopConf = new Configuration();
  final Path parquetPath = new Path("/partition/path/f1_1-0-1_000.parquet");
  HoodieFileReader<IndexedRecord> parquetReader = HoodieFileReaderFactory.getFileReader(hadoopConf, parquetPath);
  assertTrue(parquetReader instanceof HoodieParquetReader);

  // other file format exception.
  final Path logPath = new Path("/partition/path/f.b51192a8-574b-4a85-b246-bcfec03ac8bf_100.log.2_1-0-1");
  final Throwable thrown = assertThrows(UnsupportedOperationException.class, () -> {
    HoodieFileReader<IndexedRecord> logWriter = HoodieFileReaderFactory.getFileReader(hadoopConf, logPath);
  }, "should fail since log storage reader is not supported yet.");
  assertTrue(thrown.getMessage().contains("format not supported yet."));
}
 

@Test
public void testGetCurrent() {
    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-1"), dummyCredential));
        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertTrue(reader.start());
        IndexedRecord current = reader.getCurrent();
        assertNotNull(current);
        assertEquals("queue-1", current.get(0));

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testIncludeAllFields() throws Exception {
    TypeDesc basicTypeDesc = connection.getBasicMetaData().getTypeInfo("Opportunity");
    Schema schema = getDynamicSchema();

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(connection, schema, basicTypeDesc.getTypeName());

    SearchResultSet<Record> rs = connection.newSearch()
            .target(basicTypeDesc.getTypeName())
            .search();

    TypeDesc typeDesc = connection.getMetaDataSource().getTypeInfo(basicTypeDesc.getTypeName());

    int count = 0;
    while (count++ < connection.getSearchPageSize() && rs.next()) {
        Record record = rs.get();
        IndexedRecord indexedRecord = transducer.read(record);
        logger.debug("Indexed record: {}", indexedRecord);

        Schema recordSchema = indexedRecord.getSchema();
        assertEquals(typeDesc.getFields().size(), recordSchema.getFields().size());

        for (FieldDesc fieldDesc : typeDesc.getFields()) {
            String fieldName = fieldDesc.getName();
            Schema.Field field = recordSchema.getField(fieldName);
            assertNotNull(field);

            Object value = indexedRecord.get(field.pos());
        }
    }
    if (count == 0) {
        throw new IllegalStateException("No records");
    }
}
 

/**
 * Try to find a field in either the key index record or the value index record
 * 
 * @param fieldPath the field name. Can be a path for hierarchical element
 * @param keyRecord an Indexed record
 * @param valueRecord an another Indexed record
 * @return the Object matching to the fieldName if it was found, null otherwise
 */
public static Object getFieldFromKV(String fieldPath, IndexedRecord keyRecord, IndexedRecord valueRecord) {
    // Start with the value record, there is an higher chance to find the field
    Object outputField = getFieldValue(fieldPath, valueRecord);
    if (outputField != null) {
        return outputField;
    } else {
        return getFieldValue(fieldPath, keyRecord);
    }
}
 

@Override
public IndexedRecord apply(GeneratorContext input) {
    if (schema == null) {
        schema = new Schema.Parser().parse(jsonSchema);
    }
    GenericData.Record record = new GenericData.Record(schema);
    for (int i = 0; i < inner.length; i++) {
        record.put(i, inner[i].apply(input));
    }
    return record;
}
 

@Override
public String apply(IndexedRecord input) {
    try {
        JsonNode jsonNode = mapper.readValue(String.valueOf(input.get(0)), JsonNode.class);
        return jsonNode.path("field").asText();
    } catch (IOException e) {
        return null;
    }
}
 

@Override
public IndexedRecord convertToAvro(String delimitedString) {
	String[] fields = delimitedString.split(delimiter);
	if (fields.length != size) {
		throw new IllegalArgumentException("Input string has wrong number of fields");
	}

	IndexedRecord record = new GenericData.Record(schema);
	for (int i = 0; i < size; i++) {
		Object value = converters[i].convertToAvro(fields[i]);
		record.put(i, value);
	}

	return record;
}
 

protected static void doWriteRows(SalesforceConnectionModuleProperties props, List<IndexedRecord> outputRows) throws Exception {
    SalesforceSink salesforceSink = new SalesforceSink();
    salesforceSink.initialize(adaptor, props);
    salesforceSink.validate(adaptor);
    SalesforceWriteOperation writeOperation = salesforceSink.createWriteOperation();
    Writer<Result> saleforceWriter = writeOperation.createWriter(adaptor);
    writeRows(saleforceWriter, outputRows);
}
 

@Test
public void testSyncLead() throws Exception {
    outProperties.outputOperation.setValue(OutputOperation.syncLead);
    outProperties.operationType.setValue(OperationType.createOrUpdate);
    outProperties.lookupField.setValue(RESTLookupFields.email);
    outProperties.deDupeEnabled.setValue(false);
    outProperties.updateSchemaRelated();
    outProperties.updateOutputSchemas();
    MarketoSource source = new MarketoSource();
    source.initialize(null, outProperties);
    MarketoClientService client = source.getClientService(null);
    //
    // test attributes
    List<Field> fields = new ArrayList<>();
    Field field = new Schema.Field("accountType", Schema.create(Schema.Type.STRING), null, (Object) null);
    fields.add(field);
    Schema s = MarketoUtils.newSchema(outProperties.schemaInput.schema.getValue(), "leadAttribute", fields);
    IndexedRecord record = new GenericData.Record(s);
    record.put(0, null);
    record.put(1, "[email protected]");
    record.put(2, "ForeignPersonSysId");
    record.put(3, "SFDC");// CUSTOM, SFDC, NETSUITE;
    record.put(4, "My firstName");
    List<IndexedRecord> leads = Arrays.asList(record);
    outProperties.schemaInput.schema.setValue(s);
    outProperties.beforeMappingInput();
    ///
    MarketoSyncResult result = client.syncLead(outProperties, record);
    LOG.debug("result = {}.", result);
    List<SyncStatus> changes = result.getRecords();
    assertTrue(changes.size() > 0);
    for (SyncStatus r : changes) {
        assertNotNull(r);
        assertNotNull(r.getId());
        LOG.debug("r = {}.", r);
    }
}