Java Code Examples for org.apache.spark.sql.Row#getAs()

@Override
public boolean check(Row row) {
  for (String field : fields) {
    Object value = row.getAs(field);
    if (value != null) {
      if (!(value instanceof Comparable)) {
        throw new RuntimeException("Range checkInternal on non-comparable type");
      }
      if (!checkInternal((Comparable<?>)value, lower, upper)) {
        return false;
      }
    }
    else {
      return ignoreNulls;
    }
  }
  return true;
}
 

@Override
public Iterable<Row> translate(Row message) {
  String value = message.getAs(Translator.VALUE_FIELD_NAME);

  String[] stringValues = value.split((delimiterRegex) ?
                          delimiter : Pattern.quote(delimiter), schema.length());
  values.clear();

  for (int valuePos = 0; valuePos < schema.length(); valuePos++) {
    Object rowVal = null; 
    if (valuePos < stringValues.length) {
      String fieldValue = stringValues[valuePos];
      DataType fieldType = schema.fields()[valuePos].dataType();

      if (fieldValue.length() > 0) {
        rowVal = RowUtils.toRowValue(fieldValue, fieldType, rowValueMetadata);
      }
    }
    values.add(rowVal); 
  }

  Row row = new RowWithSchema(schema, values.toArray());

  return Collections.singleton(row);
}
 

/**
 *
 * @param collectorItemMetricDetail
 * @param itemRow
 *
 * Update collectorItemMetric details with latest timeSeries data and summary.
 *
 */
private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail,Row itemRow) {

    Date timeWindowDt = itemRow.getAs(STR_TIME_WINDOW);
    List<String> testTypes = Arrays.asList(STR_AUTOMATED, STR_MANUAL);
    GenericRowWithSchema javaCollection = itemRow.getAs(STR_TRACEABILITY);
    testTypes.forEach(testType -> {
        double value = javaCollection.getAs(testType);
        MetricCount mc = getMetricCount("", value, testType);
        if (mc != null) {
            collectorItemMetricDetail.setStrategy(getCollectionStrategy());
            collectorItemMetricDetail.addCollectorItemMetricCount(timeWindowDt, mc);
            collectorItemMetricDetail.setLastScanDate(timeWindowDt);
        }
    });
}
 

private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail, Row row) {
    Date timeWindowDt = row.getAs(STR_TIMEWINDOW);
    String auditStatus = row.getAs(STR_AUDIT_STATUS);
    String auditType = row.getAs("auditType");
    String auditTypeStatus = row.getAs("auditTypeStatus");
    auditMap.put("auditStatus", auditStatus);
    auditMap.put("auditType", auditType);
    auditMap.put("auditTypeStatus", auditTypeStatus);

    double value = 0.0;
    if ("OK".equalsIgnoreCase(auditStatus)){
        value = 1.0;
    }
    MetricCount mc = getMetricCount("", value, auditType);
    if (!mc.getLabel().isEmpty()) {
        collectorItemMetricDetail.setStrategy(getCollectionStrategy());
        collectorItemMetricDetail.addCollectorItemMetricCount(timeWindowDt, mc);
        collectorItemMetricDetail.setLastScanDate(timeWindowDt);
    }
}
 

private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail, Row itemRow) {
    Date timeWindowDt = itemRow.getAs("timeWindow");
    List<String> metricList = Arrays.asList("High", "Medium", "Low");
    Collection<Object> javaCollection = JavaConversions.asJavaCollection(((WrappedArray) itemRow.getAs("metrics")).toList());

    Optional.ofNullable(javaCollection)
            .orElseGet(Collections::emptyList)
            .forEach(m -> {
                GenericRowWithSchema genericRowWithSchema = (GenericRowWithSchema) m;
                String existingLabelName = genericRowWithSchema.getAs("name");
                if (metricList.contains(existingLabelName)) {
                    String valueStr = genericRowWithSchema.getAs("value");
                    try {
                        double value = Double.parseDouble(valueStr);
                        MetricCount mc = getMetricCount("", value, existingLabelName);
                        if (mc != null) {
                            collectorItemMetricDetail.setStrategy(getCollectionStrategy());
                            collectorItemMetricDetail.addCollectorItemMetricCount(timeWindowDt, mc);
                            collectorItemMetricDetail.setLastScanDate(timeWindowDt);
                        }
                    } catch (NumberFormatException e) {
                        LOGGER.info("Exception: Not a number, 'value' = "+valueStr,e);
                    }
                }
            });
}
 

private boolean matchesValueFilter(Row row, Row filter) {
  for (String filterFieldName : filter.schema().fieldNames()) {
    Object rowValue = row.get(row.fieldIndex(filterFieldName));
    Object filterValue = filter.getAs(filterFieldName);
    
    if (!rowValue.equals(filterValue)) {
      return false;
    }
  }
  
  return true;
}
 

private Row carryForwardWhenNull(Row into, Row from) {
  if (!config.hasPath(CARRY_FORWARD_CONFIG_NAME) || !config.getBoolean(CARRY_FORWARD_CONFIG_NAME)) {
    return into;
  }

  for (StructField field : into.schema().fields()) {
    String fieldName = field.name();
    if (into.getAs(fieldName) == null && from.getAs(fieldName) != null) {
      into = RowUtils.set(into, fieldName, from.getAs(fieldName));
    }
  }

  return into;
}
 

private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail, Row itemRow) {
    Date timeWindowDt = itemRow.getAs("timeWindow");
    LOGGER.info("TimeWindow:" +timeWindowDt );
    LOGGER.info("itemRow :" + itemRow);
    Collection<Object> javaCollection = JavaConversions.asJavaCollection(((WrappedArray) itemRow.getAs("prodStageList")).toList());

    Optional.ofNullable(javaCollection)
            .orElseGet(Collections::emptyList).stream().map(m -> (GenericRowWithSchema) m).forEach(genericRowWithSchema -> {
        Long pipelineTimeL = genericRowWithSchema.getAs("timestamp");
        Date dateObj = new Timestamp(new Date(pipelineTimeL).getTime());
        LOGGER.info("Date Object :" + dateObj);
        Long scmTimeL = genericRowWithSchema.getAs("scmCommitTimestamp");
        Long pipelineTimeAfterIgnore = pipelineTimeL/1000;
        Long scmTimeAfterIgnore = scmTimeL/1000;
        try {
            Long diffTimestamp = Math.abs(pipelineTimeAfterIgnore - scmTimeAfterIgnore);
            String strTimestampInsec = Long.toString(diffTimestamp);
            double value = Double.parseDouble(strTimestampInsec);
            MetricCount mc = getMetricCount("", value, "pipeline-lead-time");
            if (mc != null) {
                collectorItemMetricDetail.setStrategy(getCollectionStrategy());
                collectorItemMetricDetail.addCollectorItemMetricCount(dateObj, mc);
                collectorItemMetricDetail.setLastScanDate(dateObj);
            }
        } catch (NumberFormatException e) {
            LOGGER.info("Exception: Not a number, 'value' = " + scmTimeAfterIgnore, e);
        }
    });
}
 

private Row carryForwardWhenNull(Row into, Row from) {
  if (!doesCarryForward()) {
    return into;
  }

  for (StructField field : into.schema().fields()) {
    String fieldName = field.name();
    if (into.getAs(fieldName) == null && from.getAs(fieldName) != null) {
      into = RowUtils.set(into, fieldName, from.getAs(fieldName));
    }
  }

  return into;
}
 

private Config performSubstitutions(Config c, Row row) {
  Config substConfig = c;
  for (StructField field : row.schema().fields()) {
    Object value = row.getAs(field.name());
    substConfig = ConfigUtils.findReplaceStringValues(substConfig, "\\$\\{" + field.name() + "\\}",
      value != null ? value : "null");
  }
  return substConfig;
}
 

@Override
public Iterable<Row> translate(Row message) throws Exception {
  byte[] value = message.getAs(Translator.VALUE_FIELD_NAME);

  Decoder decoder = DecoderFactory.get().binaryDecoder(value, null);
  GenericRecord record = reader.read(null, decoder);
  Row row = rowForRecord(record);

  return Collections.singleton(row);
}
 

@Override
public Row getTime(Row row) {
  return new RowWithSchema(getSchema(), row.getAs(field.name()));
}
 

@Override
public Iterable<Row> translate(Row message) throws Exception {
  Object key = message.getAs("key");
  Object value = message.getAs(Translator.VALUE_FIELD_NAME);

  // Get the Morphline Command pipeline
  if (null == this.pipeline) {
    this.pipeline = MorphlineUtils.getPipeline(this.morphlineFile, this.morphlineId);

    // If null, then instantiate the pipeline
    if (null == this.pipeline) {
      this.pipeline = MorphlineUtils.setPipeline(this.morphlineFile, this.morphlineId, new MorphlineUtils.Collector(),true);
    }
  }

  // Construct the input Record
  Record inputRecord = new Record();

  // Set up the message as _attachment_body (standard Morphline convention)
  inputRecord.put(Fields.ATTACHMENT_BODY, value);
  inputRecord.put(Fields.ATTACHMENT_CHARSET, this.messageEncoding);

  // Add the key as a custom Record field
  if (null != key) {
    inputRecord.put(TRANSLATOR_KEY, key);
    inputRecord.put(TRANSLATOR_KEY_CHARSET, this.keyEncoding);
  }

  // TODO : Consider using the MorphlineContext exception handler
  // Execute the pipeline (runtime errors are not caught)
  List<Record> outputRecords = MorphlineUtils.executePipeline(this.pipeline, inputRecord);

  // Convert output to Rows
  List<Row> outputRows = Lists.newArrayListWithCapacity(outputRecords.size());
  for (Record output: outputRecords) {
    Row outputRow = MorphlineUtils.convertToRow(this.schema, output);

    outputRows.add(outputRow);
  }

  return outputRows;
}
 

@Override
public List<Feature> encodeFeatures(SparkMLEncoder encoder){
	ImputerModel transformer = getTransformer();

	Double missingValue = transformer.getMissingValue();
	String strategy = transformer.getStrategy();
	Dataset<Row> surrogateDF = transformer.surrogateDF();

	MissingValueTreatmentMethod missingValueTreatmentMethod = parseStrategy(strategy);

	List<Row> surrogateRows = surrogateDF.collectAsList();
	if(surrogateRows.size() != 1){
		throw new IllegalArgumentException();
	}

	Row surrogateRow = surrogateRows.get(0);

	InOutMode inputMode = getInputMode();

	List<Feature> result = new ArrayList<>();

	String[] inputCols = inputMode.getInputCols(transformer);
	for(String inputCol : inputCols){
		Feature feature = encoder.getOnlyFeature(inputCol);

		Field<?> field = feature.getField();

		if(field instanceof DataField){
			DataField dataField = (DataField)field;

			Object surrogate = surrogateRow.getAs(inputCol);

			encoder.addDecorator(dataField, new MissingValueDecorator(missingValueTreatmentMethod, surrogate));

			if(missingValue != null && !missingValue.isNaN()){
				PMMLUtil.addValues(dataField, Collections.singletonList(missingValue), Value.Property.MISSING);
			}
		} else

		{
			throw new IllegalArgumentException();
		}

		result.add(feature);
	}

	return result;
}
 

@Override
public Row call(Row row) throws Exception {
  return new RowWithSchema(SCHEMA, name, row.<Long>getAs("count") == thisExpected);
}
 

private boolean hadError(Row row) {
  return row.getAs(TranslateFunction.HAD_ERROR_FIELD_NAME);
}
 

@Override
public Row getTime(Row row) {
  return new RowWithSchema(getSchema(), 
      row.getAs(nanoField.name()), row.getAs(seqNumField.name()));
}
 

@Override
public Row merge(Row a, Row b) {
  return new RowWithSchema(SCHEMA, name,
      a.<Boolean>getAs("result") && b.<Boolean>getAs("result"));
}
 

@Override
public Row getTime(Row row) {
  return new RowWithSchema(getSchema(), row.getAs(field.name()));
}
 

/**
 * Returns true if the given CodeableConcept row has a Coding belonging to the ValueSet having the
 * given reference name, or false otherwise.
 */
private static Boolean inValueSet(Row codeableRow,
    String referenceName,
    BroadcastableValueSets valueSets) {

  boolean found = false;

  if (codeableRow != null) {

    List<Row> codingArray = codeableRow.getList(1);

    if (codingArray != null) {

      for (Row coding : codingArray) {

        String system = coding.getAs("system");
        String code = coding.getAs("code");

        // If there exists a matching code, return true.
        if (valueSets.hasCode(referenceName, system, code)) {

          found = true;

          break;
        }
      }
    }
  }

  return found;
}