Java Code Examples for org.kitesdk.morphline.api.Record#get()

/** Returns the values of a {@link Record} referred to by the given field expression */
public List evaluate(Record record) {
  // fast path (not functionally necessary):
  if (fields.size() == 1) {
    Object first = fields.get(0);
    if (first instanceof String) {
      return Collections.singletonList(first);
    } else {
      String ref = ((Field) first).getName();
      if (ref.length() != 0) {
        List resolvedValues = record.get(ref);
        return resolvedValues;
      }
    }
  }

  // slow path:
  ArrayList<Object> results = new ArrayList<Object>(1);
  evaluate2(0, record, new StringBuilder(), results);
  return results;
}
 

private void evaluate2(int from, Record record, StringBuilder buf, ArrayList<Object> results) {
  if (from >= fields.size()) {
    results.add(buf.toString());
    return;
  }
  
  Object item = fields.get(from);
  if (item instanceof String) {
    buf.append(item);
    evaluate2(from + 1, record, buf, results);
  } else {
    String ref = ((Field) item).getName();
    if (ref.length() == 0) {
      buf.append(record.toString()); // @{} means dump string representation of entire record
      evaluate2(from + 1, record, buf, results);
    } else {
      List resolvedValues = record.get(ref);
      for (Object value : resolvedValues) {
        StringBuilder buf2 = new StringBuilder(buf);
        buf2.append(value.toString());
        evaluate2(from + 1, record, buf2, results);
      }
    }
  }
}
 

@Override
protected boolean doProcess(Record record) {
  try {
    List outputValues = record.get(outputFieldName);
    for (Object value : record.get(inputFieldName)) {
      reader.setValue(value.toString());
      TokenStream tokenStream = analyzer.tokenStream("content", reader);
      tokenStream.reset();
      while (tokenStream.incrementToken()) {
        if (token.length() > 0) { // incrementToken() updates the token!
          String tokenStr = new String(token.buffer(), 0, token.length());
          outputValues.add(tokenStr);
        }
      }
      tokenStream.end();
      tokenStream.close();
    }
  } catch (IOException e) {
    throw new MorphlineRuntimeException(e);
  }
  
  // pass record to next command in chain:
  return super.doProcess(record);
}
 

protected Charset detectCharset(Record record, Charset charset) {
  if (charset != null) {
    return charset;
  }
  List charsets = record.get(Fields.ATTACHMENT_CHARSET);
  if (charsets.size() == 0) {
    // TODO try autodetection (AutoDetectReader)
    throw new MorphlineRuntimeException("Missing charset for record: " + record); 
  }
  String charsetName = (String) charsets.get(0);        
  return Charset.forName(charsetName);
}
 

@Override
protected boolean doProcess(Record record) {      
  List attachments = record.get(Fields.ATTACHMENT_BODY);
  List mimeTypes = record.get(Fields.ATTACHMENT_MIME_TYPE);
  List charsets = record.get(Fields.ATTACHMENT_CHARSET);
  List names = record.get(Fields.ATTACHMENT_NAME);
  for (int i = 0; i < attachments.size(); i++) {
    Record outputRecord = record.copy();        
    outputRecord.getFields().replaceValues(Fields.ATTACHMENT_BODY, Collections.singletonList(attachments.get(i)));
    
    List<Object> replacement;
    replacement = i < mimeTypes.size() ? Collections.singletonList(mimeTypes.get(i)) : Collections.emptyList();
    outputRecord.getFields().replaceValues(Fields.ATTACHMENT_MIME_TYPE, replacement);
    
    replacement = i < charsets.size() ? Collections.singletonList(charsets.get(i)) : Collections.emptyList();
    outputRecord.getFields().replaceValues(Fields.ATTACHMENT_CHARSET, replacement);
    
    replacement = i < names.size() ? Collections.singletonList(names.get(i)) : Collections.emptyList();
    outputRecord.getFields().replaceValues(Fields.ATTACHMENT_NAME, replacement);
    
    // pass record to next command in chain:
    if (!super.doProcess(outputRecord)) {
      return false;
    }
  }
  return true;
}
 

@Override
protected boolean doProcess(Record record) {
  for (Object value : record.get(inputFieldName)) {
    Iterable<String> columns = splitter.split(value.toString());
    if (outputFieldNames == null) {
      record.getFields().putAll(outputFieldName, columns);
    } else {
      extractColumns(record, columns);
    }
  }
  
  // pass record to next command in chain:
  return super.doProcess(record);
}
 

private void doProcessFast(Record record) {
  for (String name : nameMatcher.getLiteralsOnly()) {
    List values = record.get(name);
    for (int i = values.size(); --i >= 0; ) {
      if (valueMatcher.matches(values.get(i).toString())) {
        if (replacement != null) {
          values.set(i, replacement);
        } else {
          values.remove(i);
        }
      }
    }
  }
}
 

@Override
protected boolean doProcess(Record record) {
  for (Object item : record.get(inputFieldName)) {
    String str = item.toString();
    int start;
    int end;
    if (regex != null) {
      if (regex.reset(str).find()) {
        start = regex.start();
        end = regex.end();
      } else {
        start = -1;
        end = -1;
      }   
    } else if (separator.length() == 1) {
      start = str.indexOf(separator.charAt(0));
      end = start + 1;
    } else {
      start = str.indexOf(separator);
      end = start + separator.length();
    }
    
    String key = str;
    String value = "";        
    if (start >= 0) { // found?
      key = str.substring(0, start);
      value = str.substring(end, str.length());
      value = trim(value);        
    }
    if (value.length() > 0 || addEmptyStrings) {
      record.put(concat(outputFieldPrefix, trim(key)), value);
    }
  }
  
  // pass record to next command in chain:
  return super.doProcess(record);
}
 

@Override
protected boolean doProcess(Record record) {      
  for (Object value : record.get(inputFieldName)) {
    Preconditions.checkNotNull(value);
    String stringValue = value.toString().trim();
    for (Mapping mapping : mappings) {
      mapping.apply(record, stringValue);
    }
  }
  
  // pass record to next command in chain:
  return super.doProcess(record);
}
 

private boolean doMatch(Record inputRecord, Record outputRecord, boolean doExtract) {
  for (Regex regex : regexes) {        
    Matcher matcher = regex.getMatcher();
    List values = inputRecord.get(regex.getRecordInputField());
    int todo = values.size();
    int minMatches = 1;
    int maxMatches = Integer.MAX_VALUE;
    switch (numRequiredMatches) {
      case once : { 
        maxMatches = 1;
        break;
      }
      case all : { 
        minMatches = todo;
        break;
      }
      default: {
        break;
      }
    }        
    int numMatches = 0;
    for (Object value : values) {
      matcher.reset(value.toString());
      if (!findSubstrings) {
        if (matcher.matches()) {
          numMatches++;
          if (numMatches > maxMatches) {
            LOG.debug("grok failed because it found too many matches for values: {} for grok command: {}",
                      values, renderedConfig);
            return false;
          }
          extract(outputRecord, regex, doExtract);
        }
      } else {
        int previousNumMatches = numMatches;
        while (matcher.find()) {
          if (numMatches == previousNumMatches) {
            numMatches++;
            if (numMatches > maxMatches) {
              LOG.debug("grok failed because it found too many matches for values: {} for grok command: {}",
                        values, renderedConfig);
              return false;
            }
            if (!doExtract && numMatches >= minMatches && maxMatches == Integer.MAX_VALUE) {
              break; // fast path
            }
          }
          extract(outputRecord, regex, doExtract);
        }
      }
      todo--;
      if (!doExtract && numMatches >= minMatches && maxMatches == Integer.MAX_VALUE) {
        break; // fast path
      }
    }
    if (numMatches + todo < minMatches) {
      LOG.debug("grok failed because it found too few matches for values: {} for grok command: {}", 
                values, renderedConfig);
      return false;          
    }
  }
  return true;
}
 

@Override
protected boolean doProcess(Record record) {
  for (Object uri : record.get(inputFieldName)) {
    String rawURIQuery = "";
    try {
      rawURIQuery = new URI(uri.toString()).getRawQuery();
    } catch (URISyntaxException e) {
      LOG.debug("Invalid URI: {}", uri);
      if (failOnInvalidURI) {
        return false;
      }
    }
    if (rawURIQuery == null) {
      rawURIQuery = "";
    }
    
    int len = rawURIQuery.length();
    int i = 0;
    int numParameters = 0;
    while (numParameters < maxParameters && i < len) {
      int j = rawURIQuery.indexOf('&', i);
      if (j < 0) {
        j = len;
      }
      for (int k = i; k < j; k++) {
        if (rawURIQuery.charAt(k) == '=') {
          String key = decodeString(rawURIQuery.substring(i, k));
          if (parameterName.equals(key)) {
            String value = decodeString(rawURIQuery.substring(k + 1, j));
            record.put(outputFieldName, value);
            numParameters++;
          }              
          break; // found '=' separator
        }
      }
      i = j;
      i++;
    }
  }
  
  // pass record to next command in chain:
  return super.doProcess(record);
}
 

/**
 * Get all lifecycle events from the given record.
 * @return A {@link List} of {@link LifecycleEvent} enumerations representing all lifecycles stored in the given notification.
 */
public static List getLifecycleEvents(Record notification) {
  return notification.get(LIFE_CYLCLE);
}