nl.basjes.parse.core.Casts Java Examples

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    String name = extractFieldName(inputname, outputname);

    switch (name) {
        case "asn.number":
            wantAsnNumber = true;
            return STRING_OR_LONG;

        case "asn.organization":
            wantAsnOrganization = true;
            return STRING_ONLY;

        default:
            return NO_CASTS;
    }
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    EnumSet<Casts> result = super.prepareForDissect(inputname, outputname);
    if (!result.isEmpty()) {
        return result;
    }
    String name = extractFieldName(inputname, outputname);

    switch (name) {
        case "isp.name":
            wantIspName = true;
            return STRING_ONLY;

        case "isp.organization":
            wantIspOrganization = true;
            return STRING_ONLY;

        default:
            return NO_CASTS;
    }
}
 

public void run() throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
    HttpdLoglineParser<MyRecord> parser = new HttpdLoglineParser<>(MyRecord.class, logFormat);

    List<String> allPossiblePaths = parser.getPossiblePaths();
    parser.addParseTarget(MyRecord.class.getMethod("setter", String.class, String.class), allPossiblePaths);

    System.out.println("class MyRecord {\n");

    for (String field : parser.getPossiblePaths()) {
        for (Casts cast : parser.getCasts(field)) {
            System.out.println("    @Field{\"" + field + "\"}\n" +
                    "    public void setter(String name, " + castToJavaType(cast) + " value) {\n" +
                    "        System.out.println(\"SETTER CALLED FOR \\\"\" + name + \"\\\" = \\\"\" + value + \"\\\"\");\n" +
                    "    }\n");
        }
    }
    System.out.println("}\n");
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    String name = extractFieldName(inputname, outputname);
    if ("epoch".equals(name)) {
        wantTime = true;
        return STRING_OR_LONG;
    }
    if ("ip".equals(name)) {
        wantIp = true;
        return STRING_OR_LONG;
    }
    if ("processid".equals(name)) {
        wantProcessId = true;
        return STRING_OR_LONG;
    }
    if ("counter".equals(name)) {
        wantCounter = true;
        return STRING_OR_LONG;
    }
    if ("threadindex".equals(name)) {
        wantThreadIndex = true;
        return STRING_OR_LONG;
    }
    return NO_CASTS;
}
 

public NamedTokenParser(
        final String nLogFormatToken,
        final String nValueName,
        final String nValueType,
        final EnumSet<Casts> nCasts,
        final String nRegex,
        final int prio) {
    super(nLogFormatToken, nValueName, nValueType, nCasts, nRegex, prio);

    // Compile the regular expression
    pattern = Pattern.compile(getLogFormatToken());
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    String name = extractFieldName(inputname, outputname);

    switch (name) {
        case "continent.name":
            wantContinentName = true;
            wantAnyContinent = true;
            return STRING_ONLY;

        case "continent.code":
            wantContinentCode = true;
            wantAnyContinent = true;
            return STRING_ONLY;

        case "country.name":
            wantCountryName = true;
            wantAnyCountry = true;
            return STRING_ONLY;

        case "country.iso":
            wantCountryIso = true;
            wantAnyCountry = true;
            return STRING_ONLY;

        case "country.getconfidence":
            wantCountryGetConfidence = true;
            wantAnyCountry = true;
            return STRING_OR_LONG;

        case "country.isineuropeanunion":
            wantCountryIsInEuropeanUnion = true;
            wantAnyCountry = true;
            return STRING_OR_LONG;

        default:
            return NO_CASTS;
    }
}
 

@Override
public EnumSet<Casts> prepareForDissect(String inputname, String outputname) {
    String name = extractFieldName(inputname, outputname);
    if ("width".equals(name)) {
        wantWidth = true;
        return STRING_OR_LONG;
    }
    if ("height".equals(name)) {
        wantHeight = true;
        return STRING_OR_LONG;
    }
    return NO_CASTS;
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    String name = extractFieldName(inputname, outputname);
    if ("protocol".equals(name)) {
        wantProtocol = true;
        return STRING_ONLY;
    }
    if ("userinfo".equals(name)) {
        wantUserinfo = true;
        return STRING_ONLY;
    }
    if ("host".equals(name)) {
        wantHost = true;
        return STRING_ONLY;
    }
    if ("port".equals(name)) {
        wantPort = true;
        return STRING_OR_LONG;
    }
    if ("path".equals(name)) {
        wantPath = true;
        return STRING_ONLY;
    }
    if ("query".equals(name)) {
        wantQuery = true;
        return STRING_ONLY;
    }
    if ("ref".equals(name)) {
        wantRef = true;
        return STRING_ONLY;
    }
    return NO_CASTS;
}
 

@Override
public TokenParser addOutputField(String type, String name, EnumSet<Casts> casts) {
    if (getOutputFields().isEmpty()) {
        return super.addOutputField(type, name, casts);
    }
    throw new UnsupportedOperationException("A ParameterizedTokenParser only supports ONE outputfield.");
}
 

public ParameterizedTokenParser(
        final String nLogFormatToken,
        final String nValueName,
        final String nValueType,
        final EnumSet<Casts> nCasts,
        final String nRegex,
        final int prio,
        final Dissector customDissector) {
    super(nLogFormatToken, nValueName, nValueType, nCasts, nRegex, prio, customDissector);

    // Compile the regular expression
    pattern = Pattern.compile(getLogFormatToken());
}
 

/**
 * This record will be used with a single parser. For each field that is to be parsed a setter will be called. It
 * registers a setter method for each field being parsed. It also builds the data writers to hold the data beings
 * parsed.
 *
 * @param parser
 * @param mapWriter
 * @param type
 * @param parserFieldName
 * @param drillFieldName
 * @throws NoSuchMethodException
 */
public void addField(final Parser<HttpdLogRecord> parser, final MapWriter mapWriter, final EnumSet<Casts> type, final String parserFieldName, final String drillFieldName) throws NoSuchMethodException {
  final boolean hasWildcard = parserFieldName.endsWith(HttpdParser.PARSER_WILDCARD);

  /**
   * This is a dynamic way to map the setter for each specified field type. <br/>
   * e.g. a TIME.STAMP may map to a LONG while a referrer may map to a STRING
   */
  if (hasWildcard) {
    final String cleanName = parserFieldName.substring(0, parserFieldName.length() - HttpdParser.PARSER_WILDCARD.length());
    LOG.debug("Adding WILDCARD parse target: {} as {}, with field name: {}", parserFieldName, cleanName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, String.class), parserFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Double.class), parserFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Long.class), parserFieldName);
    wildcards.put(cleanName, mapWriter.map(drillFieldName));
  } else if (type.contains(Casts.DOUBLE)) {
    LOG.debug("Adding DOUBLE parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("set", String.class, Double.class), parserFieldName);
    doubles.put(parserFieldName, mapWriter.float8(drillFieldName));
  } else if (type.contains(Casts.LONG)) {
    LOG.debug("Adding LONG parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("set", String.class, Long.class), parserFieldName);
    longs.put(parserFieldName, mapWriter.bigInt(drillFieldName));
  } else {
    LOG.debug("Adding STRING parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    if (parserFieldName.startsWith("TIME.STAMP:")) {
      parser.addParseTarget(this.getClass().getMethod("setTimestamp", String.class, String.class), parserFieldName);
      times.put(parserFieldName, mapWriter.timeStamp(drillFieldName));
    } else {
      parser.addParseTarget(this.getClass().getMethod("set", String.class, String.class), parserFieldName);
      strings.put(parserFieldName, mapWriter.varChar(drillFieldName));
    }
  }
}
 

public TokenParser(final String nLogFormatToken,
        final String nValueName,
        final String nValueType,
        final EnumSet<Casts> nCasts,
        final String nRegex,
        final int nPrio,
       final Dissector nCustomDissector) {
    logFormatToken = nLogFormatToken;
    regex = nRegex;
    prio = nPrio;
    customDissector = nCustomDissector;
    addOutputField(nValueType, nValueName, nCasts);
}
 

public TokenParser(final String nLogFormatToken,
                   final String nValueName,
                   final String nValueType,
                   final EnumSet<Casts> nCasts,
                   final String nRegex,
                   final int nPrio) {
    this(nLogFormatToken, nValueName, nValueType, nCasts, nRegex, nPrio, null);
}
 

public TokenParser(final String nLogFormatToken,
        final String nValueName,
        final String nValueType,
        final EnumSet<Casts> nCasts,
        final String nRegex) {
    this(nLogFormatToken, nValueName, nValueType, nCasts, nRegex, 10);
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputName, final String outputName) {
    requestedFields.add(outputName);
    for (Token token: logFormatTokens) {
        for (TokenOutputField tokenOutputField: token.getOutputFields()) {
            if (outputName.equals(tokenOutputField.getName())) {
                tokenOutputField.wasUsed();
                return tokenOutputField.getCasts();
            }
        }
    }
    return STRING_ONLY;
}
 

public NamedTokenParser(
        final String nLogFormatToken,
        final String nValueName,
        final String nValueType,
        final EnumSet<Casts> nCasts,
        final String nRegex) {
    this(nLogFormatToken, nValueName, nValueType, nCasts, nRegex, 0);
}
 

@Override
public EnumSet<Casts> prepareForDissect(String inputname, String outputname) { // NOSONAR: MUST be EnumSet
    String name = extractFieldName(inputname, outputname);
    String fieldName = dissectionNameToFieldName(name);

    if (fieldName == null) {
        LOG.error("There is NO fieldname for the requested \"{}\" ({})", outputname, name);
        return NO_CASTS;
    }
    requestedFieldNames.add(fieldName);
    return STRING_ONLY; // We ONLY do Strings here
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    String name = extractFieldName(inputname, outputname);
    switch (name) {
        case "value":   return STRING_ONLY;
        case "expires": return STRING_OR_LONG;
        case "path":    return STRING_ONLY;
        case "domain":  return STRING_ONLY;
        case "comment": return STRING_ONLY;
        default:        return STRING_ONLY;
    }
}
 

@Override
public EnumSet<Casts> prepareForDissect(String inputname, String outputname) {
    String name = extractFieldName(inputname, outputname);

    if (name.endsWith(OUTPUT_ORIGINAL_NAME)) {
        return outputOriginalCasts;
    }
    if (name.endsWith(OUTPUT_REDIRECTED_NAME)) {
        return outputRedirectedCasts;
    }
    return NO_CASTS;
}
 

public UpstreamListDissector(String inputType,
                             String outputOriginalType, EnumSet<Casts> outputOriginalCasts,
                             String outputRedirectedType, EnumSet<Casts> outputRedirectedCasts) {
    this.inputType = inputType;
    this.outputOriginalType     = outputOriginalType;
    this.outputOriginalCasts    = outputOriginalCasts;
    this.outputRedirectedType   = outputRedirectedType;
    this.outputRedirectedCasts  = outputRedirectedCasts;
}
 

public EnumSet<Casts> getCasts(String name) throws IOException {
    if (outputAllPossibleFields) {
        return allCasts.get(name);
    }
    try {
        return getParser().getCasts(name);
    } catch (MissingDissectorsException | InvalidDissectorException e) {
        throw new IOException("Fatal error in the parser", e);
    }
}
 

@Override
public ResourceSchema getSchema(String location, Job job) throws IOException {
    ResourceSchema rs = new ResourceSchema();
    List<ResourceFieldSchema> fieldSchemaList = new ArrayList<>();

    for (String fieldName : requestedFields) {
        ResourceFieldSchema rfs = new ResourceFieldSchema();
        rfs.setName(fieldName);
        rfs.setDescription(fieldName);

        if(fieldName.endsWith(".*")) {
            rfs.setType(DataType.MAP);
        } else {
            EnumSet<Casts> casts = theInputFormat.getRecordReader().getCasts(fieldName);
            if (casts != null) {
                if (casts.contains(Casts.LONG)) {
                    rfs.setType(DataType.LONG);
                } else {
                    if (casts.contains(Casts.DOUBLE)) {
                        rfs.setType(DataType.DOUBLE);
                    } else {
                        rfs.setType(DataType.CHARARRAY);
                    }
                }
            } else {
                rfs.setType(DataType.BYTEARRAY);
            }
        }
        fieldSchemaList.add(rfs);
    }

    rs.setFields(fieldSchemaList.toArray(new ResourceFieldSchema[fieldSchemaList.size()]));
    return rs;
}
 

private String castToJavaType(Casts casts) {
    switch (casts) {
        case STRING:
            return "String";
        case LONG:
            return "Long";
        case DOUBLE:
            return "Double";
        default:
            return null;
    }
}
 

private List<TokenParser> createFirstAndLastTokenParsers(
    final String nLogFormatToken,
    final String nValueName,
    final String nValueType,
    final EnumSet<Casts> nCasts,
    final String nRegex) {
    return createFirstAndLastTokenParsers(nLogFormatToken, nValueName, nValueType, nCasts, nRegex, 0);
}
 

@Override
public EnumSet<Casts> prepareForDissect(String inputname, String outputname) {
    if (dissectors.isEmpty()) {
        return NO_CASTS;
    }

    EnumSet<Casts> result = EnumSet.noneOf(Casts.class); // Start empty
    for (Dissector dissector : dissectors) {
        result.addAll(dissector.prepareForDissect(inputname, outputname));
    }
    return result;
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    requestedCookies.add(extractFieldName(inputname, outputname));
    return STRING_ONLY;
}
 

@Override
public EnumSet<Casts> prepareForDissect(String inputname, String outputname) {
    return STRING_ONLY;
}
 

@Override
public EnumSet<Casts> prepareForDissect(String inputname, String outputname) {
    return STRING_ONLY;
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    EnumSet<Casts> result = super.prepareForDissect(inputname, outputname);
    if (!result.isEmpty()) {
        return result;
    }

    String name = extractFieldName(inputname, outputname);

    switch (name) {

        case "subdivision.name":
            wantSubdivisionName = true;
            wantAnySubdivision = true;
            return STRING_ONLY;

        case "subdivision.iso":
            wantSubdivisionIso = true;
            wantAnySubdivision = true;
            return STRING_ONLY;

        // ---------------------------------

        case "city.name":
            wantCityName = true;
            wantAnyCity = true;
            return STRING_ONLY;

        case "city.confidence":
            wantCityConfidence = true;
            wantAnyCity = true;
            return STRING_OR_LONG;

        case "city.geonameid":
            wantCityGeoNameId = true;
            wantAnyCity = true;
            return STRING_OR_LONG;

        // ---------------------------------

        case "postal.code":
            wantPostalCode = true;
            wantAnyPostal = true;
            return STRING_ONLY;

        case "postal.confidence":
            wantPostalConfidence = true;
            wantAnyPostal = true;
            return STRING_OR_LONG;

        // ---------------------------------

        case "location.latitude":
            wantLocationLatitude = true;
            wantAnyLocation = true;
            return STRING_OR_DOUBLE;

        case "location.longitude":
            wantLocationLongitude = true;
            wantAnyLocation = true;
            return STRING_OR_DOUBLE;

        case "location.accuracyradius":
            wantLocationAccuracyradius = true;
            wantAnyLocation = true;
            return STRING_OR_LONG;

        case "location.timezone":
            wantLocationTimezone = true;
            wantAnyLocation = true;
            return STRING_ONLY;

        case "location.averageincome":
            wantLocationAverageincome = true;
            wantAnyLocation = true;
            return STRING_OR_LONG;

        case "location.metrocode":
            wantLocationMetrocode = true;
            wantAnyLocation = true;
            return STRING_OR_LONG;

        case "location.populationdensity":
            wantLocationPopulationdensity = true;
            wantAnyLocation = true;
            return STRING_OR_LONG;
        default:
            return NO_CASTS;
    }
}
 

@Override
public EnumSet<Casts> prepareForDissect(final String inputname, final String outputname) {
    requestedCookies.add(extractFieldName(inputname, outputname));
    return STRING_ONLY;
}