Java Code Examples for org.datavec.api.writable.Writable#toString()

public static void append(List<Writable> c, StringBuilder sb, String delim, String quote) {
    boolean first = true;
    for (Writable w : c) {
        if (!first)
            sb.append(delim);
        String s = w.toString();
        boolean needQuotes = s.contains(delim);
        if (needQuotes && quote != null) {
            sb.append(quote);
            s = s.replace(quote, quote + quote);
        }
        sb.append(s);
        if (needQuotes && quote != null) {
            sb.append(quote);
        }
        first = false;
    }
}
 

@Override
public LongQuality call(LongQuality v1, Writable writable) throws Exception {

    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long nonLong = v1.getCountNonLong();

    if (meta.isValid(writable))
        valid++;
    else if (writable instanceof NullWritable
                    || writable instanceof Text && (writable.toString() == null || writable.toString().isEmpty()))
        countMissing++;
    else
        invalid++;

    String str = writable.toString();
    try {
        Long.parseLong(str);
    } catch (NumberFormatException e) {
        nonLong++;
    }

    return new LongQuality(valid, invalid, countMissing, countTotal, nonLong);
}
 

public static void append(List<Writable> c, StringBuilder sb, String delim, String quote) {
    boolean first = true;
    for (Writable w : c) {
        if (!first)
            sb.append(delim);
        String s = w.toString();
        boolean needQuotes = s.contains(delim);
        if (needQuotes && quote != null) {
            sb.append(quote);
            s = s.replace(quote, quote + quote);
        }
        sb.append(s);
        if (needQuotes && quote != null) {
            sb.append(quote);
        }
        first = false;
    }
}
 

@Override
public CategoricalQuality call(CategoricalQuality v1, Writable writable) throws Exception {

    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;

    if (meta.isValid(writable))
        valid++;
    else if (writable instanceof NullWritable
                    || writable instanceof Text && (writable.toString() == null || writable.toString().isEmpty()))
        countMissing++;
    else
        invalid++;

    return new CategoricalQuality(valid, invalid, countMissing, countTotal);
}
 

@Override
public StringQuality apply(StringQuality v1, Writable writable) {
    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long empty = v1.getCountEmptyString();
    long alphabetic = v1.getCountAlphabetic();
    long numerical = v1.getCountNumerical();
    long word = v1.getCountWordCharacter();
    long whitespaceOnly = v1.getCountWhitespace();
    HyperLogLogPlus hll = v1.getHll();

    String str = writable.toString();

    if (writable instanceof NullWritable)
        countMissing++;
    else if (meta.isValid(writable))
        valid++;
    else
        invalid++;

    if (str == null || str.isEmpty()) {
        empty++;
    } else {
        if (str.matches("[a-zA-Z]"))
            alphabetic++;
        if (str.matches("\\d+"))
            numerical++;
        if (str.matches("\\w+"))
            word++;
        if (str.matches("\\s+"))
            whitespaceOnly++;
    }

    hll.offer(str);
    return new StringQuality(valid, invalid, countMissing, countTotal, empty, alphabetic, numerical, word,
                    whitespaceOnly, hll);
}
 

@Override
public Boolean call(Writable v1) throws Exception {
    boolean valid = meta.isValid(v1);
    if (excludeMissing && (v1 instanceof NullWritable
                    || v1 instanceof Text && (v1.toString() == null || v1.toString().isEmpty())))
        return false; //Remove (spark)
    if (keepValid)
        return valid; //Spark: return true to keep
    else
        return !valid;
}
 

@Override
public Boolean apply(Writable v1) {
    boolean valid = meta.isValid(v1);
    if (excludeMissing && (v1 instanceof NullWritable
                    || v1 instanceof Text && (v1.toString() == null || v1.toString().isEmpty())))
        return false; //Remove
    if (keepValid)
        return valid; //return true to keep
    else
        return !valid;
}
 

@Override
public boolean isValid(Writable writable) {
    String str = writable.toString();
    int len = str.length();
    if (minLength != null && len < minLength)
        return false;
    if (maxLength != null && len > maxLength)
        return false;

    return regex == null || str.matches(regex);
}
 

@Override
public StringQuality call(StringQuality v1, Writable writable) throws Exception {
    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long empty = v1.getCountEmptyString();
    long alphabetic = v1.getCountAlphabetic();
    long numerical = v1.getCountNumerical();
    long word = v1.getCountWordCharacter();
    long whitespaceOnly = v1.getCountWhitespace();
    HyperLogLogPlus hll = v1.getHll();

    String str = writable.toString();

    if (writable instanceof NullWritable)
        countMissing++;
    else if (meta.isValid(writable))
        valid++;
    else
        invalid++;

    if (str == null || str.isEmpty()) {
        empty++;
    } else {
        if (str.matches("[a-zA-Z]"))
            alphabetic++;
        if (str.matches("\\d+"))
            numerical++;
        if (str.matches("\\w+"))
            word++;
        if (str.matches("\\s+"))
            whitespaceOnly++;
    }

    hll.offer(str);
    return new StringQuality(valid, invalid, countMissing, countTotal, empty, alphabetic, numerical, word,
                    whitespaceOnly, hll);
}
 

@Override
public DoubleQuality call(DoubleQuality v1, Writable writable) throws Exception {

    long valid = v1.getCountValid();
    long invalid = v1.getCountInvalid();
    long countMissing = v1.getCountMissing();
    long countTotal = v1.getCountTotal() + 1;
    long nonReal = v1.getCountNonReal();
    long nan = v1.getCountNaN();
    long infinite = v1.getCountInfinite();

    if (meta.isValid(writable))
        valid++;
    else if (writable instanceof NullWritable
                    || writable instanceof Text && (writable.toString() == null || writable.toString().isEmpty()))
        countMissing++;
    else
        invalid++;

    String str = writable.toString();
    double d;
    try {
        d = Double.parseDouble(str);
        if (Double.isNaN(d))
            nan++;
        if (Double.isInfinite(d))
            infinite++;
    } catch (NumberFormatException e) {
        nonReal++;
    }

    return new DoubleQuality(valid, invalid, countMissing, countTotal, nonReal, nan, infinite);
}
 

@Override
public CategoricalAnalysisCounter add(Writable writable) {
    String value = writable.toString();

    long newCount = 0;
    if (counts.containsKey(value)) {
        newCount = counts.get(value);
    }
    newCount++;
    counts.put(value, newCount);

    countTotal++;

    return this;
}
 

@Override
public Text map(Writable writable) {
    String s = writable.toString();
    if (s == null || s.isEmpty())
        return new Text(value);
    else if (writable instanceof Text)
        return (Text) writable;
    else
        return new Text(writable.toString());
}
 

@Override
public Text map(Writable writable) {
    String str = writable.toString();
    if (exceptions.contains(str)) {
        return new Text(str);
    } else {
        return new Text(newValue);
    }
}
 

@Override
public List<Writable> map(List<Writable> writables) {
    if (writables.size() != inputSchema.numColumns()) {
        throw new IllegalStateException("Cannot execute transform: input writables list length (" + writables.size()
                        + ") does not " + "match expected number of elements (schema: " + inputSchema.numColumns()
                        + "). Transform = " + toString());
    }
    int idx = getColumnIdx();

    int n = stateNames.size();
    List<Writable> out = new ArrayList<>(writables.size() + n);

    int i = 0;
    for (Writable w : writables) {

        if (i++ == idx) {
            //Do conversion
            String str = w.toString();
            Integer classIdx = statesMap.get(str);
            if (classIdx == null) {
                throw new IllegalStateException("Cannot convert categorical value to one-hot: input value (\"" + str
                        + "\") is not in the list of known categories (state names/categories: " + stateNames + ")");
            }
            for (int j = 0; j < n; j++) {
                if (j == classIdx)
                    out.add(new IntWritable(1));
                else
                    out.add(new IntWritable(0));
            }
        } else {
            //No change to this column
            out.add(w);
        }
    }
    return out;
}
 

protected List<String> tokensFromRecord(Writable writable)
{

	String text = writable.toString();
	Tokenizer tokenizer = tokenizerFactory.create(text);
	List<String> tokens = new ArrayList<String>();
	while (tokenizer.hasMoreTokens())
        tokens.add(tokenizer.nextToken());
	return tokens;
}
 

@Override
public Writable convert(Writable writable) throws WritableConverterException {
    String label = writable.toString();
    return new IntWritable(labels.indexOf(label));
}
 

@Override
public String call(Writable writable) throws Exception {
    return writable.toString();
}
 

@Override
public Text map(final Writable writable) {
    String value = writable.toString();
    value = replaceAll(value);
    return new Text(value);
}
 

public String convertString(Writable writable) {
    return writable.toString();
}
 

/**
 * Transform the writable in to a
 * string
 *
 * @param writable the writable to transform
 * @return the string form of this writable
 */
@Override
public Text map(Writable writable) {
    return new Text(writable.toString());
}