org.apache.parquet.filter2.predicate.Operators.FloatColumn Java Examples

The following examples show how to use org.apache.parquet.filter2.predicate.Operators.FloatColumn. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParquetTableSource.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Nullable
private FilterPredicate lessThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThan, "exp has to be LessThan");

	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.lt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.lt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.lt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.lt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}
Example #2
Source File: ParquetTableSource.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Nullable
private FilterPredicate lessThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThan, "exp has to be LessThan");

	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.lt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.lt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.lt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.lt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}
Example #3
Source File: DictionaryFilterTest.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGtFloat() throws Exception {
  FloatColumn f = floatColumn("float_field");
  float highest = Float.MIN_VALUE;
  for (int value : intValues) {
    highest = Math.max(highest, toFloat(value));
  }

  assertTrue("Should drop: > highest value",
      canDrop(gt(f, highest), ccmd, dictionaries));
  assertFalse("Should not drop: > (highest value - 1.0)",
      canDrop(gt(f, highest - 1.0f), ccmd, dictionaries));

  assertFalse("Should not drop: contains matching values",
      canDrop(gt(f, Float.MIN_VALUE), ccmd, dictionaries));
}
Example #4
Source File: ParquetTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nullable
private FilterPredicate greaterThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThan, "exp has to be GreaterThan");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}
Example #5
Source File: ParquetTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nullable
private FilterPredicate greaterThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThanOrEqual, "exp has to be GreaterThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gtEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gtEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gtEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gtEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}
Example #6
Source File: ParquetTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nullable
private FilterPredicate lessThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThanOrEqual, "exp has to be LessThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.ltEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.ltEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.ltEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.ltEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}
Example #7
Source File: ParquetTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nullable
private FilterPredicate greaterThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThan, "exp has to be GreaterThan");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}
Example #8
Source File: ParquetTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nullable
private FilterPredicate greaterThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThanOrEqual, "exp has to be GreaterThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gtEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gtEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gtEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gtEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}
Example #9
Source File: ParquetTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nullable
private FilterPredicate lessThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThanOrEqual, "exp has to be LessThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.ltEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.ltEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.ltEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.ltEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}
Example #10
Source File: FilterApi.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
public static FloatColumn floatColumn(String columnPath) {
  return new FloatColumn(ColumnPath.fromDotString(columnPath));
}
Example #11
Source File: TestColumnIndexBuilder.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBuildFloat() {
  PrimitiveType type = Types.required(FLOAT).named("test_float");
  ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  assertThat(builder, instanceOf(FloatColumnIndexBuilder.class));
  assertNull(builder.build());
  FloatColumn col = floatColumn("test_col");

  StatsBuilder sb = new StatsBuilder();
  builder.add(sb.stats(type, -4.2f, -4.1f));
  builder.add(sb.stats(type, -11.7f, 7.0f, null));
  builder.add(sb.stats(type, 2.2f, 2.2f, null, null));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, 1.9f, 2.32f));
  builder.add(sb.stats(type, -21.0f, 8.1f));
  assertEquals(6, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  ColumnIndex columnIndex = builder.build();
  assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0);
  assertCorrectNullPages(columnIndex, false, false, false, true, false, false);
  assertCorrectValues(columnIndex.getMaxValues(), -4.1f, 7.0f, 2.2f, null, 2.32f, 8.1f);
  assertCorrectValues(columnIndex.getMinValues(), -4.2f, -11.7f, 2.2f, null, 1.9f, -21.0f);
  assertCorrectFiltering(columnIndex, eq(col, 0.0f), 1, 5);
  assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
  assertCorrectFiltering(columnIndex, notEq(col, 2.2f), 0, 1, 2, 3, 4, 5);
  assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4, 5);
  assertCorrectFiltering(columnIndex, gt(col, 2.2f), 1, 4, 5);
  assertCorrectFiltering(columnIndex, gtEq(col, 2.2f), 1, 2, 4, 5);
  assertCorrectFiltering(columnIndex, lt(col, 0.0f), 0, 1, 5);
  assertCorrectFiltering(columnIndex, ltEq(col, 1.9f), 0, 1, 4, 5);
  assertCorrectFiltering(columnIndex, userDefined(col, FloatIsInteger.class), 1, 4, 5);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, FloatIsInteger.class)), 0, 1, 2, 3, 4, 5);

  builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  sb = new StatsBuilder();
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, -532.3f, -345.2f, null, null));
  builder.add(sb.stats(type, -300.6f, -234.7f, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, -234.6f, 2.99999f));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, 3.0f, 42.83f));
  builder.add(sb.stats(type, null, null));
  assertEquals(9, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  columnIndex = builder.build();
  assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2);
  assertCorrectNullPages(columnIndex, true, false, false, true, true, false, true, false, true);
  assertCorrectValues(columnIndex.getMaxValues(), null, -345.2f, -234.7f, null, null, 2.99999f, null, 42.83f, null);
  assertCorrectValues(columnIndex.getMinValues(), null, -532.3f, -300.6f, null, null, -234.6f, null, 3.0f, null);
  assertCorrectFiltering(columnIndex, eq(col, 0.0f), 5);
  assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 6, 8);
  assertCorrectFiltering(columnIndex, notEq(col, 2.2f), 0, 1, 2, 3, 4, 5, 6, 7, 8);
  assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 5, 7);
  assertCorrectFiltering(columnIndex, gt(col, 2.2f), 5, 7);
  assertCorrectFiltering(columnIndex, gtEq(col, -234.7f), 2, 5, 7);
  assertCorrectFiltering(columnIndex, lt(col, -234.6f), 1, 2);
  assertCorrectFiltering(columnIndex, ltEq(col, -234.6f), 1, 2, 5);
  assertCorrectFiltering(columnIndex, userDefined(col, FloatIsInteger.class), 1, 2, 5, 7);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, FloatIsInteger.class)), 0, 1, 2, 3, 4, 5, 6, 7, 8);

  builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  sb = new StatsBuilder();
  builder.add(sb.stats(type, null, null, null, null, null));
  builder.add(sb.stats(type, 532.3f, 345.2f));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, 234.7f, 234.6f, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, 234.6f, -2.99999f));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, -3.0f, -42.83f));
  assertEquals(9, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  columnIndex = builder.build();
  assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0);
  assertCorrectNullPages(columnIndex, true, false, true, false, true, false, true, true, false);
  assertCorrectValues(columnIndex.getMaxValues(), null, 532.3f, null, 234.7f, null, 234.6f, null, null, -3.0f);
  assertCorrectValues(columnIndex.getMinValues(), null, 345.2f, null, 234.6f, null, -2.99999f, null, null, -42.83f);
  assertCorrectFiltering(columnIndex, eq(col, 234.65f), 3);
  assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 6, 7);
  assertCorrectFiltering(columnIndex, notEq(col, 2.2f), 0, 1, 2, 3, 4, 5, 6, 7, 8);
  assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 5, 8);
  assertCorrectFiltering(columnIndex, gt(col, 2.2f), 1, 3, 5);
  assertCorrectFiltering(columnIndex, gtEq(col, 2.2f), 1, 3, 5);
  assertCorrectFiltering(columnIndex, lt(col, 0.0f), 5, 8);
  assertCorrectFiltering(columnIndex, ltEq(col, 0.0f), 5, 8);
  assertCorrectFiltering(columnIndex, userDefined(col, FloatIsInteger.class), 1, 5, 8);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, FloatIsInteger.class)), 0, 1, 2, 3, 4, 5, 6, 7, 8);
}