org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc Java Examples

public DecomposedPredicate pushPredicate(Map<String, String> hiveTypeMapping, ExprNodeDesc
    predicate) {
  log.info("Checking predicates for pushdown in DynamoDB query");
  List<IndexSearchCondition> searchConditions = getGenericSearchConditions(hiveTypeMapping,
      predicate);
  log.info("Pushed predicates: " + searchConditions);
  if (searchConditions.isEmpty()) {
    return null;
  } else {
    List<IndexSearchCondition> finalSearchCondition =
        prioritizeSearchConditions(searchConditions);
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate =
        analyzer.translateSearchConditions(finalSearchCondition);
    decomposedPredicate.residualPredicate = (ExprNodeGenericFuncDesc) predicate;
    return decomposedPredicate;
  }
}
 

@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 

@Test
public void testExpressionConversionWithNestedStringAndTimestampColumns() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val(testDate).timestampCol("timestamp").pred("=", 2)
        .val("test").strCol("strCol").pred("=", 2)
        .pred("and", 2)
        .val(testDate).timestampCol("timestamp").pred("=", 2)
        .val("test").strCol("strCol").pred("=", 2)
        .pred("and", 2)
        .pred("and", 2).build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(((strCol = 'test') and (timestamp = '1970-01-01 00:02:03.456')) " +
              "and " +
              "((strCol = 'test') and (timestamp = '1970-01-01 00:02:03.456')))",
        filter);
}
 

@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 

@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 

@Test
public void testListPartitionsByExpression() throws Exception {
  String dbName = testDB.getName();
  Table table = getTestTable();
  ExprBuilder e = new ExprBuilder(table.getName()).val("test").strCol("location").pred("=", 2);
  ExprNodeGenericFuncDesc exprTree = e.build();
  byte[] payload = hiveShims.getSerializeExpression(exprTree);
  List<org.apache.hadoop.hive.metastore.api.Partition> results = new ArrayList<>();

  Partition newPartition = new Partition()
    .withDatabaseName(dbName).withTableName(table.getName())
    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(null);
  when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
          .thenReturn(new GetPartitionsResult().withPartitions(newPartition));
  boolean status = metastoreClient.listPartitionsByExpr(dbName,table.getName(),payload,null,(short)5,results);
  assertFalse(status);
}
 

@Test
public void testExpressionConversionWithTwoNotInAndOneIn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .vals(Arrays.asList("data1", "data2")).strCol("notInCol1").pred("in", 3)
        .pred("not", 1)
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("notInCol2").pred("in", 4)
        .pred("not", 1)
        .pred("and", 2)
        .vals(Arrays.asList("abc", "def", "ghi", "jkl")).strCol("inCol1").pred("in", 5)
        .pred("and", 2)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("((inCol1) IN ('jkl', 'ghi', 'def', 'abc') and (((notInCol2) NOT IN ('val3', 'val2', 'val1')) and " +
        "((notInCol1) NOT IN ('data2', 'data1'))))", filter);
}
 

private List<ObjectPair<Integer, byte[]>> getDropPartitionExpressions(List<Partition> partitions) throws Exception {
  List<ObjectPair<Integer, byte[]>> partExprs = Lists.newArrayList();

  for (int i = 0; i < partitions.size(); i++) {
    Partition partition = partitions.get(i);
    // the catalogTable has only one partition key
    String partitionKey = catalogTable.getPartitionKeys().get(0).getName();
    String partitionValue = partition.getValues().get(0);
    ExprNodeGenericFuncDesc expr = new ExprBuilder(catalogTable.getName())
        .val(partitionValue).strCol(partitionKey).pred("=", 2).build();
    byte[] payload = hiveShims.getSerializeExpression(expr);
    partExprs.add(new ObjectPair<>(i, payload));
  }

  return partExprs;
}
 

private static ExprNodeGenericFuncDesc deserializeExpr(byte[] exprBytes) throws MetaException {
  ExprNodeGenericFuncDesc expr = null;
  try {
    expr = ShimsLoader.getHiveShims().getDeserializeExpression(exprBytes);
  } catch (Exception ex) {
    logger.error("Failed to deserialize the expression", ex);
    throw new MetaException(ex.getMessage());
  }
  if (expr == null) {
    throw new MetaException("Failed to deserialize expression - ExprNodeDesc not present");
  }
  return expr;
}
 

public IExpressionNode createExpressionNode( final List<ExprNodeGenericFuncDesc> filterExprs ){
  HiveExprOrNode hiveOrNode = new HiveExprOrNode();
  for( ExprNodeGenericFuncDesc filterExpr : filterExprs ){
    if( filterExpr != null ){
      hiveOrNode.addChildNode( filterExpr );
    }
  }

  return hiveOrNode.getPushDownFilterNode();
}
 

public static IExtractNode getExtractNodeFromGenericFunc( final ExprNodeGenericFuncDesc target ){
  GenericUDF udf = target.getGenericUDF();
  if( ! ( udf instanceof GenericUDFIndex ) ){
    return null;
  }
  return getExtractNodeFromGenericIndex( target , (GenericUDFIndex)udf );
}
 

public static IExtractNode getExtractNode(final ExprNodeDesc target ){
  if( target instanceof ExprNodeGenericFuncDesc ){
    return getExtractNodeFromGenericFunc( (ExprNodeGenericFuncDesc)target );
  }
  else if( target instanceof ExprNodeFieldDesc ){
    return getExtractNodeFromField( (ExprNodeFieldDesc)target  );
  }
  else if( target instanceof ExprNodeColumnDesc ){
    if( ( (ExprNodeColumnDesc)target ).getIsPartitionColOrVirtualCol() ){
      return null;
    }
    return getExtractNodeFromColumn( (ExprNodeColumnDesc)target  );
  }
  return null;
}
 

public HiveExprNotNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 

public HiveExprAndNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 

public HiveExprOrNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 

@Test
public void testExpressionConversionWithTwoNotInsSameColumn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .vals(Arrays.asList("data1", "data2")).strCol("strCol").pred("in", 3)
        .pred("not", 1)
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .pred("not", 1)
        .pred("or", 2)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(((strCol) NOT IN ('val3', 'val2', 'val1')) or ((strCol) NOT IN ('data2', 'data1')))", filter);
}
 

@Test
public void testExpressionConversionWithIn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(strCol) IN ('val3', 'val2', 'val1')", filter);
}
 

@Test
public void testExpressionConversionWithNotIn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .pred("not", 1)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("((strCol) NOT IN ('val3', 'val2', 'val1'))", filter);
}
 

@Test
public void testExpressionConversionWithDateCharAndBooleanColumn() throws Exception {
  ExprNodeGenericFuncDesc expr = new ExprBuilder("fooTable")
          // see https://goo.gl/eGEUh2, Hive uses the same method to convert query string value to Date object
          .val(java.sql.Date.valueOf("2017-11-14")).dateCol("dateCol").pred("=", 2)
          .val('c').charCol("charCol").pred("=", 2).pred("and", 2)
          .val(true).booleanCol("booleanCol").pred("=", 2).pred("and", 2).build();
  byte[] payload = hiveShims.getSerializeExpression(expr);

  String catalogExpression = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("((booleanCol = true) and ((charCol = 'c') and (dateCol = '2017-11-14')))",
          catalogExpression);
}
 

@Test
public void testExpressionConversionWithStringAndTimestampColumn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val(testDate).timestampCol("timestamp").pred("=", 2)
        .val("test").strCol("strCol").pred("=", 2)
        .pred("and", 2).build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("((strCol = 'test') and (timestamp = '1970-01-01 00:02:03.456'))", filter);
}
 

@Test
public void testExpressionConversionByTimestamp() throws Exception {
  Table table = getTestTable();
  ExprBuilder e = new ExprBuilder(table.getName()).val(testDate).timestampCol("timestamp").pred("=", 2);
  ExprNodeGenericFuncDesc exprTree = e.build();
  byte[] payload = hiveShims.getSerializeExpression(exprTree);
  String expression = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(timestamp = '1970-01-01 00:02:03.456')", expression);
}
 

private ExprBuilder fn(String name, TypeInfo ti, int args) throws Exception {
  List<ExprNodeDesc> children = new ArrayList<>();
  for (int i = 0; i < args; ++i) {
    children.add(stack.pop());
  }
  stack.push(new ExprNodeGenericFuncDesc(ti, FunctionRegistry.getFunctionInfo(name).getGenericUDF(), children));
  return this;
}
 

@Test
public void testExpressionConversionWithBetween() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val("val100").val("val1").strCol("strCol").val(false).pred("between", 4)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("strCol BETWEEN 'val1' AND 'val100'", filter);
}
 

@Test
public void testExpressionConversionWithNotBetween() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val("val100" ).val("val1").strCol("strCol").val(true).pred("between", 4)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("strCol NOT BETWEEN 'val1' AND 'val100'", filter);
}
 

private static void addColumnNamesOfNotInExpressionToSet(ExprNodeDesc childNode, ExprNodeDesc parentNode, Set<String> columnsInNotInExpression) {
  if (parentNode != null && childNode != null && parentNode instanceof ExprNodeGenericFuncDesc && childNode instanceof ExprNodeGenericFuncDesc) {
    ExprNodeGenericFuncDesc parentFuncNode = (ExprNodeGenericFuncDesc) parentNode;
    ExprNodeGenericFuncDesc childFuncNode = (ExprNodeGenericFuncDesc) childNode;
    if(parentFuncNode.getGenericUDF() instanceof GenericUDFOPNot && childFuncNode.getGenericUDF() instanceof GenericUDFIn) {
      // The current parent child pair represents a "NOT IN" expression. Add name of the column to the set.
      columnsInNotInExpression.addAll(childFuncNode.getCols());
    }
  }
}
 

public static String convertHiveExpressionToCatalogExpression(byte[] exprBytes) throws MetaException {
  ExprNodeGenericFuncDesc exprTree = deserializeExpr(exprBytes);
  Set<String> columnNamesInNotInExpression = Sets.newHashSet();
  fieldEscaper(exprTree.getChildren(), exprTree, columnNamesInNotInExpression);
  String expression = rewriteExpressionForNotIn(exprTree.getExprString(), columnNamesInNotInExpression);
  return expression;
}
 

@Test
public void testExpressionConversionWithMultipleNots() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val("val100" ).val("val1").strCol("strCol").val(true).pred("between", 4)
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .pred("not", 1)
        .pred("and", 2)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(((strCol) NOT IN ('val3', 'val2', 'val1')) and strCol NOT BETWEEN 'val1' AND 'val100')", filter);
}
 

@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  throw new UnsupportedOperationException( "IHiveExprNode node can not have child node." );
}
 

@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  throw new UnsupportedOperationException( "IHiveExprNode node can not have child node." );
}
 

public HiveReaderSetting( final FileSplit split, final JobConf job ){
  config = new Configuration();

  disableSkipBlock = job.getBoolean( "mds.disable.block.skip" , false );
  disableFilterPushdown = job.getBoolean( "mds.disable.filter.pushdown" , false );

  Set<String> pathNameSet= createPathSet( split.getPath() );
  List<ExprNodeGenericFuncDesc> filterExprs = new ArrayList<ExprNodeGenericFuncDesc>();
  String filterExprSerialized = job.get( TableScanDesc.FILTER_EXPR_CONF_STR );
  if( filterExprSerialized != null ){
    filterExprs.add( Utilities.deserializeExpression(filterExprSerialized) );
  }

  MapWork mapWork;
  try{
    mapWork = Utilities.getMapWork(job);
  }catch( Exception e ){
    mapWork = null;
  }

  if( mapWork == null ){
    node = createExpressionNode( filterExprs );
    isVectorModeFlag = false;
    return;
  }

  node = createExpressionNode( filterExprs );

  for( Map.Entry<String,PartitionDesc> pathsAndParts: mapWork.getPathToPartitionInfo().entrySet() ){
    if( ! pathNameSet.contains( pathsAndParts.getKey() ) ){
      continue;
    }
    Properties props = pathsAndParts.getValue().getTableDesc().getProperties();
    if( props.containsKey( "mds.expand" ) ){
      config.set( "spread.reader.expand.column" , props.getProperty( "mds.expand" ) );
    }
    if( props.containsKey( "mds.flatten" ) ){
      config.set( "spread.reader.flatten.column" , props.getProperty( "mds.flatten" ) );
    }
  }

  config.set( "spread.reader.read.column.names" , createReadColumnNames( job.get( ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR , null ) ) );

  // Next Hive vesion;
  // Utilities.getUseVectorizedInputFileFormat(job)
  isVectorModeFlag = Utilities.isVectorMode( job );
}