org.apache.pig.Expression.OpType Java Examples

private org.apache.iceberg.expressions.Expression convert(OpType op, Column col, Const constant) {
  String name = col.getName();
  Object value = constant.getValue();

  switch (op) {
    case OP_GE: return Expressions.greaterThanOrEqual(name, value);
    case OP_GT: return Expressions.greaterThan(name, value);
    case OP_LE: return Expressions.lessThanOrEqual(name, value);
    case OP_LT: return Expressions.lessThan(name, value);
    case OP_EQ: return Expressions.equal(name, value);
    case OP_NE: return Expressions.notEqual(name, value);
  }

  throw new RuntimeException(
      String.format("[%s]: Failed to pushdown expression: %s %s %s", signature, col, op, constant));
}
 

/**
 * Return cql where clauses for the corresponding partition filter. Make sure the data format matches
 * Only support the following Pig data types: int, long, float, double, boolean and chararray
 * */
private String partitionFilterToWhereClauseString(Expression expression) throws IOException
{
    Expression.BinaryExpression be = (Expression.BinaryExpression) expression;
    OpType op = expression.getOpType();
    String opString = op.toString();
    switch (op)
    {
        case OP_EQ:
            opString = " = ";
        case OP_GE:
        case OP_GT:
        case OP_LE:
        case OP_LT:
            String name = be.getLhs().toString();
            String value = be.getRhs().toString();
            return String.format("%s %s %s", name, opString, value);
        case OP_AND:
            return String.format("%s AND %s", partitionFilterToWhereClauseString(be.getLhs()), partitionFilterToWhereClauseString(be.getRhs()));
        default:
            throw new IOException("Unsupported expression type: " + opString);
    }
}
 

@Override
public List<OpType> getSupportedExpressionTypes() {
    List<OpType> types = new ArrayList<OpType>();
    types.add(OpType.OP_EQ);
    types.add(OpType.OP_NE);
    types.add(OpType.OP_GT);
    types.add(OpType.OP_GE);
    types.add(OpType.OP_LT);
    types.add(OpType.OP_LE);
    types.add(OpType.OP_IN);
    types.add(OpType.OP_BETWEEN);
    types.add(OpType.OP_NULL);
    types.add(OpType.OP_NOT);
    types.add(OpType.OP_AND);
    types.add(OpType.OP_OR);
    return types;
}
 

@VisibleForTesting
SearchArgument getSearchArgument(Expression expr) {
    if (expr == null) {
        return null;
    }
    Builder builder = SearchArgumentFactory.newBuilder();
    boolean beginWithAnd = !(expr.getOpType().equals(OpType.OP_AND) || expr.getOpType().equals(OpType.OP_OR) || expr.getOpType().equals(OpType.OP_NOT));
    if (beginWithAnd) {
        builder.startAnd();
    }
    buildSearchArgument(expr, builder);
    if (beginWithAnd) {
        builder.end();
    }
    SearchArgument sArg = builder.build();
    return sArg;
}
 

@Override
public void visit() throws FrontendException {
    super.visit();
    if (supportedOpTypes.contains(OpType.OP_BETWEEN)) {
        // TODO: Collapse multiple ORs into BETWEEN
    } else if (supportedOpTypes.contains(OpType.OP_IN)) {
        // TODO: Collapse multiple ORs into IN
    }
}
 

@Override
protected boolean isSupportedOpType(BinaryExpression binOp) {
    if(binOp instanceof AddExpression) {
        return supportedOpTypes.contains(OpType.OP_PLUS );
    } else if(binOp instanceof SubtractExpression) {
        return supportedOpTypes.contains(OpType.OP_MINUS);
    } else if(binOp instanceof MultiplyExpression) {
        return supportedOpTypes.contains(OpType.OP_TIMES);
    } else if(binOp instanceof DivideExpression) {
        return supportedOpTypes.contains(OpType.OP_DIV);
    } else if(binOp instanceof ModExpression) {
        return supportedOpTypes.contains(OpType.OP_MOD);
    } else if(binOp instanceof AndExpression) {
        return supportedOpTypes.contains(OpType.OP_AND);
    } else if(binOp instanceof OrExpression) {
        return supportedOpTypes.contains(OpType.OP_OR);
    } else if(binOp instanceof EqualExpression) {
        return supportedOpTypes.contains(OpType.OP_EQ);
    } else if(binOp instanceof NotEqualExpression) {
        return supportedOpTypes.contains(OpType.OP_NE);
    } else if(binOp instanceof GreaterThanExpression) {
        return supportedOpTypes.contains(OpType.OP_GT);
    } else if(binOp instanceof GreaterThanEqualExpression) {
        return supportedOpTypes.contains(OpType.OP_GE);
    } else if(binOp instanceof LessThanExpression) {
        return supportedOpTypes.contains(OpType.OP_LT);
    } else if(binOp instanceof LessThanEqualExpression) {
        return supportedOpTypes.contains(OpType.OP_LE);
    } else if(binOp instanceof RegexExpression) {
        return supportedOpTypes.contains(OpType.OP_MATCH);
    } else {
        return false;
    }
}
 

@Override
protected boolean isSupportedOpType(UnaryExpression unaryOp) {
    if (unaryOp instanceof CastExpression) {
        return true;
    } else if(unaryOp instanceof IsNullExpression) {
        return supportedOpTypes.contains(OpType.OP_NULL);
    } else if(unaryOp instanceof NotExpression) {
        return supportedOpTypes.contains(OpType.OP_NOT);
    } else {
        return false;
    }
}
 

@BeforeClass
public static void oneTimeSetup() throws Exception{
    cluster = MiniGenericCluster.buildCluster();
    Util.copyFromLocalToCluster(cluster, basedir + "orc-file-11-format.orc", basedir + "orc-file-11-format.orc");
    Util.copyFromLocalToCluster(cluster, basedir + "charvarchar.orc", basedir + "charvarchar.orc");
    createInputData();

    if(Util.WINDOWS){
        OUTPUT1 = OUTPUT1.replace("\\", "/");
        OUTPUT2 = OUTPUT2.replace("\\", "/");
        OUTPUT3 = OUTPUT3.replace("\\", "/");
        OUTPUT4 = OUTPUT4.replace("\\", "/");
    }

    supportedOpTypes = new ArrayList<OpType>();
    supportedOpTypes.add(OpType.OP_EQ);
    supportedOpTypes.add(OpType.OP_NE);
    supportedOpTypes.add(OpType.OP_GT);
    supportedOpTypes.add(OpType.OP_GE);
    supportedOpTypes.add(OpType.OP_LT);
    supportedOpTypes.add(OpType.OP_LE);
    supportedOpTypes.add(OpType.OP_IN);
    supportedOpTypes.add(OpType.OP_BETWEEN);
    supportedOpTypes.add(OpType.OP_NULL);
    supportedOpTypes.add(OpType.OP_NOT);
    supportedOpTypes.add(OpType.OP_AND);
    supportedOpTypes.add(OpType.OP_OR);

    Logger logger = Logger.getLogger(ColumnPruneVisitor.class);
    logger.removeAllAppenders();
    logger.setLevel(Level.INFO);
    SimpleLayout layout = new SimpleLayout();
    logFile = File.createTempFile("log", "");
    FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
    logger.addAppender(appender);
}
 

@Override
public ImmutableList<OpType> getSupportedExpressionTypes() {
  LOG.info("[{}]: getSupportedExpressionTypes()", signature);
  return ImmutableList.of(OpType.OP_AND, OpType.OP_OR, OpType.OP_EQ, OpType.OP_NE, OpType.OP_NOT, OpType.OP_GE,
      OpType.OP_GT, OpType.OP_LE, OpType.OP_LT, OpType.OP_BETWEEN, OpType.OP_IN, OpType.OP_NULL);
}
 

private org.apache.iceberg.expressions.Expression convert(Expression expression) throws IOException {
  OpType op = expression.getOpType();

  if (expression instanceof BinaryExpression) {
    Expression lhs = ((BinaryExpression) expression).getLhs();
    Expression rhs = ((BinaryExpression) expression).getRhs();

    switch (op) {
      case OP_AND:
        return Expressions.and(convert(lhs), convert(rhs));
      case OP_OR:
        return Expressions.or(convert(lhs), convert(rhs));
      case OP_BETWEEN:
        BetweenExpression between = (BetweenExpression) rhs;
        return Expressions.and(
            convert(OpType.OP_GE, (Column) lhs, (Const) between.getLower()),
            convert(OpType.OP_LE, (Column) lhs, (Const) between.getUpper())
        );
      case OP_IN:
        return ((InExpression) rhs).getValues().stream()
            .map(value -> convert(OpType.OP_EQ, (Column) lhs, (Const) value))
            .reduce(Expressions.alwaysFalse(), Expressions::or);
      default:
        if (lhs instanceof Column && rhs instanceof Const) {
          return convert(op, (Column) lhs, (Const) rhs);
        } else if (lhs instanceof Const && rhs instanceof Column) {
          throw new FrontendException("Invalid expression ordering " + expression);
        }
    }

  } else if (expression instanceof UnaryExpression) {
    Expression unary = ((UnaryExpression) expression).getExpression();

    switch (op) {
      case OP_NOT:  return Expressions.not(convert(unary));
      case OP_NULL: return Expressions.isNull(((Column) unary).getName());
      default: throw new FrontendException("Unsupported unary operator" + op);
    }
  }

  throw new FrontendException("Failed to pushdown expression " + expression);
}
 

public Expression getExpression(LogicalExpression op) throws FrontendException
{
    if(op == null) {
        return null;
    }
    if(op instanceof ConstantExpression) {
        ConstantExpression constExpr =(ConstantExpression)op ;
        return new Expression.Const( constExpr.getValue() );
    } else if (op instanceof ProjectExpression) {
        ProjectExpression projExpr = (ProjectExpression)op;
        String fieldName = projExpr.getFieldSchema().alias;
        return new Expression.Column(fieldName);
    } else if(op instanceof BinaryExpression) {
        BinaryExpression binOp = (BinaryExpression)op;
        if(binOp instanceof AddExpression) {
            return getExpression( binOp, OpType.OP_PLUS );
        } else if(binOp instanceof SubtractExpression) {
            return getExpression(binOp, OpType.OP_MINUS);
        } else if(binOp instanceof MultiplyExpression) {
            return getExpression(binOp, OpType.OP_TIMES);
        } else if(binOp instanceof DivideExpression) {
            return getExpression(binOp, OpType.OP_DIV);
        } else if(binOp instanceof ModExpression) {
            return getExpression(binOp, OpType.OP_MOD);
        } else if(binOp instanceof AndExpression) {
            return getExpression(binOp, OpType.OP_AND);
        } else if(binOp instanceof OrExpression) {
            return getExpression(binOp, OpType.OP_OR);
        } else if(binOp instanceof EqualExpression) {
            return getExpression(binOp, OpType.OP_EQ);
        } else if(binOp instanceof NotEqualExpression) {
            return getExpression(binOp, OpType.OP_NE);
        } else if(binOp instanceof GreaterThanExpression) {
            return getExpression(binOp, OpType.OP_GT);
        } else if(binOp instanceof GreaterThanEqualExpression) {
            return getExpression(binOp, OpType.OP_GE);
        } else if(binOp instanceof LessThanExpression) {
            return getExpression(binOp, OpType.OP_LT);
        } else if(binOp instanceof LessThanEqualExpression) {
            return getExpression(binOp, OpType.OP_LE);
        } else if(binOp instanceof RegexExpression) {
            return getExpression(binOp, OpType.OP_MATCH);
        } else {
            LOG.error("Unsupported conversion of BinaryExpression to Expression: " + op.getName());
            throw new FrontendException("Unsupported conversion of BinaryExpression to Expression: " + op.getName());
        }
    } else if(op instanceof UnaryExpression) {
        UnaryExpression unaryOp = (UnaryExpression)op;
        if(unaryOp instanceof IsNullExpression) {
            return getExpression(unaryOp, OpType.OP_NULL);
        } else if(unaryOp instanceof NotExpression) {
            return getExpression(unaryOp, OpType.OP_NOT);
        } else if(unaryOp instanceof CastExpression) {
            return getExpression(unaryOp.getExpression());
        } else {
            LOG.error("Unsupported conversion of UnaryExpression to Expression: " + op.getName());
            throw new FrontendException("Unsupported conversion of UnaryExpression to Expression: " + op.getName());
        }
    } else {
        LOG.error("Unsupported conversion of LogicalExpression to Expression: " + op.getName());
        throw new FrontendException("Unsupported conversion of LogicalExpression to Expression: " + op.getName());
    }
}
 

protected Expression getExpression(BinaryExpression binOp, OpType
        opType) throws FrontendException {
    return new Expression.BinaryExpression(getExpression(binOp.getLhs())
            , getExpression(binOp.getRhs()), opType);
}
 

protected Expression getExpression(UnaryExpression unaryOp, OpType
        opType) throws FrontendException {
    return new Expression.UnaryExpression(getExpression(unaryOp.getExpression()), opType);
}
 

public PredicatePushDownFilterExtractor(LogicalExpressionPlan plan, List<String> predicateCols,
        List<OpType> supportedOpTypes) {
    super(plan);
    this.predicateCols = predicateCols;
    this.supportedOpTypes = supportedOpTypes;
}