Java Code Examples for org.apache.hadoop.hive.ql.hooks.WriteEntity#getWriteType()

The following examples show how to use org.apache.hadoop.hive.ql.hooks.WriteEntity#getWriteType() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveITBase.java    From atlas with Apache License 2.0 6 votes vote down vote up 
protected static boolean addQueryType(HiveOperation op, WriteEntity entity) {
    if (entity.getWriteType() != null && HiveOperation.QUERY.equals(op)) {
        switch (entity.getWriteType()) {
            case INSERT:
            case INSERT_OVERWRITE:
            case UPDATE:
            case DELETE:
                return true;
            case PATH_WRITE:
                //Add query type only for DFS paths and ignore local paths since they are not added as outputs
                if ( !Entity.Type.LOCAL_DIR.equals(entity.getType())) {
                    return true;
                }
                break;
            default:
        }
    }
    return false;
}
Example 2
Source File: HiveHook.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up 
private boolean isSelectQuery(HiveEventContext event) {
    if (event.getOperation() == HiveOperation.QUERY) {
        //Select query has only one output
        if (event.getOutputs().size() == 1) {
            WriteEntity output = event.getOutputs().iterator().next();
            /* Strangely select queries have DFS_DIR as the type which seems like a bug in hive. Filter out by checking if the path is a temporary URI
             * Insert into/overwrite queries onto local or dfs paths have DFS_DIR or LOCAL_DIR as the type and WriteType.PATH_WRITE and tempUri = false
             * Insert into a temporary table has isTempURI = false. So will not skip as expected
             */
            if (output.getType() == Type.DFS_DIR || output.getType() == Type.LOCAL_DIR) {
                if (output.getWriteType() == WriteEntity.WriteType.PATH_WRITE &&
                    output.isTempURI()) {
                    return true;
                }
            }
        }
    }
    return false;
}
Example 3
Source File: HiveHook.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up 
private static boolean addQueryType(HiveOperation op, WriteEntity entity) {
    if (entity.getWriteType() != null && HiveOperation.QUERY.equals(op)) {
        switch (entity.getWriteType()) {
        case INSERT:
        case INSERT_OVERWRITE:
        case UPDATE:
        case DELETE:
            return true;
        case PATH_WRITE:
            //Add query type only for DFS paths and ignore local paths since they are not added as outputs
            if ( !Type.LOCAL_DIR.equals(entity.getType())) {
                return true;
            }
            break;
        default:
        }
    }
    return false;
}
Example 4
Source File: CreateHiveProcess.java    From atlas with Apache License 2.0 5 votes vote down vote up 
private boolean skipProcess() {
    Set<ReadEntity>  inputs  = getInputs();
    Set<WriteEntity> outputs = getOutputs();

    boolean ret = CollectionUtils.isEmpty(inputs) && CollectionUtils.isEmpty(outputs);

    if (!ret) {
        if (getContext().getHiveOperation() == HiveOperation.QUERY) {
            // Select query has only one output
            if (outputs.size() == 1) {
                WriteEntity output = outputs.iterator().next();

                if (output.getType() == Entity.Type.DFS_DIR || output.getType() == Entity.Type.LOCAL_DIR) {
                    if (output.getWriteType() == WriteEntity.WriteType.PATH_WRITE && output.isTempURI()) {
                        ret = true;
                    }
                }
                // DELETE and UPDATE initially have one input and one output.
                // Since they do not support sub-query, they won't create a lineage that have one input and one output. (One input only)
                // It's safe to filter them out here.
                if (output.getWriteType() == WriteEntity.WriteType.DELETE || output.getWriteType() == WriteEntity.WriteType.UPDATE) {
                    ret = true;
                }
            }
        }
    }

    return ret;
}