Java Code Examples for org.apache.kylin.common.util.Pair#getSecond()

public List<Pair<TableDesc, TableExtDesc>> extractHiveTableMeta(String[] tables, String project) throws Exception { // de-dup
    SetMultimap<String, String> db2tables = LinkedHashMultimap.create();
    for (String fullTableName : tables) {
        String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
        db2tables.put(parts[0], parts[1]);
    }

    // load all tables first
    List<Pair<TableDesc, TableExtDesc>> allMeta = Lists.newArrayList();
    ProjectInstance projectInstance = getProjectManager().getProject(project);
    ISourceMetadataExplorer explr = SourceManager.getSource(projectInstance).getSourceMetadataExplorer();
    for (Map.Entry<String, String> entry : db2tables.entries()) {
        Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
        TableDesc tableDesc = pair.getFirst();
        Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase(Locale.ROOT)));
        Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase(Locale.ROOT)));
        Preconditions.checkState(tableDesc.getIdentity()
                .equals(entry.getKey().toUpperCase(Locale.ROOT) + "." + entry.getValue().toUpperCase(Locale.ROOT)));
        TableExtDesc extDesc = pair.getSecond();
        Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity()));
        allMeta.add(pair);
    }
    return allMeta;
}
 

/**
 * Re-encode with measures in Object[] format.
 * @param key
 * @param value
 * @return
 * @throws IOException
 */
public Pair<Text, Object[]> reEncode2(Text key, Text value) throws IOException {
    if (initialized == false) {
        throw new IllegalStateException("Not initialized");
    }

    Object[] measureObjs = new Object[measureDescs.size()];
    codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs);
    // re-encode measures if dictionary is used
    if (dictMeasures.size() > 0) {
        for (Pair<Integer, MeasureIngester> pair : dictMeasures) {
            int i = pair.getFirst();
            MeasureIngester ingester = pair.getSecond();
            measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts);
        }

    }
    return Pair.newPair(processKey(key), measureObjs);
}
 

private SqlNode convertSqlCall(SqlCall sqlCall) {
    SqlOperator operator = sqlCall.getOperator();
    if (operator != null) {
        Pair<SqlNode, SqlNode> matched = convMaster.matchSqlFunc(sqlCall);

        if (matched != null) {
            Preconditions.checkState(matched.getFirst() instanceof SqlCall);
            SqlCall sourceTmpl = (SqlCall) matched.getFirst();

            Preconditions.checkState(sourceTmpl.operandCount() == sqlCall.operandCount());
            SqlNode targetTmpl = matched.getSecond();

            boolean isWindowCall = sourceTmpl.getOperator() instanceof SqlOverOperator;
            SqlParamsFinder sqlParamsFinder = SqlParamsFinder.newInstance(sourceTmpl, sqlCall, isWindowCall);
            return targetTmpl.accept(new SqlFuncFiller(sqlParamsFinder.getParamNodes(), isWindowCall));
        }
    }
    return null;
}
 

/**
 * This method overrides the parent putBinary() method. It will split the large value bytes into multiple chunks to fit into the internal Cache.
 * It generates a KeyHook to store the splitted chunked keys.
 */
@Override
public void putBinary(String keyS, byte[] valueB, int expiration) {
    if (Strings.isNullOrEmpty(keyS)) {
        return;
    }
    int nSplit = getValueSplit(config, keyS, valueB.length);
    Pair<KeyHook, byte[][]> keyValuePair = getKeyValuePair(nSplit, keyS, valueB);
    KeyHook keyHook = keyValuePair.getFirst();
    byte[][] splitValueB = keyValuePair.getSecond();

    if (logger.isDebugEnabled()) {
        logger.debug("put key hook:{} to cache for hash key", keyHook);
    }
    super.putBinary(keyS, serializeValue(keyHook), expiration);
    if (nSplit > 1) {
        for (int i = 0; i < nSplit; i++) {
            if (logger.isDebugEnabled()) {
                logger.debug("Chunk[" + i + "] bytes size before encoding  = " + splitValueB[i].length);
            }
            super.putBinary(keyHook.getChunkskey()[i], splitValueB[i], expiration);
        }
    }
}
 

public List<Pair<TableDesc, TableExtDesc>> extractHiveTableMeta(String[] tables, String project) throws Exception { // de-dup
    SetMultimap<String, String> db2tables = LinkedHashMultimap.create();
    for (String fullTableName : tables) {
        String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
        db2tables.put(parts[0], parts[1]);
    }

    // load all tables first
    List<Pair<TableDesc, TableExtDesc>> allMeta = Lists.newArrayList();
    ProjectInstance projectInstance = getProjectManager().getProject(project);
    ISourceMetadataExplorer explr = SourceManager.getSource(projectInstance).getSourceMetadataExplorer();
    for (Map.Entry<String, String> entry : db2tables.entries()) {
        Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project);
        TableDesc tableDesc = pair.getFirst();
        Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase(Locale.ROOT)));
        Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase(Locale.ROOT)));
        Preconditions.checkState(tableDesc.getIdentity()
                .equals(entry.getKey().toUpperCase(Locale.ROOT) + "." + entry.getValue().toUpperCase(Locale.ROOT)));
        TableExtDesc extDesc = pair.getSecond();
        Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity()));
        allMeta.add(pair);
    }
    return allMeta;
}
 

public static Map<String, Integer> getValidEncodings() {
    if (factoryMap == null)
        initFactoryMap();

    Map<String, Integer> result = Maps.newTreeMap();
    for (Pair<String, Integer> p : factoryMap.keySet()) {
        if (result.containsKey(p.getFirst())) {
            if (result.get(p.getFirst()) > p.getSecond()) {
                continue;//skip small versions
            }
        }

        result.put(p.getFirst(), p.getSecond());
    }
    result.put(DictionaryDimEnc.ENCODING_NAME, 1);
    return result;
}
 

public static void applyHBaseColums(Scan scan, List<Pair<byte[], byte[]>> hbaseColumns) {
    for (Pair<byte[], byte[]> hbaseColumn : hbaseColumns) {
        byte[] byteFamily = hbaseColumn.getFirst();
        byte[] byteQualifier = hbaseColumn.getSecond();
        scan.addColumn(byteFamily, byteQualifier);
    }
}
 

public static String replaceAliasInExpr(String expr, Map<String, String> renaming) {
    String prefix = "select ";
    String suffix = " from t";
    String sql = prefix + expr + suffix;
    SqlNode sqlNode = CalciteParser.getOnlySelectNode(sql);

    final Set<SqlIdentifier> s = Sets.newHashSet();
    SqlVisitor sqlVisitor = new SqlBasicVisitor() {
        @Override
        public Object visit(SqlIdentifier id) {
            Preconditions.checkState(id.names.size() == 2);
            s.add(id);
            return null;
        }
    };

    sqlNode.accept(sqlVisitor);
    List<SqlIdentifier> sqlIdentifiers = Lists.newArrayList(s);

    CalciteParser.descSortByPosition(sqlIdentifiers);

    for (SqlIdentifier sqlIdentifier : sqlIdentifiers) {
        Pair<Integer, Integer> replacePos = CalciteParser.getReplacePos(sqlIdentifier, sql);
        int start = replacePos.getFirst();
        int end = replacePos.getSecond();
        String aliasInExpr = sqlIdentifier.names.get(0);
        String col = sqlIdentifier.names.get(1);
        String renamedAlias = renaming.get(aliasInExpr);
        Preconditions.checkNotNull(renamedAlias,
                "rename for alias " + aliasInExpr + " in expr (" + expr + ") is not found");
        sql = sql.substring(0, start) + renamedAlias + "." + col + sql.substring(end);
    }

    return sql.substring(prefix.length(), sql.length() - suffix.length());
}
 

public HiveProducerRecord parseToHiveProducerRecord(String tableName, Map<String, String> partitionKVs,
                                                    Map<String, Object> rawValue) throws Exception {
    Pair<String, String> tableNameSplits = ActiveReservoirReporter.getTableNameSplits(tableName);
    List<FieldSchema> fields = tableFieldSchemaCache.get(tableNameSplits).getSecond();
    List<Object> columnValues = Lists.newArrayListWithExpectedSize(fields.size());
    for (FieldSchema fieldSchema : fields) {
        columnValues.add(rawValue.get(fieldSchema.getName().toUpperCase(Locale.ROOT)));
    }

    return new HiveProducerRecord(tableNameSplits.getFirst(), tableNameSplits.getSecond(), partitionKVs,
            columnValues);
}
 

@Override
public TableSignature getSignature() throws IOException {
    Pair<Long, Long> sizeAndLastModified;
    try {
        sizeAndLastModified = getSizeAndLastModified(path);
    } catch (FileNotFoundException ex) {
        sizeAndLastModified = Pair.newPair(-1L, 0L);
    }
    return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond());
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    try {
        logger.info("executing:" + getCmd());
        final PatternedLogger patternedLogger = new PatternedLogger(logger);
        final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(), patternedLogger, null);
        getManager().addJobInfo(getId(), patternedLogger.getInfo());
        return result.getFirst() == 0 ? new ExecuteResult(ExecuteResult.State.SUCCEED, result.getSecond())
                : ExecuteResult.createFailed(new ShellException(result.getSecond()));
    } catch (IOException e) {
        logger.error("job:" + getId() + " execute finished with exception", e);
        return ExecuteResult.createError(e);
    }
}
 

@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
    try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
        String tableName = segment.getStorageLocationIdentifier();

        HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
        Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
        long totalSize = 0;
        for (Long size : sizeMap.values()) {
            totalSize += size;
        }
        if (totalSize == 0) {
            return;
        }
        Map<byte[], Pair<Integer, Integer>> countMap = cal.getRegionHFileCountMap();
        // check if there's region contains more than one hfile, which means the hfile config take effects
        boolean hasMultiHFileRegions = false;
        for (Pair<Integer, Integer> count : countMap.values()) {
            // check if hfile count is greater than store count
            if (count.getSecond() > count.getFirst()) {
                hasMultiHFileRegions = true;
                break;
            }
        }
        if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() == 0 && hasMultiHFileRegions) {
            throw new IOException("hfile size set to 0, but found region contains more than one hfiles");
        } else if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() > 0 && !hasMultiHFileRegions) {
            throw new IOException("hfile size set greater than 0, but all regions still has only one hfile");
        }
    }
}
 

private static DimensionEncoding[] getDimensionEncodings(FunctionDesc function, List<TblColRef> literalCols,
        Map<TblColRef, Dictionary<String>> dictionaryMap) {
    final DimensionEncoding[] dimensionEncodings = new DimensionEncoding[literalCols.size()];
    for (int i = 0; i < literalCols.size(); i++) {
        TblColRef colRef = literalCols.get(i);

        Pair<String, String> topNEncoding = TopNMeasureType.getEncoding(function, colRef);
        String encoding = topNEncoding.getFirst();
        String encodingVersionStr = topNEncoding.getSecond();
        if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
            dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef));
        } else {
            int encodingVersion = 1;
            if (!StringUtils.isEmpty(encodingVersionStr)) {
                try {
                    encodingVersion = Integer.parseInt(encodingVersionStr);
                } catch (NumberFormatException e) {
                    throw new RuntimeException(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName()
                            + " has to be an integer");
                }
            }
            Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
            String encodingName = (String) encodingConf[0];
            String[] encodingArgs = (String[]) encodingConf[1];

            encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName,
                    literalCols.get(i).getType());

            dimensionEncodings[i] = DimensionEncodingFactory.create(encodingName, encodingArgs, encodingVersion);
        }
    }

    return dimensionEncodings;
}
 

private CheckResult checkForEqValue(Pair<Long, Long> timeUnitRange) {
    if (timeUnitRange.getFirst() <= timeStart && timeUnitRange.getSecond() >= timeEnd) {
        return CheckResult.INCLUDED;
    }
    if (timeUnitRange.getSecond() <= timeStart || timeUnitRange.getFirst() >= timeEnd) {
        return CheckResult.EXCLUDED;
    }
    return CheckResult.OVERLAP;
}
 

private static double calculateRollupRatio(Pair<Long, Long> rollupStats) {
    double rollupInputCount = (double) rollupStats.getFirst() + rollupStats.getSecond();
    return rollupInputCount == 0 ? 0 : 1.0 * rollupStats.getFirst() / rollupInputCount;
}
 

@Override
public int compare(Pair<AbstractExecutable, Integer> o1, Pair<AbstractExecutable, Integer> o2) {
    return o1.getSecond() > o2.getSecond() ? -1 : 1;
}
 

public CheckResult check(CompareTupleFilter compFilter, TimeDerivedColumnType timeDerivedColumnType, long timezoneOffset) {
    Object timestampValue = compFilter.getFirstValue();
    Set conditionValues = compFilter.getValues();
    Pair<Long, Long> timeUnitRange;
    if (timeDerivedColumnType != TimeDerivedColumnType.MINUTE_START
            && timeDerivedColumnType != TimeDerivedColumnType.HOUR_START) {
        timeUnitRange = timezoneOffset == 0 ? timeDerivedColumnType.getTimeUnitRange(timestampValue)
                : timeDerivedColumnType.getTimeUnitRangeTimezoneAware(timestampValue, timezoneOffset);
    } else {
        timeUnitRange = timeDerivedColumnType.getTimeUnitRange(timestampValue);
    }
    switch (compFilter.getOperator()) {
    case EQ:
        return checkForEqValue(timeUnitRange);
    case NEQ:
        if (timeUnitRange.getFirst() <= timeStart && timeUnitRange.getSecond() >= timeEnd) {
            return CheckResult.EXCLUDED;
        }
        if (timeUnitRange.getSecond() <= timeStart || timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.INCLUDED;
        }
        return CheckResult.OVERLAP;
    case LT:
        if ((!endClose && timeUnitRange.getFirst() >= timeEnd) || (endClose && timeUnitRange.getFirst() > timeEnd)) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getFirst() <= timeStart) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case LTE:
        if (timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getSecond() < timeStart) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case GT:
        if (timeUnitRange.getSecond() < timeStart) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case GTE:
        if (timeUnitRange.getFirst() <= timeStart) {
            return CheckResult.INCLUDED;
        }
        if ((!endClose && timeUnitRange.getFirst() >= timeEnd) || (endClose && timeUnitRange.getFirst() > timeEnd)) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case IN:
        return checkForInValues(timeDerivedColumnType, conditionValues, timezoneOffset);
    default:
        return CheckResult.OVERLAP;
    }
}
 

@Override
public Tuple2<ByteArray, Object[]> map(Tuple2<Text, Text> textTextTuple2) throws Exception {
    Pair<Text, Object[]> encodedPair = segmentReEncoder.reEncode2(textTextTuple2.f0, textTextTuple2.f1);
    return new Tuple2(new ByteArray(encodedPair.getFirst().getBytes()), encodedPair.getSecond());
}
 

@Override
public Tuple2<ByteArray, Object[]> map(Tuple2<Text, Text> textTextTuple2) throws Exception {
    Pair<Text, Object[]> encodedPair = segmentReEncoder.reEncode2(textTextTuple2.f0, textTextTuple2.f1);
    return new Tuple2(new ByteArray(encodedPair.getFirst().getBytes()), encodedPair.getSecond());
}
 

public static PushdownResponse submitPushDownTask(String sql) {
    SparkSession ss = SparderContext.getSparkSession();
    Pair<List<List<String>>, List<StructField>> pair = SparkSqlClient.executeSql(ss, sql, UUID.randomUUID());
    return new PushdownResponse(pair.getSecond(), pair.getFirst());
}