org.apache.kylin.common.util.Array Java Examples

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
 

public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols,
        Collection<TblColRef> wantedCols) {
    Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>();
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) {
        Array<TblColRef> hostCols = entry.getKey();
        boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data));
        if (!hostOnRow)
            continue;

        List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>();
        for (DeriveInfo info : entry.getValue()) {
            if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has any wanted columns?
                wantedInfo.add(info);
        }

        if (wantedInfo.size() > 0)
            result.put(hostCols, wantedInfo);
    }
    return result;
}
 

public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols,
        Collection<TblColRef> wantedCols) {
    Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>();
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) {
        Array<TblColRef> hostCols = entry.getKey();
        boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data));
        if (!hostOnRow)
            continue;

        List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>();
        for (DeriveInfo info : entry.getValue()) {
            if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has any wanted columns?
                wantedInfo.add(info);
        }

        if (wantedInfo.size() > 0)
            result.put(hostCols, wantedInfo);
    }
    return result;
}
 

private boolean matchAllCompositeKeys(TreeMap<String, String> lookupCol2FactTableCol, LinkedList<String> columnValues) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();

    for (String lookupTable : lookupTableKeys.keySet()) {
        if (lookupTableKeys.get(lookupTable).size() == 1)
            continue;

        String[] comboKey = new String[lookupTableKeys.get(lookupTable).size()];
        int index = 0;
        for (String column : lookupTableKeys.get(lookupTable)) {
            String key = lookupTable + "/" + column;
            String factTableCol = lookupCol2FactTableCol.get(key);
            int cardinal = MetadataManager.getInstance(config).getTableDesc(factTableName).findColumnByName(factTableCol).getZeroBasedIndex();
            comboKey[index] = columnValues.get(cardinal);

            index++;
        }
        Array<String> wrap = new Array<String>(comboKey);
        if (!lookupTableCompositeKeyValues.get(lookupTable).contains(wrap)) {
            // System.out.println("Try " + wrap + " Failed, continue...");
            return false;
        }
    }
    return true;
}
 

public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) {
    Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>();
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) {
        Array<TblColRef> hostCols = entry.getKey();
        boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data));
        if (!hostOnRow)
            continue;

        List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>();
        for (DeriveInfo info : entry.getValue()) {
            if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has
                                                                                                              // any
                                                                                                              // wanted
                                                                                                              // columns?
                wantedInfo.add(info);
        }

        if (wantedInfo.size() > 0)
            result.put(hostCols, wantedInfo);
    }
    return result;
}
 

private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
    if (hostCols.length == 0 || derivedCols.length == 0)
        throw new IllegalStateException("host/derived columns must not be empty");

    Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
    List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
    if (infoList == null) {
        hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
    }
    infoList.add(new DeriveInfo(type, dimension, derivedCols, false));

    for (int i = 0; i < derivedCols.length; i++) {
        TblColRef derivedCol = derivedCols[i];
        boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
        derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
    }
}
 

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
 

public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) {
    Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>();
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) {
        Array<TblColRef> hostCols = entry.getKey();
        boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data));
        if (!hostOnRow)
            continue;

        List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>();
        for (DeriveInfo info : entry.getValue()) {
            if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has
                                                                                                              // any
                                                                                                              // wanted
                                                                                                              // columns?
                wantedInfo.add(info);
        }

        if (wantedInfo.size() > 0)
            result.put(hostCols, wantedInfo);
    }
    return result;
}
 

private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
    if (hostCols.length == 0 || derivedCols.length == 0)
        throw new IllegalStateException("host/derived columns must not be empty");

    Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
    List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
    if (infoList == null) {
        hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
    }
    infoList.add(new DeriveInfo(type, dimension, derivedCols, false));

    for (int i = 0; i < derivedCols.length; i++) {
        TblColRef derivedCol = derivedCols[i];
        boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
        derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
    }
}
 

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
 

@Test
public void testGet() throws Exception {
    int getNum = 3000;
    List<String[]> keys = Lists.newArrayList();
    for (int i = 0; i < getNum; i++) {
        String[] keyi = new String[] { "keyyyyy" + random.nextInt(sourceRowNum) };
        keys.add(keyi);
    }

    long start = System.currentTimeMillis();
    for (int i = 0; i < getNum; i++) {
        String[] row = lookupTable.getRow(new Array<>(keys.get(i)));
        if (row == null) {
            System.out.println("null value for key:" + Arrays.toString(keys.get(i)));
        }
    }
    long take = System.currentTimeMillis() - start;
    System.out.println("muliti get " + getNum + " rows, take " + take + " ms");
}
 

@Test
public void testGet() throws Exception {
    int getNum = 3000;
    List<String[]> keys = Lists.newArrayList();
    for (int i = 0; i < getNum; i++) {
        String[] keyi = new String[] { "keyyyyy" + random.nextInt(sourceRowNum) };
        keys.add(keyi);
    }

    long start = System.currentTimeMillis();
    for (int i = 0; i < getNum; i++) {
        String[] row = lookupTable.getRow(new Array<>(keys.get(i)));
        if (row == null) {
            System.out.println("null value for key:" + Arrays.toString(keys.get(i)));
        }
    }
    long take = System.currentTimeMillis() - start;
    System.out.println("muliti get " + getNum + " rows, take " + take + " ms");
}
 

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
 

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
 

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
 

private static void collect(String[] row, int[] pi, Set<Array<String>> satisfyingHostRecords) {
    // TODO when go beyond IN_THRESHOLD, only keep min/max is enough
    String[] rec = new String[pi.length];
    for (int i = 0; i < pi.length; i++) {
        rec[i] = row[pi[i]];
    }
    satisfyingHostRecords.add(new Array<String>(rec));
}
 

private static List<String> asValues(Set<Array<String>> satisfyingHostRecords) {
    List<String> values = Lists.newArrayListWithCapacity(satisfyingHostRecords.size());
    for (Array<String> rec : satisfyingHostRecords) {
        values.add(rec.data[0]);
    }
    return values;
}
 

private static void findMinMax(Set<Array<String>> satisfyingHostRecords, TblColRef[] hostCols, String[] min, String[] max) {

        DataTypeOrder[] orders = new DataTypeOrder[hostCols.length];
        for (int i = 0; i < hostCols.length; i++) {
            orders[i] = hostCols[i].getType().getOrder();
        }

        for (Array<String> rec : satisfyingHostRecords) {
            String[] row = rec.data;
            for (int i = 0; i < row.length; i++) {
                min[i] = orders[i].min(min[i], row[i]);
                max[i] = orders[i].max(max[i], row[i]);
            }
        }
    }
 

@Override
public String[] getRow(Array<String> key) {
    byte[] encodedKey = encoder.encodeRowKey(key.data);
    Get get = new Get(encodedKey);
    try {
        Result result = table.get(get);
        if (result.isEmpty()) {
            return null;
        }
        return encoder.decode(new HBaseRow(result.getRow(), result.getFamilyMap(HBaseLookupRowEncoder.CF)));
    } catch (IOException e) {
        throw new RuntimeException("error when get row from hBase", e);
    }
}
 

@SuppressWarnings("unchecked")
private void initRow(String[] cols, int[] keyIndex) {
    T[] value = convertRow(cols);
    T[] keyCols = (T[]) java.lang.reflect.Array.newInstance(getType(), keyIndex.length);
    for (int i = 0; i < keyCols.length; i++)
        keyCols[i] = value[keyIndex[i]];

    Array<T> key = new Array<T>(keyCols);

    if (data.containsKey(key))
        throw new IllegalStateException("The table: " + tableDesc.getName() + " Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value));

    data.put(key, value);
}
 

private Pair<Integer, Integer> countDerivedInfo(Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo) {
    int pkfkCount = 0;
    int lookupCount = 0;
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
        for (DeriveInfo deriveInfo : entry.getValue()) {
            if (deriveInfo.type == DeriveType.PK_FK)
                pkfkCount++;
            if (deriveInfo.type == DeriveType.LOOKUP)
                lookupCount++;
        }
    }
    return Pair.newPair(pkfkCount, lookupCount);
}
 

public LookupTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException {
    this.tableDesc = tableDesc;
    this.keyColumns = keyColumns;
    this.table = table;
    this.data = new ConcurrentHashMap<Array<T>, T[]>();
    init();
}
 

@SuppressWarnings("unchecked")
private void initRow(String[] cols, int[] keyIndex) {
    T[] value = convertRow(cols);
    T[] keyCols = (T[]) java.lang.reflect.Array.newInstance(value[0].getClass(), keyIndex.length);
    for (int i = 0; i < keyCols.length; i++)
        keyCols[i] = value[keyIndex[i]];

    Array<T> key = new Array<T>(keyCols);

    if (data.containsKey(key))
        throw new IllegalStateException("Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value));

    data.put(key, value);
}
 

public LookupFiller(int[] hostIndex, LookupStringTable lookupTable, DeriveInfo deriveInfo, String[] derivedFieldNames) {
    this.hostIndex = hostIndex;
    this.hostLen = hostIndex.length;
    this.lookupKey = new Array<String>(new String[hostLen]);
    this.lookupTable = lookupTable;
    this.derivedIndex = new int[deriveInfo.columns.length];
    this.derivedLen = derivedIndex.length;
    this.derivedFieldNames = derivedFieldNames;

    for (int i = 0; i < derivedLen; i++) {
        derivedIndex[i] = deriveInfo.columns[i].getColumn().getZeroBasedIndex();
    }
}
 

private TupleInfo buildTupleInfo(Cuboid cuboid) {
    TupleInfo info = new TupleInfo();
    int index = 0;
    rowKeyDecoder.setCuboid(cuboid);
    List<TblColRef> rowColumns = rowKeyDecoder.getColumns();
    List<String> colNames = rowKeyDecoder.getNames(context.getAliasMap());
    for (int i = 0; i < rowColumns.size(); i++) {
        TblColRef column = rowColumns.get(i);
        if (!dimensions.contains(column)) {
            continue;
        }
        // add normal column
        info.setField(colNames.get(i), rowColumns.get(i), rowColumns.get(i).getType().getName(), index++);
    }

    // derived columns and filler
    Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cubeSeg.getCubeDesc().getHostToDerivedInfo(rowColumns, null);
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
        TblColRef[] hostCols = entry.getKey().data;
        for (DeriveInfo deriveInfo : entry.getValue()) {
            // mark name for each derived field
            for (TblColRef derivedCol : deriveInfo.columns) {
                String derivedField = getFieldName(derivedCol, context.getAliasMap());
                info.setField(derivedField, derivedCol, derivedCol.getType().getName(), index++);
            }
            // add filler
            info.addDerivedColumnFiller(Tuple.newDerivedColumnFiller(rowColumns, hostCols, deriveInfo, info, CubeManager.getInstance(this.cube.getConfig()), cubeSeg));
        }
    }

    for (RowValueDecoder rowValueDecoder : this.rowValueDecoders) {
        List<String> names = rowValueDecoder.getNames();
        MeasureDesc[] measures = rowValueDecoder.getMeasures();
        for (int i = 0; i < measures.length; i++) {
            String dataType = measures[i].getFunction().getSQLType();
            info.setField(names.get(i), null, dataType, index++);
        }
    }
    return info;
}
 

private static void collect(String[] row, int[] pi, Set<Array<String>> satisfyingHostRecords) {
    // TODO when go beyond IN_THRESHOLD, only keep min/max is enough
    String[] rec = new String[pi.length];
    for (int i = 0; i < pi.length; i++) {
        rec[i] = row[pi[i]];
    }
    satisfyingHostRecords.add(new Array<String>(rec));
}
 

private static List<String> asValues(Set<Array<String>> satisfyingHostRecords) {
    List<String> values = Lists.newArrayListWithCapacity(satisfyingHostRecords.size());
    for (Array<String> rec : satisfyingHostRecords) {
        values.add(rec.data[0]);
    }
    return values;
}
 

private static void findMinMax(Set<Array<String>> satisfyingHostRecords, TblColRef[] hostCols, String[] min, String[] max) {

        RowKeyColumnOrder[] orders = new RowKeyColumnOrder[hostCols.length];
        for (int i = 0; i < hostCols.length; i++) {
            orders[i] = RowKeyColumnOrder.getInstance(hostCols[i].getType());
        }

        for (Array<String> rec : satisfyingHostRecords) {
            String[] row = rec.data;
            for (int i = 0; i < row.length; i++) {
                min[i] = orders[i].min(min[i], row[i]);
                max[i] = orders[i].max(max[i], row[i]);
            }
        }
    }
 

private ILookupTable getLookupTableWithRandomData(final int rowNum) {
    return new ILookupTable() {
        @Override
        public String[] getRow(Array<String> key) {
            return new String[0];
        }

        @Override
        public void close() throws IOException {

        }

        @Override
        public Iterator<String[]> iterator() {
            return new Iterator<String[]>() {
                private int iterCnt = 0;

                @Override
                public boolean hasNext() {
                    return iterCnt < rowNum;
                }

                @Override
                public String[] next() {
                    iterCnt++;
                    return genRandomRow(iterCnt);
                }

                @Override
                public void remove() {

                }
            };
        }
    };
}
 

public LookupTable(TableDesc tableDesc, String[] keyColumns, IReadableTable table) throws IOException {
    this.tableDesc = tableDesc;
    this.keyColumns = keyColumns;
    this.table = table;
    this.data = new HashMap<Array<T>, T[]>();
    init();
}