io.crate.types.ArrayType Java Examples

public static SystemTable<ColumnContext> create() {
    return SystemTable.<ColumnContext>builder(IDENT)
        .add("attrelid", INTEGER, c -> relationOid(c.tableInfo))
        .add("attname", STRING, c -> c.info.column().fqn())
        .add("atttypid", INTEGER, c -> PGTypes.get(c.info.valueType()).oid())
        .add("attstattarget", INTEGER, c -> 0)
        .add("attlen", SHORT, c -> PGTypes.get(c.info.valueType()).typeLen())
        .add("attnum", INTEGER, c -> c.ordinal)
        .add("attndims", INTEGER, c -> isArray(c.info.valueType()) ? 1 : 0)
        .add("attcacheoff", INTEGER, c -> -1)
        .add("atttypmod", INTEGER, c -> -1)
        .add("attbyval", BOOLEAN, c -> false)
        .add("attstorage", STRING, c -> null)
        .add("attalign", STRING, c -> null)
        .add("attnotnull", BOOLEAN, c -> !c.info.isNullable())
        .add("atthasdef", BOOLEAN, c -> false) // don't support default values
        .add("attidentity", STRING, c -> "")
        .add("attisdropped", BOOLEAN, c -> false) // don't support dropping columns
        .add("attislocal", BOOLEAN, c -> true)
        .add("attinhcount", INTEGER, c -> 0)
        .add("attcollation", INTEGER, c -> 0)
        .add("attacl", new ArrayType<>(DataTypes.UNTYPED_OBJECT), c -> null)
        .add("attoptions", STRING_ARRAY, c -> null)
        .add("attfdwoptions", STRING_ARRAY, c -> null)
        .build();
}
 

private ImmutableMap<ColumnIdent, String> getAnalyzers(ColumnIdent columnIdent, Map<String, Object> propertiesMap) {
    ImmutableMap.Builder<ColumnIdent, String> builder = ImmutableMap.builder();
    for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) {
        Map<String, Object> columnProperties = (Map) columnEntry.getValue();
        DataType columnDataType = getColumnDataType(columnProperties);
        ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey());
        columnProperties = furtherColumnProperties(columnProperties);
        if (columnDataType == DataTypes.OBJECT
            || (columnDataType.id() == ArrayType.ID
                && ((ArrayType) columnDataType).innerType() == DataTypes.OBJECT)) {
            if (columnProperties.get("properties") != null) {
                builder.putAll(getAnalyzers(newIdent, (Map<String, Object>) columnProperties.get("properties")));
            }
        }
        String analyzer = (String) columnProperties.get("analyzer");
        if (analyzer != null) {
            builder.put(newIdent, analyzer);
        }
    }
    return builder.build();
}
 

@Override
public Object[] evaluate(Input[] args) {

    DataType innerType   = ((ArrayType)this.info().returnType()).innerType();
    Set<Object> uniqueSet = new LinkedHashSet<>();
    for(Input array : args){
        if(array == null || array.value() == null){
            continue;
        }
        Object[] arg = (Object[]) array.value();
        for(Object element: arg){
            uniqueSet.add(innerType.value(element));
        }
    }

    return uniqueSet.toArray();
}
 

protected ReferenceInfo checkNestedArray(ColumnIdent ci, ReferenceInfo referenceInfo) {
    // TODO: build type correctly as array when the tableInfo is created and remove the conversion here
    if (!ci.isColumn() && hasMatchingParent(referenceInfo, IS_OBJECT_ARRAY)) {
        if (DataTypes.isCollectionType(referenceInfo.type())) {
            // TODO: remove this limitation with next type refactoring
            throw new UnsupportedOperationException("cannot query for arrays inside object arrays explicitly");
        }
        // for child fields of object arrays
        // return references of primitive types as array
        referenceInfo = new ReferenceInfo(
                referenceInfo.ident(),
                referenceInfo.granularity(),
                new ArrayType(referenceInfo.type()),
                referenceInfo.columnPolicy(),
                referenceInfo.indexType());
    }
    return referenceInfo;
}
 

@Test
public void testUpdateDynamicNestedArrayParam() throws Exception {
    Object[] params = {
        new Object[]{
            new Object[]{
                1.9, 4.8
            },
            new Object[]{
                9.7, 12.7
            }
        }
    };
    AnalyzedUpdateStatement update = analyze("update users set new=? where id=1");
    Assignments assignments = Assignments.convert(update.assignmentByTargetCol(), e.functions());
    Symbol[] sources = assignments.bindSources(
        ((DocTableInfo) update.table().tableInfo()), new RowN(params), SubQueryResults.EMPTY);

    DataType dataType = sources[0].valueType();
    assertThat(dataType, is(new ArrayType(new ArrayType(DoubleType.INSTANCE))));
}
 

@Override
public FunctionImplementation<Function> getForTypes(List<DataType> dataTypes) throws IllegalArgumentException {
    Preconditions.checkArgument(dataTypes.size() == 2, "array_cat function requires 2 arguments");

    for (int i = 0; i < dataTypes.size(); i++) {
        Preconditions.checkArgument(dataTypes.get(i) instanceof ArrayType, String.format(Locale.ENGLISH,
                "Argument %d of the array_cat function cannot be converted to array", i + 1));
    }

    DataType innerType0 = ((ArrayType) dataTypes.get(0)).innerType();
    DataType innerType1 = ((ArrayType) dataTypes.get(1)).innerType();

    Preconditions.checkArgument(!innerType0.equals(DataTypes.UNDEFINED) || !innerType1.equals(DataTypes.UNDEFINED),
            "One of the arguments of the array_cat function can be of undefined inner type, but not both");

    if(!innerType0.equals(DataTypes.UNDEFINED)){
        Preconditions.checkArgument(innerType1.isConvertableTo(innerType0),
                String.format(Locale.ENGLISH,
                        "Second argument's inner type (%s) of the array_cat function cannot be converted to the first argument's inner type (%s)",
                        innerType1,innerType0));
    }

    return new ArrayCatFunction(createInfo(dataTypes));
}
 

@Test
public void testCreateFunctionWithComplexComplexTypes() {
    AnalyzedStatement analyzedStatement = e.analyze(
        "CREATE FUNCTION" +
        "   bar(array(integer)," +
        "   object, ip," +
        "   timestamp with time zone" +
        ") RETURNS array(geo_point) LANGUAGE dummy_lang AS 'function() { return 1; }'");
    assertThat(analyzedStatement, instanceOf(AnalyzedCreateFunction.class));

    AnalyzedCreateFunction analysis = (AnalyzedCreateFunction) analyzedStatement;
    assertThat(analysis.name(), is("bar"));
    assertThat(analysis.replace(), is(false));
    assertThat(analysis.returnType(), is(new ArrayType(DataTypes.GEO_POINT)));
    assertThat(analysis.arguments().get(0), is(FunctionArgumentDefinition.of(new ArrayType(DataTypes.INTEGER))));
    assertThat(analysis.arguments().get(1), is(FunctionArgumentDefinition.of(DataTypes.UNTYPED_OBJECT)));
    assertThat(analysis.arguments().get(2), is(FunctionArgumentDefinition.of(DataTypes.IP)));
    assertThat(analysis.arguments().get(3), is(FunctionArgumentDefinition.of(DataTypes.TIMESTAMPZ)));
    assertThat(analysis.language(), is(Literal.of("dummy_lang")));
}
 

@Test
public void testInConvertedToAnyIfOnlyLiterals() throws Exception {
    StringBuilder sb = new StringBuilder("select id from sys.shards where id in (");
    int i = 0;
    for (; i < 1500; i++) {
        sb.append(i);
        sb.append(',');
    }
    sb.append(i++);
    sb.append(')');
    String s = sb.toString();

    WhereClause whereClause = analyzeSelectWhere(s);
    assertThat(whereClause.query(), isFunction(AnyOperators.Type.EQ.opName(),
                                               List.of(DataTypes.INTEGER, new ArrayType<>(DataTypes.INTEGER))));
}
 

@Override
public Scalar<List<Object>, List<Object>> compile(List<Symbol> arguments) {
    Symbol symbol = arguments.get(1);

    if (!symbol.symbolType().isValueSymbol()) {
        // arguments are no values, we can't compile
        return this;
    }

    Input<?> input = (Input<?>) symbol;
    Object inputValue = input.value();

    DataType<?> innerType = ((ArrayType<?>) this.boundSignature.getReturnType().createType()).innerType();
    List<Object> values = (List<Object>) inputValue;
    Set<Object> subtractSet;
    if (values == null) {
        subtractSet = Collections.emptySet();
    } else {
        subtractSet = new HashSet<>();
        for (Object element : values) {
            subtractSet.add(innerType.value(element));
        }
    }
    return new ArrayDifferenceFunction(signature, boundSignature, subtractSet);
}
 

private static List<String> targetUriToStringList(TransactionContext txnCtx,
                                                  Functions functions,
                                                  Symbol targetUri) {
    Object value = SymbolEvaluator.evaluate(txnCtx, functions, targetUri, Row.EMPTY, SubQueryResults.EMPTY);
    if (DataTypes.isSameType(targetUri.valueType(), DataTypes.STRING)) {
        String uri = (String) value;
        return Collections.singletonList(uri);
    } else if (DataTypes.isArray(targetUri.valueType()) &&
               DataTypes.isSameType(ArrayType.unnest(targetUri.valueType()), DataTypes.STRING)) {
        //noinspection unchecked
        return (List<String>) value;
    }

    // this case actually never happens because the check is already done in the analyzer
    throw AnalyzedCopyFrom.raiseInvalidType(targetUri.valueType());
}
 

private void printSubscriptFunction(StringBuilder builder, Style style) {
    Symbol base = arguments.get(0);
    if (base instanceof Reference && base.valueType() instanceof ArrayType && ((Reference) base).column().path().size() > 0) {
        Reference firstArgument = (Reference) base;
        builder.append(firstArgument.column().name());
        builder.append("[");
        builder.append(arguments.get(1).toString(style));
        builder.append("]");
        builder.append("['");
        builder.append(firstArgument.column().path().get(0));
        builder.append("']");
    } else {
        builder.append(base.toString(style));
        builder.append("[");
        builder.append(arguments.get(1).toString(style));
        builder.append("]");
    }
}
 

@Override
public Symbol apply(Function operator,
                    Captures captures,
                    Functions functions) {
    var literal = operator.arguments().get(0);
    var castFunction = captures.get(castCapture);
    var reference = castFunction.arguments().get(0);
    DataType<?> targetType = reference.valueType();
    if (targetType.id() != ArrayType.ID) {
        return null;
    }
    targetType = ((ArrayType<?>) targetType).innerType();

    var operatorName = operator.name();
    if (List.of(AnyOperators.Type.EQ.opName(), AnyOperators.Type.NEQ.opName()).contains(operatorName) == false
        && literal.valueType().id() == ArrayType.ID) {
        // this is not supported and will fail later on with more verbose error than a cast error
        return null;
    }

    return functionResolver.apply(
        operator.name(),
        List.of(literal.cast(targetType), reference)
    );
}
 

public static void flattenSettings(Settings.Builder settingsBuilder,
                                   String key,
                                   Object value) {
    if (value instanceof Map) {
        for (Map.Entry<String, Object> setting : ((Map<String, Object>) value).entrySet()) {
            flattenSettings(
                settingsBuilder,
                String.join(".", key, setting.getKey()),
                setting.getValue()
            );
        }
    } else if (value == null) {
        throw new IllegalArgumentException(
            "Cannot set \"" + key + "\" to `null`. Use `RESET [GLOBAL] \"" + key +
            "\"` to reset a setting to its default value");
    } else if (value instanceof List) {
        ArrayType<String> strArray = new ArrayType<>(DataTypes.STRING);
        List<String> values = strArray.value(value);
        settingsBuilder.put(key, String.join(",", values));
    } else {
        settingsBuilder.put(key, value.toString());
    }
}
 

private ImmutableMap<ColumnIdent, String> getAnalyzers(ColumnIdent columnIdent, Map<String, Object> propertiesMap) {
    ImmutableMap.Builder<ColumnIdent, String> builder = ImmutableMap.builder();
    for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) {
        Map<String, Object> columnProperties = (Map) columnEntry.getValue();
        DataType columnDataType = getColumnDataType(columnProperties);
        ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey());
        columnProperties = furtherColumnProperties(columnProperties);
        if (columnDataType.id() == ObjectType.ID
            || (columnDataType.id() == ArrayType.ID
                && ((ArrayType) columnDataType).innerType().id() == ObjectType.ID)) {
            if (columnProperties.get("properties") != null) {
                builder.putAll(getAnalyzers(newIdent, (Map<String, Object>) columnProperties.get("properties")));
            }
        }
        String analyzer = (String) columnProperties.get("analyzer");
        if (analyzer != null) {
            builder.put(newIdent, analyzer);
        }
    }
    return builder.build();
}
 

public static SystemTable<JobContextLog> create(Supplier<DiscoveryNode> localNode) {
    return SystemTable.<JobContextLog>builder(IDENT)
        .add("id", STRING, x -> x.id().toString())
        .add("username", STRING, JobContextLog::username)
        .add("stmt", STRING, JobContextLog::statement)
        .add("started", TIMESTAMPZ, JobContextLog::started)
        .add("ended", TIMESTAMPZ, JobContextLog::ended)
        .add("error", STRING, JobContextLog::errorMessage)
        .startObject("classification")
            .add("type", STRING, x -> x.classification().type().name())
            .add("labels", new ArrayType<>(STRING), x -> List.copyOf(x.classification().labels()))
        .endObject()
        .startObject("node")
            .add("id", STRING, ignored -> localNode.get().getId())
            .add("name", STRING, ignored -> localNode.get().getName())
        .endObject()
        .withRouting((state, routingProvider, sessionContext) -> Routing.forTableOnAllNodes(IDENT, state.getNodes()))
        .setPrimaryKeys(new ColumnIdent("id"))
        .build();
}
 

private static IntUnaryOperator getNumTermsPerDocFunction(LeafReader reader, Reference ref) {
    DataType elementType = ArrayType.unnest(ref.valueType());
    switch (elementType.id()) {
        case BooleanType.ID:
        case ByteType.ID:
        case ShortType.ID:
        case IntegerType.ID:
        case LongType.ID:
        case TimestampType.ID_WITH_TZ:
        case TimestampType.ID_WITHOUT_TZ:
        case FloatType.ID:
        case DoubleType.ID:
        case GeoPointType.ID:
            return numValuesPerDocForSortedNumeric(reader, ref.column());

        case StringType.ID:
            return numValuesPerDocForString(reader, ref.column());

        case IpType.ID:
            return numValuesPerDocForIP(reader, ref.column());

        default:
            throw new UnsupportedOperationException("NYI: " + elementType);
    }
}
 

private static Query literalMatchesAnyArrayRef(Function any,
                                               Literal candidate,
                                               Reference array,
                                               LuceneQueryBuilder.Context context) {
    MappedFieldType fieldType = context.getFieldTypeOrNull(array.column().fqn());
    if (fieldType == null) {
        if (ArrayType.unnest(array.valueType()).id() == ObjectType.ID) {
            return genericFunctionFilter(any, context); // {x=10} = any(objects)
        }
        return Queries.newMatchNoDocsQuery("column doesn't exist in this index");
    }
    if (DataTypes.isArray(candidate.valueType())) {
        return arrayLiteralEqAnyArray(any, fieldType, candidate.value(), context);
    }
    return fieldType.termQuery(candidate.value(), context.queryShardContext());
}
 

@Test
public void testStreaming() throws Exception {
    RelationName relationName = new RelationName("doc", "test");
    ReferenceIdent referenceIdent = new ReferenceIdent(relationName, "object_column");
    Reference reference = new Reference(
        referenceIdent,
        RowGranularity.DOC,
        new ArrayType<>(DataTypes.UNTYPED_OBJECT),
        ColumnPolicy.STRICT,
        Reference.IndexType.ANALYZED,
        false,
        null,
        Literal.of(Map.of("f", 10)
        )
    );

    BytesStreamOutput out = new BytesStreamOutput();
    Reference.toStream(reference, out);

    StreamInput in = out.bytes().streamInput();
    Reference reference2 = Reference.fromStream(in);

    assertThat(reference2, is(reference));
}
 

private Symbol castArrayElements(DataType<?> newDataType, CastMode... modes) {
    DataType<?> innerType = ((ArrayType<?>) newDataType).innerType();
    ArrayList<Symbol> newArgs = new ArrayList<>(arguments.size());
    for (Symbol arg : arguments) {
        try {
            newArgs.add(arg.cast(innerType, modes));
        } catch (ConversionException e) {
            throw new ConversionException(returnType, newDataType);
        }
    }
    return new Function(
        signature,
        newArgs,
        newDataType,
        null
    );
}
 

@Test
public void testCompareNestedArrayValues() throws Exception {
    ArrayType<List<Integer>> intTypeNestedArr = new ArrayType<>(new ArrayType<>(DataTypes.INTEGER));

    Literal val1 = Literal.of(intTypeNestedArr, List.of(List.of(1, 2, 3)));
    Literal val2 = Literal.of(intTypeNestedArr, List.of(List.of(4, 5, 6)));
    assertThat(val1.equals(val2), is(false));

    val1 = Literal.of(intTypeNestedArr, List.of(List.of(1, 2, 3)));
    val2 = Literal.of(intTypeNestedArr, List.of(List.of(1, 2, 3)));
    assertThat(val1.equals(val2), is(true));

    val1 = Literal.of(intTypeNestedArr, List.of(List.of(3, 2, 1)));
    val2 = Literal.of(intTypeNestedArr, List.of(List.of(1, 2, 3)));
    assertThat(val1.equals(val2), is(false));

    val1 = Literal.of(intTypeNestedArr, List.of(List.of(1, 2, 3, 4, 5)));
    val2 = Literal.of(intTypeNestedArr, List.of(List.of(1, 2, 3)));
    assertThat(val1.equals(val2), is(false));
}
 

@Test
public void testUpdateElementOfObjectArrayUsingParameterExpressionInsideFunctionResultsInCorrectlyTypedParameterSymbol() {
    AnalyzedUpdateStatement stmt = e.analyze("UPDATE bag SET ob = array_cat([?], [{obb=1}]) WHERE id = ?");
    assertThat(
        stmt.assignmentByTargetCol().keySet(),
        contains(isReference("ob", new ArrayType(DataTypes.UNTYPED_OBJECT))));
    assertThat(
        stmt.assignmentByTargetCol().values(),
        contains(isFunction("array_cat",
            isFunction("_array", singletonList(DataTypes.UNTYPED_OBJECT)),
            instanceOf(Literal.class)
        )));
}
 

@Test
public void testUpdateElementOfObjectArrayUsingParameterExpressionResultsInCorrectlyTypedParameterSymbol() {
    AnalyzedUpdateStatement stmt = e.analyze("UPDATE bag SET ob = [?] WHERE id = ?");
    assertThat(
        stmt.assignmentByTargetCol().keySet(),
        contains(isReference("ob", new ArrayType<>(DataTypes.UNTYPED_OBJECT))));
    assertThat(
        stmt.assignmentByTargetCol().values(),
        contains(isFunction("_array", singletonList(DataTypes.UNTYPED_OBJECT))));
}
 

@Test
public void test_fs_data_is_a_object_array() {
    var table = SysNodesTableInfo.create();
    Reference ref = table.getReference(new ColumnIdent("fs", "data"));
    assertThat(ref.valueType().id(), Matchers.is(ArrayType.ID));
    assertThat(((ArrayType<?>) ref.valueType()).innerType().id(), is(ObjectType.ID));
}
 

private static Iterator<Object> createIterator(List<Object> objects, ArrayType<?> type) {
    if (objects == null) {
        return Collections.emptyIterator();
    }
    if (type.innerType() instanceof ArrayType) {
        @SuppressWarnings("unchecked")
        List<Iterator<Object>> iterators = Lists2.map(
            objects,
            x -> createIterator((List<Object>) x, (ArrayType<?>) type.innerType())
        );
        return Iterators.concat(iterators.iterator());
    } else {
        return objects.iterator();
    }
}
 

@Test
public void testListCanContainMixedTypeIfSafeCastIsPossible() {
    List<Object> objects = Arrays.asList(1, null, 1.2f, 0);
    Collections.shuffle(objects);
    DataType<?> dataType = DataTypes.guessType(objects);
    assertThat(dataType, Matchers.is(new ArrayType(DataTypes.FLOAT)));
}
 

@Test
public void testEvaluate() throws Exception {
    assertEvaluate(
        "collection_avg(long_array)",
        5.0,
        Literal.of(List.of(3L, 7L), new ArrayType<>(LongType.INSTANCE))
    );
}
 

public PgExpandArray(Signature signature, Signature boundSignature) {
    this.signature = signature;
    this.boundSignature = boundSignature;
    ArrayType<?> argType = (ArrayType<?>) boundSignature.getArgumentDataTypes().get(0);
    resultType = new RowType(
        List.of(argType.innerType(), DataTypes.INTEGER),
        List.of("x", "n")
    );
}
 

public void test_undefined_array_type_estimate_size_for_object() {
    var sizeEstimator = SizeEstimatorFactory.create(new ArrayType<>(DataTypes.UNDEFINED));
    assertThat(
        sizeEstimator.estimateSize(List.of("", "")),
        // 16 bytes for the list memory overhead
        is(RamUsageEstimator.UNKNOWN_DEFAULT_RAM_BYTES_USED * 2L + 16L));
}
 

@Test
public void testPgArray2CrateType() {
    assertThat(PGTypes.fromOID(PGArray.CHAR_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.INT2_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.INT4_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.INT8_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.FLOAT4_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.FLOAT8_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.BOOL_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.TIMESTAMPZ_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.TIMESTAMP_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.VARCHAR_ARRAY.oid()), instanceOf(ArrayType.class));
    assertThat(PGTypes.fromOID(PGArray.JSON_ARRAY.oid()), instanceOf(ArrayType.class));
}
 

private static LuceneCollectorExpression<?> typeSpecializedExpression(final FieldTypeLookup fieldTypeLookup, final Reference ref) {
    final String fqn = ref.column().fqn();
    final MappedFieldType fieldType = fieldTypeLookup.get(fqn);
    if (fieldType == null) {
        return NO_FIELD_TYPES_IDS.contains(unnest(ref.valueType()).id()) || isIgnoredDynamicReference(ref)
            ? DocCollectorExpression.create(toSourceLookup(ref))
            : new NullValueCollectorExpression();
    }
    if (!fieldType.hasDocValues()) {
        return DocCollectorExpression.create(toSourceLookup(ref));
    }
    switch (ref.valueType().id()) {
        case ByteType.ID:
            return new ByteColumnReference(fqn);
        case ShortType.ID:
            return new ShortColumnReference(fqn);
        case IpType.ID:
            return new IpColumnReference(fqn);
        case StringType.ID:
            return new BytesRefColumnReference(fqn);
        case DoubleType.ID:
            return new DoubleColumnReference(fqn);
        case BooleanType.ID:
            return new BooleanColumnReference(fqn);
        case FloatType.ID:
            return new FloatColumnReference(fqn);
        case LongType.ID:
        case TimestampType.ID_WITH_TZ:
        case TimestampType.ID_WITHOUT_TZ:
            return new LongColumnReference(fqn);
        case IntegerType.ID:
            return new IntegerColumnReference(fqn);
        case GeoPointType.ID:
            return new GeoPointColumnReference(fqn);
        case ArrayType.ID:
            return DocCollectorExpression.create(toSourceLookup(ref));
        default:
            throw new UnhandledServerException("Unsupported type: " + ref.valueType().getName());
    }
}