org.elasticsearch.common.bytes.BytesArray Java Examples

public LocalRestRequest(final String uri, final Method method) {
    this.headers = new HashMap<>();
    if (uri.startsWith("\"") && uri.endsWith("\"")) {
        this.uri = uri.substring(1, uri.length()-1);
    } else {
        this.uri = uri;
    }
    this.method = method;
    this.params = new HashMap<>();
    this.content = BytesArray.EMPTY;

    int pathEndPos = this.uri.indexOf('?');
    if (pathEndPos < 0) {
        this.rawPath = this.uri;
    } else {
        this.rawPath = this.uri.substring(0, pathEndPos);
        RestUtils.decodeQueryString(this.uri, pathEndPos + 1, params);
    }
}
 

private void test_get(final Client client, final String index,
        final String type, final String id, final byte[] hash1,
        final byte[] hash2, final byte[] hash3) {
    final GetResponse response = client.prepareGet(index, type, id)
            .setStoredFields(new String[] { "_source", "minhash_value1", "minhash_value2", "minhash_value3" }).execute()
            .actionGet();
    assertTrue(response.isExists());
    final Map<String, Object> source = response.getSourceAsMap();
    assertEquals("test " + Integer.parseInt(id) % 100, source.get("msg"));

    final DocumentField field1 = response.getField("minhash_value1");
    final BytesArray value1 = (BytesArray) field1.getValue();
    assertEquals(hash1.length, value1.length());
    Assert.assertArrayEquals(hash1, value1.array());

    final DocumentField field2 = response.getField("minhash_value2");
    final BytesArray value2 = (BytesArray) field2.getValue();
    assertEquals(hash2.length, value2.length());
    Assert.assertArrayEquals(hash2, value2.array());

    final DocumentField field3 = response.getField("minhash_value3");
    final BytesArray value3 = (BytesArray) field3.getValue();
    assertEquals(hash3.length, value3.length());
    Assert.assertArrayEquals(hash3, value3.array());
}
 

@Override
public void putIndex(String index, String source) {
    var watch = new StopWatch();
    try {
        IndicesClient client = client().indices();
        CreateIndexRequest request = new CreateIndexRequest(index).source(new BytesArray(source), XContentType.JSON);
        boolean exists = client.exists(new GetIndexRequest(index), RequestOptions.DEFAULT);
        if (!exists) {
            client.create(request, RequestOptions.DEFAULT);
        } else {
            // only try to update mappings, as for settings it generally requires to close index first then open after update
            logger.info("index already exists, update mapping, index={}", index);
            client.putMapping(new PutMappingRequest(index).source(request.mappings(), XContentType.JSON), RequestOptions.DEFAULT);
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        logger.info("put index, index={}, source={}, elapsed={}", index, source, watch.elapsed());
    }
}
 

@Override
public Map<String, Object> generateSourceAndCheckConstraints(Object[] values) {
    String rawSource = (String) values[0];
    Map<String, Object> source = XContentHelper.toMap(new BytesArray(rawSource), XContentType.JSON);
    mixinDefaults(source, defaults);
    for (int i = 0; i < expressions.size(); i++) {
        expressions.get(i).setNextRow(source);
    }
    for (Map.Entry<Reference, Input<?>> entry : generatedCols.entrySet()) {
        var reference = entry.getKey();
        var value = entry.getValue().value();
        var valueForInsert = reference
            .valueType()
            .valueForInsert(value);
        source.putIfAbsent(reference.column().fqn(), valueForInsert);
    }
    return source;
}
 

private Set<BytesArray> getExistingDigestsFromTarget(byte prefix) {
    BlobStartPrefixResponse response =
        (BlobStartPrefixResponse)transportService.submitRequest(
            request.targetNode(),
            BlobRecoveryTarget.Actions.START_PREFIX,
            new BlobStartPrefixSyncRequest(request.recoveryId(), request.shardId(), prefix),
            TransportRequestOptions.EMPTY,
            new FutureTransportResponseHandler<TransportResponse>() {
                @Override
                public TransportResponse newInstance() {
                    return new BlobStartPrefixResponse();
                }
            }
        ).txGet();

    Set<BytesArray> result = new HashSet<BytesArray>();
    for (byte[] digests : response.existingDigests) {
        result.add(new BytesArray(digests));
    }
    return result;
}
 

public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException {
    if (bytes.hasArray()) {
        return convertToJson(bytes.array(), bytes.arrayOffset(), bytes.length(), reformatJson, prettyPrint);
    }
    XContentType xContentType = XContentFactory.xContentType(bytes);
    if (xContentType == XContentType.JSON && !reformatJson) {
        BytesArray bytesArray = bytes.toBytesArray();
        return new String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length(), Charsets.UTF_8);
    }
    try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput())) {
        parser.nextToken();
        XContentBuilder builder = XContentFactory.jsonBuilder();
        if (prettyPrint) {
            builder.prettyPrint();
        }
        builder.copyCurrentStructure(parser);
        return builder.string();
    }
}
 

@VisibleForTesting
Map<String, Object> buildMapFromSource(Reference[] insertColumns,
                                       Object[] insertValues,
                                       boolean isRawSourceInsert) {
    Map<String, Object> sourceAsMap;
    if (isRawSourceInsert) {
        BytesRef source = (BytesRef) insertValues[0];
        sourceAsMap = XContentHelper.convertToMap(new BytesArray(source), true).v2();
    } else {
        sourceAsMap = new LinkedHashMap<>(insertColumns.length);
        for (int i = 0; i < insertColumns.length; i++) {
            sourceAsMap.put(insertColumns[i].ident().columnIdent().fqn(), insertValues[i]);
        }
    }
    return sourceAsMap;
}
 

private Status start(ChannelBuffer buffer, boolean last) {
    logger.trace("start blob upload");
    assert (transferId == null);
    StartBlobRequest request = new StartBlobRequest(
            index,
            Hex.decodeHex(digest),
            new BytesArray(buffer.array()),
            last
    );
    transferId = request.transferId();
    size += buffer.readableBytes();

    startResponse = client.execute(StartBlobAction.INSTANCE, request).actionGet();
    status = startResponse.status();
    return status;
}
 

public static void toXContent(AliasMetaData aliasMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
    builder.startObject(aliasMetaData.alias());

    boolean binary = params.paramAsBoolean("binary", false);

    if (aliasMetaData.filter() != null) {
        if (binary) {
            builder.field("filter", aliasMetaData.filter.compressed());
        } else {
            builder.field("filter", XContentHelper.convertToMap(new BytesArray(aliasMetaData.filter().uncompressed()), true).v2());
        }
    }
    if (aliasMetaData.indexRouting() != null) {
        builder.field("index_routing", aliasMetaData.indexRouting());
    }
    if (aliasMetaData.searchRouting() != null) {
        builder.field("search_routing", aliasMetaData.searchRouting());
    }

    if (aliasMetaData.writeIndex() != null) {
        builder.field("is_write_index", aliasMetaData.writeIndex());
    }

    builder.endObject();
}
 

/**
 * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
 *
 * @param blobContainer blob container
 * @param blobName blob name
 */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    try (InputStream inputStream = blobContainer.readBlob(blobName)) {
        byte[] bytes = ByteStreams.toByteArray(inputStream);
        final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
        try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) {
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize);
            return read(bytesReference);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 

/**
 * Reads snapshot index file
 * <p>
 * This file can be used by read-only repositories that are unable to list files in the repository
 *
 * @return list of snapshots in the repository
 * @throws IOException I/O errors
 */
protected List<SnapshotId> readSnapshotList() throws IOException {
    try (InputStream blob = snapshotsBlobContainer.readBlob(SNAPSHOTS_FILE)) {
        final byte[] data = ByteStreams.toByteArray(blob);
        ArrayList<SnapshotId> snapshots = new ArrayList<>();
        try (XContentParser parser = XContentHelper.createParser(new BytesArray(data))) {
            if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
                if (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
                    String currentFieldName = parser.currentName();
                    if ("snapshots".equals(currentFieldName)) {
                        if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
                            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
                                snapshots.add(new SnapshotId(repositoryName, parser.text()));
                            }
                        }
                    }
                }
            }
        }
        return Collections.unmodifiableList(snapshots);
    }
}
 

@Override
public void verify(String seed) {
    BlobContainer testBlobContainer = blobStore.blobContainer(basePath.add(testBlobPrefix(seed)));
    DiscoveryNode localNode = clusterService.localNode();
    if (testBlobContainer.blobExists("master.dat")) {
        try  {
            testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", new BytesArray(seed));
        } catch (IOException exp) {
            throw new RepositoryVerificationException(repositoryName, "store location [" + blobStore + "] is not accessible on the node [" + localNode + "]", exp);
        }
    } else {
        throw new RepositoryVerificationException(repositoryName, "a file written by master to the store [" + blobStore + "] cannot be accessed on the node [" + localNode + "]. "
                + "This might indicate that the store [" + blobStore + "] is not shared between this node and the master node or "
                + "that permissions on the store don't allow reading files written by the master node");
    }
}
 

@Test
public void testPutChunkReplicaRequestSerialization() throws Exception {
    UUID transferId = UUID.randomUUID();

    PutChunkReplicaRequest requestOut = new PutChunkReplicaRequest(
        new ShardId("foo", UUIDs.randomBase64UUID(), 1),
        "nodeId",
        transferId,
        0,
        new BytesArray(new byte[]{0x65, 0x66}),
        false
    );
    requestOut.index("foo");
    BytesStreamOutput outputStream = new BytesStreamOutput();
    requestOut.writeTo(outputStream);
    StreamInput inputStream = outputStream.bytes().streamInput();

    PutChunkReplicaRequest requestIn = new PutChunkReplicaRequest(inputStream);

    assertEquals(requestOut.currentPos, requestIn.currentPos);
    assertEquals(requestOut.isLast, requestIn.isLast);
    assertEquals(requestOut.content, requestIn.content);
    assertEquals(requestOut.transferId, requestIn.transferId);
    assertEquals(requestOut.index(), requestIn.index());
}
 

private void initMemory(Terms curTerms, int termFreq) {
    // init memory for performance reasons
    if (curTerms.hasPositions()) {
        currentPositions = ArrayUtil.grow(currentPositions, termFreq);
    }
    if (curTerms.hasOffsets()) {
        currentStartOffset = ArrayUtil.grow(currentStartOffset, termFreq);
        currentEndOffset = ArrayUtil.grow(currentEndOffset, termFreq);
    }
    if (curTerms.hasPayloads()) {
        currentPayloads = new BytesArray[termFreq];
    }
}
 

@Override
public Map<String, Object> generateSourceAndCheckConstraints(Object[] values) throws IOException {
    return JsonXContent.JSON_XCONTENT.createParser(
        NamedXContentRegistry.EMPTY,
        DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
        new BytesArray(((String) values[0])).array()
    ).map();
}
 

@Override
public BulkRequest buildRequest(Client client, BulkRequestBuilder builder, Collection<SearchGuardACLDocument> docs) throws IOException{
    for (SearchGuardACLDocument doc : docs) {
        logContent("Expired doc {} to be: {}", doc.getType(), doc);
        Map<String, Object> content = new HashMap<>();
        content.put(doc.getType(), new BytesArray(XContentHelper.toString(doc)));
        IndexRequestBuilder indexBuilder = client
                .prepareIndex(searchGuardIndex, doc.getType(), SEARCHGUARD_CONFIG_ID)
                .setOpType(OpType.INDEX)
                .setVersion(doc.getVersion())
                .setSource(content);
        builder.add(indexBuilder.request());
    }
    return builder.request();
}
 

@Test
public void testCoordinateSearchApi() throws IOException, RestException, ExecutionException, InterruptedException {
  assertAcked(prepareCreate("index1").addMapping("type", "id", "type=string", "foreign_key", "type=string"));
  assertAcked(prepareCreate("index2").addMapping("type", "id", "type=string", "tag", "type=string"));

  ensureGreen();

  indexRandom(true,
    client().prepareIndex("index1", "type", "1").setSource("id", "1", "foreign_key", new String[]{"1", "3"}),
    client().prepareIndex("index1", "type", "2").setSource("id", "2"),
    client().prepareIndex("index1", "type", "3").setSource("id", "3", "foreign_key", new String[]{"2"}),
    client().prepareIndex("index1", "type", "4").setSource("id", "4", "foreign_key", new String[]{"1", "4"}),

    client().prepareIndex("index2", "type", "1").setSource("id", "1", "tag", "aaa"),
    client().prepareIndex("index2", "type", "2").setSource("id", "2", "tag", "aaa"),
    client().prepareIndex("index2", "type", "3").setSource("id", "3", "tag", "bbb"),
    client().prepareIndex("index2", "type", "4").setSource("id", "4", "tag", "ccc") );

  // Check body search query with filter join
  String q = boolQuery().filter(
              filterJoin("foreign_key").indices("index2").types("type").path("id").query(
                  boolQuery().filter(termQuery("tag", "aaa"))
          )).toString();
  String body = "{ \"query\" : " + q + "}";

  HttpResponse response = httpClient().method("GET").path("/_coordinate_search").body(body).execute();
  assertThat(response.getStatusCode(), equalTo(RestStatus.OK.getStatus()));
  Map<String, Object> map = XContentHelper.convertToMap(new BytesArray(response.getBody().getBytes("UTF-8")), false).v2();
  assertThat((Integer) ((Map) map.get("hits")).get("total"), equalTo(3));

  // Check uri search
  response = httpClient().method("GET").path("/_coordinate_search").addParam("q", "tag:aaa").execute();
  assertThat(response.getStatusCode(), equalTo(RestStatus.OK.getStatus()));
  map = XContentHelper.convertToMap(new BytesArray(response.getBody().getBytes("UTF-8")), false).v2();
  assertThat((Integer) ((Map) map.get("hits")).get("total"), equalTo(2));
}
 

private BytesReference getContent(final RestRequest request, final OpenshiftRequestContext context) {
    String content = request.content().utf8ToString();
    if(OpenshiftRequestContext.EMPTY != context && content.contains("_index\":\"" + defaultKibanaIndex)) {
        LOGGER.debug("Replacing the content that references the default kibana index");
        String replaced = content.replaceAll("_index\":\"" + defaultKibanaIndex + "\"", "_index\":\"" + context.getKibanaIndex() + "\"");
        return new BytesArray(replaced);
    }
    return request.content();
}
 

/**
 * Reads a bytes reference from this stream, might hold an actual reference to the underlying
 * bytes of the stream.
 */
public BytesReference readBytesReference(int length) throws IOException {
    if (length == 0) {
        return BytesArray.EMPTY;
    }
    byte[] bytes = new byte[length];
    readBytes(bytes, 0, length);
    return new BytesArray(bytes, 0, length);
}
 

/** Return the uncompressed bytes. */
public byte[] uncompressed() {
    try {
        return CompressorFactory.uncompress(new BytesArray(bytes)).toBytes();
    } catch (IOException e) {
        throw new IllegalStateException("Cannot decompress compressed string", e);
    }
}
 

/**
 * Guesses the content type based on the provided input stream without consuming it.
 */
public static XContentType xContentType(InputStream si) throws IOException {
    if (si.markSupported() == false) {
        throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass());
    }
    si.mark(GUESS_HEADER_LENGTH);
    try {
        final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH];
        final int read = Streams.readFully(si, firstBytes);
        return xContentType(new BytesArray(firstBytes, 0, read));
    } finally {
        si.reset();
    }
}
 

@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
    if (sourceFieldName.equals(fieldInfo.name)) {
        source = new BytesArray(value);
    } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
        id = Uid.decodeId(value);
    } else {
        addValue(fieldInfo.name, new BytesRef(value));
    }
}
 

private IndexRequest createIndexRequest(byte[] messageBytes) {
    IndexRequest request = new IndexRequest();

    request.index(indexName);
    request.source(new BytesArray(messageBytes), XContentType.JSON);

    return request;
}
 

protected static void expectErrorResponseWithMessage(HttpResponse response,
                                                     int expectedStatus,
                                                     String messageSubstring) throws IOException {
   byte[] responseBytes = response.getEntity().getContent().readAllBytes();
   String responseString = new String(responseBytes, StandardCharsets.UTF_8);
   String responseDescription =
       "HTTP response status: " + response.getStatusLine().toString() + ", " +
           "HTTP body: " + responseString;

   Assertions.assertThat(response.getStatusLine().getStatusCode())
       .describedAs(responseDescription)
       .isEqualTo(expectedStatus);

   XContentType xContentType = XContentType.fromMediaTypeOrFormat(
       response.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue());
   try (XContentParser parser = XContentHelper.createParser(
       NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
       new BytesArray(responseBytes), xContentType)) {
      Map<String, Object> responseJson = parser.mapOrdered();

      Assertions.assertThat(responseJson)
          .describedAs(responseString)
          .containsKey("error");

      Assertions.assertThat(responseJson.get("error").toString())
          .describedAs(responseString)
          .contains(messageSubstring);
   }
}
 

@Nullable
Map<String, Object> sourceAsMap() {
    if (parsedSource == null) {
        if (rawSource != null) {
            try {
                parsedSource = XContentHelper.toMap(new BytesArray(rawSource), XContentType.JSON);
            } catch (ElasticsearchParseException | NotXContentException e) {
                throw new RuntimeException("JSON parser error: " + e.getMessage(), e);
            }
        }
    }
    return parsedSource;
}
 

private void givenKibanaConfigWithDefaultIndex(String index) {
    GetResponse response = new GetResponse(new GetResult(context.getKibanaIndex(), 
            "config", ConfigurationSettings.DEFAULT_KIBANA_VERSION, 1L, true, 
            new BytesArray("{\"defaultIndex\":\"" + index + "\"}"), null));
    when(pluginClient.getDocument(eq(context.getKibanaIndex()), eq("config"),
            eq(ConfigurationSettings.DEFAULT_KIBANA_VERSION))).thenReturn(response);
}
 

private BytesReference evaluateContentForKibanaIndex(BytesReference contentRef, OpenshiftRequestContext context, String defaultKibanaIndex) {
    if (context == null || context == OpenshiftRequestContext.EMPTY) {
        return contentRef;
    }
    String content = contentRef.utf8ToString();
    if(content.contains("_index\":\"" + context.getKibanaIndex())) {
        LOGGER.debug("Replacing the content that references the kibana index");
        String replaced = content.replaceAll("_index\":\"" + context.getKibanaIndex() + "\"", "_index\":\"" + defaultKibanaIndex + "\"");
        return new BytesArray(replaced);
    }
    return contentRef;
}
 

@Override
public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
    if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
        source = new BytesArray(value);
    } else {
        addValue(fieldInfo.name, new BytesRef(value));
    }
}
 

private void writeTo(final T obj, final String blobName, final CheckedConsumer<BytesArray, IOException> consumer) throws IOException {
    final BytesReference bytes;
    try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
        if (compress) {
            try (StreamOutput compressedStreamOutput = CompressorFactory.COMPRESSOR.streamOutput(bytesStreamOutput)) {
                write(obj, compressedStreamOutput);
            }
        } else {
            write(obj, bytesStreamOutput);
        }
        bytes = bytesStreamOutput.bytes();
    }
    try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
        final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
        try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, outputStream, BUFFER_SIZE)) {
            CodecUtil.writeHeader(indexOutput, codec, VERSION);
            try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
                @Override
                public void close() {
                    // this is important since some of the XContentBuilders write bytes on close.
                    // in order to write the footer we need to prevent closing the actual index input.
                }
            }) {
                bytes.writeTo(indexOutputOutputStream);
            }
            CodecUtil.writeFooter(indexOutput);
        }
        consumer.accept(new BytesArray(outputStream.toByteArray()));
    }
}
 

@Override
public void binaryField(final FieldInfo fieldInfo, final byte[] value) throws IOException {

    if (fieldInfo.name.equals("_source")) {
        final BytesReference bytesRef = new BytesArray(value);
        final Tuple<XContentType, Map<String, Object>> bytesRefTuple = XContentHelper.convertToMap(bytesRef, false, XContentType.JSON);
        Map<String, Object> filteredSource = bytesRefTuple.v2();
        MapUtils.deepTraverseMap(filteredSource, HASH_CB);
        final XContentBuilder xBuilder = XContentBuilder.builder(bytesRefTuple.v1().xContent()).map(filteredSource);
        delegate.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xBuilder)));
    } else {
        delegate.binaryField(fieldInfo, value);
    }
}