org.apache.lucene.store.OutputStreamDataOutput Java Examples

@Override
public boolean store(OutputStream output) throws IOException {
  DataOutput dataOut = new OutputStreamDataOutput(output);
  try {
    if (fst == null) {
      return false;
    }

    fst.save(dataOut);
    dataOut.writeVInt(maxAnalyzedPathsForOneInput);
    dataOut.writeByte((byte) (hasPayloads ? 1 : 0));
  } finally {
    IOUtils.close(output);
  }
  return true;
}
 

@Override
public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException {
    if (weight < -1 || weight > Integer.MAX_VALUE) {
        throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE");
    }
    for (int i = 0; i < surfaceForm.length; i++) {
        if (surfaceForm.bytes[i] == UNIT_SEPARATOR) {
            throw new IllegalArgumentException(
                    "surface form cannot contain unit separator character U+001F; this character is reserved");
        }
    }
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream);
    output.writeVLong(weight + 1);
    output.writeVInt(surfaceForm.length);
    output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
    output.writeVInt(payload.length);
    output.writeBytes(payload.bytes, 0, payload.length);

    output.close();
    return new BytesRef(byteArrayOutputStream.toByteArray());
}
 

public static TranslogWriter create(Type type, ShardId shardId, String translogUUID, long fileGeneration, Path file, Callback<ChannelReference> onClose, int bufferSize, ChannelFactory channelFactory) throws IOException {
    final BytesRef ref = new BytesRef(translogUUID);
    final int headerLength = getHeaderLength(ref.length);
    final FileChannel channel = channelFactory.open(file);
    try {
        // This OutputStreamDataOutput is intentionally not closed because
        // closing it will close the FileChannel
        final OutputStreamDataOutput out = new OutputStreamDataOutput(java.nio.channels.Channels.newOutputStream(channel));
        CodecUtil.writeHeader(out, TRANSLOG_CODEC, VERSION);
        out.writeInt(ref.length);
        out.writeBytes(ref.bytes, ref.offset, ref.length);
        channel.force(true);
        writeCheckpoint(headerLength, 0, file.getParent(), fileGeneration, StandardOpenOption.WRITE);
        final TranslogWriter writer = type.create(shardId, fileGeneration, new ChannelReference(file, fileGeneration, channel, onClose), bufferSize);
        return writer;
    } catch (Throwable throwable){
        // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that
        // file exists we remove it. We only apply this logic to the checkpoint.generation+1 any other file with a higher generation is an error condition
        IOUtils.closeWhileHandlingException(channel);
        throw throwable;
    }
}
 

public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 

public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 

private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 2) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 2 characters");
        }
        out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());
      }
    }
  }
}
 

@Test
public void testTokenStream() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  SuggestField suggestField = new SuggestField("field", "input", 1);
  BytesRef surfaceForm = new BytesRef("input");
  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
    output.writeVInt(surfaceForm.length);
    output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
    output.writeVInt(1 + 1);
    output.writeByte(SuggestField.TYPE);
  }
  BytesRef payload = new BytesRef(byteArrayOutputStream.toByteArray());
  TokenStream stream = new PayloadAttrToTypeAttrFilter(suggestField.tokenStream(analyzer, null));
  assertTokenStreamContents(stream, new String[] {"input"}, null, null, new String[]{payload.utf8ToString()}, new int[]{1}, null, null);

  CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer);
  stream = new PayloadAttrToTypeAttrFilter(suggestField.tokenStream(completionAnalyzer, null));
  assertTokenStreamContents(stream, new String[] {"input"}, null, null, new String[]{payload.utf8ToString()}, new int[]{1}, null, null);
}
 

public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 

private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte)0);
        out.writeByte((byte)0);
        out.writeByte((byte)0);
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 3) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 3 characters");
        }
        out.writeString(data[0]);
        out.writeString(data[1]);
        out.writeString(data[2]);
      }
    }
  }
}
 

public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 

@Test
public void testTokenStream() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  ContextSuggestField field = new ContextSuggestField("field", "input", 1, "context1", "context2");
  BytesRef surfaceForm = new BytesRef("input");
  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
    output.writeVInt(surfaceForm.length);
    output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
    output.writeVInt(1 + 1);
    output.writeByte(ContextSuggestField.TYPE);
  }
  BytesRef payload = new BytesRef(byteArrayOutputStream.toByteArray());
  String[] expectedOutputs = new String[2];
  CharsRefBuilder builder = new CharsRefBuilder();
  builder.append("context1");
  builder.append(((char) ContextSuggestField.CONTEXT_SEPARATOR));
  builder.append((char) ConcatenateGraphFilter.SEP_LABEL);
  builder.append("input");
  expectedOutputs[0] = builder.toCharsRef().toString();
  builder.clear();
  builder.append("context2");
  builder.append(((char) ContextSuggestField.CONTEXT_SEPARATOR));
  builder.append((char) ConcatenateGraphFilter.SEP_LABEL);
  builder.append("input");
  expectedOutputs[1] = builder.toCharsRef().toString();
  TokenStream stream = new TestSuggestField.PayloadAttrToTypeAttrFilter(field.tokenStream(analyzer, null));
  assertTokenStreamContents(stream, expectedOutputs, null, null, new String[]{payload.utf8ToString(), payload.utf8ToString()}, new int[]{1, 0}, null, null);

  CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer);
  stream = new TestSuggestField.PayloadAttrToTypeAttrFilter(field.tokenStream(completionAnalyzer, null));
  assertTokenStreamContents(stream, expectedOutputs, null, null, new String[]{payload.utf8ToString(), payload.utf8ToString()}, new int[]{1, 0}, null, null);
}
 

/**
 * Writes an automaton to a file. 
 */
public void save(final Path path) throws IOException {
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) {
    DataOutput out = new OutputStreamDataOutput(os);
    save(out, out);
  }
}
 

protected URI createFst(SortedSet<String> values) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException {
    FST fst = DatawaveFieldIndexListIteratorJexl.getFST(values);
    
    // now serialize to our file system
    CompressionCodec codec = null;
    String extension = "";
    if (config.getHdfsFileCompressionCodec() != null) {
        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
        if (classLoader == null) {
            classLoader = this.getClass().getClassLoader();
        }
        Class<? extends CompressionCodec> clazz = Class.forName(config.getHdfsFileCompressionCodec(), true, classLoader).asSubclass(CompressionCodec.class);
        codec = clazz.newInstance();
        extension = codec.getDefaultExtension();
    }
    int fstCount = config.getFstCount().incrementAndGet();
    Path fstFile = new Path(fstHdfsUri, "PushdownLargeFileFst." + fstCount + ".fst" + extension);
    
    OutputStream fstFileOut = new BufferedOutputStream(fs.create(fstFile, false));
    if (codec != null) {
        fstFileOut = codec.createOutputStream(fstFileOut);
    }
    
    OutputStreamDataOutput outStream = new OutputStreamDataOutput(fstFileOut);
    fst.save(outStream);
    outStream.close();
    
    return fstFile.toUri();
}
 

private BytesRef buildSuggestPayload() {
  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
    output.writeVInt(surfaceForm.length);
    output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
    output.writeVInt(weight + 1);
    output.writeByte(type());
  } catch (IOException e) {
    throw new RuntimeException(e); // not possible, it's a ByteArrayOutputStream!
  }
  return new BytesRef(byteArrayOutputStream.toByteArray());
}
 

/**
 * Calls {@link #store(DataOutput)} after converting
 * {@link OutputStream} to {@link DataOutput}
 */
public boolean store(OutputStream output) throws IOException {
  DataOutput dataOut = new OutputStreamDataOutput(output);
  try {
    return store(dataOut);
  } finally {
    IOUtils.close(output);
  }
}
 

public Connection(int tcpPort) throws IOException {
  this.destTCPPort = tcpPort;
  this.s = new Socket(InetAddress.getLoopbackAddress(), tcpPort);
  this.sockIn = s.getInputStream();
  this.in = new InputStreamDataInput(sockIn);
  this.bos = new BufferedOutputStream(s.getOutputStream());
  this.out = new OutputStreamDataOutput(bos);
  if (Node.VERBOSE_CONNECTIONS) {
    System.out.println("make new client Connection socket=" + this.s + " destPort=" + tcpPort);
  }
}
 

@Override
public void run() {
  boolean success = false;
  try {
    //node.message("using stream buffer size=" + bufferSize);
    InputStream is = new BufferedInputStream(socket.getInputStream(), bufferSize);
    DataInput in = new InputStreamDataInput(is);
    BufferedOutputStream bos = new BufferedOutputStream(socket.getOutputStream(), bufferSize);
    DataOutput out = new OutputStreamDataOutput(bos);

    if (node instanceof SimplePrimaryNode) {
      ((SimplePrimaryNode) node).handleOneConnection(random(), ss, stop, is, socket, in, out, bos);
    } else {
      ((SimpleReplicaNode) node).handleOneConnection(ss, stop, is, socket, in, out, bos);
    }

    bos.flush();
    if (Node.VERBOSE_CONNECTIONS) {
      node.message("bos.flush done");
    }

    success = true;
  } catch (Throwable t) {
    if (t instanceof SocketException == false && t instanceof NodeCommunicationException == false) {
      node.message("unexpected exception handling client connection; now failing test:");
      t.printStackTrace(System.out);
      IOUtils.closeWhileHandlingException(ss);
      // Test should fail with this:
      throw new RuntimeException(t);
    } else {
      node.message("exception handling client connection; ignoring:");
      t.printStackTrace(System.out);
    }
  } finally {
    if (success) {
      try {
        IOUtils.close(socket);
      } catch (IOException ioe) {
        throw new RuntimeException(ioe);
      }
    } else {
      IOUtils.closeWhileHandlingException(socket);
    }
  }
  if (Node.VERBOSE_CONNECTIONS) {
    node.message("socket.close done");
  }
}
 

/**
 *
 * @param inputStream the input stream
 * @param outputStream the output stream
 * @throws IOException if compilation fails
 */
public void compile(InputStream inputStream, OutputStream outputStream) throws IOException {
    final HashSet<BytesRef> words = new HashSet<>();
    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
    String line;
    String last = null;
    StringBuilder stringBuilder = new StringBuilder();
    while ((line = reader.readLine()) != null) {
        if (line.indexOf('#') >= 0) {
            continue;
        }
        line = pattern.split(line)[0].trim();
        line = line.toLowerCase(Locale.ROOT);
        if (line.equals(last)) {
            continue;
        }
        last = line;
        /*
         * Add the word to the hash set in left-to-right characters order and reversed
         * for easier matching later on.
         */
        stringBuilder.setLength(0);
        stringBuilder.append(line);
        final int len = stringBuilder.length();
        stringBuilder.append('>');
        words.add(new BytesRef(stringBuilder));
        stringBuilder.setLength(len);
        stringBuilder.reverse().append('<');
        words.add(new BytesRef(stringBuilder));
    }
    reader.close();
    final BytesRef [] all = new BytesRef[words.size()];
    words.toArray(all);
    Arrays.sort(all, BytesRef::compareTo);
    final Object nothing = NoOutputs.getSingleton().getNoOutput();
    final Builder<Object> builder = new Builder<>(INPUT_TYPE.BYTE4, NoOutputs.getSingleton());
    final IntsRefBuilder intsRef = new IntsRefBuilder();
    for (BytesRef bytesRef : all) {
        intsRef.clear();
        intsRef.copyUTF8Bytes(bytesRef);
        builder.add(intsRef.get(), nothing);
    }
    final FST<Object> fst = builder.finish();
    try (OutputStreamDataOutput out = new OutputStreamDataOutput(outputStream)) {
        fst.save(out);
    }
}