org.apache.lucene.store.InputStreamDataInput Java Examples

private static void recompileAndWalk(String fstFilePath) throws IOException {
  try (InputStreamDataInput in = new InputStreamDataInput(newInputStream(Paths.get(fstFilePath)))) {

    System.out.println("Reading FST");
    long startTimeMs = System.currentTimeMillis();
    FST<CharsRef> originalFst = new FST<>(in, in, CharSequenceOutputs.getSingleton());
    long endTimeMs = System.currentTimeMillis();
    System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");

    for (float oversizingFactor : List.of(0f, 0f, 0f, 1f, 1f, 1f)) {
      System.out.println("\nFST construction (oversizingFactor=" + oversizingFactor + ")");
      startTimeMs = System.currentTimeMillis();
      FST<CharsRef> fst = recompile(originalFst, oversizingFactor);
      endTimeMs = System.currentTimeMillis();
      System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
      System.out.println("FST RAM = " + fst.ramBytesUsed() + " B");

      System.out.println("FST enum");
      startTimeMs = System.currentTimeMillis();
      walk(fst);
      endTimeMs = System.currentTimeMillis();
      System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
    }
  }
}
 

@Override
public boolean load(InputStream input) throws IOException {
  DataInput dataIn = new InputStreamDataInput(input);
  try {
    this.fst = new FST<>(dataIn, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
    maxAnalyzedPathsForOneInput = dataIn.readVInt();
    hasPayloads = dataIn.readByte() == 1;
  } finally {
    IOUtils.close(input);
  }
  return true;
}
 

@Override
public void parsePayload(BytesRef payload, SuggestPayload ref) throws IOException {
    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(payload.bytes, payload.offset, payload.length);
    InputStreamDataInput input = new InputStreamDataInput(byteArrayInputStream);
    ref.weight = input.readVLong() - 1;
    int len = input.readVInt();
    ref.surfaceForm.grow(len);
    ref.surfaceForm.setLength(len);
    input.readBytes(ref.surfaceForm.bytes(), 0, ref.surfaceForm.length());
    len = input.readVInt();
    ref.payload.grow(len);
    ref.payload.setLength(len);
    input.readBytes(ref.payload.bytes(), 0, ref.payload.length());
    input.close();
}
 

public Connection(int tcpPort) throws IOException {
  this.destTCPPort = tcpPort;
  this.s = new Socket(InetAddress.getLoopbackAddress(), tcpPort);
  this.sockIn = s.getInputStream();
  this.in = new InputStreamDataInput(sockIn);
  this.bos = new BufferedOutputStream(s.getOutputStream());
  this.out = new OutputStreamDataOutput(bos);
  if (Node.VERBOSE_CONNECTIONS) {
    System.out.println("make new client Connection socket=" + this.s + " destPort=" + tcpPort);
  }
}
 

/**
 * Calls {@link #load(DataInput)} after converting
 * {@link InputStream} to {@link DataInput}
 */
public boolean load(InputStream input) throws IOException {
  DataInput dataIn = new InputStreamDataInput(input);
  try {
    return load(dataIn);
  } finally {
    IOUtils.close(input);
  }
}
 

/**
 * @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
 * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
 * this class's name as the path.
 */
public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
  super(resourceScheme, resourcePath);
  FST<Long> fst;
  try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {
    DataInput in = new InputStreamDataInput(is);
    fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
  }
  // TODO: some way to configure?
  this.fst = new TokenInfoFST(fst, true);
}
 

/**
 * @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
 * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
 * this class's name as the path.
 */
public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
  super(resourceScheme, resourcePath);
  FST<Long> fst;
  try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {
    DataInput in = new InputStreamDataInput(is);
    fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
  }
  this.fst = new TokenInfoFST(fst);
}
 

/**
 * Reads an automaton from a file. 
 */
public static <T> FST<T> read(Path path, Outputs<T> outputs) throws IOException {
  try (InputStream is = Files.newInputStream(path)) {
    DataInput in = new InputStreamDataInput(new BufferedInputStream(is));
    return new FST<>(in, in, outputs);
  }
}
 

public FstDecompounder(InputStream inputStream, List<String> glue) throws IOException {
    try {
        this.surfaceForms = new FST<>(new InputStreamDataInput(inputStream), NoOutputs.getSingleton());
        // set up glue morphemes
        this.glueMorphemes = createGlueMorphemes(glue != null && glue.size() > 0 ? glue :morphemes);
    } finally {
        inputStream.close();
    }
}
 

public static Checkpoint read(Path path) throws IOException {
    try (InputStream in = Files.newInputStream(path)) {
        return new Checkpoint(new InputStreamDataInput(in));
    }
}
 

@Override
public void run() {
  boolean success = false;
  try {
    //node.message("using stream buffer size=" + bufferSize);
    InputStream is = new BufferedInputStream(socket.getInputStream(), bufferSize);
    DataInput in = new InputStreamDataInput(is);
    BufferedOutputStream bos = new BufferedOutputStream(socket.getOutputStream(), bufferSize);
    DataOutput out = new OutputStreamDataOutput(bos);

    if (node instanceof SimplePrimaryNode) {
      ((SimplePrimaryNode) node).handleOneConnection(random(), ss, stop, is, socket, in, out, bos);
    } else {
      ((SimpleReplicaNode) node).handleOneConnection(ss, stop, is, socket, in, out, bos);
    }

    bos.flush();
    if (Node.VERBOSE_CONNECTIONS) {
      node.message("bos.flush done");
    }

    success = true;
  } catch (Throwable t) {
    if (t instanceof SocketException == false && t instanceof NodeCommunicationException == false) {
      node.message("unexpected exception handling client connection; now failing test:");
      t.printStackTrace(System.out);
      IOUtils.closeWhileHandlingException(ss);
      // Test should fail with this:
      throw new RuntimeException(t);
    } else {
      node.message("exception handling client connection; ignoring:");
      t.printStackTrace(System.out);
    }
  } finally {
    if (success) {
      try {
        IOUtils.close(socket);
      } catch (IOException ioe) {
        throw new RuntimeException(ioe);
      }
    } else {
      IOUtils.closeWhileHandlingException(socket);
    }
  }
  if (Node.VERBOSE_CONNECTIONS) {
    node.message("socket.close done");
  }
}