org.apache.solr.client.solrj.io.Tuple Java Examples

private List<Future<Tuple>> callShards(List<String> baseUrls) throws IOException {

    List<Future<Tuple>> futures = new ArrayList<>();
    for (String baseUrl : baseUrls) {
      LogitCall lc = new LogitCall(baseUrl,
          this.params,
          this.field,
          this.terms,
          this.weights,
          this.outcome,
          this.positiveLabel,
          this.learningRate,
          this.iteration);

      Future<Tuple> future = executorService.submit(lc);
      futures.add(future);
    }

    return futures;
  }
 

public void testSimpleDeleteStreamInvalidCredentials() throws Exception {
  assertEquals(0,
               (setBasicAuthCredentials(new UpdateRequest(), WRITE_X_USER)
                .add(sdoc("id", "42"))
                .commit(cluster.getSolrClient(), COLLECTION_X)).getStatus());
  assertEquals(1L, commitAndCountDocsInCollection(COLLECTION_X, WRITE_X_USER));
  
  final SolrStream solrStream = new SolrStream(solrUrl + "/" + COLLECTION_X,
                                               params("qt", "/stream", "expr",
                                                      "update("+COLLECTION_X+",batchSize=1," +
                                                      "tuple(id=42))"));
  // "WRITE" credentials should be required for 'update(...)'
  solrStream.setCredentials(WRITE_X_USER, "BOGUS_PASSWORD");
  
  // NOTE: Can't make any assertions about Exception: SOLR-14226
  expectThrows(Exception.class, () -> {
      final List<Tuple> ignored = getTuples(solrStream);
    });
  
  assertEquals(1L, commitAndCountDocsInCollection(COLLECTION_X, WRITE_X_USER));
}
 

protected Tuple _read() throws IOException {
  TupleWrapper tw = tuples.pollFirst();
  if(tw != null) {
    Tuple t = tw.getTuple();

    if (trace) {
      t.put("_COLLECTION_", this.collection);
    }

    if(tw.next()) {
      tuples.add(tw);
    }
    return t;
  } else {
    Tuple tuple = Tuple.EOF();
    if(trace) {
      tuple.put("_COLLECTION_", this.collection);
    }
    return tuple;
  }
}
 

public void testIndirectUpdateStreamInsufficientCredentials() throws Exception {
  
  // regardless of how it's routed, WRITE_Y should NOT have authz to stream updates to X...
  for (String path : Arrays.asList(COLLECTION_X, COLLECTION_Y)) {
    final SolrStream solrStream = new SolrStream(solrUrl + "/" + path,
                                                 params("qt", "/stream", "expr",
                                                        "update("+COLLECTION_X+",batchSize=1," +
                                                        "tuple(id=42,a_i=1,b_i=5))"));
    solrStream.setCredentials(WRITE_Y_USER, WRITE_Y_USER);
  
    // NOTE: Can't make any assertions about Exception: SOLR-14226
    expectThrows(Exception.class, () -> {
        final List<Tuple> ignored = getTuples(solrStream);
      });
  }

  assertEquals(0L, commitAndCountDocsInCollection(COLLECTION_X, WRITE_X_USER));
}
 

@Test//(expected = NumberFormatException.class)
public void addTwoFieldsWithMissingField() throws Exception{
  StreamEvaluator evaluator = factory.constructEvaluator("add(a,b)");
  Object result;
  
  values.clear();
  values.put("a", 1);
  result = evaluator.evaluate(new Tuple(values));
  Assert.assertNull(result);
  
  values.clear();
  values.put("a", 1.1);
  result = evaluator.evaluate(new Tuple(values));
  Assert.assertNull(result);
  
  values.clear();
  values.put("b", 1.1);    
  result = evaluator.evaluate(new Tuple(values));
  Assert.assertNull(result);
}
 

@Test
public void testSearchBacktick() throws Exception {
  UpdateRequest updateRequest = new UpdateRequest();
  updateRequest.add(id, "hello", "test_t", "l b c d c e");
  updateRequest.add(id, "hello1", "test_t", "l b c d c");
  updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);

  String expr = "search("+COLLECTIONORALIAS+", q=\"`c d c e`\", fl=\"id,test_t\", sort=\"id desc\")";

  ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
  paramsLoc.set("expr", expr);
  paramsLoc.set("qt", "/stream");

  String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
  TupleStream solrStream = new SolrStream(url, paramsLoc);

  StreamContext context = new StreamContext();
  solrStream.setStreamContext(context);
  List<Tuple> tuples = getTuples(solrStream);
  assertTrue(tuples.size() == 1);
  Tuple tuple = tuples.get(0);
  assertTrue(tuple.get("id").equals("hello"));
  assertTrue(tuple.get("test_t").equals("l b c d c e"));
}
 

@Test
public void testCatStreamMultipleFilesOneEmpty() throws Exception {
  final String catStream = "cat(\"topLevel1.txt,topLevel-empty.txt\")";
  ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
  paramsLoc.set("expr", catStream);
  paramsLoc.set("qt", "/stream");
  String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+FILESTREAM_COLLECTION;

  SolrStream solrStream = new SolrStream(url, paramsLoc);

  StreamContext context = new StreamContext();
  solrStream.setStreamContext(context);
  List<Tuple> tuples = getTuples(solrStream);

  assertEquals(4, tuples.size());

  for (int i = 0; i < 4; i++) {
    Tuple t = tuples.get(i);
    assertEquals("topLevel1.txt line " + String.valueOf(i+1), t.get("line"));
    assertEquals("topLevel1.txt", t.get("file"));
  }
}
 

private void checkDaemonKilled(String daemon) throws IOException, InterruptedException {
  TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);

  while (timeout.hasTimedOut() == false) {
    List<Tuple> tuples = getTuples(TestSQLHandler.mapParams("qt", "/stream", "action", "list"));
    Boolean foundIt = false;
    for (Tuple tuple : tuples) {
      if (tuple.get("id").equals(daemon)) {
        foundIt = true;
      }
    }
    if (foundIt == false) return;
    TimeUnit.MILLISECONDS.sleep(100);
  }
  fail("'" + daemonOfInterest + "' did not disappear in 10 seconds");
}
 

@Test
public void arrayLongSortDescTest() throws IOException{
  StreamEvaluator evaluator = factory.constructEvaluator("array(a,b,c, sort=desc)");
  StreamContext context = new StreamContext();
  evaluator.setStreamContext(context);
  Object result;
  
  values.put("a", 1L);
  values.put("b", 3L);
  values.put("c", 2L);
  
  result = evaluator.evaluate(new Tuple(values));
  
  Assert.assertTrue(result instanceof List<?>);
  
  Assert.assertEquals(3, ((List<?>)result).size());
  Assert.assertEquals(3D, ((List<?>)result).get(0));
  Assert.assertEquals(2D, ((List<?>)result).get(1));
  Assert.assertEquals(1D, ((List<?>)result).get(2));
}
 

public void testSimpleDeleteStream() throws Exception {
  assertEquals(0,
               (setBasicAuthCredentials(new UpdateRequest(), WRITE_X_USER)
                .add(sdoc("id", "42"))
                .commit(cluster.getSolrClient(), COLLECTION_X)).getStatus());
  assertEquals(1L, commitAndCountDocsInCollection(COLLECTION_X, WRITE_X_USER));
  
  final SolrStream solrStream = new SolrStream(solrUrl + "/" + COLLECTION_X,
                                               params("qt", "/stream", "expr",
                                                      "delete("+COLLECTION_X+",batchSize=1," +
                                                      "tuple(id=42))"));
  solrStream.setCredentials(WRITE_X_USER, WRITE_X_USER);
  final List<Tuple> tuples = getTuples(solrStream);
  assertEquals(1, tuples.size());
  assertEquals(1L, tuples.get(0).get("totalIndexed"));
  
  assertEquals(0L, commitAndCountDocsInCollection(COLLECTION_X, WRITE_X_USER));

}
 

public void testIndirectDeleteStreamInsufficientCredentials() throws Exception {
  assertEquals(0,
               (setBasicAuthCredentials(new UpdateRequest(), WRITE_X_USER)
                .add(sdoc("id", "42"))
                .commit(cluster.getSolrClient(), COLLECTION_X)).getStatus());
  assertEquals(1L, commitAndCountDocsInCollection(COLLECTION_X, WRITE_X_USER));
  
  // regardless of how it's routed, WRITE_Y should NOT have authz to delete from X...
  for (String path : Arrays.asList(COLLECTION_X, COLLECTION_Y)) {
    final SolrStream solrStream = new SolrStream(solrUrl + "/" + path,
                                                 params("qt", "/stream", "expr",
                                                        "delete("+COLLECTION_X+",batchSize=1," +
                                                        "tuple(id=42))"));
    solrStream.setCredentials(WRITE_Y_USER, WRITE_Y_USER);
  
    // NOTE: Can't make any assertions about Exception: SOLR-14226
    expectThrows(Exception.class, () -> {
        final List<Tuple> ignored = getTuples(solrStream);
      });
  }

  assertEquals(1L, commitAndCountDocsInCollection(COLLECTION_X, WRITE_X_USER));
}
 

@Override
public Object evaluate(Tuple tuple) throws IOException {    
  try{
    
    Object firstLevel = containedEvaluators.get(0).evaluate(tuple);
    
    if(!(firstLevel instanceof List<?>) || ((List<?>) firstLevel).stream().anyMatch(value -> !(value instanceof Tuple))){
      throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting a list of tuples but found %s", toExpression(constructingFactory), firstLevel.getClass().getSimpleName()));
    }

    List<Object> column = new ArrayList<>();
    for(Object innerTuple : (List<?>)firstLevel){
      column.add(containedEvaluators.get(1).evaluate((Tuple)innerTuple));
    }

    return normalizeOutputType(column);
  }
  catch(UncheckedIOException e){
    throw e.getCause();
  }
}
 

@Test
public void testCatStreamSingleFile() throws Exception {
  final String catStream = "cat(\"topLevel1.txt\")";
  ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
  paramsLoc.set("expr", catStream);
  paramsLoc.set("qt", "/stream");
  String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+FILESTREAM_COLLECTION;

  SolrStream solrStream = new SolrStream(url, paramsLoc);

  StreamContext context = new StreamContext();
  solrStream.setStreamContext(context);
  List<Tuple> tuples = getTuples(solrStream);
  assertEquals(4, tuples.size());

  for (int i = 0; i < 4; i++) {
    Tuple t = tuples.get(i);
    assertEquals("topLevel1.txt line " + String.valueOf(i+1), t.get("line"));
    assertEquals("topLevel1.txt", t.get("file"));
  }
}
 

protected List<Tuple> getTuples(TupleStream tupleStream) throws IOException {
  tupleStream.open();
  List<Tuple> tuples = new ArrayList<>();
  for(;;) {
    Tuple t = tupleStream.read();
    if(t.EOF) {
      break;
    } else {
      tuples.add(t);
    }
  }
  tupleStream.close();
  return tuples;
}
 

public void operate(Tuple tuple) {
  if(priorityQueue.size() >= size) {
    Tuple peek = priorityQueue.peek();
    if(streamComparator.compare(tuple, peek) < 0) {
      priorityQueue.poll();
      priorityQueue.add(tuple);
    }
  } else {
    priorityQueue.add(tuple);
  }
}
 

public void testEchoStreamInvalidCredentials() throws Exception {
  final SolrStream solrStream = new SolrStream(solrUrl + "/" + COLLECTION_X,
                                               params("qt", "/stream", 
                                                      "expr", "echo(hello world)"));
  solrStream.setCredentials(READ_ONLY_USER, "BOGUS_PASSWORD");
  
  // NOTE: Can't make any assertions about Exception: SOLR-14226
  expectThrows(Exception.class, () -> {
      final List<Tuple> ignored = getTuples(solrStream);
    });
}
 

@SuppressWarnings({"unchecked"})
public Tuple read() throws IOException {
  Tuple tuple = this.tupleStream.read();
  if(tuple.EOF) {
    long totalTime = (System.nanoTime() - begin) / 1000000;
    tuple.put(StreamParams.RESPONSE_TIME, totalTime);
  }
  return tuple;
}
 

@Test
public void testParallelUniqueStream() throws Exception {

  new UpdateRequest()
      .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "0")
      .add(id, "2", "a_s", "hello2", "a_i", "2", "a_f", "0")
      .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
      .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
      .add(id, "1", "a_s", "hello1", "a_i", "1", "a_f", "1")
      .add(id, "5", "a_s", "hello1", "a_i", "10", "a_f", "1")
      .add(id, "6", "a_s", "hello1", "a_i", "11", "a_f", "5")
      .add(id, "7", "a_s", "hello1", "a_i", "12", "a_f", "5")
      .add(id, "8", "a_s", "hello1", "a_i", "13", "a_f", "4")
      .commit(cluster.getSolrClient(), COLLECTIONORALIAS);

  StreamContext streamContext = new StreamContext();
  SolrClientCache solrClientCache = new SolrClientCache();
  streamContext.setSolrClientCache(solrClientCache);

  try {

    SolrParams sParams = mapParams("q", "*:*", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i asc", "partitionKeys", "a_f", "qt", "/export");
    CloudSolrStream stream = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParams);
    UniqueStream ustream = new UniqueStream(stream, new FieldEqualitor("a_f"));
    ParallelStream pstream = parallelStream(ustream, new FieldComparator("a_f", ComparatorOrder.ASCENDING));
    attachStreamFactory(pstream);
    pstream.setStreamContext(streamContext);
    List<Tuple> tuples = getTuples(pstream);
    assertEquals(5, tuples.size());
    assertOrder(tuples, 0, 1, 3, 4, 6);

    //Test the eofTuples

    Map<String, Tuple> eofTuples = pstream.getEofTuples();
    assertEquals(numWorkers, eofTuples.size()); //There should be an EOF tuple for each worker.
  }finally {
    solrClientCache.close();
  }

}
 

public Tuple read() throws IOException {
  while(true) {
    Tuple tuple = stream.read();
    if(tuple.EOF) {
      return tuple;
    }

    streamContext.getTupleContext().clear();
    if((boolean)evaluator.evaluate(tuple)){
      return tuple;
    }
  }
}
 

protected boolean assertOrder(List<Tuple> tuples, int... ids) throws Exception {
  int i = 0;
  for(int val : ids) {
    Tuple t = tuples.get(i);
    String tip = (String)t.get("id");
    String valStr = Integer.toString(val);
    if(!tip.equals(valStr)) {
      assertEquals("Found value:"+tip+" expecting:"+valStr, val, tip);
    }
    ++i;
  }
  return true;
}
 

protected List<Tuple> getTuples(TupleStream tupleStream) throws IOException {
  tupleStream.open();
  List<Tuple> tuples = new ArrayList<>();
  for(;;) {
    Tuple t = tupleStream.read();
    if(t.EOF) {
      break;
    } else {
      tuples.add(t);
    }
  }
  tupleStream.close();
  return tuples;
}
 

@Test(expected = IOException.class)
public void lengthNullValue() throws Exception{
  StreamEvaluator evaluator = factory.constructEvaluator("length(a)");
  
  values.clear();
  values.put("a", null);
  Object result = evaluator.evaluate(new Tuple(values));
  assertNull(result);
}
 

public Tuple read() throws IOException {
  if(tuple.EOF) {
    return tuple;
  } else {
    Tuple t = tuple;
    tuple = EOFTuple;
    return t;
  }
}
 

@Test(expected = IOException.class)
public void gtDifferentTypes3() throws Exception{
  StreamEvaluator evaluator = factory.constructEvaluator("gt(a,b)");
  
  values.clear();
  values.put("a", "1");
  values.put("b",1);
  evaluator.evaluate(new Tuple(values));
}
 

@Test
public void testJDBCJoin() throws Exception {
  
  // Load Database Data
  try (Connection connection = DriverManager.getConnection("jdbc:hsqldb:mem:.");
        Statement statement = connection.createStatement()) {
    statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('US', 'United States')");
    statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('NL', 'Netherlands')");
    statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('NP', 'Nepal')");
    statement.executeUpdate("insert into COUNTRIES (CODE,COUNTRY_NAME) values ('NO', 'Norway')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (11,'Emma','NL')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (12,'Grace','NI')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (13,'Hailey','NG')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (14,'Isabella','NF')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (15,'Lily','NE')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (16,'Madison','NC')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (17,'Mia','NL')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (18,'Natalie','NZ')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (19,'Olivia','NL')");
    statement.executeUpdate("insert into PEOPLE (ID, NAME, COUNTRY_CODE) values (20,'Samantha','NR')");
  }
  
  TupleStream stream;
  List<Tuple> tuples;
  
  // Simple 1
  stream = new JDBCStream("jdbc:hsqldb:mem:.", "select PEOPLE.ID, PEOPLE.NAME, COUNTRIES.COUNTRY_NAME from PEOPLE inner join COUNTRIES on PEOPLE.COUNTRY_CODE = COUNTRIES.CODE where COUNTRIES.CODE = 'NL' order by PEOPLE.ID", new FieldComparator("ID", ComparatorOrder.ASCENDING));
  tuples = getTuples(stream);
  
  assertEquals(3, tuples.size());
  assertOrderOf(tuples, "ID", 11, 17, 19);
  assertOrderOf(tuples, "NAME", "Emma", "Mia", "Olivia");    
}
 

protected INDArray getInputsFromTuple(Tuple tuple) {
  final double[] inputs = new double[inputKeys.length];
  for (int ii=0; ii<inputKeys.length; ++ii)
  {
    inputs[ii] = tuple.getDouble(inputKeys[ii]).doubleValue();
  }
  return Nd4j.create(new double[][]{ inputs });
}
 

public Tuple read() throws IOException {

    while(true) {
      Tuple t = stream.read();

      if(t.EOF) {
       if(needsReduce) {
         stream.pushBack(t);
         needsReduce = false;
         return op.reduce();
       } else {
         return t;
       }
      }

      if(currentGroupHead == null) {
        currentGroupHead = t;
        op.operate(t);
        needsReduce = true;
      } else {
        if(eq.test(currentGroupHead, t)) {
          op.operate(t);
          needsReduce = true;
        } else {
          stream.pushBack(t);
          currentGroupHead = null;
          needsReduce = false;
          return op.reduce();
        }
      }
    }
  }
 

@Test
public void test() throws IOException {
  double[] l1 = new double[] {3.4, 6.7, 4.5};
  
  values.clear();
  values.put("l1", l1);

  @SuppressWarnings({"rawtypes"})
  List result = ((List<?>)factory.constructEvaluator("reverse(l1)").evaluate(new Tuple(values)));

  Assert.assertEquals(4.5, result.get(0));
  Assert.assertEquals(6.7, result.get(1));
  Assert.assertEquals(3.4, result.get(2));
}
 

private void getTuples(@SuppressWarnings({"rawtypes"})NamedList response,
                              Bucket[] buckets,
                              Metric[] metrics) {

  Tuple tuple = new Tuple();
  @SuppressWarnings({"rawtypes"})
  NamedList facets = (NamedList)response.get("facets");
  fillTuples(0,
             tuples,
             tuple,
             facets,
             buckets,
             metrics);

}
 

public Tuple read() throws IOException {
  Tuple tuple = originalStream.read();
  ++lineNumber;
  if(tuple.EOF) {
    return tuple;
  } else {
    String file = formatFile(tuple.getString("file"));
    String line = tuple.getString("line");
    if (file.equals(currentFile)) {
      String[] fields = split(line);
      if(fields.length != headers.length) {
        throw new IOException("Headers and lines must have the same number of fields [file:"+file+" line number:"+lineNumber+"]");
      }
      Tuple out = new Tuple();
      out.put("id", file+"_"+lineNumber);
      for(int i=0; i<headers.length; i++) {
        if(fields[i] != null && fields[i].length() > 0) {
          out.put(headers[i], fields[i]);
        }
      }
      return out;
    } else {
      this.currentFile = file;
      this.headers = split(line);
      this.lineNumber = 1; //New file so reset the lineNumber
      return read();
    }
  }
}