Java Code Examples for org.apache.arrow.vector.ValueVector#getObject()

private List<List<String>> getOutput(List<QueryDataBatch> batches) throws SchemaChangeException {
  List<List<String>> output = new ArrayList<>();
  RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
  int last = 0;
  for(QueryDataBatch batch : batches) {
    int rows = batch.getHeader().getRowCount();
    if(batch.getData() != null) {
      loader.load(batch.getHeader().getDef(), batch.getData());
      // TODO:  Clean:  DRILL-2933:  That load(...) no longer throws
      // SchemaChangeException, so check/clean throws clause above.
      for (int i = 0; i < rows; ++i) {
        output.add(new ArrayList<String>());
        for (VectorWrapper<?> vw: loader) {
          ValueVector vv = vw.getValueVector();
          Object o = vv.getObject(i);
          output.get(last).add(o == null ? null: o.toString());
        }
        ++last;
      }
    }
    loader.clear();
    batch.release();
  }
  return output;
}
 

@SuppressWarnings("unchecked")
private <T> void assertField(ValueVector valueVector, int index,
    TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {

  if (expectedMinorType == TypeProtos.MinorType.STRUCT) {
    return;
  }

  final T val;
  try {
    val = (T) valueVector.getObject(index);
  } catch (Throwable ex) {
    throw ex;
  }

  if (val instanceof byte[]) {
    assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
  } else {
    assertEquals(value, val);
  }
}
 

public void printColumnMajor(ValueVector vv) {
  if (ParquetRecordReaderTest.VERBOSE_DEBUG){
    System.out.println("\n" + vv.getField().getName());
  }
  for (int j = 0; j < vv.getValueCount(); j++) {
    if (ParquetRecordReaderTest.VERBOSE_DEBUG){
      Object o = vv.getObject(j);
      if (o instanceof byte[]) {
        try {
          o = new String((byte[])o, "UTF-8");
        } catch (UnsupportedEncodingException e) {
          throw new RuntimeException(e);
        }
      }
      System.out.print(Strings.padStart(o + "", 20, ' ') + " ");
      System.out.print(", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
    }
  }
  if (ParquetRecordReaderTest.VERBOSE_DEBUG) {
    System.out.println("\n" + vv.getValueCount());
  }
}
 

private static void printBatchNoSV(VectorAccessible batch) {
  List<String> columns = Lists.newArrayList();
  List<ValueVector> vectors = Lists.newArrayList();
  for (VectorWrapper<?> vw : batch) {
    columns.add(vw.getValueVector().getField().getName());
    vectors.add(vw.getValueVector());
  }
  int width = columns.size();
  int rows = batch.getRecordCount();
  rows = Math.min(ROWS_TO_PRINT, rows);
  for (int row = 0; row < rows; row++) {
    if (row%50 == 0) {
      System.out.println(StringUtils.repeat("-", width * 52 + 1));
      for (String column : columns) {
        System.out.printf("| %-50s", width <= 50 ? column : column.substring(0, Math.min(column.length(), 49)));
      }
      System.out.printf("|\n");
      System.out.println(StringUtils.repeat("-", width*52 + 1));
    }
    for (ValueVector vv : vectors) {
      Object o = vv.getObject(row);
      String value;
      if (o == null) {
        value = "null";
      } else
      if (o instanceof byte[]) {
        value = new String((byte[]) o);
      } else {
        value = o.toString();
      }
      System.out.printf("| %-50s",value.length() <= 50 ? value : value.substring(0, 49));
    }
    System.out.printf("|\n");
  }
}
 

private static void printBatchSV2(VectorAccessible batch, SelectionVector2 sv2) {
  List<String> columns = Lists.newArrayList();
  List<ValueVector> vectors = Lists.newArrayList();
  for (VectorWrapper<?> vw : batch) {
    columns.add(vw.getValueVector().getField().getName());
    vectors.add(vw.getValueVector());
  }
  int width = columns.size();
  int rows = batch.getRecordCount();
  rows = Math.min(ROWS_TO_PRINT, rows);
  for (int i = 0; i < rows; i++) {
    if (i%50 == 0) {
      System.out.println(StringUtils.repeat("-", width * 32 + 1));
      for (String column : columns) {
        System.out.printf("| %-30s", width <= 30 ? column : column.substring(0, 29));
      }
      System.out.printf("|\n");
      System.out.println(StringUtils.repeat("-", width*32 + 1));
    }
    int row = sv2.getIndex(i);
    for (ValueVector vv : vectors) {
      Object o = vv.getObject(row);
      String value;
      if (o == null) {
        value = "null";
      } else
      if (o instanceof byte[]) {
        value = new String((byte[]) o);
      } else {
        value = o.toString();
      }
      System.out.printf("| %-30s",value.length() <= 30 ? value : value.substring(0, 29));
    }
    System.out.printf("|\n");
  }
}
 

@Override
public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) {
  QueryData queryHeader = result.getHeader();
  int rows = queryHeader.getRowCount();
  try {
    if ( result.hasData() ) {
      ArrowBuf data = result.getData();
      loader.load(queryHeader.getDef(), data);
      for (int i = 0; i < rows; i++) {
         Map<String,String> record = Maps.newHashMap();
        for (VectorWrapper<?> vw : loader) {
          final String field = vw.getValueVector().getField().getName();
          final ValueVector vv = vw.getValueVector();
          final Object value = i < vv.getValueCount() ? vv.getObject(i) : null;
          final String display = value == null ? null : value.toString();
          record.put(field, display);
        }
        records.add(record);
      }
      loader.clear();
    }
    result.release();
  } catch (SchemaChangeException e) {
    fail(e.getMessage());
  }

}
 

@Test
  @Ignore
  public void testParseParquetPhysicalPlan() throws Exception {
    try (ClusterCoordinator clusterCoordinator = LocalClusterCoordinator.newRunningCoordinator();
         SabotNode bit1 = new SabotNode(DEFAULT_SABOT_CONFIG, clusterCoordinator, CLASSPATH_SCAN_RESULT, true); DremioClient client = new DremioClient(DEFAULT_SABOT_CONFIG, clusterCoordinator)) {
      bit1.run();
      client.connect();
      List<QueryDataBatch> results = client.runQuery(com.dremio.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
      RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
      int count = 0;
      for (QueryDataBatch b : results) {
        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
        count += b.getHeader().getRowCount();
        loader.load(b.getHeader().getDef(), b.getData());
        for (VectorWrapper vw : loader) {
          System.out.print(vw.getValueVector().getField().getName() + ": ");
          ValueVector vv = vw.getValueVector();
          for (int i = 0; i < vv.getValueCount(); i++) {
            Object o = vv.getObject(i);
            if (o instanceof byte[]) {
              System.out.print(" [" + new String((byte[]) o) + "]");
            } else {
              System.out.print(" [" + vv.getObject(i) + "]");
            }
//            break;
          }
          System.out.println();
        }
        loader.clear();
        b.release();
      }
      client.close();
      System.out.println(String.format("Got %d total results", count));
    }
  }
 

protected static String getValueInFirstRecord(String sql, String columnName)
    throws Exception {
  final List<QueryDataBatch> results = testSqlWithResults(sql);
  final RecordBatchLoader loader = new RecordBatchLoader(getSabotContext().getAllocator());
  final StringBuilder builder = new StringBuilder();
  final boolean silent = config != null && config.getBoolean(QueryTestUtil.TEST_QUERY_PRINTING_SILENT);

  for (final QueryDataBatch b : results) {
    if (!b.hasData()) {
      continue;
    }

    loader.load(b.getHeader().getDef(), b.getData());

    final VectorWrapper<?> vw;
    try {
        vw = loader.getValueAccessorById(
            VarCharVector.class,
            loader.getValueVectorId(SchemaPath.getSimplePath(columnName)).getFieldIds());
    } catch (Throwable t) {
      throw new Exception("Looks like you did not provide an explain plan query, please add EXPLAIN PLAN FOR to the beginning of your query.");
    }

    if (!silent) {
      System.out.println(vw.getValueVector().getField().getName());
    }
    final ValueVector vv = vw.getValueVector();
    for (int i = 0; i < vv.getValueCount(); i++) {
      final Object o = vv.getObject(i);
      builder.append(o);
      if (!silent) {
        System.out.println(o);
      }
    }
    loader.clear();
    b.release();
  }

  return builder.toString();
}
 

@Test
public void testMultipleProvidersMixedSizes() throws Exception {
  try (final ClusterCoordinator clusterCoordinator = LocalClusterCoordinator.newRunningCoordinator();
       final SabotNode bit1 = new SabotNode(DEFAULT_SABOT_CONFIG, clusterCoordinator, CLASSPATH_SCAN_RESULT, true);
       final SabotNode bit2 = new SabotNode(DEFAULT_SABOT_CONFIG, clusterCoordinator, CLASSPATH_SCAN_RESULT, false);
       final DremioClient client = new DremioClient(DEFAULT_SABOT_CONFIG, clusterCoordinator);) {

    bit1.run();
    bit2.run();
    client.connect();
    final List<QueryDataBatch> results =
        client.runQuery(com.dremio.exec.proto.UserBitShared.QueryType.PHYSICAL,
            Files.toString(FileUtils.getResourceAsFile("/mergerecv/multiple_providers.json"),
                Charsets.UTF_8));
    int count = 0;
    final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getRecordAllocator());
    // print the results
    Long lastBlueValue = null;
    for (final QueryDataBatch b : results) {
      final QueryData queryData = b.getHeader();
      final int batchRowCount = queryData.getRowCount();
      count += batchRowCount;
      batchLoader.load(queryData.getDef(), b.getData());
      for (final VectorWrapper vw : batchLoader) {
        final ValueVector vv = vw.getValueVector();
        final Field materializedField = vv.getField();
        final int numValues = vv.getValueCount();
        for(int valueIdx = 0; valueIdx < numValues; ++valueIdx) {
          if (materializedField.getName().equals("blue")) {
            Long val = (Long) vv.getObject(valueIdx);
            if (val != null) {
              final long longValue = ((Long) vv.getObject(valueIdx)).longValue();
              // check that order is ascending
              if (lastBlueValue != null) {
                assertTrue(longValue >= lastBlueValue);
              }
              lastBlueValue = longValue;
            }
          }
        }
      }
      b.release();
      batchLoader.clear();
    }
    assertEquals(400000, count);
  }
}