Java Code Examples for org.apache.spark.sql.catalyst.InternalRow#isNullAt()

public UnsafeRow apply(InternalRow i) {
  holder.reset();

  rowWriter.zeroOutNullBytes();


  boolean isNull = i.isNullAt(0);
  long value = isNull ? -1L : (i.getLong(0));
  if (isNull) {
    rowWriter.setNullAt(0);
  } else {
    rowWriter.write(0, value);
  }


  boolean isNull1 = i.isNullAt(1);
  UTF8String value1 = isNull1 ? null : (i.getUTF8String(1));
  if (isNull1) {
    rowWriter.setNullAt(1);
  } else {
    rowWriter.write(1, value1);
  }
  result.setTotalSize(holder.totalSize());
  return result;
}
 

public static void assertEqualsBatch(Types.StructType struct, Iterator<Record> expected, ColumnarBatch batch,
                                     boolean checkArrowValidityVector) {
  for (int rowId = 0; rowId < batch.numRows(); rowId++) {
    List<Types.NestedField> fields = struct.fields();
    InternalRow row = batch.getRow(rowId);
    Record rec = expected.next();
    for (int i = 0; i < fields.size(); i += 1) {
      Type fieldType = fields.get(i).type();
      Object expectedValue = rec.get(i);
      Object actualValue = row.isNullAt(i) ? null : row.get(i, convert(fieldType));
      assertEqualsUnsafe(fieldType, expectedValue, actualValue);

      if (checkArrowValidityVector) {
        ColumnVector columnVector = batch.column(i);
        ValueVector arrowVector = ((IcebergArrowColumnVector) columnVector).vectorAccessor().getVector();
        Assert.assertEquals("Nullability doesn't match", expectedValue == null, arrowVector.isNull(rowId));
      }
    }
  }
}
 

@Override
public Object get(InternalRow row) {
  if (row.isNullAt(position)) {
    return null;
  }
  return row.get(position, type);
}
 

@Override
public Object get(InternalRow row) {
  if (row.isNullAt(p)) {
    return null;
  }
  return ((Decimal) row.get(p, type)).toJavaBigDecimal();
}
 

@Override
public Object get(InternalRow row) {
  if (row.isNullAt(p)) {
    return null;
  }
  return row.get(p, type).toString();
}
 

@Override
public Object get(InternalRow row) {
  if (row.isNullAt(p)) {
    return null;
  }
  return row.get(p, type);
}
 

@Override
public void write(InternalRow row, Encoder encoder) throws IOException {
  for (int i = 0; i < types.length; i += 1) {
    if (row.isNullAt(i)) {
      writers[i].write(null, encoder);
    } else {
      write(row, i, writers[i], encoder);
    }
  }
}
 

public static void assertEqualsUnsafe(Types.StructType struct, Record rec, InternalRow row) {
  List<Types.NestedField> fields = struct.fields();
  for (int i = 0; i < fields.size(); i += 1) {
    Type fieldType = fields.get(i).type();

    Object expectedValue = rec.get(i);
    Object actualValue = row.isNullAt(i) ? null : row.get(i, convert(fieldType));

    assertEqualsUnsafe(fieldType, expectedValue, actualValue);
  }
}
 

@Override
public Object get(InternalRow row) {
  if (row.isNullAt(position())) {
    return null;
  }
  return ByteBuffer.wrap((byte[]) row.get(position(), type()));
}
 

@Override
public Object get(InternalRow row) {
  if (row.isNullAt(position())) {
    return null;
  }
  return ((Decimal) row.get(position(), type())).toJavaBigDecimal();
}
 

@Override
public Object get(InternalRow row) {
  if (row.isNullAt(position())) {
    return null;
  }
  return row.get(position(), type()).toString();
}
 

@Override
public void write(InternalRow row, Encoder encoder) throws IOException {
  for (int i = 0; i < types.length; i += 1) {
    if (row.isNullAt(i)) {
      writers[i].write(null, encoder);
    } else {
      write(row, i, writers[i], encoder);
    }
  }
}
 

private static Object getValue(SpecializedGetters container, int ord,
                               Type type) {
  if (container.isNullAt(ord)) {
    return null;
  }
  switch (type.typeId()) {
    case BOOLEAN:
      return container.getBoolean(ord);
    case INTEGER:
      return container.getInt(ord);
    case LONG:
      return container.getLong(ord);
    case FLOAT:
      return container.getFloat(ord);
    case DOUBLE:
      return container.getDouble(ord);
    case STRING:
      return container.getUTF8String(ord).toString();
    case BINARY:
    case FIXED:
    case UUID:
      return container.getBinary(ord);
    case DATE:
      return new DateWritable(container.getInt(ord)).get();
    case TIMESTAMP:
      return DateTimeUtils.toJavaTimestamp(container.getLong(ord));
    case DECIMAL: {
      Types.DecimalType dt = (Types.DecimalType) type;
      return container.getDecimal(ord, dt.precision(), dt.scale()).toJavaBigDecimal();
    }
    case STRUCT:
      Types.StructType struct = type.asStructType();
      InternalRow internalRow = container.getStruct(ord, struct.fields().size());
      Object[] data = new Object[struct.fields().size()];
      for (int i = 0; i < data.length; i += 1) {
        if (internalRow.isNullAt(i)) {
          data[i] = null;
        } else {
          data[i] = getValue(internalRow, i, struct.fields().get(i).type());
        }
      }
      return new GenericRow(data);
    default:
      throw new IllegalArgumentException("Unhandled type " + type);
  }
}
 

public UnsafeRow apply(InternalRow i) {
  holder.reset();

  rowWriter.zeroOutNullBytes();


  boolean isNull = i.isNullAt(0);
  InternalRow value = isNull ? null : (i.getStruct(0, 1));
  if (isNull) {
    rowWriter.setNullAt(0);
  } else {
    // Remember the current cursor so that we can calculate how many bytes are
    // written later.
    final int tmpCursor = holder.cursor;

    if (value instanceof UnsafeRow) {

      final int sizeInBytes = ((UnsafeRow) value).getSizeInBytes();
      // grow the global buffer before writing data.
      holder.grow(sizeInBytes);
      ((UnsafeRow) value).writeToMemory(holder.buffer, holder.cursor);
      holder.cursor += sizeInBytes;

    } else {
      rowWriter1.reset();


      boolean isNull1 = value.isNullAt(0);
      float value1 = isNull1 ? -1.0f : value.getFloat(0);

      if (isNull1) {
        rowWriter1.setNullAt(0);
      } else {
        rowWriter1.write(0, value1);
      }
    }

    rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
  }
  result.setTotalSize(holder.totalSize());
  return result;
}
 

public UnsafeRow apply(InternalRow i) {
  holder.reset();

  rowWriter.zeroOutNullBytes();


  boolean isNull = i.isNullAt(0);
  InternalRow value = isNull ? null : (i.getStruct(0, 2));
  if (isNull) {
    rowWriter.setNullAt(0);
  } else {
    // Remember the current cursor so that we can calculate how many bytes are
    // written later.
    final int tmpCursor = holder.cursor;

    if (value instanceof UnsafeRow) {

      final int sizeInBytes = ((UnsafeRow) value).getSizeInBytes();
      // grow the global buffer before writing data.
      holder.grow(sizeInBytes);
      ((UnsafeRow) value).writeToMemory(holder.buffer, holder.cursor);
      holder.cursor += sizeInBytes;

    } else {
      rowWriter1.reset();


      boolean isNull1 = value.isNullAt(0);
      float value1 = isNull1 ? -1.0f : value.getFloat(0);

      if (isNull1) {
        rowWriter1.setNullAt(0);
      } else {
        rowWriter1.write(0, value1);
      }


      boolean isNull2 = value.isNullAt(1);
      float value2 = isNull2 ? -1.0f : value.getFloat(1);

      if (isNull2) {
        rowWriter1.setNullAt(1);
      } else {
        rowWriter1.write(1, value2);
      }
    }

    rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
  }
  result.setTotalSize(holder.totalSize());
  return result;
}
 

public UnsafeRow apply(InternalRow i) {
  holder.reset();

  rowWriter.zeroOutNullBytes();


  boolean isNull = i.isNullAt(0);
  MapData value = isNull ? null : (i.getMap(0));
  if (isNull) {
    rowWriter.setNullAt(0);
  } else {
    // Remember the current cursor so that we can calculate how many bytes are
    // written later.
    final int tmpCursor = holder.cursor;

    if (value instanceof UnsafeMapData) {

      final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
      // grow the global buffer before writing data.
      holder.grow(sizeInBytes);
      ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
      holder.cursor += sizeInBytes;

    } else {
      final ArrayData keys = value.keyArray();
      final ArrayData values = value.valueArray();

      // preserve 8 bytes to write the key array numBytes later.
      holder.grow(8);
      holder.cursor += 8;

      // Remember the current cursor so that we can write numBytes of key array later.
      final int tmpCursor1 = holder.cursor;


      if (keys instanceof UnsafeArrayData) {

        final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
        // grow the global buffer before writing data.
        holder.grow(sizeInBytes1);
        ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
        holder.cursor += sizeInBytes1;

      } else {
        final int numElements = keys.numElements();
        arrayWriter.initialize(holder, numElements, 8);

        for (int index = 0; index < numElements; index++) {
          if (keys.isNullAt(index)) {
            arrayWriter.setNull(index);
          } else {
            final UTF8String element = keys.getUTF8String(index);
            arrayWriter.write(index, element);
          }
        }
      }

      // Write the numBytes of key array into the first 8 bytes.
      Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);


      if (values instanceof UnsafeArrayData) {

        final int sizeInBytes2 = ((UnsafeArrayData) values).getSizeInBytes();
        // grow the global buffer before writing data.
        holder.grow(sizeInBytes2);
        ((UnsafeArrayData) values).writeToMemory(holder.buffer, holder.cursor);
        holder.cursor += sizeInBytes2;

      } else {
        final int numElements1 = values.numElements();
        arrayWriter1.initialize(holder, numElements1, 8);

        for (int index1 = 0; index1 < numElements1; index1++) {
          if (values.isNullAt(index1)) {
            arrayWriter1.setNull(index1);
          } else {
            final UTF8String element1 = values.getUTF8String(index1);
            arrayWriter1.write(index1, element1);
          }
        }
      }

    }

    rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
  }
  result.setTotalSize(holder.totalSize());
  return result;
}
 

public UnsafeRow apply(InternalRow i) {
  holder.reset();

  rowWriter.zeroOutNullBytes();


  boolean isNull = i.isNullAt(0);
  ArrayData value = isNull ? null : (i.getArray(0));
  if (isNull) {
    rowWriter.setNullAt(0);
  } else {
    // Remember the current cursor so that we can calculate how many bytes are
    // written later.
    final int tmpCursor = holder.cursor;

    if (value instanceof UnsafeArrayData) {

      final int sizeInBytes1 = ((UnsafeArrayData) value).getSizeInBytes();
      // grow the global buffer before writing data.
      holder.grow(sizeInBytes1);
      ((UnsafeArrayData) value).writeToMemory(holder.buffer, holder.cursor);
      holder.cursor += sizeInBytes1;

    } else {
      final int numElements = value.numElements();
      arrayWriter.initialize(holder, numElements, 8);

      for (int index = 0; index < numElements; index++) {
        if (value.isNullAt(index)) {
          arrayWriter.setNull(index);
        } else {
          final InternalRow element = value.getStruct(index, 2);

          final int tmpCursor1 = holder.cursor;

          if (element instanceof UnsafeRow) {

            final int sizeInBytes = ((UnsafeRow) element).getSizeInBytes();
            // grow the global buffer before writing data.
            holder.grow(sizeInBytes);
            ((UnsafeRow) element).writeToMemory(holder.buffer, holder.cursor);
            holder.cursor += sizeInBytes;

          } else {
            rowWriter1.reset();


            boolean isNull1 = element.isNullAt(0);
            int value1 = isNull1 ? -1 : element.getInt(0);

            if (isNull1) {
              rowWriter1.setNullAt(0);
            } else {
              rowWriter1.write(0, value1);
            }


            boolean isNull2 = element.isNullAt(1);
            int value2 = isNull2 ? -1 : element.getInt(1);

            if (isNull2) {
              rowWriter1.setNullAt(1);
            } else {
              rowWriter1.write(1, value2);
            }
          }

          arrayWriter.setOffsetAndSize(index, tmpCursor1, holder.cursor - tmpCursor1);

        }
      }
    }

    rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
  }
  result.setTotalSize(holder.totalSize());
  return result;
}
 

public UnsafeRow apply(InternalRow i) {
  holder.reset();

  rowWriter.zeroOutNullBytes();


  boolean isNull = i.isNullAt(0);
  MapData value = isNull ? null : (i.getMap(0));
  if (isNull) {
    rowWriter.setNullAt(0);
  } else {
    // Remember the current cursor so that we can calculate how many bytes are
    // written later.
    final int tmpCursor = holder.cursor;

    if (value instanceof UnsafeMapData) {

      final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
      // grow the global buffer before writing data.
      holder.grow(sizeInBytes);
      ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
      holder.cursor += sizeInBytes;

    } else {
      final ArrayData keys = value.keyArray();
      final ArrayData values = value.valueArray();

      // preserve 8 bytes to write the key array numBytes later.
      holder.grow(8);
      holder.cursor += 8;

      // Remember the current cursor so that we can write numBytes of key array later.
      final int tmpCursor1 = holder.cursor;


      if (keys instanceof UnsafeArrayData) {

        final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
        // grow the global buffer before writing data.
        holder.grow(sizeInBytes1);
        ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
        holder.cursor += sizeInBytes1;

      } else {
        final int numElements = keys.numElements();
        arrayWriter.initialize(holder, numElements, 8);

        for (int index = 0; index < numElements; index++) {
          if (keys.isNullAt(index)) {
            arrayWriter.setNull(index);
          } else {
            final UTF8String element = keys.getUTF8String(index);
            arrayWriter.write(index, element);
          }
        }
      }

      // Write the numBytes of key array into the first 8 bytes.
      Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);


      if (values instanceof UnsafeArrayData) {

        final int sizeInBytes2 = ((UnsafeArrayData) values).getSizeInBytes();
        // grow the global buffer before writing data.
        holder.grow(sizeInBytes2);
        ((UnsafeArrayData) values).writeToMemory(holder.buffer, holder.cursor);
        holder.cursor += sizeInBytes2;

      } else {
        final int numElements1 = values.numElements();
        arrayWriter1.initialize(holder, numElements1, 8);

        for (int index1 = 0; index1 < numElements1; index1++) {
          if (values.isNullAt(index1)) {
            arrayWriter1.setNull(index1);
          } else {
            final UTF8String element1 = values.getUTF8String(index1);
            arrayWriter1.write(index1, element1);
          }
        }
      }

    }

    rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
  }
  result.setTotalSize(holder.totalSize());
  return result;
}
 

private static Object getValue(SpecializedGetters container, int ord,
                               Type type) {
  if (container.isNullAt(ord)) {
    return null;
  }
  switch (type.typeId()) {
    case BOOLEAN:
      return container.getBoolean(ord);
    case INTEGER:
      return container.getInt(ord);
    case LONG:
      return container.getLong(ord);
    case FLOAT:
      return container.getFloat(ord);
    case DOUBLE:
      return container.getDouble(ord);
    case STRING:
      return container.getUTF8String(ord).toString();
    case BINARY:
    case FIXED:
    case UUID:
      return container.getBinary(ord);
    case DATE:
      return new DateWritable(container.getInt(ord)).get();
    case TIMESTAMP:
      return DateTimeUtils.toJavaTimestamp(container.getLong(ord));
    case DECIMAL: {
      Types.DecimalType dt = (Types.DecimalType) type;
      return container.getDecimal(ord, dt.precision(), dt.scale()).toJavaBigDecimal();
    }
    case STRUCT:
      Types.StructType struct = type.asStructType();
      InternalRow internalRow = container.getStruct(ord, struct.fields().size());
      Object[] data = new Object[struct.fields().size()];
      for (int i = 0; i < data.length; i += 1) {
        if (internalRow.isNullAt(i)) {
          data[i] = null;
        } else {
          data[i] = getValue(internalRow, i, struct.fields().get(i).type());
        }
      }
      return new GenericRow(data);
    default:
      throw new IllegalArgumentException("Unhandled type " + type);
  }
}
 

public void write(InternalRow row) {
    for (int colId = 0; colId < sqlTypes.length; colId++) {
        SQLType type = sqlTypes[colId];
        switch (type) {
            case INT:
                if (row.isNullAt(colId)) {
                    rowBuilder.appendInt(0);
                } else {
                    rowBuilder.appendInt(row.getInt(colId));
                }
                break;
            case BIGINT:
                if (row.isNullAt(colId)) {
                    rowBuilder.appendLong(0L);
                } else {
                    rowBuilder.appendLong(row.getLong(colId));
                }
                break;
            case FLOAT:
                if (row.isNullAt(colId)) {
                    rowBuilder.appendFloat(0f);
                } else {
                    rowBuilder.appendFloat(row.getFloat(colId));
                }
                break;
            case DOUBLE:
                if (row.isNullAt(colId)) {
                    rowBuilder.appendDouble(0d);
                } else {
                    rowBuilder.appendDouble(row.getDouble(colId));
                }
                break;
            case VARCHAR:
                if (row.isNullAt(colId)) {
                    rowBuilder.appendString("");
                } else {
                    rowBuilder.appendUTF8String(row.getUTF8String(colId));
                }
                break;
            case DATE:
                if (row.isNullAt(colId)) {
                    rowBuilder.appendLong(0);
                } else {
                    int days = row.getInt(colId);
                    rowBuilder.appendLong(days * DateTimeUtil.MILLIS_PER_DAY);
                }
                break;
            case DATETIME:
                if (row.isNullAt(colId)) {
                    rowBuilder.appendLong(0);
                } else {
                    // microsecond in spark, indexr only have millisecond.
                    rowBuilder.appendLong(row.getLong(colId) / 1000);
                }
                break;
            default:
                throw new RuntimeException("can't recognize this type [" + type + "]");
        }
    }
    try {
        segmentGen.add(rowBuilder.buildAndReset());
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}