org.apache.pig.ResourceSchema.ResourceFieldSchema#setType

Source File: GenRandomData.java From spork with Apache License 2.0

6 votes

public static ResourceFieldSchema getFloatDataBagFieldSchema(int column) throws IOException {
    ResourceFieldSchema intfs = new ResourceFieldSchema();
    intfs.setType(DataType.INTEGER);
    
    ResourceSchema tupleSchema = new ResourceSchema();
    ResourceFieldSchema[] fss = new ResourceFieldSchema[column];
    for (int i=0;i<column;i++) {
        fss[i] = intfs;
    }
    tupleSchema.setFields(fss);
    ResourceFieldSchema tuplefs = new ResourceFieldSchema();
    tuplefs.setSchema(tupleSchema);
    tuplefs.setType(DataType.TUPLE);
    
    ResourceSchema bagSchema = new ResourceSchema();
    bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
    ResourceFieldSchema bagfs = new ResourceFieldSchema();
    bagfs.setSchema(bagSchema);
    bagfs.setType(DataType.BAG);
    
    return bagfs;
}

Source File: GenRandomData.java From spork with Apache License 2.0

6 votes

public static ResourceFieldSchema getMixedTupleToConvertFieldSchema() throws IOException {
    ResourceFieldSchema stringfs = new ResourceFieldSchema();
    stringfs.setType(DataType.CHARARRAY);
    ResourceFieldSchema intfs = new ResourceFieldSchema();
    intfs.setType(DataType.INTEGER);
    ResourceFieldSchema longfs = new ResourceFieldSchema();
    longfs.setType(DataType.LONG);
    ResourceFieldSchema floatfs = new ResourceFieldSchema();
    floatfs.setType(DataType.FLOAT);
    ResourceFieldSchema doublefs = new ResourceFieldSchema();
    doublefs.setType(DataType.DOUBLE);
    ResourceFieldSchema boolfs = new ResourceFieldSchema();
    boolfs.setType(DataType.BOOLEAN);
    ResourceFieldSchema dtfs = new ResourceFieldSchema();
    dtfs.setType(DataType.DATETIME);
    
    ResourceSchema tupleSchema = new ResourceSchema();
    tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, longfs, intfs, doublefs, floatfs, stringfs, intfs, doublefs, floatfs, boolfs, dtfs});
    ResourceFieldSchema tuplefs = new ResourceFieldSchema();
    tuplefs.setSchema(tupleSchema);
    tuplefs.setType(DataType.TUPLE);
    
    return tuplefs;
}

Source File: AegisthusLoader.java From aegisthus with Apache License 2.0

6 votes

protected ResourceSchema columnSchema() throws IOException {
	ResourceSchema schema = new ResourceSchema();
	List<ResourceFieldSchema> fields = new ArrayList<>();

	fields.add(field("name", DataType.BYTEARRAY));
	fields.add(field("value", DataType.BYTEARRAY));
	fields.add(field("ts", DataType.LONG));
	fields.add(field("status", DataType.CHARARRAY));
	fields.add(field("ttl", DataType.LONG));

	ResourceSchema tuple = new ResourceSchema();
	tuple.setFields(fields.toArray(new ResourceFieldSchema[0]));

	ResourceFieldSchema fs = new ResourceFieldSchema();
	fs.setName("column");
	fs.setType(DataType.TUPLE);

	fs.setSchema(tuple);
	fields.clear();
	fields.add(fs);
	schema.setFields(fields.toArray(new ResourceFieldSchema[0]));

	return schema;
}

Source File: AvroStorageUtils.java From spork with Apache License 2.0

5 votes

/** wrap a pig schema as tuple */
public static ResourceFieldSchema wrapAsTuple(ResourceFieldSchema subFieldSchema) throws IOException {
    ResourceSchema listSchema = new ResourceSchema();
    listSchema.setFields(new ResourceFieldSchema[] { subFieldSchema });

    ResourceFieldSchema tupleWrapper = new ResourceFieldSchema();
    tupleWrapper.setType(DataType.TUPLE);
    tupleWrapper.setName(PIG_TUPLE_WRAPPER);
    tupleWrapper.setSchema(listSchema);

    return tupleWrapper;
}

Source File: GenRandomData.java From spork with Apache License 2.0

5 votes

public static ResourceFieldSchema getFullTupTextDataBagFieldSchema() throws IOException{
    ResourceFieldSchema tuplefs = getSmallBagTextTupleFieldSchema();
    
    ResourceSchema outBagSchema = new ResourceSchema();
    outBagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
    ResourceFieldSchema outBagfs = new ResourceFieldSchema();
    outBagfs.setSchema(outBagSchema);
    outBagfs.setType(DataType.BAG);
    
    return outBagfs;
}

Source File: GenRandomData.java From spork with Apache License 2.0

5 votes

public static ResourceFieldSchema getSmallTupDataBagFieldSchema() throws IOException {
    ResourceFieldSchema tuplefs = getSmallTupleFieldSchema();
    
    ResourceSchema bagSchema = new ResourceSchema();
    bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
    ResourceFieldSchema bagfs = new ResourceFieldSchema();
    bagfs.setSchema(bagSchema);
    bagfs.setType(DataType.BAG);
    
    return bagfs;
}

Source File: Loader.java From logparser with Apache License 2.0

5 votes

@Override
public ResourceSchema getSchema(String location, Job job) throws IOException {
    ResourceSchema rs = new ResourceSchema();
    List<ResourceFieldSchema> fieldSchemaList = new ArrayList<>();

    for (String fieldName : requestedFields) {
        ResourceFieldSchema rfs = new ResourceFieldSchema();
        rfs.setName(fieldName);
        rfs.setDescription(fieldName);

        if(fieldName.endsWith(".*")) {
            rfs.setType(DataType.MAP);
        } else {
            EnumSet<Casts> casts = theInputFormat.getRecordReader().getCasts(fieldName);
            if (casts != null) {
                if (casts.contains(Casts.LONG)) {
                    rfs.setType(DataType.LONG);
                } else {
                    if (casts.contains(Casts.DOUBLE)) {
                        rfs.setType(DataType.DOUBLE);
                    } else {
                        rfs.setType(DataType.CHARARRAY);
                    }
                }
            } else {
                rfs.setType(DataType.BYTEARRAY);
            }
        }
        fieldSchemaList.add(rfs);
    }

    rs.setFields(fieldSchemaList.toArray(new ResourceFieldSchema[fieldSchemaList.size()]));
    return rs;
}

Source File: GenRandomData.java From spork with Apache License 2.0

5 votes

public static ResourceFieldSchema getRandMapFieldSchema() throws IOException {
    ResourceFieldSchema bytefs = new ResourceFieldSchema();
    bytefs.setType(DataType.BYTEARRAY);
    ResourceSchema mapSchema = new ResourceSchema();
    mapSchema.setFields(new ResourceFieldSchema[]{bytefs});
    ResourceFieldSchema mapfs = new ResourceFieldSchema();
    mapfs.setSchema(mapSchema);
    mapfs.setType(DataType.MAP);

    return mapfs;
}

Source File: TestTextDataParser.java From spork with Apache License 2.0

5 votes

public ResourceFieldSchema getBagFieldSchema() throws IOException{
    ResourceFieldSchema tuplefs = getTupleFieldSchema();

    ResourceSchema outBagSchema = new ResourceSchema();
    outBagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
    ResourceFieldSchema outBagfs = new ResourceFieldSchema();
    outBagfs.setSchema(outBagSchema);
    outBagfs.setType(DataType.BAG);

    return outBagfs;
}

Source File: OrcStorage.java From spork with Apache License 2.0

5 votes

@Override
public void checkSchema(ResourceSchema rs) throws IOException {
    ResourceFieldSchema fs = new ResourceFieldSchema();
    fs.setType(DataType.TUPLE);
    fs.setSchema(rs);
    typeInfo = OrcUtils.getTypeInfo(fs);
    Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
    p.setProperty(signature + SchemaSignatureSuffix, ObjectSerializer.serialize(typeInfo));
}

Source File: JsonStorage.java From spork with Apache License 2.0

5 votes

public ResourceSchema fixSchema(ResourceSchema s){
  for (ResourceFieldSchema filed : s.getFields()) {
    if(filed.getType() == DataType.NULL)
      filed.setType(DataType.BYTEARRAY);
  }
  return s;
}

Source File: CqlNativeStorage.java From stratio-cassandra with Apache License 2.0

5 votes

/** schema: (value, value, value) where keys are in the front. */
public ResourceSchema getSchema(String location, Job job) throws IOException
{
    setLocation(location, job);
    CfInfo cfInfo = getCfInfo(loadSignature);
    CfDef cfDef = cfInfo.cfDef;
    // top-level schema, no type
    ResourceSchema schema = new ResourceSchema();

    // get default marshallers and validators
    Map<MarshallerType, AbstractType> marshallers = getDefaultMarshallers(cfDef);
    Map<ByteBuffer, AbstractType> validators = getValidatorMap(cfDef);

    // will contain all fields for this schema
    List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();

    for (ColumnDef cdef : cfDef.column_metadata)
    {
        ResourceFieldSchema valSchema = new ResourceFieldSchema();
        AbstractType validator = validators.get(cdef.name);
        if (validator == null)
            validator = marshallers.get(MarshallerType.DEFAULT_VALIDATOR);
        valSchema.setName(new String(cdef.getName()));
        valSchema.setType(getPigType(validator));
        allSchemaFields.add(valSchema);
    }

    // top level schema contains everything
    schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
    return schema;
}

Source File: AvroStorageUtils.java From Cubert with Apache License 2.0

5 votes

/** wrap a pig schema as tuple */
public static ResourceFieldSchema wrapAsTuple(ResourceFieldSchema subFieldSchema) throws IOException {
    ResourceSchema listSchema = new ResourceSchema();
    listSchema.setFields(new ResourceFieldSchema[] { subFieldSchema });

    ResourceFieldSchema tupleWrapper = new ResourceFieldSchema();
    tupleWrapper.setType(DataType.TUPLE);
    tupleWrapper.setName(PIG_TUPLE_WRAPPER);
    tupleWrapper.setSchema(listSchema);

    return tupleWrapper;
}

Source File: SchemaUtil.java From iceberg with Apache License 2.0

5 votes

private static ResourceFieldSchema convert(Type type) throws IOException {
  ResourceFieldSchema result = new ResourceFieldSchema();
  result.setType(convertType(type));

  if (!type.isPrimitiveType()) {
    result.setSchema(convertComplex(type));
  }

  return result;
}

Source File: PigTuple.java From elasticsearch-hadoop with Apache License 2.0

5 votes

public void setSchema(ResourceSchema schema) {
    schemaField = new ResourceFieldSchema();
    schemaField.setType(DataType.TUPLE);
    try {
        schemaField.setSchema(schema);
    } catch (IOException ex) {
        throw new EsHadoopIllegalStateException(String.format("Cannot use schema [%s]", schema), ex);
    }
}

Source File: AvroSchema2Pig.java From spork with Apache License 2.0

4 votes

/**
 * Convert a schema with field name to a pig schema
 */
 private static ResourceFieldSchema inconvert(Schema in, String fieldName, Set<Schema> visitedRecords)
         throws IOException {

    AvroStorageLog.details("InConvert avro schema with field name " + fieldName);

    Schema.Type avroType = in.getType();
    ResourceFieldSchema fieldSchema = new ResourceFieldSchema();
    fieldSchema.setName(fieldName);

    if (avroType.equals(Schema.Type.RECORD)) {

        AvroStorageLog.details("convert to a pig tuple");

        if (visitedRecords.contains(in)) {
            fieldSchema.setType(DataType.BYTEARRAY);
        } else {
            visitedRecords.add(in);
            fieldSchema.setType(DataType.TUPLE);
            ResourceSchema tupleSchema = new ResourceSchema();
            List<Schema.Field> fields = in.getFields();
            ResourceFieldSchema[] childFields = new ResourceFieldSchema[fields.size()];
            int index = 0;
            for (Schema.Field field : fields) {
                childFields[index++] = inconvert(field.schema(), field.name(), visitedRecords);
            }

            tupleSchema.setFields(childFields);
            fieldSchema.setSchema(tupleSchema);
            visitedRecords.remove(in);
        }

    } else if (avroType.equals(Schema.Type.ARRAY)) {

        AvroStorageLog.details("convert array to a pig bag");
        fieldSchema.setType(DataType.BAG);
        Schema elemSchema = in.getElementType();
        ResourceFieldSchema subFieldSchema = inconvert(elemSchema, ARRAY_FIELD, visitedRecords);
        add2BagSchema(fieldSchema, subFieldSchema);

    } else if (avroType.equals(Schema.Type.MAP)) {

        AvroStorageLog.details("convert map to a pig map");
        fieldSchema.setType(DataType.MAP);

    } else if (avroType.equals(Schema.Type.UNION)) {

        if (AvroStorageUtils.isAcceptableUnion(in)) {
            Schema acceptSchema = AvroStorageUtils.getAcceptedType(in);
            ResourceFieldSchema realFieldSchema = inconvert(acceptSchema, null, visitedRecords);
            fieldSchema.setType(realFieldSchema.getType());
            fieldSchema.setSchema(realFieldSchema.getSchema());
        } else
            throw new IOException("Do not support generic union:" + in);

    } else if (avroType.equals(Schema.Type.FIXED)) {
         fieldSchema.setType(DataType.BYTEARRAY);
    } else if (avroType.equals(Schema.Type.BOOLEAN)) {
        fieldSchema.setType(DataType.BOOLEAN);
    } else if (avroType.equals(Schema.Type.BYTES)) {
        fieldSchema.setType(DataType.BYTEARRAY);
    } else if (avroType.equals(Schema.Type.DOUBLE)) {
        fieldSchema.setType(DataType.DOUBLE);
    } else if (avroType.equals(Schema.Type.ENUM)) {
        fieldSchema.setType(DataType.CHARARRAY);
    } else if (avroType.equals(Schema.Type.FLOAT)) {
        fieldSchema.setType(DataType.FLOAT);
    } else if (avroType.equals(Schema.Type.INT)) {
        fieldSchema.setType(DataType.INTEGER);
    } else if (avroType.equals(Schema.Type.LONG)) {
        fieldSchema.setType(DataType.LONG);
    } else if (avroType.equals(Schema.Type.STRING)) {
        fieldSchema.setType(DataType.CHARARRAY);
    } else if (avroType.equals(Schema.Type.NULL)) {
        // value of NULL is always NULL
        fieldSchema.setType(DataType.INTEGER);
    } else {
        throw new IOException("Unsupported avro type:" + avroType);
    }
    return fieldSchema;
}

Source File: TestStore.java From spork with Apache License 2.0

4 votes

@Test
public void testStoreComplexDataWithNull() throws Exception {
    Tuple inputTuple = GenRandomData.genRandSmallBagTextTupleWithNulls(new Random(), 10, 100);
    inpDB = DefaultBagFactory.getInstance().newDefaultBag();
    inpDB.add(inputTuple);
    storeAndCopyLocally(inpDB);
    PigStorage ps = new PigStorage("\t");
    BufferedReader br = new BufferedReader(new FileReader(outputFileName));
    for(String line=br.readLine();line!=null;line=br.readLine()){
        System.err.println("Complex data: ");
        System.err.println(line);
        String[] flds = line.split("\t",-1);
        Tuple t = new DefaultTuple();

        ResourceFieldSchema stringfs = new ResourceFieldSchema();
        stringfs.setType(DataType.CHARARRAY);
        ResourceFieldSchema intfs = new ResourceFieldSchema();
        intfs.setType(DataType.INTEGER);
        ResourceFieldSchema bytefs = new ResourceFieldSchema();
        bytefs.setType(DataType.BYTEARRAY);

        ResourceSchema tupleSchema = new ResourceSchema();
        tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, intfs});
        ResourceFieldSchema tuplefs = new ResourceFieldSchema();
        tuplefs.setSchema(tupleSchema);
        tuplefs.setType(DataType.TUPLE);

        ResourceSchema bagSchema = new ResourceSchema();
        bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
        ResourceFieldSchema bagfs = new ResourceFieldSchema();
        bagfs.setSchema(bagSchema);
        bagfs.setType(DataType.BAG);

        ResourceSchema mapSchema = new ResourceSchema();
        mapSchema.setFields(new ResourceFieldSchema[]{bytefs});
        ResourceFieldSchema mapfs = new ResourceFieldSchema();
        mapfs.setSchema(mapSchema);
        mapfs.setType(DataType.MAP);

        t.append(flds[0].compareTo("")!=0 ? ps.getLoadCaster().bytesToBag(flds[0].getBytes(), bagfs) : null);
        t.append(flds[1].compareTo("")!=0 ? new DataByteArray(flds[1].getBytes()) : null);
        t.append(flds[2].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[2].getBytes()) : null);
        t.append(flds[3].compareTo("")!=0 ? ps.getLoadCaster().bytesToDouble(flds[3].getBytes()) : null);
        t.append(flds[4].compareTo("")!=0 ? ps.getLoadCaster().bytesToFloat(flds[4].getBytes()) : null);
        t.append(flds[5].compareTo("")!=0 ? ps.getLoadCaster().bytesToInteger(flds[5].getBytes()) : null);
        t.append(flds[6].compareTo("")!=0 ? ps.getLoadCaster().bytesToLong(flds[6].getBytes()) : null);
        t.append(flds[7].compareTo("")!=0 ? ps.getLoadCaster().bytesToMap(flds[7].getBytes(), mapfs) : null);
        t.append(flds[8].compareTo("")!=0 ? ps.getLoadCaster().bytesToTuple(flds[8].getBytes(), tuplefs) : null);
        t.append(flds[9].compareTo("")!=0 ? ps.getLoadCaster().bytesToBoolean(flds[9].getBytes()) : null);
        t.append(flds[10].compareTo("")!=0 ? ps.getLoadCaster().bytesToDateTime(flds[10].getBytes()) : null);
        t.append(flds[11].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[10].getBytes()) : null);
        assertEquals(inputTuple, t);
    }
    br.close();
}

Source File: GenRandomData.java From spork with Apache License 2.0

4 votes

public static ResourceFieldSchema getSmallBagTextTupleFieldSchema() throws IOException{
    ResourceFieldSchema dbafs = new ResourceFieldSchema();
    dbafs.setType(DataType.BYTEARRAY);
    
    ResourceFieldSchema stringfs = new ResourceFieldSchema();
    stringfs.setType(DataType.CHARARRAY);
    
    ResourceFieldSchema intfs = new ResourceFieldSchema();
    intfs.setType(DataType.INTEGER);
	
    ResourceFieldSchema bagfs = getSmallTupDataBagFieldSchema();
    
    ResourceFieldSchema floatfs = new ResourceFieldSchema();
    floatfs.setType(DataType.FLOAT);
    
    ResourceFieldSchema doublefs = new ResourceFieldSchema();
    doublefs.setType(DataType.DOUBLE);
    
    ResourceFieldSchema longfs = new ResourceFieldSchema();
    longfs.setType(DataType.LONG);
    
    ResourceFieldSchema mapfs = new ResourceFieldSchema();
    mapfs.setType(DataType.MAP);
    
    ResourceFieldSchema tuplefs = getSmallTupleFieldSchema();

    ResourceFieldSchema boolfs = new ResourceFieldSchema();
    boolfs.setType(DataType.BOOLEAN);

    ResourceFieldSchema dtfs = new ResourceFieldSchema();
    dtfs.setType(DataType.DATETIME);
    
    ResourceSchema outSchema = new ResourceSchema();
    outSchema.setFields(new ResourceFieldSchema[]{bagfs, dbafs, stringfs, doublefs, floatfs,
            intfs, longfs, mapfs, tuplefs, boolfs, dtfs});
    ResourceFieldSchema outfs = new ResourceFieldSchema();
    outfs.setSchema(outSchema);
    outfs.setType(DataType.TUPLE);
    
    return outfs;
}

Source File: SchemaUtil.java From iceberg with Apache License 2.0

4 votes

private static ResourceSchema convertComplex(Type type) throws IOException {
  ResourceSchema result = new ResourceSchema();

  switch (type.typeId()) {
    case STRUCT:
      StructType structType = type.asStructType();

      List<ResourceFieldSchema> fields = Lists.newArrayList();

      for (Types.NestedField f : structType.fields()) {
        fields.add(convert(f));
      }

      result.setFields(fields.toArray(new ResourceFieldSchema[0]));

      return result;
    case LIST:
      ListType listType = type.asListType();

      ResourceFieldSchema [] elementFieldSchemas = new ResourceFieldSchema[]{convert(listType.elementType())};

      if (listType.elementType().isStructType()) {
        result.setFields(elementFieldSchemas);
      } else {
        //Wrap non-struct types in tuples
        ResourceSchema elementSchema = new ResourceSchema();
        elementSchema.setFields(elementFieldSchemas);

        ResourceFieldSchema tupleSchema = new ResourceFieldSchema();
        tupleSchema.setType(DataType.TUPLE);
        tupleSchema.setSchema(elementSchema);

        result.setFields(new ResourceFieldSchema[]{tupleSchema});
      }

      return result;
    case MAP:
      MapType mapType = type.asMapType();

      if (mapType.keyType().typeId() != Type.TypeID.STRING) {
        throw new FrontendException("Unsupported map key type: " + mapType.keyType());
      }
      result.setFields(new ResourceFieldSchema[]{convert(mapType.valueType())});

      return result;
    default:
      throw new FrontendException("Unsupported complex type: " + type);
  }
}

Source File: SchemaUtil.java From iceberg with Apache License 2.0

4 votes

private static ResourceSchema convertComplex(Type type) throws IOException {
  ResourceSchema result = new ResourceSchema();

  switch (type.typeId()) {
    case STRUCT:
      Types.StructType structType = type.asStructType();

      List<ResourceFieldSchema> fields = Lists.newArrayList();

      for (Types.NestedField f : structType.fields()) {
        fields.add(convert(f));
      }

      result.setFields(fields.toArray(new ResourceFieldSchema[0]));

      return result;
    case LIST:
      Types.ListType listType = type.asListType();

      ResourceFieldSchema [] elementFieldSchemas = new ResourceFieldSchema[]{convert(listType.elementType())};

      if (listType.elementType().isStructType()) {
        result.setFields(elementFieldSchemas);
      } else {
        //Wrap non-struct types in tuples
        ResourceSchema elementSchema = new ResourceSchema();
        elementSchema.setFields(elementFieldSchemas);

        ResourceFieldSchema tupleSchema = new ResourceFieldSchema();
        tupleSchema.setType(DataType.TUPLE);
        tupleSchema.setSchema(elementSchema);

        result.setFields(new ResourceFieldSchema[]{tupleSchema});
      }

      return result;
    case MAP:
      Types.MapType mapType = type.asMapType();

      if (mapType.keyType().typeId() != Type.TypeID.STRING) {
        throw new FrontendException("Unsupported map key type: " + mapType.keyType());
      }
      result.setFields(new ResourceFieldSchema[]{convert(mapType.valueType())});

      return result;
    default:
      throw new FrontendException("Unsupported complex type: " + type);
  }
}

Java Code Examples for org.apache.pig.ResourceSchema.ResourceFieldSchema#setType()