Java Code Examples for org.apache.pig.StoreFuncInterface#setStoreLocation()

private static POStore configureStorer(JobConf jobConf,
        PhysicalOperator physicalOperator) throws IOException {
    ArrayList<POStore> storeLocations = Lists.newArrayList();
    POStore poStore = (POStore) physicalOperator;
    storeLocations.add(poStore);
    StoreFuncInterface sFunc = poStore.getStoreFunc();
    sFunc.setStoreLocation(poStore.getSFile().getFileName(),
            new org.apache.hadoop.mapreduce.Job(jobConf));
    poStore.setInputs(null);
    poStore.setParentPlan(null);

    jobConf.set(JobControlCompiler.PIG_MAP_STORES,
            ObjectSerializer.serialize(Lists.newArrayList()));
    jobConf.set(JobControlCompiler.PIG_REDUCE_STORES,
            ObjectSerializer.serialize(storeLocations));
    return poStore;
}
 

/**
 * Before delegating calls to underlying OutputFormat or OutputCommitter
 * Pig needs to ensure the Configuration in the JobContext contains
 * the output location and StoreFunc
 * for the specific store - so set these up in the context for this specific
 * store
 * @param jobContext the {@link JobContext}
 * @param store the POStore
 * @throws IOException on failure
 */
public static void setLocation(JobContext jobContext, POStore store) throws
IOException {
    Job storeJob = new Job(jobContext.getConfiguration());
    StoreFuncInterface storeFunc = store.getStoreFunc();
    String outputLocation = store.getSFile().getFileName();
    storeFunc.setStoreLocation(outputLocation, storeJob);

    // the setStoreLocation() method would indicate to the StoreFunc
    // to set the output location for its underlying OutputFormat.
    // Typically OutputFormat's store the output location in the
    // Configuration - so we need to get the modified Configuration
    // containing the output location (and any other settings the
    // OutputFormat might have set) and merge it with the Configuration
    // we started with so that when this method returns the Configuration
    // supplied as input has the updates.
    ConfigurationUtil.mergeConf(jobContext.getConfiguration(),
            storeJob.getConfiguration());
}
 

private static POStore createPOStoreForFileBasedSystem(long size, StoreFuncInterface storeFunc,
        Configuration conf) throws Exception {

    File file = File.createTempFile("tempFile", ".tmp");
    file.deleteOnExit();
    RandomAccessFile f = new RandomAccessFile(file, "rw");
    f.setLength(size);
    f.close();

    storeFunc.setStoreLocation(file.getAbsolutePath(), new Job(conf));
    FuncSpec funcSpec = new FuncSpec(storeFunc.getClass().getCanonicalName());
    POStore poStore = new POStore(new OperatorKey());
    poStore.setSFile(new FileSpec(file.getAbsolutePath(), funcSpec));
    poStore.setStoreFunc(storeFunc);
    poStore.setUp();

    return poStore;
}
 

private static POStore createPOStoreForNonFileBasedSystem(StoreFuncInterface storeFunc,
        Configuration conf) throws Exception {

    String nonFileBasedUri = "hbase://tableName";
    storeFunc.setStoreLocation(nonFileBasedUri, new Job(conf));
    FuncSpec funcSpec = new FuncSpec(storeFunc.getClass().getCanonicalName());
    POStore poStore = new POStore(new OperatorKey());
    poStore.setSFile(new FileSpec(nonFileBasedUri, funcSpec));
    poStore.setStoreFunc(storeFunc);
    poStore.setUp();

    return poStore;
}
 

public static void write(String out) throws IOException, ParserException,
    InterruptedException, ExecException {
  {
    StringBuilder schemaString = new StringBuilder("a0: chararray");
    for (int i = 1; i < COLUMN_COUNT; i++) {
      schemaString.append(", a" + i + ": chararray");
    }

    String location = out;
    String schema = schemaString.toString();

    StoreFuncInterface storer = new ParquetStorer();
    Job job = new Job(conf);
    storer.setStoreFuncUDFContextSignature("sig");
    String absPath = storer.relToAbsPathForStoreLocation(location, new Path(new File(".").getAbsoluteFile().toURI()));
    storer.setStoreLocation(absPath, job);
    storer.checkSchema(new ResourceSchema(Utils.getSchemaFromString(schema)));
    @SuppressWarnings("unchecked") // that's how the base class is defined
    OutputFormat<Void, Tuple> outputFormat = storer.getOutputFormat();
    // it's ContextUtil.getConfiguration(job) and not just conf !
    JobContext jobContext = ContextUtil.newJobContext(ContextUtil.getConfiguration(job), new JobID("jt", jobid ++));
    outputFormat.checkOutputSpecs(jobContext);
    if (schema != null) {
      ResourceSchema resourceSchema = new ResourceSchema(Utils.getSchemaFromString(schema));
      storer.checkSchema(resourceSchema);
      if (storer instanceof StoreMetadata) {
        ((StoreMetadata)storer).storeSchema(resourceSchema, absPath, job);
      }
    }
    TaskAttemptContext taskAttemptContext = ContextUtil.newTaskAttemptContext(ContextUtil.getConfiguration(job), new TaskAttemptID("jt", jobid, true, 1, 0));
    RecordWriter<Void, Tuple> recordWriter = outputFormat.getRecordWriter(taskAttemptContext);
    storer.prepareToWrite(recordWriter);

    for (int i = 0; i < ROW_COUNT; i++) {
      Tuple tuple = TupleFactory.getInstance().newTuple(COLUMN_COUNT);
      for (int j = 0; j < COLUMN_COUNT; j++) {
        tuple.set(j, "a" + i + "_" + j);
      }
      storer.putNext(tuple);
    }

    recordWriter.close(taskAttemptContext);
    OutputCommitter outputCommitter = outputFormat.getOutputCommitter(taskAttemptContext);
    outputCommitter.commitTask(taskAttemptContext);
    outputCommitter.commitJob(jobContext);

  }
}