org.apache.hadoop.mrunit.mapreduce.MapDriver Java Examples

public static void verifyMapReduce(SmartMapper mapper, SmartReducer reducer, Object key, Object input)
    throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  MapReduceDriver mapReduceDriver = new MapReduceDriver();
  mapReduceDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  mapReduceDriver = new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
  writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapReduceDriver.withInput(writableKey, writableValue);
  mapReduceDriver.setMapper(mapper);
  mapReduceDriver.setReducer(reducer);
  List finalResults = mapReduceDriver.run();
  String text = String.format("[%s]\n\n -> maps via %s to -> \n\n%s\n\n -> reduces via %s to -> \n\n%s", input,
      mapper.getClass().getSimpleName(), ArrayUtils.toString(results, Echo.INSTANCE),
      reducer.getClass().getSimpleName(), ArrayUtils.toString(finalResults, Echo.INSTANCE));
  Approvals.verify(text);
}
 

@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    inMemCuboidMapper = new InMemCuboidMapper<>();
    mapDriver = MapDriver.newMapDriver(inMemCuboidMapper);

    PowerMockito.stub(PowerMockito.method(CuboidSchedulerUtil.class, "getCuboidSchedulerByMode", CubeSegment.class,
            String.class)).toReturn(cube.getCuboidScheduler());
    IMRBatchCubingInputSide mockInputSide = createMockInputSide();
    PowerMockito.stub(PowerMockito.method(MRUtil.class, "getBatchCubingInputSide")).toReturn(mockInputSide);

}
 

@Test
public void testTableMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    TripleRowResolver trr = new WholeRowTripleResolver();
    Map<TABLE_LAYOUT,TripleRow> map = trr.serialize(rya);
    TripleRow tr = map.get(TABLE_LAYOUT.SPO);
    byte[] b = new byte[0];
    Key key = new Key(tr.getRow(), tr.getColumnFamily(),
        tr.getColumnQualifier(), b, 1);
    Value val = new Value(b);
    new MapDriver<Key, Value, Fact, Derivation>()
        .withMapper(new DuplicateElimination.DuplicateTableMapper())
        .withInput(key, val)
        .withOutput(X_SUB_Y, X_SUB_Y.getDerivation())
        .runTest();
}
 

@Test
public void testTableMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    TripleRowResolver trr = new WholeRowTripleResolver();
    Map<TABLE_LAYOUT,TripleRow> map = trr.serialize(rya);
    TripleRow tr = map.get(TABLE_LAYOUT.SPO);
    byte[] b = new byte[0];
    Key key = new Key(tr.getRow(), tr.getColumnFamily(),
        tr.getColumnQualifier(), b, 1);
    Value val = new Value(b);
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<Key, Value, ResourceWritable, Fact>()
        .withMapper(new ForwardChain.TableMapper(schema))
        .withInput(key, val)
        .withOutput(rw1, X_SUB_Y)
        .withOutput(rw2, X_SUB_Y)
        .runTest();
}
 

@Test
public void testRdfMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    RyaStatementWritable rsw = new RyaStatementWritable();
    rsw.setRyaStatement(rya);
    LongWritable l = new LongWritable();
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<LongWritable, RyaStatementWritable, ResourceWritable,
            Fact>()
        .withMapper(new ForwardChain.RdfMapper(schema))
        .withInput(l, rsw)
        .withOutput(rw1, X_SUB_Y)
        .withOutput(rw2, X_SUB_Y)
        .runTest();
}
 

@Before
public void setUp() {
    LouvainTableSynthesizerMapper mapper = new LouvainTableSynthesizerMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
    LouvainTableSynthesizerReducer reducer = new LouvainTableSynthesizerReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 

public static void verifyMapping(SmartMapper mapper, Object key, Object input) throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  String header = String.format("[%s]\n\n -> maps via %s to -> \n", input, mapper.getClass().getSimpleName());
  Approvals.verifyAll(header, results, Echo.INSTANCE);
}
 

@Before
public void setUp() throws Exception {
  DerivedColumnTransformationPhaseMapper mapper = new DerivedColumnTransformationPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");

  ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  Schema inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  Schema outputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(TRANSFORMATION_SCHEMA));
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_SCHEMA.toString(),
      outputSchema.toString());

  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_TOPK_PATH.toString(),
      ClassLoader.getSystemResource(TOPK_PATH).toString());

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_PATH.toString(), outputPath);

}
 

@Before
public void setUp() throws Exception {
  DerivedColumnNoTransformationPhaseMapper mapper = new DerivedColumnNoTransformationPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");


  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");

  ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  Schema inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  Schema outputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(NO_TRANSFORMATION_SCHEMA));
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_SCHEMA.toString(),
      outputSchema.toString());

  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_TOPK_PATH.toString(),
      TOPK_PATH);

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_PATH.toString(), outputPath);
}
 

@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));

    BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>();
    mapDriver = MapDriver.newMapDriver(mapper);
}
 

@Before
public void setUp() {
    CommunityCompression.Map mapper = new CommunityCompression.Map();
    mapDriver = MapDriver.newMapDriver(mapper);
    CommunityCompression.Reduce reducer = new CommunityCompression.Reduce();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
 

@Test
public void testInconsistencyMapperOutput() throws Exception {
    Fact empty = new Fact();
    empty.setDerivation(X_DISJOINT);
    new MapDriver<Derivation, NullWritable, Fact, Derivation>()
        .withMapper(new DuplicateElimination.InconsistencyMapper())
        .withInput(X_DISJOINT, NullWritable.get())
        .withOutput(empty, X_DISJOINT)
        .runTest();
}
 

@Test
public void testRdfMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    RyaStatementWritable rsw = new RyaStatementWritable();
    rsw.setRyaStatement(rya);
    LongWritable l = new LongWritable();
    new MapDriver<LongWritable, RyaStatementWritable, Fact,
        Derivation>()
        .withMapper(new DuplicateElimination.DuplicateRdfMapper())
        .withInput(l, rsw)
        .withOutput(X_SUB_Y, X_SUB_Y.getDerivation())
        .runTest();
}
 

@Test
public void testFileMapperOutput() throws Exception {
    new MapDriver<Fact, NullWritable, Fact, Derivation>()
        .withMapper(new DuplicateElimination.DuplicateFileMapper())
        .withInput(X_SUB_Y, NullWritable.get())
        .withOutput(X_SUB_Y, X_SUB_Y.getDerivation())
        .runTest();
}
 

@BeforeClass
public static void init() throws IOException {
  LindenMapper mapper = new LindenMapper();
  mDriver = MapDriver.newMapDriver(mapper);
  int numShards = 1;
  Shard[] shards = LindenJob.createShards(indexPath, numShards);
  Shard.setIndexShards(mDriver.getConfiguration(), shards);
}
 

@Test
public void testFullTripleEntry() throws InterruptedException, IOException {

  TripleEntry te1 = new TripleEntry(new Text("urn:gem:etype#1234"), new Text("urn:gem#pred"), new Text("subject"), new Text("predicate"), new Text("object"));
  CardList cl = new CardList(34, 52, 63, 0, 0, 0);
  TripleEntry te2 = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("object"));
  TripleEntry te3 = new TripleEntry(new Text("urn:gem#pred"), new Text(""), new Text("predicate"), new Text(""), new Text("object"));

  new MapDriver<TripleEntry,CardList,TripleEntry,CardList>().withMapper(new JoinSelectStatisticsSum.CardinalityIdentityMapper()).withInput(te1, cl)
      .withOutput(te2, cl).withOutput(te3, cl).withOutput(te1, cl).runTest();

}
 

@Before
public void setUp()
{
    SMSCDRMapper mapper = new SMSCDRMapper();
    SMSCDRReducer reducer = new SMSCDRReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

@Test
public void testOutput() throws InterruptedException, IOException {

    String s = "urn:gem:etype#1234";
    String p = "urn:gem#pred";

    String ts = "798497748386999999";
    
    Text t1 = new Text(TripleValueType.subject.name() + DELIM + s + DELIM + 1);
    Text t2 = new Text(TripleValueType.predicate.name() + DELIM + p + DELIM + 2);
    Text t3 = new Text(TripleValueType.subjectpredicate.name() + DELIM + s + DELIM + p + DELIM + ts);

    byte[] b = new byte[0];
    byte[] c = "25".getBytes();
    byte[] d = "47".getBytes();
    byte[] e = "15".getBytes();

    Key key1 = new Key(t1.getBytes(), b, b, b, 1);
    Key key2 = new Key(t2.getBytes(), b, b, b, 1);
    Key key3 = new Key(t3.getBytes(), b, b, b, 1);
    Value val1 = new Value(c);
    Value val2 = new Value(d);
    Value val3 = new Value(e);
    
   

    // System.out.println("Keys are " + key1 + " and " + key2);

    new MapDriver<Key, Value, CompositeType, TripleCard>()
            .withMapper(new JoinSelectProspectOutput.CardinalityMapper())
            .withInput(key1, val1)
            .withInput(key2, val2)
            .withInput(key3, val3)
            .withOutput(new CompositeType(s, 1), new TripleCard(new CardinalityType(25, "subject", 1)))
            .withOutput(new CompositeType(p, 1), new TripleCard(new CardinalityType(47, "predicate", 2)))
            .withOutput(new CompositeType(s + DELIM + p, 1),
                    new TripleCard(new CardinalityType(15, "subjectpredicate", Long.parseLong(ts)))).runTest();

}
 

@Test
public void testOutput() throws InterruptedException, IOException {

  String s = "urn:gem:etype#1234";
  String p = "urn:gem#pred";

  Text t1 = new Text(TripleValueType.subject.name() + DELIM + s + DELIM + 1);
  Text t2 = new Text(TripleValueType.predicate.name() + DELIM + p + DELIM + 2);
  Text t3 = new Text(TripleValueType.subjectpredicate.name() + DELIM + s + DELIM + p + DELIM + 3);

  byte[] b = new byte[0];
  byte[] c = "25".getBytes();
  byte[] d = "47".getBytes();
  byte[] e = "15".getBytes();

  Key key1 = new Key(t1.getBytes(), b, b, b, 1);
  Key key2 = new Key(t2.getBytes(), b, b, b, 1);
  Key key3 = new Key(t3.getBytes(), b, b, b, 1);
  Value val1 = new Value(c);
  Value val2 = new Value(d);
  Value val3 = new Value(e);

  // System.out.println("Keys are " + key1 + " and " + key2);

  new MapDriver<Key,Value,CompositeType,TripleCard>().withMapper(new JoinSelectProspectOutput.CardinalityMapper()).withInput(key1, val1)
      .withInput(key2, val2).withInput(key3, val3).withOutput(new CompositeType(s, 1), new TripleCard(new CardinalityType(25, "subject", 1)))
      .withOutput(new CompositeType(p, 1), new TripleCard(new CardinalityType(47, "predicate", 2)))
      .withOutput(new CompositeType(s + DELIM + p, 1), new TripleCard(new CardinalityType(15, "subjectpredicate", 3))).runTest();

}
 

@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    FactDistinctColumnsMapper<LongWritable> factDistinctColumnsMapper = new FactDistinctColumnsMapper<>();
    mapDriver = MapDriver.newMapDriver(factDistinctColumnsMapper);
}
 

@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    CalculateStatsFromBaseCuboidMapper calStatsFromBasicCuboidMapper = new CalculateStatsFromBaseCuboidMapper();
    mapDriver = MapDriver.newMapDriver(calStatsFromBasicCuboidMapper);
}
 

@Test
public void testFileMapperOutput() throws Exception {
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<Fact, NullWritable, ResourceWritable, Fact>()
        .withMapper(new ForwardChain.FileMapper(schema))
        .withInput(X_SUB_Y, NullWritable.get())
        .withOutput(rw1, X_SUB_Y)
        .withOutput(rw2, X_SUB_Y)
        .runTest();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Before
public void setUp() {
    ColumnCardinalityMapper mapper = new ColumnCardinalityMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}
 

@Before
public void setUp() throws IOException {
  GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put(
      "memory",
      new MemoryStoreFactoryFamily());
  final NNMapReduce.NNMapper<SimpleFeature> nnMapper = new NNMapReduce.NNMapper<>();
  final NNMapReduce.NNReducer<SimpleFeature, Text, Text, Boolean> nnReducer =
      new NNMapReduce.NNSimpleFeatureIDOutputReducer();

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "0.0002,0.0002");

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          CommonParameters.Common.DISTANCE_FUNCTION_CLASS),
      FeatureCentroidOrthodromicDistanceFn.class,
      DistanceFn.class);
  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      0.001);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);

  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
  internalAdapterId = InternalAdapterStoreImpl.getLazyInitialAdapterId(adapter.getTypeName());
  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      internalAdapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      internalAdapterId);

  serializations();
}
 

@Before
public void setUp() throws IOException {

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "10,10");

  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      10);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          DBScanMapReduce.class,
          HullParameters.Hull.PROJECTION_CLASS),
      SimpleFeatureProjection.class,
      Projection.class);

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);
  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);

  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  serializations();
}
 

@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION_NAMES.toString(), "d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION.toString() + ".d3", "xyz2");
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);

  // Mapper config
  TopKPhaseMapper mapper = new TopKPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  TopKPhaseReducer reducer = new TopKPhaseReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(TopKPhaseConstants.TOPK_PHASE_OUTPUT_PATH.toString(), outputPath);

}
 

@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.HOURS.toString());
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.MILLISECONDS.toString());
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  aggPhaseConfig = AggregationPhaseConfig.fromThirdEyeConfig(thirdeyeConfig);

  // Mapper config
  AggregationMapper mapper = new AggregationMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  AggregationReducer reducer = new AggregationReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  Schema reducerSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  configuration.set(AggregationPhaseConstants.AGG_PHASE_AVRO_SCHEMA.toString(), reducerSchema.toString());

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(AggregationPhaseConstants.AGG_PHASE_OUTPUT_PATH.toString(), outputPath);
  setUpAvroSerialization(reduceDriver.getConfiguration(), reducerSchema);

}
 

@Before
public void setUp() throws IOException {
  _mapper = new CsvBlurMapper();
  _mapDriver = MapDriver.newMapDriver(_mapper);
}
 

@Before
public void setUp() {
  mapper = new Mapper<Text, Text, Text, Text>();
  driver = new MapDriver<Text, Text, Text, Text>(mapper);
}
 

@Before
public void setUp() {
  mapper = new Mapper<Text, Text, Text, Text>();
  driver = new MapDriver<Text, Text, Text, Text>(mapper);
}