Java Code Examples for org.apache.hadoop.mrunit.mapreduce.MapDriver#newMapDriver()

The following examples show how to use org.apache.hadoop.mrunit.mapreduce.MapDriver#newMapDriver() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: InMemCuboidMapperTest.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    inMemCuboidMapper = new InMemCuboidMapper<>();
    mapDriver = MapDriver.newMapDriver(inMemCuboidMapper);

    PowerMockito.stub(PowerMockito.method(CuboidSchedulerUtil.class, "getCuboidSchedulerByMode", CubeSegment.class,
            String.class)).toReturn(cube.getCuboidScheduler());
    IMRBatchCubingInputSide mockInputSide = createMockInputSide();
    PowerMockito.stub(PowerMockito.method(MRUtil.class, "getBatchCubingInputSide")).toReturn(mockInputSide);

}
Example 2
Source File: LindenMapredTest.java    From linden with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void init() throws IOException {
  LindenMapper mapper = new LindenMapper();
  mDriver = MapDriver.newMapDriver(mapper);
  int numShards = 1;
  Shard[] shards = LindenJob.createShards(indexPath, numShards);
  Shard.setIndexShards(mDriver.getConfiguration(), shards);
}
Example 3
Source File: DerivedColumnTransformationTest.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
  DerivedColumnTransformationPhaseMapper mapper = new DerivedColumnTransformationPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");

  ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  Schema inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  Schema outputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(TRANSFORMATION_SCHEMA));
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_SCHEMA.toString(),
      outputSchema.toString());

  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_TOPK_PATH.toString(),
      ClassLoader.getSystemResource(TOPK_PATH).toString());

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_PATH.toString(), outputPath);

}
Example 4
Source File: DerivedColumnNoTransformationTest.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
  DerivedColumnNoTransformationPhaseMapper mapper = new DerivedColumnNoTransformationPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");


  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");

  ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  Schema inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  Schema outputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(NO_TRANSFORMATION_SCHEMA));
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_SCHEMA.toString(),
      outputSchema.toString());

  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_TOPK_PATH.toString(),
      TOPK_PATH);

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(DerivedColumnTransformationPhaseConstants.DERIVED_COLUMN_TRANSFORMATION_PHASE_OUTPUT_PATH.toString(), outputPath);
}
Example 5
Source File: BaseCuboidMapperTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("../job/meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));

    BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>();
    mapDriver = MapDriver.newMapDriver(mapper);
}
Example 6
Source File: CommunityCompressionTest.java    From distributed-graph-analytics with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
    CommunityCompression.Map mapper = new CommunityCompression.Map();
    mapDriver = MapDriver.newMapDriver(mapper);
    CommunityCompression.Reduce reducer = new CommunityCompression.Reduce();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
Example 7
Source File: LouvainTableSynthesizerTest.java    From distributed-graph-analytics with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
    LouvainTableSynthesizerMapper mapper = new LouvainTableSynthesizerMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
    LouvainTableSynthesizerReducer reducer = new LouvainTableSynthesizerReducer();
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
}
Example 8
Source File: CalculateStatsFromBaseCuboidMapperTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    CalculateStatsFromBaseCuboidMapper calStatsFromBasicCuboidMapper = new CalculateStatsFromBaseCuboidMapper();
    mapDriver = MapDriver.newMapDriver(calStatsFromBasicCuboidMapper);
}
Example 9
Source File: FactDistinctColumnsMapperTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws Exception {
    createTestMetadata();
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    cubeName = "test_kylin_cube_with_slr_1_new_segment";
    cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    FactDistinctColumnsMapper<LongWritable> factDistinctColumnsMapper = new FactDistinctColumnsMapper<>();
    mapDriver = MapDriver.newMapDriver(factDistinctColumnsMapper);
}
Example 10
Source File: MRUnitTest.java    From dkpro-c4corpus with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp()
{
    SMSCDRMapper mapper = new SMSCDRMapper();
    SMSCDRReducer reducer = new SMSCDRReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
Example 11
Source File: CubeHFileMapperTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() {
    CubeHFileMapper mapper = new CubeHFileMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}
Example 12
Source File: CubeHFileMapperTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() {
    CubeHFileMapper mapper = new CubeHFileMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}
Example 13
Source File: NNMapReduceTest.java    From geowave with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws IOException {
  GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put(
      "memory",
      new MemoryStoreFactoryFamily());
  final NNMapReduce.NNMapper<SimpleFeature> nnMapper = new NNMapReduce.NNMapper<>();
  final NNMapReduce.NNReducer<SimpleFeature, Text, Text, Boolean> nnReducer =
      new NNMapReduce.NNSimpleFeatureIDOutputReducer();

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "0.0002,0.0002");

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          CommonParameters.Common.DISTANCE_FUNCTION_CLASS),
      FeatureCentroidOrthodromicDistanceFn.class,
      DistanceFn.class);
  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      0.001);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);

  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
  internalAdapterId = InternalAdapterStoreImpl.getLazyInitialAdapterId(adapter.getTypeName());
  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      internalAdapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      internalAdapterId);

  serializations();
}
Example 14
Source File: RandomKeyDistributionMapperTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Before
@SuppressWarnings({ "rawtypes", "unchecked" })
public void setUp() {
    RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}
Example 15
Source File: DBScanMapReduceTest.java    From geowave with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws IOException {

  mapDriver = MapDriver.newMapDriver(nnMapper);
  reduceDriver = ReduceDriver.newReduceDriver(nnReducer);

  mapDriver.getConfiguration().set(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.DISTANCE_THRESHOLDS),
      "10,10");

  reduceDriver.getConfiguration().setDouble(
      GeoWaveConfiguratorBase.enumToConfKey(
          NNMapReduce.class,
          PartitionParameters.Partition.MAX_DISTANCE),
      10);

  ftype =
      AnalyticFeature.createGeometryFeatureAdapter(
          "centroid",
          new String[] {"extra1"},
          BasicFeatureTypes.DEFAULT_NAMESPACE,
          ClusteringUtils.CLUSTERING_CRS).getFeatureType();

  reduceDriver.getConfiguration().setClass(
      GeoWaveConfiguratorBase.enumToConfKey(
          DBScanMapReduce.class,
          HullParameters.Hull.PROJECTION_CLASS),
      SimpleFeatureProjection.class,
      Projection.class);

  final Index index = new SpatialDimensionalityTypeProvider().createIndex(new SpatialOptions());
  final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
  adapter.init(index);
  JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);

  JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
  JobContextInternalAdapterStore.addTypeName(
      mapDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  JobContextInternalAdapterStore.addTypeName(
      reduceDriver.getConfiguration(),
      adapter.getTypeName(),
      adapterId);
  serializations();
}
Example 16
Source File: ColumnCardinalityMapperTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings({ "rawtypes", "unchecked" })
@Before
public void setUp() {
    ColumnCardinalityMapper mapper = new ColumnCardinalityMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}
Example 17
Source File: CsvBlurMapperTest.java    From incubator-retired-blur with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws IOException {
  _mapper = new CsvBlurMapper();
  _mapDriver = MapDriver.newMapDriver(_mapper);
}
Example 18
Source File: AggregationPhaseTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.HOURS.toString());
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_SIZE.toString(), "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_INPUT_TIMECOLUMN_TYPE.toString(), TimeUnit.MILLISECONDS.toString());
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
  aggPhaseConfig = AggregationPhaseConfig.fromThirdEyeConfig(thirdeyeConfig);

  // Mapper config
  AggregationMapper mapper = new AggregationMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  AggregationReducer reducer = new AggregationReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  Schema reducerSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  configuration.set(AggregationPhaseConstants.AGG_PHASE_AVRO_SCHEMA.toString(), reducerSchema.toString());

  configuration.set(AggregationPhaseConstants.AGG_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(AggregationPhaseConstants.AGG_PHASE_OUTPUT_PATH.toString(), outputPath);
  setUpAvroSerialization(reduceDriver.getConfiguration(), reducerSchema);

}
Example 19
Source File: TopkPhaseTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {

  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TABLE_NAME.toString(), "collection");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_NAMES.toString(), "d1,d2,d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_DIMENSION_TYPES.toString(), "STRING,LONG,STRING");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString(), "m1,m2");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), "INT,INT");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TIMECOLUMN_NAME.toString(), "hoursSinceEpoch");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_DIMENSION_NAMES.toString(), "d2,");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_METRICS.toString() + ".d2", "m1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_TOPK_KVALUES.toString() + ".d2", "1");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION_NAMES.toString(), "d3");
  props.setProperty(ThirdEyeConfigProperties.THIRDEYE_WHITELIST_DIMENSION.toString() + ".d3", "xyz2");
  thirdeyeConfig = ThirdEyeConfig.fromProperties(props);

  // Mapper config
  TopKPhaseMapper mapper = new TopKPhaseMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  inputSchema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(AVRO_SCHEMA));
  setUpAvroSerialization(mapDriver.getConfiguration(), inputSchema);

  // Reducer config
  TopKPhaseReducer reducer = new TopKPhaseReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");

  configuration.set(TopKPhaseConstants.TOPK_PHASE_THIRDEYE_CONFIG.toString(),
      OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(TopKPhaseConstants.TOPK_PHASE_OUTPUT_PATH.toString(), outputPath);

}
Example 20
Source File: CubeHFileMapperTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() {
    CubeHFileMapper mapper = new CubeHFileMapper();
    mapDriver = MapDriver.newMapDriver(mapper);
}