Java Code Examples for org.apache.tez.runtime.library.common.ConfigUtils#mergeConfsWithExclusions()

The following examples show how to use org.apache.tez.runtime.library.common.ConfigUtils#mergeConfsWithExclusions() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: OrderedPartitionedKVOutputConfig.java    From tez with Apache License 2.0 6 votes vote down vote up 
/**
 * Set serialization class and the relevant comparator to be used for sorting.
 * Providing custom serialization class could change the way, keys needs to be compared in
 * sorting. Providing invalid comparator here could create invalid results.
 *
 * @param serializationClassName
 * @param comparatorClassName
 * @param serializerConf         the serializer configuration. This can be null, and is a
 *                               {@link java.util.Map} of key-value pairs. The keys should be limited
 *                               to the ones required by the comparator.
 * @return this object for further chained method calls
 */
public Builder setKeySerializationClass(String serializationClassName,
    String comparatorClassName, @Nullable Map<String, String> serializerConf) {
  Preconditions.checkArgument(serializationClassName != null,
      "serializationClassName cannot be null");
  Preconditions.checkArgument(comparatorClassName != null,
      "comparator cannot be null");
  this.conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, serializationClassName + ","
      + conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY));
  setKeyComparatorClass(comparatorClassName, null);
  if (serializerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, serializerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 2
Source File: OrderedGroupedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
@Override
public Builder setCombiner(String combinerClassName, Map<String, String> combinerConf) {
  this.conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, combinerClassName);
  if (combinerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, combinerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 3
Source File: UnorderedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Set serialization class responsible for providing serializer/deserializer for key/value and
 * the corresponding comparator class to be used as key comparator.
 *
 * @param serializationClassName
 * @param serializerConf         the serializer configuration. This can be null, and is a
 *                               {@link java.util.Map} of key-value pairs. The keys should be limited
 *                               to the ones required by the comparator.
 * @return this object for further chained method calls
 */
public Builder setKeySerializationClass(String serializationClassName,
                                        @Nullable Map<String, String> serializerConf) {
  Preconditions.checkArgument(serializationClassName != null,
      "serializationClassName cannot be null");
  this.conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, serializationClassName + ","
      + conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY));
  if (serializerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, serializerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 4
Source File: UnorderedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
                              @Nullable Map<String, String> codecConf) {
  this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);
  if (enabled && compressionCodec != null) {
    this.conf
        .set(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, compressionCodec);
  }
  if (codecConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, codecConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 5
Source File: UnorderedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
                              @Nullable Map<String, String> codecConf) {
  this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);
  if (enabled && compressionCodec != null) {
    this.conf
        .set(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, compressionCodec);
  }
  if (codecConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, codecConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 6
Source File: UnorderedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Set serialization class responsible for providing serializer/deserializer for values.
 *
 * @param serializationClassName
 * @param serializerConf         the serializer configuration. This can be null, and is a
 *                               {@link java.util.Map} of key-value pairs. The keys should be limited
 *                               to the ones required by the comparator.
 * @return this object for further chained method calls
 */
public Builder setValueSerializationClass(String serializationClassName,
                                          @Nullable Map<String, String> serializerConf) {
  Preconditions.checkArgument(serializationClassName != null,
      "serializationClassName cannot be null");
  this.conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, serializationClassName + ","
      + conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY));
  if (serializerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, serializerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 7
Source File: UnorderedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Set serialization class responsible for providing serializer/deserializer for keys.
 *
 * @param serializationClassName
 * @param serializerConf         the serializer configuration. This can be null, and is a
 *                               {@link java.util.Map} of key-value pairs. The keys should be limited
 *                               to the ones required by the comparator.
 * @return this object for further chained method calls
 */
public Builder setKeySerializationClass(String serializationClassName,
                                        @Nullable Map<String, String> serializerConf) {
  Preconditions.checkArgument(serializationClassName != null,
      "serializationClassName cannot be null");
  this.conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, serializationClassName + ","
      + conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY));
  if (serializerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, serializerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 8
Source File: OrderedGroupedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Serialization class to be used for serializing values.
 *
 * @param serializationClassName
 * @param serializerConf         the serializer configuration. This can be null, and is a
 *                               {@link java.util.Map} of key-value pairs. The keys should be limited
 *                               to the ones required by the comparator.
 * @return this object for further chained method calls
 */
public Builder setValueSerializationClass(String serializationClassName,
                                          @Nullable Map<String, String> serializerConf) {
  Preconditions.checkArgument(serializationClassName != null,
      "serializationClassName cannot be null");
  this.conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, serializationClassName + ","
      + conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY));
  if (serializerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, serializerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 9
Source File: UnorderedPartitionedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
@InterfaceAudience.Private
Builder setPartitioner(String partitionerClassName, Map<String, String> partitionerConf) {
  Objects.requireNonNull(partitionerClassName, "Partitioner class name cannot be null");
  this.conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, partitionerClassName);
  if (partitionerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, partitionerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 10
Source File: OrderedGroupedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
                              @Nullable Map<String, String> codecConf) {
  this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);
  if (enabled && compressionCodec != null) {
    this.conf
        .set(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, compressionCodec);
  }
  if (codecConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, codecConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 11
Source File: OrderedGroupedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Set the key comparator class and it's associated configuration. This method should only be
 * used if the comparator requires some specific configuration, which is typically not the
 * case. {@link #setKeyComparatorClass(String)} is the preferred method for setting a
 * comparator.
 *
 * @param comparatorClassName the key comparator class name
 * @param comparatorConf      the comparator configuration. This can be null, and is a {@link
 *                            java.util.Map} of key-value pairs. The keys should be limited to
 *                            the ones required by the comparator.
 * @return instance of the current builder
 */
public Builder setKeyComparatorClass(String comparatorClassName,
                                     @Nullable Map<String, String> comparatorConf) {
  Objects.requireNonNull(comparatorClassName, "Comparator class name cannot be null");
  this.conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS,
      comparatorClassName);
  if (comparatorConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, comparatorConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 12
Source File: ShuffledMergedInputConfiguration.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Override
public Builder setCombiner(String combinerClassName, Configuration combinerConf) {
  this.conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, combinerClassName);
  if (combinerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, combinerConf,
        TezJobConfig.getRuntimeConfigKeySet());
  }
  return this;
}
Example 13
Source File: UnorderedKVInputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Set serialization class responsible for providing serializer/deserializer for values.
 *
 * @param serializationClassName
 * @param serializerConf         the serializer configuration. This can be null, and is a
 *                               {@link java.util.Map} of key-value pairs. The keys should be limited
 *                               to the ones required by the comparator.
 * @return this object for further chained method calls
 */
public Builder setValueSerializationClass(String serializationClassName,
                                          @Nullable Map<String, String> serializerConf) {
  Preconditions.checkArgument(serializationClassName != null,
      "serializationClassName cannot be null");
  this.conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, serializationClassName + ","
      + conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY));
  if (serializerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, serializerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 14
Source File: UnorderedPartitionedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
                              @Nullable Map<String, String> codecConf) {
  this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);
  if (enabled && compressionCodec != null) {
    this.conf
        .set(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, compressionCodec);
  }
  if (codecConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, codecConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 15
Source File: OrderedPartitionedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Set the key comparator class and it's associated configuration. This method should only be
 * used if the comparator requires some specific configuration, which is typically not the
 * case. {@link #setKeyComparatorClass(String)} is the preferred method for setting a
 * comparator.
 *
 * @param comparatorClassName the key comparator class name
 * @param comparatorConf      the comparator configuration. This can be null, and is a {@link
 *                            java.util.Map} of key-value pairs. The keys should be limited to
 *                            the ones required by the comparator.
 * @return instance of the current builder
 */
public Builder setKeyComparatorClass(String comparatorClassName,
                                     @Nullable Map<String, String> comparatorConf) {
  Objects.requireNonNull(comparatorClassName, "Comparator class name cannot be null");
  this.conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS,
      comparatorClassName);
  if (comparatorConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, comparatorConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 16
Source File: OrderedPartitionedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
@Override
public Builder setCombiner(String combinerClassName, Map<String, String> combinerConf) {
  this.conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, combinerClassName);
  if (combinerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, combinerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 17
Source File: OrderedPartitionedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up 
@InterfaceAudience.Private
Builder setPartitioner(String partitionerClassName, @Nullable Map<String, String> partitionerConf) {
  Objects.requireNonNull(partitionerClassName, "Partitioner class name cannot be null");
  this.conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, partitionerClassName);
  if (partitionerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, partitionerConf,
        TezRuntimeConfiguration.getRuntimeConfigKeySet());
  }
  return this;
}
Example 18
Source File: OnFileSortedOutputConfiguration.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Override
public Builder setCombiner(String combinerClassName, Configuration combinerConf) {
  this.conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, combinerClassName);
  if (combinerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, combinerConf,
        TezJobConfig.getRuntimeConfigKeySet());
  }
  return this;
}
Example 19
Source File: OnFileSortedOutputConfiguration.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@InterfaceAudience.Private
Builder setPartitioner(String partitionerClassName, Configuration partitionerConf) {
  Preconditions.checkNotNull(partitionerClassName, "Partitioner class name cannot be null");
  this.conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, partitionerClassName);
  if (partitionerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, partitionerConf,
        TezJobConfig.getRuntimeConfigKeySet());
  }
  return this;
}
Example 20
Source File: OnFileUnorderedPartitionedKVOutputConfiguration.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@InterfaceAudience.Private
Builder setPartitioner(String partitionerClassName, Configuration partitionerConf) {
  Preconditions.checkNotNull(partitionerClassName, "Partitioner class name cannot be null");
  this.conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, partitionerClassName);
  if (partitionerConf != null) {
    // Merging the confs for now. Change to be specific in the future.
    ConfigUtils.mergeConfsWithExclusions(this.conf, partitionerConf,
        TezJobConfig.getRuntimeConfigKeySet());
  }
  return this;
}