org.apache.flink.table.api.config.ExecutionConfigOptions Java Examples

@Test
public void testGetValidShuffleMode() {
	final Configuration configuration = new Configuration();

	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, GlobalDataExchangeMode.ALL_EDGES_BLOCKING.toString());
	assertEquals(GlobalDataExchangeMode.ALL_EDGES_BLOCKING, ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration));

	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, GlobalDataExchangeMode.FORWARD_EDGES_PIPELINED.toString());
	assertEquals(GlobalDataExchangeMode.FORWARD_EDGES_PIPELINED, ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration));

	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, GlobalDataExchangeMode.POINTWISE_EDGES_PIPELINED.toString());
	assertEquals(GlobalDataExchangeMode.POINTWISE_EDGES_PIPELINED, ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration));

	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, GlobalDataExchangeMode.ALL_EDGES_PIPELINED.toString());
	assertEquals(GlobalDataExchangeMode.ALL_EDGES_PIPELINED, ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration));
}
 

@Test
public void testConfiguration() throws Exception {
	final ExecutionContext<?> context = createConfigurationExecutionContext();
	final TableEnvironment tableEnv = context.getTableEnvironment();

	Configuration conf = tableEnv.getConfig().getConfiguration();
	assertEquals(100, conf.getInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_DEFAULT_LIMIT));
	assertTrue(conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED));
	assertEquals("128kb", conf.getString(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE));

	assertTrue(conf.getBoolean(OptimizerConfigOptions.TABLE_OPTIMIZER_JOIN_REORDER_ENABLED));

	// these options are not modified and should be equal to their default value
	assertEquals(
		ExecutionConfigOptions.TABLE_EXEC_SORT_ASYNC_MERGE_ENABLED.defaultValue(),
		conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SORT_ASYNC_MERGE_ENABLED));
	assertEquals(
		ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE.defaultValue(),
		conf.getString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE));
	assertEquals(
		OptimizerConfigOptions.TABLE_OPTIMIZER_BROADCAST_JOIN_THRESHOLD.defaultValue().longValue(),
		conf.getLong(OptimizerConfigOptions.TABLE_OPTIMIZER_BROADCAST_JOIN_THRESHOLD));
}
 

private boolean failOrFilterNullValues(RowData row) {
	for (int index : notNullFieldIndices) {
		if (row.isNullAt(index)) {
			if (notNullEnforcer == NotNullEnforcer.ERROR) {
				String optionKey = ExecutionConfigOptions.TABLE_EXEC_SINK_NOT_NULL_ENFORCER.key();
				throw new TableException(
					String.format("Column '%s' is NOT NULL, however, a null value is being written into it. " +
						"You can set job configuration '" + optionKey + "'='drop' " +
						"to suppress this exception and drop such records silently.", allFieldNames[index]));
			} else {
				// simply drop the record
				return true;
			}
		}
	}
	return false;
}
 

private void testSourceConfig(boolean fallbackMR, boolean inferParallelism) throws Exception {
	HiveTableFactory tableFactorySpy = spy((HiveTableFactory) hiveCatalog.getTableFactory().get());

	doAnswer(invocation -> {
		TableSourceFactory.Context context = invocation.getArgument(0);
		return new TestConfigSource(
				new JobConf(hiveCatalog.getHiveConf()),
				context.getConfiguration(),
				context.getObjectIdentifier().toObjectPath(),
				context.getTable(),
				fallbackMR,
				inferParallelism);
	}).when(tableFactorySpy).createTableSource(any(TableSourceFactory.Context.class));

	HiveCatalog catalogSpy = spy(hiveCatalog);
	doReturn(Optional.of(tableFactorySpy)).when(catalogSpy).getTableFactory();

	TableEnvironment tableEnv = HiveTestUtils.createTableEnvWithBlinkPlannerBatchMode();
	tableEnv.getConfig().getConfiguration().setBoolean(
			HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER, fallbackMR);
	tableEnv.getConfig().getConfiguration().setBoolean(
			HiveOptions.TABLE_EXEC_HIVE_INFER_SOURCE_PARALLELISM, inferParallelism);
	tableEnv.getConfig().getConfiguration().setInteger(
			ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 2);
	tableEnv.registerCatalog(catalogSpy.getName(), catalogSpy);
	tableEnv.useCatalog(catalogSpy.getName());

	List<Row> results = Lists.newArrayList(
			tableEnv.sqlQuery("select * from db1.src order by x").execute().collect());
	assertEquals("[1,a, 2,b]", results.toString());
}
 

@Test
public void testGetShuffleModeIgnoreCases() {
	final Configuration configuration = new Configuration();

	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, "Forward_edges_PIPELINED");
	assertEquals(GlobalDataExchangeMode.FORWARD_EDGES_PIPELINED, ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration));

	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, "Pipelined");
	assertEquals(GlobalDataExchangeMode.ALL_EDGES_PIPELINED, ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration));
}
 

@Test
public void testGetLegacyShuffleMode() {
	final Configuration configuration = new Configuration();

	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, ShuffleModeUtils.ALL_EDGES_BLOCKING_LEGACY);
	assertEquals(GlobalDataExchangeMode.ALL_EDGES_BLOCKING, ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration));

	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, ShuffleModeUtils.ALL_EDGES_PIPELINED_LEGACY);
	assertEquals(GlobalDataExchangeMode.ALL_EDGES_PIPELINED, ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration));
}
 

static GlobalDataExchangeMode getShuffleModeAsGlobalDataExchangeMode(final Configuration configuration) {
	final String value = configuration.getString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE);

	try {
		return GlobalDataExchangeMode.valueOf(convertLegacyShuffleMode(value).toUpperCase());
	} catch (IllegalArgumentException e) {
		throw new IllegalArgumentException(
			String.format("Unsupported value %s for config %s.", value, ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE.key()));
	}
}
 

@Before
public void setup() {
	TypeInformation[] types = new TypeInformation[]{Types.INT, Types.INT};
	this.buildSideSerializer = new BinaryRowDataSerializer(types.length);
	this.probeSideSerializer = new BinaryRowDataSerializer(types.length);

	this.ioManager = new IOManagerAsync();

	conf = new Configuration();
	conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, useCompress);
}
 

@Before
public void init() {
	TypeInformation[] types = new TypeInformation[]{Types.INT, Types.INT};
	this.buildSideSerializer = new BinaryRowDataSerializer(types.length);
	this.probeSideSerializer = new BinaryRowDataSerializer(types.length);
	this.ioManager = new IOManagerAsync();

	conf = new Configuration();
	conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, useCompress);
}
 

public BufferedKVExternalSorterTest(
		int spillNumber, int recordNumberPerFile, boolean spillCompress) {
	ioManager = new IOManagerAsync();
	conf = new Configuration();
	conf.setInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES, 5);
	if (!spillCompress) {
		conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, false);
	}
	this.spillNumber = spillNumber;
	this.recordNumberPerFile = recordNumberPerFile;
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
	this.serializer = new BinaryRowDataSerializer(2);
	this.conf.setInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES, 128);
}
 

public BinaryExternalSorterTest(
		boolean spillCompress,
		boolean asyncMerge) {
	ioManager = new IOManagerAsync();
	conf = new Configuration();
	if (!spillCompress) {
		conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, false);
	}
	if (asyncMerge) {
		conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SORT_ASYNC_MERGE_ENABLED, true);
	}
}
 

public BufferedKVExternalSorter(
		IOManager ioManager,
		BinaryRowDataSerializer keySerializer,
		BinaryRowDataSerializer valueSerializer,
		NormalizedKeyComputer nKeyComputer,
		RecordComparator comparator,
		int pageSize,
		Configuration conf) throws IOException {
	this.keySerializer = keySerializer;
	this.valueSerializer = valueSerializer;
	this.nKeyComputer = nKeyComputer;
	this.comparator = comparator;
	this.pageSize = pageSize;
	this.sorter = new QuickSort();
	this.maxNumFileHandles = conf.getInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES);
	this.compressionEnable = conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED);
	this.compressionCodecFactory = this.compressionEnable
		? BlockCompressionFactory.createBlockCompressionFactory(
			BlockCompressionFactory.CompressionFactoryName.LZ4.toString())
		: null;
	this.compressionBlockSize = (int) MemorySize.parse(
		conf.getString(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE)).getBytes();
	this.ioManager = ioManager;
	this.enumerator = this.ioManager.createChannelEnumerator();
	this.channelManager = new SpillChannelManager();
	this.merger = new BinaryKVExternalMerger(
			ioManager, pageSize,
			maxNumFileHandles, channelManager,
			keySerializer, valueSerializer, comparator,
			compressionEnable,
		compressionCodecFactory,
			compressionBlockSize);
}
 

@Test
public void testParallelismOnLimitPushDown() {
	final String dbName = "source_db";
	final String tblName = "test_parallelism_limit_pushdown";
	TableEnvironment tEnv = createTableEnv();
	tEnv.getConfig().getConfiguration().setBoolean(
			HiveOptions.TABLE_EXEC_HIVE_INFER_SOURCE_PARALLELISM, false);
	tEnv.getConfig().getConfiguration().setInteger(
			ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 2);
	tEnv.executeSql("CREATE TABLE source_db.test_parallelism_limit_pushdown " +
				"(`year` STRING, `value` INT) partitioned by (pt int)");
	HiveTestUtils.createTextTableInserter(hiveShell, dbName, tblName)
				.addRow(new Object[]{"2014", 3})
				.addRow(new Object[]{"2014", 4})
				.commit("pt=0");
	HiveTestUtils.createTextTableInserter(hiveShell, dbName, tblName)
				.addRow(new Object[]{"2015", 2})
				.addRow(new Object[]{"2015", 5})
				.commit("pt=1");
	Table table = tEnv.sqlQuery("select * from hive.source_db.test_parallelism_limit_pushdown limit 1");
	PlannerBase planner = (PlannerBase) ((TableEnvironmentImpl) tEnv).getPlanner();
	RelNode relNode = planner.optimize(TableTestUtil.toRelNode(table));
	ExecNode execNode = planner.translateToExecNodePlan(toScala(Collections.singletonList(relNode))).get(0);
	@SuppressWarnings("unchecked")
	Transformation transformation = execNode.translateToPlan(planner);
	Assert.assertEquals(1, ((PartitionTransformation) ((OneInputTransformation) transformation).getInput())
		.getInput().getParallelism());
}
 

private Map<String, ConfigOption> extractTableConfigOptions() {
  Map<String, ConfigOption> configOptions = new HashMap<>();
  configOptions.putAll(extractConfigOptions(ExecutionConfigOptions.class));
  configOptions.putAll(extractConfigOptions(OptimizerConfigOptions.class));
  configOptions.putAll(extractConfigOptions(PythonOptions.class));
  return configOptions;
}
 

@Test
public void testConfiguration() throws Exception {
	final ExecutionContext<?> context = createConfigurationExecutionContext();
	final TableEnvironment tableEnv = context.createEnvironmentInstance().getTableEnvironment();

	assertEquals(
		100,
		tableEnv.getConfig().getConfiguration().getInteger(
			ExecutionConfigOptions.TABLE_EXEC_SORT_DEFAULT_LIMIT));
	assertTrue(
		tableEnv.getConfig().getConfiguration().getBoolean(
			ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED));
	assertEquals(
		"128kb",
		tableEnv.getConfig().getConfiguration().getString(
			ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE));

	assertTrue(
		tableEnv.getConfig().getConfiguration().getBoolean(
			OptimizerConfigOptions.TABLE_OPTIMIZER_JOIN_REORDER_ENABLED));

	// these options are not modified and should be equal to their default value
	assertEquals(
		ExecutionConfigOptions.TABLE_EXEC_SORT_ASYNC_MERGE_ENABLED.defaultValue(),
		tableEnv.getConfig().getConfiguration().getBoolean(
			ExecutionConfigOptions.TABLE_EXEC_SORT_ASYNC_MERGE_ENABLED));
	assertEquals(
		ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE.defaultValue(),
		tableEnv.getConfig().getConfiguration().getString(
			ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE));
	assertEquals(
		OptimizerConfigOptions.TABLE_OPTIMIZER_BROADCAST_JOIN_THRESHOLD.defaultValue().longValue(),
		tableEnv.getConfig().getConfiguration().getLong(
			OptimizerConfigOptions.TABLE_OPTIMIZER_BROADCAST_JOIN_THRESHOLD));
}
 

private boolean isShuffleModeAllBatch() {
	String value = tableConfig.getConfiguration().getString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE);
	if (value.equalsIgnoreCase(ShuffleMode.BATCH.toString())) {
		return true;
	} else if (!value.equalsIgnoreCase(ShuffleMode.PIPELINED.toString())) {
		throw new IllegalArgumentException(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE.key() +
				" can only be set to " + ShuffleMode.BATCH.toString() + " or " + ShuffleMode.PIPELINED.toString());
	}
	return false;
}
 

@Before
public void setup() {
	TypeInformation[] types = new TypeInformation[]{Types.INT, Types.INT};
	this.buildSideSerializer = new BinaryRowSerializer(types.length);
	this.probeSideSerializer = new BinaryRowSerializer(types.length);

	this.ioManager = new IOManagerAsync();

	conf = new Configuration();
	conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, useCompress);
}
 

@Before
public void init() {
	TypeInformation[] types = new TypeInformation[]{Types.INT, Types.INT};
	this.buildSideSerializer = new BinaryRowSerializer(types.length);
	this.probeSideSerializer = new BinaryRowSerializer(types.length);
	this.ioManager = new IOManagerAsync();

	conf = new Configuration();
	conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, useCompress);
}
 

public BufferedKVExternalSorterTest(
		int spillNumber, int recordNumberPerFile, boolean spillCompress) {
	ioManager = new IOManagerAsync();
	conf = new Configuration();
	conf.setInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES, 5);
	if (!spillCompress) {
		conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, false);
	}
	this.spillNumber = spillNumber;
	this.recordNumberPerFile = recordNumberPerFile;
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.serializer = new BinaryRowSerializer(2);
	this.conf.setInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES, 128);
}
 

public BinaryExternalSorterTest(
		boolean spillCompress,
		boolean asyncMerge) {
	ioManager = new IOManagerAsync();
	conf = new Configuration();
	if (!spillCompress) {
		conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED, false);
	}
	if (asyncMerge) {
		conf.setBoolean(ExecutionConfigOptions.TABLE_EXEC_SORT_ASYNC_MERGE_ENABLED, true);
	}
}
 

public BufferedKVExternalSorter(
		IOManager ioManager,
		BinaryRowSerializer keySerializer,
		BinaryRowSerializer valueSerializer,
		NormalizedKeyComputer nKeyComputer,
		RecordComparator comparator,
		int pageSize,
		Configuration conf) throws IOException {
	this.keySerializer = keySerializer;
	this.valueSerializer = valueSerializer;
	this.nKeyComputer = nKeyComputer;
	this.comparator = comparator;
	this.pageSize = pageSize;
	this.sorter = new QuickSort();
	this.maxNumFileHandles = conf.getInteger(ExecutionConfigOptions.TABLE_EXEC_SORT_MAX_NUM_FILE_HANDLES);
	this.compressionEnable = conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED);
	this.compressionCodecFactory = this.compressionEnable
		? BlockCompressionFactory.createBlockCompressionFactory(
			BlockCompressionFactory.CompressionFactoryName.LZ4.toString())
		: null;
	this.compressionBlockSize = (int) MemorySize.parse(
		conf.getString(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE)).getBytes();
	this.ioManager = ioManager;
	this.enumerator = this.ioManager.createChannelEnumerator();
	this.channelManager = new SpillChannelManager();
	this.merger = new BinaryKVExternalMerger(
			ioManager, pageSize,
			maxNumFileHandles, channelManager,
			keySerializer, valueSerializer, comparator,
			compressionEnable,
		compressionCodecFactory,
			compressionBlockSize);
}
 

/**
 * Prepare TableEnvironment for query.
 *
 * @param sourceTablePath
 * @return
 */
private static TableEnvironment prepareTableEnv(String sourceTablePath, Boolean useTableStats) {
	//init Table Env
	EnvironmentSettings environmentSettings = EnvironmentSettings
			.newInstance()
			.useBlinkPlanner()
			.inBatchMode()
			.build();
	TableEnvironment tEnv = TableEnvironment.create(environmentSettings);

	//config Optimizer parameters
	tEnv.getConfig().getConfiguration()
			.setInteger(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 4);
	tEnv.getConfig().getConfiguration()
			.setLong(OptimizerConfigOptions.TABLE_OPTIMIZER_BROADCAST_JOIN_THRESHOLD, 10 * 1024 * 1024);
	tEnv.getConfig().getConfiguration()
			.setBoolean(OptimizerConfigOptions.TABLE_OPTIMIZER_JOIN_REORDER_ENABLED, true);

	//register TPC-DS tables
	TPCDS_TABLES.forEach(table -> {
		TpcdsSchema schema = TpcdsSchemaProvider.getTableSchema(table);
		CsvTableSource.Builder builder = CsvTableSource.builder();
		builder.path(sourceTablePath + FILE_SEPARATOR + table + DATA_SUFFIX);
		for (int i = 0; i < schema.getFieldNames().size(); i++) {
			builder.field(
					schema.getFieldNames().get(i),
					TypeConversions.fromDataTypeToLegacyInfo(schema.getFieldTypes().get(i)));
		}
		builder.fieldDelimiter(COL_DELIMITER);
		builder.emptyColumnAsNull();
		builder.lineDelimiter("\n");
		CsvTableSource tableSource = builder.build();
		ConnectorCatalogTable catalogTable = ConnectorCatalogTable.source(tableSource, true);
		tEnv.getCatalog(tEnv.getCurrentCatalog()).ifPresent(catalog -> {
			try {
				catalog.createTable(new ObjectPath(tEnv.getCurrentDatabase(), table), catalogTable, false);
			} catch (Exception e) {
				throw new RuntimeException(e);
			}
		});
	});
	// register statistics info
	if (useTableStats) {
		TpcdsStatsProvider.registerTpcdsStats(tEnv);
	}
	return tEnv;
}
 

@Test(expected = IllegalArgumentException.class)
public void testGetInvalidShuffleMode() {
	final Configuration configuration = new Configuration();
	configuration.setString(ExecutionConfigOptions.TABLE_EXEC_SHUFFLE_MODE, "invalid-value");
	ShuffleModeUtils.getShuffleModeAsGlobalDataExchangeMode(configuration);
}
 

public BaseHybridHashTable(
		Configuration conf,
		Object owner,
		MemoryManager memManager,
		long reservedMemorySize,
		IOManager ioManager,
		int avgRecordLen,
		long buildRowCount,
		boolean tryDistinctBuildRow) {

	//TODO: read compression config from configuration
	this.compressionEnable = conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED);
	this.compressionCodecFactory = this.compressionEnable
			? BlockCompressionFactory.createBlockCompressionFactory(BlockCompressionFactory.CompressionFactoryName.LZ4.toString())
			: null;
	this.compressionBlockSize = (int) MemorySize.parse(
		conf.getString(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE)).getBytes();

	this.avgRecordLen = avgRecordLen;
	this.buildRowCount = buildRowCount;
	this.tryDistinctBuildRow = tryDistinctBuildRow;
	this.totalNumBuffers = (int) (reservedMemorySize / memManager.getPageSize());
	// some sanity checks first
	checkArgument(totalNumBuffers >= MIN_NUM_MEMORY_SEGMENTS);
	this.internalPool = new LazyMemorySegmentPool(owner, memManager, totalNumBuffers);
	this.ioManager = ioManager;

	this.segmentSize = memManager.getPageSize();

	checkArgument(MathUtils.isPowerOf2(segmentSize));

	// take away the write behind buffers
	this.buildSpillReturnBuffers = new LinkedBlockingQueue<>();

	this.segmentSizeBits = MathUtils.log2strict(segmentSize);
	this.segmentSizeMask = segmentSize - 1;

	// open builds the initial table by consuming the build-side input
	this.currentRecursionDepth = 0;

	// create the partitions
	this.initPartitionFanOut = Math.min(getPartitioningFanOutNoEstimates(), maxNumPartition());

	this.closed.set(false);

	LOG.info(String.format("Initialize hash table with %d memory segments, each size [%d], the memory %d MB.",
			totalNumBuffers, segmentSize, (long) totalNumBuffers * segmentSize / 1024 / 1024));
}
 

@Test
public void testSetTableConfig() throws InterpreterException, IOException {
  hiveShell.execute("create table source_table (id int, name string)");
  hiveShell.execute("insert into source_table values(1, 'a'), (2, 'b')");

  File destDir = Files.createTempDirectory("flink_test").toFile();
  FileUtils.deleteDirectory(destDir);
  InterpreterResult result = sqlInterpreter.interpret(
          "CREATE TABLE sink_table (\n" +
                  "id int,\n" +
                  "name string" +
                  ") WITH (\n" +
                  "'format.field-delimiter'=',',\n" +
                  "'connector.type'='filesystem',\n" +
                  "'format.derive-schema'='true',\n" +
                  "'connector.path'='" + destDir.getAbsolutePath() + "',\n" +
                  "'format.type'='csv'\n" +
                  ");", getInterpreterContext());
  assertEquals(InterpreterResult.Code.SUCCESS, result.code());
  
  // set parallelism then insert into
  InterpreterContext context = getInterpreterContext();
  result = sqlInterpreter.interpret(
          "set table.exec.resource.default-parallelism=10;" +
                  "insert into sink_table select * from source_table", context);
  assertEquals(InterpreterResult.Code.SUCCESS, result.code());
  List<InterpreterResultMessage> resultMessages = context.out.toInterpreterResultMessage();
  assertEquals("Insertion successfully.\n", resultMessages.get(0).getData());
  assertEquals(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM.defaultValue(),
          sqlInterpreter.tbenv.getConfig().getConfiguration().get(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM));

  // set then insert into
  destDir.delete();
  context = getInterpreterContext();
  result = sqlInterpreter.interpret(
          "set table.optimizer.source.predicate-pushdown-enabled=false;" +
                  "insert into sink_table select * from source_table", context);
  assertEquals(InterpreterResult.Code.SUCCESS, result.code());
  resultMessages = context.out.toInterpreterResultMessage();
  assertEquals("Insertion successfully.\n", resultMessages.get(0).getData());
  assertEquals(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM.defaultValue(),
          sqlInterpreter.tbenv.getConfig().getConfiguration().get(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM));
  assertEquals(OptimizerConfigOptions.TABLE_OPTIMIZER_SOURCE_PREDICATE_PUSHDOWN_ENABLED.defaultValue(),
          sqlInterpreter.tbenv.getConfig().getConfiguration().get(OptimizerConfigOptions.TABLE_OPTIMIZER_SOURCE_PREDICATE_PUSHDOWN_ENABLED));

  // invalid config
  destDir.delete();
  context = getInterpreterContext();
  result = sqlInterpreter.interpret(
          "set table.invalid_config=false;" +
                  "insert into sink_table select * from source_table", context);
  assertEquals(InterpreterResult.Code.ERROR, result.code());
  resultMessages = context.out.toInterpreterResultMessage();
  assertTrue(resultMessages.get(0).getData(),
          resultMessages.get(0).getData().contains("table.invalid_config is not a valid table/sql config"));
}
 

public BaseHybridHashTable(
		Configuration conf,
		Object owner,
		MemoryManager memManager,
		long reservedMemorySize,
		long preferredMemorySize,
		long perRequestMemorySize,
		IOManager ioManager,
		int avgRecordLen,
		long buildRowCount,
		boolean tryDistinctBuildRow) {

	//TODO: read compression config from configuration
	this.compressionEnable = conf.getBoolean(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_ENABLED);
	this.compressionCodecFactory = this.compressionEnable
			? BlockCompressionFactory.createBlockCompressionFactory(BlockCompressionFactory.CompressionFactoryName.LZ4.toString())
			: null;
	this.compressionBlockSize = (int) MemorySize.parse(
		conf.getString(ExecutionConfigOptions.TABLE_EXEC_SPILL_COMPRESSION_BLOCK_SIZE)).getBytes();

	this.owner = owner;
	this.avgRecordLen = avgRecordLen;
	this.buildRowCount = buildRowCount;
	this.tryDistinctBuildRow = tryDistinctBuildRow;
	this.reservedNumBuffers = (int) (reservedMemorySize / memManager.getPageSize());
	// some sanity checks first
	checkArgument(reservedNumBuffers >= MIN_NUM_MEMORY_SEGMENTS);
	this.maxNumBuffers = (int) (preferredMemorySize / memManager.getPageSize());
	this.perRequestNumBuffers = (int) (perRequestMemorySize / memManager.getPageSize());
	this.availableMemory = new ArrayList<>(this.reservedNumBuffers);
	try {
		List<MemorySegment> allocates = memManager.allocatePages(owner, this.reservedNumBuffers);
		this.availableMemory.addAll(allocates);
		allocates.clear();
	} catch (MemoryAllocationException e) {
		LOG.error("Out of memory", e);
		throw new RuntimeException(e);
	}
	this.memManager = memManager;
	this.ioManager = ioManager;

	this.segmentSize = memManager.getPageSize();

	checkArgument(MathUtils.isPowerOf2(segmentSize));

	// take away the write behind buffers
	this.buildSpillReturnBuffers = new LinkedBlockingQueue<>();

	this.segmentSizeBits = MathUtils.log2strict(segmentSize);
	this.segmentSizeMask = segmentSize - 1;

	// open builds the initial table by consuming the build-side input
	this.currentRecursionDepth = 0;

	// create the partitions
	this.initPartitionFanOut = Math.min(getPartitioningFanOutNoEstimates(), maxNumPartition());

	this.closed.set(false);

	LOG.info(String.format("Initialize hash table with %d memory segments, each size [%d], the reserved memory %d" +
					" MB, the max memory %d MB, per allocate {} segments from floating memory pool.",
			reservedNumBuffers, segmentSize, (long) reservedNumBuffers * segmentSize / 1024 / 1024,
			(long) maxNumBuffers * segmentSize / 1024 / 1024), perRequestNumBuffers);
}