Java Code Examples for org.apache.flink.core.io.InputSplit#getSplitNumber()

The following examples show how to use org.apache.flink.core.io.InputSplit#getSplitNumber() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SlsRecordReader.java    From alibaba-flink-connectors with Apache License 2.0 4 votes vote down vote up 
@Override
public void open(
		InputSplit split, RuntimeContext context) throws IOException {
	SlsInputSplit slsInputSplit = (SlsInputSplit) split;
	LOG.info(String.format("open project[%s] logStore[%s],consumer[%s]-%s  startTime[%d)", project, logStore,
						accessKeyId,
						slsInputSplit.toString(), startInSec));

	int curRetry = 0;
	while (curRetry++ < maxRetryTime) {
		try {
			List<Shard> shardsList  = getShardsList();
			if (initPartitionCount != shardsList.size()){
				ErrorUtils.throwException(
						String.format("Source {%s} partitions number has changed from {%s} to {%s} \n " +
										"Wait the failover finish, blink is trying to recovery from " +
										"source partition change", getReaderName(),
								String.valueOf(initPartitionCount), String.valueOf(getPartitionsNums())));
			}
			this.shardId = split.getSplitNumber();
			for (Shard shard: shardsList) {
				if (shard.GetShardId() == this.shardId){
					this.shard = shard;
					break;
				}
			}
			if (shard.getStatus().equalsIgnoreCase("readonly")) {
				LOG.info("ShardId " + shard.GetShardId() + " status:readOnly");
				isReadOnlyShard = true;
				this.endCursor = getSlsClientProvider().getClient().GetCursor(project, logStore, shardId, Consts
						.CursorMode.END).GetCursor();
			} else {
				LOG.info("ShardId " + shard.GetShardId() + " status:readwrite");
				isReadOnlyShard = false;
			}
			this.nextBeginCursor = getSlsClientProvider().getClient()
														.GetCursor(project, logStore, shardId, startInSec)
														.GetCursor();
			if (stopInSec == Integer.MAX_VALUE) {
				this.stopCursor = null;

			} else {
				this.stopCursor = getSlsClientProvider().getClient()
						.GetCursor(project, logStore, shardId, stopInSec).GetCursor();
			}

			if (consumerGroup == null) {
				LOG.info(String.format(
						"Open method get init cursor, " +
						"project[%s]-logStore[%s]-shardId[%d]-startInSec[%d]-Cursor[%s]",
						project,
						logStore,
						shardId,
						startInSec,
						nextBeginCursor));
			} else {
				LOG.info(String.format(
						"Open method get init cursor, " +
						"project[%s]-logStore[%s]-shardId[%d]-startInSec[%d]-Cursor[%s]-ConsumerGroup[%s]",
						project,
						logStore,
						shardId,
						startInSec,
						nextBeginCursor,
						consumerGroup));
			}
			break;
		} catch (LogException e) {
			LOG.error("Error in get shard list", e);
			// refresh sts account
			getSlsClientProvider().getClient(true, true);
			if (curRetry == maxRetryTime) {
				ErrorUtils.throwException(
						e.getMessage());
			}
			try {
				Thread.sleep(curRetry * 500);
			} catch (Exception e1) {

			}
		}
	}
	initPartitionNumsListener();
}
Example 2
Source File: JDBCInputFormat.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Connects to the source database and executes the query in a <b>parallel
 * fashion</b> if
 * this {@link InputFormat} is built using a parameterized query (i.e. using
 * a {@link PreparedStatement})
 * and a proper {@link ParameterValuesProvider}, in a <b>non-parallel
 * fashion</b> otherwise.
 *
 * @param inputSplit which is ignored if this InputFormat is executed as a
 *        non-parallel source,
 *        a "hook" to the query parameters otherwise (using its
 *        <i>splitNumber</i>)
 * @throws IOException if there's an error during the execution of the query
 */
@Override
public void open(InputSplit inputSplit) throws IOException {
	try {
		if (inputSplit != null && parameterValues != null) {
			for (int i = 0; i < parameterValues[inputSplit.getSplitNumber()].length; i++) {
				Object param = parameterValues[inputSplit.getSplitNumber()][i];
				if (param instanceof String) {
					statement.setString(i + 1, (String) param);
				} else if (param instanceof Long) {
					statement.setLong(i + 1, (Long) param);
				} else if (param instanceof Integer) {
					statement.setInt(i + 1, (Integer) param);
				} else if (param instanceof Double) {
					statement.setDouble(i + 1, (Double) param);
				} else if (param instanceof Boolean) {
					statement.setBoolean(i + 1, (Boolean) param);
				} else if (param instanceof Float) {
					statement.setFloat(i + 1, (Float) param);
				} else if (param instanceof BigDecimal) {
					statement.setBigDecimal(i + 1, (BigDecimal) param);
				} else if (param instanceof Byte) {
					statement.setByte(i + 1, (Byte) param);
				} else if (param instanceof Short) {
					statement.setShort(i + 1, (Short) param);
				} else if (param instanceof Date) {
					statement.setDate(i + 1, (Date) param);
				} else if (param instanceof Time) {
					statement.setTime(i + 1, (Time) param);
				} else if (param instanceof Timestamp) {
					statement.setTimestamp(i + 1, (Timestamp) param);
				} else if (param instanceof Array) {
					statement.setArray(i + 1, (Array) param);
				} else {
					//extends with other types if needed
					throw new IllegalArgumentException("open() failed. Parameter " + i + " of type " + param.getClass() + " is not handled (yet).");
				}
			}
			if (LOG.isDebugEnabled()) {
				LOG.debug(String.format("Executing '%s' with parameters %s", queryTemplate, Arrays.deepToString(parameterValues[inputSplit.getSplitNumber()])));
			}
		}
		resultSet = statement.executeQuery();
		hasNext = resultSet.next();
	} catch (SQLException se) {
		throw new IllegalArgumentException("open() failed." + se.getMessage(), se);
	}
}
Example 3
Source File: JDBCInputFormat.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Connects to the source database and executes the query in a <b>parallel
 * fashion</b> if
 * this {@link InputFormat} is built using a parameterized query (i.e. using
 * a {@link PreparedStatement})
 * and a proper {@link ParameterValuesProvider}, in a <b>non-parallel
 * fashion</b> otherwise.
 *
 * @param inputSplit which is ignored if this InputFormat is executed as a
 *        non-parallel source,
 *        a "hook" to the query parameters otherwise (using its
 *        <i>splitNumber</i>)
 * @throws IOException if there's an error during the execution of the query
 */
@Override
public void open(InputSplit inputSplit) throws IOException {
	try {
		if (inputSplit != null && parameterValues != null) {
			for (int i = 0; i < parameterValues[inputSplit.getSplitNumber()].length; i++) {
				Object param = parameterValues[inputSplit.getSplitNumber()][i];
				if (param instanceof String) {
					statement.setString(i + 1, (String) param);
				} else if (param instanceof Long) {
					statement.setLong(i + 1, (Long) param);
				} else if (param instanceof Integer) {
					statement.setInt(i + 1, (Integer) param);
				} else if (param instanceof Double) {
					statement.setDouble(i + 1, (Double) param);
				} else if (param instanceof Boolean) {
					statement.setBoolean(i + 1, (Boolean) param);
				} else if (param instanceof Float) {
					statement.setFloat(i + 1, (Float) param);
				} else if (param instanceof BigDecimal) {
					statement.setBigDecimal(i + 1, (BigDecimal) param);
				} else if (param instanceof Byte) {
					statement.setByte(i + 1, (Byte) param);
				} else if (param instanceof Short) {
					statement.setShort(i + 1, (Short) param);
				} else if (param instanceof Date) {
					statement.setDate(i + 1, (Date) param);
				} else if (param instanceof Time) {
					statement.setTime(i + 1, (Time) param);
				} else if (param instanceof Timestamp) {
					statement.setTimestamp(i + 1, (Timestamp) param);
				} else if (param instanceof Array) {
					statement.setArray(i + 1, (Array) param);
				} else {
					//extends with other types if needed
					throw new IllegalArgumentException("open() failed. Parameter " + i + " of type " + param.getClass() + " is not handled (yet).");
				}
			}
			if (LOG.isDebugEnabled()) {
				LOG.debug(String.format("Executing '%s' with parameters %s", queryTemplate, Arrays.deepToString(parameterValues[inputSplit.getSplitNumber()])));
			}
		}
		resultSet = statement.executeQuery();
		hasNext = resultSet.next();
	} catch (SQLException se) {
		throw new IllegalArgumentException("open() failed." + se.getMessage(), se);
	}
}
Example 4
Source File: JdbcInputFormat.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Connects to the source database and executes the query in a <b>parallel
 * fashion</b> if
 * this {@link InputFormat} is built using a parameterized query (i.e. using
 * a {@link PreparedStatement})
 * and a proper {@link JdbcParameterValuesProvider}, in a <b>non-parallel
 * fashion</b> otherwise.
 *
 * @param inputSplit which is ignored if this InputFormat is executed as a
 *        non-parallel source,
 *        a "hook" to the query parameters otherwise (using its
 *        <i>splitNumber</i>)
 * @throws IOException if there's an error during the execution of the query
 */
@Override
public void open(InputSplit inputSplit) throws IOException {
	try {
		if (inputSplit != null && parameterValues != null) {
			for (int i = 0; i < parameterValues[inputSplit.getSplitNumber()].length; i++) {
				Object param = parameterValues[inputSplit.getSplitNumber()][i];
				if (param instanceof String) {
					statement.setString(i + 1, (String) param);
				} else if (param instanceof Long) {
					statement.setLong(i + 1, (Long) param);
				} else if (param instanceof Integer) {
					statement.setInt(i + 1, (Integer) param);
				} else if (param instanceof Double) {
					statement.setDouble(i + 1, (Double) param);
				} else if (param instanceof Boolean) {
					statement.setBoolean(i + 1, (Boolean) param);
				} else if (param instanceof Float) {
					statement.setFloat(i + 1, (Float) param);
				} else if (param instanceof BigDecimal) {
					statement.setBigDecimal(i + 1, (BigDecimal) param);
				} else if (param instanceof Byte) {
					statement.setByte(i + 1, (Byte) param);
				} else if (param instanceof Short) {
					statement.setShort(i + 1, (Short) param);
				} else if (param instanceof Date) {
					statement.setDate(i + 1, (Date) param);
				} else if (param instanceof Time) {
					statement.setTime(i + 1, (Time) param);
				} else if (param instanceof Timestamp) {
					statement.setTimestamp(i + 1, (Timestamp) param);
				} else if (param instanceof Array) {
					statement.setArray(i + 1, (Array) param);
				} else {
					//extends with other types if needed
					throw new IllegalArgumentException("open() failed. Parameter " + i + " of type " + param.getClass() + " is not handled (yet).");
				}
			}
			if (LOG.isDebugEnabled()) {
				LOG.debug(String.format("Executing '%s' with parameters %s", queryTemplate, Arrays.deepToString(parameterValues[inputSplit.getSplitNumber()])));
			}
		}
		resultSet = statement.executeQuery();
		hasNext = resultSet.next();
	} catch (SQLException se) {
		throw new IllegalArgumentException("open() failed." + se.getMessage(), se);
	}
}
Example 5
Source File: JdbcRowDataInputFormat.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Connects to the source database and executes the query in a <b>parallel
 * fashion</b> if
 * this {@link InputFormat} is built using a parameterized query (i.e. using
 * a {@link PreparedStatement})
 * and a proper {@link JdbcParameterValuesProvider}, in a <b>non-parallel
 * fashion</b> otherwise.
 *
 * @param inputSplit which is ignored if this InputFormat is executed as a
 *                   non-parallel source,
 *                   a "hook" to the query parameters otherwise (using its
 *                   <i>splitNumber</i>)
 * @throws IOException if there's an error during the execution of the query
 */
@Override
public void open(InputSplit inputSplit) throws IOException {
	try {
		if (inputSplit != null && parameterValues != null) {
			for (int i = 0; i < parameterValues[inputSplit.getSplitNumber()].length; i++) {
				Object param = parameterValues[inputSplit.getSplitNumber()][i];
				if (param instanceof String) {
					statement.setString(i + 1, (String) param);
				} else if (param instanceof Long) {
					statement.setLong(i + 1, (Long) param);
				} else if (param instanceof Integer) {
					statement.setInt(i + 1, (Integer) param);
				} else if (param instanceof Double) {
					statement.setDouble(i + 1, (Double) param);
				} else if (param instanceof Boolean) {
					statement.setBoolean(i + 1, (Boolean) param);
				} else if (param instanceof Float) {
					statement.setFloat(i + 1, (Float) param);
				} else if (param instanceof BigDecimal) {
					statement.setBigDecimal(i + 1, (BigDecimal) param);
				} else if (param instanceof Byte) {
					statement.setByte(i + 1, (Byte) param);
				} else if (param instanceof Short) {
					statement.setShort(i + 1, (Short) param);
				} else if (param instanceof Date) {
					statement.setDate(i + 1, (Date) param);
				} else if (param instanceof Time) {
					statement.setTime(i + 1, (Time) param);
				} else if (param instanceof Timestamp) {
					statement.setTimestamp(i + 1, (Timestamp) param);
				} else if (param instanceof Array) {
					statement.setArray(i + 1, (Array) param);
				} else {
					//extends with other types if needed
					throw new IllegalArgumentException("open() failed. Parameter " + i + " of type " + param.getClass() + " is not handled (yet).");
				}
			}
			if (LOG.isDebugEnabled()) {
				LOG.debug(String.format("Executing '%s' with parameters %s", queryTemplate, Arrays.deepToString(parameterValues[inputSplit.getSplitNumber()])));
			}
		}
		resultSet = statement.executeQuery();
		hasNext = resultSet.next();
	} catch (SQLException se) {
		throw new IllegalArgumentException("open() failed." + se.getMessage(), se);
	}
}