com.aliyun.openservices.log.common.Shard Java Examples

@Override
public List<Tuple2<InputSplit, String>> reAssignInputSplitsForCurrentSubTask(
		int numberOfParallelSubTasks, int indexOfThisSubTask, List<InnerProgress<String>> allSplitsInState)
		throws IOException {
	List<Tuple2<InputSplit, String>> initialProgess = new ArrayList<>();
	List<Shard> subscribedPartitions = modAssign(numberOfParallelSubTasks, indexOfThisSubTask);
	for (Shard shard : subscribedPartitions) {
		boolean existBefore = false;
		for (InnerProgress<String> progress: allSplitsInState) {
			if (shard.GetShardId() == progress.getInputSplit().getSplitNumber()){
				initialProgess.add(new Tuple2<>(progress.getInputSplit(), progress.getCursor()));
				existBefore = true;
				break;
			}
		}
		if (!existBefore) {
			// 新增加的shardId 标识0为shard的开头
			initialProgess.add(Tuple2.of(new SlsInputSplit(shard.GetShardId()), NEW_SLS_START_FLAG));
		}
	}

	return initialProgess;
}
 

public SlsRecordReader(
		String endPoint,
		String accessKeyId,
		String accessKeySecret,
		String project,
		String logStore,
		int startInSec,
		int stopInSec,
		int maxRetryTime,
		int batchGetSize,
		List<Shard> initShardList,
		Configuration properties,
		String consumerGroup) {
	this.endPoint = endPoint;
	this.accessKeyId = accessKeyId;
	this.accessKeySecret = accessKeySecret;
	this.project = project;
	this.logStore = logStore;
	this.startInSec = startInSec;
	this.stopInSec = stopInSec;
	this.maxRetryTime = maxRetryTime;
	this.batchGetSize = batchGetSize;
	setInitPartitionCount(null == initShardList ? 0 : initShardList.size());
	this.properties = properties;
	this.consumerGroup = consumerGroup;
}
 

public SlsRecordReader(
		String endPoint,
		Configuration properties,
		String project,
		String logStore,
		int startInSec,
		int stopInSec,
		int maxRetryTime,
		int batchGetSize,
		List<Shard> initShardList,
		String consumerGroup) {
	this.endPoint = endPoint;
	this.project = project;
	this.logStore = logStore;
	this.startInSec = startInSec;
	this.stopInSec = stopInSec;
	this.maxRetryTime = maxRetryTime;
	this.batchGetSize = batchGetSize;
	setInitPartitionCount(null == initShardList ? 0 : initShardList.size());
	this.properties = properties;
	this.consumerGroup = consumerGroup;
}
 

@Override
public InputSplit[] createInputSplitsForCurrentSubTask(
		int numberOfParallelSubTasks, int indexOfThisSubTask) throws IOException {
	List<Shard> subscribedPartitions =
			modAssign(numberOfParallelSubTasks, indexOfThisSubTask);

	SlsInputSplit[] inputSplits = new SlsInputSplit[subscribedPartitions.size()];
	int i = 0;
	for (Shard shard : subscribedPartitions) {
		inputSplits[i++] = new SlsInputSplit(shard.GetShardId());
	}
	return inputSplits;
}
 

@Override
public List<String> getPartitionList() throws Exception {
	List<String> partitions = new ArrayList<>();
	List<Shard> shards = getSlsClientProvider().getClient().ListShard(project, logStore).GetShards();
	for (Shard shard : shards) {
		partitions.add("" + shard.GetShardId());
	}
	return partitions;
}
 

private void initShardList() {
	if (null != initShardList) {
		try {
			initShardList = getSlsClientProvider().getClient().ListShard(project, logStore).GetShards();
			Collections.sort(initShardList, new Comparator<Shard>() {
				@Override
				public int compare(Shard o1, Shard o2) {
					return o1.GetShardId() - o2.GetShardId();
				}
			});
		} catch (Exception e){
			throw new RuntimeException("", e);
		}
	}
}
 

private List<Shard> modAssign(
		int consumerCount,
		int consumerIndex) {
	List<Shard> assignedShards = new LinkedList<>();

	for (Shard shard: initShardList) {
		if (shard.GetShardId() % consumerCount == consumerIndex) {
			assignedShards.add(shard);
		}
	}
	return assignedShards;
}
 

private List<LogstoreShardMeta> listAssignedShards() throws Exception {
    List<String> logstores = getLogstores();
    List<LogstoreShardMeta> shardMetas = new ArrayList<>();
    for (String logstore : logstores) {
        List<Shard> shards = logClient.listShards(project, logstore);
        for (Shard shard : shards) {
            LogstoreShardMeta shardMeta = new LogstoreShardMeta(logstore, shard.GetShardId(), shard.getStatus());
            if (shardAssigner.assign(shardMeta, totalNumberOfSubtasks) % totalNumberOfSubtasks == indexOfThisSubtask) {
                shardMetas.add(shardMeta);
            }
        }
    }
    return shardMetas;
}
 

public List<Shard> getShardsList() throws LogException {
	slsClientProvider = getSlsClientProvider();
	List<Shard> shards = slsClientProvider.getClient().ListShard(project, logStore).GetShards();
	return shards;
}
 

@Override
public void open(
		InputSplit split, RuntimeContext context) throws IOException {
	SlsInputSplit slsInputSplit = (SlsInputSplit) split;
	LOG.info(String.format("open project[%s] logStore[%s],consumer[%s]-%s  startTime[%d)", project, logStore,
						accessKeyId,
						slsInputSplit.toString(), startInSec));

	int curRetry = 0;
	while (curRetry++ < maxRetryTime) {
		try {
			List<Shard> shardsList  = getShardsList();
			if (initPartitionCount != shardsList.size()){
				ErrorUtils.throwException(
						String.format("Source {%s} partitions number has changed from {%s} to {%s} \n " +
										"Wait the failover finish, blink is trying to recovery from " +
										"source partition change", getReaderName(),
								String.valueOf(initPartitionCount), String.valueOf(getPartitionsNums())));
			}
			this.shardId = split.getSplitNumber();
			for (Shard shard: shardsList) {
				if (shard.GetShardId() == this.shardId){
					this.shard = shard;
					break;
				}
			}
			if (shard.getStatus().equalsIgnoreCase("readonly")) {
				LOG.info("ShardId " + shard.GetShardId() + " status:readOnly");
				isReadOnlyShard = true;
				this.endCursor = getSlsClientProvider().getClient().GetCursor(project, logStore, shardId, Consts
						.CursorMode.END).GetCursor();
			} else {
				LOG.info("ShardId " + shard.GetShardId() + " status:readwrite");
				isReadOnlyShard = false;
			}
			this.nextBeginCursor = getSlsClientProvider().getClient()
														.GetCursor(project, logStore, shardId, startInSec)
														.GetCursor();
			if (stopInSec == Integer.MAX_VALUE) {
				this.stopCursor = null;

			} else {
				this.stopCursor = getSlsClientProvider().getClient()
						.GetCursor(project, logStore, shardId, stopInSec).GetCursor();
			}

			if (consumerGroup == null) {
				LOG.info(String.format(
						"Open method get init cursor, " +
						"project[%s]-logStore[%s]-shardId[%d]-startInSec[%d]-Cursor[%s]",
						project,
						logStore,
						shardId,
						startInSec,
						nextBeginCursor));
			} else {
				LOG.info(String.format(
						"Open method get init cursor, " +
						"project[%s]-logStore[%s]-shardId[%d]-startInSec[%d]-Cursor[%s]-ConsumerGroup[%s]",
						project,
						logStore,
						shardId,
						startInSec,
						nextBeginCursor,
						consumerGroup));
			}
			break;
		} catch (LogException e) {
			LOG.error("Error in get shard list", e);
			// refresh sts account
			getSlsClientProvider().getClient(true, true);
			if (curRetry == maxRetryTime) {
				ErrorUtils.throwException(
						e.getMessage());
			}
			try {
				Thread.sleep(curRetry * 500);
			} catch (Exception e1) {

			}
		}
	}
	initPartitionNumsListener();
}
 

public String getBeginHash(String project, String logstore, String hash) {
	if (hash == null) {
		return hash;
	}
	Client client = clientPool.getClient(project);
	if (client == null) {
		return hash;
	}
	String key = project + "|" + logstore;
	rwLock.readLock().lock();
	ShardHash h = shardHash.get(key);
	if (h == null) {
		rwLock.readLock().unlock();
		rwLock.writeLock().lock();
		h = shardHash.get(key);
		if (h == null) {
			h = new ShardHash();
			shardHash.put(key, h);
		}
		h.rwLock.readLock().lock();
		rwLock.writeLock().unlock();
	} else {
		h.rwLock.readLock().lock();
		rwLock.readLock().unlock();
	}
	long cur = System.currentTimeMillis();
	if (cur - h.updateTime >= producerConfig.shardHashUpdateIntervalInMS) {
		h.rwLock.readLock().unlock();
		h.rwLock.writeLock().lock();
		if (cur - h.updateTime >= producerConfig.shardHashUpdateIntervalInMS) {
			ArrayList<Shard> shards = null;
			try {
				shards = client.ListShard(project, logstore).GetShards();
			} catch (LogException e) {
			}
			if (shards != null) {
				h.hash = new ArrayList<String>();
				for (Shard s : shards) {
					if(s.getStatus().compareToIgnoreCase("readonly") != 0){
						h.hash.add(s.getInclusiveBeginKey());
					}
				}
				Collections.sort(h.hash);
				h.updateTime = cur;
			}
		}
		h.rwLock.writeLock().unlock();
		h.rwLock.readLock().lock();
	}
	if (h.hash != null && !h.hash.isEmpty()) {
		int low = 0, high = h.hash.size() - 1, mid = 0;
		int action = 0;
		while (low <= high) {
			mid = (low + high) / 2;
			int cmp = h.hash.get(mid).compareTo(hash);
			if (cmp == 0) {
				action = 0;
				break;
			} else if (cmp < 0) {
				action = 1;
				low = mid + 1;
			} else {
				action = -1;
				high = mid - 1;
			}
		}
		if (action == -1 && mid > 0) {
			--mid;
		}
		h.rwLock.readLock().unlock();
		return h.hash.get(mid);
	}
	h.rwLock.readLock().unlock();
	return hash;
}
 

public List<Shard> listShards(final String project, final String logstore) throws LogException {
    return RetryUtil.call((Callable<List<Shard>>) () -> client.ListShard(project, logstore).GetShards(), "Error while listing shards");
}