com.aliyun.openservices.log.Client Java Examples

private Client buildClient(ProjectConfig projectConfig) {
  Client client =
      new Client(
          projectConfig.getEndpoint(),
          projectConfig.getAccessKeyId(),
          projectConfig.getAccessKeySecret(),
          serviceClient);
  String userAgent = projectConfig.getUserAgent();
  if (userAgent != null) {
    client.setUserAgent(userAgent);
  }
  String stsToken = projectConfig.getStsToken();
  if (stsToken != null) {
    client.setSecurityToken(stsToken);
  }
  return client;
}
 

public SendProducerBatchTask(
    ProducerBatch batch,
    ProducerConfig producerConfig,
    Map<String, Client> clientPool,
    RetryQueue retryQueue,
    BlockingQueue<ProducerBatch> successQueue,
    BlockingQueue<ProducerBatch> failureQueue,
    AtomicInteger batchCount) {
  this.batch = batch;
  this.producerConfig = producerConfig;
  this.clientPool = clientPool;
  this.retryQueue = retryQueue;
  this.successQueue = successQueue;
  this.failureQueue = failureQueue;
  this.batchCount = batchCount;
}
 

void transferProducerBatch(
    IOThreadPool ioThreadPool,
    ProducerConfig producerConfig,
    Map<String, Client> clientPool,
    RetryQueue retryQueue,
    BlockingQueue<ProducerBatch> successQueue,
    BlockingQueue<ProducerBatch> failureQueue,
    AtomicInteger batchCount) {
  if (producerBatch == null) {
    return;
  }
  ioThreadPool.submit(
      new SendProducerBatchTask(
          producerBatch,
          producerConfig,
          clientPool,
          retryQueue,
          successQueue,
          failureQueue,
          batchCount));
  producerBatch = null;
}
 

public LogAccumulator(
    String producerHash,
    ProducerConfig producerConfig,
    Map<String, Client> clientPool,
    Semaphore memoryController,
    RetryQueue retryQueue,
    BlockingQueue<ProducerBatch> successQueue,
    BlockingQueue<ProducerBatch> failureQueue,
    IOThreadPool ioThreadPool,
    AtomicInteger batchCount) {
  this.producerHash = producerHash;
  this.producerConfig = producerConfig;
  this.clientPool = clientPool;
  this.memoryController = memoryController;
  this.retryQueue = retryQueue;
  this.successQueue = successQueue;
  this.failureQueue = failureQueue;
  this.ioThreadPool = ioThreadPool;
  this.batchCount = batchCount;
  this.batches = new ConcurrentHashMap<GroupKey, ProducerBatchHolder>();
  this.appendsInProgress = new AtomicInteger(0);
  this.closed = false;
}
 

public Mover(
    String name,
    ProducerConfig producerConfig,
    Map<String, Client> clientPool,
    LogAccumulator accumulator,
    RetryQueue retryQueue,
    BlockingQueue<ProducerBatch> successQueue,
    BlockingQueue<ProducerBatch> failureQueue,
    IOThreadPool ioThreadPool,
    AtomicInteger batchCount) {
  super(name, true);
  this.producerConfig = producerConfig;
  this.clientPool = clientPool;
  this.accumulator = accumulator;
  this.retryQueue = retryQueue;
  this.successQueue = successQueue;
  this.failureQueue = failureQueue;
  this.ioThreadPool = ioThreadPool;
  this.batchCount = batchCount;
  this.closed = false;
}
 

@Override
protected Client produceStsClient(String accessId, String accessKey, String securityToken) {
	Client client = new Client(endPoint, accessId, accessKey);
	client.setUserAgent("Blink-sts" + "-" + String.valueOf(consumerGroup) + "-" +
						getHostName());
	client.SetSecurityToken(securityToken);
	return client;
}
 

private Client buildClient(final ProjectConfig config) {
    Client client = new Client(config.endpoint, config.accessKeyId, config.accessKey);
    client.setUserAgent(producerConfig.userAgent);
    if (config.stsToken != null) {
        client.SetSecurityToken(config.stsToken);
    }
    return client;
}
 

private PutLogsResponse sendRequest(Client clt, BlockedData bd) throws LogException {
    PutLogsRequest request = buildRequest(bd);
    List<TagContent> tags = new ArrayList<TagContent>();
    tags.add(new TagContent("__pack_id__", bd.data.getPackageId()));
    request.SetTags(tags);
    request.setContentType(producerConfig.logsFormat.equals("protobuf") ?
            Consts.CONST_PROTO_BUF
            : Consts.CONST_SLS_JSON);
    PutLogsResponse response = clt.PutLogs(request);
    long tmpBytes = sendLogBytes.get();
    sendLogBytes.set(tmpBytes + bd.bytes);
    return response;
}
 

private PutLogsResponse doSendData(BlockedData bd) throws LogException, InterruptedException {
    try {
        Client clt = clientPool.getClient(bd.data.project);
        if (clt == null) {
            throw new LogException("ProjectConfigNotExist",
                    "the config of project " + bd.data.project + " is not exist", "");
        } else {
            int retry = 0;
            int sleepTimeInSecs = 1;
            while (true) {
                try {
                    return sendRequest(clt, bd);
                } catch (LogException e) {
                    if (e.GetErrorCode().equals("RequestError") || e.GetErrorCode().equals("Unauthorized") || e.GetErrorCode().equals("WriteQuotaExceed") || e.GetErrorCode().equals("InternalServerError") || e.GetErrorCode().equals("ServerBusy")) {
                        if (retry > producerConfig.retryTimes) {
                            throw e;
                        }
                        LOGGER.warn("Failed to send request, errorCode=" + e.GetErrorCode() + ", errorMessage=" + e.GetErrorMessage() + ", requestId=" + e.GetRequestId() + ", retry=" + retry);
                        Thread.sleep(sleepTimeInSecs * 1000);
                        if (sleepTimeInSecs < 512) {
                            sleepTimeInSecs = sleepTimeInSecs * 2;
                        }
                        ++retry;
                    } else {
                        throw e;
                    }
                }
            }
        }
    } finally {
        packageManager.releaseBytes(bd.bytes);
    }
}
 

@Override
protected Client produceNormalClient(String accessId, String accessKey) {
	Client client = new Client(endPoint, accessId, accessKey);
	if (directMode){
		client.EnableDirectMode();
	}
	client.setUserAgent("Blink-ak" + "-" + String.valueOf(consumerGroup) + "-" +
						getHostName());
	return client;
}
 

public Client getClient(final String project) {
    return clientPool.get(project);
}
 

public Client updateClient(final ProjectConfig config) {
    Client client = buildClient(config);
    clientPool.put(config.projectName, client);
    return client;
}
 

public String getBeginHash(String project, String logstore, String hash) {
	if (hash == null) {
		return hash;
	}
	Client client = clientPool.getClient(project);
	if (client == null) {
		return hash;
	}
	String key = project + "|" + logstore;
	rwLock.readLock().lock();
	ShardHash h = shardHash.get(key);
	if (h == null) {
		rwLock.readLock().unlock();
		rwLock.writeLock().lock();
		h = shardHash.get(key);
		if (h == null) {
			h = new ShardHash();
			shardHash.put(key, h);
		}
		h.rwLock.readLock().lock();
		rwLock.writeLock().unlock();
	} else {
		h.rwLock.readLock().lock();
		rwLock.readLock().unlock();
	}
	long cur = System.currentTimeMillis();
	if (cur - h.updateTime >= producerConfig.shardHashUpdateIntervalInMS) {
		h.rwLock.readLock().unlock();
		h.rwLock.writeLock().lock();
		if (cur - h.updateTime >= producerConfig.shardHashUpdateIntervalInMS) {
			ArrayList<Shard> shards = null;
			try {
				shards = client.ListShard(project, logstore).GetShards();
			} catch (LogException e) {
			}
			if (shards != null) {
				h.hash = new ArrayList<String>();
				for (Shard s : shards) {
					if(s.getStatus().compareToIgnoreCase("readonly") != 0){
						h.hash.add(s.getInclusiveBeginKey());
					}
				}
				Collections.sort(h.hash);
				h.updateTime = cur;
			}
		}
		h.rwLock.writeLock().unlock();
		h.rwLock.readLock().lock();
	}
	if (h.hash != null && !h.hash.isEmpty()) {
		int low = 0, high = h.hash.size() - 1, mid = 0;
		int action = 0;
		while (low <= high) {
			mid = (low + high) / 2;
			int cmp = h.hash.get(mid).compareTo(hash);
			if (cmp == 0) {
				action = 0;
				break;
			} else if (cmp < 0) {
				action = 1;
				low = mid + 1;
			} else {
				action = -1;
				high = mid - 1;
			}
		}
		if (action == -1 && mid > 0) {
			--mid;
		}
		h.rwLock.readLock().unlock();
		return h.hash.get(mid);
	}
	h.rwLock.readLock().unlock();
	return hash;
}
 

private Client getClient(String project) {
  return clientPool.get(project);
}
 

/** Add or update a project config. */
@Override
public void putProjectConfig(ProjectConfig projectConfig) {
  Client client = buildClient(projectConfig);
  clientPool.put(projectConfig.getProject(), client);
}
 

public LogClientProxy(String endpoint, String accessKeyId, String accessKey, String userAgent) {
    this.client = new Client(endpoint, accessKeyId, accessKey);
    this.client.setUserAgent(userAgent);
}
 

public AliyunSLSLogger(String key, String secret, String endPoint, String project, String source) {
    this.project = project;
    client = new Client(endPoint, key, secret, source, true);
}
 

/**
 * 创建阿里云日志服务工具类
 * <br/>阿里云日志服务控制台： <a href="https://sls.console.aliyun.com">https://sls.console.aliyun.com</a>
 * @param endpoint 如 cn-hongkong.log.aliyuncs.com
 * @param accessKeyId 阿里云访问密钥 AccessKeyId
 * @param accessKeySecret 阿里云 AccessKeySecret
 * @param project 日志服务中，创建的项目名称
 * @param logstore 日志服务中，创建的项目下的日志库名称
 */
public AliyunLogUtil(String endpoint, String accessKeyId, String accessKeySecret, String project, String logstore) {
	this.project = project;
	this.logstore = logstore;
	client = new Client(endpoint, accessKeyId, accessKeySecret);
	
	logGroupCache = new Vector<LogItem>();
}