Java Code Examples for com.aliyun.openservices.log.common.LogItem#PushBack

/**
 * 增加动作日志
 * @param goalid 操作的目标id,若没有，则为0
 * @param action 动作
 * @param remark 备注，说明
 */
public static void addActionLog(int goalid, String action, String remark){
	if(aliyunLogUtil == null){
		//不使用日志服务，终止即可
		return;
	}
	Site site = Func.getCurrentSite();
	
	String siteName = "";
	String siteDomain = "";
	int siteid = 0;
	if(site != null){
		siteName = site.getName();
		siteDomain = Func.getDomain(site);
		siteid = site.getId();
	}
	
	LogItem logItem = new LogItem((int) (new Date().getTime() / 1000));
	logItem.PushBack("siteName", siteName);
	logItem.PushBack("siteDomain", siteDomain);
	logItem.PushBack("siteid", siteid+"");
	insert(logItem, null, goalid, action, remark);
}
 

@Override
public void log(String store, String topic, String source, Object... args) {
    List<LogItem> logs = new ArrayList<LogItem>();

    LogItem item = new LogItem((int) (System.currentTimeMillis() / 1000));
    for (int i = 0; i < args.length; ++i) {
        item.PushBack(String.valueOf(args[i]), String.valueOf(args[++i]));
    }
    logs.add(item);

    PutLogsRequest req = new PutLogsRequest(project, store, topic, source, logs);
    try {
        client.PutLogs(req);
    } catch (LogException ex) {
        //Logger.log(ex);
    }
}
 

public static void requestLog(HttpServletRequest request, RequestInfo requestInfo, SImpleSiteVO simpleSiteVO){
	//未开启日志记录
	if(aliyunLogUtil == null){
		return;
	}
	if(simpleSiteVO.getResult() - SImpleSiteVO.FAILURE == 0){
		//失败的，没有访问正规站点的，可能是ip直接访问的，或者访问的未绑定上的域名过来的。这样的不与记录。
		return;
	}
	
	LogItem logItem = new LogItem(DateUtil.timeForUnix10());
	logItem.PushBack("ip", requestInfo.getIp());
	logItem.PushBack("referer", requestInfo.getReferer());
	logItem.PushBack("userAgent", requestInfo.getUserAgent());
	logItem.PushBack("htmlFile", requestInfo.getHtmlFile());
	logItem.PushBack("siteid", simpleSiteVO.getSimpleSite().getId()+"");
	logItem.PushBack("serverName", requestInfo.getServerName());
	
	try {
		aliyunLogUtil.cacheLog(logItem);
	} catch (LogException e1) {
		e1.printStackTrace();
	}
}
 

/**
 * 增加动作日志
 * @param action 动作
 * @param remark 备注，说明
 */
public static void addActionLog(String action, String remark){
	if(aliyunLogUtil == null){
		//不使用日志服务，终止即可
		return;
	}
	Site site = Func.getCurrentSite();
	
	String siteName = "";
	String siteDomain = "";
	int siteid = 0;
	if(site != null){
		siteName = site.getName();
		siteDomain = Func.getDomain(site);
		siteid = site.getId();
	}
	
	LogItem logItem = new LogItem((int) (new Date().getTime() / 1000));
	logItem.PushBack("siteName", siteName);
	logItem.PushBack("siteDomain", siteDomain);
	logItem.PushBack("siteid", siteid+"");
	insert(logItem, null, 0, action, remark);
}
 

@Test
public void testCalculateLogNullValue() {
  LogItem logItem = new LogItem();
  logItem.PushBack("key1", "val1");
  logItem.PushBack("key2", "val2");
  logItem.PushBack("null_value", null);
  int sizeInBytes = LogSizeCalculator.calculate(logItem);
  Assert.assertEquals(30, sizeInBytes);
}
 

/**
 * 插入一条日志
 * @param logItem 传入要保存的logItem，若为空，则会创建一个新的。此项主要为扩展使用，可自行增加其他信息记录入日志
 * @param request HttpServletRequest
 * @param goalid 操作的目标的id，若无，可为0，也可为空
 * @param action 动作的名字，如：用户登录、更改密码
 * @param remark 动作的描述，如用户将名字张三改为李四
 */
public static synchronized void insert(LogItem logItem, HttpServletRequest request, int goalid, String action, String remark){
	if(aliyunLogUtil == null){
		//不使用日志服务，终止即可
		return;
	}
	if(logItem == null){
		logItem = aliyunLogUtil.newLogItem();
	}
	
	/*用户相关信息,只有用户登录后，才会记录用户信息*/
	User user = ShiroFunc.getUser();
	if(user != null){
		logItem.PushBack("userid", user.getId()+"");
		logItem.PushBack("username", user.getUsername());
	}
	
	/* 动作相关 */
	logItem.PushBack("goalid", goalid+"");
	logItem.PushBack("action", action);
	logItem.PushBack("remark", remark);
	
	/*浏览器自动获取的一些信息*/
	if(request != null){
		logItem.PushBack("ip", IpUtil.getIpAddress(request));
		logItem.PushBack("param", request.getQueryString());
		logItem.PushBack("url", request.getRequestURL().toString());
		logItem.PushBack("referer", request.getHeader("referer"));
		logItem.PushBack("userAgent", request.getHeader("User-Agent"));
	}
	try {
		aliyunLogUtil.cacheLog(logItem);
	} catch (LogException e) {
		e.printStackTrace();
	}
}
 

@Test
public void testMaxBatchSizeInBytes() throws InterruptedException, ProducerException {
  ProducerConfig producerConfig = new ProducerConfig();
  producerConfig.setBatchSizeThresholdInBytes(27);
  Producer producer = new LogProducer(producerConfig);
  producer.putProjectConfig(buildProjectConfig());
  LogItem logItem = new LogItem();
  logItem.PushBack("key1", "val1");
  logItem.PushBack("key2", "val2");
  logItem.PushBack("key3", "val3");
  int sizeInBytes = LogSizeCalculator.calculate(logItem);
  Assert.assertEquals(28, sizeInBytes);
  producer.send("project", "logStore", new LogItem());
}
 

/**
 * 缓存日志。缓存的并不会立即联网提交到阿里云日志服务，而是达到一定条数 {@link #setCacheLogMaxNumber(int)} 达到这里设置的条数后才会触发 {@link #cacheCommit()} 提交到阿里云日志服务
 * <br/>这里缓存的日志，没提供设置topic、source，可以将其记录到日志本身中去 
 * @param logItem 要缓存的日志，可用 {@link AliyunLogUtil#newLogItem()} 获取创建 {@link LogItem} 对象，然后将要记录的日志类似JSON， {@link LogItem#PushBack(String, String)} 加入
 * @throws LogException
 */
public void cacheLog(LogItem logItem) throws LogException{
	if(logItem == null){
		return;
	}
	
	/*使用的类的信息，来源位置*/
	if(stack_trace_deep > 0){
		StackTraceElement st = Thread.currentThread().getStackTrace()[stack_trace_deep];
		logItem.PushBack("className", st.getClassName());
		logItem.PushBack("methodName", st.getMethodName());
		logItem.PushBack("fileName", st.getFileName());
	}
	
	logGroupCache.add(logItem);
	
	boolean submit = false;	//提交日志
	if(logGroupCache.size() > cacheLogMaxNumber){
		//超过定义的缓存最大值，那么将缓存中的日志数据提交到阿里日志服务中去
		submit = true;
	}else{
		int currentTime = DateUtil.timeForUnix10();	//当前时间
		if(lastSubmitTime + cacheLogMaxTime < currentTime){
			submit = true;
			lastSubmitTime = currentTime;
		}
	}
	
	if(submit){
		cacheCommit();
	}
}
 

private static LogItem generateLogItem(int r) {
  LogItem logItem = new LogItem();
  logItem.PushBack("content_key_1", "1abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r);
  logItem.PushBack("content_key_2", "2abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r);
  logItem.PushBack("content_key_3", "3abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r);
  logItem.PushBack("content_key_4", "4abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r);
  logItem.PushBack("content_key_5", "5abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r);
  logItem.PushBack("content_key_6", "6abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r);
  logItem.PushBack("content_key_7", "7abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r);
  logItem.PushBack("content_key_8", "8abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_" + r);
  return logItem;
}
 

@Test
public void testCalculateLogNullKey() {
  LogItem logItem = new LogItem();
  logItem.PushBack("key1", "val1");
  logItem.PushBack("key2", "val2");
  logItem.PushBack(null, "null_key");
  int sizeInBytes = LogSizeCalculator.calculate(logItem);
  Assert.assertEquals(28, sizeInBytes);
}
 

@Test
public void testCalculateLog() {
  ProducerTest.buildLogItem();
  int sizeInBytes = LogSizeCalculator.calculate(ProducerTest.buildLogItem());
  Assert.assertEquals(12, sizeInBytes);

  LogItem logItem = new LogItem();
  logItem.PushBack("key1", "val1");
  logItem.PushBack("key2", "val2");
  logItem.PushBack("key3", "val3");
  sizeInBytes = LogSizeCalculator.calculate(logItem);
  Assert.assertEquals(28, sizeInBytes);
}
 

private List<LogItem> getLogItems() {
    List<LogItem> logItems = new ArrayList<LogItem>();

    LogItem logItem1 = new LogItem((int) (new Date().getTime() / 1000));
    logItem1.PushBack("key1", "val1");
    LogItem logItem2 = new LogItem((int) (new Date().getTime() / 1000));
    logItem2.PushBack("key2", "val2");

    logItems.add(logItem1);
    logItems.add(logItem2);
    return logItems;
}
 

private List<LogItem> getLogItems() {
    List<LogItem> logItems = new ArrayList<LogItem>();

    LogItem logItem1 = new LogItem((int) (new Date().getTime() / 1000));
    logItem1.PushBack("key1", "val1");
    LogItem logItem2 = new LogItem((int) (new Date().getTime() / 1000));
    logItem2.PushBack("key2", "val2");

    logItems.add(logItem1);
    logItems.add(logItem2);
    return logItems;
}
 

private List<LogItem> getLogItems() {
    List<LogItem> logItems = new ArrayList<LogItem>();

    LogItem logItem = new LogItem((int) (new Date().getTime() / 1000));
    logItem.PushBack("", "val");
    logItems.add(logItem);
    return logItems;
}
 

private List<LogItem> getLogItems() {
    List<LogItem> logItems = new ArrayList<LogItem>();
    LogItem logItem = new LogItem((int) (new Date().getTime() / 1000));
    for (int i = 0; i < 1000; ++i) {
        logItem.PushBack("key", "val");
    }
    logItems.add(logItem);
    return logItems;
}
 

private void appendEvent(E eventObject) {
    //init Event Object
    if (!(eventObject instanceof LoggingEvent)) {
        return;
    }
    LoggingEvent event = (LoggingEvent) eventObject;

    List<LogItem> logItems = new ArrayList<LogItem>();
    LogItem item = new LogItem();
    logItems.add(item);
    item.SetTime((int) (event.getTimeStamp() / 1000));

    if(formatter!=null){
        DateTime dateTime = new DateTime(event.getTimeStamp());
        item.PushBack("time", dateTime.toString(formatter));
    }else {
        Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
        item.PushBack("time", formatter1.format(instant));
    }

    item.PushBack("level", event.getLevel().toString());
    item.PushBack("thread", event.getThreadName());

    StackTraceElement[] caller = event.getCallerData();
    if (caller != null && caller.length > 0) {
        item.PushBack("location", caller[0].toString());
    }

    String message = event.getFormattedMessage();
    item.PushBack("message", message);

    IThrowableProxy iThrowableProxy = event.getThrowableProxy();
    if (iThrowableProxy != null) {
        String throwable = getExceptionInfo(iThrowableProxy);
        throwable += fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
        item.PushBack("throwable", throwable);
    }

    if (this.encoder != null) {
        item.PushBack("log", new String(this.encoder.encode(eventObject)));
    }

    Optional.ofNullable(mdcFields).ifPresent(
            f->event.getMDCPropertyMap().entrySet().stream()
                    .filter(v->Arrays.stream(f.split(",")).anyMatch(i->i.equals(v.getKey())))
                    .forEach(map-> item.PushBack(map.getKey(),map.getValue()))
    );
    try {
        producer.send(projectConfig.getProject(), logStore, topic, source, logItems, new LoghubAppenderCallback<E>(this,
                projectConfig.getProject(), logStore, topic, source, logItems));
    } catch (Exception e) {
        this.addError(
                "Failed to send log, project=" + project
                        + ", logStore=" + logStore
                        + ", topic=" + topic
                        + ", source=" + source
                        + ", logItem=" + logItems, e);
    }
}
 

public static LogItem buildLogItem() {
  LogItem logItem = new LogItem();
  logItem.PushBack("k1", "v1");
  logItem.PushBack("k2", "v2");
  return logItem;
}
 

@Override
public void invoke(T value, Context context) {
    if (this.producer == null) {
        throw new IllegalStateException("Flink log producer has not been initialized yet!");
    }
    RawLogGroup logGroup = schema.serialize(value);
    if (logGroup == null) {
        LOG.info("The serialized log group is null, will not send any data to log service");
        return;
    }
    String shardHashKey = null;
    if (customPartitioner != null) {
        shardHashKey = customPartitioner.getHashKey(value);
    }
    List<LogItem> logs = new ArrayList<>();
    for (RawLog rawLog : logGroup.getLogs()) {
        if (rawLog == null) {
            continue;
        }
        LogItem record = new LogItem(rawLog.getTime());
        for (Map.Entry<String, String> kv : rawLog.getContents().entrySet()) {
            record.PushBack(kv.getKey(), kv.getValue());
        }
        logs.add(record);
    }
    if (logs.isEmpty()) {
        return;
    }
    try {
        ListenableFuture<Result> future = producer.send(project,
                logstore,
                logGroup.getTopic(),
                logGroup.getSource(),
                shardHashKey,
                logs);
        Futures.addCallback(future, callback, executor);
        buffered.incrementAndGet();
    } catch (InterruptedException | ProducerException e) {
        LOG.error("Error while sending logs", e);
        throw new RuntimeException(e);
    }
}
 

public static void main(String args[]) throws InterruptedException {
	System.out.println(System.currentTimeMillis());
	ProducerConfig producerConfig = new ProducerConfig();
	// 使用默认producer配置
	final LogProducer producer = new LogProducer(producerConfig);
	// 添加多个project配置
	producer.setProjectConfig(buildProjectConfig1());
	producer.setProjectConfig(buildProjectConfig2());
	// 生成日志集合，用于测试
	final Vector<Vector<LogItem>> logGroups = new Vector<Vector<LogItem>>();
	for (int i = 0; i < 100000; ++i) {
		Vector<LogItem> tmpLogGroup = new Vector<LogItem>();
		LogItem logItem = new LogItem((int) (new Date().getTime() / 1000));
		logItem.PushBack("level", "info " + System.currentTimeMillis());
		logItem.PushBack("message", "mmmmmdekdekjdefjekjfek"
				+ RandomString(50));
		logItem.PushBack("method", "SenderToServer " + RandomString(10));
		tmpLogGroup.add(logItem);
		logGroups.add(tmpLogGroup);
	}
	// 并发调用send发送日志
	Thread[] threads = new Thread[ThreadsCount];
	for (int i = 0; i < ThreadsCount; ++i) {
		threads[i] = new Thread(null, new Runnable() {
			Random random = new Random();

			public void run() {
				int j = 0, rand = random.nextInt(99999);
				while (++j < 10) {
					producer.send(System.getenv("project1"), "store_1s", "topic1", MOCK_IP,
							logGroups.get(rand),
							new CallbackSample(System.getenv("project1"), "store_1s",
									"topic1", MOCK_IP, null, logGroups.get(rand), producer));
				}
			}
		}, i + "");
		threads[i].start();
	}
	
	//等待发送线程退出
	Thread.sleep(5 * 1000);
	for (int i = 0; i < ThreadsCount; ++i) {
		threads[i].interrupt();
	}
	//主动刷新缓存起来的还没有被发送的日志
	producer.flush();
	//关闭后台io线程
	producer.close();
}
 

@Override
protected void append(LoggingEvent event) {
  LogItem logItem = new LogItem();
  logItem.SetTime((int) (event.getTimeStamp() / 1000));
  DateTime dateTime = new DateTime(event.getTimeStamp());
  logItem.PushBack("time", dateTime.toString(formatter));
  logItem.PushBack("level", event.getLevel().toString());
  logItem.PushBack("thread", event.getThreadName());
  logItem.PushBack("location", event.getLocationInformation().fullInfo);
  Object message = event.getMessage();
  if(message==null){
    logItem.PushBack("message", "");
  }else{
    logItem.PushBack("message", event.getMessage().toString());
  }

  String throwable = getThrowableStr(event);
  if (throwable != null) {
    logItem.PushBack("throwable", throwable);
  }

  if (getLayout() != null) {
    logItem.PushBack("log", getLayout().format(event));
  }

  Map properties = event.getProperties();
  if (properties.size() > 0) {
    Object[] keys = properties.keySet().toArray();
    Arrays.sort(keys);
    for (int i = 0; i < keys.length; i++) {
      logItem.PushBack(keys[i].toString(), properties.get(keys[i])
          .toString());
    }
  }
  try {
    producer.send(projectConfig.getProject(), logStore, topic, source, logItem,
        new LoghubAppenderCallback(projectConfig.getProject(), logStore, topic, source, logItem));
  } catch (Exception e) {
    LogLog.error(
        "Failed to send log, project=" + project
            + ", logStore=" + logStore
            + ", topic=" + topic
            + ", source=" + source
            + ", logItem=" + logItem, e);
  }
}