Java Code Examples for org.frameworkset.elasticsearch.entity.ESDatas#getScrollId()

The following examples show how to use org.frameworkset.elasticsearch.entity.ESDatas#getScrollId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestScrollQuery.java From elasticsearch-gradle-example with Apache License 2.0

4 votes

@Test
	public void testScroll(){
		ClientInterface clientUtil = ElasticSearchHelper.getConfigRestClientUtil("esmapper/scroll.xml");
		//scroll分页检索
		List<String > scrollIds = new ArrayList<>();
		long starttime = System.currentTimeMillis();
		Map params = new HashMap();

		params.put("size", 100);//每页100条记录
		ESDatas<Map> response = clientUtil.searchList("agentstat-*/_search?scroll=1m","scrollQuery",params,Map.class);
		List<Map> datas = response.getDatas();
		long realTotalSize = datas.size();
		long totalSize = response.getTotalSize();
		String scrollId = response.getScrollId();
		if(scrollId != null)
			scrollIds.add(scrollId);
		System.out.println("totalSize:"+totalSize);
		System.out.println("scrollId:"+scrollId);
		if(datas != null && datas.size() > 0) {//每页1000条记录，迭代scrollid，遍历scroll分页结果
			do {

				response = clientUtil.searchScroll("1m",scrollId,Map.class);
				scrollId = response.getScrollId();
				totalSize = response.getTotalSize();
				System.out.println("scroll totalSize:"+totalSize);
				if(scrollId != null)
					scrollIds.add(scrollId);
				datas = response.getDatas();

				if(datas == null || datas.size() == 0){
					break;
				}
				realTotalSize = realTotalSize + datas.size();
			} while (true);
		}
		long endtime = System.currentTimeMillis();
		System.out.println("耗时："+(endtime - starttime)+",realTotalSize："+realTotalSize+",totalSize:"+totalSize);
		//查询存在es服务器上的scroll上下文信息
		String scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
//		System.out.println(scrolls);
		//处理完毕后清除scroll上下文信息
		if(scrollIds.size() > 0) {
			scrolls = clientUtil.deleteScrolls(scrollIds);
//			System.out.println(scrolls);
		}
		//清理完毕后查看scroll上下文信息
		scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
//		System.out.println(scrolls);
	}

Example 2

Source File: TestScrollQuery.java From elasticsearch-gradle-example with Apache License 2.0

4 votes

/**
 * 串行方式执行slice scroll操作
 */
@Test
public void testSliceScroll() {
	ClientInterface clientUtil = ElasticSearchHelper.getConfigRestClientUtil("esmapper/scroll.xml");
	List<String> scrollIds = new ArrayList<>();
	long starttime = System.currentTimeMillis();
	//scroll slice分页检索
	int max = 6;
	long realTotalSize = 0;
	for (int i = 0; i < max; i++) {
		Map params = new HashMap();
		params.put("id", i);
		params.put("max", max);//最多6个slice，不能大于share数
		params.put("size", 100);//每页100条记录
		ESDatas<Map> sliceResponse = clientUtil.searchList("agentstat-*/_search?scroll=1m",
				"scrollSliceQuery", params,Map.class);
		List<Map> sliceDatas = sliceResponse.getDatas();
		realTotalSize = realTotalSize + sliceDatas.size();
		long totalSize = sliceResponse.getTotalSize();
		String scrollId = sliceResponse.getScrollId();
		if (scrollId != null)
			scrollIds.add(scrollId);
		System.out.println("totalSize:" + totalSize);
		System.out.println("scrollId:" + scrollId);
		if (sliceDatas != null && sliceDatas.size() >= 100) {//每页100条记录，迭代scrollid，遍历scroll分页结果
			do {
				sliceResponse = clientUtil.searchScroll("1m", scrollId, Map.class);
				String sliceScrollId = sliceResponse.getScrollId();
				if (sliceScrollId != null)
					scrollIds.add(sliceScrollId);
				sliceDatas = sliceResponse.getDatas();
				if (sliceDatas == null || sliceDatas.size() < 100) {
					break;
				}
				realTotalSize = realTotalSize + sliceDatas.size();
			} while (true);
		}
	}
	long endtime = System.currentTimeMillis();
	System.out.println("耗时："+(endtime - starttime)+",realTotalSize："+realTotalSize);
	//查询存在es服务器上的scroll上下文信息
	String scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
	System.out.println(scrolls);
	//处理完毕后清除scroll上下文信息
	if(scrollIds.size() > 0) {
		scrolls = clientUtil.deleteScrolls(scrollIds);
		System.out.println(scrolls);
	}
	//清理完毕后查看scroll上下文信息
	scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
	System.out.println(scrolls);
}

Example 3

Source File: TestScrollQuery.java From elasticsearch-gradle-example with Apache License 2.0

4 votes

/**
	 * 并行方式执行slice scroll操作
	 */
	@Test
	public void testParralSliceScroll() {
		final ClientInterface clientUtil = ElasticSearchHelper.getConfigRestClientUtil("esmapper/scroll.xml");
		final List<String> scrollIds = new ArrayList<>();
		long starttime = System.currentTimeMillis();
		//scroll slice分页检索
		final int max = 6;
		final CountDownLatch countDownLatch = new CountDownLatch(max);//线程任务完成计数器，每个线程对应一个sclice,每运行完一个slice任务,countDownLatch计数减去1

		for (int j = 0; j < max; j++) {
			final int i = j;
			Thread sliceThread = new Thread(new Runnable() {//多线程并行执行scroll操作做，每个线程对应一个sclice

				@Override
				public void run() {
					Map params = new HashMap();
					params.put("id", i);
					params.put("max", max);//最多6个slice，不能大于share数
					params.put("size", 100);//每页100条记录
					ESDatas<Map> sliceResponse = clientUtil.searchList("agentstat-*/_search?scroll=1m",
							"scrollSliceQuery", params,Map.class);
					List<Map> sliceDatas = sliceResponse.getDatas();
					incrementSize( sliceDatas.size());//统计实际处理的文档数量
					long totalSize = sliceResponse.getTotalSize();
					String scrollId = sliceResponse.getScrollId();
					if (scrollId != null)
						scrollIds.add(scrollId);
					System.out.println("totalSize:" + totalSize);
					System.out.println("scrollId:" + scrollId);
					if (sliceDatas != null && sliceDatas.size() >= 100) {//每页100条记录，迭代scrollid，遍历scroll分页结果
						do {
							sliceResponse = clientUtil.searchScroll("1m", scrollId, Map.class);
							String sliceScrollId = sliceResponse.getScrollId();
							if (sliceScrollId != null)
								scrollIds.add(sliceScrollId);
							sliceDatas = sliceResponse.getDatas();
							if (sliceDatas == null || sliceDatas.size() < 100) {
								break;
							}
							incrementSize( sliceDatas.size());//统计实际处理的文档数量
						} while (true);
					}
					countDownLatch.countDown();//slice检索完毕后计数器减1
				}

			});
			sliceThread.start();
		}
		try {
			countDownLatch.await();//等待所有的线程执行完毕,计数器变成0
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		long endtime = System.currentTimeMillis();
		System.out.println("耗时："+(endtime - starttime)+",realTotalSize："+realTotalSize);
		//查询存在es服务器上的scroll上下文信息
		String scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
//		System.out.println(scrolls);
		//处理完毕后清除scroll上下文信息
		if(scrollIds.size() > 0) {
			scrolls = clientUtil.deleteScrolls(scrollIds);
//			System.out.println(scrolls);
		}
		//清理完毕后查看scroll上下文信息
		scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
//		System.out.println(scrolls);
	}

Example 4

Source File: TestScrollQuery.java From elasticsearch-gradle-example with Apache License 2.0

4 votes

@Test
	public void testScroll(){
		ClientInterface clientUtil = ElasticSearchHelper.getConfigRestClientUtil("esmapper/scroll.xml");
		//scroll分页检索
		List<String > scrollIds = new ArrayList<>();
		long starttime = System.currentTimeMillis();
		Map params = new HashMap();

		params.put("size", 100);//每页100条记录
		ESDatas<Map> response = clientUtil.searchList("agentstat-*/_search?scroll=1m","scrollQuery",params,Map.class);
		List<Map> datas = response.getDatas();
		long realTotalSize = datas.size();
		long totalSize = response.getTotalSize();
		String scrollId = response.getScrollId();
		if(scrollId != null)
			scrollIds.add(scrollId);
		System.out.println("totalSize:"+totalSize);
		System.out.println("scrollId:"+scrollId);
		if(datas != null && datas.size() > 0) {//每页1000条记录，迭代scrollid，遍历scroll分页结果
			do {

				response = clientUtil.searchScroll("1m",scrollId,Map.class);
				scrollId = response.getScrollId();
				totalSize = response.getTotalSize();
				System.out.println("scroll totalSize:"+totalSize);
				if(scrollId != null)
					scrollIds.add(scrollId);
				datas = response.getDatas();

				if(datas == null || datas.size() == 0){
					break;
				}
				realTotalSize = realTotalSize + datas.size();
			} while (true);
		}
		long endtime = System.currentTimeMillis();
		System.out.println("耗时："+(endtime - starttime)+",realTotalSize："+realTotalSize+",totalSize:"+totalSize);
		//查询存在es服务器上的scroll上下文信息
		String scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
//		System.out.println(scrolls);
		//处理完毕后清除scroll上下文信息
		if(scrollIds.size() > 0) {
			scrolls = clientUtil.deleteScrolls(scrollIds);
//			System.out.println(scrolls);
		}
		//清理完毕后查看scroll上下文信息
		scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
//		System.out.println(scrolls);
	}

Example 5

Source File: TestScrollQuery.java From elasticsearch-gradle-example with Apache License 2.0

4 votes

/**
 * 串行方式执行slice scroll操作
 */
@Test
public void testSliceScroll() {
	ClientInterface clientUtil = ElasticSearchHelper.getConfigRestClientUtil("esmapper/scroll.xml");
	List<String> scrollIds = new ArrayList<>();
	long starttime = System.currentTimeMillis();
	//scroll slice分页检索
	int max = 6;
	long realTotalSize = 0;
	for (int i = 0; i < max; i++) {
		Map params = new HashMap();
		params.put("id", i);
		params.put("max", max);//最多6个slice，不能大于share数
		params.put("size", 100);//每页100条记录
		ESDatas<Map> sliceResponse = clientUtil.searchList("agentstat-*/_search?scroll=1m",
				"scrollSliceQuery", params,Map.class);
		List<Map> sliceDatas = sliceResponse.getDatas();
		realTotalSize = realTotalSize + sliceDatas.size();
		long totalSize = sliceResponse.getTotalSize();
		String scrollId = sliceResponse.getScrollId();
		if (scrollId != null)
			scrollIds.add(scrollId);
		System.out.println("totalSize:" + totalSize);
		System.out.println("scrollId:" + scrollId);
		if (sliceDatas != null && sliceDatas.size() >= 100) {//每页100条记录，迭代scrollid，遍历scroll分页结果
			do {
				sliceResponse = clientUtil.searchScroll("1m", scrollId, Map.class);
				String sliceScrollId = sliceResponse.getScrollId();
				if (sliceScrollId != null)
					scrollIds.add(sliceScrollId);
				sliceDatas = sliceResponse.getDatas();
				if (sliceDatas == null || sliceDatas.size() < 100) {
					break;
				}
				realTotalSize = realTotalSize + sliceDatas.size();
			} while (true);
		}
	}
	long endtime = System.currentTimeMillis();
	System.out.println("耗时："+(endtime - starttime)+",realTotalSize："+realTotalSize);
	//查询存在es服务器上的scroll上下文信息
	String scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
	System.out.println(scrolls);
	//处理完毕后清除scroll上下文信息
	if(scrollIds.size() > 0) {
		scrolls = clientUtil.deleteScrolls(scrollIds);
		System.out.println(scrolls);
	}
	//清理完毕后查看scroll上下文信息
	scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
	System.out.println(scrolls);
}

Example 6

Source File: TestScrollQuery.java From elasticsearch-gradle-example with Apache License 2.0

4 votes

/**
	 * 并行方式执行slice scroll操作
	 */
	@Test
	public void testParralSliceScroll() {
		final ClientInterface clientUtil = ElasticSearchHelper.getConfigRestClientUtil("esmapper/scroll.xml");
		final List<String> scrollIds = new ArrayList<>();
		long starttime = System.currentTimeMillis();
		//scroll slice分页检索
		final int max = 6;
		final CountDownLatch countDownLatch = new CountDownLatch(max);//线程任务完成计数器，每个线程对应一个sclice,每运行完一个slice任务,countDownLatch计数减去1

		for (int j = 0; j < max; j++) {
			final int i = j;
			Thread sliceThread = new Thread(new Runnable() {//多线程并行执行scroll操作做，每个线程对应一个sclice

				@Override
				public void run() {
					Map params = new HashMap();
					params.put("id", i);
					params.put("max", max);//最多6个slice，不能大于share数
					params.put("size", 100);//每页100条记录
					ESDatas<Map> sliceResponse = clientUtil.searchList("agentstat-*/_search?scroll=1m",
							"scrollSliceQuery", params,Map.class);
					List<Map> sliceDatas = sliceResponse.getDatas();
					incrementSize( sliceDatas.size());//统计实际处理的文档数量
					long totalSize = sliceResponse.getTotalSize();
					String scrollId = sliceResponse.getScrollId();
					if (scrollId != null)
						scrollIds.add(scrollId);
					System.out.println("totalSize:" + totalSize);
					System.out.println("scrollId:" + scrollId);
					if (sliceDatas != null && sliceDatas.size() >= 100) {//每页100条记录，迭代scrollid，遍历scroll分页结果
						do {
							sliceResponse = clientUtil.searchScroll("1m", scrollId, Map.class);
							String sliceScrollId = sliceResponse.getScrollId();
							if (sliceScrollId != null)
								scrollIds.add(sliceScrollId);
							sliceDatas = sliceResponse.getDatas();
							if (sliceDatas == null || sliceDatas.size() < 100) {
								break;
							}
							incrementSize( sliceDatas.size());//统计实际处理的文档数量
						} while (true);
					}
					countDownLatch.countDown();//slice检索完毕后计数器减1
				}

			});
			sliceThread.start();
		}
		try {
			countDownLatch.await();//等待所有的线程执行完毕,计数器变成0
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		long endtime = System.currentTimeMillis();
		System.out.println("耗时："+(endtime - starttime)+",realTotalSize："+realTotalSize);
		//查询存在es服务器上的scroll上下文信息
		String scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
//		System.out.println(scrolls);
		//处理完毕后清除scroll上下文信息
		if(scrollIds.size() > 0) {
			scrolls = clientUtil.deleteScrolls(scrollIds);
//			System.out.println(scrolls);
		}
		//清理完毕后查看scroll上下文信息
		scrolls = clientUtil.executeHttp("_nodes/stats/indices/search", ClientUtil.HTTP_GET);
//		System.out.println(scrolls);
	}