redis.clients.jedis.ScanParams Java Examples

/**
 * For the given key prefix, find all actual keys depending on the type of the key.
 *
 * @param split The split for this request, mostly used to get the redis endpoint and config details.
 * @param redisCursor The previous Redis cursor (aka continuation token).
 * @param keys The collections of keys we collected so far. Any new keys we find are added to this.
 * @return The Redis cursor to use when continuing the scan.
 */
private ScanResult<String> loadKeys(Split split, ScanResult<String> redisCursor, Set<String> keys)
{
    try (Jedis client = getOrCreateClient(split.getProperty(REDIS_ENDPOINT_PROP))) {
        KeyType keyType = KeyType.fromId(split.getProperty(KEY_TYPE));
        String keyPrefix = split.getProperty(KEY_PREFIX_TABLE_PROP);
        if (keyType == KeyType.ZSET) {
            long start = Long.valueOf(split.getProperty(SPLIT_START_INDEX));
            long end = Long.valueOf(split.getProperty(SPLIT_END_INDEX));
            keys.addAll(client.zrange(keyPrefix, start, end));
            return new ScanResult<String>(END_CURSOR, Collections.EMPTY_LIST);
        }
        else {
            String cursor = (redisCursor == null) ? SCAN_POINTER_START : redisCursor.getCursor();
            ScanParams scanParam = new ScanParams();
            scanParam.count(SCAN_COUNT_SIZE);
            scanParam.match(split.getProperty(KEY_PREFIX_TABLE_PROP));

            ScanResult<String> newCursor = client.scan(cursor, scanParam);
            keys.addAll(newCursor.getResult());
            return newCursor;
        }
    }
}
 

public Set<String> smembers(String key) {
    logger.trace("smembers {}", key);
    JedisCommands client = dynoClient;
    Set<String> r = new HashSet<>();
    int cursor = 0;
    ScanParams sp = new ScanParams();
    sp.count(50);

    do {
        ScanResult<String> scanResult = client.sscan(key, "" + cursor, sp);
        cursor = Integer.parseInt(scanResult.getCursor());
        r.addAll(scanResult.getResult());

    } while (cursor > 0);

    return r;

}
 

@Test
public void zscanTest() {
    final String key = "zscan.test";
    final String prefix = "zscan.test-";
    try {
        final Map<Object, Double> map = Maps.newHashMap();
        for (int idx = 0; idx < 1000; idx++) {
            map.put(prefix + idx, Double.valueOf(idx));
        }

        redisClient.zadd(key, map);
        final AtomicLong cursor = new AtomicLong(-1);
        final ScanParams params = new ScanParams().count(10);
        while (cursor.get() == -1 || cursor.get() > 0) {
            if (cursor.get() == -1) {
                cursor.set(0);
            }

            final ScanResult<Entry<String, Double>> res = redisClient.zscan(key, cursor.get(), params);
            cursor.set(Long.valueOf(res.getStringCursor()));
        }
    } finally {
        redisClient.del(key);
    }
}
 

@Override
public void withKeyScan(String pattern, int count, Consumer<RedisScanResult> f) {
  ScanParams params = new ScanParams().match(pattern).count(count);
  String cursor = ScanParams.SCAN_POINTER_START;

  try (Jedis jedis = jedisPool.getResource()) {
    do {
      ScanResult<String> result = jedis.scan(cursor, params);

      final List<String> results = result.getResult();
      f.accept(() -> results);

      cursor = result.getCursor();
    } while (!"0".equals(cursor));
  }
}
 

/**
 * Crates new {@link ScanCursor}
 * 
 * @param cluster
 *            JedisCluster
 * @param cursor
 * @param param
 *            Defaulted to {@link ScanParams#NONE} if nulled.
 */
public ScanCursor(JedisCluster cluster, CursorWrapper cursor, Class<?> valueType, ScanParams param) {
	notNull(cluster, "jedisCluster must not be null");
	if (isNull(valueType)) {
		valueType = ResolvableType.forClass(getClass()).getSuperType().getGeneric(0).resolve();
	}
	this.valueType = valueType;
	notNull(valueType, "No scan value java type is specified. Use constructs that can set value java type.");
	this.cluster = cluster;
	this.params = param != null ? param : NONE_PARAMS;
	this.nodePools = cluster.getClusterNodes().values().stream().map(n -> n).collect(toList());
	this.state = CursorState.READY;
	this.cursor = cursor;
	this.iter = new ScanIterable<>(cursor, emptyList());
	CursorWrapper.validate(cursor);
	notEmptyOf(nodePools, "Jedis nodes is empty.");
}
 

@Test
public void sscanTest() {
    final String key = "sscan.test";
    final String prefix = "sscan.test-";
    try {
        final List<String> values = Lists.newArrayList();
        for (int idx = 0; idx < 1000; idx++) {
            values.add(prefix + idx);
        }

        redisClient.sadd(key, values.toArray(new String[values.size()]));
        final AtomicLong cursor = new AtomicLong(-1);
        final ScanParams params = new ScanParams().count(10);
        while (cursor.get() == -1 || cursor.get() > 0) {
            if (cursor.get() == -1) {
                cursor.set(0);
            }

            final ScanResult<String> res = redisClient.sscan(key, cursor.get(), params);
            cursor.set(Long.valueOf(res.getStringCursor()));
        }
    } finally {
        redisClient.del(key);
    }
}
 

@Test
public void scanTest() {
    final String prefix = "scan.test-";
    final Map<String, Object> map = Maps.newHashMap();
    for (int idx = 0; idx < 100; idx++) {
        map.put(prefix + idx, idx);
    }

    redisClient.set(map);

    final ScanParams params = new ScanParams().match(prefix + '*');
    long cursor = -1;
    while (cursor == -1 || cursor > 0) {
        final ScanResult<String> res = redisClient.scan(cursor == -1 ? 0 : cursor, params);
        final String nextCursor = res.getStringCursor();
        cursor = Long.parseLong(nextCursor);
        final List<String> keys = res.getResult();
        LOGGER.debug("{}", keys);
        if (cursor > 0) {
            Assert.assertTrue(keys.size() > 0);
        }
    }

    redisClient.del(map.keySet().toArray(new String[map.size()]));
}
 

@ProcessElement
public void processElement(ProcessContext c) {
  ScanParams scanParams = new ScanParams();
  scanParams.match(c.element());

  String cursor = ScanParams.SCAN_POINTER_START;
  boolean finished = false;
  while (!finished) {
    ScanResult<String> scanResult = jedis.scan(cursor, scanParams);
    List<String> keys = scanResult.getResult();
    for (String k : keys) {
      c.output(k);
    }
    cursor = scanResult.getCursor();
    if (cursor.equals(ScanParams.SCAN_POINTER_START)) {
      finished = true;
    }
  }
}
 

/**
 * Construct the iterator to start at the cursor with no match results.
 *
 * @param redisClient The client to use; may not be {@code null}.
 * @param cursor      The cursor to start at; may not be {@code null}.
 * @param params      The parameters; {@code null} means no parameters.
 */
public ScanSpliterator(Redis redisClient, String cursor, ScanParams params) throws Exception {
  Objects.requireNonNull(redisClient, "Client for Redis");
  Objects.requireNonNull(cursor, "The cursor to start from (you probably meanto use STARTING_CURSOR)");

  ScanResult<String> scanResult = redisClient.withRedis(jedis -> {
    if (params == null) {
      return jedis.scan(cursor);
    } else {
      return jedis.scan(cursor, params);
    }
  });

  results = scanResult.getResult().toArray(new String[scanResult.getResult().size()]);
  if (scanResult.getStringCursor().equals("0")) {
    this.next = null;
  } else {
    this.next = ForkJoinPool.commonPool().submit(() -> {
      return new ScanSpliterator(redisClient, scanResult.getStringCursor(), params);
    });
  }
}
 

@Override
public void setup(OperatorContext context)
{
  super.setup(context);
  sleepTimeMillis = context.getValue(context.SPIN_MILLIS);
  getWindowDataManager().setup(context);
  this.context = context;
  scanOffset = 0;
  scanComplete = false;
  scanParameters = new ScanParams();
  scanParameters.count(scanCount);

  // For the 1st window after checkpoint, windowID - 1 would not have recovery
  // offset stored in windowDataManager
  // But recoveryOffset is non-transient, so will be recovered with
  // checkPointing
  // Offset recovery from idempotency storage can be skipped in this case
  scanOffset = recoveryState.scanOffsetAtBeginWindow;
  skipOffsetRecovery = true;
}
 

@Override
public ScanResult<String> scan(final long cursor, final ScanParams params) {
    ShardedJedis shardedJedis = null;
    try {
        shardedJedis = POOL.getJedis(config.getRedisType());
        final Collection<Jedis> allJedis = shardedJedis.getAllShards();
        if (allJedis.size() == 1) {
            final Jedis jedis = allJedis.iterator().next();
            if (params == null) {
                return jedis.scan(String.valueOf(cursor), DEFAULT_SCAN_PARAMS);
            } else {
                return jedis.scan(String.valueOf(cursor), params);
            }
        } else {
            throw new RedisClientException("不支持对多节点Sharded模式进行Scan操作");
        }
    } catch (final Throwable e) {
        if (e instanceof RedisClientException) {
            throw e;
        }

        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(shardedJedis);
    }
}
 

@Override
public ScanResult<String> scan(final String cursor, final ScanParams params) {
	String matchPattern = null;

	if (params == null || (matchPattern = scanMatch(params)) == null || matchPattern.isEmpty()) {
		throw new IllegalArgumentException(
				getClass().getSimpleName() + " only supports SCAN commands with non-empty MATCH patterns");
	}

	if (JedisClusterHashTagUtil.isClusterCompliantMatchPattern(matchPattern)) {

		return new EnhancedJedisClusterCommand<ScanResult<String>>(connectionHandler, maxAttempts) {
			@Override
			public ScanResult<String> doExecute(Jedis connection) {
				return connection.scan(cursor, params);
			}
		}.runBinary(SafeEncoder.encode(matchPattern));
	} else {
		throw new IllegalArgumentException(getClass().getSimpleName()
				+ " only supports SCAN commands with MATCH patterns containing hash-tags ( curly-brackets enclosed strings )");
	}
}
 

public List<String> scanKeys(String key) {
    Jedis jedis = null;
    try {
        jedis = jedisPool.getResource();
        List<String> keys = new ArrayList<String>();
        String cursor = "0";
        ScanParams sp = new ScanParams();
        sp.match("*" + key + "*");
        sp.count(100);
        do {
            ScanResult<String> ret = jedis.scan(cursor, sp);
            List<String> result = ret.getResult();
            if (result != null && result.size() > 0) {
                keys.addAll(result);
            }
            //再处理cursor
            cursor = ret.getStringCursor();
        } while (!cursor.equals("0"));
        return keys;
    } finally {
        if (jedis != null) {
            jedis.close();
        }
    }
}
 

@Override
public ScanResult<Entry<String, Double>> zscan(final String key, final long cursor, final ScanParams params) {
    Assert.hasText(key);
    Assert.notNull(params);

    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final ScanResult<Tuple> res = jedis.zscan(key, String.valueOf(cursor), params);
        final List<Tuple> tuples = res.getResult();
        if (CollectionUtils.isEmpty(tuples)) {
            return new ScanResult<>(res.getStringCursor(), Collections.emptyList());
        }

        final List<Entry<String, Double>> newTuples = Lists.newArrayList();
        tuples.forEach(tuple -> newTuples.add(new AbstractMap.SimpleEntry<>(tuple.getElement(), tuple.getScore())));
        return new ScanResult<>(res.getStringCursor(), newTuples);
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public ScanResult<String> sscan(String key, String cursor, ScanParams params)
{
    ScanResult<byte[]> scanResult = jedisCluster.sscan(key.getBytes(), cursor.getBytes(), params);
    List<String> results = scanResult.getResult().stream().map(String::new).collect(Collectors.toList());
    return new ScanResult<>(scanResult.getCursorAsBytes(), results);
}
 

@Override
public ScanResult<Map.Entry<String, String>> hscan(String key, String cursor, ScanParams params) {
    ScanResult<Map.Entry<byte[], byte[]>> scanResult = jedisCluster.hscan(key.getBytes(), cursor.getBytes(), params);
    List<Map.Entry<String, String>> results = scanResult.getResult()
                                                        .stream()
                                                        .map(entry -> new AbstractMap.SimpleEntry<>(new String(entry.getKey()),
                                                                                                    new String(entry.getValue())))
                                                        .collect(Collectors.toList());

    return new ScanResult<>(scanResult.getCursorAsBytes(), results);
}
 

@Override
public ScanResult<String> sscan(String key, String cursor, ScanParams params) {
	try {
		org.rarefiedredis.redis.ScanResult<Set<String>> sr = redis.sscan(key, Long.valueOf(cursor), "count", "1000000");
		List<String> list = sr.results.stream().collect(Collectors.toList());
           ScanResult<String> result = new ScanResult<String>("0", list);
           return result;
       }
       catch (Exception e) {
           throw new JedisException(e);
       }
}
 

/**
 * Creates a key scan stream that matches keys with the given pattern.
 *
 * @param pattern The Redis pattern to match.
 */
protected Stream<String> createKeyScanStream(String pattern) throws Exception {
  return redis.fullScan(() -> {
    ScanParams param = new ScanParams();
    param.match(pattern);
    return param;
  }).parallel();
}
 

@Override
public ScanResult<String> sscan(final String key, final String cursor, final ScanParams params) {
    writeAsync(key, new Callable<OperationResult<ScanResult<String>>>() {
        @Override
        public OperationResult<ScanResult<String>> call() throws Exception {
            return shadowClient.d_sscan(key, cursor, params);
        }
    });

    return DynoDualWriterClient.super.sscan(key, cursor, params);
}
 

@Override
public ScanResult<String> sscan(final String key, final long cursor, final ScanParams params) {
    Assert.hasText(key);
    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        return jedis.sscan(key, String.valueOf(cursor), params);
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

@Override
public <T> ScanResult<T> scan(final long cursor, final ScanParams params, final TypeReference<T> type) {
    final ScanResult<String> res = scan(cursor, params);
    final String nextCursor = res.getStringCursor();
    final long next = Long.parseLong(nextCursor);
    if (next == 0) {
        return new ScanResult<>(START_CURSOR, Collections.emptyList());
    }

    final List<String> values = res.getResult();
    final List<T> newValues = values.stream().map(value -> JSON.parseObject(value, type)).collect(Collectors.toList());
    return new ScanResult<>(nextCursor, newValues);
}
 

private Set<String> scanSet(String key) {
  final Set<String> results = new HashSet<>();
  final AtomicReference<String> cursor = new AtomicReference<>(ScanParams.SCAN_POINTER_START);
  do {
    final ScanResult<String> result =
        redisClientDelegate.withCommandsClient(
            jedis -> {
              return jedis.sscan(key, cursor.get());
            });
    results.addAll(result.getResult());
    cursor.set(result.getCursor());
  } while (!"0".equals(cursor.get()));
  return results;
}
 

@Override
public ScanResult<Entry<String, String>> hscan(String key, String cursor, ScanParams params) {
  Span span = helper.buildSpan("hscan", key);
  span.setTag("cursor", cursor);
  span.setTag("params", TracingHelper.toString(params.getParams()));
  try {
    return super.hscan(key, cursor, params);
  } catch (Exception e) {
    onError(e, span);
    throw e;
  } finally {
    span.finish();
  }
}
 

private static void deleteExistingKeys(Jedis node) throws Exception {
  String nextCursor = "0";
  Set<String> matchingKeys = new HashSet<>();
  ScanParams params = new ScanParams();
  params.match("*");

  // get all of the keys for the particular node
  do {
    ScanResult scanResult = node.scan(nextCursor, params);
    List<String> keys = scanResult.getResult();
    nextCursor = scanResult.getCursor();

    matchingKeys.addAll(keys);

  } while (!nextCursor.equals("0"));

  if (matchingKeys.size() == 0) {
    return;
  }

  // we cannot pass all of the keys to del because of the following error:
  // "CROSSSLOT Keys in request don't hash to the same slot"
  // so iterate over and delete them individually.
  for (String key : matchingKeys.toArray(new String[matchingKeys.size()])) {
    node.del(key);
  }
}
 

@Override
public ScanResult<Tuple> zscan(byte[] key, byte[] cursor, ScanParams params) {
  Span span = helper.buildSpan("zscan", key);
  span.setTag("cursor", Arrays.toString(cursor));
  span.setTag("params", TracingHelper.toString(params.getParams()));
  try {
    return super.zscan(key, cursor, params);
  } catch (Exception e) {
    onError(e, span);
    throw e;
  } finally {
    span.finish();
  }
}
 

@Override
public ScanResult<Entry<String, String>> hscan(final String key, final long cursor, final ScanParams params) {
    Assert.hasText(key);
    Assert.notNull(params);
    try {
        return cluster.hscan(key, String.valueOf(cursor), params);
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    }
}
 

@Override
public ScanResult<String> sscan(String key, String cursor, ScanParams params) {
  Span span = helper.buildSpan("sscan", key);
  span.setTag("cursor", cursor);
  span.setTag("params", TracingHelper.toString(params.getParams()));
  try {
    return super.sscan(key, cursor, params);
  } catch (Exception e) {
    onError(e, span);
    throw e;
  } finally {
    span.finish();
  }
}
 

@Override
public ScanResult<Entry<byte[], byte[]>> hscan(byte[] key, byte[] cursor, ScanParams params) {
  Span span = helper.buildSpan("hscan", key);
  span.setTag("cursor", Arrays.toString(cursor));
  span.setTag("params", TracingHelper.toString(params.getParams()));
  try {
    return super.hscan(key, cursor, params);
  } catch (Exception e) {
    onError(e, span);
    throw e;
  } finally {
    span.finish();
  }
}
 

@Override
public ScanResult<Tuple> zscan(String key, String cursor, ScanParams params) {
  Span span = helper.buildSpan("zscan", key);
  span.setTag("cursor", cursor);
  span.setTag("params", TracingHelper.toString(params.getParams()));
  try {
    return super.zscan(key, cursor, params);
  } catch (Exception e) {
    onError(e, span);
    throw e;
  } finally {
    span.finish();
  }
}
 

@Override
public ScanResult<byte[]> scan(byte[] cursor, ScanParams params) {
  Span span = helper.buildSpan("scan");
  span.setTag("cursor", Arrays.toString(cursor));
  span.setTag("params", TracingHelper.toString(params.getParams()));
  try {
    return super.scan(cursor, params);
  } catch (Exception e) {
    onError(e, span);
    throw e;
  } finally {
    span.finish();
  }
}