redis.clients.jedis.Pipeline Java Examples

@Test
public void no_reconnect_during_initialization() throws InterruptedException {
    final Jedis firstInitTryResult = mock(Jedis.class);
    final Jedis reconnectResult = mock(Jedis.class);

    final Pipeline reconnectPipeline = mock(Pipeline.class);
    when(reconnectResult.pipelined()).thenReturn(reconnectPipeline);

    when(clientProvider.getJedisClient()).thenReturn(Optional.empty())
            .thenReturn(Optional.of(firstInitTryResult))
            .thenReturn(Optional.of(reconnectResult));
    try (JedisClient jedisClient = new JedisClient(clientProvider, MAX_INIT_RETRIES, INIT_RETRIES_INTERVAL_MILLIS)) {

        jedisClient.reconnect(); // won't work, client is initializing
        assertEquals(null, jedisClient.getPipeline().orElse(null));
        TimeUnit.MILLISECONDS.sleep(INIT_RETRIES_INTERVAL_MILLIS * 2);
        // initialization resulted in firstInitTryResult
        jedisClient.reconnect();

        verify(firstInitTryResult).close();
        assertEquals(reconnectPipeline, jedisClient.getPipeline().orElse(null));
    }
}
 

@Test
public void givenMultipleIndependentOperations_whenNetworkOptimizationIsImportant_thenWrapThemInAPipeline() {
    String userOneId = "4352523";
    String userTwoId = "4849888";

    Pipeline p = jedis.pipelined();
    p.sadd("searched#" + userOneId, "paris");
    p.zadd("ranking", 126, userOneId);
    p.zadd("ranking", 325, userTwoId);
    Response<Boolean> pipeExists = p.sismember("searched#" + userOneId, "paris");
    Response<Set<String>> pipeRanking = p.zrange("ranking", 0, -1);
    p.sync();

    Assert.assertTrue(pipeExists.get());
    Assert.assertEquals(2, pipeRanking.get().size());
}
 

@Test
public void testEvalshaKeyAndArg() {
  String key = "test";
  String arg = "3";
  String script = "redis.call('INCRBY', KEYS[1], ARGV[1]) redis.call('INCRBY', KEYS[1], ARGV[1])";
  String sha1 = jedis.scriptLoad(script);

  assertTrue(jedis.scriptExists(sha1));

  Pipeline p = jedis.pipelined();
  p.set(key, "0");
  Response<Object> result0 = p.evalsha(sha1, Arrays.asList(key), Arrays.asList(arg));
  p.incr(key);
  Response<Object> result1 = p.evalsha(sha1, Arrays.asList(key), Arrays.asList(arg));
  Response<String> result2 = p.get(key);
  p.sync();

  assertNull(result0.get());
  assertNull(result1.get());
  assertEquals("13", result2.get());
}
 

@Test
public void testCloseable() throws IOException {
  // we need to test with fresh instance of Jedis
  Jedis jedis2 = new Jedis(hnp.getHost(), hnp.getPort(), 500);
  jedis2.auth("foobared");

  Pipeline pipeline = jedis2.pipelined();
  Response<String> retFuture1 = pipeline.set("a", "1");
  Response<String> retFuture2 = pipeline.set("b", "2");

  pipeline.close();

  // it shouldn't meet any exception
  retFuture1.get();
  retFuture2.get();
}
 

@Test
public void testEvalKeyAndArg() {
  String key = "test";
  String arg = "3";
  String script = "redis.call('INCRBY', KEYS[1], ARGV[1]) redis.call('INCRBY', KEYS[1], ARGV[1])";

  Pipeline p = jedis.pipelined();
  p.set(key, "0");
  Response<Object> result0 = p.eval(script, Arrays.asList(key), Arrays.asList(arg));
  p.incr(key);
  Response<Object> result1 = p.eval(script, Arrays.asList(key), Arrays.asList(arg));
  Response<String> result2 = p.get(key);
  p.sync();

  assertNull(result0.get());
  assertNull(result1.get());
  assertEquals("13", result2.get());
}
 

@Override
public void saveFile(String fileLengthKey, String fileDataKey, String fileName, List<byte[]> values, long fileLength) {
    Jedis jedis = null;
    try {
        jedis = jedisPool.getResource();
        Pipeline pipelined = jedis.pipelined();
        pipelined.hset(fileLengthKey.getBytes(), fileName.getBytes(), Longs.toByteArray(fileLength));
        Long blockSize = getBlockSize(fileLength);
        for (int i = 0; i < blockSize; i++) {
            pipelined.hset(fileDataKey.getBytes(), getBlockName(fileName, i), compressFilter(values.get(i)));
            if (i % Constants.SYNC_COUNT == 0) {
                pipelined.sync();
                pipelined = jedis.pipelined();
            }
        }
        values.clear();
        pipelined.sync();
    } finally {
        jedis.close();
    }
}
 

@Override
public void rename(String fileLengthKey, String fileDataKey, String oldField, String newField, List<byte[]> values, long
        fileLength) {
    long blockSize = 0;
    Jedis jedis = null;
    try {
        jedis = jedisPool.getResource();
        Pipeline pipelined = jedis.pipelined();
        //add new file length
        pipelined.hset(fileLengthKey.getBytes(), newField.getBytes(), Longs.toByteArray(fileLength));
        //add new file content
        blockSize = getBlockSize(fileLength);
        for (int i = 0; i < blockSize; i++) {
            pipelined.hset(fileDataKey.getBytes(), getBlockName(newField, i), compressFilter(values.get(i)));
        }
        values.clear();
        pipelined.sync();
    } finally {
        jedis.close();
        deleteFile(fileLengthKey, fileDataKey, oldField, blockSize);
    }
}
 

/**
 * Use transactions to delete index file
 *
 * @param fileLengthKey
 * @param fileDataKey
 * @param field
 * @param blockSize
 */
@Override
public void deleteFile(String fileLengthKey, String fileDataKey, String field, long blockSize) {
    Jedis jedis = null;
    try {
        jedis = jedisPool.getResource();
        Pipeline pipelined = jedis.pipelined();
        //delete file length
        pipelined.hdel(fileLengthKey.getBytes(), field.getBytes());
        //delete file content
        for (int i = 0; i < blockSize; i++) {
            byte[] blockName = getBlockName(field, i);
            pipelined.hdel(fileDataKey.getBytes(), blockName);
        }
        pipelined.sync();
    } finally {
        jedis.close();
    }
}
 

@Override
public void saveFile(String fileLengthKey, String fileDataKey, String fileName, List<byte[]> values, long fileLength) {
    Jedis jedis = openJedis();
    Pipeline pipelined = jedis.pipelined();
    pipelined.hset(fileLengthKey.getBytes(), fileName.getBytes(), Longs.toByteArray(fileLength));
    Long blockSize = getBlockSize(fileLength);
    for (int i = 0; i < blockSize; i++) {
        pipelined.hset(fileDataKey.getBytes(), getBlockName(fileName, i), compressFilter(values.get(i)));
        if (i % Constants.SYNC_COUNT == 0) {
            pipelined.sync();
            pipelined = jedis.pipelined();
        }
    }
    pipelined.sync();
    jedis.close();
    values.clear();
}
 

public void testCombPipelineTrans() {// 0.099秒
	Jedis jedis = new Jedis("120.25.241.144", 6379);
	jedis.auth("b840fc02d52404542994");

	long start = System.currentTimeMillis();
	Pipeline pipeline = jedis.pipelined();
	pipeline.multi();
	for (int i = 0; i < 1000; i++) {
		pipeline.set("n" + i, "n" + i);
		System.out.println(i);
	}
	pipeline.exec();
	pipeline.syncAndReturnAll();
	long end = System.currentTimeMillis();
	System.out.println("共花费：" + (end - start) / 1000.0 + "秒");

	jedis.disconnect();
	try {
		Closeables.close(jedis, true);
	} catch (IOException e) {
		e.printStackTrace();
	}
}
 

private void registerFakeSession(int count, User user) {
	//Clean all sessions
	Jedis jedis = JedisFactory.getJedis();
	Pipeline pipeline = jedis.pipelined();
	Set<byte[]> strs = jedis.keys("*".getBytes());
	for ( byte[] key : strs ) {
		pipeline.del(key);
	}
	pipeline.sync();
	
	UserId userId = user.get_id();
	SessionKey sessionKey = SessionKey.createSessionKeyFromRandomString();
	user.setSessionKey(sessionKey);
	
	//Store it with machineid to redis
	jedis = JedisFactory.getJedis();
	pipeline = jedis.pipelined();
	for ( int i=0; i<count; i++ ) {
		pipeline.hset(sessionKey.toString(), SessionManager.H_MACHINE_KEY, "localhost:10000");
		pipeline.hset(userId.toString(), SessionManager.H_MACHINE_KEY, "localhost:10000");
		pipeline.hset(userId.toString(), SessionManager.H_SESSION_KEY, sessionKey.toString());
	}
	pipeline.sync();
}
 

private boolean sendValuesToRedis(String... values) {
    if (values.length == 0) {
        return true;
    }
    synchronized (client) {
        /*
         * RedisBatchAppender-doc stated, that jedis client is not thread safe.
         * logback's AppenderBase.doAppend is synchronized, so no concurrent logs can access this method,
         * but flushing thread could be active
         */
        try {
            final Pipeline pipeline = client.getPipeline().orElse(null);
            if (pipeline != null) {
                final long start = System.currentTimeMillis();
                addValuesToPipeline(pipeline, values);
                pipeline.sync();
                logSendStatistics(values.length, start);
                return true;
            }
        } catch (JedisException ex) {
            log.info("unable to send {} events, reconnecting to redis", values.length, ex);
        }
        client.reconnect();
        return false;
    }
}
 

@Test
public void testEvalKeyAndArgWithBinary() {
  // binary
  byte[] bKey = SafeEncoder.encode("test");
  byte[] bArg = SafeEncoder.encode("3");
  byte[] bScript = SafeEncoder
      .encode("redis.call('INCRBY', KEYS[1], ARGV[1]) redis.call('INCRBY', KEYS[1], ARGV[1])");

  Pipeline bP = jedis.pipelined();
  bP.set(bKey, SafeEncoder.encode("0"));
  Response<Object> bResult0 = bP.eval(bScript, Arrays.asList(bKey), Arrays.asList(bArg));
  bP.incr(bKey);
  Response<Object> bResult1 = bP.eval(bScript, Arrays.asList(bKey), Arrays.asList(bArg));
  Response<byte[]> bResult2 = bP.get(bKey);
  bP.sync();

  assertNull(bResult0.get());
  assertNull(bResult1.get());
  assertArrayEquals(SafeEncoder.encode("13"), bResult2.get());
}
 

@Override
public void rename(String fileLengthKey, String fileDataKey, String oldField, String newField, List<byte[]> values, long
        fileLength) {
    Jedis jedis = openJedis();
    Pipeline pipelined = jedis.pipelined();
    //add new file length
    pipelined.hset(fileLengthKey.getBytes(), newField.getBytes(), Longs.toByteArray(fileLength));
    //add new file content
    Long blockSize = getBlockSize(fileLength);
    for (int i = 0; i < blockSize; i++) {
        pipelined.hset(fileDataKey.getBytes(), getBlockName(newField, i), compressFilter(values.get(i)));
    }
    pipelined.sync();
    jedis.close();
    values.clear();
    deleteFile(fileLengthKey, fileDataKey, oldField, blockSize);
}
 

@Test
public void multiWithSync() {
  jedis.set("foo", "314");
  jedis.set("bar", "foo");
  jedis.set("hello", "world");
  Pipeline p = jedis.pipelined();
  Response<String> r1 = p.get("bar");
  p.multi();
  Response<String> r2 = p.get("foo");
  p.exec();
  Response<String> r3 = p.get("hello");
  p.sync();

  // before multi
  assertEquals("foo", r1.get());
  // It should be readable whether exec's response was built or not
  assertEquals("314", r2.get());
  // after multi
  assertEquals("world", r3.get());
}
 

@Override
public void saveFile(String fileLengthKey, String fileDataKey, String fileName, List<byte[]> values, long fileLength) {
    Jedis jedis = null;
    try {
        jedis = jedisPool.getResource();
        Pipeline pipelined = jedis.pipelined();
        pipelined.hset(fileLengthKey.getBytes(), fileName.getBytes(), Longs.toByteArray(fileLength));
        Long blockSize = getBlockSize(fileLength);
        for (int i = 0; i < blockSize; i++) {
            pipelined.hset(fileDataKey.getBytes(), getBlockName(fileName, i), compressFilter(values.get(i)));
            if (i % Constants.SYNC_COUNT == 0) {
                pipelined.sync();
                pipelined = jedis.pipelined();
            }
        }
        values.clear();
        pipelined.sync();
    } finally {
        jedis.close();
    }
}
 

@Override
public void rename(String fileLengthKey, String fileDataKey, String oldField, String newField, List<byte[]> values, long
        fileLength) {
    long blockSize = 0;
    Jedis jedis = null;
    try {
        jedis = jedisPool.getResource();
        Pipeline pipelined = jedis.pipelined();
        //add new file length
        pipelined.hset(fileLengthKey.getBytes(), newField.getBytes(), Longs.toByteArray(fileLength));
        //add new file content
        blockSize = getBlockSize(fileLength);
        for (int i = 0; i < blockSize; i++) {
            pipelined.hset(fileDataKey.getBytes(), getBlockName(newField, i), compressFilter(values.get(i)));
        }
        values.clear();
        pipelined.sync();
    } finally {
        jedis.close();
        deleteFile(fileLengthKey, fileDataKey, oldField, blockSize);
    }
}
 

/**
 * Use transactions to delete index file
 *
 * @param fileLengthKey
 * @param fileDataKey
 * @param field
 * @param blockSize
 */
@Override
public void deleteFile(String fileLengthKey, String fileDataKey, String field, long blockSize) {
    Jedis jedis = null;
    try {
        jedis = jedisPool.getResource();
        Pipeline pipelined = jedis.pipelined();
        //delete file length
        pipelined.hdel(fileLengthKey.getBytes(), field.getBytes());
        //delete file content
        for (int i = 0; i < blockSize; i++) {
            byte[] blockName = getBlockName(field, i);
            pipelined.hdel(fileDataKey.getBytes(), blockName);
        }
        pipelined.sync();
    } finally {
        jedis.close();
    }
}
 

@Test
public void new_pipeline_on_successful_reconnect() throws InterruptedException {
    Jedis newJedis = mock(Jedis.class);
    Pipeline newPipeline = mock(Pipeline.class);
    when(newJedis.pipelined()).thenReturn(newPipeline);

    when(clientProvider.getJedisClient()).thenReturn(Optional.of(jedis)).thenReturn(Optional.of(newJedis));
    try (JedisClient jedisClient = new JedisClient(clientProvider, MAX_INIT_RETRIES, INIT_RETRIES_INTERVAL_MILLIS)) {
        verify(clientProvider, times(1)).getJedisClient();
        assertEquals(pipeline, jedisClient.getPipeline().orElse(null));

        jedisClient.reconnect();

        verify(clientProvider, times(2)).getJedisClient();
        assertEquals(newPipeline, jedisClient.getPipeline().orElse(null));
    }
}
 

protected void doActualFlush(Jedis jedis, Pipeline pipeline,
                             ArrayList<Callable<Integer>> pipelineCallables,
                             long batch, boolean closeAfterFlush) {
  LOG.debug("Flushing pipeline. batch " + batch + " size = " +
            pipelineCallables.size() + " pending batches " +
            pendingPipelineBatches.addAndGet(1));
  int count = 0;
  try {
    pipeline.sync();
    for (Callable<Integer> c : pipelineCallables) {
      count += c.call();
    }
    if (closeAfterFlush) {
      pipeline.close();
      jedis.close();
    }
  } catch (Exception e) {
    throw new RuntimeException(
      "Caught Exception from redis pipeline " + getRedisServerInUse(), e);
  } finally {
    pipelineCallables.clear();
    numOpsRespondedThisRound.addAndGet(count);
  }
  LOG.debug("Processed batch  " + batch + " count " + count + " responses."
            + " pending batches " + pendingPipelineBatches.addAndGet(-1));
}
 

private void doUpsertString(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException {
  if (value != null && value.getType() == Field.Type.STRING) {
    String val = value.getValueAsString();
    pipeline.set(key, val);
    tempRecords.add(new ErrorRecord(record, "String", key, val));
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), " value should be String");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be String"
        )
    );
  }
}
 

private void sendSetCommand(String keyPrefix, Row row) {
    List<String> keyValues = new ArrayList<>(row.getArity() * 2);
    List<String> keys = new ArrayList<>(row.getArity());
    for (int i = 0; i < fieldNames.length; i++) {
        if (containKey(fieldNames[i])) {
            continue;
        }
        StringBuilder builder = new StringBuilder();
        builder.append(keyPrefix).append(":").append(fieldNames[i]);
        String key = builder.toString();
        keys.add(key);
        keyValues.add(key);
        keyValues.add(row.getField(i).toString());
    }
    if (keyValues.size() == 2) {
        jedis.set(keyValues.get(0), keyValues.get(1));
        expire(keys);
    } else if (keyValues.size() > 2) {
        Pipeline pipeline = jedis.pipelined();
        pipeline.mset(keyValues.toArray(new String[keyValues.size()]));
        expire(pipeline, keys);
        pipeline.sync();
    }
}
 

@Override
public boolean smove(final String source, final String destination, final String member) {
    Assert.hasText(source);
    Assert.hasText(destination);
    Assert.hasText(member);
    ShardedJedis jedis = null;
    try {
        jedis = POOL.getJedis(config.getRedisType());
        final Collection<Jedis> allShards;
        if ((allShards = jedis.getAllShards()).size() == 1) {
            return isSuccess(allShards.iterator().next().smove(source, destination, member));
        } else if (allShards.size() > 1) {
            final AtomicLong val = new AtomicLong();
            allShards.parallelStream().forEach(shard -> {
                Pipeline pipeline = shard.pipelined();
                pipeline.sismember(source, member);
                Response<Long> response = pipeline.smove(source, destination, member);
                pipeline.sync();
                val.addAndGet(response.get());
            });

            if (val.get() > 0) {
                return true;
            }
        }

        return false;
    } catch (final Throwable e) {
        throw new RedisClientException(e.getMessage(), e);
    } finally {
        POOL.close(jedis);
    }
}
 

/**
 * Store a {@link org.quartz.Calendar}
 * @param name the name of the calendar
 * @param calendar the calendar object to be stored
 * @param replaceExisting if true, any existing calendar with the same name will be overwritten
 * @param updateTriggers if true, any existing triggers associated with the calendar will be updated
 * @param jedis a thread-safe Redis connection
 * @throws JobPersistenceException
 */
@Override
public void storeCalendar(String name, Calendar calendar, boolean replaceExisting, boolean updateTriggers, Jedis jedis) throws JobPersistenceException{
    final String calendarHashKey = redisSchema.calendarHashKey(name);
    if(!replaceExisting && jedis.exists(calendarHashKey)){
        throw new ObjectAlreadyExistsException(String.format("Calendar with key %s already exists.", calendarHashKey));
    }
    Map<String, String> calendarMap = new HashMap<>();
    calendarMap.put(CALENDAR_CLASS, calendar.getClass().getName());
    try {
        calendarMap.put(CALENDAR_JSON, mapper.writeValueAsString(calendar));
    } catch (JsonProcessingException e) {
        throw new JobPersistenceException("Unable to serialize calendar.", e);
    }

    Pipeline pipe = jedis.pipelined();
    pipe.hmset(calendarHashKey, calendarMap);
    pipe.sadd(redisSchema.calendarsSet(), calendarHashKey);
    pipe.sync();

    if(updateTriggers){
        final String calendarTriggersSetKey = redisSchema.calendarTriggersSetKey(name);
        Set<String> triggerHashKeys = jedis.smembers(calendarTriggersSetKey);
        for (String triggerHashKey : triggerHashKeys) {
            OperableTrigger trigger = retrieveTrigger(redisSchema.triggerKey(triggerHashKey), jedis);
            long removed = jedis.zrem(redisSchema.triggerStateKey(RedisTriggerState.WAITING), triggerHashKey);
            trigger.updateWithNewCalendar(calendar, misfireThreshold);
            if(removed == 1){
                setTriggerState(RedisTriggerState.WAITING, (double) trigger.getNextFireTime().getTime(), triggerHashKey, jedis);
            }
        }
    }
}
 

@Test
public void pipelineWithPubSub() {
  Pipeline pipelined = jedis.pipelined();
  Response<Long> p1 = pipelined.publish("foo", "bar");
  Response<Long> p2 = pipelined.publish("foo".getBytes(), "bar".getBytes());
  pipelined.sync();
  assertEquals(0, p1.get().longValue());
  assertEquals(0, p2.get().longValue());
}
 

@Test
public void testDiscardInPipeline() {
  Pipeline pipeline = jedis.pipelined();
  pipeline.multi();
  pipeline.set("foo", "bar");
  Response<String> discard = pipeline.discard();
  Response<String> get = pipeline.get("foo");
  pipeline.sync();
  discard.get();
  get.get();
}
 

@Test
public void testResetStateWhenInPipeline() {
  Pipeline pipeline = jedis.pipelined();
  pipeline.set("foo", "3");
  jedis.resetState();
  String result = jedis.get("foo");
  assertEquals(result, "3");
}
 

@Test
public void testReuseJedisWhenPipelineIsEmpty() {
  Pipeline pipeline = jedis.pipelined();
  pipeline.set("foo", "3");
  pipeline.sync();
  String result = jedis.get("foo");
  assertEquals(result, "3");
}
 

@Test
public void multi() {
  Pipeline p = jedis.pipelined();
  p.multi();
  Response<Long> r1 = p.hincrBy("a", "f1", -1);
  Response<Long> r2 = p.hincrBy("a", "f1", -2);
  Response<List<Object>> r3 = p.exec();
  List<Object> result = p.syncAndReturnAll();

  assertEquals(new Long(-1), r1.get());
  assertEquals(new Long(-3), r2.get());

  assertEquals(4, result.size());

  assertEquals("OK", result.get(0));
  assertEquals("QUEUED", result.get(1));
  assertEquals("QUEUED", result.get(2));

  // 4th result is a list with the results from the multi
  @SuppressWarnings("unchecked")
  List<Object> multiResult = (List<Object>) result.get(3);
  assertEquals(new Long(-1), multiResult.get(0));
  assertEquals(new Long(-3), multiResult.get(1));

  assertEquals(new Long(-1), r3.get().get(0));
  assertEquals(new Long(-3), r3.get().get(1));

}
 

@Test
public void canRetrieveUnsetKey() {
  Pipeline p = jedis.pipelined();
  Response<String> shouldNotExist = p.get(UUID.randomUUID().toString());
  p.sync();
  assertNull(shouldNotExist.get());
}