Java Code Examples for com.google.common.hash.Hashing#crc32()

private long generateFingerprint(Collection<URL> extensions) {
    Collection<Integer> hashes = new ArrayList<>();
    for (URL extUrl : extensions) {
        try {
            HashingInputStream hin = new HashingInputStream(
                    Hashing.crc32(), extUrl.openStream());
            //noinspection StatementWithEmptyBody
            while (hin.read() != -1) {
                // Do nothing. Reading all input stream to update hash.
            }
            hashes.add(hin.hash().asInt());
        } catch (IOException e) {
            throw new IllegalArgumentException(e);
        }
    }
    //  FIXME: how to include behaviours in the hash?
    int low = Arrays.hashCode(hashes.toArray());
    int high = pipelineModel.hashCode();
    return ByteBuffer.allocate(8).putInt(high).putInt(low).getLong(0);
}
 

/**
 * Returns the HashFunction named by algorithm
 *
 * See the Hasher class docs for a list of algorithms and guidance on selection.
 *
 * @param algorithm the hash algorithm to use
 * @throws IllegalArgumentException for an invalid seed given the algorithm
 * @throws RuntimeException when the seed cannot be parsed
 */
private void makeHashFunc(String algorithm) throws IllegalArgumentException, RuntimeException
{
  if (hash_func != null) { throw new RuntimeException("The hash function should only be set once per instance"); }

  if (algorithm.startsWith("good-")) {
    int bits = Integer.parseInt(algorithm.substring(5));
    hash_func = Hashing.goodFastHash(bits);
  }
  else if (algorithm.equals("murmur3-32")) { hash_func = Hashing.murmur3_32();  }
  else if (algorithm.equals("murmur3-128")){ hash_func = Hashing.murmur3_128(); }
  else if (algorithm.equals("sip24"))      { hash_func = Hashing.sipHash24();   }
  else if (algorithm.equals("sha1"))       { hash_func = Hashing.sha1();        }
  else if (algorithm.equals("sha256"))     { hash_func = Hashing.sha256();      }
  else if (algorithm.equals("sha512"))     { hash_func = Hashing.sha512();      }
  else if (algorithm.equals("md5"))        { hash_func = Hashing.md5();         }
  else if (algorithm.equals("adler32"))    { hash_func = Hashing.adler32();     }
  else if (algorithm.equals("crc32"))      { hash_func = Hashing.crc32();       }
  else { throw new IllegalArgumentException("No hash function found for algorithm "+algorithm+". Allowed values include "+HASH_NAMES); }
}
 

public static HashFunction getHasher(HashType hashType) {
  switch(hashType) {
    case MURMUR3_128:
      return Hashing.murmur3_128();
    case MURMUR3_32:
      return Hashing.murmur3_32();
    case SIPHASH24:
      return Hashing.sipHash24();
    case MD5:
      return Hashing.md5();
    case SHA1:
      return Hashing.sha1();
    case SHA256:
      return Hashing.sha256();
    case SHA512:
      return Hashing.sha512();
    case ADLER32:
      return Hashing.adler32();
    case CRC32:
      return Hashing.crc32();
    case CRC32C:
      return Hashing.crc32c();
    default:
      throw new IllegalArgumentException(Utils.format("Unsupported Hashing Algorithm: {}", hashType.name()));
  }
}
 

@BeforeMethod
public void setup() throws Exception {
    pulsar = mock(PulsarService.class);
    executor = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("test").build();
    zkc = MockZooKeeper.newInstance();
    zkCache = new LocalZooKeeperCache(zkc, 30, executor);
    localCache = new LocalZooKeeperCacheService(zkCache, null);

    // set mock pulsar localzkcache
    LocalZooKeeperCacheService localZkCache = mock(LocalZooKeeperCacheService.class);
    ZooKeeperDataCache<LocalPolicies> poilciesCache = mock(ZooKeeperDataCache.class);
    when(pulsar.getLocalZkCacheService()).thenReturn(localZkCache);
    when(localZkCache.policiesCache()).thenReturn(poilciesCache);
    doNothing().when(poilciesCache).registerListener(any());
    bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());

    doReturn(zkCache).when(pulsar).getLocalZkCache();
    doReturn(localCache).when(pulsar).getLocalZkCacheService();
}
 

@Test
public void whenConcatenatingHashFunction_concatenatedHashShouldBeReturned() throws Exception {
    int inputData = 15;

    HashFunction hashFunction = Hashing.concatenating(Hashing.crc32(), Hashing.crc32());
    HashFunction crc32Function = Hashing.crc32();

    HashCode hashCode = hashFunction.hashInt(inputData);
    HashCode crc32HashCode = crc32Function.hashInt(inputData);

    assertEquals(crc32HashCode.toString() + crc32HashCode.toString(), hashCode.toString());
}
 

public static HashFunction forName(String name) {
    switch (name) {
        case "murmur3_32":
            return Hashing.murmur3_32();
        case "murmur3_128":
            return Hashing.murmur3_128();
        case "crc32":
            return Hashing.crc32();
        case "md5":
            return Hashing.md5();
        default:
            throw new IllegalArgumentException("Can't find hash function with name " + name);
    }
}
 

/**
 * Default constructor.
 *
 * @throws PulsarServerException
 */
public NamespaceService(PulsarService pulsar) {
    this.pulsar = pulsar;
    host = pulsar.getAdvertisedAddress();
    this.config = pulsar.getConfiguration();
    this.loadManager = pulsar.getLoadManager();
    this.bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());
    this.ownershipCache = new OwnershipCache(pulsar, bundleFactory, this);
    this.namespaceClients = new ConcurrentOpenHashMap<>();
    this.bundleOwnershipListeners = new CopyOnWriteArrayList<>();
}
 

@BeforeMethod
@Override
public void setup() throws Exception {
    conf.setLoadBalancerEnabled(true);
    conf.setBrokerServicePortTls(Optional.of(0));
    conf.setWebServicePortTls(Optional.of(0));
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);

    super.internalSetup();

    bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());

    adminTls = spy(PulsarAdmin.builder().tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
            .serviceHttpUrl(brokerUrlTls.toString()).build());

    // create otherbroker to test redirect on calls that need
    // namespace ownership
    mockPulsarSetup = new MockedPulsarService(this.conf);
    mockPulsarSetup.setup();
    otherPulsar = mockPulsarSetup.getPulsar();
    otheradmin = mockPulsarSetup.getAdmin();

    // Setup namespaces
    admin.clusters().createCluster("use", new ClusterData(pulsar.getWebServiceAddress()));
    TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("use"));
    admin.tenants().createTenant("prop-xyz", tenantInfo);
    admin.namespaces().createNamespace("prop-xyz/use/ns1");
}
 

@BeforeMethod
@Override
public void setup() throws Exception {
    conf.setLoadBalancerEnabled(true);
    conf.setBrokerServicePortTls(Optional.of(0));
    conf.setWebServicePortTls(Optional.of(0));
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    conf.setMessageExpiryCheckIntervalInMinutes(1);
    conf.setSubscriptionExpiryCheckIntervalInMinutes(1);
    conf.setBrokerDeleteInactiveTopicsEnabled(false);

    super.internalSetup();

    bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());

    adminTls = spy(PulsarAdmin.builder().tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
            .serviceHttpUrl(brokerUrlTls.toString()).build());

    // create otherbroker to test redirect on calls that need
    // namespace ownership
    mockPulsarSetup = new MockedPulsarService(this.conf);
    mockPulsarSetup.setup();
    otherPulsar = mockPulsarSetup.getPulsar();
    otheradmin = mockPulsarSetup.getAdmin();

    // Setup namespaces
    admin.clusters().createCluster("test", new ClusterData(pulsar.getWebServiceAddress()));
    TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
    admin.tenants().createTenant("prop-xyz", tenantInfo);
    admin.namespaces().createNamespace("prop-xyz/ns1", Sets.newHashSet("test"));
}
 

@BeforeMethod
public void setup() throws Exception {
    final int port = 8080;
    selfBrokerUrl = "tcp://localhost:" + port;
    pulsar = mock(PulsarService.class);
    config = mock(ServiceConfiguration.class);
    executor = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("test").build();
    zkc = MockZooKeeper.newInstance();
    zkCache = new LocalZooKeeperCache(zkc, 30, executor);
    localCache = spy(new LocalZooKeeperCacheService(zkCache, null));
    ZooKeeperDataCache<LocalPolicies> poilciesCache = mock(ZooKeeperDataCache.class);
    when(pulsar.getLocalZkCacheService()).thenReturn(localCache);
    when(localCache.policiesCache()).thenReturn(poilciesCache);
    doNothing().when(poilciesCache).registerListener(any());

    bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());
    nsService = mock(NamespaceService.class);
    brokerService = mock(BrokerService.class);
    doReturn(CompletableFuture.completedFuture(1)).when(brokerService).unloadServiceUnit(any(), anyBoolean(), anyInt(), any());

    doReturn(zkCache).when(pulsar).getLocalZkCache();
    doReturn(localCache).when(pulsar).getLocalZkCacheService();
    doReturn(config).when(pulsar).getConfiguration();
    doReturn(nsService).when(pulsar).getNamespaceService();
    doReturn(Optional.ofNullable(new Integer(port))).when(config).getBrokerServicePort();
    doReturn(Optional.ofNullable(null)).when(config).getWebServicePort();
    doReturn(brokerService).when(pulsar).getBrokerService();
    doReturn(selfBrokerUrl).when(pulsar).getSafeBrokerServiceUrl();
}
 

@BeforeMethod
void setup() throws Exception {
    executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>());
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // Start broker 1
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config1.setClusterName("use");
    config1.setWebServicePort(Optional.of(0));
    config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config1.setBrokerServicePort(Optional.of(0));
    config1.setFailureDomainsEnabled(true);
    config1.setLoadBalancerEnabled(true);
    config1.setAdvertisedAddress("localhost");
    createCluster(bkEnsemble.getZkClient(), config1);
    pulsar1 = new PulsarService(config1);
    pulsar1.start();

    primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get());
    url1 = new URL("http://127.0.0.1" + ":" + pulsar1.getListenPortHTTP().get());
    admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();

    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config2.setClusterName("use");
    config2.setWebServicePort(Optional.of(0));
    config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config2.setBrokerServicePort(Optional.of(0));
    config2.setFailureDomainsEnabled(true);
    config2.setAdvertisedAddress("localhost");
    pulsar2 = new PulsarService(config2);
    pulsar2.start();

    secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get());

    url2 = new URL("http://127.0.0.1" + ":" + config2.getWebServicePort().get());
    admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();

    primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
    secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
    nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
    Thread.sleep(100);
}
 

public static String getHashedKey(String key, String hashingAlgorithm) {
    final long start = System.nanoTime();
    HashFunction hf = null; 
    switch(hashingAlgorithm.toLowerCase()) {
        case "murmur3" :
            hf = Hashing.murmur3_128();
            break;

        case "adler32" :
            hf = Hashing.adler32();
            break;

        case "crc32" :
            hf = Hashing.crc32();
            break;

        case "sha1" :
            hf = Hashing.sha1();
            break;

        case "sha256" :
            hf = Hashing.sha256();
            break;

        case "siphash24" :
            hf = Hashing.sipHash24();
            break;

        case "goodfasthash" :
            hf = Hashing.goodFastHash(128);
            break;

        case "md5" :
        default :
            hf = Hashing.md5();
            break;
    }

    final HashCode hc = hf.newHasher().putString(key, Charsets.UTF_8).hash();
    final byte[] digest = hc.asBytes();
    if(log.isDebugEnabled()) log.debug("Key : " + key +"; digest length : " + digest.length + "; byte Array contents : " + Arrays.toString(digest) );
    final String hKey = encoder.encodeToString(digest);
    if(log.isDebugEnabled()) log.debug("Key : " + key +"; Hashed & encoded key : " + hKey + "; Took " + (System.nanoTime() - start) + " nanos");
    return hKey;
}
 

@BeforeMethod
void setup() throws Exception {
    executor = new ThreadPoolExecutor(1, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>());

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // Start broker 1
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config1.setClusterName("use");
    config1.setWebServicePort(Optional.of(0));
    config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());

    config1.setAdvertisedAddress("localhost");
    config1.setBrokerServicePort(Optional.of(0));
    config1.setBrokerServicePortTls(Optional.of(0));
    config1.setWebServicePortTls(Optional.of(0));
    pulsar1 = new PulsarService(config1);
    pulsar1.start();

    primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get());
    url1 = new URL(pulsar1.getWebServiceAddress());
    admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();

    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config2.setClusterName("use");
    config2.setWebServicePort(Optional.of(0));
    config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config2.setAdvertisedAddress("localhost");
    config2.setBrokerServicePort(Optional.of(0));
    config2.setBrokerServicePortTls(Optional.of(0));
    config2.setWebServicePortTls(Optional.of(0));
    pulsar2 = new PulsarService(config2);
    pulsar2.start();

    secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get());
    url2 = new URL(pulsar2.getWebServiceAddress());
    admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();

    primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
    secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
    nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
    Thread.sleep(100);
}
 

public Crc32ByteArrayPartitioner() {
  hashFunction = Hashing.crc32();
  random = new Random();
}
 

@Test
public void testWriterWithHeadersInjectorEnabled() throws Exception {
  SingerLog singerLog = new SingerLog(createSingerLogConfig("test", "/a/b/c"));
  LogStream logStream = new LogStream(singerLog, "test.tmp");
  KafkaMessagePartitioner partitioner = new Crc32ByteArrayPartitioner();
  KafkaProducerConfig config = new KafkaProducerConfig();
  SingerSettings.setSingerConfig(new SingerConfig());
  KafkaProducerManager.injectTestProducer(config, producer);
  KafkaWriter writer = new KafkaWriter(logStream, config, partitioner, "topicx", false,
      Executors.newCachedThreadPool(), true);
  List<PartitionInfo> partitions = ImmutableList.copyOf(Arrays.asList(
      new PartitionInfo("topicx", 1, new Node(2, "broker2", 9092, "us-east-1b"), null, null),
      new PartitionInfo("topicx", 0, new Node(1, "broker1", 9092, "us-east-1a"), null, null),
      new PartitionInfo("topicx", 2, new Node(3, "broker3", 9092, "us-east-1c"), null, null),
      new PartitionInfo("topicx", 6, new Node(2, "broker2", 9092, "us-east-1b"), null, null),
      new PartitionInfo("topicx", 3, new Node(4, "broker4", 9092, "us-east-1a"), null, null),
      new PartitionInfo("topicx", 5, new Node(1, "broker1", 9092, "us-east-1a"), null, null),
      new PartitionInfo("topicx", 7, new Node(3, "broker3", 9092, "us-east-1c"), null, null),
      new PartitionInfo("topicx", 4, new Node(5, "broker5", 9092, "us-east-1b"), null, null),
      new PartitionInfo("topicx", 8, new Node(4, "broker4", 9092, "us-east-1a"), null, null),
      new PartitionInfo("topicx", 9, new Node(5, "broker5", 9092, "us-east-1b"), null, null),
      new PartitionInfo("topicx", 10, new Node(1, "broker1", 9092, "us-east-1a"), null, null)));
  when(producer.partitionsFor("topicx")).thenReturn(partitions);
  when(producer.send(any())).thenReturn(ConcurrentUtils.constantFuture(
      new RecordMetadata(new TopicPartition("topicx", 0), 0L, 0L, 0L, 0L, 0, 0)));

  Map<Integer, List<LogMessage>> msgPartitionMap = new HashMap<>();
  HashFunction crc32 = Hashing.crc32();
  List<LogMessage> logMessages = new ArrayList<>();
  int pid = new Random().nextInt();
  long session = System.currentTimeMillis();
  TSerializer serializer = new TSerializer();

  for (int i = 0; i < NUM_EVENTS; i++) {
    LogMessage logMessage = new LogMessage();
    logMessage.setKey(ByteBuffer.allocate(100).put(String.valueOf(i).getBytes()));
    logMessage.setMessage(ByteBuffer.allocate(100).put(String.valueOf(i).getBytes()));
    LoggingAuditHeaders headers = new LoggingAuditHeaders()
        .setHost("host-name")
        .setLogName("topicx")
        .setPid(pid)
        .setSession(session)
        .setLogSeqNumInSession(i);
    logMessage.setLoggingAuditHeaders(headers);
    logMessages.add(logMessage);

    int partitionId = Math.abs(crc32.hashBytes(logMessage.getKey()).asInt() % partitions.size());
    List<LogMessage> list = msgPartitionMap.get(partitionId);
    if (list == null) {
      list = new ArrayList<>();
      msgPartitionMap.put(partitionId, list);
    }
    list.add(logMessage);
  }

  Map<Integer, Map<Integer, LoggingAuditHeaders>> mapOfHeadersMap = new HashMap<>();
  Map<Integer, List<ProducerRecord<byte[], byte[]>>> messageCollation = writer
      .messageCollation(partitions, "topicx", logMessages, mapOfHeadersMap);
  for (int i = 0; i < messageCollation.size(); i++) {
    List<ProducerRecord<byte[], byte[]>> writerOutput = messageCollation.get(i);
    List<LogMessage> originalData = msgPartitionMap.get(i);
    if (originalData == null) {
      assertEquals(0, writerOutput.size());
      continue;
    }
    assertEquals(originalData.size(), writerOutput.size());
    for (int j = 0; j < writerOutput.size(); j++) {
      assertTrue(Arrays.equals(originalData.get(j).getKey(), writerOutput.get(j).key()));
      boolean foundLoggingAuditHeaders = false;
      for(Header header : writerOutput.get(j).headers().toArray()){
        if (header.key().equals(LoggingAuditHeadersInjector.getHeaderKey())){
          byte[] expected = serializer.serialize(originalData.get(j).getLoggingAuditHeaders());
          assertArrayEquals(expected, header.value());
          foundLoggingAuditHeaders = true;
          break;
        }
      }
      assertTrue(foundLoggingAuditHeaders);
    }
  }
  // validate if writes are throwing any error
  writer.writeLogMessages(logMessages);
  writer.close();
}
 

@Test
public void testWriteLogMessagesWithCrcPartitioning() throws Exception {
  KafkaMessagePartitioner partitioner = new Crc32ByteArrayPartitioner();
  KafkaProducerConfig config = new KafkaProducerConfig();
  SingerSettings.setSingerConfig(new SingerConfig());
  KafkaProducerManager.injectTestProducer(config, producer);
  // default value for skip noleader partition is false
  KafkaWriter writer = new KafkaWriter(config, partitioner, "topicx", false, Executors.newCachedThreadPool());

  List<PartitionInfo> partitions = ImmutableList.copyOf(Arrays.asList(
      new PartitionInfo("topicx", 1, new Node(2, "broker2", 9092, "us-east-1b"), null, null),
      new PartitionInfo("topicx", 0, new Node(1, "broker1", 9092, "us-east-1a"), null, null),
      new PartitionInfo("topicx", 2, new Node(3, "broker3", 9092, "us-east-1c"), null, null),
      new PartitionInfo("topicx", 6, new Node(2, "broker2", 9092, "us-east-1b"), null, null),
      new PartitionInfo("topicx", 3, new Node(4, "broker4", 9092, "us-east-1a"), null, null),
      new PartitionInfo("topicx", 5, new Node(1, "broker1", 9092, "us-east-1a"), null, null),
      new PartitionInfo("topicx", 7, new Node(3, "broker3", 9092, "us-east-1c"), null, null),
      new PartitionInfo("topicx", 4, new Node(5, "broker5", 9092, "us-east-1b"), null, null),
      new PartitionInfo("topicx", 8, new Node(4, "broker4", 9092, "us-east-1a"), null, null),
      new PartitionInfo("topicx", 9, new Node(5, "broker5", 9092, "us-east-1b"), null, null),
      new PartitionInfo("topicx", 10, new Node(1, "broker1", 9092, "us-east-1a"), null, null)));

  when(producer.partitionsFor("topicx")).thenReturn(partitions);

  // message with same key will be put together in the same bucket (same partition);
  List<String> keys = IntStream.range(0, NUM_KEYS).mapToObj(i->"key"+i).collect(Collectors.toList());
  Map<Integer, List<LogMessage>> msgPartitionMap = new HashMap<>();
  Map<Integer, List<ProducerRecord<byte[], byte[]>>> recordPartitionMap = new HashMap<>();
  Map<Integer, List<RecordMetadata>> metadataPartitionMap = new HashMap<>();
  HashFunction crc32 = Hashing.crc32();
  List<LogMessage> logMessages = new ArrayList<>();
  for(int i = 0; i < NUM_KEYS; i++){
    for(int j = 0; j < NUM_EVENTS / NUM_KEYS; j++){
       LogMessage logMessage = new LogMessage();
       logMessage.setKey(keys.get(i).getBytes());
       logMessage.setMessage(ByteBuffer.allocate(100).put(String.valueOf(i).getBytes()));
       logMessages.add(logMessage);
       int partitionId = Math.abs(crc32.hashBytes(logMessage.getKey()).asInt() % partitions.size());
       ProducerRecord<byte[], byte[]> record = new ProducerRecord<byte[], byte[]>("topicx", partitionId, logMessage.getKey(), logMessage.getMessage());
       RecordMetadata recordMetadata = new RecordMetadata(new TopicPartition(  record.topic(),
          record.partition()), 0, 0, 0, 0L, record.key().length, record.value().length);
       when(producer.send(record)).thenReturn(ConcurrentUtils.constantFuture(recordMetadata));

       if (msgPartitionMap.containsKey(partitionId)){
         msgPartitionMap.get(partitionId).add(logMessage);
         recordPartitionMap.get(partitionId).add(record);
         metadataPartitionMap.get(partitionId).add(recordMetadata);
       } else {
         msgPartitionMap.put(partitionId, new ArrayList<>());
         recordPartitionMap.put(partitionId, new ArrayList<>());
         metadataPartitionMap.put(partitionId, new ArrayList<>());
         msgPartitionMap.get(partitionId).add(logMessage);
         recordPartitionMap.get(partitionId).add(record);
         metadataPartitionMap.get(partitionId).add(recordMetadata);
       }
    }
  }

  List<PartitionInfo> sortedPartitions = new ArrayList<>(partitions);
  Collections.sort(sortedPartitions, new PartitionComparator());

  Map<Integer, Map<Integer, LoggingAuditHeaders>> mapOfHeadersMap = new HashMap<>();
  Map<Integer, List<ProducerRecord<byte[], byte[]>>> messageCollation = writer.messageCollation(partitions, "topicx", logMessages, mapOfHeadersMap);

  for(int partitionId = 0; partitionId < messageCollation.keySet().size(); partitionId++) {
    if (messageCollation.get(partitionId).size() == 0) {
      continue;
    }
    List<ProducerRecord<byte[], byte[]>> writerOutput = messageCollation.get(partitionId);

    // verify the message order is what is expected by calling messageCollation()
    List<ProducerRecord<byte[], byte[]>> expectedRecords = recordPartitionMap.get(partitionId);
    assertEquals(expectedRecords.size(), writerOutput.size());
    for(int j = 0; j < writerOutput.size(); j++){
      assertEquals(expectedRecords.get(j), writerOutput.get(j));
    }

    // verify the content of LogMessage and the content of ProducerRecord match
    List<LogMessage> originalData = msgPartitionMap.get(partitionId);
    assertEquals(originalData.size(), writerOutput.size());
    for (int j = 0; j < writerOutput.size(); j++) {
      assertTrue(Arrays.equals(originalData.get(j).getKey(), writerOutput.get(j).key()));
      assertTrue(Arrays.equals(originalData.get(j).getMessage(), writerOutput.get(j).value()));
    }

    // verify the RecordMetadata that corresponds to record send to certain partitions are put
    // together into a list and the order of the RecordMetadata is same as the original message order
    List<RecordMetadata> expectedRecordMetadata = metadataPartitionMap.get(partitionId);
    KafkaWritingTaskResult kafkaWritingTaskResult  = writer.getClusterThreadPool().submit(new
        KafkaWritingTask(producer, writerOutput, 0, sortedPartitions)).get();
    assertEquals(expectedRecordMetadata.size(), kafkaWritingTaskResult.getRecordMetadataList().size());
    for(int j = 0; j < expectedRecordMetadata.size(); j++){
      assertEquals(expectedRecordMetadata.get(j), kafkaWritingTaskResult.getRecordMetadataList().get(j));
    }
  }

  // validate if writes are throwing any error
  writer.writeLogMessages(logMessages);
  writer.close();
}
 

public Crc32ByteArrayPartitioner() {
  hashFunction = Hashing.crc32();
  random = new Random();
}