Python mmh3.hash64() Examples

def copy_bitmap(self, shm, num, size, bitmap, payload, payload_size, effector_mode=False):
        new_hash = mmh3.hash64(bitmap)
        if not (self.crashed or self.kasan or self.timeout):
            if new_hash in self.lookup.non_finding:
                if effector_mode:
                    shm.seek(size * num)
                    shm.write(bitmap)
                    return True
                else:
                    shm.seek((size * num) + len(bitmap))
                    return False

        if not (self.crashed or self.kasan or self.timeout) and not self.check_for_unseen_bits(bitmap):
            self.lookup.non_finding[new_hash] = None
            return False
        if not (self.timeout):
            bitmap = self.verifiy_input(payload, bitmap, payload_size)
            shm.seek(size * num)
            shm.write(bitmap)
        self.lookup.non_finding[new_hash] = None
        return True 

def sim_shi4_mm3_text(text):
    text = HTML_COMMENT_RE.sub(" ", text)
    text = SCRIPT_RE.sub(" ", text)
    text = STYLE_RE.sub(" ", text)
    text = HTML_TAGS_RE.sub(" ", text)
    text = HTML_ENTITIES_RE.sub(" ", text)
    text = NONTEXT_RE.sub(" ", text)  # replace all runs of spaces and punctuation

    i1, i2 = itertools.tee(WORD_RE.finditer(text))
    for _ in xrange(3):  # 4 words per shingle
        next(i2, None)

    hash64 = mmh3.hash64
    # NB: `array` of i64 & ui64 is Python 3.3+
    mm = [
        hash64(text[m1.start() : m2.end()])[0] & 0xFFFFFFFFFFFFFFFF
        for m1, m2 in itertools.izip(i1, i2)
    ]
    r = simhash.compute(mm)
    return r - 0x10000000000000000 if r > 0x7FFFFFFFFFFFFFFF else r 

def debug_execution(config, execs, qemu_verbose=False, notifiers=True):
    log_info("Starting...")

    zero_hash = mmh3.hash64(("\x00" * config.config_values['BITMAP_SHM_SIZE']))
    q = qemu(1337, config, debug_mode=True, notifiers=notifiers)
    q.start(verbose=qemu_verbose)
    q.set_payload(open(config.argument_values["payload"][0]).read())
    # try:
    start = time.time()
    for i in range(execs):
        if i % 3 == 0:
            q.set_payload(open(config.argument_values["payload"][0]).read())
        # time.sleep(0.01 * randint(0, 9))
        print("+----------------------------------------------+")
        # a = str(q.send_payload())
        # hexdump(a)
        current_hash = q.send_payload().hash()
        if zero_hash == current_hash:
            print("Hash: " + str(
                current_hash) + common.color.WARNING + " (WARNING: Zero hash found!)" + common.color.ENDC)
        else:
            print("Hash: " + str(current_hash))
    end = time.time()
    print("Performance: " + str(execs / (end - start)) + "t/s")
    # except:
    #    print("EXC")

    q.__del__()
    try:
        for i in range(512):
            if os.path.exists("/tmp/kAFL_printf.txt." + str(i)):
                os.remove("/tmp/kAFL_printf.txt." + str(i))
            else:
                break
    except:
        pass
    os.system("stty sane")
    return 0 

def murmur3_64bit(obj):
    """
    Use murmur3_64bit for 64 bit hash by passing this method:
    hasher=DeepHash.murmur3_64bit
    """
    if isinstance(obj, str):
        obj = obj.encode('utf-8')
    # This version of murmur3 returns two 64bit integers.
    return mmh3.hash64(obj, MURMUR_SEED)[0] 

def __req_effector_tag_handler(self, request):
        #log_mapserver("New Effector Map (" + str(len(request.data)) + ")")
        self.effector_initial_bitmap = mmh3.hash64(request.data)
        for i in range(self.config.config_values['PAYLOAD_SHM_SIZE']):
            self.effector_map.append(False) 

def hash_string(string: str) -> int:
    """Create a hash for a given string. 
    Hashes created by this functions will be used everywhere by
    SyferText to represent tokens.
    """

    key = mmh3.hash64(string, signed=False, seed=1)[0]

    return key 

def sim_shi4_mm3_layout(text):
    text = SPACE_RE.sub(" ", text)  # replace all runs of spaces

    i1, i2 = itertools.tee(WORD_RE.finditer(text))
    for _ in xrange(3):  # 4 words per shingle
        next(i2, None)

    hash64 = mmh3.hash64
    # NB: `array` of i64 & ui64 is Python 3.3+
    mm = [
        hash64(text[m1.start() : m2.end()])[0] & 0xFFFFFFFFFFFFFFFF
        for m1, m2 in itertools.izip(i1, i2)
    ]
    r = simhash.compute(mm)
    return r - 0x10000000000000000 if r > 0x7FFFFFFFFFFFFFFF else r 

def sim_shi4_mm3(text):
    # NB: It makes quite little sense to use both 64bit numbers to compare
    # hashes as pairwise Hamming distance using high 64bit is highly correlated
    # with the distance computed using low 64bit. It's actually expected, but
    # it means, that summing these distances is not linear and should be avoided.
    # -- https://gist.github.com/darkk/e2b2762c4fe053a3cf8a299520f0490e
    i1, i2 = itertools.tee(WORD_RE.finditer(text))
    for _ in xrange(3): # 4 words per shingle
        next(i2, None)
    mm = [mmh3.hash64(text[m1.start():m2.end()]) for m1, m2 in itertools.izip(i1, i2)]
    return (simhash.compute([_[0] & 0xffffffffffffffff for _ in mm]),
            simhash.compute([_[1] & 0xffffffffffffffff for _ in mm])) 

def main():
    simhash.Simhash(unicode(TEXT, 'utf-8'), reg=RE_WORD, hashfunc=lambda x: mmh3.hash64(x)[0]) 

def triple_hash(self, item):
        '''Returns a triple i, val, aug hashed values, where i is bucketbits,
        val is the position of the leading one in a 64-bit integer, and aug is the bits
        to go in the subbuckets'''

        y, h2 = mmh3.hash64(str(item).encode())
        val = 64 + 1 - int(np.uint64(y)).bit_length()
        val = min(val, 2**self.bucketsize)

        h2prime = int(np.uint64(h2))
        i = h2prime >> self._bucketbit_shift
        aug = h2prime & self._bbit_mask

        return (i, val, aug) 

def benchmark(config):
    log_info("Starting...")

    q = qemu(1337, config, debug_mode=False)
    q.start(verbose=False)
    q.set_payload(open(config.argument_values["payload"][0]).read())
    print(mmh3.hash64(q.send_payload()))
    try:
        while True:
            start = time.time()
            execs = 0
            while (time.time()-start < REFRESH):
                q.set_payload(open(config.argument_values["payload"][0]).read())
                q.send_payload()
                execs += 1
            end = time.time()
            stdout.write(common.color.FLUSH_LINE + "Performance: " + str(execs/(end - start)) + "t/s")
            stdout.flush()
    except:
        print("\nExit")
  
    q.__del__()
    try:
        for i in range(512):
            if os.path.exists("/tmp/kAFL_printf.txt." + str(i)):
                os.remove("/tmp/kAFL_printf.txt." + str(i))
            else:
                break
    except:
        pass
    return 0 

def debug_execution(config, execs, qemu_verbose=False, notifiers=True):
    log_info("Starting...")

    zero_hash = mmh3.hash64(("\xFF" * config.config_values['BITMAP_SHM_SIZE']))
    q = qemu(1337, config, debug_mode=True, notifiers=notifiers)
    q.start(verbose=qemu_verbose)
    q.set_payload(open(config.argument_values["payload"][0]).read())
    start = time.time()
    for i in range(execs):
        print("+----------------------------------------------+")
        current_hash = mmh3.hash64(q.send_payload())
        if zero_hash == current_hash:
            print("Hash: " + str(current_hash) + common.color.WARNING + " (WARNING: Zero hash found!)" + common.color.ENDC)
        else:
            print("Hash: " + str(current_hash))
    end = time.time()
    print("Performance: " + str(execs/(end - start)) + "t/s")
  
    q.__del__()
    try:
        for i in range(512):
            if os.path.exists("/tmp/kAFL_printf.txt." + str(i)):
                os.remove("/tmp/kAFL_printf.txt." + str(i))
            else:
                break
    except:
        pass
    os.system("stty sane")
    return 0 

def __req_effector_tag_handler(self, request):
        log_mapserver("New Effector Map (" + str(len(request.data)) + ")")
        self.effector_initial_bitmap = mmh3.hash64(request.data)
        for i in range(self.config.config_values['PAYLOAD_SHM_SIZE']):
            self.effector_map.append(False) 

def __respond_bitmap_hash_req(self, response):
        self.q.set_payload(response.data)
        while True:
            try:
                bitmap = self.q.send_payload()
                break
            except:
                log_slave("__respond_bitmap_hash_req failed...", self.slave_id)
                log_slave("%s"%traceback.format_exc(), self.slave_id)
                self.__restart_vm()
        send_msg(KAFL_TAG_REQ_BITMAP_HASH, mmh3.hash64(bitmap), self.comm.to_master_from_slave_queue, source=self.slave_id) 

def __result_tag_handler(self, request):
        self.comm.slave_locks_B[request.source].acquire()
        results = request.data
        payloads = []
        bitmaps = []
        payload_shm = self.comm.get_mapserver_payload_shm(request.source)
        bitmap_shm = self.comm.get_bitmap_shm(request.source)

        for result in results:
            if result.new_bits:
                bitmap_shm.seek(result.pos * self.comm.get_bitmap_shm_size())
                payload_shm.seek(result.pos * self.comm.get_mapserver_payload_shm_size())
                length = payload_shm.read(4)
                data_len = (ord(length[3]) << 24) + (ord(length[2]) << 16) + (ord(length[1]) << 8) + \
                           (ord(length[0]))
                payloads.append(payload_shm.read(data_len))
                bitmaps.append(bitmap_shm.read(self.comm.get_bitmap_shm_size()))
            else:
                payloads.append(None)
                bitmaps.append(None)
                #log_mapserver("[MAPS]\t\ SKIP")
        self.comm.slave_locks_A[request.source].release()
        for i in range(len(results)):
            if results[i].reloaded:
                self.abortion_counter += 1

            if results[i].new_bits:
                if results[i].timeout:
                    self.mapserver_state_obj.timeout += 1
                new_hash = mmh3.hash64(bitmaps[i])

                self.__check_hash(new_hash, bitmaps[i], payloads[i], results[i].crash, results[i].timeout, results[i].kasan, results[i].slave_id, results[i].reloaded, results[i].performance, results[i].qid, results[i].pos)
                self.last_hash = new_hash
                self.round_counter += 1
                if self.effector_initial_bitmap:
                    if self.effector_initial_bitmap != new_hash:
                        for j in results[i].affected_bytes:
                            if not self.effector_map[j]:
                                self.effector_map[j] = True
            else:
                self.round_counter += 1

        # TODO: Replace const value by performance*(1/50)s
        if self.abortion_counter >= self.abortion_threshold:
            if not self.abortion_alredy_sent:
                log_mapserver("Stage abortion limit (" + str(self.abortion_threshold) + ") reached!")
                send_msg(KAFL_TAG_ABORT_REQ, self.mapserver_state_obj, self.comm.to_master_queue)
                self.abortion_alredy_sent = True
                self.comm.stage_abortion_notifier.value = True 

def __respond_verification(self, response):

        jobs = response.data[0]
        methods = response.data[1]

        results = []
        i = 0
        self.comm.slave_locks_A[self.slave_id].acquire()

        while True:
                payload, payload_shm_size = self.q.copy_master_payload(self.comm.get_master_payload_shm(self.slave_id), i, self.comm.get_master_payload_shm_size())

                payload_content_len_init = struct.unpack("I", payload[0:4])[0]

                payload_content_len = perform_trim(payload_content_len_init, self.q.send_payload, self.q.modify_payload_size, self.error_handler)

                if payload_content_len_init != payload_content_len:
                    log_slave("TRIM: " + "{0:.2f}".format(((payload_content_len*1.0)/(payload_content_len_init*1.0))*100.0) + "% (" + str(payload_content_len) + "/" + str(payload_content_len_init) + ")", self.slave_id)

                patches = jobs[0]
                if len(patches) > 0:
                    log_slave("Got payload to fix with size: %d and patches %s"%( payload_content_len, patches), self.slave_id )

                    if len(patches):
                        log_redq("Slave "+str(self.slave_id)+" Orig  Payload: " + repr(payload[4:4+payload_content_len]))
                        hash = HashFixer(self.q, self.redqueen_state)
                        new_payload = hash.try_fix_data(payload[4:4+payload_content_len])

                        if new_payload:
                            log_redq("Slave "+str(self.slave_id)+"Fixed Payload: " + repr("".join(map(chr,new_payload))))
                            payload = payload[:4]+"".join(map(chr,new_payload))
                            self.q.set_payload(new_payload)

                start_time = time.time()
                bitmap = self.q.send_payload(apply_patches=False)
                performance = time.time() - start_time
                log_slave("performance: " + str(1.0/performance) + " -> " + str(performance), self.slave_id)
                break
                    

        if not bitmap:
            log_slave("SHM ERROR....", self.slave_id)

        new_bits = self.q.copy_bitmap(self.comm.get_bitmap_shm(self.slave_id), i, self.comm.get_bitmap_shm_size(),
                                      bitmap, payload, payload_shm_size, effector_mode_hash=None, apply_patches = False)
        if new_bits:
            self.q.copy_mapserver_payload(self.comm.get_mapserver_payload_shm(self.slave_id), i, self.comm.get_mapserver_payload_shm_size())
        
        results.append(FuzzingResult(i, self.q.crashed, self.q.timeout, self.q.kasan, jobs[i], self.slave_id, performance, methods[i], mmh3.hash64(bitmap), reloaded=(self.q.timeout or self.q.crashed or self.q.kasan), new_bits=new_bits, qid=self.slave_id))

        self.comm.slave_locks_B[self.slave_id].release()
        send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)