Python threading.Barrier() Examples

def wait(self, timeout=None):
        self._cond.acquire()
        try:
            if self._flag.acquire(False):
                self._flag.release()
            else:
                self._cond.wait(timeout)

            if self._flag.acquire(False):
                self._flag.release()
                return True
            return False
        finally:
            self._cond.release()

#
# Barrier
# 

def test_allocator_thread_local(self):
        def thread_body(self):
            new_pool = memory.MemoryPool()
            with cupy.cuda.using_allocator(new_pool.malloc):
                assert memory.get_allocator() == new_pool.malloc
                threading.Barrier(2)
                arr = cupy.zeros(128, dtype=cupy.int64)
                threading.Barrier(2)
                self.assertEqual(arr.data.mem.size, new_pool.used_bytes())
                threading.Barrier(2)
            assert memory.get_allocator() == self.pool.malloc

        with cupy.cuda.Device(0):
            t = threading.Thread(target=thread_body, args=(self,))
            t.daemon = True
            t.start()
            threading.Barrier(2)
            assert memory.get_allocator() == self.pool.malloc
            arr = cupy.ones(256, dtype=cupy.int64)
            threading.Barrier(2)
            self.assertEqual(arr.data.mem.size, self.pool.used_bytes())
            threading.Barrier(2)
            t.join() 

def run_multiple_writers(self, writer, num_writers=32):
        barrier = threading.Barrier(num_writers)

        def writer_proxy(thread_index, results):
            barrier.wait()
            # Attempts to operate on a table while locked should raise a RuntimeError
            try:
                writer(thread_index, results)
                results[thread_index] = 0
            except RuntimeError:
                results[thread_index] = 1

        results = run_threads(writer_proxy, num_writers)
        failures = sum(results)
        successes = num_writers - failures
        # Note: we would like to insist that #failures is > 0, but this is too
        # stochastic to guarantee for test purposes.
        self.assertGreaterEqual(failures, 0)
        self.assertGreater(successes, 0) 

def __init__(self, rank, world_size, init_ttp=False):
        self.world_size = world_size
        self.rank = rank
        self.reset_communication_stats()
        self._name = f"rank{rank}"

        with InProcessCommunicator.lock:
            if InProcessCommunicator.mailbox is None:
                InProcessCommunicator.mailbox = [
                    Queue() for _ in range(self.world_size)
                ]

                # This prevents one thread from running ahead of the others and doing
                # multiple puts that would show up in the get calls below
                InProcessCommunicator.barrier = threading.Barrier(self.world_size)

        # logging:
        level = logging.getLogger().level
        logging.getLogger().setLevel(logging.INFO)
        logging.info("==================")
        logging.info("InProcessCommunicator with rank %d" % self.rank)
        logging.info("==================")

        logging.info("World size = %d" % self.get_world_size())
        logging.getLogger().setLevel(level) 

def test_multiple_keys_multiple_times_different_thread(self):
        name1 = str(uuid.uuid4())
        name2 = str(uuid.uuid4())
        name3 = str(uuid.uuid4())
        barrier = threading.Barrier(5)

        threads = []

        lock1 = NamedReverseSemaphore(name1)
        with lock1:
            threads.insert(0, ExThread(target=thread_run, args=(name2, True, barrier), daemon=True))
            threads[0].start()
            threads.insert(0, ExThread(target=thread_run, args=(name2, True, barrier), daemon=True))
            threads[0].start()
            threads.insert(0, ExThread(target=thread_run, args=(name3, True, barrier), daemon=True))
            threads[0].start()
            threads.insert(0, ExThread(target=thread_run, args=(name3, True, barrier), daemon=True))
            threads[0].start()

            barrier.wait()
            # FIXME: hack to make sure we acquired the lock in the other thread
            time.sleep(0.2)

        for thread in threads:
            thread.join() 

def test_multiple_keys_different_thread(self):
        name1 = str(uuid.uuid4())
        name2 = str(uuid.uuid4())
        name3 = str(uuid.uuid4())
        barrier = threading.Barrier(3)

        threads = []

        lock1 = NamedReverseSemaphore(name1)
        with lock1:
            threads.insert(0, ExThread(target=thread_run, args=(name2, True, barrier), daemon=True))
            threads[0].start()
            threads.insert(0, ExThread(target=thread_run, args=(name3, True, barrier), daemon=True))
            threads[0].start()

            barrier.wait()
            # FIXME: hack to make sure we acquired the lock in the other thread
            time.sleep(0.2)

        for thread in threads:
            thread.join() 

def test_safe_logging():
    barrier = Barrier(2)
    counter = itertools.count()

    sink = NonSafeSink(1)
    logger.add(sink, format="{message}", catch=False)

    def threaded():
        barrier.wait()
        logger.info("___{}___", next(counter))

    threads = [Thread(target=threaded) for _ in range(2)]

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()

    logger.remove()

    assert sink.written in ("___0___\n___1___\n", "___1___\n___0___\n") 

def test_different_key_different_thread_wait(self):
        name1 = str(uuid.uuid4())
        name2 = str(uuid.uuid4())
        barrier = threading.Barrier(2)

        thread = ExThread(target=thread_run, args=(name2, True), daemon=True)
        thread.start()
        thread.join()

        lock1 = NamedReverseSemaphore(name1)
        with lock1:
            thread = ExThread(target=thread_run, args=(name2, True, barrier), daemon=True)
            thread.start()
            barrier.wait()
            # FIXME: hack to make sure we acquired the lock in the other thread
            time.sleep(0.2)
        thread.join() 

def test_safe_removing_while_logging(capsys):
    barrier = Barrier(2)
    counter = itertools.count()

    sink = NonSafeSink(1)
    i = logger.add(sink, format="{message}", catch=False)

    def thread_1():
        barrier.wait()
        logger.info("aaa{}bbb", next(counter))

    def thread_2():
        barrier.wait()
        time.sleep(0.5)
        logger.remove(i)
        logger.info("ccc{}ddd", next(counter))

    threads = [Thread(target=thread_1), Thread(target=thread_2)]

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()

    out, err = capsys.readouterr()
    assert out == ""
    assert err == ""
    assert sink.written == "aaa0bbb\n" 

def test_default_timeout(self):
        """
        Test the barrier's default timeout
        """
        barrier = self.Barrier(self.N, timeout=0.5)
        results = self.DummyList()
        self.run_threads(self._test_default_timeout_f, (barrier, results))
        self.assertEqual(len(results), barrier.parties) 

def test_lru_cache_threaded2(self):
        # Simultaneous call with the same arguments
        n, m = 5, 7
        start = threading.Barrier(n+1)
        pause = threading.Barrier(n+1)
        stop = threading.Barrier(n+1)
        @self.module.lru_cache(maxsize=m*n)
        def f(x):
            pause.wait(10)
            return 3 * x
        self.assertEqual(f.cache_info(), (0, 0, m*n, 0))
        def test():
            for i in range(m):
                start.wait(10)
                self.assertEqual(f(i), 3 * i)
                stop.wait(10)
        threads = [threading.Thread(target=test) for k in range(n)]
        with support.start_threads(threads):
            for i in range(m):
                start.wait(10)
                stop.reset()
                pause.wait(10)
                start.reset()
                stop.wait(10)
                pause.reset()
                self.assertEqual(f.cache_info(), (0, (i+1)*n, m*n, i+1)) 

def test_base_tuner_distribution(tmp_dir):
    num_workers = 3
    barrier = threading.Barrier(num_workers)

    def _test_base_tuner():
        def build_model(hp):
            return hp.Int('a', 1, 100)

        tuner = SimpleTuner(
            oracle=kt.oracles.RandomSearch(
                objective=kt.Objective('score', 'max'),
                max_trials=10),
            hypermodel=build_model,
            directory=tmp_dir)
        tuner.search()

        # Only worker makes it to this point, server runs until thread stops.
        assert dist_utils.has_chief_oracle()
        assert not dist_utils.is_chief_oracle()
        assert isinstance(tuner.oracle, kt.distribute.oracle_client.OracleClient)

        barrier.wait(60)

        # Model is just a score.
        scores = tuner.get_best_models(10)
        assert len(scores)
        assert scores == sorted(copy.copy(scores), reverse=True)

    mock_distribute.mock_distribute(_test_base_tuner, num_workers=num_workers) 

def _reset_barrier(self):
        self.barrier = threading.Barrier(self.num_participants, action=self._post_barrier_action) 

def __init__(self, sess, put_ops, batch_group_size, use_python32_barrier):
    self.sess = sess
    self.num_gets = 0
    self.put_ops = put_ops
    self.batch_group_size = batch_group_size
    self.done_event = threading.Event()
    if (use_python32_barrier and
        sys.version_info[0] == 3 and sys.version_info[1] >= 2):
      self.put_barrier = threading.Barrier(2)
    else:
      self.put_barrier = Barrier(2) 

def roll_numpy_batches(array, batch_size, shift_ratio):
  """Moves a proportion of batches from start to the end of the array.

  This function moves a proportion of batches, specified by `shift_ratio`, from
  the starts of the array to the end. The number of batches moved is rounded
  down to the nearest integer. For example,

  ```
  roll_numpy_batches([1, 2, 3, 4, 5, 6], 2, 0.34) == [3, 4, 5, 6, 1, 2]
  ```

  Args:
    array: A Numpy array whose first dimension is the batch dimension.
    batch_size: The batch size.
    shift_ratio: Proportion of batches to move from the start of the array to
      the end of the array.
  Returns:
    A new Numpy array, with a proportion of the batches at the start of `array`
    moved to the end.
  """
  num_items = array.shape[0]
  assert num_items % batch_size == 0
  num_batches = num_items // batch_size
  starting_batch = int(num_batches * shift_ratio)
  starting_item = starting_batch * batch_size
  return np.roll(array, -starting_item, axis=0)


# For Python 2.7 compatibility, we do not use threading.Barrier. 

def test_contextualize_thread(writer):
    logger.add(writer, format="{message} {extra[i]}")

    def task():
        logger.info("Processing")

    def worker(entry_barrier, exit_barrier, i):
        with logger.contextualize(i=i):
            entry_barrier.wait()
            task()
            exit_barrier.wait()

    entry_barrier = threading.Barrier(5)
    exit_barrier = threading.Barrier(5)

    threads = [
        threading.Thread(target=worker, args=(entry_barrier, exit_barrier, i)) for i in range(5)
    ]

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()

    assert sorted(writer.read().splitlines()) == ["Processing %d" % i for i in range(5)] 

def test_safe_writing_after_removing(capsys):
    barrier = Barrier(2)

    i = logger.add(NonSafeSink(1), format="{message}", catch=False)
    j = logger.add(NonSafeSink(1), format="{message}", catch=False)

    def write():
        barrier.wait()
        logger.info("Writing")

    def remove():
        barrier.wait()
        time.sleep(0.5)
        logger.remove(j)

    threads = [Thread(target=write), Thread(target=remove)]

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()

    logger.remove()

    out, err = capsys.readouterr()
    assert out == ""
    assert err == "" 

def test_abort_and_reset(self):
        """
        Test that a barrier can be reset after being broken.
        """
        results1 = self.DummyList()
        results2 = self.DummyList()
        results3 = self.DummyList()
        barrier2 = self.Barrier(self.N)

        self.run_threads(self._test_abort_and_reset_f,
                         (self.barrier, barrier2, results1, results2, results3))
        self.assertEqual(len(results1), 0)
        self.assertEqual(len(results2), self.N-1)
        self.assertEqual(len(results3), self.N) 

def test_single_thread(self):
        b = self.Barrier(1)
        b.wait()
        b.wait() 

def test_lru_cache_threaded2(self):
        # Simultaneous call with the same arguments
        n, m = 5, 7
        start = threading.Barrier(n+1)
        pause = threading.Barrier(n+1)
        stop = threading.Barrier(n+1)
        @self.module.lru_cache(maxsize=m*n)
        def f(x):
            pause.wait(10)
            return 3 * x
        self.assertEqual(f.cache_info(), (0, 0, m*n, 0))
        def test():
            for i in range(m):
                start.wait(10)
                self.assertEqual(f(i), 3 * i)
                stop.wait(10)
        threads = [threading.Thread(target=test) for k in range(n)]
        with support.start_threads(threads):
            for i in range(m):
                start.wait(10)
                stop.reset()
                pause.wait(10)
                start.reset()
                stop.wait(10)
                pause.reset()
                self.assertEqual(f.cache_info(), (0, (i+1)*n, m*n, i+1)) 

def __init__(self, socket_address, args, interactive):
        self.socket_address = socket_address
        self.input = args
        self.interactive = interactive
        self._output = None
        self.event_loop = asyncio.new_event_loop()
        self.thread = threading.Thread(target=self._run_event_loop)
        self.barrier = threading.Barrier(2) 

def __init__(self, socket_address, libdir, options, cache, output, secrets=None):
        self.socket_address = socket_address
        self.libdir = libdir
        self.cache = cache
        self.output = output
        self.options = options or {}
        self.secrets = secrets or {}
        self.barrier = threading.Barrier(2)
        self.event_loop = None
        self.thread = None 

def __init__(self, socket_address):
        self.socket_address = socket_address
        self.devs = []
        self.ctl = loop.LoopControl()
        self.event_loop = asyncio.new_event_loop()
        self.thread = threading.Thread(target=self._run_event_loop)
        self.barrier = threading.Barrier(2) 

def fileServer(directory):
    with netns():
        # This is leaked until the program exits, but inaccessible after the with
        # due to the network namespace.
        barrier = threading.Barrier(2)
        thread = threading.Thread(target=runFileServer, args=(barrier, directory))
        thread.daemon = True
        thread.start()
        barrier.wait()
        yield 

def run(self):
        """This is the run method from threading.Thread"""
        #TODO threading.Barrier to sync with ribbapi
        #print("Starting")

        self.started = time.time()
        self._running = True
        self.animate()

    # def start(self): 

def __init__(self, tellos: list):
		"""Initialize a TelloSwarm instance

		Arguments:
			tellos: list of [Tello][tello] instances
		"""
		self.tellos = tellos
		self.barrier = Barrier(len(tellos))
		self.funcBarrier = Barrier(len(tellos) + 1)
		self.funcQueues = [Queue() for tello in tellos]

		def worker(i):
			queue = self.funcQueues[i]
			tello = self.tellos[i]

			while True:
				func = queue.get()
				self.funcBarrier.wait()
				func(i, tello)
				self.funcBarrier.wait()

		self.threads = []
		for i, tello in enumerate(tellos):
			thread = Thread(target=worker, daemon=True, args=(i,))
			thread.start()
			self.threads.append(thread) 

def run_failing_reader(self, writer, reader, num_readers=32):
        """
        Runs a test in which a single writer acceses some tables
        and a bunch of other threads try to read the data.
        """
        barrier = threading.Barrier(num_readers + 1)

        def writer_proxy():
            barrier.wait()
            writer()

        def reader_proxy(thread_index, results):
            barrier.wait()
            # Attempts to operate on a table while locked should raise a RuntimeError
            try:
                reader(thread_index, results)
                results[thread_index] = 0
            except RuntimeError:
                results[thread_index] = 1

        writer_thread = threading.Thread(target=writer_proxy)
        writer_thread.start()
        results = run_threads(reader_proxy, num_readers)
        writer_thread.join()

        failures = sum(results)
        successes = num_readers - failures
        # Note: we would like to insist that #failures is > 0, but this is too
        # stochastic to guarantee for test purposes.
        self.assertGreaterEqual(failures, 0)
        self.assertGreater(successes, 0) 

def __init__(self, sess, put_ops, batch_group_size):
    self.sess = sess
    self.num_gets = 0
    self.put_ops = put_ops
    self.batch_group_size = batch_group_size
    self.done_event = threading.Event()
    if (FLAGS.use_python32_barrier and
        sys.version_info[0] == 3 and sys.version_info[1] >= 2):
      self.put_barrier = threading.Barrier(2)
    else:
      self.put_barrier = Barrier(2) 

def log_fn(log):
  print(log)
  if FLAGS.flush_stdout:
    sys.stdout.flush()


# For Python 2.7 compatibility, we do not use threading.Barrier. 

def _get_barrier(self, parties=2):
        """Lazy-instantiate a barrier"""
        with self._barrier_lock:
            if not self._barrier:
                self._barrier = threading.Barrier(parties, timeout=self._max_wait)
            return self._barrier