Python contextlib.closing() Examples

def test_copy_data_from(self):
        try:
            tmpdir = tempfile.mkdtemp()

            # create new database
            with testing.postgresql.Postgresql(base_dir=tmpdir) as pgsql:
                conn = pg8000.connect(**pgsql.dsn())
                with closing(conn.cursor()) as cursor:
                    cursor.execute("CREATE TABLE hello(id int, value varchar(256))")
                    cursor.execute("INSERT INTO hello values(1, 'hello'), (2, 'ciao')")
                conn.commit()
                conn.close()

            # create another database from first one
            data_dir = os.path.join(tmpdir, 'data')
            with testing.postgresql.Postgresql(copy_data_from=data_dir) as pgsql:
                conn = pg8000.connect(**pgsql.dsn())
                with closing(conn.cursor()) as cursor:
                    cursor.execute('SELECT * FROM hello ORDER BY id')
                    self.assertEqual(cursor.fetchall(), ([1, 'hello'], [2, 'ciao']))
                conn.close()
        finally:
            rmtree(tmpdir) 

def test_PostgresqlFactory_with_initialized_handler(self):
        def handler(pgsql):
            conn = pg8000.connect(**pgsql.dsn())
            with closing(conn.cursor()) as cursor:
                cursor.execute("CREATE TABLE hello(id int, value varchar(256))")
                cursor.execute("INSERT INTO hello values(1, 'hello'), (2, 'ciao')")
            conn.commit()
            conn.close()

        Postgresql = testing.postgresql.PostgresqlFactory(cache_initialized_db=True,
                                                          on_initialized=handler)
        try:
            with Postgresql() as pgsql:
                conn = pg8000.connect(**pgsql.dsn())
                with closing(conn.cursor()) as cursor:
                    cursor.execute('SELECT * FROM hello ORDER BY id')
                    self.assertEqual(cursor.fetchall(), ([1, 'hello'], [2, 'ciao']))
                conn.close()
        finally:
            Postgresql.clear_cache() 

def test_handler_callback_file_object(self):
        """The handler callback receives the same fd object it passed in."""
        server_sock, port = bind_unused_port()
        fds = []

        def handle_connection(fd, events):
            fds.append(fd)
            conn, addr = server_sock.accept()
            conn.close()
            self.stop()
        self.io_loop.add_handler(server_sock, handle_connection, IOLoop.READ)
        with contextlib.closing(socket.socket()) as client_sock:
            client_sock.connect(('127.0.0.1', port))
            self.wait()
        self.io_loop.remove_handler(server_sock)
        self.io_loop.add_handler(server_sock.fileno(), handle_connection,
                                 IOLoop.READ)
        with contextlib.closing(socket.socket()) as client_sock:
            client_sock.connect(('127.0.0.1', port))
            self.wait()
        self.assertIs(fds[0], server_sock)
        self.assertEqual(fds[1], server_sock.fileno())
        self.io_loop.remove_handler(server_sock.fileno())
        server_sock.close() 

def _AxisInfo(fontfile):
  """Gets variable axes info.

  Args:
    fontfile: Font file to look at for variation info

  Returns:
    Variable axes info
  """
  with contextlib.closing(ttLib.TTFont(fontfile)) as font:
    if 'fvar' not in font:
      return frozenset()
    else:
      fvar = font['fvar']
      axis_info = [
          (a.axisTag, a.minValue, a.maxValue) for a in fvar.axes
      ]
      return tuple(sorted(axis_info)) 

def main(argv):
  if len(argv) < 2:
    sys.exit(('{}\n'
              'usage:\n'
              '       gftools varfont-info fontfile.ttf').format(__doc__))
  for filename in argv[1:]:
    with contextlib.closing(ttLib.TTFont(filename)) as ttf:
      print(filename)
      if 'fvar' not in ttf:
        print("This font file lacks an 'fvar' table.")
      else:
        fvar = ttf['fvar']
        print(' axes')
        axes = [(a.axisTag, a.minValue, a.defaultValue, a.maxValue)
                for a in fvar.axes]
        for tag, minv, defv, maxv in axes:
          print("  '%s' %d-%d, default %d" % (tag, minv, maxv, defv))

        if fvar.instances:
          print(' named-instances')
          for inst in fvar.instances:
            print('   %s %s' % (_ResolveName(ttf, inst.postscriptNameID),
                                inst.coordinates)) 

def test_max_clients(self):
        AsyncHTTPClient.configure(SimpleAsyncHTTPClient)
        with closing(AsyncHTTPClient(
                self.io_loop, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 10)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=11, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 11)

        # Now configure max_clients statically and try overriding it
        # with each way max_clients can be passed
        AsyncHTTPClient.configure(SimpleAsyncHTTPClient, max_clients=12)
        with closing(AsyncHTTPClient(
                self.io_loop, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 12)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=13, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 13)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=14, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 14) 

def _CheckFontInternalValues(path):
  """Validates fonts internal metadata matches METADATA.pb values.

  In particular, checks 'OS/2' {usWeightClass, fsSelection, fsType} and 'name'
  {fullName, postScriptName} values.

  Args:
    path: A directory containing a METADATA.pb file.
  Returns:
    A list of ResultMessageTuple, one per validation performed.
  """
  results = []
  metadata = fonts.Metadata(path)
  name = metadata.name

  for font in metadata.fonts:
    font_file = font.filename
    with contextlib.closing(ttLib.TTFont(os.path.join(path, font_file))) as ttf:
      results.extend(_CheckFontOS2Values(path, font, ttf))
      results.extend(_CheckFontNameValues(path, name, font, ttf))
      results.extend(_CheckLSB0ForEmptyGlyphs(path, font, ttf))

  return results 

def test_multi_line_headers(self):
        # Multi-line http headers are rare but rfc-allowed
        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
        sock, port = bind_unused_port()
        with closing(sock):
            def write_response(stream, request_data):
                if b"HTTP/1." not in request_data:
                    self.skipTest("requires HTTP/1.x")
                stream.write(b"""\
HTTP/1.1 200 OK
X-XSS-Protection: 1;
\tmode=block

""".replace(b"\n", b"\r\n"), callback=stream.close)

            def accept_callback(conn, address):
                stream = IOStream(conn, io_loop=self.io_loop)
                stream.read_until(b"\r\n\r\n",
                                  functools.partial(write_response, stream))
            netutil.add_accept_handler(sock, accept_callback, self.io_loop)
            self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop)
            resp = self.wait()
            resp.rethrow()
            self.assertEqual(resp.headers['X-XSS-Protection'], "1; mode=block")
            self.io_loop.remove_handler(sock.fileno()) 

def test_add_callback_while_closing(self):
        # Issue #635: add_callback() should raise a clean exception
        # if called while another thread is closing the IOLoop.
        closing = threading.Event()

        def target():
            other_ioloop.add_callback(other_ioloop.stop)
            other_ioloop.start()
            closing.set()
            other_ioloop.close(all_fds=True)
        other_ioloop = IOLoop()
        thread = threading.Thread(target=target)
        thread.start()
        closing.wait()
        for i in range(1000):
            try:
                other_ioloop.add_callback(lambda: None)
            except RuntimeError as e:
                self.assertEqual("IOLoop is closing", str(e))
                break 

def test_handler_callback_file_object(self):
        """The handler callback receives the same fd object it passed in."""
        server_sock, port = bind_unused_port()
        fds = []

        def handle_connection(fd, events):
            fds.append(fd)
            conn, addr = server_sock.accept()
            conn.close()
            self.stop()
        self.io_loop.add_handler(server_sock, handle_connection, IOLoop.READ)
        with contextlib.closing(socket.socket()) as client_sock:
            client_sock.connect(('127.0.0.1', port))
            self.wait()
        self.io_loop.remove_handler(server_sock)
        self.io_loop.add_handler(server_sock.fileno(), handle_connection,
                                 IOLoop.READ)
        with contextlib.closing(socket.socket()) as client_sock:
            client_sock.connect(('127.0.0.1', port))
            self.wait()
        self.assertIs(fds[0], server_sock)
        self.assertEqual(fds[1], server_sock.fileno())
        self.io_loop.remove_handler(server_sock.fileno())
        server_sock.close() 

def _add_device(self, device):
        if self._check_port:
            # check that the device port is opened
            with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
                sock.settimeout(_DEFAULT_TIMEOUT)
                try:
                    res = sock.connect_ex((device.ip_addr, device.port))
                except (socket.error, OSError):
                    self.logger.debug("{} is unreachable".format(device.ip_addr))
                    return
                if res != 0:
                    self.logger.debug("{}:{} is closed".format(device.ip_addr, device.port))
                    return
        # add this device to the "discovered" devices
        f = self._thread_loop.run_async(self._do_add_device, device)
        try:
            f.result_or_cancel(timeout=_DEFAULT_TIMEOUT)
        except concurrent.futures.TimeoutError:
            self.logger.error("raw discovery timedout for {}:{}".format(
                device.ip_addr, device.port)) 

def process_module(self, node):
        """Process the module."""
        if os.path.basename(os.path.splitext(node.file)[0]) == '__init__':
            return
        max_lineno = 1
        with contextlib.closing(node.stream()) as stream:
            for (lineno, line) in enumerate(stream):
                if lineno == 1 and line.startswith(b'#!'):
                    max_lineno += 1
                    continue
                elif line.startswith(b'# vim:'):
                    if lineno > max_lineno:
                        self.add_message('modeline-position', line=lineno)
                    if (line.rstrip() != b'# vim: ft=python '
                                         b'fileencoding=utf-8 sts=4 sw=4 et:'):
                        self.add_message('invalid-modeline', line=lineno)
                    break
            else:
                self.add_message('modeline-missing', line=1) 

def open(self, mode: QIODevice.OpenMode) -> contextlib.closing:
        """Open the underlying device and ensure opening succeeded.

        Raises OSError if opening failed.

        Args:
            mode: QIODevice::OpenMode flags.

        Return:
            A contextlib.closing() object so this can be used as
            contextmanager.
        """
        ok = self.dev.open(mode)
        if not ok:
            raise QtOSError(self.dev)
        return contextlib.closing(self) 

def test_max_clients(self):
        AsyncHTTPClient.configure(SimpleAsyncHTTPClient)
        with closing(AsyncHTTPClient(
                self.io_loop, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 10)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=11, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 11)

        # Now configure max_clients statically and try overriding it
        # with each way max_clients can be passed
        AsyncHTTPClient.configure(SimpleAsyncHTTPClient, max_clients=12)
        with closing(AsyncHTTPClient(
                self.io_loop, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 12)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=13, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 13)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=14, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 14) 

def read(self, filename):
        with closing(open(filename, 'r')) as r:
            csvfile = csv.reader(r)
            csvfile.next() # skip header
            for row in csvfile:
                b_low = float(row[0])
                b_high = float(row[1])
                h_low = float(row[2])
                h_high = float(row[3])                

                if len(row) > 5:
                    count = int(row[4])
                    unique = int(row[5])
                else:
                    # older format with only count/unique stored
                    count = int( float(row[4]) * 10.0 )
                    unique = 10

                self.import_sector(b_low, b_high, h_low, h_high, count, unique) 

def write(self, filename):
        with closing(open(filename + '.new', 'w')) as w:
            c = csv.writer(w)
            c.writerow(['bearing_start','bearing_end','bin_start','bin_end','samples','unique'])
            for b_low,b_high,histo in self.values():
                # make sure we write at least one value per sector,
                # it makes things a little easier when plotting
                first = True
                for h_low,h_high,count,unique in histo.values():
                    if unique or first:
                        c.writerow(['%f' % b_low,
                                    '%f' % b_high,
                                    '%f' % h_low,
                                    '%f' % h_high,
                                    '%d' % count,
                                    '%d' % unique])
                        first = False
        os.rename(filename + '.new', filename) 

def ping_sim_tcp_server(server_port, ping_ip, client, address):
    _, client_port = address
    logger = logging.getLogger('threaded.ping.tcp-server({} -> {})'.format(server_port,
                                                                           client_port))
    logger.debug('connection accepted - client at tcp://{}:{}'.format(*address))
    ping_out = ping_output.replace("10.0.2.15", ping_ip)
    ping_lines = ping_out.splitlines(True)
    with closing(client):
        for ping_line in ping_lines:
            data = ping_line.encode(encoding='utf-8')
            try:
                client.sendall(data)
            except socket.error:  # client is gone
                break
            time.sleep(1)  # simulate delay between ping lines
    logger.info('Connection closed') 

def get_max_range(baseurl):
    with closing(urlopen(baseurl + '/data/receiver.json', None, 5.0)) as f:
        receiver = json.load(f)

        if not (receiver.has_key('lat') and receiver.has_key('lon')):
            return None

        rlat = receiver['lat']
        rlon = receiver['lon']

        maxrange = None
        with closing(urlopen(baseurl + '/data/aircraft.json', None, 5.0)) as f:
            aircraft = json.load(f)
            for ac in aircraft['aircraft']:
                if ac.has_key('seen_pos') and ac['seen_pos'] < 300:
                    alat = ac['lat']
                    alon = ac['lon']
                    distance = greatcircle(rlat, rlon, alat, alon)
                    if maxrange is None or distance > maxrange:
                        maxrange = distance
                        
        return maxrange 

def tcp_connection(address, moler_conn):
    """Forwarder reading from tcp network transport layer"""
    logger = logging.getLogger('threaded.tcp-connection({}:{})'.format(*address))
    logger.debug('... connecting to tcp://{}:{}'.format(*address))
    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    client_socket.connect(address)

    with closing(client_socket):
        while True:
            data = client_socket.recv(128)
            if data:
                logger.debug('<<< {!r}'.format(data))
                # Forward received data into Moler's connection
                moler_conn.data_received(data)
                yield data
            else:
                logger.debug("... closed")
                break 

def list(uri):
  # Create instance of Engine 
  engine = Engine(uri)
  files = []
  # Ensure we'll close engine on exception 
  with closing(engine):
    # Start engine 
    engine.start()
    # Wait until files received 
    while not files and not xbmc.abortRequested:
        # Will list only video files in torrent
        files = engine.list(media_types=[MediaType.VIDEO])
        # Check if there is loading torrent error and raise exception 
        engine.check_torrent_error()
        xbmc.sleep(200)
  return files 

def tcp_connection(address):
    """Generator reading from tcp network transport layer"""
    logger = logging.getLogger('threaded.tcp-connection')
    logger.debug('... connecting to tcp://{}:{}'.format(*address))
    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    client_socket.connect(address)

    with closing(client_socket):
        while True:
            data = client_socket.recv(128)
            if data:
                logger.debug('<<< {!r}'.format(data))
                yield data
            else:
                logger.debug("... closed")
                break 

def tcp_connection(address, moler_conn):
    """Forwarder reading from tcp network transport layer"""
    logger = logging.getLogger('threaded.tcp-connection({}:{})'.format(*address))
    logger.debug('... connecting to tcp://{}:{}'.format(*address))
    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    client_socket.connect(address)

    with closing(client_socket):
        while True:
            data = client_socket.recv(128)
            if data:
                logger.debug('<<< {!r}'.format(data))
                # Forward received data into Moler's connection
                moler_conn.data_received(data)
                yield data
            else:
                logger.debug("... closed")
                break 

def test_multi_line_headers(self):
        # Multi-line http headers are rare but rfc-allowed
        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
        sock, port = bind_unused_port()
        with closing(sock):
            def write_response(stream, request_data):
                if b"HTTP/1." not in request_data:
                    self.skipTest("requires HTTP/1.x")
                stream.write(b"""\
HTTP/1.1 200 OK
X-XSS-Protection: 1;
\tmode=block

""".replace(b"\n", b"\r\n"), callback=stream.close)

            def accept_callback(conn, address):
                stream = IOStream(conn, io_loop=self.io_loop)
                stream.read_until(b"\r\n\r\n",
                                  functools.partial(write_response, stream))
            netutil.add_accept_handler(sock, accept_callback, self.io_loop)
            self.http_client.fetch("http://127.0.0.1:%d/" % port, self.stop)
            resp = self.wait()
            resp.rethrow()
            self.assertEqual(resp.headers['X-XSS-Protection'], "1; mode=block")
            self.io_loop.remove_handler(sock.fileno()) 

def test_add_callback_while_closing(self):
        # Issue #635: add_callback() should raise a clean exception
        # if called while another thread is closing the IOLoop.
        closing = threading.Event()

        def target():
            other_ioloop.add_callback(other_ioloop.stop)
            other_ioloop.start()
            closing.set()
            other_ioloop.close(all_fds=True)
        other_ioloop = IOLoop()
        thread = threading.Thread(target=target)
        thread.start()
        closing.wait()
        for i in range(1000):
            try:
                other_ioloop.add_callback(lambda: None)
            except RuntimeError as e:
                self.assertEqual("IOLoop is closing", str(e))
                break 

def shelf(filename, ttl=0):
    import shelve
    filename = os.path.join(CACHE_DIR, filename)
    with LOCKS.get(filename, threading.RLock()):
        with closing(shelve.open(filename, writeback=True)) as d:
            import time
            if not d:
                d.update({
                    "created_at": time.time(),
                    "data": {},
                })
            elif ttl > 0 and (time.time() - d["created_at"]) > ttl:
                d["data"] = {}
            yield d["data"] 

def CodepointsInFont(font_filename):
  """Returns the set of codepoints present in the font file specified.

  Args:
    font_filename: The name of a font file.
  Returns:
    A set of integers, each representing a codepoint present in font.
  """

  font_cps = set()
  with contextlib.closing(ttLib.TTFont(font_filename)) as font:
    for t in UnicodeCmapTables(font):
      font_cps.update(t.cmap.keys())

  return font_cps 

def ExtractName(font_or_file, name_id, default):
  """Extracts a name table field (first value if many) from a font.

  Args:
    font_or_file: path to a font file or a TTFont.
    name_id: the ID of the name desired. Use NAME_* constant.
    default: result if no value is present.
  Returns:
    The value of the first entry for name_id or default if there isn't one.
  """
  value = default
  names = []
  if isinstance(font_or_file, ttLib.TTFont):
    names = ExtractNames(font_or_file, name_id)
  else:
    with contextlib.closing(ttLib.TTFont(font_or_file)) as font:
      names = ExtractNames(font, name_id)

  if names:
    value = names[0]

  return value 

def main(argv):
  for font_file in argv[1:]:
    filename = os.path.basename(font_file)
    try:
      with contextlib.closing(ttLib.TTFont(font_file)) as ttf:
        if 'name' not in ttf:
          continue
        for name in ttf['name'].names:
          print('%s %d %d %d %s %s' % (filename, name.platformID,
                                       name.platEncID, name.langID, name.nameID,
                                       name.toUnicode()))
    except ttLib.TTLibError as e:
      print('BAD_FILE', font_file, e) 

def to_representation(self, value):
        value.open()
        with closing(value) as value:
            return value.read().decode("utf-8") 

def consolidate_output(job, job_vars, output_ids):
    """
    Combine the contents of separate zipped outputs into one via streaming

    job_vars: tuple     Tuple of dictionaries: input_args and ids
    output_ids: tuple   Nested tuple of all the output fileStore IDs
    """
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    # Retrieve IDs
    rseq_id, exon_id, rsem_id = flatten(output_ids)
    # Retrieve output file paths to consolidate
    # map_tar = job.fileStore.readGlobalFile(map_id, os.path.join(work_dir, 'map.tar.gz'))
    qc_tar = job.fileStore.readGlobalFile(rseq_id, os.path.join(work_dir, 'qc.tar.gz'))
    exon_tar = job.fileStore.readGlobalFile(exon_id, os.path.join(work_dir, 'exon.tar.gz'))
    rsem_tar = job.fileStore.readGlobalFile(rsem_id, os.path.join(work_dir, 'rsem.tar.gz'))
    # I/O
    out_tar = os.path.join(work_dir, uuid + '.tar.gz')
    # Consolidate separate tarballs
    with tarfile.open(os.path.join(work_dir, out_tar), 'w:gz') as f_out:
        for tar in [rsem_tar, exon_tar, qc_tar]:
            with tarfile.open(tar, 'r') as f_in:
                for tarinfo in f_in:
                    with closing(f_in.extractfile(tarinfo)) as f_in_file:
                        if tar == qc_tar:
                            tarinfo.name = os.path.join(uuid, 'rseq_qc', os.path.basename(tarinfo.name))
                        else:
                            tarinfo.name = os.path.join(uuid, os.path.basename(tarinfo.name))
                        f_out.addfile(tarinfo, fileobj=f_in_file)
    # Move to output directory of selected
    if input_args['output_dir']:
        output_dir = input_args['output_dir']
        mkdir_p(output_dir)
        copy_to_output_dir(work_dir, output_dir, uuid=None, files=[uuid + '.tar.gz'])
    # Write output file to fileStore
    ids['uuid.tar.gz'] = job.fileStore.writeGlobalFile(out_tar)
    # If S3 bucket argument specified, upload to S3
    if input_args['s3_dir']:
        job.addChildJobFn(upload_output_to_s3, job_vars)