Python pkg_resources.resource_stream() Examples

def open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    filename = os.path.join(os.path.dirname(__file__),
                            'zoneinfo', *name_parts)
    if not os.path.exists(filename):
        # http://bugs.launchpad.net/bugs/383171 - we avoid using this
        # unless absolutely necessary to help when a broken version of
        # pkg_resources is installed.
        try:
            from pkg_resources import resource_stream
        except ImportError:
            resource_stream = None

        if resource_stream is not None:
            return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    filename = os.path.join(os.path.dirname(__file__),
                            'zoneinfo', *name_parts)
    if not os.path.exists(filename):
        # http://bugs.launchpad.net/bugs/383171 - we avoid using this
        # unless absolutely necessary to help when a broken version of
        # pkg_resources is installed.
        try:
            from pkg_resources import resource_stream
        except ImportError:
            resource_stream = None

        if resource_stream is not None:
            return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def __init__(self):
        resource_package = __name__

        yelp_acc_path = 'acc_yelp.bin'
        yelp_ppl_path = 'ppl_yelp.binary'
        yelp_ref0_path = 'yelp.refs.0'
        yelp_ref1_path = 'yelp.refs.1'

        
        yelp_acc_file = pkg_resources.resource_stream(resource_package, yelp_acc_path)
        yelp_ppl_file = pkg_resources.resource_stream(resource_package, yelp_ppl_path)
        yelp_ref0_file = pkg_resources.resource_stream(resource_package, yelp_ref0_path)
        yelp_ref1_file = pkg_resources.resource_stream(resource_package, yelp_ref1_path)

        
        self.yelp_ref = []
        with open(yelp_ref0_file.name, 'r') as fin:
            self.yelp_ref.append(fin.readlines())
        with open(yelp_ref1_file.name, 'r') as fin:
            self.yelp_ref.append(fin.readlines())
        self.classifier_yelp = fasttext.load_model(yelp_acc_file.name)
        self.yelp_ppl_model = kenlm.Model(yelp_ppl_file.name) 

def schematron(cls, schema):
        transforms = [
            "xml/schematron/iso_dsdl_include.xsl",
            "xml/schematron/iso_abstract_expand.xsl",
            "xml/schematron/iso_svrl_for_xslt1.xsl",
            ]
        if isinstance(schema, file):
            compiled = etree.parse(schema)
        else:
            compiled = schema
        for filename in transforms:
            with resource_stream(
                    __name__, filename) as stream:
                xform_xml = etree.parse(stream)
                xform = etree.XSLT(xform_xml)
                compiled = xform(compiled)
        return etree.XSLT(compiled) 

def test_read_yaml_file_bad_extract(tmpdir, caplog):
    class FakeProvider(object):
        def get_resource_stream(self, pkg, rsc):
            raise IOError

    # pkg_resources.resource_stream() cannot be mocked directly
    # Instead mock the module-level function it calls.
    (flexmock(pkg_resources)
        .should_receive('get_provider')
        .and_return(FakeProvider()))

    config_path = os.path.join(str(tmpdir), 'config.yaml')
    with open(config_path, 'w'):
        pass

    with pytest.raises(IOError):
        read_yaml_from_file_path(config_path, 'schemas/container.json')
    assert "unable to extract JSON schema, cannot validate" in caplog.text 

def open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    filename = os.path.join(os.path.dirname(__file__),
                            'zoneinfo', *name_parts)
    if not os.path.exists(filename):
        # http://bugs.launchpad.net/bugs/383171 - we avoid using this
        # unless absolutely necessary to help when a broken version of
        # pkg_resources is installed.
        try:
            from pkg_resources import resource_stream
        except ImportError:
            resource_stream = None

        if resource_stream is not None:
            return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def __open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    filename = os.path.join(os.path.dirname(__file__),
                            'zoneinfo', *name_parts)
    if not os.path.exists(filename) and resource_stream is not None:
        # http://bugs.launchpad.net/bugs/383171 - we avoid using this
        # unless absolutely necessary to help when a broken version of
        # pkg_resources is installed.
        return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def test_invalid_schema_resource(self, tmpdir, caplog, schema):
        class FakeProvider(object):
            def get_resource_stream(self, pkg, rsc):
                return io.BufferedReader(io.BytesIO(schema))

        # pkg_resources.resource_stream() cannot be mocked directly
        # Instead mock the module-level function it calls.
        (flexmock(pkg_resources)
            .should_receive('get_provider')
            .and_return(FakeProvider()))

        filename = os.path.join(str(tmpdir), 'config.yaml')
        with open(filename, 'w'):
            pass

        tasker, workflow = self.prepare()
        plugin = ReactorConfigPlugin(tasker, workflow, config_path=str(tmpdir))
        with caplog.at_level(logging.ERROR), pytest.raises(Exception):
            plugin.run()

        captured_errs = [x.message for x in caplog.records]
        assert any("cannot validate" in x for x in captured_errs) 

def run(self):
        # XXX: do this once only
        fd = pkg_resources.resource_stream ('crocoite', 'data/click.yaml')
        config = list (yaml.safe_load_all (fd))

        l = nodes.definition_list ()
        for site in config:
            urls = set ()
            v = nodes.definition ()
            vl = nodes.bullet_list ()
            v += vl
            for s in site['selector']:
                i = nodes.list_item ()
                i += nodes.paragraph (text=s['description'])
                vl += i
                urls.update (map (lambda x: URL(x).with_path ('/'), s.get ('urls', [])))

            item = nodes.definition_list_item ()
            term = ', '.join (map (lambda x: x.host, urls)) if urls else site['match']
            k = nodes.term (text=term)
            item += k

            item += v
            l += item
        return [l] 

def open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    filename = os.path.join(os.path.dirname(__file__),
                            'zoneinfo', *name_parts)
    if not os.path.exists(filename):
        # http://bugs.launchpad.net/bugs/383171 - we avoid using this
        # unless absolutely necessary to help when a broken version of
        # pkg_resources is installed.
        try:
            from pkg_resources import resource_stream
        except ImportError:
            resource_stream = None

        if resource_stream is not None:
            return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def _cache_tlds(self, tlds):
        '''Logs a diff of the new TLDs and caches them on disk, according to
        settings passed to __init__.'''
        if LOG.isEnabledFor(logging.DEBUG):
            import difflib
            snapshot_stream = pkg_resources.resource_stream(__name__, '.tld_set_snapshot')
            with closing(snapshot_stream) as snapshot_file:
                snapshot = sorted(
                    json.loads(snapshot_file.read().decode('utf-8'))
                )
            new = sorted(tlds)
            LOG.debug('computed TLD diff:\n' + '\n'.join(difflib.unified_diff(
                snapshot,
                new,
                fromfile=".tld_set_snapshot",
                tofile=self.cache_file
            )))

        if self.cache_file:
            try:
                with open(self.cache_file, 'w') as cache_file:
                    json.dump(tlds, cache_file)
            except IOError as ioe:
                LOG.warn("unable to cache TLDs in file %s: %s", self.cache_file, ioe) 

def ensure_builder_units():
    gallery_builder_service = 'tljh-voila-gallery-builder.service'
    with resource_stream(__name__, f'./systemd-units/{gallery_builder_service}') as f:
        builder_unit_template = f.read().decode('utf-8')

    gallery_builder_timer = 'tljh-voila-gallery-builder.timer'
    with resource_stream(__name__, f'./systemd-units/{gallery_builder_timer}') as f:
        builder_timer_template = f.read().decode('utf-8')

    unit_params = dict(
        python_interpreter_path=sys.executable,
    )

    systemd.install_unit(gallery_builder_service, builder_unit_template.format(**unit_params))
    systemd.install_unit(gallery_builder_timer, builder_timer_template.format(**unit_params))

    for unit in [gallery_builder_service, gallery_builder_timer]:
        systemd.restart_service(unit)
        systemd.enable_service(unit)

    systemd.reload_daemon() 

def open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    filename = os.path.join(os.path.dirname(__file__),
                            'zoneinfo', *name_parts)
    if not os.path.exists(filename):
        # http://bugs.launchpad.net/bugs/383171 - we avoid using this
        # unless absolutely necessary to help when a broken version of
        # pkg_resources is installed.
        try:
            from pkg_resources import resource_stream
        except ImportError:
            resource_stream = None

        if resource_stream is not None:
            return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def test_post_upload_list(db, client, token):
    """
    Test uploading a report.
    """
    count_1 = db.session.query(models.Upload).count()

    rv = client.post(
        "/rest_api/v1/uploads",
        data={"report": resource_stream("tests", "multiqc_data.json")},
        headers={
            "access_token": token,
            "Content-Type": "multipart/form-data",
            "Accept": "application/json",
        },
    )

    # Check the request was successful
    assert rv.status_code == 201, rv.json

    # Validate the response
    schemas.UploadSchema().validate(rv.json)

    # Check that there is a new Upload
    count_2 = db.session.query(models.Upload).count()
    assert count_2 == count_1 + 1 

def open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    filename = os.path.join(os.path.dirname(__file__),
                            'zoneinfo', *name_parts)
    if not os.path.exists(filename) and resource_stream is not None:
        # http://bugs.launchpad.net/bugs/383171 - we avoid using this
        # unless absolutely necessary to help when a broken version of
        # pkg_resources is installed.
        return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def load_resource(path: str, encoding: str = None) -> TextIO:
    """
    Open a resource file located in a python package or the local filesystem.

    Args:
        path: The resource path in the form of `dir/file` or `package:dir/file`
    Returns:
        A file-like object representing the resource
    """
    components = path.rsplit(":", 1)
    try:
        if len(components) == 1:
            return open(components[0], encoding=encoding)
        else:
            bstream = pkg_resources.resource_stream(components[0], components[1])
            if encoding:
                return TextIOWrapper(bstream, encoding=encoding)
            return bstream
    except IOError:
        pass 

def get_gallery():
    with resource_stream(__name__, 'gallery.yaml') as f:
        gallery = yaml.load(f)
    images = _get_docker_images()
    gallery['examples'] = {
        name: ex for name, ex in gallery.get('examples', {}).items()
        if f"{name}:latest" in images
    }
    return gallery 

def open_local_resource(cls, uri):
        """Overwrite the superclass's open_local_resource() method.

        This is done to allow periods in the folder and file names, to allow a
        final .map extension, and to allow a hashtag at the end.
        """
        assert re.match(
            r'^public/([a-zA-Z0-9\.\-_]+/)*[a-zA-Z0-9\.\-_]+\.'
            '(jpg|jpeg|png|gif|js|css|json|map)(#\.+)?$', uri
        ), uri
        assert '..' not in uri
        return pkg_resources.resource_stream(cls.__module__, uri) 

def load_titles():
    """
    load job titles as generator from txt.gz file included in the library
    """
    with resource_stream('find_job_titles',
                         'data/titles_combined.txt.gz') as fhandle:
        with gzip.GzipFile(fileobj=fhandle, mode='r') as gzf:
            for line in gzf:
                # Note: decode rather than "rt" for py2 compat
                # TODO: using pyahocorasick this should now be 'str' again ;(
                yield line.decode('utf-8').strip() 

def load_base(self) -> Optional[RecursiveDict[CommentedMap]]:
        if self.base_path.startswith("pkg://"):
            url = URL(self.base_path)
            return RecursiveDict(yaml.load(pkg_resources.resource_stream(url.host, url.path)),
                                 CommentedMap)
        try:
            with open(self.base_path, 'r') as stream:
                return RecursiveDict(yaml.load(stream), CommentedMap)
        except OSError:
            pass
        return None 

def addJSONBlocksFromFile(self, filename):
        try:
            import pkg_resources

            f = pkg_resources.resource_stream(__name__, filename)
        except (ImportError, IOError), e:
            log.debug("Cannot get resource_stream for %s %s"%(filename, e))
            root = os.environ.get("PYMCLEVEL_YAML_ROOT", "pymclevel")  # fall back to cwd as last resort
            path = join(root, filename)

            log.debug("Failed to read %s using pkg_resources. Trying %s instead." % (filename, path))

            f = file(path) 

def sendfile(self, writer, fname, content_type=None, headers=None):
        if not content_type:
            content_type = get_mime_type(fname)
        try:
            with pkg_resources.resource_stream(self.pkg, fname) as f:
                yield from start_response(writer, content_type, "200", headers)
                yield from sendstream(writer, f)
        except OSError as e:
            if e.args[0] == uerrno.ENOENT:
                yield from http_error(writer, "404")
            else:
                raise 

def load(cls, name):
        """Loads the runtime of the specified name.
        """
        f = resource_stream(__name__, "runtimes/{}/runtime.yml".format(name))
        data = yaml.safe_load(f)
        return Runtime(data['name'], data['version'], data) 

def open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.

    It is possible to specify different location for zoneinfo
    subdir by using the PYTZ_TZDATADIR environment variable.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    zoneinfo_dir = os.environ.get('PYTZ_TZDATADIR', None)
    if zoneinfo_dir is not None:
        filename = os.path.join(zoneinfo_dir, *name_parts)
    else:
        filename = os.path.join(os.path.dirname(__file__),
                                'zoneinfo', *name_parts)
        if not os.path.exists(filename):
            # http://bugs.launchpad.net/bugs/383171 - we avoid using this
            # unless absolutely necessary to help when a broken version of
            # pkg_resources is installed.
            try:
                from pkg_resources import resource_stream
            except ImportError:
                resource_stream = None

            if resource_stream is not None:
                return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def test_extractor_from_entry_points():
    import testme
    extract = extractor_from_entry_points()
    readable = resource_stream(__name__, 'fixtures/get_this_that')
    for value in Response.values_from_readable(extract, readable):
        pass
    hostname = 'httpbin.org'
    assert list(extract.extractors.keys()) == [hostname]
    extractors = set(extract.extractors[hostname].extractors)
    expected = set([testme.example, testme.example_with_hostname_suffix])
    assert expected.issubset(extractors) 

def main():
    # prune stopped containers and dangling images
    client = docker.from_env()
    client.containers.prune()
    client.images.prune()

    # build the new images
    with resource_stream(__name__, GALLERY_PATH) as fp:
        examples = parse_gallery_config(fp)
    for example in examples:
        logging.info(f'Building image {example.image}')
        build_image(example) 

def open_resource(name):
    """Open a resource from the zoneinfo subdir for reading.

    Uses the pkg_resources module if available and no standard file
    found at the calculated location.

    It is possible to specify different location for zoneinfo
    subdir by using the PYTZ_TZDATADIR environment variable.
    """
    name_parts = name.lstrip('/').split('/')
    for part in name_parts:
        if part == os.path.pardir or os.path.sep in part:
            raise ValueError('Bad path segment: %r' % part)
    zoneinfo_dir = os.environ.get('PYTZ_TZDATADIR', None)
    if zoneinfo_dir is not None:
        filename = os.path.join(zoneinfo_dir, *name_parts)
    else:
        filename = os.path.join(os.path.dirname(__file__),
                                'zoneinfo', *name_parts)
        if not os.path.exists(filename):
            # http://bugs.launchpad.net/bugs/383171 - we avoid using this
            # unless absolutely necessary to help when a broken version of
            # pkg_resources is installed.
            try:
                from pkg_resources import resource_stream
            except ImportError:
                resource_stream = None

            if resource_stream is not None:
                return resource_stream(__name__, 'zoneinfo/' + name)
    return open(filename, 'rb') 

def spin(parser, args):
    """
    Creates a Vagrantfile meant for user-interaction in the current directory.
    """
    if os.path.isfile('Vagrantfile'):
        print("Vagrantfile already exists. Please move it away. Exiting...")
        sys.exit(EXIT_VAGRANTFILE_ALREADY_EXISTS)

    config, _ = prepare_config(args)

    config['template'] = args.template
    config['name'] = args.name

    print("Creating a Vagrantfile")
    if config['hypervisor'] == 'virtualbox':
        with open("Vagrantfile", 'w') as f:
            _prepare_vagrantfile(config, "analyst_single.rb", f)
    elif config['hypervisor'] == 'vsphere':
        with open("Vagrantfile", 'w') as f:
            _prepare_vagrantfile(config, "analyst_vsphere.rb", f)
    elif config['hypervisor'] == 'aws':
        with open("Vagrantfile", 'w') as f:
            config['aws_ami_id'] = get_AMI_ID_by_template(config,
            config['template'])
            _prepare_vagrantfile(config, "analyst_aws.rb", f)
    print("Vagrantfile generated. You can move it in your analysis directory "
          "and issue a `vagrant up` to get started with your VM.")

    if config.get("windows_defender", "false") == "false" \
        and config.get("os") == "Windows10" and config.get("os_version") >= 1903:
        _r = resource_stream(__name__, 'messages/defender-1903.txt')
        print(_r.read().decode()) 

def resource_stream(subpath):
    module_name = external_caller()
    return pkg_resource_stream(module_name, subpath) 

def whitelist():
    return Whitelist.load(pkg_resources.resource_stream(
        'vulnix', 'tests/fixtures/whitelist.toml'))