Python glob2.glob() Examples

def create_mod_data(mod_paths):

    module_data = {}
    # load mod paths
    for m in mod_paths:
        if os.path.isdir(m):
            files = glob2.glob(os.path.join(m, "**/*.py"))
            pkg_root = os.path.abspath(os.path.dirname(m))
        else:
            pkg_root = os.path.abspath(os.path.dirname(m))
            files = [m]
        for f in files:
            f = os.path.abspath(f)
            mod_str = open(f, 'rb').read()

            dest_filename = f[len(pkg_root)+1:].replace(os.sep, "/")
            module_data[dest_filename] = bytes_to_b64str(mod_str)

    return module_data 

def __init__(self, panels_path, ocr_file, ad_path, dims, max_panels, max_boxes, max_words, max_vocab_size, h5path):

        self.panels_path = panels_path
        # self.images_path = images_path

        # self.n_pages = glob2.glob(join(self.images_path, '*', '*.jpg')) 
        # takes a really long time

        self.ocr_file = ocr_file
        self.dev_idx, self.test_idx, self.n_pages = self.compute_fold_starts()

        self.n_pages -= self.subtract_ad_pages(ad_path)

        self.dims = dims
        self.h5path = h5path
        self.max_panels = max_panels
        self.max_boxes = max_boxes
        self.max_words = max_words
        self.max_vocab_size = max_vocab_size
        self.dump_vocabulary('./data/comics_vocab.p') 

def list_files_in(directory, ignore_git_folder, ignore_file):
    '''Receives a path to a directory and returns
    paths to all files along with each mimetype'''
    file_list = []
    for root, dirs, files in os.walk(directory):
        for file in files:
            file_list.append(os.path.join(root, file))

    file_list = set(file_list)

    glob_rules = []

    if ignore_git_folder:
            glob_rules.append('.git/**')

    #Check if ignore file was provided
    if ignore_file is not None:
        glob_rules += parse_ignore_file(ignore_file)

    if len(glob_rules):
        glob_matches = match_glob_rules_in_directory(glob_rules, directory)
        #Remove files in file_list that matched any glob rule
        file_list = file_list - glob_matches

    return file_list 

def _reset_filelist(self):
        sample_set = self.sample_set
        datatype = self.datatype
        print(datatype)
        print(sample_set)
        print(self.train_dir)
        if sample_set == 'train':
            if datatype == 'detail_data':
                self.filelist = glob2.glob(self.train_dir + '/**/*_rgb.npy')
                random.shuffle(self.filelist)
        if sample_set == 'valid':
            if datatype == 'detail_data':
                self.filelist = glob2.glob(self.valid_dir + '/**/*_rgb.npy')
                random.shuffle(self.filelist)
        if sample_set == 'test':
            if datatype == 'detail_data':
                self.filelist = glob2.glob(self.test_dir + '/**/*_rgb.npy')
                random.shuffle(self.filelist)
        self.currentindex = 0
        self.datanum = len(self.filelist) 

def getFiles(inputElement, extensions = PC_FILE_FORMATS, recursive = False):
    """ Get the list of files with certain extensions contained in the folder (and possible
subfolders) given by inputElement. If inputElement is directly a file it
returns a list with only one element, the given file """
    # If extensions is not a list but a string we converted to a list
    if type(extensions) == str:
        extensions = [extensions,]
    # If input element is file, return it
    if(os.path.isfile(inputElement)):
        fname,fext = os.path.splitext(inputElement)
        return [inputElement] if fext.lower() in extensions else []
    # Else, use recursive globbing
    files = []
    globpath = os.path.join(inputElement,'**') if recursive else inputElement
    for ext in extensions:
        files.extend(glob2.glob(os.path.join(globpath,'*.' + ext)))
        files.extend(glob2.glob(os.path.join(globpath,'*.' + ext.upper())))
    return list(set(files)) 

def copy_runtime(tgt_dir):
    files = glob(os.path.join(pywren.SOURCE_DIR, "jobrunner/*.py"))
    for f in files:
        shutil.copy(f, os.path.join(tgt_dir, os.path.basename(f))) 

def match_glob_rules_in_directory(glob_rules, directory):
    #Append directory to each glob_rule
    glob_rules = [os.path.join(directory, rule) for rule in glob_rules]
    glob_matches = [glob.glob(rule) for rule in glob_rules]
    #Flatten matches
    glob_matches = reduce(lambda x,y: x+y, glob_matches)
    #Convert to a set to remove duplicates
    return set(glob_matches) 

def produce_normal_file(data_dir):
    filelist = glob2.glob(data_dir + '/**/*_rgb.png')
    print('Total {} images'.format(len(filelist)))

    for curidx in range(len(filelist)):
        if curidx % 100 ==0:
            print('Processing image number', curidx)
        name = filelist[curidx]
        frameindex = name[-12:-8]
        # name = '/home/sicong/detail_data/data/3/0235_rgb.png'
        try:
            img_full = io.imread(name)
        except:
            continue
        depth_full = io.imread(name[0:-8] + '_depth.png')
        depthcount = np.sum(depth_full > 100)
        if depthcount < 100 * 100:
            continue
        rot = 0
        scale = util.getScale_detail(depth_full)
        center = util.getCenter_detail(depth_full)
        if (center[0] < 1 or center[1] < 1 or center[1] > img_full.shape[0] or center[0] > img_full.shape[1]):
            continue

        ori_mask = depth_full > 100
        pcd = PointCloud(np.expand_dims(depth_full, 0), np.expand_dims(ori_mask, 0))
        ori_normal = pcd.get_normal().squeeze(0)
        gt_normal = util_detail.cropfor3d(ori_normal, center, scale, rot, 256, 'nearest')
        gt_normal = normal_util.normalize(gt_normal)
        normal_file_name = name[0:-8] + '_normal.npy'
        np.save(normal_file_name, gt_normal) 

def LoadModules(self):
        # loop and assign key and name
        warnings.filterwarnings('ignore', '.*Parent module*',)
        x = 1
        for name in glob2.glob('Modules/**/*.py'):
            if name.endswith(".py") and ("__init__" not in name):
                loaded_modules = imp.load_source(
                    name.replace("/", ".").rstrip('.py'), name)
                self.Modules[name] = loaded_modules
                self.Dmodules[x] = loaded_modules
                x += 1
        # print self.Dmodules
        # print self.Modules 

def _RmGlob(file_wildcard, root, include_hidden):
  """Removes files matching 'file_wildcard' in root and its subdirectories, if
  any exists.

  An exception is thrown if root doesn't exist."""
  wildcard = os.path.join(os.path.realpath(root), file_wildcard)
  for item in glob2.glob(wildcard, include_hidden=include_hidden):
    try:
      os.remove(item)
    except OSError, e:
      if e.errno != errno.ENOENT:
        raise 

def live():
    """Run livereload server"""
    from livereload import Server

    server = Server(app)

    map(server.watch, glob2.glob("application/pages/**/*.*"))  # pages
    map(server.watch, glob2.glob("application/macros/**/*.html"))  # macros
    map(server.watch, glob2.glob("application/static/**/*.*"))  # public assets

    server.serve(port=PORT) 

def live():
    """Run livereload server"""
    from livereload import Server

    server = Server(app)

    map(server.watch, glob2.glob("application/pages/**/*.*"))  # pages
    map(server.watch, glob2.glob("application/macros/**/*.html"))  # macros
    map(server.watch, glob2.glob("application/static/**/*.*"))  # public assets

    server.serve(port=PORT) 

def test_maps():
    sample_dir = os.path.join(os.path.dirname(__file__), "mapfiles")
    pth = sample_dir + r'/**/*.map'
    mapfiles = glob2.glob(pth)
    mapfiles = [f for f in mapfiles if "basemaps" not in f]

    for fn in mapfiles:
        logging.info("Processing {}".format(fn))
        fn = os.path.join(sample_dir, fn)
        pr = cProfile.Profile()
        pr.enable()
        output(fn)
        pr.disable()
        # pr.print_stats(sort='time') 

def list_serial_ports(ports_filter=None):
    ports = list(serial.tools.list_ports.comports())
    if ports_filter is not None:
        ports = filter(ports_filter, ports)
    if is_mac():
        ports = ports + [[x] for x in glob('/dev/tty.*') if x not in map(lambda x: x[0], ports)]
    return list(ports) 

def find_files(path, patterns):
    if not isinstance(patterns, (list, tuple, set)):
        patterns = [patterns]
    files = []
    for pattern in patterns:
        files += glob2.glob(path + os.sep + pattern)
    return list(set(files)) 

def get_module_string(tests_path):
    path = get_module_path()
    tests_path = os.path.abspath(os.path.join(path, os.path.pardir, tests_path))
    test_files = glob2.glob(tests_path)
    relative_test_files = [get_ws_relative_route(test_file) for test_file in test_files]
    module_strings = [".".join(test_file)[:-3] for test_file in relative_test_files]
    return module_strings 

def _Glob(base, pattern, include_hidden):
  base = os.path.realpath(base)
  hits = glob2.glob(os.path.join(base, pattern), include_hidden=include_hidden)
  if hits:
    print('\n'.join(sorted((os.path.relpath(hit, start=base) for hit in hits)))) 

def register_metas(self):
        """register metas"""

        # concatenate some attributes to global lists:
        aggregated = {'build': [], 'watch': []}
        for attribute, values in aggregated.items():
            for info in self.info['analyses'] + [self.info]:
                if attribute in info:
                    values.append(info[attribute])

        for meta in self.metas:
            log.debug('Registering meta information {}'.format(meta.name))

            # grab routes
            self.routes += [(r'/{}/{}'.format(meta.name, route),
                             handler, data)
                            for route, handler, data in meta.routes]

        # process files to watch for autoreload
        if aggregated['watch']:
            to_watch = [expr for w in aggregated['watch'] for expr in w]
            log.info('watching additional files: {}'.format(to_watch))

            cwd = os.getcwd()
            os.chdir(self.analyses_path)
            if glob2:
                files = [os.path.join(self.analyses_path, fn)
                         for expr in to_watch for fn in glob2.glob(expr)]
            else:
                files = [os.path.join(self.analyses_path, fn)
                         for expr in to_watch for fn in glob.glob(expr)]
                if any('**' in expr for expr in to_watch):
                    log.warning('Please run "pip install glob2" to properly '
                                'process watch patterns with "**".')
            os.chdir(cwd)

            for fn in files:
                log.debug('watch file {}'.format(fn))
                tornado.autoreload.watch(fn)

        # save build commands
        self.build_cmds = aggregated['build'] 

def _reset_filelist(self, datatype=None, sample_set = 'train'):
        if sample_set == 'train':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.train_dir + '/pose_prepared/91/500/up-p91/' +  '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.train_dir + '/**/*.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.train_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.train_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.train_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        if sample_set == 'valid':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.valid_dir + '/pose_prepared/91/500/up-p91/' + '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.valid_dir + '/**/*.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.valid_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.valid_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.valid_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        if sample_set == 'test':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.test_dir + '/pose_prepared/91/500/up-p91/' + '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.test_dir + '/**/**.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.test_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.test_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.test_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        self.currentindex = 0
        self.datanum = len(self.filelist)
        #print('finding data in folder:',self.train_dir, 'with ending:', ) 

def _reset_filelist(self, datatype=None, sample_set='train'):
        if sample_set == 'train':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.train_dir + '/pose_prepared/91/500/up-p91/' + '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.train_dir + '/**/*.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.train_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.train_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data2':
                self.filelist = glob2.glob(self.train_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.train_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        if sample_set == 'valid':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.valid_dir + '/pose_prepared/91/500/up-p91/' + '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.valid_dir + '/**/*.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.valid_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.valid_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            elif datatype == 'detail_dat2':
                self.filelist = glob2.glob(self.valid_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.valid_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        if sample_set == 'test':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.test_dir + '/pose_prepared/91/500/up-p91/' + '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.test_dir + '/**/**.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.test_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.test_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data2':
                self.filelist = glob2.glob(self.test_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.test_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        self.currentindex = 0
        self.datanum = len(self.filelist) 

def _reset_filelist(self):
        sample_set = self.sample_set
        datatype = self.datatype
        if sample_set == 'train':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.train_dir + '/pose_prepared/91/500/up-p91/' + '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.train_dir + '/**/*.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.train_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.train_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.train_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        if sample_set == 'valid':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.valid_dir + '/pose_prepared/91/500/up-p91/' + '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.valid_dir + '/**/*.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.valid_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.valid_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.valid_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        if sample_set == 'test':
            if datatype == 'up-3d':
                self.filelist = glob2.glob(self.test_dir + '/pose_prepared/91/500/up-p91/' + '/**/*_image.png')
                random.shuffle(self.filelist)
            elif datatype == 'normal_dataset':
                self.filelist = glob2.glob(self.test_dir + '/**/**.tiff')
                random.shuffle(self.filelist)
            elif datatype == 'detail_data':
                self.filelist = glob2.glob(self.test_dir + '/**/*_rgb.png')
                random.shuffle(self.filelist)
            elif datatype == 'realtest':
                self.filelist = glob2.glob(self.test_dir + '/**/*.jpg')
                random.shuffle(self.filelist)
            else:
                self.filelist = glob2.glob(self.test_dir + '/**/*.mp4')
                random.shuffle(self.filelist)
        if sample_set == 'test_seq':
            self.filelist = sorted(glob2.glob(self.test_dir + '/*_rgb.png'))
        self.currentindex = 0
        self.datanum = len(self.filelist)
        print('{} in dir'.format(self.datanum))
        # print('finding data in folder:',self.train_dir, 'with ending:', )