Python utils.get_files() Examples

def batch_gen(folder, batch_shape):
    '''Resize images to 512, randomly crop a 256 square, and normalize'''
    files = np.asarray(get_files(folder))
    while True:
        X_batch = np.zeros(batch_shape, dtype=np.float32)

        idx = 0

        while idx < batch_shape[0]:  # Build batch sample by sample
            try:
                f = np.random.choice(files)

                X_batch[idx] = get_img_random_crop(f, resize=512, crop=256).astype(np.float32)
                X_batch[idx] /= 255.    # Normalize between [0,1]
                
                assert(not np.isnan(X_batch[idx].min()))
            except Exception as e:
                # Do not increment idx if we failed 
                print(e)
                continue
            idx += 1

        yield X_batch 

def validate(schema, jsonfiles):
    """Validate a JSON files against a JSON schema.

    \b
    SCHEMA: JSON schema to validate against. Required.
    JSONFILE: JSON files to validate. Required.
    """
    schema = json.loads(schema.read())
    success = True
    for path in utils.get_files(jsonfiles):
        with open(path) as f:
            try:
                jsonfile = json.loads(f.read())
            except ValueError:
                logging.error("Error loading json file " + path)
                raise Exception("Invalid json file")
        try:
            jsonschema.validate(jsonfile, schema)
        except Exception as e:
            success = False
            logging.error("Error validating file " + path)
            logging.error(str(e))

    if not success:
        sys.exit(-1) 

def batch_gen(folder, batch_shape):
    '''Resize images to 512, randomly crop a 256 square, and normalize'''
    files = np.asarray(get_files(folder))
    while True:
        X_batch = np.zeros(batch_shape, dtype=np.float32)

        idx = 0

        while idx < batch_shape[0]:  # Build batch sample by sample
            try:
                f = np.random.choice(files)

                X_batch[idx] = get_img_random_crop(f, resize=512, crop=256).astype(np.float32)
                X_batch[idx] /= 255.    # Normalize between [0,1]
                
                assert(not np.isnan(X_batch[idx].min()))
            except Exception as e:
                # Do not increment idx if we failed 
                print(e)
                continue
            idx += 1

        yield X_batch 

def main():
    parser = build_parser()
    options = parser.parse_args()
    check_opts(options)

    style_image = utils.load_image(options.style)
    style_image = np.ndarray.reshape(style_image, (1,) + style_image.shape)

    content_targets = utils.get_files(options.train_path)
    content_shape = utils.load_image(content_targets[0]).shape

    device = '/gpu:0' if options.use_gpu else '/cpu:0'

    style_transfer = FastStyleTransfer(
        vgg_path=VGG_PATH,
        style_image=style_image,
        content_shape=content_shape,
        content_weight=options.content_weight,
        style_weight=options.style_weight,
        tv_weight=options.style_weight,
        batch_size=options.batch_size,
        device=device)

    for iteration, network, first_image, losses in style_transfer.train(
        content_training_images=content_targets,
        learning_rate=options.learning_rate,
        epochs=options.epochs,
        checkpoint_iterations=options.checkpoint_iterations
    ):
        print_losses(losses)

        saver = tf.train.Saver()
        if (iteration % 100 == 0):
            saver.save(network, opts.save_path + '/fast_style_network.ckpt')

        saver.save(network, opts.save_path + '/fast_style_network.ckpt') 

def __init__(self, style_path, img_size=512, scale=1, alpha=1, interpolate=False):
        self.style_imgs = get_files(style_path)

        # Create room for two styles for interpolation
        self.style_rgbs = [None, None]

        self.img_size = img_size
        self.crop_size = 256
        self.scale = scale
        self.alpha = alpha

        cv2.namedWindow('Style Controls')
        if len(self.style_imgs) > 1:
            # Select style image by index
            cv2.createTrackbar('index','Style Controls', 0, len(self.style_imgs)-1, self.set_idx)
        
        # Blend param for AdaIN transform
        cv2.createTrackbar('alpha','Style Controls', 100, 100, self.set_alpha)

        # Resize style to this size before cropping
        cv2.createTrackbar('size','Style Controls', img_size, 2048, self.set_size)

        # Size of square crop box for style
        cv2.createTrackbar('crop size','Style Controls', 256, 2048, self.set_crop_size)

        # Scale the content before processing
        cv2.createTrackbar('scale','Style Controls', int(scale*100), 200, self.set_scale)

        self.set_style(random=True, window='Style Controls', style_idx=0)

        if interpolate:
            # Create a window to show second style image for interpolation
            cv2.namedWindow('style2')
            self.interp_weight = 1.
            cv2.createTrackbar('interpolation','Style Controls', 100, 100, self.set_interp)
            self.set_style(random=True, style_idx=1, window='style2') 

def validate_path(schema, jsonfiles):
    schema = json.loads(schema.read())

    for path in utils.get_files(jsonfiles):
        path_components = utils.get_path_parts(path)

        regex = schema[path_components[0]]
        if not re.compile(regex).match(path):
            raise AssertionError('Path "%s" does not match spec "%s"' % (path, regex)) 

def backup_all(password_file=None):
    for file_ in get_files('.'):
        backup(file_, password_file) 

def restore_all(password_file=None):
    for file_ in get_files(ATK_VAULT):
        if os.path.basename(file_) == 'encrypted':

            # Get the path without the atk vault base and encrypted filename
            original_path = os.path.join(*split_path(file_)[1:-1])
            restore(original_path, password_file) 

def __init__(self, style_path, img_size=512, crop_size=512, scale=1, alpha=1, swap5=False, ss_alpha=1, passes=1):
        if os.path.isdir(style_path):
            self.style_imgs = get_files(style_path)
        else:
            self.style_imgs = [style_path]  # Single image instead of folder

        self.style_rgb = None

        self.img_size = img_size
        self.crop_size = crop_size
        self.scale = scale
        self.alpha = alpha
        self.ss_alpha = ss_alpha
        self.passes = passes

        cv2.namedWindow('Style Controls')
        if len(self.style_imgs) > 1:
            # Select style image by index
            cv2.createTrackbar('Index','Style Controls', 0, len(self.style_imgs)-1, self.set_idx)
        
        # Blend param for WCT/AdaIN transform
        cv2.createTrackbar('WCT/AdaIN alpha','Style Controls', int(self.alpha*100), 100, self.set_alpha)

        # Separate blend setting for style-swap
        cv2.createTrackbar('Style-swap alpha','Style Controls', int(self.ss_alpha*100), 100, self.set_ss_alpha)

        # Resize style to this size before cropping
        cv2.createTrackbar('Style size','Style Controls', self.img_size, 1280, self.set_size)

        # Size of square crop box for style
        cv2.createTrackbar('Style crop','Style Controls', self.crop_size, 1280, self.set_crop_size)

        # Scale the content before processing
        cv2.createTrackbar('Content scale','Style Controls', int(self.scale*100), 200, self.set_scale)

        # Num times to repeat the stylization pipeline
        cv2.createTrackbar('# of passes','Style Controls', self.passes, 5, self.set_passes)

        self.set_style(random=True, window='Style Controls') 

def vectorTiling(output, sources, catalog, min_zoom, max_zoom, layer):
    """ Generate an MBTiles file of vector tiles from the output of an OpenBounds project.

    \b
    PARAMS: 
        - sources : A directory containing geojson files, or a list of geojson files"
        - output : file.mbtiles for the generated data 
    """

    if os.path.exists(output):
        logging.error("Error, output path already exists")
        sys.exit(-1)

    if catalog:
        with open(catalog, "rb") as f:
            geojson = json.load(f)
        source_paths = [item["properties"]["path"] for item in geojson["features"]]
    else:
        source_paths = []
        for arg in sources:
            for item in utils.get_files(arg):
                if (
                    os.path.splitext(item)[1] == ".geojson"
                    and os.path.basename(item) != "catalog.geojson"
                ):
                    source_paths.append(item)

    logging.info("{} geojson files found".format(len(source_paths)))

    command = (
        "tippecanoe -o "
        + output
        + " "
        + " ".join(source_paths)
        + " --no-progress-indicator "
        + " --no-polygon-splitting "
        + "--coalesce --reverse --reorder "
        + "--detect-shared-borders "  # try really hard to coalesce polygons with the same properties
        + " -l "  # Avoid small gaps between polygons when simplifying
        + layer
        + " -z {} -Z {}".format(max_zoom, min_zoom)  # force to use a single layer
    )
    logging.info(command)
    subprocess.call(command, shell=True) 

def main():

    # parse arguments
    args = parse_args()
    if args is None:
        exit()

    # initiate VGG19 model
    model_file_path = args.vgg_model + '/' + vgg19.MODEL_FILE_NAME
    vgg_net = vgg19.VGG19(model_file_path)

    # get file list for training
    content_images = utils.get_files(args.trainDB_path)

    # load style image
    style_image = utils.load_image(args.style)

    # create a map for content layers info
    CONTENT_LAYERS = {}
    for layer, weight in zip(args.content_layers,args.content_layer_weights):
        CONTENT_LAYERS[layer] = weight

    # create a map for style layers info
    STYLE_LAYERS = {}
    for layer, weight in zip(args.style_layers, args.style_layer_weights):
        STYLE_LAYERS[layer] = weight

    # open session
    sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))

    # build the graph for train
    trainer = style_transfer_trainer.StyleTransferTrainer(session=sess,
                                                          content_layer_ids=CONTENT_LAYERS,
                                                          style_layer_ids=STYLE_LAYERS,
                                                          content_images=content_images,
                                                          style_image=add_one_dim(style_image),
                                                          net=vgg_net,
                                                          num_epochs=args.num_epochs,
                                                          batch_size=args.batch_size,
                                                          content_weight=args.content_weight,
                                                          style_weight=args.style_weight,
                                                          tv_weight=args.tv_weight,
                                                          learn_rate=args.learn_rate,
                                                          save_path=args.output,
                                                          check_period=args.checkpoint_every,
                                                          test_image=args.test,
                                                          max_size=args.max_size,
                                                          )
    # launch the graph in a session
    trainer.train()

    # close session
    sess.close()