Python tensorflow.initialize_all_variables() Examples

def build_model(self, sess):
        self.init_opt()
        sess.run(tf.initialize_all_variables())

        if len(self.model_path) > 0:
            print("Reading model parameters from %s" % self.model_path)
            restore_vars = tf.all_variables()
            # all_vars = tf.all_variables()
            # restore_vars = [var for var in all_vars if
            #                 var.name.startswith('g_') or
            #                 var.name.startswith('d_')]
            saver = tf.train.Saver(restore_vars)
            saver.restore(sess, self.model_path)

            istart = self.model_path.rfind('_') + 1
            iend = self.model_path.rfind('.')
            counter = self.model_path[istart:iend]
            counter = int(counter)
        else:
            print("Created model with fresh parameters.")
            counter = 0
        return counter 

def main(argv=None):
    X_train, Y_train, X_test, Y_test = gtsrb(FLAGS.train_dataset, FLAGS.test_dataset, labels_filename=FLAGS.labels)
    print 'Loaded GTSRB data'

    X_train = np.asarray(map(lambda x: pre_process_image(x), X_train.astype(np.uint8)),dtype=np.float32)
    X_test = np.asarray(map(lambda x: pre_process_image(x), X_test.astype(np.uint8)),dtype=np.float32)
    global total_iterations 
    global best_validation_accuracy
    global last_improvement
    global best_test_accuracy 
    
    global val_acc_list 
    global batch_acc_list 
    global test_acc_list


    with tf.Session() as sess:
        model = YadavModel()
	sess.run(tf.initialize_all_variables())
        #X_train, Y_train = gen_transformed_data(X_train,Y_train,43,10,30,5,5,1)
	print(X_train.shape)
	print(Y_train.shape)
	optimize(sess, model, X_train, Y_train, X_test, Y_test, 10000, 128) 

def restore_best_model(self):
    """Load bestmodel file from eval directory, add variables for adagrad, and save to train directory"""
    tf.logging.info("Restoring bestmodel for training...")

    # Initialize all vars in the model
    sess = tf.Session(config=util.get_config())
    print("Initializing all variables...")
    sess.run(tf.initialize_all_variables())

    # Restore the best model from eval dir
    saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name])
    print("Restoring all non-adagrad variables from best model in eval dir...")
    curr_ckpt = util.load_ckpt(saver, sess, "eval")
    print("Restored %s." % curr_ckpt)

    # Save this model to train dir and quit
    new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model")
    new_fname = os.path.join(FLAGS.log_root, "train", new_model_name)
    print("Saving model to %s..." % (new_fname))
    new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables
    new_saver.save(sess, new_fname)
    print("Saved.")
    exit() 

def SplitApplyMerge(self):
    # Repeatability.  SGD has a tendency to jump around, even here.
    tf.set_random_seed(1)

    # Use sampling to train REINFORCE
    with st.value_type(st.SampleAndReshapeValue(n=1)):
      (route_selection,
        routing_loss,
        final_loss) = build_split_apply_merge_model()

    sgd = tf.train.GradientDescentOptimizer(1.0).minimize(final_loss)

    tf.initialize_all_variables().run()

    for i in range(10):
        # Run loss and inference step.  This toy problem converges VERY quickly.
      (routing_loss_v, final_loss_v, route_selection_v, _) = sess.run(
          [routing_loss, final_loss, tf.identity(route_selection), sgd])
      print(
          "Iteration %d, routing loss: %s, final_loss: %s, "
          "route selection: %s"
          % (i, routing_loss_v, final_loss_v, route_selection_v)) 

def main():
    sys.stdout.write("start ptb")
    raw_data = reader.ptb_raw_data("")
    train_data, valid_data, test_data, word_to_id = raw_data

    with tf.Graph().as_default(), tf.Session() as session:
        initializer = tf.random_uniform_initializer(-0.04, 0.04)
        with tf.variable_scope("model", reuse=None, initializer=initializer):
            model = PTBModel()

        saver = tf.train.Saver()
        tf.initialize_all_variables().run()
        model.train_writer = tf.train.SummaryWriter('./train', graph=session.graph)

        for i in range(13):
            sys.stdout.write("Epoch: %d\n" % (i + 1))
            train_perplexity = model.train(session, train_data)
            sys.stdout.write("Epoch: %d Train Perplexity: %.3f\n" % (i + 1, train_perplexity))
            valid_perplexity = model.evaluate(session, valid_data)
            sys.stdout.write("Epoch: %d Valid Perplexity: %.3f\n" % (i + 1, valid_perplexity))
            test_perplexity = model.evaluate(session, test_data)
            sys.stdout.write("Epoch: %d Test Perplexity: %.3f\n" % (i + 1, test_perplexity))

        # model.predict(session, test_data, word_to_id)
        saver.save(session, 'model.ckpt') 

def restore_best_model(self):
    """Load bestmodel file from eval directory, add variables for adagrad, and save to train directory"""
    tf.logging.info("Restoring bestmodel for training...")

    # Initialize all vars in the model
    sess = tf.Session(config=util.get_config())
    print("Initializing all variables...")
    sess.run(tf.initialize_all_variables())

    # Restore the best model from eval dir
    saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name])
    print("Restoring all non-adagrad variables from best model in eval dir...")
    curr_ckpt = util.load_ckpt(saver, sess, "eval")
    print("Restored %s." % curr_ckpt)

    # Save this model to train dir and quit
    new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model")
    new_fname = os.path.join(FLAGS.log_root, "train", new_model_name)
    print("Saving model to %s..." % (new_fname))
    new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables
    new_saver.save(sess, new_fname)
    print("Saved.")
    exit() 

def train(self):

	self.train_op = self.optim.minimize(self.loss, global_step=self.global_step)
        self.writer = tf.train.SummaryWriter("./logs/D_pretrained", self.sess.graph)
	self.summary_op = tf.merge_all_summaries()
        tf.initialize_all_variables().run()
        self.saver = tf.train.Saver(var_list=self.D_params_dict, max_to_keep=self.max_to_keep)
        count = 0
	for idx in range(self.max_iter//3000):
            self.save(self.checkpoint_dir, count)
            self.evaluate('test', count)
	    self.evaluate('train', count)
            for k in tqdm(range(3000)):
		right_images, right_text, _ = self.dataset.sequential_sample(self.batch_size)
		right_length = np.sum((right_text!=self.NOT)+0, 1)
		fake_images, fake_text, _ = self.negative_dataset.sequential_sample(self.batch_size)
		fake_length = np.sum((fake_text!=self.NOT)+0, 1)
		wrong_text = self.dataset.get_wrong_text(self.batch_size)
		wrong_length = np.sum((wrong_text!=self.NOT)+0, 1)
		feed_dict = {self.right_images:right_images, self.right_text:right_text, self.right_length:right_length, 
				self.fake_images:fake_images, self.fake_text:fake_text, self.fake_length:fake_length, 
				self.wrong_images:right_images, self.wrong_text:wrong_text, self.wrong_length:wrong_length}
		_, loss, summary_str = self.sess.run([self.train_op, self.loss, self.summary_op], feed_dict)
		self.writer.add_summary(summary_str, count)
                count += 1 

def export(self, last_checkpoint, output_dir):
    """Builds a prediction graph and xports the model.

    Args:
      last_checkpoint: The latest checkpoint from training.
      output_dir: Path to the folder to be used to output the model.
    """
    logging.info('Exporting prediction graph to %s', output_dir)
    with tf.Session(graph=tf.Graph()) as sess:
      # Build and save prediction meta graph and trained variable values.
      self.build_prediction_graph()
      # Remove this if once Tensorflow 0.12 is standard.
      try:
        init_op = tf.global_variables_initializer()
      except AttributeError:
        init_op = tf.initialize_all_variables()
      sess.run(init_op)
      trained_saver = tf.train.Saver()
      trained_saver.restore(sess, last_checkpoint)
      saver = tf.train.Saver()
      saver.export_meta_graph(filename=os.path.join(output_dir, 'export.meta'))
      saver.save(
          sess, os.path.join(output_dir, 'export'), write_meta_graph=False) 

def test_lm(self):
        hps = get_test_hparams()

        with tf.variable_scope("model"):
            model = LM(hps)

        with self.test_session() as sess:
            tf.initialize_all_variables().run()
            tf.initialize_local_variables().run()

            loss = 1e5
            for i in range(50):
                x, y, w = simple_data_generator(hps.batch_size, hps.num_steps)
                loss, _ = sess.run([model.loss, model.train_op], {model.x: x, model.y: y, model.w: w})
                print("%d: %.3f %.3f" % (i, loss, np.exp(loss)))
                if np.isnan(loss):
                    print("NaN detected")
                    break

            self.assertLess(loss, 1.0) 

def build_model(self, sess):
        self.init_opt()

        sess.run(tf.initialize_all_variables())
        if len(self.model_path) > 0:
            print("Reading model parameters from %s" % self.model_path)
            all_vars = tf.trainable_variables()
            # all_vars = tf.all_variables()
            restore_vars = []
            for var in all_vars:
                if var.name.startswith('g_') or var.name.startswith('d_'):
                    restore_vars.append(var)
                    # print(var.name)
            saver = tf.train.Saver(restore_vars)
            saver.restore(sess, self.model_path)

            istart = self.model_path.rfind('_') + 1
            iend = self.model_path.rfind('.')
            counter = self.model_path[istart:iend]
            counter = int(counter)
        else:
            print("Created model with fresh parameters.")
            counter = 0
        return counter 

def train(train_corpus, config, val_corpus, eval_train_corpus=None):
    iterator = Iterator(train_corpus)

    with tf.Session(config=config.cf) as sess:
        model = DecompAtt(config)
        saver = tf.train.Saver()
        sess.run(tf.initialize_all_variables())
        for epoch in xrange(config.num_epochs):
            count = 0
            for batch_x in iterator.next(config.batch_size, shuffle=True):
                batch_qids, batch_q, batch_aids, batch_ap, labels = zip(*batch_x)
                batch_q = np.asarray(batch_q)
                batch_ap = np.asarray(batch_ap)
                labels = np.asarray(labels).astype(np.int32)
                _, loss = sess.run([model.train_op, model.total_loss], 
                                   feed_dict={model.q:batch_q, 
                                              model.a:batch_ap,
                                              model.y:labels,
                                              model.keep_prob:config.keep_prob})
                count += 1
                if count % 10 == 0:
                    print('[epoch {}, batch {}]Loss:{}'.format(epoch, count, loss))
            saver.save(sess,'{}/my_model'.format(model_path), global_step=epoch)
            if eval_train_corpus is not None:
                train_res = evaluate(sess, model, eval_train_corpus, config)
                print('[train] ' + train_res)
            if val_corpus is not None:
                val_res = evaluate(sess, model, val_corpus, config)
                print('[eval] ' + val_res) 

def train(train_corpus, config, val_corpus, eval_train_corpus=None):
    iterator = Iterator(train_corpus)

    with tf.Session(config=config.cf) as sess:
        model = BiMPM(config)
        saver = tf.train.Saver()
        sess.run(tf.initialize_all_variables())
        for epoch in xrange(config.num_epochs):
            count = 0
            for batch_x in iterator.next(config.batch_size, shuffle=True):
                batch_qids, batch_q, batch_ql, batch_aids, batch_ap, batch_al, labels = zip(*batch_x)
                batch_q = np.asarray(batch_q)
                batch_ap = np.asarray(batch_ap)
                labels = np.asarray(labels).astype(np.int32)
                _, loss = sess.run([model.train_op, model.total_loss], 
                                   feed_dict={model.q:batch_q, 
                                              model.a:batch_ap,
                                              model.question_lengths:batch_ql,
                                              model.passage_lengths:batch_al,
                                              model.y:labels,
                                              model.keep_prob:config.keep_prob})
                count += 1
                if count % 10 == 0:
                    print('[epoch {}, batch {}]Loss:{}'.format(epoch, count, loss))
            saver.save(sess,'{}/my_model'.format(model_path), global_step=epoch)
            if eval_train_corpus is not None:
                train_res = evaluate(sess, model, eval_train_corpus, config)
                print('[train] ' + train_res)
            if val_corpus is not None:
                val_res = evaluate(sess, model, val_corpus, config)
                print('[eval] ' + val_res) 

def test_batcher_when_batch_size_is_one(self):
    with self.test_session() as sess:
      batch_size = 1
      num_batches = 2
      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
      counter = examples.count_up_to(num_batches * batch_size + 2)
      image = tf.reshape(
          tf.range(counter * counter), tf.stack([counter, counter]))
      batch_queue = batcher.BatchQueue(
          tensor_dict={'image': image},
          batch_size=batch_size,
          batch_queue_capacity=100,
          num_batch_queue_threads=1,
          prefetch_queue_capacity=100)
      batch = batch_queue.dequeue()

      for tensor_dict in batch:
        for tensor in tensor_dict.values():
          self.assertAllEqual([None, None], tensor.get_shape().as_list())

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        i = 2
        for _ in range(num_batches):
          batch_np = sess.run(batch)
          for tensor_dict in batch_np:
            for tensor in tensor_dict.values():
              self.assertAllEqual(tensor, np.arange(i * i).reshape((i, i)))
              i += 1
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(batch) 

def test_batch_and_unpad_2d_tensors_of_same_size_in_all_dimensions(self):
    with self.test_session() as sess:
      batch_size = 3
      num_batches = 2
      examples = tf.Variable(tf.constant(1, dtype=tf.int32))
      counter = examples.count_up_to(num_batches * batch_size + 1)
      image = tf.reshape(tf.range(1, 13), [4, 3]) * counter
      batch_queue = batcher.BatchQueue(
          tensor_dict={'image': image},
          batch_size=batch_size,
          batch_queue_capacity=100,
          num_batch_queue_threads=1,
          prefetch_queue_capacity=100)
      batch = batch_queue.dequeue()

      for tensor_dict in batch:
        for tensor in tensor_dict.values():
          self.assertAllEqual([4, 3], tensor.get_shape().as_list())

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        i = 1
        for _ in range(num_batches):
          batch_np = sess.run(batch)
          for tensor_dict in batch_np:
            for tensor in tensor_dict.values():
              self.assertAllEqual(tensor, np.arange(1, 13).reshape((4, 3)) * i)
              i += 1
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(batch) 

def test_batch_and_unpad_2d_tensors_of_different_sizes_in_all_dimensions(
      self):
    with self.test_session() as sess:
      batch_size = 3
      num_batches = 2
      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
      counter = examples.count_up_to(num_batches * batch_size + 2)
      image = tf.reshape(
          tf.range(counter * counter), tf.stack([counter, counter]))
      batch_queue = batcher.BatchQueue(
          tensor_dict={'image': image},
          batch_size=batch_size,
          batch_queue_capacity=100,
          num_batch_queue_threads=1,
          prefetch_queue_capacity=100)
      batch = batch_queue.dequeue()

      for tensor_dict in batch:
        for tensor in tensor_dict.values():
          self.assertAllEqual([None, None], tensor.get_shape().as_list())

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        i = 2
        for _ in range(num_batches):
          batch_np = sess.run(batch)
          for tensor_dict in batch_np:
            for tensor in tensor_dict.values():
              self.assertAllEqual(tensor, np.arange(i * i).reshape((i, i)))
              i += 1
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(batch) 

def test_batch_and_unpad_2d_tensors_of_different_sizes_in_1st_dimension(self):
    with self.test_session() as sess:
      batch_size = 3
      num_batches = 2
      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
      counter = examples.count_up_to(num_batches * batch_size + 2)
      boxes = tf.tile(
          tf.reshape(tf.range(4), [1, 4]), tf.stack([counter, tf.constant(1)]))
      batch_queue = batcher.BatchQueue(
          tensor_dict={'boxes': boxes},
          batch_size=batch_size,
          batch_queue_capacity=100,
          num_batch_queue_threads=1,
          prefetch_queue_capacity=100)
      batch = batch_queue.dequeue()

      for tensor_dict in batch:
        for tensor in tensor_dict.values():
          self.assertAllEqual([None, 4], tensor.get_shape().as_list())

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        i = 2
        for _ in range(num_batches):
          batch_np = sess.run(batch)
          for tensor_dict in batch_np:
            for tensor in tensor_dict.values():
              self.assertAllEqual(tensor, np.tile(np.arange(4), (i, 1)))
              i += 1
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(batch) 

def test_prefetch_tensors_with_fully_defined_shapes(self):
    with self.test_session() as sess:
      batch_size = 10
      image_size = 32
      num_batches = 5
      examples = tf.Variable(tf.constant(0, dtype=tf.int64))
      counter = examples.count_up_to(num_batches)
      image = tf.random_normal([batch_size, image_size,
                                image_size, 3],
                               dtype=tf.float32,
                               name='images')
      label = tf.random_uniform([batch_size, 1], 0, 10,
                                dtype=tf.int32, name='labels')

      prefetch_queue = prefetcher.prefetch(tensor_dict={'counter': counter,
                                                        'image': image,
                                                        'label': label},
                                           capacity=100)
      tensor_dict = prefetch_queue.dequeue()

      self.assertAllEqual(tensor_dict['image'].get_shape().as_list(),
                          [batch_size, image_size, image_size, 3])
      self.assertAllEqual(tensor_dict['label'].get_shape().as_list(),
                          [batch_size, 1])

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        for _ in range(num_batches):
          results = sess.run(tensor_dict)
          self.assertEquals(results['image'].shape,
                            (batch_size, image_size, image_size, 3))
          self.assertEquals(results['label'].shape, (batch_size, 1))
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(tensor_dict) 

def avg_checkpoints(model_dir, output_model_dir, last_k):
  tf.reset_default_graph()

  checkpoint_state = tf.train.get_checkpoint_state(model_dir)
  checkpoints = checkpoint_state.all_model_checkpoint_paths[- last_k:]
  var_list = tf.contrib.framework.list_variables(checkpoints[0])
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step"):
      var_values[name] = np.zeros(shape)
  for checkpoint in checkpoints:
    reader = tf.contrib.framework.load_checkpoint(checkpoint)
    for name in var_values:
      tensor = reader.get_tensor(name)
      var_dtypes[name] = tensor.dtype
      var_values[name] += tensor
    tf.logging.info("Read from checkpoint %s", checkpoint)
  for name in var_values:  # Average.
    var_values[name] /= len(checkpoints)

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.all_variables())

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})
    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
        global_step=global_step) 

def evaluate():
    """Evaluated current agent, and records a video with it's performance"""
    envwrap = GymWrapperFactory.make(FLAGS.env,
                                     actrep=FLAGS.action_repeat,
                                     memlen=FLAGS.memory_len,
                                     w=FLAGS.width,
                                     h=FLAGS.height)
    with tf.Session() as sess:
        agent = QlearningAgent(session=sess,
                               action_size=envwrap.action_size,
                               h=FLAGS.height,
                               w=FLAGS.width,
                               channels=FLAGS.memory_len,
                               opt=tf.train.AdamOptimizer(FLAGS.lr))
        sess.run(tf.initialize_all_variables())
        if not os.path.exists(FLAGS.logdir):
            print('ERROR! No', FLAGS.logdir, 'folder found!')
            return
        ckpt = tf.train.latest_checkpoint(FLAGS.logdir)
        if ckpt is not None:
            tf.train.Saver().restore(sess, ckpt)
            agent.update_target()
            print('Session was restored from %s' % ckpt)
        else:
            print('ERROR! No checkpoint found at', FLAGS.logdir)
            return
        envwrap.env.monitor.start(os.path.join(FLAGS.evaldir, FLAGS.env))
        total_reward = 0
        for _ in range(FLAGS.eval_iter):
            s = envwrap.reset()
            terminal = False
            while not terminal:
                reward_per_action = agent.predict_rewards(s)
                s, r, terminal, info = envwrap.step(np.argmax(reward_per_action), test=True)
                total_reward += r
                envwrap.render()
        envwrap.env.monitor.close()
        print('Evaluation finished.')
        print('Average reward per episode: %0.4f' % (total_reward / FLAGS.eval_iter)) 

def main():
    batch = 20
    memory = 10
    input = 200
    steps = 30
    output = 300

    with tf.Session() as sess:
        model = FSMN(memory, input, output)
        model._memory_weights = tf.Variable(np.arange(memory), dtype=tf.float32)
        tf.initialize_all_variables().run()
        w1 = model._W1.eval()
        w2 = model._W2.eval()
        bias = model._bias.eval()
        memory_weights = model._memory_weights.eval()
        inputs = np.random.rand(batch, steps, input).astype(np.float32)
        start = time.time()
        ret = sess.run(model(tf.constant(inputs, dtype=tf.float32)))
        print(str(time.time() - start), "(s)")

    expect_first_batch = []
    for i in range(steps):
        hidden = np.sum([memory_weights[j] * inputs[0][i - j] for j in range(0, min(memory, i + 1))], axis=0)
        expect_first_batch.append(np.dot(w1.T, inputs[0][i]) + np.dot(w2.T, hidden) + bias)

    expect_first_batch = np.array(expect_first_batch)
    real_first_batch = ret[0].reshape(-1, output)
    assert (np.absolute(expect_first_batch - real_first_batch) < 0.0001).all()
    tf.reset_default_graph() 

def test_prefetch_tensors_with_fully_defined_shapes(self):
    with self.test_session() as sess:
      batch_size = 10
      image_size = 32
      num_batches = 5
      examples = tf.Variable(tf.constant(0, dtype=tf.int64))
      counter = examples.count_up_to(num_batches)
      image = tf.random_normal([batch_size, image_size,
                                image_size, 3],
                               dtype=tf.float32,
                               name='images')
      label = tf.random_uniform([batch_size, 1], 0, 10,
                                dtype=tf.int32, name='labels')

      prefetch_queue = prefetcher.prefetch(tensor_dict={'counter': counter,
                                                        'image': image,
                                                        'label': label},
                                           capacity=100)
      tensor_dict = prefetch_queue.dequeue()

      self.assertAllEqual(tensor_dict['image'].get_shape().as_list(),
                          [batch_size, image_size, image_size, 3])
      self.assertAllEqual(tensor_dict['label'].get_shape().as_list(),
                          [batch_size, 1])

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        for _ in range(num_batches):
          results = sess.run(tensor_dict)
          self.assertEquals(results['image'].shape,
                            (batch_size, image_size, image_size, 3))
          self.assertEquals(results['label'].shape, (batch_size, 1))
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(tensor_dict) 

def test_batcher_when_batch_size_is_one(self):
    with self.test_session() as sess:
      batch_size = 1
      num_batches = 2
      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
      counter = examples.count_up_to(num_batches * batch_size + 2)
      image = tf.reshape(
          tf.range(counter * counter), tf.stack([counter, counter]))
      batch_queue = batcher.BatchQueue(
          tensor_dict={'image': image},
          batch_size=batch_size,
          batch_queue_capacity=100,
          num_batch_queue_threads=1,
          prefetch_queue_capacity=100)
      batch = batch_queue.dequeue()

      for tensor_dict in batch:
        for tensor in tensor_dict.values():
          self.assertAllEqual([None, None], tensor.get_shape().as_list())

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        i = 2
        for _ in range(num_batches):
          batch_np = sess.run(batch)
          for tensor_dict in batch_np:
            for tensor in tensor_dict.values():
              self.assertAllEqual(tensor, np.arange(i * i).reshape((i, i)))
              i += 1
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(batch) 

def test_batch_and_unpad_2d_tensors_of_same_size_in_all_dimensions(self):
    with self.test_session() as sess:
      batch_size = 3
      num_batches = 2
      examples = tf.Variable(tf.constant(1, dtype=tf.int32))
      counter = examples.count_up_to(num_batches * batch_size + 1)
      image = tf.reshape(tf.range(1, 13), [4, 3]) * counter
      batch_queue = batcher.BatchQueue(
          tensor_dict={'image': image},
          batch_size=batch_size,
          batch_queue_capacity=100,
          num_batch_queue_threads=1,
          prefetch_queue_capacity=100)
      batch = batch_queue.dequeue()

      for tensor_dict in batch:
        for tensor in tensor_dict.values():
          self.assertAllEqual([4, 3], tensor.get_shape().as_list())

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        i = 1
        for _ in range(num_batches):
          batch_np = sess.run(batch)
          for tensor_dict in batch_np:
            for tensor in tensor_dict.values():
              self.assertAllEqual(tensor, np.arange(1, 13).reshape((4, 3)) * i)
              i += 1
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(batch) 

def test_batch_and_unpad_2d_tensors_of_different_sizes_in_all_dimensions(
      self):
    with self.test_session() as sess:
      batch_size = 3
      num_batches = 2
      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
      counter = examples.count_up_to(num_batches * batch_size + 2)
      image = tf.reshape(
          tf.range(counter * counter), tf.stack([counter, counter]))
      batch_queue = batcher.BatchQueue(
          tensor_dict={'image': image},
          batch_size=batch_size,
          batch_queue_capacity=100,
          num_batch_queue_threads=1,
          prefetch_queue_capacity=100)
      batch = batch_queue.dequeue()

      for tensor_dict in batch:
        for tensor in tensor_dict.values():
          self.assertAllEqual([None, None], tensor.get_shape().as_list())

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        i = 2
        for _ in range(num_batches):
          batch_np = sess.run(batch)
          for tensor_dict in batch_np:
            for tensor in tensor_dict.values():
              self.assertAllEqual(tensor, np.arange(i * i).reshape((i, i)))
              i += 1
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(batch) 

def test_batch_and_unpad_2d_tensors_of_different_sizes_in_1st_dimension(self):
    with self.test_session() as sess:
      batch_size = 3
      num_batches = 2
      examples = tf.Variable(tf.constant(2, dtype=tf.int32))
      counter = examples.count_up_to(num_batches * batch_size + 2)
      boxes = tf.tile(
          tf.reshape(tf.range(4), [1, 4]), tf.stack([counter, tf.constant(1)]))
      batch_queue = batcher.BatchQueue(
          tensor_dict={'boxes': boxes},
          batch_size=batch_size,
          batch_queue_capacity=100,
          num_batch_queue_threads=1,
          prefetch_queue_capacity=100)
      batch = batch_queue.dequeue()

      for tensor_dict in batch:
        for tensor in tensor_dict.values():
          self.assertAllEqual([None, 4], tensor.get_shape().as_list())

      tf.initialize_all_variables().run()
      with slim.queues.QueueRunners(sess):
        i = 2
        for _ in range(num_batches):
          batch_np = sess.run(batch)
          for tensor_dict in batch_np:
            for tensor in tensor_dict.values():
              self.assertAllEqual(tensor, np.tile(np.arange(4), (i, 1)))
              i += 1
        with self.assertRaises(tf.errors.OutOfRangeError):
          sess.run(batch) 

def start(self, model, data, sess):
        self.logger.info('Initializing all variables')
        sess.run(tf.initialize_all_variables())
        self.logger.info("Skipping training") 

def initialize_graph(self, input_dim):
        self.input_dim = input_dim
        self._setup_base_graph()
        with self.graph.as_default():
            self.sess = tf.Session()
            self.init_op = tf.initialize_all_variables()
            self.summary = tf.merge_all_summaries()
            self.sess.run(self.init_op)
        self.initialized = True 

def train(self):

	self.train_op = self.optim.minimize(self.loss, global_step=self.global_step, var_list=self.D_params_train)
	#self.train_op = self.optim.minimize(self.loss, global_step=self.global_step)
        self.writer = tf.train.SummaryWriter("./logs/D_CNN_pretrained_sample", self.sess.graph)
        tf.initialize_all_variables().run()
        self.saver = tf.train.Saver(var_list=self.D_params_dict, max_to_keep=30)
	# assign the G matrix to D pretrain
	self.sess.run(self.embedding_assign_op)
        count = 0
	for idx in range(self.max_iter//3000):
            self.save(self.checkpoint_dir, count)
            self.evaluate('test', count)
	    self.evaluate('train', count)
            for k in tqdm(range(3000)):
		right_images, right_text, _ = self.dataset.sequential_sample(self.batch_size)
                fake_images, fake_text, _ = self.negative_dataset.sequential_sample(self.batch_size)
                wrong_text = self.dataset.get_wrong_text(self.batch_size)

		images = np.concatenate((right_images, right_images, fake_images), axis=0)
		text = np.concatenate((right_text, wrong_text, fake_text.astype('int32')), axis=0)
		label = np.zeros((text.shape[0], self.num_classes))
		# right -> first entry
		# wrong -> second entry
		# fake -> third entry
		label[:self.batch_size, 0] = 1
		label[self.batch_size:2*self.batch_size, 1] = 1
		label[2*self.batch_size:, 2] = 1
                _, loss, summary_str = self.sess.run([self.train_op, self.loss, self.loss_sum],{
                                self.text: text.astype('int32'),
				self.images: images, 
                                self.label: label
                                })
                self.writer.add_summary(summary_str, count)
                count += 1 

def main():
  net = build_vgg19(VGG_MODEL)
  sess = tf.Session()
  sess.run(tf.initialize_all_variables())
  noise_img = np.random.uniform(-20, 20, (1, IMAGE_H, IMAGE_W, 3)).astype('float32')
  content_img = read_image(CONTENT_IMG)
  style_img = read_image(STYLE_IMG)

  sess.run([net['input'].assign(content_img)])
  cost_content = sum(map(lambda l,: l[1]*build_content_loss(sess.run(net[l[0]]) ,  net[l[0]])
    , CONTENT_LAYERS))

  sess.run([net['input'].assign(style_img)])
  cost_style = sum(map(lambda l: l[1]*build_style_loss(sess.run(net[l[0]]) ,  net[l[0]])
    , STYLE_LAYERS))

  cost_total = cost_content + STYLE_STRENGTH * cost_style
  optimizer = tf.train.AdamOptimizer(2.0)

  train = optimizer.minimize(cost_total)
  sess.run(tf.initialize_all_variables())
  sess.run(net['input'].assign( INI_NOISE_RATIO* noise_img + (1.-INI_NOISE_RATIO) * content_img))

  if not os.path.exists(OUTOUT_DIR):
      os.mkdir(OUTOUT_DIR)

  for i in range(ITERATION):
    sess.run(train)
    if i%100 ==0:
      result_img = sess.run(net['input'])
      print sess.run(cost_total)
      write_image(os.path.join(OUTOUT_DIR,'%s.png'%(str(i).zfill(4))),result_img)
  
  write_image(os.path.join(OUTOUT_DIR,OUTPUT_IMG),result_img) 

def export_intermediate(FLAGS, sess, dataset):
    # Models
    x = tf.placeholder(tf.float32, shape=[
        None, IMAGE_SIZE['resized'][0], IMAGE_SIZE['resized'][1], 3])
    dropout = tf.placeholder(tf.float32)
    feat_model = discriminator(x, reuse=False, dropout=dropout, int_feats=True)

    # Init
    init_op = tf.group(tf.initialize_all_variables(),
                       tf.initialize_local_variables())
    sess.run(init_op)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord)

    # Restore
    saver = tf.train.Saver()
    checkpoint = tf.train.latest_checkpoint(FLAGS.logdir)
    saver.restore(sess, checkpoint)

    # Run
    all_features = np.zeros((dataset['size'], feat_model.get_shape()[1]))
    all_paths = []
    for i in itertools.count():
        try:
            images, paths = sess.run(dataset['batch'])
        except tf.errors.OutOfRangeError:
            break
        if i % 10 == 0:
            print(i * FLAGS.batch_size, dataset['size'])
        im_features = sess.run(feat_model, feed_dict={x: images, dropout: 1, })
        all_features[FLAGS.batch_size * i:FLAGS.batch_size * i + im_features.shape[0]] = im_features
        all_paths += list(paths)

    # Finish off the filename queue coordinator.
    coord.request_stop()
    coord.join(threads)

    return all_features, all_paths