Python gflags.FLAGS Examples

def run_main():
  """Function to be used as setuptools script entry point.

  Appcommands assumes that it always runs as __main__, but launching
  via a setuptools-generated entry_point breaks this rule. We do some
  trickery here to make sure that appcommands and flags find their
  state where they expect to by faking ourselves as __main__.
  """

  # Put the flags for this module somewhere the flags module will look
  # for them.
  # pylint: disable=protected-access
  new_name = sys.argv[0]
  sys.modules[new_name] = sys.modules['__main__']
  for flag in FLAGS.FlagsByModuleDict().get(__name__, []):
    FLAGS._RegisterFlagByModule(new_name, flag)
    for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []):
      FLAGS._RegisterKeyFlagForModule(new_name, key_flag)
  # pylint: enable=protected-access

  # Now set __main__ appropriately so that appcommands will be
  # happy.
  sys.modules['__main__'] = sys.modules[__name__]
  appcommands.Run()
  sys.modules['__main__'] = sys.modules.pop(new_name) 

def build_cost(logits, targets):
    """
    Build a classification cost function.
    """
    # Clip gradients coming from the cost function.
    logits = theano.gradient.grad_clip(
        logits, -1. * FLAGS.clipping_max_value, FLAGS.clipping_max_value)

    predicted_dist = T.nnet.softmax(logits)

    costs = T.nnet.categorical_crossentropy(predicted_dist, targets)
    cost = costs.mean()

    pred = T.argmax(logits, axis=1)
    acc = 1. - T.mean(T.cast(T.neq(pred, targets), theano.config.floatX))

    return cost, acc 

def evaluate(eval_fn, eval_set, logger, step):
    # Evaluate
    acc_accum = 0.0
    action_acc_accum = 0.0
    eval_batches = 0.0
    for (eval_X_batch, eval_transitions_batch, eval_y_batch, eval_num_transitions_batch) in eval_set[1]:
        acc_value, action_acc_value = eval_fn(
            eval_X_batch, eval_transitions_batch,
            eval_y_batch, eval_num_transitions_batch, 0.0,  # Eval mode: Don't apply dropout.
            int(FLAGS.allow_gt_transitions_in_eval),  # Allow GT transitions to be used according to flag.
            float(FLAGS.allow_gt_transitions_in_eval))  # If flag not set, used scheduled sampling
                                                        # p(ground truth) = 0.0,
                                                        # else SS p(ground truth) = 1.0
        acc_accum += acc_value
        action_acc_accum += action_acc_value
        eval_batches += 1.0
    logger.Log("Step: %i\tEval acc: %f\t %f\t%s" %
              (step, acc_accum / eval_batches, action_acc_accum / eval_batches, eval_set[0]))
    return acc_accum / eval_batches 

def ShowPlots(subplot=False):
  for log_ind, path in enumerate(FLAGS.path.split(":")):
    log = Log(path)
    if subplot:
      plt.subplot(len(FLAGS.path.split(":")), 1, log_ind + 1)
    for index in FLAGS.index.split(","):
      index = int(index)
      for attr in ["pred_acc", "parse_acc", "total_cost", "xent_cost", "l2_cost", "action_cost"]:
        if getattr(FLAGS, attr):
          if "cost" in attr:
            assert index == 0, "costs only associated with training log"
          steps, val = zip(*[(l.step, getattr(l, attr)) for l in log.corpus[index] if l.step < FLAGS.iters])
          dct = {}
          for k, v in zip(steps, val):
            dct[k] = max(v, dct[k]) if k in dct else v
          steps, val = zip(*sorted(dct.iteritems()))
          plt.plot(steps, val, label="Log%d:%s-%d" % (log_ind, attr, index))
    
  plt.xlabel("No. of training iteration")
  plt.ylabel(FLAGS.ylabel)
  if FLAGS.legend:
    plt.legend()
  plt.show() 

def getExcludeFilename(prefix=None, infile=None, log_file=None, job=None):
    try:
        if not prefix:
            prefix = 'exclude'
        if not infile:
            infile = FLAGS.infile
        if not log_file:
            log_file = FLAGS.log_file
        if not job:
            job = FLAGS.job
        # the exclude file is stored as ./log/$prefix-$job_name-$in_filename
        inname = os.path.basename(infile)
        log_dir = os.path.dirname(log_file)
        exclude_filename = os.path.join(log_dir, '%s-%s-%s' % (prefix, job, inname))
    except Exception as e:
        exclude_filename = None
    return exclude_filename 

def __init__(self, external_file=None):
    """Initialize the error fixer.

    Args:
      external_file: If included, all output will be directed to this file
          instead of overwriting the files the errors are found in.
    """
    errorhandler.ErrorHandler.__init__(self)

    self._file_name = None
    self._file_token = None
    self._external_file = external_file

    try:
      self._fix_error_codes = set([errors.ByName(error.upper()) for error in
                                   FLAGS.fix_error_codes])
    except KeyError as ke:
      raise ValueError('Unknown error code ' + ke.args[0]) 

def _GetRecursiveFiles(suffixes):
  """Returns files to be checked specified by the --recurse flag.

  Args:
    suffixes: Expected suffixes for the file type being checked.

  Returns:
    A list of files to be checked.
  """
  lint_files = []
  # Perform any request recursion
  if FLAGS.recurse:
    for start in FLAGS.recurse:
      for root, subdirs, files in os.walk(start):
        for f in files:
          if MatchesSuffixes(f, suffixes):
            lint_files.append(os.path.join(root, f))
  return lint_files 

def build_cost(logits, targets):
    """
    Build a classification cost function.
    """
    # Clip gradients coming from the cost function.
    logits = theano.gradient.grad_clip(
        logits, -1. * FLAGS.clipping_max_value, FLAGS.clipping_max_value)

    predicted_dist = T.nnet.softmax(logits)

    costs = T.nnet.categorical_crossentropy(predicted_dist, targets)
    cost = costs.mean()

    pred = T.argmax(logits, axis=1)
    acc = 1. - T.mean(T.cast(T.neq(pred, targets), theano.config.floatX))

    return cost, acc 

def GetAllSpecifiedFiles(argv, suffixes):
  """Returns all files specified by the user on the commandline.

  Args:
    argv: Sequence of command line arguments. The second and following arguments
      are assumed to be files that should be linted.
    suffixes: Expected suffixes for the file type

  Returns:
    A list of all files specified directly or indirectly (via flags) on the
    command line by the user.
  """
  files = _GetUserSpecifiedFiles(argv, suffixes)

  if FLAGS.recurse:
    files += _GetRecursiveFiles(suffixes)

  return FilterFiles(files) 

def MakeErrorRecord(path, error):
  """Make an error record with correctly formatted error string.

  Errors are not able to be serialized (pickled) over processes because of
  their pointers to the complex token/context graph.  We use an intermediary
  serializable class to pass back just the relevant information.

  Args:
    path: Path of file the error was found in.
    error: An error.Error instance.

  Returns:
    _ErrorRecord instance.
  """
  new_error = error.code in errors.NEW_ERRORS

  if FLAGS.unix_mode:
    error_string = erroroutput.GetUnixErrorOutput(
        path, error, new_error=new_error)
  else:
    error_string = erroroutput.GetErrorOutput(error, new_error=new_error)

  return ErrorRecord(path, error_string, new_error) 

def ShouldCheck(rule):
  """Returns whether the optional rule should be checked.

  Computes different flags (strict, jslint_error, jslint_noerror) to find out if
  this specific rule should be checked.

  Args:
    rule: Name of the rule (see Rule).

  Returns:
    True if the rule should be checked according to the flags, otherwise False.
  """
  if 'no_' + rule in FLAGS.jslint_error:
    return False
  if rule in FLAGS.jslint_error or Rule.ALL in FLAGS.jslint_error:
    return True
  # Checks strict rules.
  return FLAGS.strict and rule in Rule.CLOSURE_RULES 

def process_flags(flags=[]):
    """Uses the command-line flags to set the logging level.

    Args:
    argv: List of command line arguments passed to the python script.
    """

    # Let the gflags module process the command-line arguments.
    try:
        FLAGS(flags)
    except gflags.FlagsError as e:
        print('%s\nUsage: %s ARGS\n%s' % (e, str(flags), FLAGS))
        sys.exit(1)

    # Set the logging according to the command-line flag.
    logging.getLogger().setLevel(getattr(logging, FLAGS.logging_level)) 

def test_defeat_zerglings(self):
    FLAGS(sys.argv)

    with sc2_env.SC2Env(
        "DefeatZerglingsAndBanelings",
        step_mul=self.step_mul,
        visualize=True,
        game_steps_per_episode=self.steps * self.step_mul) as env:
      obs = env.step(actions=[sc2_actions.FunctionCall(_NO_OP, [])])
      player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]

      # Break Point!!
      print(player_relative)

      agent = random_agent.RandomAgent()
      run_loop.run_loop([agent], env, self.steps)

    self.assertEqual(agent.steps, self.steps) 

def evaluate(eval_fn, eval_set, logger, step, zero_fn):
    # Evaluate
    acc_accum = 0.0
    action_acc_accum = 0.0
    eval_batches = 0.0
    for (eval_X_batch, eval_transitions_batch, eval_y_batch, eval_num_transitions_batch) in eval_set[1]:
        acc_value, action_acc_value = eval_fn(
            eval_X_batch, eval_transitions_batch,
            eval_y_batch, eval_num_transitions_batch, 0.0,  # Eval mode: Don't apply dropout.
            int(FLAGS.allow_gt_transitions_in_eval),  # Allow GT transitions to be used according to flag.
            float(FLAGS.allow_gt_transitions_in_eval))  # If flag not set, used scheduled sampling
                                                        # p(ground truth) = 0.0,
                                                        # else SS p(ground truth) = 1.0
        acc_accum += acc_value
        action_acc_accum += action_acc_value
        eval_batches += 1.0

        # Zero out all auxiliary variables.
        zero_fn()
    logger.Log("Step: %i\tEval acc: %f\t %f\t%s" %
              (step, acc_accum / eval_batches, action_acc_accum / eval_batches, eval_set[0]))
    return acc_accum / eval_batches 

def main():
  try:
    gflags.FLAGS(sys.argv)
  except gflags.FlagsError as e:
    print '%s\nUsage: %s ARGS\n%s' % (e, sys.argv[0], gflags.FLAGS)
    sys.exit(1)
  logging.basicConfig(level=logging.INFO)

  if gflags.FLAGS.mindmap_file is None:
    print 'Usage: %s ARGS\n%s' % (sys.argv[0], gflags.FLAGS)
    sys.exit(1)

  org = Organization(
    codecs.open(gflags.FLAGS.mindmap_file, 'r', 'utf8').read())
  if gflags.FLAGS.html_file is not None:
    org.OutputToHTML(gflags.FLAGS.html_file)

  if gflags.FLAGS.beamer_latex_file is not None:
    org.OutputToBeamerLatex(gflags.FLAGS.beamer_latex_file)

  if gflags.FLAGS.latex_file is not None:
    org.OutputToLatex(gflags.FLAGS.latex_file) 

def pull_profile_and_essays(username):
    '''Given a username, fetches the profile page and parses it.'''
    url = PROFILE_URL % username
    print "Fetching profile HTML for", username + "... ",
    html = None
    for i in range(3):
        if i > 0:
            print "Retrying...",
        try:
            r = requests.get(url, cookies={'session': FLAGS.session_cookie})
            if r.status_code != 200:
                print "Error, HTTP status code was %d, not 200" % r.status_code
            else:
                html = r.content
                break
        except requests.exceptions.RequestException, e:
            print "Error completing HTTP request:", e 

def _GetRunFlowFlags(args=None):
    """Retrieves command line flags based on gflags module."""
    # There's one rare situation where gsutil will not have argparse
    # available, but doesn't need anything depending on argparse anyway,
    # since they're bringing their own credentials. So we just allow this
    # to fail with an ImportError in those cases.
    #
    parser = argparse.ArgumentParser(parents=[tools.argparser])
    # Get command line argparse flags.
    flags, _ = parser.parse_known_args(args=args)

    # Allow `gflags` and `argparse` to be used side-by-side.
    if hasattr(FLAGS, 'auth_host_name'):
        flags.auth_host_name = FLAGS.auth_host_name
    if hasattr(FLAGS, 'auth_host_port'):
        flags.auth_host_port = FLAGS.auth_host_port
    if hasattr(FLAGS, 'auth_local_webserver'):
        flags.noauth_local_webserver = (not FLAGS.auth_local_webserver)
    return flags


# TODO(craigcitro): Switch this from taking a path to taking a stream. 

def map(self, key, value):
        if type(value) is not str:
            value = str(value)
        files = [value]
        image_lib = FLAGS.image_lib.lower()
        if image_lib == 'pil':
            resize_crop = PILResizeCrop()
        else:
            resize_crop = OpenCVResizeCrop()
        for i, line in enumerate(files):
            try:
                line = line.replace(FLAGS.input_folder, '').strip()
                line = line.split()
                image_file_name = line[0]
                input_file = os.path.join(FLAGS.input_folder, image_file_name)
                output_file = os.path.join(FLAGS.output_folder, image_file_name)
                output_dir = output_file[:output_file.rfind('/')]
                if not os.path.exists(output_dir):
                    os.makedirs(output_dir)
                feat = resize_crop.resize_and_crop_image(input_file, output_file,
                                                              FLAGS.output_side_length)
            except Exception, e:
                # we ignore the exception (maybe the image is corrupted?)
                print line, Exception, e 

def map(self, key, value):
        if type(value) is not str:
            value = str(value)
        files = [value]
        image_lib = FLAGS.image_lib.lower()
        if image_lib == 'pil':
            resize_crop = PILResizeCrop()
        else:
            resize_crop = OpenCVResizeCrop()
        for i, line in enumerate(files):
            try:
                line = line.replace(FLAGS.input_folder, '').strip()
                line = line.split()
                image_file_name = line[0]
                input_file = os.path.join(FLAGS.input_folder, image_file_name)
                output_file = os.path.join(FLAGS.output_folder, image_file_name)
                output_dir = output_file[:output_file.rfind('/')]
                if not os.path.exists(output_dir):
                    os.makedirs(output_dir)
                feat = resize_crop.resize_and_crop_image(input_file, output_file,
                                                              FLAGS.output_side_length)
            except Exception, e:
                # we ignore the exception (maybe the image is corrupted?)
                print line, Exception, e 

def ParseFlagsWithUsage(argv):
  """Parse the flags, exiting (after printing usage) if they are unparseable.

  Args:
    argv: command line arguments

  Returns:
    remaining command line arguments after parsing flags
  """
  # Update the global commands.
  # pylint: disable=global-statement
  global _cmd_argv
  try:
    _cmd_argv = FLAGS(argv)
    return _cmd_argv
  except flags.FlagsError, error:
    ShortHelpAndExit('FATAL Flags parsing error: %s' % error) 

def testFlags(self):
    if FLAGS.testid == 1:
      self.assertEqual(FLAGS.test_random_seed, 301)
      self.assert_(FLAGS.test_tmpdir.startswith('/'))
      self.assert_(os.access(FLAGS.test_tmpdir, os.W_OK))
    elif FLAGS.testid == 2:
      self.assertEqual(FLAGS.test_random_seed, 321)
      self.assertEqual(FLAGS.test_srcdir, 'cba')
      self.assertEqual(FLAGS.test_tmpdir, 'fed')
    elif FLAGS.testid == 3:
      self.assertEqual(FLAGS.test_random_seed, 123)
      self.assertEqual(FLAGS.test_srcdir, 'abc')
      self.assertEqual(FLAGS.test_tmpdir, 'def')
    elif FLAGS.testid == 4:
      self.assertEqual(FLAGS.test_random_seed, 123)
      self.assertEqual(FLAGS.test_srcdir, 'abc')
      self.assertEqual(FLAGS.test_tmpdir, 'def') 

def Run(self, unused_argv):
    """Output 'Command1' and flag info.

    Args:
      unused_argv: Remaining arguments after parsing flags and command

    Returns:
      Value of flag fail1
    """
    print 'Command1'
    if FLAGS.hint:
      print "Hint1:'%s'" % FLAGS.hint
    print "Foo1:'%s'" % FLAGS.foo
    print "Bar1:'%s'" % FLAGS.bar
    if FLAGS.allhelp:
      print "AllHelp:'%s'" % self._all_commands_help
    return FLAGS.fail1 * 1 

def testRmDirs(self):
    test_sandbox = os.path.join(FLAGS.test_tmpdir, 'test-rm-dirs')
    test_dir = os.path.join(test_sandbox, 'test', 'dir')

    os.makedirs(test_sandbox)
    with open(os.path.join(test_sandbox, 'file'), 'w'):
      pass
    os.makedirs(test_dir)
    with open(os.path.join(test_dir, 'file'), 'w'):
      pass

    file_util.RmDirs(test_dir)

    self.assertFalse(os.path.exists(os.path.join(test_sandbox, 'test')))
    self.assertTrue(os.path.exists(os.path.join(test_sandbox, 'file')))

    shutil.rmtree(test_sandbox) 

def map(self, key, value):
        if type(value) is not str:
            value = str(value)
        files = [value]
        image_lib = FLAGS.image_lib.lower()
        if image_lib == 'pil':
            resize_crop = PILResizeCrop()
        else:
            resize_crop = OpenCVResizeCrop()
        for i, line in enumerate(files):
            try:
                line = line.replace(FLAGS.input_folder, '').strip()
                line = line.split()
                image_file_name = line[0]
                input_file = os.path.join(FLAGS.input_folder, image_file_name)
                output_file = os.path.join(FLAGS.output_folder, image_file_name)
                output_dir = output_file[:output_file.rfind('/')]
                if not os.path.exists(output_dir):
                    os.makedirs(output_dir)
                feat = resize_crop.resize_and_crop_image(input_file, output_file,
                                                              FLAGS.output_side_length)
            except Exception, e:
                # we ignore the exception (maybe the image is corrupted?)
                print line, Exception, e 

def _SetupTestFlags(self, **kwargs):
    defaults = {
        'skip_leading_rows': None,
        'allow_quoted_newlines': None,
    }
    defaults.update(kwargs)
    for k, v in defaults.iteritems():
      self.stubs.Set(load_lib.FLAGS, k, v) 

def setUp(self):
    """Run once for each test in the class."""
    self.mox = mox.Mox()
    self.stubs = stubout.StubOutForTesting()
    os.environ['TMPDIR'] = FLAGS.test_tmpdir
    self.dirname = tempfile.mkdtemp() 

def DiffTestStringFile(data, golden):
  """Diff data agains a golden file."""
  data_file = os.path.join(FLAGS.test_tmpdir, 'provided.dat')
  _WriteTestData(data, data_file)
  _Diff(data_file, golden) 

def CaptureTestStderr(outfile='', expected_output_filepath=None):
  """Capture the stderr stream to a file.

  If expected_output_filepath, then this function returns a context manager
  that stops capturing and performs a diff when the context is exited.

    with basetest.CaptureTestStderr(expected_output_filepath=some_filepath):
      sys.stderr.print(....)

  Otherwise, StopCapturing() must be called to stop capturing stderr, and then
  DiffTestStderr() must be called to do the comparison.

  Args:
    outfile: The path to the local filesystem file to which to capture output;
        if omitted, a standard filepath in --test_tmpdir will be used.
    expected_output_filepath: The path to the local filesystem file containing
        the expected output, to be diffed against when the context is exited.
  Returns:
    A context manager if expected_output_filepath is specified, otherwise
        None.
  """
  if not outfile:
    outfile = os.path.join(FLAGS.test_tmpdir, 'captured.err')
    outdir = FLAGS.test_tmpdir
  else:
    outdir = os.path.dirname(outfile)
  _MaybeNotifyAboutTestOutput(outdir)
  _CaptureTestOutput(sys.stderr, outfile)
  if expected_output_filepath is not None:
    return _DiffingTestOutputContext(
        lambda: DiffTestStderr(expected_output_filepath)) 

def CaptureTestStdout(outfile='', expected_output_filepath=None):
  """Capture the stdout stream to a file.

  If expected_output_filepath, then this function returns a context manager
  that stops capturing and performs a diff when the context is exited.

    with basetest.CaptureTestStdout(expected_output_filepath=some_filepath):
      sys.stdout.print(....)

  Otherwise, StopCapturing() must be called to stop capturing stdout, and then
  DiffTestStdout() must be called to do the comparison.

  Args:
    outfile: The path to the local filesystem file to which to capture output;
        if omitted, a standard filepath in --test_tmpdir will be used.
    expected_output_filepath: The path to the local filesystem file containing
        the expected output to be diffed against when the context is exited.
  Returns:
    A context manager if expected_output_filepath is specified, otherwise
        None.
  """
  if not outfile:
    outfile = os.path.join(FLAGS.test_tmpdir, 'captured.out')
    outdir = FLAGS.test_tmpdir
  else:
    outdir = os.path.dirname(outfile)
  _MaybeNotifyAboutTestOutput(outdir)
  _CaptureTestOutput(sys.stdout, outfile)
  if expected_output_filepath is not None:
    return _DiffingTestOutputContext(
        lambda: DiffTestStdout(expected_output_filepath)) 

def parse_flags_with_usage(args):
  """Try parsing the flags, printing usage and exiting if unparseable."""
  try:
    argv = FLAGS(args)
    return argv
  except flags.FlagsError, error:
    sys.stderr.write('FATAL Flags parsing error: %s\n' % error)
    sys.stderr.write('Pass --helpshort or --helpfull to see help on flags.\n')
    sys.exit(1)