Python logging.basicConfig() Examples

def init_logging(log_file=None, append=False, console_loglevel=logging.INFO):
    """Set up logging to file and console."""
    if log_file is not None:
        if append:
            filemode_val = 'a'
        else:
            filemode_val = 'w'
        logging.basicConfig(level=logging.DEBUG,
                            format="%(asctime)s %(levelname)s %(threadName)s %(name)s %(message)s",
                            # datefmt='%m-%d %H:%M',
                            filename=log_file,
                            filemode=filemode_val)
    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(console_loglevel)
    # set a format which is simpler for console use
    formatter = logging.Formatter("%(message)s")
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)
    global LOG
    LOG = logging.getLogger(__name__) 

def parse_args():
    parser = argparse.ArgumentParser(description = "Bass")
    parser.add_argument("-v", "--verbose", action = "count", default = 0, help = "Increase verbosity")
    parser.add_argument("samples", metavar = "sample", nargs = "+", help = "Sample path") 

    args = parser.parse_args()

    try:
        loglevel = {
            0: logging.ERROR,
            1: logging.WARN,
            2: logging.INFO
        }[args.verbose]
    except KeyError:
        loglevel = logging.DEBUG

    logging.basicConfig(level = loglevel)
    logging.getLogger().setLevel(loglevel)

    return args 

def parse_args():
    parser = argparse.ArgumentParser(description = "Find common ngrams in binary files")
    parser.add_argument("-v", "--verbose", action = "count", default = 0, help = "Increase verbosity")
    parser.add_argument("--output", type = str, default = None, help = "Output to file instead of stdout")
    parser.add_argument("--url", type = str, default = "http://localhost:5000", help = "URL of BASS server")
    parser.add_argument("samples", metavar = "sample", nargs = "+", help = "Cluster samples")

    args = parser.parse_args()

    try:
        loglevel = {
            0: logging.ERROR,
            1: logging.WARN,
            2: logging.INFO}[args.verbose]
    except KeyError:
        loglevel = logging.DEBUG
    logging.basicConfig(level = loglevel)
    logging.getLogger().setLevel(loglevel)

    return args 

def parse_args():
    parser = argparse.ArgumentParser(description = "Add samples to BASS whitelist")
    parser.add_argument("-v", "--verbose", action = "count", default = 0, help = "Increase verbosity")
    parser.add_argument("--url", type = str, default = "http://localhost:5000", help = "URL of BASS server")
    parser.add_argument("sample", help = "Whitelist sample")

    args = parser.parse_args()

    try:
        loglevel = {
            0: logging.ERROR,
            1: logging.WARN,
            2: logging.INFO}[args.verbose]
    except KeyError:
        loglevel = logging.DEBUG
    logging.basicConfig(level = loglevel)
    logging.getLogger().setLevel(loglevel)

    return args 

def log_config(log_dir=None, log_file=None, prefix=None, rank=0):
    reload(logging)
    head = '%(asctime)-15s Node[' + str(rank) + '] %(message)s'
    if log_dir:
        logging.basicConfig(level=logging.DEBUG, format=head)
        if not os.path.exists(log_dir):
            os.makedirs(log_dir)
        if not log_file:
            log_file = (prefix if prefix else '') + datetime.now().strftime('_%Y_%m_%d-%H_%M.log')
            log_file = log_file.replace('/', '-')
        else:
            log_file = log_file
        log_file_full_name = os.path.join(log_dir, log_file)
        handler = logging.FileHandler(log_file_full_name, mode='w')
        formatter = logging.Formatter(head)
        handler.setFormatter(formatter)
        logging.getLogger().addHandler(handler)
        logging.info('start with arguments %s', args)
    else:
        logging.basicConfig(level=logging.DEBUG, format=head)
        logging.info('start with arguments %s', args) 

def test_log(app):
    log_stream = StringIO()
    for handler in logging.root.handlers[:]:
        logging.root.removeHandler(handler)
    logging.basicConfig(
        format=logging_format, level=logging.DEBUG, stream=log_stream
    )
    log = logging.getLogger()
    rand_string = str(uuid.uuid4())

    @app.route("/")
    def handler(request):
        log.info(rand_string)
        return text("hello")

    request, response = app.test_client.get("/")
    log_text = log_stream.getvalue()
    assert rand_string in log_text 

def setup_logging(args):
    """Setup the logging level and configure the basic logger
    """
    if args.verbose == 1:
        level = logging.INFO
    elif args.verbose >= 2:
        level = logging.DEBUG
    else:
        level = logging.WARN

    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(message)s",
        level=level,
    )

    global LOG
    LOG = logging.getLogger(__name__) 

def main(argv):
    logging.basicConfig(level=logging.INFO)

    log_history = LogHistory(argv.config, argv.flow_dir, argv.port_dir, int(argv.num_steps), debug=True)

    channel = "sdx_stats"
    address = "192.168.99.100"
    port = 6379
    db = 0

    publisher = Publisher(channel, address, port)

    log_replay = LogReplay(log_history, publisher, int(argv.timestep), debug=True)

    # start replay
    replay_thread = Thread(target=log_replay.start)
    replay_thread.daemon = True
    replay_thread.start()

    while replay_thread.is_alive():
        try:
            replay_thread.join(1)
        except KeyboardInterrupt:
            log_replay.stop() 

def lambda_handler(event, context):
    logger = logging.getLogger()

    if logger.handlers:
        for handler in logger.handlers:
            logger.removeHandler(handler)

    # change logging levels for boto and others
    logging.getLogger("boto3").setLevel(logging.ERROR)
    logging.getLogger("botocore").setLevel(logging.ERROR)
    logging.getLogger("urllib3").setLevel(logging.ERROR)

    # set logging format
    logging.basicConfig(
        format="[%(levelname)s] %(message)s (%(filename)s, %(funcName)s(), line %(lineno)d)",
        level=os.environ.get("LOGLEVEL", "WARNING").upper(),
    )

    # instantiate class
    retry = Retry(logging)

    # run functions
    retry.retry_security_events() 

def _controller(self):
        """Return the server controller."""
        def server_controller(cmd_id, cmd_body, _):
            """Server controler."""
            if not self.init_logginig:
                # the reason put the codes here is because we cannot get
                # kvstore.rank earlier
                head = '%(asctime)-15s Server[' + str(
                    self.kvstore.rank) + '] %(message)s'
                logging.basicConfig(level=logging.DEBUG, format=head)
                self.init_logginig = True

            if cmd_id == 0:
                try:
                    optimizer = pickle.loads(cmd_body)
                except:
                    raise
                self.kvstore.set_optimizer(optimizer)
            else:
                print("server %d, unknown command (%d, %s)" % (
                    self.kvstore.rank, cmd_id, cmd_body))
        return server_controller 

def test_lenet(devs, kv_type):
    logging.basicConfig(level=logging.DEBUG)
    (train, val) = mnist(batch_size = batch_size, input_shape=(1,28,28))
    # guarantee the same weight init for each run
    mx.random.seed(0)
    model = mx.model.FeedForward(
        ctx           = devs,
        symbol        = net,
        num_epoch     = 2,
        learning_rate = 0.1,
        momentum      = 0.9,
        wd            = 0.00001)
    model.fit(
        kvstore       = kv_type,
        X             = train)
    return accuracy(model, val) 

def test_profile_task():
    def makeParams():
        objects = tuple('foo' for _ in range(50))
        template = ''.join('{%d}' % i for i in range(len(objects)))
        return template, objects

    def doLog():
        template, objects = makeParams()
        for _ in range(100000):
            logging.info(template.format(*objects))

    logging.basicConfig()
    enable_profiler('test_profile_task.json')
    python_domain = profiler.Domain('PythonDomain::test_profile_task')
    task = profiler.Task(python_domain, "test_profile_task")
    task.start()
    start = time.time()
    var = mx.nd.ones((1000, 500))
    doLog()
    var.asnumpy()
    stop = time.time()
    task.stop()
    print('run took: %.3f' % (stop - start))
    profiler.set_state('stop') 

def test_profile_frame():
    def makeParams():
        objects = tuple('foo' for _ in range(50))
        template = ''.join('{%d}' % i for i in range(len(objects)))
        return template, objects

    def doLog():
        template, objects = makeParams()
        for _ in range(100000):
            logging.info(template.format(*objects))

    logging.basicConfig()
    enable_profiler('test_profile_frame.json')
    python_domain = profiler.Domain('PythonDomain::test_profile_frame')
    frame = profiler.Frame(python_domain, "test_profile_frame")
    frame.start()
    start = time.time()
    var = mx.nd.ones((1000, 500))
    doLog()
    var.asnumpy()
    stop = time.time()
    frame.stop()
    print('run took: %.3f' % (stop - start))
    profiler.set_state('stop') 

def test_cifar10():
    # print logging by default
    logging.basicConfig(level=logging.DEBUG)
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)
    logging.getLogger('').addHandler(console)
    kv = mx.kvstore.create("local")
    # test float32 input
    (train, val) = get_iterator_float32(kv)
    run_cifar10(train, val, use_module=False)
    run_cifar10(train, val, use_module=True)

    # test legecay tuple in provide_data and provide_label
    run_cifar10(CustomDataIter(train), CustomDataIter(val), use_module=False)
    run_cifar10(CustomDataIter(train), CustomDataIter(val), use_module=True)

    # test uint8 input
    (train, val) = get_iterator_uint8(kv)
    run_cifar10(train, val, use_module=False)
    run_cifar10(train, val, use_module=True) 

def main():
    """ Entry point. """
    settings = ChatDumpSettings(__doc__)

    # define the console output verbosity
    default_format = '%(levelname)s:%(message)s'
    if settings.is_verbose:
        logging.basicConfig(format=default_format, level=logging.DEBUG)
    else:
        logging.basicConfig(format=default_format, level=logging.INFO)

    metadata = DumpMetadata(settings.out_file)

    # when user specified --continue
    try:
        if settings.is_incremental_mode and settings.last_message_id == -1:
            metadata.merge_into_settings(settings)
    except MetadataError as ex:
        sprint("ERROR: %s" % ex)
        sys.exit(1)

    exporter = _load_exporter(settings.exporter)

    sys.exit(TelegramDumper(os.path.basename(__file__), settings, metadata, exporter).run()) 

def get_logger(filename):
    logging.basicConfig(filename=f"{filename}", filemode='a', format="%(message)s")
    logging.warning(f"[{datetime.datetime.now()}] {'=' * 10}")

    def log(message, error=True):
        func = inspect.currentframe().f_back.f_code
        final_msg = "(%s:%i) %s" % (
            func.co_name,
            func.co_firstlineno,
            message
        )
        if error:
            logging.warning(final_msg)
            print(f"[ERROR] {final_msg}")
        else:
            print(final_msg)

    return log 

def main():
    # Initialize the logger
    logging.basicConfig()

    # Intialize from our arguments
    insightfinder = InsightfinderStore(*sys.argv[1:])

    current_chunk = 0

    # Get all the inputs
    for line in sys.stdin:
        if insightfinder.filter_string not in line:
            continue
        map_size = len(bytearray(json.dumps(insightfinder.metrics_map)))
        if map_size >= insightfinder.flush_kb * 1000:
            insightfinder.logger.debug("Flushing chunk number: " + str(current_chunk))
            insightfinder.send_metrics()
            current_chunk += 1
        insightfinder.append(line.strip())
    insightfinder.logger.debug("Flushing chunk number: " + str(current_chunk))
    insightfinder.send_metrics()
    insightfinder.save_grouping()
    insightfinder.logger.debug("Finished sending all chunks to InsightFinder") 

def init(args):
    # init logger
    log_format = '%(asctime)-10s: %(message)s'
    if args.log_file is not None and args.log_file != "":
        Path(args.log_file).parent.mkdir(parents=True, exist_ok=True)
        logging.basicConfig(level=logging.INFO, filename=args.log_file, filemode='w', format=log_format)
        logging.warning(f'This will get logged to file: {args.log_file}')
    else:
        logging.basicConfig(level=logging.INFO, format=log_format)

    # create output dir
    if args.output_dir.is_dir() and list(args.output_dir.iterdir()):
        logging.warning(f"Output directory ({args.output_dir}) already exists and is not empty!")
    assert 'bert' in args.output_dir.name, \
        '''Output dir name has to contain `bert` or `roberta` for AutoModel.from_pretrained to correctly infer the model type'''

    args.output_dir.mkdir(parents=True, exist_ok=True)

    # set random seeds
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed) 

def create(gearkey,gearsecret, appid="", args = {}):
    if 'debugmode' in args:
        logging.basicConfig(level=logging.INFO,
                        format='%(asctime)s %(levelname)-8s %(message)s',
                        datefmt='%d/%m/%Y %I:%M:%S %p',
                        )
    else:
        logging.basicConfig(level=logging.WARNING,
                        format='%(asctime)s %(levelname)-8s %(message)s',
                        datefmt='%d/%m/%Y %I:%M:%S %p',
                        )
    microgear.gearalias = args.get('alias',"")[0:16]
    if 'scope' in args:
        matchScope = re.match( r'^(\w+:[a-zA-Z\/]+,*)+$', args['scope'])
        if matchScope:
            microgear.scope = args["scope"]
        else:
            microgear.scope = ""
            logging.warning("Specify scope is not valid")

    microgear.gearkey = gearkey
    microgear.gearsecret = gearsecret
    microgear.appid = appid 

def main(argv):
    logging.basicConfig(level=logging.DEBUG)

    config = configparser.ConfigParser()
    config.read([
        os.path.join(os.path.dirname(__file__), "config.default.ini"),
        os.path.join(os.path.dirname(__file__), "config.ini"),
    ] + argv)

    bind = config.get("server", "bind")
    port = config.getint("server", "port")

    app = make_app(config)

    print("* Server name: ", config.get("server", "name"))
    print("* Base url: ", config.get("server", "base_url"))
    aiohttp.web.run_app(app, host=bind, port=port, access_log=None) 

def main():
    '''Entry-point function for the cookiejar Transaction Processor.'''
    try:
        # Setup logging for this class.
        logging.basicConfig()
        logging.getLogger().setLevel(logging.DEBUG)

        # Register the Transaction Handler and start it.
        processor = TransactionProcessor(url=DEFAULT_URL)
        sw_namespace = _hash(FAMILY_NAME.encode('utf-8'))[0:6]
        handler = CookieJarTransactionHandler(sw_namespace)
        processor.add_handler(handler)
        processor.start()
    except KeyboardInterrupt:
        pass
    except SystemExit as err:
        raise err
    except BaseException as err:
        traceback.print_exc(file=sys.stderr)
        sys.exit(1) 

def exec_mnist(model, train_dataiter, val_dataiter):
    # print logging by default
    logging.basicConfig(level=logging.DEBUG)
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)
    logging.getLogger('').addHandler(console)

    model.fit(X=train_dataiter,
              eval_data=val_dataiter)
    logging.info('Finish fit...')
    prob = model.predict(val_dataiter)
    logging.info('Finish predict...')
    val_dataiter.reset()
    y = np.concatenate([batch.label[0].asnumpy() for batch in val_dataiter]).astype('int')
    py = np.argmax(prob, axis=1)
    acc1 = float(np.sum(py == y)) / len(y)
    logging.info('final accuracy = %f', acc1)
    assert(acc1 > 0.94)

# run as a script 

def update(image_name):
    logging.basicConfig(level=logging.DEBUG)
    os.environ["PYWREN_LOGLEVEL"] = 'DEBUG'
    update_runtime(image_name) 

def main():
    # Read config
    config = ConfigParser()
    config.readfp(open(os.path.dirname(os.path.abspath(__file__)) + "/smtpd.cfg.default"))
    config.read(["smtpd.cfg",])

    # Configure the logger
    logging.basicConfig(level=getattr(logging, config["logging"]["log_level"].upper()),
            format='%(levelname)s: %(asctime)s %(message)s',
            datefmt='%m/%d/%Y %I:%M:%S %p')

    loop = asyncio.get_event_loop()

    # Init plugin manager
    plugin_manager = PluginManager(loop)

    logger.info("Starting smtpd on {}:{}".format(config["smtpd"]["host"], config["smtpd"]["port"]))
    cont = CustomIdentController(
            MailHandler(loop, config, plugin_manager),
            loop=loop,
            ident_hostname=config["smtpd"]["hostname"],
            ident=config["smtpd"]["ident"],
            hostname=config["smtpd"]["host"],
            port=config["smtpd"]["port"])
    cont.start()

    # Ugly but whatever, wait until the controller thread finishes (wtf why do they start a thread)
    threads = threading.enumerate()
    for thread in threads:
        if not threading.current_thread() == thread:
            thread.join()

    plugin_manager.stop_plugins() 

def setUp(self):
        self.outdir = tempfile.mkdtemp()
        logging.basicConfig(level=logging.DEBUG) 

def setUp(self):
        self.outdir = tempfile.mkdtemp()
        logging.basicConfig(level=logging.DEBUG) 

def delete(image_name):
    logging.basicConfig(level=logging.DEBUG)
    os.environ["PYWREN_LOGLEVEL"] = 'DEBUG'
    delete_runtime(image_name) 

def setUp(self):
        self.outdir = tempfile.mkdtemp()
        logging.basicConfig(level=logging.DEBUG) 

def setUp(self):
        self.outdir = tempfile.mkdtemp()
        logging.basicConfig(level=logging.DEBUG) 

def create(image_name, memory):
    logging.basicConfig(level=logging.DEBUG)
    os.environ["PYWREN_LOGLEVEL"] = 'DEBUG'
    create_runtime(image_name, memory=memory)