Python logging.config.getfloat() Examples

def iterate(self, data):
        for key in data:
            t = data[key]
            if torch.is_tensor(t):
                data[key] = t.to(self.device)
        tensor = data['tensor']
        outputs = pybenchmark.profile('inference')(self.inference)(tensor)
        height, width = data['image'].size()[1:3]
        loss = pybenchmark.profile('loss')(model.Loss(self.config, data, self.limbs_index, height, width))
        losses = [loss(**output) for output in outputs]
        losses_hparam = [{name: self.loss_hparam(i, name, l) for name, l in loss.items()} for i, loss in enumerate(losses)]
        loss_total = sum(sum(loss.values()) for loss in losses_hparam)
        self.optimizer.zero_grad()
        loss_total.backward()
        try:
            clip = self.config.getfloat('train', 'clip')
            nn.utils.clip_grad_norm(self.inference.parameters(), clip)
        except configparser.NoOptionError:
            pass
        self.optimizer.step()
        return dict(
            height=height, width=width,
            data=data, outputs=outputs,
            loss_total=loss_total, losses=losses, losses_hparam=losses_hparam,
        ) 

def __init__(self, env):
        super(SummaryWorker, self).__init__()
        self.env = env
        self.config = env.config
        self.queue = multiprocessing.Queue()
        try:
            self.timer_scalar = utils.train.Timer(env.config.getfloat('summary', 'scalar'))
        except configparser.NoOptionError:
            self.timer_scalar = lambda: False
        try:
            self.timer_image = utils.train.Timer(env.config.getfloat('summary', 'image'))
        except configparser.NoOptionError:
            self.timer_image = lambda: False
        try:
            self.timer_histogram = utils.train.Timer(env.config.getfloat('summary', 'histogram'))
        except configparser.NoOptionError:
            self.timer_histogram = lambda: False
        with open(os.path.expanduser(os.path.expandvars(env.config.get('summary_histogram', 'parameters'))), 'r') as f:
            self.histogram_parameters = utils.RegexList([line.rstrip() for line in f])
        self.draw_bbox = utils.visualize.DrawBBox(env.category)
        self.draw_feature = utils.visualize.DrawFeature() 

def draw_bbox_iou(self, canvas_share, yx_min, yx_max, cls, iou, rows, cols, colors=None):
        batch_size = len(canvas_share)
        yx_min, yx_max = ([np.squeeze(a, -2) for a in np.split(a, a.shape[-2], -2)] for a in (yx_min, yx_max))
        cls, iou = ([np.squeeze(a, -1) for a in np.split(a, a.shape[-1], -1)] for a in (cls, iou))
        results = []
        for i, (yx_min, yx_max, cls, iou) in enumerate(zip(yx_min, yx_max, cls, iou)):
            mask = iou > self.config.getfloat('detect', 'threshold')
            yx_min, yx_max = (np.reshape(a, [a.shape[0], -1, 2]) for a in (yx_min, yx_max))
            cls, iou, mask = (np.reshape(a, [a.shape[0], -1]) for a in (cls, iou, mask))
            yx_min, yx_max, cls, iou, mask = ([a[b] for b in range(batch_size)] for a in (yx_min, yx_max, cls, iou, mask))
            yx_min, yx_max, cls = ([a[m] for a, m in zip(l, mask)] for l in (yx_min, yx_max, cls))
            canvas = [self.draw_bbox(canvas, yx_min.astype(np.int), yx_max.astype(np.int), cls, colors=colors) for canvas, yx_min, yx_max, cls in zip(np.copy(canvas_share), yx_min, yx_max, cls)]
            iou = [np.reshape(a, [rows, cols]) for a in iou]
            canvas = [self.draw_feature(_canvas, iou) for _canvas, iou in zip(canvas, iou)]
            results.append(canvas)
        return results 

def iterate(self, data):
        for key in data:
            t = data[key]
            if torch.is_tensor(t):
                data[key] = utils.ensure_device(t)
        tensor = torch.autograd.Variable(data['tensor'])
        pred = pybenchmark.profile('inference')(model._inference)(self.inference, tensor)
        height, width = data['image'].size()[1:3]
        rows, cols = pred['feature'].size()[-2:]
        loss, debug = pybenchmark.profile('loss')(model.loss)(self.anchors, norm_data(data, height, width, rows, cols), pred, self.config.getfloat('model', 'threshold'))
        loss_hparam = {key: loss[key] * self.config.getfloat('hparam', key) for key in loss}
        loss_total = sum(loss_hparam.values())
        self.optimizer.zero_grad()
        loss_total.backward()
        try:
            clip = self.config.getfloat('train', 'clip')
            nn.utils.clip_grad_norm(self.inference.parameters(), clip)
        except configparser.NoOptionError:
            pass
        self.optimizer.step()
        return dict(
            height=height, width=width, rows=rows, cols=cols,
            data=data, pred=pred, debug=debug,
            loss_total=loss_total, loss=loss, loss_hparam=loss_hparam,
        ) 

def draw_clusters(self, image, parts, limbs):
        try:
            interpolation = getattr(cv2, 'INTER_' + self.config.get('estimate', 'interpolation').upper())
            parts, limbs = (np.stack([cv2.resize(feature, image.shape[1::-1], interpolation=interpolation) for feature in a]) for a in (parts, limbs))
        except configparser.NoOptionError:
            pass
        clusters = pyopenpose.estimate(
            parts, limbs,
            self.env.limbs_index,
            self.config.getfloat('nms', 'threshold'),
            self.config.getfloat('integration', 'step'), tuple(map(int, self.config.get('integration', 'step_limits').split())), self.config.getfloat('integration', 'min_score'), self.config.getint('integration', 'min_count'),
            self.config.getfloat('cluster', 'min_score'), self.config.getint('cluster', 'min_count'),
        )
        scale_y, scale_x = np.array(image.shape[1::-1], parts.dtype) / np.array(parts.shape[-2:], parts.dtype)
        for cluster in clusters:
            cluster = [((i1, int(y1 * scale_y), int(x1 * scale_x)), (i2, int(y2 * scale_y), int(x2 * scale_x))) for (i1, y1, x1), (i2, y2, x2) in cluster]
            image = self.draw_cluster(image, cluster)
        return image 

def __init__(self, env):
        super(SummaryWorker, self).__init__()
        self.env = env
        self.config = env.config
        self.queue = multiprocessing.Queue()
        try:
            self.timer_scalar = utils.train.Timer(env.config.getfloat('summary', 'scalar'))
        except configparser.NoOptionError:
            self.timer_scalar = lambda: False
        try:
            self.timer_image = utils.train.Timer(env.config.getfloat('summary', 'image'))
        except configparser.NoOptionError:
            self.timer_image = lambda: False
        try:
            self.timer_histogram = utils.train.Timer(env.config.getfloat('summary', 'histogram'))
        except configparser.NoOptionError:
            self.timer_histogram = lambda: False
        with open(os.path.expanduser(os.path.expandvars(env.config.get('summary_histogram', 'parameters'))), 'r') as f:
            self.histogram_parameters = utils.RegexList([line.rstrip() for line in f])
        self.draw_points = utils.visualize.DrawPoints(env.limbs_index, colors=env.config.get('draw_points', 'colors').split())
        self._draw_points = utils.visualize.DrawPoints(env.limbs_index, thickness=1)
        self.draw_bbox = utils.visualize.DrawBBox()
        self.draw_feature = utils.visualize.DrawFeature()
        self.draw_cluster = utils.visualize.DrawCluster() 

def main_daemon(args: argparse.Namespace, config: configparser.ConfigParser) -> None:
    """Run the daemon."""

    checks = set_up_checks(
        config,
        "check",
        "activity",
        Activity,  # type: ignore
        error_none=True,
    )
    wakeups = set_up_checks(
        config, "wakeup", "wakeup", Wakeup,  # type: ignore
    )

    processor = configure_processor(args, config, checks, wakeups)
    loop(
        processor,
        config.getfloat("general", "interval", fallback=60),
        run_for=args.run_for,
        woke_up_file=get_woke_up_file(config),
        lock_file=get_lock_file(config),
        lock_timeout=get_lock_timeout(config),
    ) 

def configure_processor(
    args: argparse.Namespace,
    config: configparser.ConfigParser,
    checks: Iterable[Activity],
    wakeups: Iterable[Wakeup],
) -> Processor:
    return Processor(
        checks,
        wakeups,
        config.getfloat("general", "idle_time", fallback=300),
        config.getfloat("general", "min_sleep_time", fallback=1200),
        get_wakeup_delta(config),
        get_notify_and_suspend_func(config),
        get_schedule_wakeup_func(config),
        all_activities=args.all_checks,
    ) 

def get_lock_timeout(config: configparser.ConfigParser) -> float:
    return config.getfloat("general", "lock_timeout", fallback=30.0) 

def loss_hparam(self, i, name, loss):
        try:
            return loss * self.config.getfloat('hparam', '%s%d' % (name, i))
        except configparser.NoOptionError:
            return loss * self.config.getfloat('hparam', name) 

def __init__(self, args, config):
        self.args = args
        self.config = config
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self.model_dir = utils.get_model_dir(config)
        self.cache_dir = utils.get_cache_dir(config)
        _, self.num_parts = utils.get_dataset_mappers(config)
        self.limbs_index = utils.get_limbs_index(config)
        logging.info('use cache directory ' + self.cache_dir)
        logging.info('tensorboard --logdir ' + self.model_dir)
        if args.delete:
            logging.warning('delete model directory: ' + self.model_dir)
            shutil.rmtree(self.model_dir, ignore_errors=True)
        os.makedirs(self.model_dir, exist_ok=True)
        with open(self.model_dir + '.ini', 'w') as f:
            config.write(f)

        self.step, self.epoch, self.dnn, self.stages = self.load()
        self.inference = model.Inference(self.config, self.dnn, self.stages)
        logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in self.inference.state_dict().values())))
        if self.args.finetune:
            path = os.path.expanduser(os.path.expandvars(self.args.finetune))
            logging.info('finetune from ' + path)
            self.finetune(self.dnn, path)
        self.inference = self.inference.to(self.device)
        self.inference.train()
        self.optimizer = eval(self.config.get('train', 'optimizer'))(filter(lambda p: p.requires_grad, self.inference.parameters()), self.args.learning_rate)

        self.saver = utils.train.Saver(self.model_dir, config.getint('save', 'keep'))
        self.timer_save = utils.train.Timer(config.getfloat('save', 'secs'), False)
        try:
            self.timer_eval = utils.train.Timer(eval(config.get('eval', 'secs')), config.getboolean('eval', 'first'))
        except configparser.NoOptionError:
            self.timer_eval = lambda: False
        self.summary_worker = SummaryWorker(self)
        self.summary_worker.start() 

def get_wakeup_delta(config: configparser.ConfigParser) -> float:
    return config.getfloat("general", "wakeup_delta", fallback=30) 

def filter_visible(config, iou, yx_min, yx_max, prob):
    prob_cls, cls = torch.max(prob, -1)
    if config.getboolean('detect', 'fix'):
        mask = (iou * prob_cls) > config.getfloat('detect', 'threshold_cls')
    else:
        mask = iou > config.getfloat('detect', 'threshold')
    iou, prob_cls, cls = (t[mask].view(-1) for t in (iou, prob_cls, cls))
    _mask = torch.unsqueeze(mask, -1).repeat(1, 2)  # PyTorch's bug
    yx_min, yx_max = (t[_mask].view(-1, 2) for t in (yx_min, yx_max))
    num = prob.size(-1)
    _mask = torch.unsqueeze(mask, -1).repeat(1, num)  # PyTorch's bug
    prob = prob[_mask].view(-1, num)
    return iou, yx_min, yx_max, prob, prob_cls, cls 

def __init__(self):
        HackQ.download_nltk_resources()
        colorama.init()

        self.bearer = config.get("CONNECTION", "BEARER")
        self.timeout = config.getfloat("CONNECTION", "Timeout")
        self.show_next_info = config.getboolean("MAIN", "ShowNextShowInfo")
        self.exit_if_offline = config.getboolean("MAIN", "ExitIfShowOffline")
        self.show_bearer_info = config.getboolean("MAIN", "ShowBearerInfo")
        self.headers = {"User-Agent": "Android/1.40.0",
                        "x-hq-client": "Android/1.40.0",
                        "x-hq-country": "US",
                        "x-hq-lang": "en",
                        "x-hq-timezone": "America/New_York",
                        "Authorization": f"Bearer {self.bearer}",
                        "Connection": "close"}

        self.session = requests.Session()
        self.session.headers.update(self.headers)

        self.init_root_logger()
        self.logger = logging.getLogger(__name__)

        # Find local UTC offset
        now = time.time()
        self.local_utc_offset = datetime.fromtimestamp(now) - datetime.utcfromtimestamp(now)

        self.validate_bearer()
        self.logger.info("HackQ-Trivia initialized.\n", extra={"pre": colorama.Fore.GREEN}) 

def async_main(args: argparse.Namespace, config: ConfigParser) -> int:
    if args.op.lower() == "delete":
        async with bandersnatch.master.Master(
            config.get("mirror", "master"),
            config.getfloat("mirror", "timeout"),
            config.getfloat("mirror", "global-timeout", fallback=None),
        ) as master:
            return await bandersnatch.delete.delete_packages(config, args, master)
    elif args.op.lower() == "verify":
        return await bandersnatch.verify.metadata_verify(config, args)
    elif args.op.lower() == "sync":
        return await bandersnatch.mirror.mirror(config, args.packages)

    if args.force_check:
        storage_plugin = next(iter(storage_backend_plugins()))
        status_file = (
            storage_plugin.PATH_BACKEND(config.get("mirror", "directory")) / "status"
        )
        if status_file.exists():
            tmp_status_file = Path(gettempdir()) / "status"
            try:
                shutil.move(str(status_file), tmp_status_file)
                logger.debug(
                    "Force bandersnatch to check everything against the master PyPI"
                    + f" - status file moved to {tmp_status_file}"
                )
            except OSError as e:
                logger.error(
                    f"Could not move status file ({status_file} to "
                    + f" {tmp_status_file}): {e}"
                )
        else:
            logger.info(
                f"No status file to move ({status_file}) - Full sync will occur"
            )

    return await bandersnatch.mirror.mirror(config) 

def get_restful_params(urlstring):
    """Returns a dictionary of paired RESTful URI parameters"""
    parsed_path = urllib.parse.urlsplit(urlstring.strip("/"))
    query_params = urllib.parse.parse_qsl(parsed_path.query)
    path_tokens = parsed_path.path.split('/')

    # If first token is API version, ensure it isn't obsolete
    api_version = API_VERSION
    if len(path_tokens[0]) == 2 and path_tokens[0][0] == 'v':
        # Require latest API version
        if path_tokens[0][1] != API_VERSION:
            return None
        api_version = path_tokens.pop(0)

    path_params = list_to_dict(path_tokens)
    path_params["api_version"] = api_version
    path_params.update(query_params)
    return path_params

# this doesn't currently work
# if LOAD_TEST:
#     config = ConfigParser.RawConfigParser()
#     config.read(CONFIG_FILE)
#     TEST_CREATE_DEEP_QUOTE_DELAY = config.getfloat('general', 'test_deep_quote_delay')
#     TEST_CREATE_QUOTE_DELAY = config.getfloat('general','test_quote_delay')

# NOTE These are still used by platform init in dev in eclipse mode 

def draw_bbox_pred(self, canvas, yx_min, yx_max, cls, iou, colors=None, nms=False):
        batch_size = len(canvas)
        mask = iou > self.config.getfloat('detect', 'threshold')
        yx_min, yx_max = (np.reshape(a, [a.shape[0], -1, 2]) for a in (yx_min, yx_max))
        cls, iou, mask = (np.reshape(a, [a.shape[0], -1]) for a in (cls, iou, mask))
        yx_min, yx_max, cls, iou, mask = ([a[b] for b in range(batch_size)] for a in (yx_min, yx_max, cls, iou, mask))
        yx_min, yx_max, cls, iou = ([a[m] for a, m in zip(l, mask)] for l in (yx_min, yx_max, cls, iou))
        if nms:
            overlap = self.config.getfloat('detect', 'overlap')
            keep = [pybenchmark.profile('nms')(utils.postprocess.nms)(torch.Tensor(iou), torch.Tensor(yx_min), torch.Tensor(yx_max), overlap) if iou.shape[0] > 0 else [] for yx_min, yx_max, iou in zip(yx_min, yx_max, iou)]
            keep = [np.array(k, np.int) for k in keep]
            yx_min, yx_max, cls = ([a[k] for a, k in zip(l, keep)] for l in (yx_min, yx_max, cls))
        return [self.draw_bbox(canvas, yx_min.astype(np.int), yx_max.astype(np.int), cls, colors=colors) for canvas, yx_min, yx_max, cls in zip(canvas, yx_min, yx_max, cls)] 

def postprocess(config, iou, yx_min, yx_max, prob):
    iou, yx_min, yx_max, prob, prob_cls, cls = filter_visible(config, iou, yx_min, yx_max, prob)
    keep = pybenchmark.profile('nms')(utils.postprocess.nms)(iou, yx_min, yx_max, config.getfloat('detect', 'overlap'))
    if keep:
        keep = utils.ensure_device(torch.LongTensor(keep))
        iou, yx_min, yx_max, prob, prob_cls, cls = (t[keep] for t in (iou, yx_min, yx_max, prob, prob_cls, cls))
        if config.getboolean('detect', 'fix'):
            score = torch.unsqueeze(iou, -1) * prob
            mask = score > config.getfloat('detect', 'threshold_cls')
            indices, cls = torch.unbind(mask.nonzero(), -1)
            yx_min, yx_max = (t[indices] for t in (yx_min, yx_max))
            score = score[mask]
        else:
            score = iou
        return iou, yx_min, yx_max, cls, score