Python IPython.embed() Examples

def main(argv):
    if 'createdb' in argv:
        models.db.create_all()
    elif 'createdata' in argv:
        from scoreboard.tests import data
        models.db.create_all()
        data.create_all()
    elif 'shell' in argv:
        try:
            import IPython
            run_shell = IPython.embed
        except ImportError:
            import readline  # noqa: F401
            import code
            run_shell = code.InteractiveConsole().interact
        run_shell()
    else:
        wsgi.app.run(
                host='0.0.0.0', debug=True,
                port=wsgi.app.config.get('PORT', 9999)) 

def test():
    from tqdm import trange
    import cv2
    print('debug mode '*10 )
    args = parse_args()
    cfg = Config.fromfile(args.config)
    cfg.gpus = 1

    dataset = build_dataset(cfg.data.train)
    embed(header='123123')
    # def visual(i):
    #     img = dataset[i]['img'].data
    #     img = img.permute(1,2,0) + 100
    #     img = img.data.cpu().numpy()
    #     cv2.imwrite('./trash/resize_v1.jpg',img)

    # embed(header='check data resizer') 

def cmd_console(args):
    import code
    import platform

    d = u2.connect(args.serial)
    model = d.shell("getprop ro.product.model").output.strip()
    serial = d.serial
    try:
        import IPython
        from traitlets.config import get_config
        c = get_config()
        c.InteractiveShellEmbed.colors = "neutral"
        IPython.embed(config=c, header="IPython -- d.info is ready")
    except ImportError:
        _vars = globals().copy()
        _vars.update(locals())
        shell = code.InteractiveConsole(_vars)
        shell.interact(banner="Python: %s\nDevice: %s(%s)" %
                       (platform.python_version(), model, serial)) 

def YYT(Y, n_components=None, DEBUG=False):
    """
    Param:
        Y: n x d
        n_components: use 'mle' to guess
    Returns:
        P: d x d'
        QT: d' x d
    """
    newdata = Y.copy()
    model = PCA(n_components=n_components)

    if len(newdata.shape) != 2:
        newdata = newdata.reshape((newdata.shape[0], -1))
    #TODO center data
    model.fit(newdata)
    if DEBUG: from IPython import embed; embed()

    return model.components_.T, model.components_

#def GSVD(Z, Y):
#    NotImplementedError
#    return [U,V,X,C,S] 

def prepare_client(image):
    """prepare a client to embed in a shell with recipe parsers and writers.
    """
    # The client will announce itself (backend/database) unless it's get
    from spython.main import get_client
    from spython.main.parse import parsers
    from spython.main.parse import writers

    client = get_client()

    if image:
        client.load(image)

    # Add recipe parsers
    client.parsers = parsers
    client.writers = writers
    return client 

def shell(scope):
    if isinstance(scope, ga.account.Profile):
        profile = scope
        account = profile.account
        metrics = profile.core.metrics
        dimensions = profile.core.dimensions
        core = profile.core.query
        realtime = profile.realtime.query
        print('* global variables: profile, account, metrics, dimensions')
        print('* build queries with the `core` and `realtime` variables')
        print("  e.g. `core.metrics('pageviews').daily('yesterday').values`\n")
    else:
        print('* global variables: scope')
        print('  (provide webproperty and/or profile for additional shortcuts)\n')

    embed(local=locals()) 

def local_test(self, test_dataloader):
        self.model.eval()
        test_loss = test_acc = test_total = 0.
        with torch.no_grad():
            for x, y in test_dataloader:
                # print("test")
                # from IPython import embed
                # embed()
                if self.gpu:
                    x, y = x.cuda(), y.cuda()

                pred = self.model(x)
                loss = criterion(pred, y)
                _, predicted = torch.max(pred, 1)
                correct = predicted.eq(y).sum()

                test_acc += correct.item()
                test_loss += loss.item() * y.size(0)
                test_total += y.size(0)

        return test_acc, test_loss 

def sample(self, vis = False, stop = False):
        """
        Samples probabilities of success from the given values
        """
        #samples = np.random.beta(self.posterior_alphas_, self.posterior_betas_)
        samples = scipy.stats.beta.rvs(self.posterior_alphas_, self.posterior_betas_)
        if stop:
            IPython.embed()
        if vis:
            print('Samples')
            print(samples)
            print('Estimated mean')
            print((BetaBernoulliModel.beta_mean(self.posterior_alphas_, self.posterior_betas_)))
            print('At best index')
            print((BetaBernoulliModel.beta_mean(self.posterior_alphas_[21], self.posterior_betas_[21])))
        return samples 

def _start_repl(api):
    # type: (Iota) -> None
    """
    Starts the REPL.
    """
    _banner = (
      'IOTA API client for {uri} ({testnet}) initialized as variable `api`.\n'
      'Type `help(api)` for list of API commands.'.format(
        testnet = 'testnet' if api.testnet else 'mainnet',
        uri     = api.adapter.get_uri(),
      )
    )

    try:
      # noinspection PyUnresolvedReferences
      import IPython
    except ImportError:
      # IPython not available; use regular Python REPL.
      from code import InteractiveConsole
      InteractiveConsole(locals={'api': api}).interact(_banner)
    else:
      # Launch IPython REPL.
      IPython.embed(header=_banner) 

def parse_arch_to_model_spec_matrix_op(cell, B=5):
    matrix = np.zeros((B + 2, B + 2))
    ops = [INPUT, ]
    is_output = [True, ] * (B + 1)
    try:
        for i in range(B):
            prev_node1 = cell[2 * i]  # O as input.
            op = ALLOWED_OPS[cell[2 * i + 1]]
            ops.append(op)

            is_output[prev_node1] = False
            curr_node = i + 1
            matrix[prev_node1][curr_node] = 1
        # process output
        for input_node, connect_to_output in enumerate(is_output):
            matrix[input_node][B + 1] = 1 if connect_to_output else 0
        matrix = matrix.astype(np.int)
        ops.append(OUTPUT)
    except Exception as e:
        IPython.embed()
    return matrix, ops 

def forward(self, x, model_spec=None, steps=None, bn_train=None):
        aux_logits = None
        # self._model_spec.update_multi_gpu(self.alpha_topology, self.alpha_ops)
        out = self.stem(x)
        ws = [self.weights(i) for i in range(self.num_intermediate_nodes)]
        for stack in self.stacks.values():
            for k, module in stack.items():
                # IPython.embed()
                if 'module' in k:
                    out = module([out, ws])
                else:
                    out = module(out)

        out = F.avg_pool2d(out, out.shape[2:]).view(out.shape[:2])
        # TO match the auxiliary head, but right now it is always false for NASBench.
        return self.dense(out), aux_logits 

def install_extra_module(self, s):
        if self.extra_module_base is not None:
            try:
                extra_module_base = self.extra_module_base
                extra_module_size = self.extra_module_size
                num_of_pages = extra_module_size / 4096 + 1
                print('extra module is at memory location %x of size %x' % (extra_module_base, extra_module_size))
                for i in range(num_of_pages):
                    addr = extra_module_base + i * 4096
                    con = self.statebroker.get_a_page(self.r, addr)
                    if con is not None:
                        print('successfully get a page at:', hex(addr))
                        self.set_loader_concret_memory_region(s, addr, con, 4096)
                    else:
                        input('failed to get a page')
                print('Finished installing extra modules')
            except TypeError as e:
                print(e)
                traceback.print_exc()
                embed()
        else:
            print('do not need to print extra module')
        return 

def random_sampler(self, model, architect, args):
        # according to Aug 8 meeting. become the new topo sampler
        total = self.args.num_intermediate_nodes
        matrices_list = self.search_space.nasbench_sample_matrix_from_list(
            np.arange(1, total+1), self.search_space.nasbench_topo_sample_probs)

        for matrix in matrices_list:
            if matrix is not None:
                spec = obtain_full_model_spec(total + 2)
                try:
                    spec = ModelSpec_v2(matrix, spec.ops)
                except:
                    IPython.embed()
                self.model_spec = spec
                self.model_spec_id = None
                yield change_model_spec(model, spec) 

def save_model(self):

        if self.iter_num==-7:
            print("Error: MUST SPECIFY ITER_NUM FOR SAVER...")
            import IPython
            IPython.embed()

        # save the model under current iteration number
        # but also update the finalModel.ckpt too
        save_path1 = self.tf_saver.save(
            self.sess, self.save_dir + '/models/model_aggIter' + str(
                self.iter_num) + '.ckpt')
        save_path2 = self.tf_saver.save(
            self.sess, self.save_dir + '/models/finalModel.ckpt')
        print("Model saved at ", save_path1)

    ############################################################################################
    ##### The following 2 saves together represent a single "iteration"
    ########## (train model) + (collect new rollouts with that model) = a single "iteration"
    ############################################################################################ 

def model_hash_rank(self, full_spec=False):
        """
        Return model's hash according to its rank in given search space.
        Output:
            if not full_spec:   list[hash_1, hash_2 ...]
            if full_spec:       list[hash_1, hash_2 ...], list[model_spec_1, model_spec_2 ...]

        :param full_spec: if True, return the spec of each model architecture.
        :return:
        """
        logging.info('return the entire model hash rank. '
                     'Within current searching space, the number of total models are \n'
                     f'Total models: {len(self.hash_rank)}. ')
        # IPython.embed(header='model_hash_rank')
        if full_spec:
            model_specs = []
            for ind, h in enumerate(self.hash_rank):
                model_specs.append(self.hash_to_model_spec(h))
            return self.hash_rank, model_specs

        return self.hash_rank

    # TODO implement, hash_perfs, perfs_rank 

def forward(self, x):
        embedded = self.embedding(x)  # size = [72, 20, 96]
        embedded = self.dropout(embedded)
        if self.source_length != self.length:
            assert self.source_length % self.length == 0
            ratio = self.source_length // self.length
            embedded = embedded.view(-1, self.source_length // ratio, ratio * self.emb_size)
        out, hidden = self.rnn(embedded)
        out = F.normalize(out, 2, dim=-1)
        encoder_outputs = out
        encoder_hidden = hidden

        out = torch.mean(out, dim=1)
        out = F.normalize(out, 2, dim=-1)
        arch_emb = out  # [72, 96]

        out = self.mlp(out)
        out = self.regressor(out)
        predict_value = torch.sigmoid(out)  # accuracy prediction.
        # IPython.embed(header='Checking forward of encoder...')
        return encoder_outputs, encoder_hidden, arch_emb, predict_value 

def enforce_prologue_to_copy_to_user(self, state):
        """
        this function is actually a callback or bp over unconstrained call instructions
        :param state:
        :return:
        """
        if state.regs.rip.symbolic:
            print('trying to extract signature at prologue indirect call to copy_from_user')
            print('Call target address :', state.inspect.function_address)
            # self.dump_reg(state)  # dump registers for debug purpose
            print(colorama.Fore.RED + '[+] extracting runtime data flow signature for pairing with disclosure gadget'
                  + colorama.Style.RESET_ALL)
            data_signatures = self.extract_prologue_call_site_signature(state)
            self.current_prologue_signature = data_signatures
            print(colorama.Fore.RED + '[!] removing bp_enforce_prologue_to_copy_to_user)' + colorama.Style.RESET_ALL)
            state.inspect.remove_breakpoint('call', self.bp_enforce_prologue_to_copy_to_user)
            # embed()
        else:
            print('rip is not symbolic, we are not removing this enforcement until we finding one')
            # embed()
        return 

def enforce_indirect_jump_to_disclosure_gadget(self, state):
        """
        this function is actually a callback or bp over unconstrained jmp instruction
        :param state:
        :return:
        """
        if state.regs.rip.symbolic:
            print('trying to extract signature at prologue indirect jump to copy_from_user')
            print(colorama.Fore.RED +'jmp instruction at:', hex(state.history.addr) + colorama.Style.RESET_ALL)
            # self.dump_reg(state)
            print(colorama.Fore.RED + '[+] extracting runtime data flow signature for pairing with disclosure gadget'
                  + colorama.Style.RESET_ALL)
            data_signatures = self.extract_prologue_call_site_signature(state)
            self.current_prologue_signature = data_signatures
            print(colorama.Fore.RED + '[!] removing bp_enforce_prologue_to_copy_to_user)' + colorama.Style.RESET_ALL)
            state.inspect.remove_breakpoint('call', self.bp_enforce_prologue_to_copy_to_user)
            # embed()
        else:
            print('rip is not symbolic, this should never happen')
            embed()
        return 

def forward(self, x, weight):
        """

        :param x: input.
        :param weight: topology, op weights accordingly
        :return:
        """
        topo_weight, op_weight = weight
        # This is the Single-path way.
        # apply all projection to get real input to this vertex
        proj_iter = iter(self.current_proj_ops)
        input_iter = iter(x)
        out = next(proj_iter)(next(input_iter))
        for ind, (proj, inp) in enumerate(zip(proj_iter, input_iter)):
            out = out + topo_weight[ind] * proj(inp)
        # no such thing called current op.
        output = 0.0
        try:
            for ind, op in enumerate(self.ops.values()):
                output = output + op_weight[ind] * op(out)
        except RuntimeError as e:
            IPython.embed()
        return output 

def invBN(self, arr, Y_name):
        if isinstance(arr, int) or len(self.bns) == 0 or len(self.affines) == 0:
            return arr
        interstellar = Y_name.split('_')[0]
        for i in self.bottom_names[interstellar]:
            if i in self.bns and 'branch2c' in i:
                bn = i
                break
        for i in self.affines:
            if self.layer_bottom(i) == bn:
                affine = i
                break

        if 1: print('inverted bn', bn, affine, Y_name)
        mean, std, k, b = self.getBNaff(bn, affine)
        # (y - mean) / std * k + b
        #return (arr - b) * std / k + mean
        return arr * std / k
        #embed() 

def do(self, which_callback, *args):

        if not self.sig_raised:
            return
        self.sig_raised = False

        env = None
        if self.use_main_loop_run_caller_env:
            frame = sys._getframe()
            while frame:
                if frame.f_code is self.main_loop.run.func_code:
                    env = frame.f_back.f_locals
                    break
                frame = frame.f_back

        IPython.embed(user_ns=env) 

def enter_prologue_callback(self, state):
        """
        this is a bp on the first instruction of the prologue gadget
        we add a bp over call instruction to handle future indirect call
        TODO: BUG here: we did not consider the indirect jump
        :param state:
        :return:
        """
        self.reach_current_prologue_entry = True
        print(colorama.Fore.RED + 'enter prologue gadget' + colorama.Style.RESET_ALL)
        if not self.is_dfs_search_routine:
            state.inspect.remove_breakpoint("call", self.first_fork_site_bp)
            print('[+] removed the call bp at the first fork site..')
        #
        self.bp_enforce_prologue_to_copy_to_user = state.inspect.b("call", when=angr.BP_BEFORE
                                                                   , action=self.enforce_prologue_to_copy_to_user)
        print('[+] enforced a bp on call for disclosure')
        #import IPython; IPython.embed()
        return 

def extract_prologue_call_site_signature(self, state):
        """
        extract the data flow signature e.g., rdx rsi rdi at the indirect call in the prologue function
        :param state: the state
        :return: dict of interested register values
        """
        print('[+] extracting prologue call site signatures...')
        signature = dict()
        signature['rdx'] = state.regs.rdx
        signature['rsi'] = state.regs.rsi
        signature['rdi'] = state.regs.rdi
        print('rdx', state.regs.rdx)
        print('rsi', state.regs.rdi)
        print('rdi', state.regs.rdi)
        # import IPython; IPython.embed()
        return signature 

def forward(self, input_variable, target_variable=None):
        # IPython.embed(header='study nao_nasbench sampler, chaeck input and output')
        # Input to encoder is so-called sequence.
        encoder_outputs, encoder_hidden, arch_emb, predict_value = self.encoder(input_variable)
        decoder_hidden = (arch_emb.unsqueeze(0), arch_emb.unsqueeze(0))
        decoder_outputs, decoder_hidden, ret = self.decoder(target_variable, decoder_hidden, encoder_outputs)
        # decoder_outputs 41 x [72,12], same as ret['sequence']
        decoder_outputs = torch.stack(decoder_outputs, 0).permute(1, 0, 2)
        # decoder_outputs becomes []
        arch = torch.stack(ret['sequence'], 0).permute(1, 0, 2)
        return predict_value, decoder_outputs, arch 

def _process_rank_data_for_plotting(rank_data, gt_index_by_rank, e_dict):
    """
    :param rank_data:
    :param gt_index_by_rank:
    :param e_dict: To create panda data. {}
    :return:
    """
    epochs = rank_data.keys()

    # print(e_dict)
    # IPython.embed()
    pd_epoch = pd.DataFrame.from_dict(e_dict)
    pd_epoch.sort_values('geno_id', inplace=True)
    pd_epoch.reset_index(inplace=True, drop=True)
    pd_epoch = pd_epoch.reindex(gt_index_by_rank)
    pd_epoch.reset_index(inplace=True, drop=True)
    # print(pd_epoch)

    # rank change
    rank_df = pd_epoch
    rc_data = []  # rc = rank change
    keys_list = list(epochs)
    step = 2
    x_values = []
    for i in range(0, len(keys_list), step):
        k = keys_list[i]
        a = rank_df.sort_values(['epoch_{}'.format(k)])
        x_values.append(int(k))
        rc_data.append(a.index.values)
    # Add the final result.
    rc_data.append([i for i in range(0, len(rc_data[-1]))])
    x_values.append(x_values[-1] + 1 * (x_values[-1] - x_values[-2]) if len(x_values) > 1 else 1)

    x_values = np.array(x_values)
    rc_data = np.asanyarray(rc_data)
    rc_data = np.transpose(rc_data)
    return rc_data, x_values, rank_df 

def choose_next(self, stop = False):
        """ Returns the index of the maximal random sample, breaking ties uniformly at random"""
        if self.model_ is None:
            raise ValueError('Must set predictive model')
        sampled_values = self.model_.sample()
        if stop:
            IPython.embed()
        max_indices = np.where(sampled_values == np.max(sampled_values))[0]
        num_max_indices = max_indices.shape[0]
        next_index = np.random.choice(num_max_indices)
        return max_indices[next_index] 

def interactive(self):
        # Enable infix notation in Interactive mode
        get_env().enable_infix_notation = True
        try:
            import IPython
            print(welcome_msg)
            IPython.embed()
        except ImportError:
            import code
            code.interact(welcome_msg) 

def explore_model(self, model_name):
        """
        Initiates an interactive IPython session to run discovery queries.

        :param model_name:
        :return:
        """
        api, reporting = init_system(self._make_model_path(model_name).__str__() + '/', create_reporting=True)
        IPython.embed() 

def shell():
    """Run a Python shell in the app context."""

    try:
        import IPython
    except ImportError:
        IPython = None

    if IPython is not None:
        IPython.embed(banner1="", user_ns=current_app.make_shell_context())
    else:
        import code

        code.interact(banner="", local=current_app.make_shell_context()) 

def save_rollout_info(self, save_data):

        if self.iter_num==-7:
            print("Error: MUST SPECIFY ITER_NUM FOR SAVER...")
            import IPython
            IPython.embed()

        #info from all MPC rollouts (from this iteration)
        pickle.dump(
            save_data.rollouts_info,
            open(
                self.save_dir + '/saved_rollouts/rollouts_info_' + str(
                    self.iter_num) + '.pickle', 'wb'),
            protocol=pickle.HIGHEST_PROTOCOL)

        #save rewards and scores (for rollouts from all iterations thus far)
        np.save(self.save_dir + '/rollouts_rewardsPerIter.npy',
                save_data.rollouts_rewardsPerIter)
        np.save(self.save_dir + '/rollouts_scoresPerIter.npy',
                save_data.rollouts_scoresPerIter)

        #plot rewards and scores (for rollouts from all iterations thus far)
        rew = np.array(save_data.rollouts_rewardsPerIter)
        scores = np.array(save_data.rollouts_scoresPerIter)
        plot_mean_std(rew[:, 0], rew[:, 1], self.save_dir + '/rewards_perIter')
        plot_mean_std(scores[:, 0], scores[:, 1],
                      self.save_dir + '/scores_perIter')

        self.iter_num = -7