Python pandas.set_option() Examples

def execute(cls, ctx, op: "DataFrameDropNA"):
        try:
            pd.set_option('mode.use_inf_as_na', op.use_inf_as_na)

            in_data = ctx[op.inputs[0].key]
            if op.drop_directly:
                if in_data.ndim == 2:
                    result = in_data.dropna(axis=op.axis, how=op.how, thresh=op.thresh,
                                            subset=op.subset)
                else:
                    result = in_data.dropna(axis=op.axis, how=op.how)
                ctx[op.outputs[0].key] = result
                return

            in_counts = ctx[op.inputs[1].key]
            if op.how == 'all':
                in_counts = in_counts[in_counts > 0]
            else:
                thresh = op.subset_size if op.thresh is None else op.thresh
                in_counts = in_counts[in_counts >= thresh]

            ctx[op.outputs[0].key] = in_data.reindex(in_counts.index)
        finally:
            pd.reset_option('mode.use_inf_as_na') 

def job_table(self, project=None, recursive=True, columns=None, all_columns=False, sort_by="id", max_colwidth=200,
                  job_name_contains=''):
        if project is None:
            project = self._project
        if columns is None:
            columns = ["job", "project", "chemicalformula"]
        if all_columns:
            columns = self._columns
        if len(self._job_table) != 0:
            if recursive:
                df = self._job_table[self._job_table.project.str.contains(project)]
            else:
                df = self._job_table[self._job_table.project == project]
        else:
            df = self._job_table
        pandas.set_option("display.max_colwidth", max_colwidth)
        if len(df) == 0:
            return df
        if job_name_contains != '':
            df = df[df.job.str.contains(job_name_contains)]
        if sort_by in columns:
            return df[columns].sort_values(by=sort_by)
        return df[columns] 

def write_to_html(self):
        pandas.set_option('display.max_colwidth', -1)
        header = '{!s}'.format(self.df.index.tolist()[0])
        df = self.df.reset_index(level=['Clf.', 'Set_Type', 'Eval.'])
        if '#Rep.' in df:
            df.drop('#Rep.', 1, inplace=True)

        df.drop('Eval.', 1, inplace=True)
        df.drop('Set_Size', 1, inplace=True)
        df.drop('Set_Type', 1, inplace=True)
        df.drop('f1', 1, inplace=True)
        df.drop('precision', 1, inplace=True)
        df.columns = ['Clf', '\\ac{DGA} Type', '\\ac{ACC}', '\\ac{TPR}', '\\ac{TNR}', '\\ac{FNR}', '\\ac{FPR}']
        fname = settings.ANALYSIS_FOLDER + '/eval_full.html'
        with open(fname, 'w') as f:
            f.write(df.to_html())
        pandas.reset_option('display.max_colwidth') 

def print_para(model):
    """
    Prints parameters of a model
    :param opt:
    :return:
    """
    st = {}
    total_params = 0
    total_params_training = 0
    for p_name, p in model.named_parameters():
        # if not ('bias' in p_name.split('.')[-1] or 'bn' in p_name.split('.')[-1]):
        st[p_name] = ([str(x) for x in p.size()], np.prod(p.size()), p.requires_grad)
        total_params += np.prod(p.size())
        if p.requires_grad:
            total_params_training += np.prod(p.size())
    pd.set_option('display.max_columns', None)
    shapes_df = pd.DataFrame([(p_name, '[{}]'.format(','.join(size)), prod, p_req_grad)
                              for p_name, (size, prod, p_req_grad) in sorted(st.items(), key=lambda x: -x[1][1])],
                             columns=['name', 'shape', 'size', 'requires_grad']).set_index('name')

    print('\n {:.1f}M total parameters. {:.1f}M training \n ----- \n {} \n ----'.format(total_params / 1000000.0,
                                                                                        total_params_training / 1000000.0,
                                                                                        shapes_df.to_string()),
          flush=True)
    return shapes_df 

def list_groups(dim=3):
    """
    Function for quick print of groups and symbols

    Args:
        group: the group symbol or international number
        dim: the periodic dimension of the group
    """
    
    import pandas as pd

    keys = {3: 'space_group',
            2: 'layer_group',
            1: 'rod_group',
            0: 'point_group',
           }
    data = symbols[keys[dim]]
    df = pd.DataFrame(index=range(1, len(data)+1), 
                      data=data, 
                      columns=[keys[dim]])
    pd.set_option('display.max_rows', len(df))
    #df.set_index('Number')
    print(df) 

def daily_stats(data: (pd.Series, pd.DataFrame), **kwargs) -> pd.DataFrame:
    """
    Daily stats for given data

    Examples:
        >>> pd.set_option('precision', 2)
        >>> (
        ...     pd.concat([
        ...         pd.read_pickle('xbbg/tests/data/sample_rms_ib0.pkl'),
        ...         pd.read_pickle('xbbg/tests/data/sample_rms_ib1.pkl'),
        ...     ], sort=False)
        ...     .pipe(get_series, col='close')
        ...     .pipe(daily_stats)
        ... )['RMS FP Equity'].iloc[:, :5]
                                   count    mean   std    min    10%
        2020-01-16 00:00:00+00:00  434.0  711.16  1.11  708.6  709.6
        2020-01-17 00:00:00+00:00  437.0  721.53  1.66  717.0  719.0
    """
    if data.empty: return pd.DataFrame()
    if 'percentiles' not in kwargs: kwargs['percentiles'] = [.1, .25, .5, .75, .9]
    return data.groupby(data.index.floor('d')).describe(**kwargs) 

def df_to_string(df):
    """
    Create a formatted str representation of the DataFrame.

    Parameters
    ----------
    df: DataFrame

    Returns
    -------
    str

    """
    pd.set_option('display.expand_frame_repr', False)
    pd.set_option('precision', 8)
    pd.set_option('display.width', 1000)
    pd.set_option('display.max_colwidth', 1000)

    return df.to_string() 

def execute(cls, ctx, op):
        try:
            pd.set_option('mode.use_inf_as_na', op.use_inf_as_na)
            if op.stage == OperandStage.map:
                cls._execute_map(ctx, op)
            elif op.stage == OperandStage.combine:
                cls._execute_combine(ctx, op)
            else:
                input_data = ctx[op.inputs[0].key]
                value = getattr(op, 'value', None)
                if isinstance(op.value, (Base, Entity)):
                    value = ctx[op.value.key]
                ctx[op.outputs[0].key] = input_data.fillna(
                    value=value, method=op.method, axis=op.axis, limit=op.limit, downcast=op.downcast)
        finally:
            pd.reset_option('mode.use_inf_as_na') 

def print_para(model):
    """
    Prints parameters of a model
    :param opt:
    :return:
    """
    st = {}
    total_params = 0
    total_params_training = 0
    for p_name, p in model.named_parameters():
        # if not ('bias' in p_name.split('.')[-1] or 'bn' in p_name.split('.')[-1]):
        st[p_name] = ([str(x) for x in p.size()], np.prod(p.size()), p.requires_grad)
        total_params += np.prod(p.size())
        if p.requires_grad:
            total_params_training += np.prod(p.size())
    pd.set_option('display.max_columns', None)
    shapes_df = pd.DataFrame([(p_name, '[{}]'.format(','.join(size)), prod, p_req_grad)
                              for p_name, (size, prod, p_req_grad) in sorted(st.items(), key=lambda x: -x[1][1])],
                             columns=['name', 'shape', 'size', 'requires_grad']).set_index('name')

    print('\n {:.1f}M total parameters. {:.1f}M training \n ----- \n {} \n ----'.format(total_params / 1000000.0,
                                                                                        total_params_training / 1000000.0,
                                                                                        shapes_df.to_string()),
          flush=True)
    return shapes_df 

def execute(cls, ctx, op: "DataFrameAggregate"):
        try:
            pd.set_option('mode.use_inf_as_na', op.use_inf_as_na)
            if op.stage == OperandStage.map:
                cls._execute_map(ctx, op)
            elif op.stage == OperandStage.combine:
                cls._execute_combine(ctx, op)
            elif op.stage == OperandStage.agg:
                cls._execute_agg(ctx, op)
            elif op.raw_func == 'size':
                xp = cp if op.gpu else np
                ctx[op.outputs[0].key] = xp.array(ctx[op.inputs[0].key].agg(op.raw_func, axis=op.axis)) \
                    .reshape(op.outputs[0].shape)
            else:
                ctx[op.outputs[0].key] = ctx[op.inputs[0].key].agg(op.raw_func, axis=op.axis)
        finally:
            pd.reset_option('mode.use_inf_as_na') 

def rsi(df, n=6):
    """
    相对强弱指标（Relative Strength Index，简称RSI
    LC= REF(CLOSE,1)
    RSI=SMA(MAX(CLOSE-LC,0),N,1)/SMA(ABS(CLOSE-LC),N1,1)×100
    SMA（C,N,M）=M/N×今日收盘价+(N-M)/N×昨日SMA（N）
    """
    # pd.set_option('display.max_rows', 1000)
    _rsi = pd.DataFrame()
    _rsi['date'] = df['date']
    px = df.close - df.close.shift(1)
    px[px < 0] = 0
    _rsi['rsi'] = sma(px, n) / sma((df['close'] - df['close'].shift(1)).abs(), n) * 100
    # def tmax(x):
    #     if x < 0:
    #         x = 0
    #     return x
    # _rsi['rsi'] = sma((df['close'] - df['close'].shift(1)).apply(tmax), n) / sma((df['close'] - df['close'].shift(1)).abs(), n) * 100
    return _rsi 

def bbiboll(df, n=10, k=3):
    """
    BBI多空布林线	bbiboll(10,3)
    BBI={MA(3)+ MA(6)+ MA(12)+ MA(24)}/4
    标准差MD=根号[∑（BBI-MA(BBI，N)）^2/N]
    UPR= BBI＋k×MD
    DWN= BBI－k×MD
    """
    # pd.set_option('display.max_rows', 1000)
    _bbiboll = pd.DataFrame()
    _bbiboll['date'] = df.date
    _bbiboll['bbi'] = (_ma(df.close, 3) + _ma(df.close, 6) + _ma(df.close, 12) + _ma(df.close, 24)) / 4
    _bbiboll['md'] = _md(_bbiboll.bbi, n)
    _bbiboll['upr'] = _bbiboll.bbi + k * _bbiboll.md
    _bbiboll['dwn'] = _bbiboll.bbi - k * _bbiboll.md
    return _bbiboll 

def _set_display_options(self):
        """
        Set pandas display options. Display all rows and columns by default.
        """
        display_options = {'display.max_rows': None,
                           'display.max_columns': None,
                           'display.width': None,
                           'display.max_colwidth': None}

        for k in display_options:
            try:
                pd.set_option(k, display_options[k])
            except ValueError:
                msg = """Newer version of Pandas required to set the '{}'
                         option.""".format(k)
                warnings.warn(msg) 

def setUpClass(cls):
        pd.set_option('chained_assignment','raise')
        #print("setting up: {0}".format(cls)) 

def cli_output(conv_names, sep):
    """ Helper function for printing to the console
    """
    pd.set_option('max_rows', len(conv_names))
    if type(conv_names) is pd.DataFrame:
        if len(conv_names.columns) == 1:
            conv_names = conv_names.iloc[:, 0].tolist()
        elif len(conv_names.columns) == 2:
            for row in conv_names.iterrows():
                print(str(row[1][0]) + sep + str(row[1][1]))
            return
    print(sep.join(
        [str(etr) for etr in conv_names] if
        isinstance(conv_names, list) else [str(conv_names)])) 

def connectivity_summary_verbose(sgraph):
    """Generate a summary of each node's connectivity, with details
    about the parameters.

    Args:
        sgraph: a SummaryGraph instance
    """
    def format_list(l):
        ret = ''
        for i in l: ret += str(i) + '\n'
        return ret[:-1]

    df = pd.DataFrame(columns=['Name', 'Type', 'Inputs', 'Outputs'])
    pd.set_option('precision', 5)
    for i, op in enumerate(sgraph.ops.values()):
        outputs = []
        for blob in op['outputs']:
            if blob in sgraph.params:
                outputs.append(blob + ": " + str(sgraph.params[blob]['shape']))
        inputs = []
        for blob in op['inputs']:
            if blob in sgraph.params:
                inputs.append(blob + ": " + str(sgraph.params[blob]['shape']))
        inputs = format_list(inputs)
        outputs = format_list(outputs)
        df.loc[i] = [op['name'], op['type'], inputs, outputs]

    return df 

def init_pandas():
    pd.set_option('display.float_format', lambda x: '%.3f' % x)
    pd.set_option('display.max_rows', None)
    pd.set_option('display.max_columns', None)
    pd.set_option('display.width', 1000) 

def configure_tests():
    pandas.set_option('chained_assignment', 'raise')


# For running doctests: make np and pd names available 

def describe_task_names(dag: int):
    pd.set_option('display.max_columns', None)
    pd.set_option('display.expand_frame_repr', False)
    pd.set_option('max_colwidth', -1)

    provider = TaskProvider()
    tasks = provider.by_dag(dag)
    return pd.DataFrame([{'id': t.id, 'name': t.name} for t in tasks]) 

def print_table(table, name=None, fmt=None):
    """
    Pretty print a pandas DataFrame.
    Uses HTML output if running inside Jupyter Notebook, otherwise
    formatted text output.
    Parameters
    ----------
    table : pandas.Series or pandas.DataFrame
        Table to pretty-print.
    name : str, optional
        Table name to display in upper left corner.
    fmt : str, optional
        Formatter to use for displaying table elements.
        E.g. '{0:.2f}%' for displaying 100 as '100.00%'.
        Restores original setting after displaying.
    """

    if isinstance(table, pd.Series):
        table = pd.DataFrame(table)

    if fmt is not None:
        prev_option = pd.get_option('display.float_format')
        pd.set_option('display.float_format', lambda x: fmt.format(x))

    if name is not None:
        table.columns.name = name

    display(table)

    if fmt is not None:
        pd.set_option('display.float_format', prev_option) 

def pandas_options():
    # lazy load
    import pandas as pd
    pd.set_option('expand_frame_repr', False)
    pd.set_option('display.max_rows', 3000)
    pd.set_option('display.max_colwidth', 1000) 

def connectivity_summary(sgraph):
    """Generate a summary of each node's connectivity.

    Args:
        sgraph: a SummaryGraph instance
    """
    df = pd.DataFrame(columns=['Name', 'Type', 'Inputs', 'Outputs'])
    pd.set_option('precision', 5)
    for i, op in enumerate(sgraph.ops.values()):
        df.loc[i] = [op['name'], op['type'], op['inputs'], op['outputs']]
    return df 

def attributes_summary(sgraph, ignore_attrs):
    """Generate a summary of a graph's attributes.

    Args:
        sgraph: a SummaryGraph instance
        ignore_attrs: a list of attributes to ignore in the output datafraem

    Output:
        A Pandas dataframe
    """
    def pretty_val(val):
        if type(val) == int:
            return format(val, ",d")
        return str(val)

    def pretty_attrs(attrs, ignore_attrs):
        ret = ''
        for key, val in attrs.items():
            if key in ignore_attrs:
                continue
            ret += key + ': ' + pretty_val(val) + '\n'
        return ret

    df = pd.DataFrame(columns=['Name', 'Type', 'Attributes'])
    pd.set_option('precision', 5)
    for i, op in enumerate(sgraph.ops):
        df.loc[i] = [op['name'], op['type'], pretty_attrs(op['attrs'], ignore_attrs)]
    return df 

def display(self, title=None, dir_output=None):
        pd.set_option('display.width', 1000)
        N_trafos, dof = self._transform_params.shape
        if dof == 6:
            params = self._get_scaled_params(self._transform_params)

            # add mean value
            params = np.concatenate(
                (params,
                 np.mean(params, axis=0).reshape(1, -1),
                 np.std(params, axis=0).reshape(1, -1)
                 ))

            cols = self._labels_long[dof]

        else:
            params = self._transform_params
            cols = ["a%d" % (d + 1) for d in range(0, dof)]

        rows = ["Trafo %d" % (d + 1) for d in range(0, N_trafos)]
        rows.append("Mean")
        rows.append("Std")

        df = pd.DataFrame(params, rows, cols)
        print(df)

        if dir_output is not None:
            title = self._replace_string(title)
            filename = "%s.csv" % title
            ph.create_directory(dir_output)
            df.to_csv(os.path.join(dir_output, filename))

    ##
    # Plot figure to show parameter distribution.
    # Only works for 3D rigid transforms for now.
    # \date       2018-01-25 23:30:45+0000
    #
    # \param      self  The object
    # 

def get_consensus_hit(self, scg_raw_hits):
        pd.set_option('mode.chained_assignment', None)

        df = pd.DataFrame.from_records(scg_raw_hits)

        # remove hits that are null at the phylum level if there are still hits
        # in the df that are not null:
        not_null_hits = df[df.t_phylum.notnull()]
        if len(not_null_hits):
            df = not_null_hits

        # find the max percent identity score in the df
        max_percent_identity = max(df['percent_identity'])

        # subset the data frame to those with percent identity that match to `max_percent_identity`
        df_max_identity = df.loc[df.percent_identity == max_percent_identity]

        # if some of the competing names have null species deignations, remove them from consideration
        if len(df_max_identity.t_species.unique()) > 1:
            df_max_identity = df_max_identity[df_max_identity.t_species.notnull()]

        # find the taxonomic level where the number of unique taxon names is one
        for taxonomic_level in self.ctx.levels_of_taxonomy[::-1]:
            if len(df_max_identity[taxonomic_level].unique()) == 1:
                break

        # take one of the hits from `df_max_identity`, and assign None to all taxonomic levels
        # beyond `taxonomic_level`, which, after the loop above shows the proper level of
        # assignment for this set
        final_hit = df_max_identity.head(1)
        for taxonomic_level_to_nullify in self.ctx.levels_of_taxonomy[self.ctx.levels_of_taxonomy.index(taxonomic_level) + 1:]:
            final_hit.at[0, taxonomic_level_to_nullify] = None

        # FIXME: final hit is still not what we can trust. next, we should find out whether the percent identity
        # for the level of taxonomy at `taxonomic_level` is higher than the minimum percent identity for all sequences
        # considered that are affiliated with final_hit[taxonomic_level]

        # turn it into a Python dict before returning
        final_hit_dict = final_hit.to_dict('records')[0]

        return final_hit_dict 

def get_consensus_taxonomy(self, scg_taxonomy_dict):
        """Takes in a scg_taxonomy_dict, returns a final taxonomic string that summarize all"""

        if not len(scg_taxonomy_dict):
            return dict([(l, None) for l in self.ctx.levels_of_taxonomy])

        pd.set_option('mode.chained_assignment', None)

        scg_hits = list([v for v in scg_taxonomy_dict.values() if v['t_domain']])

        if not len(scg_hits):
            return self.get_blank_hit_template_dict()

        df = pd.DataFrame.from_records(scg_hits)

        # we have already stored a unique hash for taxonomy strings. here we will figure out most frequent
        # hash values in the df
        tax_hash_counts = df['tax_hash'].value_counts()
        tax_hash_df = tax_hash_counts.rename_axis('tax_hash').reset_index(name='frequency')
        max_frequency = tax_hash_df.frequency.max()
        tax_hash_df_most_frequent = tax_hash_df[tax_hash_df.frequency == max_frequency]

        if len(tax_hash_df_most_frequent.index) == 1:
            # if there is only a single winner, we're golden
            winner_tax_hash = tax_hash_df_most_frequent.tax_hash[0]

            # get the consensus hit based on the winner hash
            consensus_hit = df[df.tax_hash == winner_tax_hash].head(1)

            # turn it into a Python dict before returning
            return consensus_hit.to_dict('records')[0]
        else:
            # if there are competing hashes, we need to be more careful to decide
            # which taxonomic level should we use to cut things off.
            consensus_hit = self.get_blank_hit_template_dict()
            for level in self.ctx.levels_of_taxonomy[::-1]:
                if len(df[level].unique()) == 1:
                    consensus_hit[level] = df[level].unique()[0]

            return consensus_hit 

def texify_plot_table(perf_table_pd, problem_set):
    """Write a ``.tex`` file with the performance table.

    The function will create a file named `performance_table_small.tex` or
    `performance_table_large.tex` with the latex code for the performance table.

    Args:
        perf_table_pd (pandas.dataframe): Pandas data frame for the performance
            table.
        problem_set (str): Can either be ``small`` or ``large`` to switch
            between which benchmark set is being plotted.

    Returns:
        str: String of the latex code for the performance table.

    """
    if not perf_table_pd.empty:
        # Postprocessing for Latex Output
        pd.set_option('display.max_colwidth', -1)
        perf_table_pd_n = perf_table_pd.apply(
            norm, axis=1)  # normalize between 0 and 100
        perf_table_pd_n_str = perf_table_pd_n.applymap(
            add_color_coding_tex) + perf_table_pd.applymap(
                latex)  # combine normalise version with latex color code command
        perf_table_pd_n_str.columns = perf_table_pd_n_str.columns.str.replace(
            '_', r'\_')  # Texify the column headers
        tikz_code = r"\def\cca#1#2{\cellcolor{green!#1!red}\ifnum #1<50\color{white}\fi{#2}}" +\
        "\n" + r"\resizebox{\textwidth}{!}{%" + "\n" +\
        perf_table_pd_n_str.to_latex(escape=False) + r"}"
        with open('performance_table_' + problem_set + '.tex', 'w') as tex_file:
            tex_file.write(tikz_code)

        return tikz_code 

def main():
    try:
        base.seabornsetup()
    except:
        pass
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("-r", "--run", dest="run", action='store_true',
                           help="run predictions")
    parser.add_option("-a", "--analyse", dest="analyse",
                           help="analyse results of runs")
    parser.add_option("-c", "--config", dest="config",
                            help="config file")
    opts, remainder = parser.parse_args()
    pd.set_option('display.width', 600)
    if opts.run == True:
        if opts.config == None:
            config.write_default_config('srnabench.conf', defaults=srnabenchoptions)
        cp = config.parse_config(opts.config)
        options = cp._sections['base']
        print (options)
        run_all(path=options['input'], **options)
    elif opts.analyse != None:
        k,n,iso = get_results(opts.analyse)
        analyse_results(k,n)
        analyse_isomirs(iso) 

def main():
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("-r", "--run", dest="run", action='store_true',
                           help="run predictions")
    parser.add_option("-i", "--input", dest="input",
                           help="input path or file")
    parser.add_option("-c", "--config", dest="config",
                            help="config file")
    parser.add_option("-a", "--analyse", dest="analyse",
                           help="analyse results of mirdeep2")

    opts, remainder = parser.parse_args()
    pd.set_option('display.width', 800)

    if opts.run == True:
        #all other options are stored in config file
        if opts.config == None:
            print ('No config file provided.')
            base.write_default_config('mirdeep2.conf', defaults=mirdeep2options)
            return
        cp = base.parse_config(opts.config)
        if opts.input != None:
            conf.input = os.path.abspath(opts.input)
        options = base.get_options(cp)
        run_multiple(**options)
    elif opts.analyse != None:
        analyse_results(opts.analyse)
    elif opts.test == True:
        test(opts.input) 

def test_underlying_data_conversion():
    # GH 4080
    df = DataFrame({c: [1, 2, 3] for c in ['a', 'b', 'c']})
    df.set_index(['a', 'b', 'c'], inplace=True)
    s = Series([1], index=[(2, 2, 2)])
    df['val'] = 0
    df
    df['val'].update(s)

    expected = DataFrame(
        dict(a=[1, 2, 3], b=[1, 2, 3], c=[1, 2, 3], val=[0, 1, 0]))
    expected.set_index(['a', 'b', 'c'], inplace=True)
    tm.assert_frame_equal(df, expected)

    # GH 3970
    # these are chained assignments as well
    pd.set_option('chained_assignment', None)
    df = DataFrame({"aa": range(5), "bb": [2.2] * 5})
    df["cc"] = 0.0

    ck = [True] * len(df)

    df["bb"].iloc[0] = .13

    # TODO: unused
    df_tmp = df.iloc[ck]  # noqa

    df["bb"].iloc[0] = .15
    assert df['bb'].iloc[0] == 0.15
    pd.set_option('chained_assignment', 'raise')

    # GH 3217
    df = DataFrame(dict(a=[1, 3], b=[np.nan, 2]))
    df['c'] = np.nan
    df['c'].update(pd.Series(['foo'], index=[0]))

    expected = DataFrame(dict(a=[1, 3], b=[np.nan, 2], c=['foo', np.nan]))
    tm.assert_frame_equal(df, expected)