Python IPython.display.Markdown() Examples

def scatter_plot_intent_dist(workspace_pd):
    """
    takes the workspace_pd and generate a scatter distribution of the intents
    :param workspace_pd:
    :return:
    """

    label_frequency = Counter(workspace_pd["intent"]).most_common()
    frequencies = list(reversed(label_frequency))
    counter_list = list(range(1, len(frequencies) + 1))
    df = pd.DataFrame(data=frequencies, columns=["Intent", "Number of User Examples"])
    df["Intent"] = counter_list

    sns.set(rc={"figure.figsize": (15, 10)})
    display(
        Markdown(
            '## <p style="text-align: center;">Sorted Distribution of User Examples \
                     per Intent</p>'
        )
    )

    plt.ylabel("Number of User Examples", fontdict=LABEL_FONT)
    plt.xlabel("Intent", fontdict=LABEL_FONT)
    ax = sns.scatterplot(x="Intent", y="Number of User Examples", data=df, s=100) 

def print_markdown_table(headers, data):
    '''
    Renders table given headers and data
    '''
    md = ''

    for h in headers:
        md += '|' + h

    md += '|\n'

    for r in range(len(headers)):
        md += '|---'

    md += '|\n'

    for row in data:
        for element in row:
            md += '|' + str(element)
        md += '|\n'

    display(Markdown(md)) 

def display_observations(self):
        """Display the current observations using IPython.display."""
        for observation in self.observation_list.values():
            display(Markdown(f"### {observation.caption}"))
            display(Markdown(observation.description))
            display(Markdown(f"Score: {observation.score}"))
            if observation.link:
                display(Markdown(f"[Go to details](#{observation.link})"))
            if observation.tags:
                display(Markdown(f'tags: {", ".join(observation.tags)}'))
            display(observation.data)
            if observation.additional_properties:
                display(Markdown("### Additional Properties"))
                for key, val in observation.additional_properties.items():
                    display(Markdown(f"**{key}**: {val}")) 

def test_append_display_data():
    widget = widget_output.Output()

    # Try appending a Markdown object.
    widget.append_display_data(Markdown("# snakes!"))
    expected = (
        {
            'output_type': 'display_data',
            'data': {
                'text/plain': '<IPython.core.display.Markdown object>',
                'text/markdown': '# snakes!'
            },
            'metadata': {}
        },
    )
    assert widget.outputs == expected, repr(widget.outputs)

    # Now try appending an Image.
    image_data = b"foobar"
    image_data_b64 = image_data if sys.version_info[0] < 3 else 'Zm9vYmFy\n'

    widget.append_display_data(Image(image_data, width=123, height=456))
    expected += (
        {
            'output_type': 'display_data',
            'data': {
                'image/png': image_data_b64,
                'text/plain': '<IPython.core.display.Image object>'
            },
            'metadata': {
                'image/png': {
                    'width': 123,
                    'height': 456
                }
            }
        },
    )
    assert widget.outputs == expected, repr(widget.outputs) 

def help(self):
        """
        Prints general help information, or specific usage information of a function if provided
        :param function: an optional function
        """
        from IPython.display import Markdown, display

        def print_md(string):
            display(Markdown(string))

        # Check whether the request is for some specific function
        #if function is not None:
        #    print_md(self.function.__doc__)
        # If not then offer the general help menu
        #else:
        print_md("### Help Menu")
        print_md("You can use the system through an **API** object. API objects are returned"
                 "by the *init_system* function, so you can get one by doing:")
        print_md("***your_api_object = init_system('path_to_stored_model')***")
        print_md("Once you have access to an API object there are a few concepts that are useful "
                 "to use the API. **content** refers to actual values of a given field. For "
                 "example, if you have a table with an attribute called __Name__ and values *Olu, Mike, Sam*, content "
                 "refers to the actual values, e.g. Mike, Sam, Olu.")
        print_md("**schema** refers to the name of a given field. In the previous example, schema refers to the word"
                 "__Name__ as that's how the field is called.")
        print_md("Finally, **entity** refers to the *semantic type* of the content. This is in experimental state. For "
                 "the previous example it would return *'person'* as that's what those names refer to.")
        print_md("Certain functions require a *field* as input. In general a field is specified by the source name ("
                 "e.g. table name) and the field name (e.g. attribute name). For example, if we are interested in "
                 "finding content similar to the one of the attribute *year* in the table *Employee* we can provide "
                 "the field in the following way:")
        print(
            "field = ('Employee', 'year') # field = [<source_name>, <field_name>)") 

def help(self):
        """
        Prints general help information, or specific usage information of a function if provided
        :param function: an optional function
        """
        from IPython.display import Markdown, display

        def print_md(string):
            display(Markdown(string))

        # Check whether the request is for some specific function
        #if function is not None:
        #    print_md(self.function.__doc__)
        # If not then offer the general help menu
        #else:
        print_md("### Help Menu")
        print_md("You can use the system through an **API** object. API objects are returned"
                 "by the *init_system* function, so you can get one by doing:")
        print_md("***your_api_object = init_system('path_to_stored_model')***")
        print_md("Once you have access to an API object there are a few concepts that are useful "
                 "to use the API. **content** refers to actual values of a given field. For "
                 "example, if you have a table with an attribute called __Name__ and values *Olu, Mike, Sam*, content "
                 "refers to the actual values, e.g. Mike, Sam, Olu.")
        print_md("**schema** refers to the name of a given field. In the previous example, schema refers to the word"
                 "__Name__ as that's how the field is called.")
        print_md("Finally, **entity** refers to the *semantic type* of the content. This is in experimental state. For "
                 "the previous example it would return *'person'* as that's what those names refer to.")
        print_md("Certain functions require a *field* as input. In general a field is specified by the source name ("
                 "e.g. table name) and the field name (e.g. attribute name). For example, if we are interested in "
                 "finding content similar to the one of the attribute *year* in the table *Employee* we can provide "
                 "the field in the following way:")
        print(
            "field = ('Employee', 'year') # field = [<source_name>, <field_name>)") 

def print_md(string):
    display(Markdown(string))


#@DeprecationWarning 

def h4(text):
        display(Markdown('#### %s' % text)) 

def h3(text):
        display(Markdown('### %s' % text)) 

def h2(text):
        display(Markdown('## %s' % text)) 

def h1(text):
        display(Markdown('# %s' % text)) 

def printmd(*args):  # fixme: make it work without jupyter notebook
    display(Markdown(" ".join(map(str, args)))) 

def printmd(string):
    '''
    Renders markdown in notebook output of codecell
    '''
    display(Markdown(string)) 

def display_source_code(source_code, language='python'):
    display(Markdown("```%s\n%s\n```" % (language, source_code))) 

def test_ipython_display(self, clean_python_kernel, to):
        s = R.Stitch('', to=to)
        code = dedent('''\
        from IPython import display
        import math
        display.Markdown("$\\alpha^{pi:1.3f}$".format(pi=math.pi))
        ''')
        messages = R.run_code(code, clean_python_kernel)
        wrapped = s.wrap_output('', messages, None)[0]
        assert wrapped['t'] == 'Para'
        assert wrapped['c'][0]['c'][0]['t'] == 'InlineMath' 

def show_user_examples_per_intent(data):
    """
    Take the workspace dictionary and display summary statistics regarding the workspace
    :param data:
    :return:
    """

    label_frequency = Counter(data["intent"]).most_common()
    frequencies = list(reversed(label_frequency))
    df = pd.DataFrame(data=frequencies, columns=["Intent", "Number of User Examples"])
    df.index = np.arange(1, len(df) + 1)
    display(Markdown("### Sorted Distribution of User Examples per Intent"))
    display(df) 

def generate_summary_statistics(data, entities_list=None):
    """
    Take the workspace dictionary and display summary statistics regarding the workspace
    :param data:
    :param entities_list:
    :return:
    """

    total_examples = len(data["utterance"])
    label_frequency = Counter(data["intent"]).most_common()
    number_of_labels = len(label_frequency)
    average_example_per_intent = np.average(list(dict(label_frequency).values()))
    standard_deviation_of_intent = np.std(list(dict(label_frequency).values()))

    characteristics = list()
    characteristics.append(["Total User Examples", total_examples])
    characteristics.append(["Unique Intents", number_of_labels])
    characteristics.append(
        ["Average User Examples per Intent", int(np.around(average_example_per_intent))]
    )
    characteristics.append(
        [
            "Standard Deviation from Average",
            int(np.around(standard_deviation_of_intent)),
        ]
    )
    if entities_list:
        characteristics.append(["Total Number of Entities", len(entities_list)])
    else:
        characteristics.append(["Total Number of Entities", 0])

    df = pd.DataFrame(data=characteristics, columns=["Data Characteristic", "Value"])
    df.index = np.arange(1, len(df) + 1)
    display(Markdown("### Summary Statistics"))
    display(df) 

def scraps_report(
        self, scrap_names=None, notebook_names=None, include_data=False, headers=True
    ):
        """
        Display scraps as markdown structed outputs.

        Parameters
        ----------
        scrap_names : str or iterable[str] (optional)
            the scraps to display as reported outputs
        notebook_names : str or iterable[str] (optional)
            notebook names to use in filtering on scraps to report
        include_data : bool (default: False)
            indicator that data-only scraps should be reported
        header : bool (default: True)
            indicator for if the scraps should render with a header
        """
        # Keep slow import lazy
        from IPython.display import display as ip_display, Markdown

        def trim_repr(data):
            # Generate a small data representation for display purposes
            if not isinstance(data, string_types):
                data_str = repr(data)
            if len(data_str) > 102:
                data_str = data_str[:100] + "..."
            return data_str

        if isinstance(scrap_names, string_types):
            scrap_names = [scrap_names]
        scrap_names = set(scrap_names or [])

        if notebook_names is None:
            notebook_names = self._notebooks.keys()
        elif isinstance(notebook_names, string_types):
            notebook_names = [notebook_names]

        for i, nb_name in enumerate(notebook_names):
            notebook = self[nb_name]
            if headers:
                if i > 0:
                    ip_display(Markdown("<hr>"))  # tag between outputs
                ip_display(Markdown("### {}".format(nb_name)))

            for name in scrap_names or notebook.scraps.display_scraps.keys():
                if headers:
                    ip_display(Markdown("#### {}".format(name)))
                notebook.reglue(name, raise_on_missing=False, unattached=True)

            if include_data:
                for name, scrap in scrap_names or notebook.scraps.data_scraps.items():
                    if scrap.display is None and scrap.data is not None:
                        if headers:
                            ip_display(Markdown("#### {}".format(name)))
                            ip_display(trim_repr(scrap.data))
                        else:
                            ip_display("{}: {}".format(scrap.name, trim_repr(scrap.data))) 

def summary_md(configuration, expts, path='/g/data3/hh5/tmp/cosima/',
               search='https://github.com/OceansAus/access-om2/search?&q=',
               nmls=[
        'atmosphere/input_atm.nml',
        'ice/cice_in.nml',
        'ice/input_ice.nml',
        'ice/input_ice_gfdl.nml',
        'ice/input_ice_monin.nml',
        'ocean/input.nml'
                   ]):
    for nml in nmls:
        epaths = []
        for e in expts:
            # NB: only look at output000
            epaths.append(os.path.join(path, configuration, e, 'output000', nml))
        nmld = cc.nmldiff(cc.nmldict(tuple(epaths)))
        epaths = list(nmld.keys())  # redefine to handle missing paths
        epaths.sort()
        nmldss = cc.superset(nmld)
        display(Markdown('### ' + nml + ' namelist differences'))
        if len(nmldss) == 0:
            display(Markdown('no differences'))
        else:
            mdstr = '| group | variable | '
            for e in epaths:
                mdstr = mdstr + e.replace('/', '/<br>') + ' | '
            mdstr = mdstr + '\n|---|:--|' + ':-:|' * len(epaths)
            for group in sorted(nmldss):
                for mem in sorted(nmldss[group]):
                    mdstr = mdstr + '\n| ' + '&' + \
                            group + ' | ' + \
                            mem + ' | '
#                        search doesn't work on github submodules or forks
#                        '[' + group + '](' + search + group + ')' + ' | ' + \
#                        '[' + mem + '](' + search + mem + ')' + ' | '
                    for e in epaths:
                        if group in nmld[e]:
                            if mem in nmld[e][group]:
                                mdstr = mdstr + repr(nmld[e][group][mem])
                        mdstr = mdstr + ' | '
            display(Markdown(mdstr))
    return 

def _adversarial_examples_multi_thread_inference(
    wrong_examples_sorted, conversation, workspace_id
):
    """
    Perform multi threaded inference on all the adversarial examples
    :param wrong_examples_sorted:
    :param conversation:
    :param workspace_id:
    """
    all_adversarial_examples = list()
    # the adversarial labels will be label\tidx for later regrouping purposes
    all_adversarial_label_idx = list()
    # map the adversarial example: span of adversarial
    adversarial_span_dict = dict()
    for original_example in wrong_examples_sorted:

        adversarial_examples, adversarial_span = _generate_adversarial_examples(
            original_example[1], original_example[0]
        )

        if not original_example[2]:
            label = skills_util.OFFTOPIC_LABEL
        else:
            label = original_example[2]
        adversarial_label = label + "\t" + str(original_example[0])

        all_adversarial_examples.extend(adversarial_examples)
        all_adversarial_label_idx.extend(
            [adversarial_label] * len(adversarial_examples)
        )
        adversarial_span_dict.update(adversarial_span)

    adversarial_test_data_frame = pd.DataFrame(
        {"utterance": all_adversarial_examples, "intent": all_adversarial_label_idx}
    )
    adversarial_results = inferencer.inference(
        conversation,
        workspace_id,
        adversarial_test_data_frame,
        max_retries=10,
        max_thread=5,
        verbose=False,
    )
    display(Markdown("   "))
    return adversarial_results, adversarial_span_dict 

def get_highlights_in_batch_multi_thread(
    conversation,
    workspace_id,
    full_results,
    output_folder,
    confidence_threshold,
    show_worst_k,
):
    """
    Given the prediction result, rank prediction results from worst to best
    & analyze the top k worst results.
    Term level highlighting on the worst results shows the sensitivity of terms in utterance
    :param conversation: conversation object produced by watson api
    :param workspace_id: workspace id
    :param full_results: prediction result showing the ranked list of intents by confidence scores
    :param output_folder: the output folder where the highlighting images will be saved
    :param confidence_threshold: the confidence threshold for offtopic detection
    :param show_worst_k: the top worst k results based on heuristics
    :return:
    """
    wrong_examples_sorted = _filter_results(full_results, confidence_threshold)
    display(
        Markdown(
            "### Identified {} problematic utterances ".format(
                len(wrong_examples_sorted)
            )
        )
    )
    display(Markdown("  "))

    wrong_examples_sorted = wrong_examples_sorted[:show_worst_k]

    (
        adversarial_results,
        adversarial_span_dict,
    ) = _adversarial_examples_multi_thread_inference(
        wrong_examples_sorted, conversation, workspace_id
    )

    if not adversarial_results.empty:

        display(Markdown("{} examples are shown below:".format(show_worst_k)))
        for original_example in wrong_examples_sorted:
            if not original_example[2]:
                label = skills_util.OFFTOPIC_LABEL
            else:
                label = original_example[2]
            label_idx = label + "\t" + str(original_example[0])
            adversarial_result_subset = adversarial_results[
                adversarial_results["correct_intent"] == label_idx
            ]
            highlight = _highlight_scoring(
                original_example, adversarial_result_subset, adversarial_span_dict
            )
            _plot_highlight(highlight, original_example, output_folder) 

def extract_table_analysis(sorted_results, ontopic_infos, offtopic_infos):
    """
    extract informations for table analysis
    :param sorted_results:
    :return:
        analysis_df: pandas dataframe of the table for dispaly
        toa_list: list of sorted on-topic accuracy
        bot_coverage_list: list of sorted bot coverage ratio
        far_list: list of sorted false acceptance rate
        thresholds: list of sorted & unique thresholds
    """
    thresholds, sort_uniq_confs = generate_unique_thresholds(sorted_results)

    toa_list, toa_count = _get_ontopic_accuracy_list(sorted_results, thresholds)
    bot_coverage_list, bot_coverage_count = _get_bot_coverage_list(
        sorted_results, thresholds
    )

    if len(offtopic_infos) >= OFFTOPIC_CNT_THRESHOLD_FOR_DISPLAY:

        far_list, _ = _get_far_list(sorted_results, thresholds)
    else:
        display(
            Markdown(
                "Out of Domain examples fewer than **%d** thus \
            no False Acceptance Rate (FAR) calculated"
                % OFFTOPIC_CNT_THRESHOLD_FOR_DISPLAY
            )
        )
        far_list = [-1] * len(thresholds)

    analysis_df = create_display_table(
        toa_list,
        bot_coverage_list,
        bot_coverage_count,
        sorted_results,
        thresholds,
        offtopic_infos,
        far_list,
    )

    return analysis_df, toa_list, bot_coverage_list, far_list, thresholds 

def analysis_pipeline(results, intent_name=None):
    """
    perform the operation of extraction of table analysis and produce threshold graph
    :param results: list of tuples of (ground_truth, prediction, confidence) sorted by confidence
    :param intent_name:
    :return: analysis_df
    """
    sorted_results = _convert_data_format(results, intent_name=intent_name)

    ontopic_infos, offtopic_infos = extract_by_topic(sorted_results)

    # if ontopic counts or sorted results are less than 3, the graph will show almost no variation
    # if all confidence of the predicted result are the same, there will be no variation
    if (
        len(ontopic_infos) < 3
        or len(sorted_results) < 3
        or all(ele[2] == sorted_results[0][2] for ele in sorted_results)
    ):
        display(Markdown("**Inadequate Data Points**: No analysis will be conducted"))
        analysis_df = pd.DataFrame()
        return analysis_df

    (
        analysis_df,
        toa_list,
        bot_coverage_list,
        far_list,
        thresholds,
    ) = extract_table_analysis(sorted_results, ontopic_infos, offtopic_infos)

    if not intent_name and not analysis_df.empty:
        line_graph_data = pd.DataFrame(
            data={
                "Thresholded On Topic Accuracy": toa_list,
                "Bot Coverage %": bot_coverage_list,
                "False Acceptance Rate (FAR) for Out of Domain Examples": far_list,
            },
            index=thresholds,
        )

        create_threshold_graph(line_graph_data)

    return analysis_df 

def _display_analysis_metrics(display_far):
    """display the explanation for analysis metrics"""
    display(Markdown("### Threshold Metrics"))
    display(
        Markdown(
            "We calculate metrics for responses where the top intent has a confidence above the \
        threshold specified on the x-axis.  "
        )
    )

    display(
        Markdown(
            "We consider examples which are within the scope of the chatbot's problem formulation as \
         on topic or in domain and those examples which are outside the scope of the problem to be \
         out of domain or irrelevant"
        )
    )

    display(Markdown("#### 1) Thresholded On Topic Accuracy (TOA)"))
    display(
        Markdown(
            "x-axis: Confidence threshold used || "
            + "y-axis: Intent Detection Accuracy for On Topic utterances"
        )
    )

    display(Markdown("#### 2)  Bot Coverage %"))
    display(
        Markdown(
            "x-axis: Confidence threshold used || "
            + "y-axis: Fraction of All utterances above the threshold"
        )
    )

    if display_far:
        display(
            Markdown("#### 3) False Acceptance Rate for Out of Domain Examples (FAR)")
        )
        display(
            Markdown(
                "x-axis: Confidence threshold used || "
                + "y-axis: Fraction of Out of Domain utterances falsely considered on topic"
            )
        )

    display(
        Markdown(
            "#### Note: Default acceptance threshold for Watson Assistant is set at 0.2.\
        Utterances with top intent confidence < 0.2 will be considered irrelevant"
        )
    ) 

def class_imbalance_analysis(workspace_pd):
    """
    performance class imbalance analysis on the training workspace
    :param workspace_pd:
    :return:
    """

    label_frequency = Counter(workspace_pd["intent"]).most_common()
    frequencies = list(reversed(label_frequency))
    min_class, min_class_len = frequencies[0]
    max_class, max_class_len = frequencies[-1]

    if max_class_len >= 2 * min_class_len:
        display(
            Markdown(
                "### <font style='color:rgb(165, 34, 34);'> Class Imbalance Detected \
        </font>"
            )
        )
        display(
            Markdown(
                "- Data could be potentially biased towards intents with more user \
        examples"
            )
        )
        display(
            Markdown(
                "- E.g. Intent < {} > has < {} > user examples while intent < {} > has \
        just < {} > user examples ".format(
                    max_class, max_class_len, min_class, min_class_len
                )
            )
        )
        flag = True
    else:
        display(
            Markdown(
                "### <font style='color:rgb(13, 153, 34);'> No Significant Class \
        Imbalance Detected </font>"
            )
        )
        display(
            Markdown(
                "- Lower chances of inherent bias in classification towards intents with \
        more user examples"
            )
        )
        flag = False

    return flag 

def ambiguous_examples_analysis(workspace_pd, threshold=0.7):
    """
    Analyze the test workspace and find out similar utterances that belongs to different intent
    :param workspace_pd: pandas dataframe in format of [utterance,label]
    :param threshold: cut off for similarity score
    :return: pands dataframe in format of ['Intent1', 'Utterance1', 'Intent2', 'Utterance2',
                                           'similarity score']
    """
    # first create the feature matrix
    vectorizer = CountVectorizer(ngram_range=(1, 2))
    workspace_bow = vectorizer.fit_transform(workspace_pd["utterance"]).todense()
    cos_sim_score_matrix = _calculate_cosine_similarity(workspace_bow)

    # remove the lower triangle of the matrix and apply threshold
    similar_utterance_index = np.argwhere(
        (cos_sim_score_matrix - np.tril(cos_sim_score_matrix)) > threshold
    )
    similar_utterance_pd = pd.DataFrame(
        columns=["Intent1", "Utterance1", "Intent2", "Utterance2", "similarity score"]
    )

    for index in similar_utterance_index:
        if (
            workspace_pd["intent"].iloc[index[0]]
            != workspace_pd["intent"].iloc[index[1]]
        ):
            intent1 = workspace_pd["intent"].iloc[index[0]]
            utterance1 = workspace_pd["utterance"].iloc[index[0]]
            intent2 = workspace_pd["intent"].iloc[index[1]]
            utterance2 = workspace_pd["utterance"].iloc[index[1]]
            score = cos_sim_score_matrix[index[0], index[1]]
            temp_pd = pd.DataFrame(
                {
                    "Intent1": [intent1],
                    "Utterance1": [utterance1],
                    "Intent2": [intent2],
                    "Utterance2": [utterance2],
                    "similarity score": [score],
                }
            )
            similar_utterance_pd = similar_utterance_pd.append(
                temp_pd, ignore_index=True
            )

    if not similar_utterance_pd.empty:
        with pd.option_context("max_colwidth", 250):
            display(
                HTML(
                    similar_utterance_pd.sort_values(
                        by=["similarity score"], ascending=False
                    ).to_html(index=False)
                )
            )
    else:
        display(Markdown("### There are no similar utterances within different Intent"))

    return similar_utterance_pd