Python nbformat.read() Examples

def test_timeseries_controls(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def run_notebook(notebook_path):
    nb_name, _ = os.path.splitext(os.path.basename(notebook_path))

    with open(notebook_path) as f:
        nb = nbformat.read(f, as_version=4)

    proc = ExecutePreprocessor(timeout=600, kernel_name='python3')
    proc.allow_errors = True
    proc.preprocess(nb)

    for num, cell in enumerate(nb.cells):
        if 'outputs' in cell:
            for output in cell['outputs']:
                if output.output_type == 'error':
                    return cell.execution_count, output.traceback

    return None

# 7-bit C1 ANSI sequences 

def test_can_run_examples_jupyter_notebooks(self):
        print("Examples folder ", self.examples_folder)
        for filename in os.listdir(self.examples_folder):
            if not filename.endswith('.ipynb'):
                continue

            path = os.path.join(self.examples_folder, filename)
            notebook = nbformat.read(path, nbformat.NO_CONVERT)
            state = {}

            for cell in notebook.cells:
                if cell.cell_type == 'code' and not is_matplotlib_cell(cell):
                    try:
                        exec(strip_magics_and_shows(cell.source), state)
                    # coverage: ignore
                    except:
                        print('Failed to run {}.'.format(path))
                        raise 

def get_notebook_checkpoint(self, checkpoint_id, path):
        """Get the content of a checkpoint for a notebook.

        Returns a dict of the form:
        {
            "type": "notebook",
            "content": <output of nbformat.read>,
        }
        """
        self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
        cp = self._get_checkpoint_path(checkpoint_id, path)
        exists, blob = self.parent._fetch(cp)
        if not exists:
            raise web.HTTPError(404, u"No such checkpoint: %s for %s" % (
                checkpoint_id, path))
        nb = self.parent._read_notebook(blob)
        return {
            "type": "notebook",
            "content": nb
        } 

def test_timeline_controls(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def _read_all_markdown(self, cells):
        sources = [cell.source for cell in cells if cell.cell_type == 'markdown']
        contents = ('\n\n%s\n\n' % self._NEW_CELL_DELIMITER).join(sources)
        ast = MarkdownPlugin().read(contents)
        if not ast.children:
            logger.debug("Skipping empty node.")
            return
        curtree = ASTNode('root')
        for child in ast.children:
            curtree.children.append(child)
            # Create a new tree at every cell delimiter.
            if child.children and child.children[0] == self._NEW_CELL_DELIMITER:
                # Remove the delimiter node.
                curtree.children.pop()
                # Append the current cell tree and create the next one.
                self._markdown_tree.append(curtree)
                curtree = ASTNode('root')
        # Append the last cell tree if not empty.
        if curtree.children:
            self._markdown_tree.append(curtree) 

def convert(self, notebook_file, sos_file, args=None, unknown_args=None):
        '''
        Convert a ipython notebook to sos format.
        '''
        if unknown_args:
            raise ValueError(f'Unrecognized parameter {unknown_args}')
        exporter = SoS_Exporter()
        notebook = nbformat.read(notebook_file, nbformat.NO_CONVERT)
        output, _ = exporter.from_notebook_node(notebook, {})
        if not sos_file:
            sys.stdout.write(output)
        elif isinstance(sos_file, str):
            with open(sos_file, 'w') as sos:
                sos.write(output)
            env.logger.info(f'SoS script saved to {sos_file}')
        else:
            sos_file.write(output)


#
# notebook to HTML
# 

def attach(self, podoc):
        podoc.register_lang('notebook',
                            file_ext='.ipynb',
                            load_func=self.load,
                            dump_func=self.dump,
                            loads_func=self.loads,
                            dumps_func=self.dumps,
                            eq_filter=self.eq_filter,
                            )
        podoc.register_func(source='notebook', target='ast',
                            func=self.read,
                            post_filter=replace_resource_paths,
                            )
        podoc.register_func(source='ast', target='notebook',
                            func=self.write,
                            pre_filter=wrap_code_cells,
                            ) 

def run_notebook(path, workdir=None, timeout=_DEFAULT_TIMEOUT):
    resources = {
        'metadata': {
            'path': workdir
        }
    } if workdir is not None else None

    with open(path, 'r') as f:
        nb = nbformat.read(f, as_version=_DEFAULT_IPYNB_VERSION)
    ep = ExecutePreprocessor(timeout=timeout)

    # Some notebooks may generate and automatically open reports in a web browser.
    # This is inconvenient in an automated test suite, so let's disable it.
    # Overwriting $BROWSER with a dummy command will keep the notebook
    # kernel from being able to open a web browser on Linux.
    # TODO find platform-neutral solution to suppress webbrowser.open
    browser = os.environ.get('BROWSER', None)
    os.environ['BROWSER'] = 'echo %s'
    ep.preprocess(nb, resources=resources)
    os.environ['BROWSER'] = browser 

def _read_notebook(self, os_path, as_version=4):
        """Read a notebook from an os path."""
        with self.open(os_path, 'r', encoding='utf-8') as f:
            try:

                file_ext = _file_extension(os_path)

                if file_ext == '.ipynb':
                    return nbformat.read(f, as_version=as_version)
                else:
                    lang = self._podoc.get_lang_for_file_ext(file_ext)
                    return self._podoc.convert_file(os_path,
                                                    source=lang,
                                                    target='notebook',
                                                    )

            except Exception as e:  # pragma: no cover
                logger.exception(e)
                raise web.HTTPError(
                    400,
                    u"Unreadable Notebook: %s %r" % (os_path, e),
                ) 

def list_examples(self):
        all_examples = []
        for category in self.get_categories():
            directory = os.path.join(self.sample_notebook_dir, category)
            filepaths = glob(os.path.join(directory, '**', '*.ipynb'), recursive=True)
            examples = [{'filepath': os.path.abspath(fp)} for fp in filepaths]
            for example in examples:
                node = nbformat.read(example['filepath'], nbformat.NO_CONVERT)
                example['filename'] = os.path.basename(example['filepath'])
                example['metadata'] = node.metadata
                example['category'] = category
                example['basename'] = os.path.basename(example['filepath'])
                example['supporting_items'] = self.get_supporting_items(example['filepath'], example['filename'])
                notebook_folder_location = os.path.split(example['filepath'])[0]
                example['notebook_folder_name'] = os.path.split(notebook_folder_location)[1]
            all_examples.extend(examples)
        return all_examples 

def _read_notebook(self, os_path, as_version=4):
        """Read a notebook from an os path."""
        with self.open(os_path, 'r', encoding='utf-8') as f:
            try:

                # NEW
                file_ext = _file_extension(os_path)
                if file_ext == '.ipynb':
                    return nbformat.read(f, as_version=as_version)
                else:
                    return convert(os_path, from_=self.format, to='notebook')

            except Exception as e:
                raise HTTPError(
                    400,
                    u"Unreadable Notebook: %s %r" % (os_path, e),
                ) 

def test_process_tree_notebook():
    nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
    abs_path = Path(_NB_FOLDER).absolute()
    with open(nb_path) as f:
        nb = nbformat.read(f, as_version=4)
    ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

    try:
        ep.preprocess(nb, {"metadata": {"path": abs_path}})
    except CellExecutionError:
        nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
        msg = f"Error executing the notebook '{nb_path}'.\n"
        msg += f"See notebook '{nb_err}' for the traceback."
        print(msg)
        with open(nb_err, mode="w", encoding="utf-8") as f:
            nbformat.write(nb, f)
        raise 

def test_folium_map_notebook(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def read(self, notebook, context=None):
        assert isinstance(notebook, nbformat.NotebookNode)
        self.resources = {}  # Dictionary {filename: data}.
        context = context or {}
        # Get the unique key for image names: basename of the output file, if it exists.
        self._unique_key = op.basename(context.get('output', None) or '')
        self._unique_key = self._unique_key or op.basename(context.get('path', None) or '')
        self._unique_key = op.splitext(self._unique_key)[0] or None
        # Create the output tree.
        self.tree = ASTNode('root')
        # Language of the notebook.
        m = notebook.metadata
        # NOTE: if no language is available in the metadata, use Python
        # by default.
        self.language = m.get('language_info', {}).get('name', 'python')

        # NOTE: for performance reasons, we parse the Markdown of all cells at once
        # to reduce the overhead of calling pandoc.
        self._markdown_tree = []
        self._read_all_markdown(notebook.cells)

        for cell_index, cell in enumerate(notebook.cells):
            getattr(self, 'read_{}'.format(cell.cell_type))(cell, cell_index)

        return self.tree 

def test_main_with_options(tmpdir):
    # Given
    code = dedent("""
    from tex2ipy.tex2cells import Tex2Cells
    class Converter(Tex2Cells):
        def _handle_frame(self, node):
            super(Converter, self)._handle_frame(node)
            self.current['source'].append('## Overloaded\\n')
    """)
    convert = tmpdir.join('extra.py')
    convert.write(code)
    src = tmpdir.join('test.tex')
    src.write(DOCUMENT)
    dest = tmpdir.join('test.ipynb')

    # When
    main(args=[str(src), str(dest), '-c', str(convert)])

    # Then
    assert dest.check(file=1)
    nb = nbformat.read(str(dest), 4)

    src = nb.cells[0].source.splitlines()
    assert src[0] == '## Overloaded'
    assert src[1] == '## Foo' 

def convert_notebook(ramp_kit_dir='.'):
    import nbformat
    import nbconvert
    from nbconvert.exporters import HTMLExporter

    problem_name = os.path.basename(os.path.abspath(ramp_kit_dir))
    print('Testing if the notebook can be converted to html')
    notebook_filename = os.path.join(
        os.path.abspath(ramp_kit_dir),
        '{}_starting_kit.ipynb'.format(problem_name))
    notebook_html_filename = os.path.join(
        os.path.abspath(ramp_kit_dir),
        '{}_starting_kit.html'.format(problem_name))

    with open(notebook_filename) as f:
        nb = nbformat.read(f, as_version=4)
        nb_html, _ = nbconvert.export(HTMLExporter, nb)

    with open(os.path.join(os.path.abspath(ramp_kit_dir),
                           notebook_html_filename), 'wb') as f:
        f.write(nb_html.encode('utf-8'))

    delete_line_from_file(
        notebook_html_filename,
        '<link rel="stylesheet" href="custom.css">\n') 

def setup(app):
    app.add_node(
        gallery,
        html=(visit_gallery_node, depart_gallery_node),
        latex=(visit_gallery_node, depart_gallery_node),
        text=(visit_gallery_node, depart_gallery_node),
        man=(visit_gallery_node, depart_gallery_node),
        texinfo=(visit_gallery_node, depart_gallery_node))
    app.add_directive('gallery', Gallery)
    app.add_directive('include_examples', IncludeExamples)
    app.connect('builder-inited', setup_env)
    app.connect('builder-inited', notebooks_to_rst)

    app.connect('doctree-read', extract_gallery_entries)
    app.connect('doctree-resolved', add_entries_to_gallery)

    return {'version': sphinx.__display_version__,
            'parallel_read_safe': True} 

def merge_notebooks(filenames):
    merged = None
    for fname in filenames:
        with io.open(fname, 'r', encoding='utf-8') as f:
            nb = nbformat.read(f, as_version=4)
        if merged is None:
            merged = nb
        else:
            # TODO: add an optional marker between joined notebooks
            # like an horizontal rule, for example, or some other arbitrary
            # (user specified) markdown cell)
            merged.cells.extend(nb.cells)
    if not hasattr(merged.metadata, 'name'):
        merged.metadata.name = ''
    merged.metadata.name += "_merged"
    print(nbformat.writes(merged)) 

def run_notebook(filename):

    run_path = os.path.split(filename)[0]

    with open(filename) as f:
        nb = nbformat.read(f, as_version=4)

    try:
        ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
        ep.preprocess(nb, {'metadata': {'path': run_path}})

        # FIXME: use tempfile and mv to avoid interruption
        # better split the source code of the notebook and the compiled targets.
        with open(filename, 'wt') as f:
            nbformat.write(nb, f)

    except Exception as e:
        print('processing', filename, e) 

def convert(nbfile):
    basename, _ = os.path.splitext(nbfile)

    meta = {'metadata': {'path': '.'}}
    with open(nbfile, 'r', encoding='utf-8') as nbf:
        nbdata = nbformat.read(nbf, as_version=4, encoding='utf-8')

    runner = ExecutePreprocessor(timeout=600, kernel_name='probscale')
    runner.preprocess(nbdata, meta)

    img_folder = basename + '_files'
    body_raw, images = RSTExporter().from_notebook_node(nbdata)
    body_final = body_raw.replace('.. image:: ', '.. image:: {}/'.format(img_folder))

    with open(basename + '.rst', 'w', encoding='utf-8') as rst_out:
        rst_out.write(body_final)

    for img_name, img_data in images['outputs'].items():
        img_path = os.path.join(img_folder, img_name)
        with open(img_path, 'wb') as img:
            img.write(img_data) 

def get_pipeline_parameters(request, source_notebook_path):
    """Get the pipeline parameters tagged in the notebook."""
    # read notebook
    log = request.log if hasattr(request, "log") else logger
    try:
        notebook = nbformat.read(source_notebook_path,
                                 as_version=nbformat.NO_CONVERT)
        params_source = parser.get_pipeline_parameters_source(notebook)
        if params_source == '':
            raise ValueError("No pipeline parameters found. Please tag a cell"
                             " of the notebook with the `pipeline-parameters`"
                             " tag.")
        # get a dict from the 'pipeline parameters' cell source code
        params_dict = ast.parse_assignments_expressions(params_source)
    except ValueError as e:
        log.exception("Value Error during parsing of pipeline parameters")
        raise RPCInternalError(details=str(e), trans_id=request.trans_id)
    # convert dict in list so its easier to parse in js
    params = [[k, *v] for k, v in params_dict.items()]
    log.info("Pipeline parameters:")
    for ln in tabulate(params, headers=["name", "type", "value"]).split("\n"):
        log.info(ln)
    return params 

def get_pipeline_metrics(request, source_notebook_path):
    """Get the pipeline metrics tagged in the notebook."""
    # read notebook
    log = request.log if hasattr(request, "log") else logger
    try:
        notebook = nbformat.read(source_notebook_path,
                                 as_version=nbformat.NO_CONVERT)
        metrics_source = parser.get_pipeline_metrics_source(notebook)
        if metrics_source == '':
            raise ValueError("No pipeline metrics found. Please tag a cell"
                             " of the notebook with the `pipeline-metrics`"
                             " tag.")
        # get a dict from the 'pipeline parameters' cell source code
        metrics = ast.parse_metrics_print_statements(metrics_source)
    except ValueError as e:
        log.exception("Failed to parse pipeline metrics")
        raise RPCInternalError(details=str(e), trans_id=request.trans_id)
    log.info("Pipeline metrics: {}".format(metrics))
    return metrics 

def _process(self):
        config = Config()
        config.HTMLExporter.preprocessors = [CppHighlighter]
        config.HTMLExporter.template_file = 'basic'

        with self.attachment.file.open() as f:
            notebook = nbformat.read(f, as_version=4)

        html_exporter = HTMLExporter(config=config)
        body, resources = html_exporter.from_notebook_node(notebook)
        css_code = '\n'.join(resources['inlining'].get('css', []))

        nonce = str(uuid4())
        html = render_template('previewer_jupyter:ipynb_preview.html', attachment=self.attachment,
                               html_code=body, css_code=css_code, nonce=nonce)

        response = current_app.response_class(html)
        # Use CSP to restrict access to possibly malicious scripts or inline JS
        csp_header = "script-src cdn.mathjax.org 'nonce-{}';".format(nonce)
        response.headers['Content-Security-Policy'] = csp_header
        response.headers['X-Webkit-CSP'] = csp_header
        # IE10 doesn't have proper CSP support, so we need to be more strict
        response.headers['X-Content-Security-Policy'] = "sandbox allow-same-origin;"

        return response 

def runWithHTMLReturn(notebook, executable=None, **run_kw):
    """
    Run notebook's celltests in a subprocess and return html generated
    by pytest's --self-contained-html.

    rules: coverage requirements (if any).

    Note - leaves behind the following generated files for
    "/path/to/notebook.ipynb":
      * /path/to/notebook_test.py (notebook test script)
      * /path/to/notebook_test.html (pytest's html report)
    """
    name = run(notebook, **run_kw)
    html = name.replace('.py', '.html')
    executable = executable or [sys.executable, '-m', 'pytest', '-v']
    argv = executable + ['--html=' + html, '--self-contained-html', name]
    subprocess.call(argv)
    with open(html, 'r', encoding='utf-8') as fp:
        return fp.read() 

def test_invalid_notebooks(
    invalid_notebook_path, cell_location, error_name, error_value, error_output_type
):
    notebook_filename = script_relative_path(invalid_notebook_path)
    with open(notebook_filename) as f:
        nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name='python3')

        try:
            ep.preprocess(
                nb,
                {
                    'metadata': {
                        'path': script_relative_path(
                            notebook_filename[: notebook_filename.rfind('/')]
                        )
                    }
                },
            )
        except CellExecutionError:
            error_message = get_dict_value(nb, cell_location)
            assert error_message.ename == error_name
            assert bool(re.search(error_value, error_message.evalue))
            assert error_message.output_type == error_output_type 

def test_reexecute_result_notebook():
    with exec_for_test('define_hello_world_pipeline') as result:
        assert result.success

        materialization_events = [
            x for x in result.step_event_list if x.event_type_value == 'STEP_MATERIALIZATION'
        ]
        for materialization_event in materialization_events:
            result_path = get_path(materialization_event)

        if result_path.endswith('.ipynb'):
            with open(result_path) as fd:
                nb = nbformat.read(fd, as_version=4)
            ep = ExecutePreprocessor()
            ep.preprocess(nb, {})
            with open(result_path) as fd:
                assert nbformat.read(fd, as_version=4) == nb 

def run(notebook, rules=None, filename=None, kernel_name="", current_env=False):
    """
    Runs no tests: just generates test script for supplied notebook.

    kernel_name and current_env 'will be passed to nbval'.
    """
    nb = nbformat.read(notebook, 4)
    name = filename or os.path.splitext(notebook)[0] + '_test.py'
    extra_metadata = extract_extrametadata(nb)
    rules = rules or {}
    extra_metadata.update(rules)

    # TODO: Coverage shouldn't be recorded at generation time as it
    # will go stale if the notebook changes. Should move to same
    # mechanism as source/tests. However, we plan to replace coverage
    # with code coverage measured during test execution.
    coverage = []
    if 'cell_coverage' in extra_metadata:
        coverage.append((get_coverage(extra_metadata), extra_metadata['cell_coverage']))

    # output tests to test file
    with open(name, 'w', encoding='utf-8') as fp:
        fp.write(BASE.format(kernel_name=kernel_name, current_env=current_env, path_to_notebook=notebook, coverage=coverage))

    return name 

def append_scrapbook_commands(input_nb_path, output_nb_path, scrap_specs):
    notebook = nbf.read(input_nb_path, as_version=nbf.NO_CONVERT)

    scrapbook_cells = []
    # Always need to import nteract-scrapbook
    scrapbook_cells.append(nbf.v4.new_code_cell(source="import scrapbook as sb"))

    # Create a cell to store each key and value in the scrapbook
    for k, v in scrap_specs.items():
        source = "sb.glue(\"{0}\", {1})".format(k, v.code)
        scrapbook_cells.append(nbf.v4.new_code_cell(source=source))

    # Append the cells to the notebook
    [notebook['cells'].append(c) for c in scrapbook_cells]

    # Write out the new notebook
    nbf.write(notebook, output_nb_path) 

def _read_notebook(self, hdfs_path, as_version=4):
        """Read a notebook from an os path."""
        # TODO: check for open errors
        with self.hdfs.open_file(hdfs_path, 'r') as f:
            try:
                return nbformat.read(f, as_version=as_version)
            except Exception as e:
                e_orig = e

            # If use_atomic_writing is enabled, we'll guess that it was also
            # enabled when this notebook was written and look for a valid
            # atomic intermediate.
            tmp_path = path_to_intermediate(hdfs_path)

            if not self.use_atomic_writing or not self.hdfs.exists(tmp_path):
                raise HTTPError(
                    400,
                    u"Unreadable Notebook: %s %r" % (hdfs_path, e_orig),
                )

            # Move the bad file aside, restore the intermediate, and try again.
            invalid_file = path_to_invalid(hdfs_path)
            self._hdfs_move_file(hdfs_path, invalid_file)
            self._hdfs_move_file(tmp_path, hdfs_path)
            return self._read_notebook(hdfs_path, as_version)