Python nbformat.write() Examples

def main():
    parser = argparse.ArgumentParser(
        description="Jupyter Notebooks to markdown"
    )

    parser.add_argument("notebook", nargs=1, help="The notebook to be converted.")
    parser.add_argument("-o", "--output", help="output markdown file")
    args = parser.parse_args()

    old_ipynb = args.notebook[0]
    new_ipynb = 'tmp.ipynb'
    md_file = args.output
    print(md_file)
    if not md_file:
        md_file = os.path.splitext(old_ipynb)[0] + '.md'


    clear_notebook(old_ipynb, new_ipynb)
    os.system('jupyter nbconvert ' + new_ipynb + ' --to markdown --output ' + md_file)
    with open(md_file, 'a') as f:
        f.write('<!-- INSERT SOURCE DOWNLOAD BUTTONS -->')
    os.system('rm ' + new_ipynb) 

def test_timeseries_controls(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def py2ipynb(input, output, cellmark_style, other_ignores=[]):
    """Converts a .py file to a V.4 .ipynb notebook usiing `parsePy` function

    :param input: Input .py filename
    :param output: Output .ipynb filename
    :param cellmark_style: Determines cell marker based on IDE, see parsePy documentation for values
    :param other_ignores: Other lines to ignore
    """
    # Create the code cells by parsing the file in input
    cells = []
    for c in parsePy(input, cellmark_style, other_ignores):
        codecell, metadata, code = c
        cell = new_code_cell(source=code, metadata=metadata) if codecell else new_markdown_cell(source=code, metadata=metadata)
        cells.append(cell)

    # This creates a V4 Notebook with the code cells extracted above
    nb0 = new_notebook(cells=cells,
                       metadata={'language': 'python',})

    with codecs.open(output, encoding='utf-8', mode='w') as f:
        nbformat.write(nb0, f, 4) 

def update_ipynb_toc(root):
    """Change the toc code block into a list of clickable links"""
    notebooks = find_files('**/*.ipynb', root)
    for fn in notebooks:
        nb = notebook.read(fn)
        if not nb:
            continue
        for cell in nb.cells:
            if (cell.cell_type == 'markdown' and '```toc' in cell.source):
                md_cells = markdown.split_markdown(cell.source)
                for c in md_cells:
                    if c['type'] == 'code' and c['class'] == 'toc':
                        toc = []
                        for l in c['source'].split('\n'):
                            if l and not l.startswith(':'):
                                toc.append(' - [%s](%s.ipynb)'%(l,l))
                        c['source'] = '\n'.join(toc)
                        c['type'] = 'markdown'
                cell.source = markdown.join_markdown_cells(md_cells)
        with open(fn, 'w') as f:
            f.write(nbformat.writes(nb)) 

def test_widgets_notebook(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def test_timeline_controls(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def test_geoip_notebook(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()

        with open(nb_path, "rb") as f:
            nb_bytes = f.read()
        nb_text = nb_bytes.decode("utf-8")
        nb = nbformat.reads(nb_text, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def test_process_tree_notebook():
    nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
    abs_path = Path(_NB_FOLDER).absolute()
    with open(nb_path) as f:
        nb = nbformat.read(f, as_version=4)
    ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

    try:
        ep.preprocess(nb, {"metadata": {"path": abs_path}})
    except CellExecutionError:
        nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
        msg = f"Error executing the notebook '{nb_path}'.\n"
        msg += f"See notebook '{nb_err}' for the traceback."
        print(msg)
        with open(nb_err, mode="w", encoding="utf-8") as f:
            nbformat.write(nb, f)
        raise 

def export_html(nb, f):
    config = {
        'Exporter': {'template_file': 'embed',
                     'template_path': ['./sphinxext/']},
        'ExtractOutputPreprocessor': {'enabled': True},
        'CSSHTMLHeaderPreprocessor': {'enabled': True}
    }

    exporter = HTMLExporter(config)
    body, resources = exporter.from_notebook_node(
        nb, resources={'output_files_dir': f['nbname']})

    for fn, data in resources['outputs'].items():
        bfn = os.path.basename(fn)
        with open("{destdir}/{fn}".format(fn=bfn, **f), 'wb') as res_f:
            res_f.write(data)

    return body 

def test_clustering_nbdisplay_notebook(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def main(args=None):
    parser = argparse.ArgumentParser(
        "Convert LaTeX beamer slides to IPython notebooks + RISE"
    )
    parser.add_argument("input", nargs=1, help="Input file (.tex)")
    parser.add_argument("output", nargs=1, help="Output file (.ipynb)")
    parser.add_argument(
        "-c", "--converter", action="store", dest="converter", default='',
        help="Path to a Python file which defines a subclass of Tex2Cells."
    )
    args = parser.parse_args(args)
    converter = None
    if len(args.converter) > 0:
        with open(args.converter) as fp:
            converter = get_tex2cells_subclass(fp, args.converter)
    if converter is None:
        converter = Tex2Cells

    with open(args.input[0]) as f:
        code = f.read()
    nb = tex2ipy(code, converter)
    with open(args.output[0], 'w') as f:
        nbformat.write(nb, f) 

def render_notebook(filename: Path, replacements: Dict[str, str]) -> str:
    """ Takes path to the notebook, returns executed and rendered version
    :param filename: notebook
    :param replacements: dictionary with text replacements done before executing
    :return: notebook, rendered as string
    """
    with filename.open('r') as f:
        nb_as_str = f.read()
    for original, replacement in replacements.items():
        nb_as_str = nb_as_str.replace(original, replacement)

    nb = nbformat.read(StringIO(nb_as_str), nbformat.NO_CONVERT)
    ep = ExecutePreprocessor(timeout=60, kernel_name='python3')
    ep.preprocess(nb, {'metadata': {'path': str(filename.parent.absolute())}})

    result_as_stream = StringIO()
    nbformat.write(nb, result_as_stream)
    return result_as_stream.getvalue() 

def run_notebook(filename):

    run_path = os.path.split(filename)[0]

    with open(filename) as f:
        nb = nbformat.read(f, as_version=4)

    try:
        ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
        ep.preprocess(nb, {'metadata': {'path': run_path}})

        # FIXME: use tempfile and mv to avoid interruption
        # better split the source code of the notebook and the compiled targets.
        with open(filename, 'wt') as f:
            nbformat.write(nb, f)

    except Exception as e:
        print('processing', filename, e) 

def buildNotebook(appName, result, notebookPath, dataFilePath, runId):
  """
  Method to build .ipynb notebook with init code
   cell for profiles and one report cell per category.

  """
  begin = time.time()
  LOGGER.info('generating notebook %s -> ', os.path.basename(notebookPath))
  nb = nbf.new_notebook()
  numOfCategories, d3Flots = buildReportCells(nb, result, dataFilePath)
  buildInitCell(nb, numOfCategories, d3Flots, appName, runId)

  try:
    with open(notebookPath, 'w') as reportFile:
      nbformat.write(nb, reportFile)
    notebookSize = formatHumanReadable(os.path.getsize(notebookPath))
    elapsed = time.time() - begin
    LOGGER.completed('completed %s in %0.2f sec.', notebookSize, elapsed)
    return True
  except IOError:
    LOGGER.exception('Could not write to the notebook(.ipynb) file')
    return False 

def append_scrapbook_commands(input_nb_path, output_nb_path, scrap_specs):
    notebook = nbf.read(input_nb_path, as_version=nbf.NO_CONVERT)

    scrapbook_cells = []
    # Always need to import nteract-scrapbook
    scrapbook_cells.append(nbf.v4.new_code_cell(source="import scrapbook as sb"))

    # Create a cell to store each key and value in the scrapbook
    for k, v in scrap_specs.items():
        source = "sb.glue(\"{0}\", {1})".format(k, v.code)
        scrapbook_cells.append(nbf.v4.new_code_cell(source=source))

    # Append the cells to the notebook
    [notebook['cells'].append(c) for c in scrapbook_cells]

    # Write out the new notebook
    nbf.write(notebook, output_nb_path) 

def attach(self, podoc):
        podoc.register_lang('notebook',
                            file_ext='.ipynb',
                            load_func=self.load,
                            dump_func=self.dump,
                            loads_func=self.loads,
                            dumps_func=self.dumps,
                            eq_filter=self.eq_filter,
                            )
        podoc.register_func(source='notebook', target='ast',
                            func=self.read,
                            post_filter=replace_resource_paths,
                            )
        podoc.register_func(source='ast', target='notebook',
                            func=self.write,
                            pre_filter=wrap_code_cells,
                            ) 

def from_notebook_node(self, nb, resources, **kwargs):
        #
        cells = nb.cells
        with StringIO() as fh:
            fh.write('#!/usr/bin/env sos-runner\n')
            fh.write('#fileformat=SOS1.0\n\n')
            idx = 0
            for cell in cells:
                idx = self.from_notebook_cell(cell, fh, idx)
            content = fh.getvalue()
        resources['output_extension'] = '.sos'
        return content, resources


#
# Converter to Notebook
# 

def convert(self, notebook_file, sos_file, args=None, unknown_args=None):
        '''
        Convert a ipython notebook to sos format.
        '''
        if unknown_args:
            raise ValueError(f'Unrecognized parameter {unknown_args}')
        exporter = SoS_Exporter()
        notebook = nbformat.read(notebook_file, nbformat.NO_CONVERT)
        output, _ = exporter.from_notebook_node(notebook, {})
        if not sos_file:
            sys.stdout.write(output)
        elif isinstance(sos_file, str):
            with open(sos_file, 'w') as sos:
                sos.write(output)
            env.logger.info(f'SoS script saved to {sos_file}')
        else:
            sos_file.write(output)


#
# notebook to HTML
# 

def notebook_factory(self, tmpdir, request):
        class Factory:
            def __init__(self):
                self.nb = nbformat.v4.new_notebook()
                self.path = tmpdir / (request.node.name + '.ipynb')

            @property
            def filename(self):
                return str(self.path)

            def write(self):
                nbformat.write(self.nb, self.filename)

            def add_code(self, code):
                self.nb['cells'].append(nbformat.v4.new_code_cell(code))

            def add_markdown(self, text):
                self.nb['cells'].append(nbformat.v4.new_markdown_cell(text))

        return Factory() 

def convert(self, nb, filename, language, base_path, path=None):
        fl_nb = ''
        fl_html = ''
        relative_path = ''
        build_path = self.htmldir
        #Convert to HTML
        if path:
            relative_path = path.replace(base_path,'')
            relative_path = relative_path[1:]

        if relative_path != '':
            build_path = self.htmldir +  "/" + relative_path

        ensuredir(build_path)
        fl_html = build_path + "/" + "{}.html".format(filename)
        with open(fl_html, "w") as f:
            html, resources = self.html_exporter.from_notebook_node(nb)
            f.write(html)

        nb['cells'] = nb['cells'][1:] #skip first code-cell as preamble

        # #Write Executed Notebook as File
        # if (nb['metadata']['download_nb'] == True):
        #     with open(download_nb, "wt", encoding="UTF-8") as f:
        #             nbformat.write(nb, f) 

def create_jupyter_notebook_random_question(destination_filename='100_Numpy_random.ipynb'):
    """ Programmatically create jupyter notebook with the questions (and hints and solutions if required)
    saved under source files """

    # Create cells sequence
    nb = nbf.v4.new_notebook()

    nb['cells'] = []

    # - Add header:
    nb['cells'].append(nbf.v4.new_markdown_cell(HEADERS["header"]))
    nb['cells'].append(nbf.v4.new_markdown_cell(HEADERS["sub_header"]))
    nb['cells'].append(nbf.v4.new_markdown_cell(HEADERS["jupyter_instruction_rand"]))

    # - Add initialisation
    nb['cells'].append(nbf.v4.new_code_cell('%run initialise.py'))
    nb['cells'].append(nbf.v4.new_code_cell("pick()"))

    # Delete file if one with the same name is found
    if os.path.exists(destination_filename):
        os.remove(destination_filename)

    # Write sequence to file
    nbf.write(nb, destination_filename) 

def produce_dask_processing_report(self, builderSelf, params, fln= "dask-reports.json"):
        """
            produces a report of dask execution
        """
        ensuredir(builderSelf.reportdir)
        json_filename = builderSelf.reportdir + fln

        try:
            if (sys.version_info > (3, 0)):
                with open(json_filename, "w") as json_file:
                    json.dump(builderSelf.dask_log, json_file)
            else:
               with open(json_filename, "w") as json_file:
                    x = json.dumps(builderSelf.dask_log, ensure_ascii=False)
                    if isinstance(x,str):
                        x = unicode(x, 'UTF-8')
                    json_file.write(x)
        except IOError:
            self.logger.warning("Unable to save dask reports JSON file. Does the {} directory exist?".format(builderSelf.reportdir)) 

def process_and_eval_notebook(input_fn, output_fn, run_cells, timeout=20*60,
                              lang='python', tab=None, default_tab=None):
    with open(input_fn, 'r') as f:
        md = f.read()
    nb = notebook.read_markdown(md)
    if tab:
        # get the tab
        nb = notebook.split_markdown_cell(nb)
        nb = notebook.get_tab_notebook(nb, tab, default_tab)
        if not nb:
            logging.info(f"Skip to eval tab {tab} for {input_fn}")
            # write an emtpy file to track the dependencies
            open(output_fn, 'w')
            return
    # evaluate
    if run_cells:
        # change to the notebook directory to resolve the relpaths properly
        cwd = os.getcwd()
        os.chdir(os.path.join(cwd, os.path.dirname(output_fn)))
        notedown.run(nb, timeout)
        os.chdir(cwd)
    # write
    nb['metadata'].update({'language_info':{'name':lang}})
    with open(output_fn, 'w') as f:
        f.write(nbformat.writes(nb)) 

def script_to_notebook(spth, npth, cr):
    """
    Convert the script at `spth` to a notebook at `npth`. Parameter `cr`
    is a CrossReferenceLookup object.
    """

    # Read entire text of example script
    with open(spth) as f:
        stxt = f.read()
    # Process script text
    stxt = preprocess_script_string(stxt)

    # If the notebook file exists and has been executed, try to
    # update markdown cells without deleting output cells
    if os.path.exists(npth) and notebook_executed(npth):
        # Read current notebook file
        nbold = nbformat.read(npth, as_version=4)
        # Construct updated notebook
        nbnew = script_string_to_notebook_object(stxt)
        if cr is not None:
            notebook_substitute_ref_with_url(nbnew, cr)
        # If the code cells of the two notebooks match, try to
        # update markdown cells without deleting output cells
        if same_notebook_code(nbnew, nbold):
            try:
                replace_markdown_cells(nbnew, nbold)
            except:
                script_string_to_notebook_with_links(stxt, npth, cr)
            else:
                with open(npth, 'wt') as f:
                    nbformat.write(nbold, f)
        else:
            # Write changed text to output notebook file
            script_string_to_notebook_with_links(stxt, npth, cr)
    else:
        # Write changed text to output notebook file
        script_string_to_notebook_with_links(stxt, npth, cr) 

def export_python(nb, destfn):
    exporter = PythonExporter()
    body, resources = exporter.from_notebook_node(nb)
    with open(destfn, 'w') as f:
        f.write(body) 

def test_one_code_block(self, notebook_factory):
        notebook_factory.add_code('__requires__ = ["matplotlib"]')
        notebook_factory.write()
        reqs = scripts.DepsReader.try_read(notebook_factory.filename)
        assert reqs == ['matplotlib'] 

def test_multiple_code_blocks(self, notebook_factory):
        notebook_factory.add_code('__requires__ = ["matplotlib"]')
        notebook_factory.add_code("import matplotlib")
        notebook_factory.write()
        reqs = scripts.DepsReader.try_read(notebook_factory.filename)
        assert reqs == ['matplotlib'] 

def test_code_and_markdown(self, notebook_factory):
        notebook_factory.add_code('__requires__ = ["matplotlib"]')
        notebook_factory.add_markdown("Mark this down please")
        notebook_factory.write()
        reqs = scripts.DepsReader.try_read(notebook_factory.filename)
        assert reqs == ['matplotlib'] 

def test_jupyter_directives(self, notebook_factory):
        notebook_factory.add_code('__requires__ = ["matplotlib"]')
        notebook_factory.add_code("%matplotlib inline\nimport matplotlib")
        notebook_factory.write()
        reqs = scripts.DepsReader.try_read(notebook_factory.filename)
        assert reqs == ['matplotlib'] 

def touch(ctx, *args, **kwargs):
    """Update the access and modification times of each Jupyter notebook to the current time."""
    for file in kwargs["file"]:
        if not os.path.exists(file):
            nb = nbformat.v4.new_notebook()
            with open(file, "w") as f:
                nbformat.write(nb, f, version=4)
        else:
            Path(file).touch()