Python nbformat.reads() Examples

def read_source_script(filepath):
    """Read the contents of `filepath`

    Parameters
    ----------
    filepath: Str
        Absolute path to a Python file. Expected to end with '.py', or '.ipynb'

    Returns
    -------
    source: Str
        The contents of `filepath`"""
    if filepath.endswith(".ipynb"):
        with open(filepath, "r") as f:
            from nbconvert import PythonExporter
            import nbformat

            notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
            exporter = PythonExporter()
            source, _ = exporter.from_notebook_node(notebook)
    else:
        with open(filepath, "r") as f:
            source = f.read()

    return source 

def collect(self):
        with self.fspath.open() as f:
            payload = f.read()
        self.notebook_folder = self.fspath.dirname
        try:
            # Ipython 3
            self.nb = reads(payload, 3)
        except (TypeError, NBFormatError):
            # Ipython 2
            self.nb = reads(payload, 'json')
        self.runner = NotebookRunner(self.nb)

        cell_num = 1

        for cell in self.runner.iter_code_cells():
            yield IPyNbCell(self.name, self, cell_num, cell)
            cell_num += 1 

def test_geoip_notebook(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()

        with open(nb_path, "rb") as f:
            nb_bytes = f.read()
        nb_text = nb_bytes.decode("utf-8")
        nb = nbformat.reads(nb_text, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def sample_perf(nb, n=30):
    samples = pd.DataFrame(
        pd.np.NaN,
        index=pd.MultiIndex.from_product(
            (range(n), ['nbformat'] + JUPYTEXT_FORMATS), names=['sample', 'implementation']),
        columns=pd.Index(['size', 'read', 'write'], name='measure'))

    for i, fmt in samples.index:
        t0 = time.time()
        if fmt == 'nbformat':
            text = nbformat.writes(nb)
        else:
            text = jupytext.writes(nb, fmt)
        t1 = time.time()
        samples.loc[(i, fmt), 'write'] = t1 - t0
        samples.loc[(i, fmt), 'size'] = len(text)
        t0 = time.time()
        if fmt == 'nbformat':
            nbformat.reads(text, as_version=4)
        else:
            jupytext.reads(text, fmt)
        t1 = time.time()
        samples.loc[(i, fmt), 'read'] = t1 - t0
    return samples 

def download_notebook(tmpdir, request):
    notebook_url = request.param

    # check extension of file
    filename = os.path.basename(notebook_url)
    if not notebook_url.endswith('.ipynb'):
        raise ValueError(f'{filename} is not a Jupyter notebook.')

    # redirect notebook if notebook is located on nbviewer.jupyter.org
    if 'nbviewer' in notebook_url:
        r = requests.get(notebook_url)
        link = re.findall(
            '(?<=<a href=").+(?=" title="Download Notebook" ' 'download>)',
            r.content.decode('utf8'),
        )
        if len(link) > 0:
            notebook_url = link[0]

    # download notebook
    r = requests.get(notebook_url)

    nb = nbformat.reads(r.content.decode('utf8'), 4)
    nb['metadata'].update({'path': f'{tmpdir}'})

    return nb 

def md_to_notebook(text):
    """Convert a Markdown text to a Jupyter notebook, using Pandoc"""
    tmp_file = tempfile.NamedTemporaryFile(delete=False)
    tmp_file.write(text.encode("utf-8"))
    tmp_file.close()

    pandoc(
        u"--from markdown --to ipynb -s --atx-headers --wrap=preserve --preserve-tabs",
        tmp_file.name,
        tmp_file.name,
    )

    with open(tmp_file.name, encoding="utf-8") as opened_file:
        notebook = ipynb_reads(opened_file.read(), as_version=4)
    os.unlink(tmp_file.name)

    return notebook 

def divine_format(text):
    """Guess the format of the notebook, based on its content #148"""
    try:
        nbformat.reads(text, as_version=4)
        return "ipynb"
    except nbformat.reader.NotJSONError:
        pass

    lines = text.splitlines()
    for comment in ["", "#"] + _COMMENT_CHARS:
        metadata, _, _, _ = header_to_metadata_and_cell(lines, comment)
        ext = (
            metadata.get("jupytext", {}).get("text_representation", {}).get("extension")
        )
        if ext:
            return ext[1:] + ":" + guess_format(text, ext)[0]

    # No metadata, but ``` on at least one line => markdown
    for line in lines:
        if line == "```":
            return "md"

    return "py:" + guess_format(text, ".py")[0] 

def notebook_view(request_args):
    check.dict_param(request_args, 'request_args')

    # This currently provides open access to your file system - the very least we can
    # do is limit it to notebook files until we create a more permanent solution.
    path = request_args['path']
    if not path.endswith('.ipynb'):
        return 'Invalid Path', 400

    with open(os.path.abspath(path)) as f:
        read_data = f.read()
        notebook = nbformat.reads(read_data, as_version=4)
        html_exporter = HTMLExporter()
        html_exporter.template_file = 'basic'
        (body, resources) = html_exporter.from_notebook_node(notebook)
        return '<style>' + resources['inlining']['css'][0] + '</style>' + body, 200 

def __init__(self, node_or_path):
        if isinstance(node_or_path, string_types):
            path = urlparse(node_or_path).path
            if not os.path.splitext(path)[-1].endswith('ipynb'):
                raise Warning(
                    "Requires an '.ipynb' file extension. Provided path: '{}'".format(node_or_path)
                )
            self.path = node_or_path
            self.node = nbformat.reads(papermill_io.read(node_or_path), as_version=4)
        else:
            self.path = ""
            self.node = node_or_path

        # Memoized traits
        self._scraps = None
        self._outputs = None 

def notebook_node_from_string_list(string_list):
    """
    Reads a notebook from a string list and returns the NotebookNode
    object.

    :param string_list: The notebook file contents as list of strings
                        (linewise).
    :return:            The notebook as NotebookNode.
    """
    return nbformat.reads(''.join(string_list), nbformat.NO_CONVERT) 

def execute_test(file_path, result_path):
  """Executes a single notebook.

  Args:
    file_path: Path to the notebook to execute.
    result_path: Path to store the resulting notebook.

  Returns:
    bool: True if the notebook does not have any errors, False otherwise.

  Raises:
    Exception if an unexpected error occurs executing the notebook.
  """
  try:
    with open(file_path, 'r') as f:
      filedata = f.read()
      if FLAGS.override_pip_install_agents:
        filedata = filedata.replace('pip install tf-agents', 'pip --version')
      nb = nbformat.reads(filedata, as_version=4)

      ep = ExecutePreprocessor(timeout=3600, kernel_name='python3')
      try:
        ep.preprocess(nb, {'metadata': {'path': FLAGS.output_dir}})
      except CellExecutionError as cex:
        logging.error('ERROR executing:%s', file_path)
        logging.error(cex)
        return False
    with open(result_path, 'w', encoding='utf-8') as fo:
      nbformat.write(nb, fo)
    return True
  except Exception as e:  # pylint: disable=W0703
    logging.error('Unexpected ERROR: in %s', file_path)
    logging.error(e) 

def set_source(self, source):
        # Parse md and put in a root node
        source, resources = self.exporter.from_notebook_node(
            nbformat.reads(source, as_version=self.declaration.version))
        # Parse source to html
        super(NotebookComponent, self).set_source(u"<div>%s</div>" % source) 

def reads(text, fmt, as_version=nbformat.NO_CONVERT, **kwargs):
    """
    Read a notebook from a string

    :param text: the text representation of the notebook
    :param fmt: (optional) the jupytext format like `md`, `py:percent`, ...
    :param as_version: see nbformat.reads
    :param kwargs: (not used) additional parameters for nbformat.reads
    :return: the notebook
    """
    fmt = copy(fmt) if fmt else divine_format(text)
    fmt = long_form_one_format(fmt)
    ext = fmt["extension"]

    if ext == ".ipynb":
        return nbformat.reads(text, as_version, **kwargs)

    format_name = read_format_from_metadata(text, ext) or fmt.get("format_name")

    if format_name:
        format_options = {}
    else:
        format_name, format_options = guess_format(text, ext)

    if format_name:
        fmt["format_name"] = format_name

    fmt.update(format_options)
    reader = TextNotebookConverter(fmt)
    notebook = reader.reads(text, **kwargs)
    rearrange_jupytext_metadata(notebook.metadata)

    if format_name and insert_or_test_version_number():
        notebook.metadata.setdefault("jupytext", {}).setdefault(
            "text_representation", {}
        ).update({"extension": ext, "format_name": format_name})

    return notebook 

def read(fp, as_version=nbformat.NO_CONVERT, fmt=None, **kwargs):
    """"
    Read a notebook from a file name or a file object

    :param fp: a file name or a file object
    :param as_version: see nbformat.read
    :param fmt: (optional) the jupytext format like `md`, `py:percent`, ...
    :param kwargs: (not used) additional parameters for nbformat.read
    :return: the notebook
    """
    if as_version != nbformat.NO_CONVERT and not isinstance(as_version, int):
        raise TypeError(
            "Second argument 'as_version' should be either nbformat.NO_CONVERT, or an integer."
        )

    if fp == "-":
        text = sys.stdin.read()
        return reads(text, fmt)

    if not hasattr(fp, "read"):
        # Treat fp as a file name
        fp = str(fp)
        _, ext = os.path.splitext(fp)
        fmt = copy(fmt or {})
        if not isinstance(fmt, dict):
            fmt = long_form_one_format(fmt)
        fmt.update({"extension": ext})
        with io.open(fp, encoding="utf-8") as stream:
            return read(stream, as_version=as_version, fmt=fmt, **kwargs)

    if fmt is not None:
        fmt = long_form_one_format(fmt)
        if fmt["extension"] == ".ipynb":
            notebook = nbformat.read(fp, as_version, **kwargs)
            rearrange_jupytext_metadata(notebook.metadata)
            return notebook

    return reads(fp.read(), fmt, **kwargs) 

def json_to_nb_format(nb_str):
    """Converts Notebook JSON to python object"""
    nb = nbformat.reads(nb_str, as_version=4)
    return nb 

def parse_ipynb(file: Path) -> str:
    with open(file, "r") as nb_file:
        nb_str = nb_file.read()
    nb = nbformat.reads(nb_str, nbformat.NO_CONVERT)
    exporter = PythonExporter()
    script, _ = exporter.from_notebook_node(nb)
    return script 

def convert_to_html(ctx, ipynb_file_path):
    html_exporter = gen_exporter()
    ipynb_data, metadata = extract_main_and_meta(ctx, ipynb_file_path)
    nb = nbformat.reads(ipynb_data, as_version=4)
    (body, __) = html_exporter.from_notebook_node(nb)
    return body, metadata 

def reads_v2(source: str, as_version: int, **kwargs: typing.Any) -> NotebookNode:
    nb = _reads(source, as_version, **kwargs)
    MetadataValidatorV2().validate_nb(nb)
    return nb 

def reads_v3(source: str, as_version: int, **kwargs: typing.Any) -> NotebookNode:
    nb = _reads(source, as_version, **kwargs)
    MetadataValidatorV3().validate_nb(nb)
    return nb 

def reads_v1(source: str, as_version: int, **kwargs: typing.Any) -> NotebookNode:
    nb = _reads(source, as_version, **kwargs)
    MetadataValidatorV1().validate_nb(nb)
    return nb 

def write_doc(self, docname, doctree):
        # work around multiple string % tuple issues in docutils;
        # replace tuples in attribute values with lists
        doctree = doctree.deepcopy()
        destination = docutils.io.StringOutput(encoding="utf-8")

        ### output notebooks for executing for single pdfs, the urlpath should be set to website url
        self.writer._set_ref_urlpath(self.config["jupyter_pdf_urlpath"])
        self.writer._set_jupyter_download_nb_image_urlpath(None)
        self.writer.write(doctree, destination)

        # get a NotebookNode object from a string
        nb = nbformat.reads(self.writer.output, as_version=4)
        nb = self.update_Metadata(nb)

        ### execute the notebook - keep it forcefully on
        strDocname = str(docname)
        if strDocname in self.execution_vars['dependency_lists'].keys():
            self.execution_vars['delayed_notebooks'].update({strDocname: nb})
        else:        
            self._execute_notebook_class.execute_notebook(self, nb, docname, self.execution_vars, self.execution_vars['futures'])

        ### mkdir if the directory does not exist
        outfilename = os.path.join(self.outdir, os_path(docname) + self.out_suffix)
        ensuredir(os.path.dirname(outfilename))

        try:
            with codecs.open(outfilename, "w", "utf-8") as f:
                self.writer.output = nbformat.writes(nb, version=4)
                f.write(self.writer.output)
        except (IOError, OSError) as err:
            self.logger.warning("error writing file %s: %s" % (outfilename, err)) 

def load_ipynb_text(text: str):
    try:
        return nbformat.reads(text, as_version=4)
    except AttributeError as e:
        raise nbformat.reader.NotJSONError(msgfy.to_error_message(e))
    except OSError as e:
        _schema_not_found_error_handler(e)
        raise 

def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack, data_args):
    model_string = inspect.getsource(model)
    model_string = remove_imports(model_string)

    if notebook_name:
        notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name)
        with open(notebook_path, 'r') as f:
            notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
            exporter = PythonExporter()
            source, _ = exporter.from_notebook_node(notebook)
    else:
        calling_script_file = os.path.abspath(inspect.stack()[stack][1])
        with open(calling_script_file, 'r') as f:
            source = f.read()

    cleaned_source = remove_all_comments(source)
    imports = extract_imports(cleaned_source, verbose)

    parts = hyperparameter_names(model_string)
    aug_parts = augmented_names(parts)

    hyperopt_params = get_hyperparameters(model_string)
    space = get_hyperopt_space(parts, hyperopt_params, verbose)

    functions_string = retrieve_function_string(functions, verbose)
    data_string = retrieve_data_string(data, verbose, data_args)
    model = hyperopt_keras_model(model_string, parts, aug_parts, verbose)

    temp_str = temp_string(imports, model, data_string, functions_string, space)
    return temp_str 

def test_callgraph():
    res = subprocess.run(["jupyter", "nbconvert",
                          "--to", "notebook",
                          "--config=" + str(IPYTHON_CONFIG_PATH),
                          "--execute", str(EXAMPLE_NB_PATH),
                          "--ExecutePreprocessor.kernel_name=python3",
                          "--stdout",
                          ],
                         stdout=subprocess.PIPE)
    assert res.returncode == 0
    assert res.stdout is not None
    gm = read_notebook(str(EXAMPLE_NB_PATH))
    nb = nbformat.reads(res.stdout, as_version=4)
    assert nb == gm 

def loads(self, s):
        return nbformat.reads(s, _NBFORMAT_VERSION) 

def run(nb_name, nb_text, parameters, hide_input):
    '''Run the notebook and return the text

    Args:
        nb_name (string): Name of notebook
        nb_text (string): nbformat json of text of notebook to convert
        paramters (string): json parameters to use for papermill
        hide_input (boolean): hide code
    '''
    # validation for papermill
    nb = nbformat.reads(nb_text, 4)

    with TemporaryDirectory() as tempdir:
        in_file = os.path.join(tempdir, '{}.ipynb'.format(nb_name))
        out_file = os.path.join(tempdir, '{}_out.ipynb'.format(nb_name))

        nbformat.write(nb, in_file)

        if isinstance(parameters, string_types):
            parameters = json.loads(parameters)

        execute_notebook(in_file, out_file, parameters=parameters, report_mode=hide_input, start_timeout=600)

        with open(out_file, 'r') as fp:
            output_text = fp.read()

    return output_text 

def validate_instructor_nb(nb_file):
    """Ensures there is at least one BEGIN SOLUTION TAG in uploaded nb
    
    Should prevent accidentally uploading a student NB
    """
    
    solution_text = "BEGIN SOLUTION"
    found = False

    raw = nb_file.file.read()
    nb_file.file.seek(0)

    try:
        # TODO: as_verion = 4? should this come from the nb somehow?
        nb = nbformat.reads(raw, as_version=4)

    except Exception as e:
        log.exception(e)
        raise ValidationError(str(e))

    for cell in nb['cells']:
        if solution_text in cell['source']:
            log.info("Found solution cell")
            return

    raise ValidationError("No Solution Cells Found in uploaded notebook!"\
            " Are you sure this is the instructor version?") 

def _load_nb_to_dict(self, nb_path):
        """Loads the students autograded notebook"""
        if os.path.exists(nb_path):
            with open(nb_path, 'r') as f:
                raw = f.read()

        # NOTE: as_version should be...?
        nb = nbformat.reads(raw, as_version=4)
        return nb 

def _read_notebook(self, path, as_version=4):
        """Read a notebook from a path."""
        nb, format = self._read_file(path, 'text')
        return nbformat.reads(nb, as_version=as_version) 

def test_save_notebook(self):
        nb = nbformat.reads(self.NOTEBOOK, 4)
        self.contents_manager.save({
            "type": "notebook",
            "content": nb
        }, self.path("test.ipynb"))
        bucket = self.bucket
        blob = bucket.blob("test.ipynb")
        self.assertTrue(blob.exists())
        try:
            self.assertEqual(blob.download_as_string(), self.NOTEBOOK.encode())
        finally:
            blob.delete()