Python nbformat.NO_CONVERT Examples

def read_source_script(filepath):
    """Read the contents of `filepath`

    Parameters
    ----------
    filepath: Str
        Absolute path to a Python file. Expected to end with '.py', or '.ipynb'

    Returns
    -------
    source: Str
        The contents of `filepath`"""
    if filepath.endswith(".ipynb"):
        with open(filepath, "r") as f:
            from nbconvert import PythonExporter
            import nbformat

            notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
            exporter = PythonExporter()
            source, _ = exporter.from_notebook_node(notebook)
    else:
        with open(filepath, "r") as f:
            source = f.read()

    return source 

def append_scrapbook_commands(input_nb_path, output_nb_path, scrap_specs):
    notebook = nbf.read(input_nb_path, as_version=nbf.NO_CONVERT)

    scrapbook_cells = []
    # Always need to import nteract-scrapbook
    scrapbook_cells.append(nbf.v4.new_code_cell(source='import scrapbook as sb'))

    # Create a cell to store each key and value in the scrapbook
    for k, v in scrap_specs.items():
        source = "sb.glue(\"{0}\", {1})".format(k, v)
        scrapbook_cells.append(nbf.v4.new_code_cell(source=source))

    # Append the cells to the notebook
    [notebook['cells'].append(c) for c in scrapbook_cells]

    # Write out the new notebook
    nbf.write(notebook, output_nb_path) 

def convert(self, notebook_file, sos_file, args=None, unknown_args=None):
        '''
        Convert a ipython notebook to sos format.
        '''
        if unknown_args:
            raise ValueError(f'Unrecognized parameter {unknown_args}')
        exporter = SoS_Exporter()
        notebook = nbformat.read(notebook_file, nbformat.NO_CONVERT)
        output, _ = exporter.from_notebook_node(notebook, {})
        if not sos_file:
            sys.stdout.write(output)
        elif isinstance(sos_file, str):
            with open(sos_file, 'w') as sos:
                sos.write(output)
            env.logger.info(f'SoS script saved to {sos_file}')
        else:
            sos_file.write(output)


#
# notebook to HTML
# 

def list_examples(self):
        all_examples = []
        for category in self.get_categories():
            directory = os.path.join(self.sample_notebook_dir, category)
            filepaths = glob(os.path.join(directory, '**', '*.ipynb'), recursive=True)
            examples = [{'filepath': os.path.abspath(fp)} for fp in filepaths]
            for example in examples:
                node = nbformat.read(example['filepath'], nbformat.NO_CONVERT)
                example['filename'] = os.path.basename(example['filepath'])
                example['metadata'] = node.metadata
                example['category'] = category
                example['basename'] = os.path.basename(example['filepath'])
                example['supporting_items'] = self.get_supporting_items(example['filepath'], example['filename'])
                notebook_folder_location = os.path.split(example['filepath'])[0]
                example['notebook_folder_name'] = os.path.split(notebook_folder_location)[1]
            all_examples.extend(examples)
        return all_examples 

def test_can_run_examples_jupyter_notebooks(self):
        print("Examples folder ", self.examples_folder)
        for filename in os.listdir(self.examples_folder):
            if not filename.endswith('.ipynb'):
                continue

            path = os.path.join(self.examples_folder, filename)
            notebook = nbformat.read(path, nbformat.NO_CONVERT)
            state = {}

            for cell in notebook.cells:
                if cell.cell_type == 'code' and not is_matplotlib_cell(cell):
                    try:
                        exec(strip_magics_and_shows(cell.source), state)
                    # coverage: ignore
                    except:
                        print('Failed to run {}.'.format(path))
                        raise 

def get_pipeline_parameters(request, source_notebook_path):
    """Get the pipeline parameters tagged in the notebook."""
    # read notebook
    log = request.log if hasattr(request, "log") else logger
    try:
        notebook = nbformat.read(source_notebook_path,
                                 as_version=nbformat.NO_CONVERT)
        params_source = parser.get_pipeline_parameters_source(notebook)
        if params_source == '':
            raise ValueError("No pipeline parameters found. Please tag a cell"
                             " of the notebook with the `pipeline-parameters`"
                             " tag.")
        # get a dict from the 'pipeline parameters' cell source code
        params_dict = ast.parse_assignments_expressions(params_source)
    except ValueError as e:
        log.exception("Value Error during parsing of pipeline parameters")
        raise RPCInternalError(details=str(e), trans_id=request.trans_id)
    # convert dict in list so its easier to parse in js
    params = [[k, *v] for k, v in params_dict.items()]
    log.info("Pipeline parameters:")
    for ln in tabulate(params, headers=["name", "type", "value"]).split("\n"):
        log.info(ln)
    return params 

def get_pipeline_metrics(request, source_notebook_path):
    """Get the pipeline metrics tagged in the notebook."""
    # read notebook
    log = request.log if hasattr(request, "log") else logger
    try:
        notebook = nbformat.read(source_notebook_path,
                                 as_version=nbformat.NO_CONVERT)
        metrics_source = parser.get_pipeline_metrics_source(notebook)
        if metrics_source == '':
            raise ValueError("No pipeline metrics found. Please tag a cell"
                             " of the notebook with the `pipeline-metrics`"
                             " tag.")
        # get a dict from the 'pipeline parameters' cell source code
        metrics = ast.parse_metrics_print_statements(metrics_source)
    except ValueError as e:
        log.exception("Failed to parse pipeline metrics")
        raise RPCInternalError(details=str(e), trans_id=request.trans_id)
    log.info("Pipeline metrics: {}".format(metrics))
    return metrics 

def render_notebook(filename: Path, replacements: Dict[str, str]) -> str:
    """ Takes path to the notebook, returns executed and rendered version
    :param filename: notebook
    :param replacements: dictionary with text replacements done before executing
    :return: notebook, rendered as string
    """
    with filename.open('r') as f:
        nb_as_str = f.read()
    for original, replacement in replacements.items():
        nb_as_str = nb_as_str.replace(original, replacement)

    nb = nbformat.read(StringIO(nb_as_str), nbformat.NO_CONVERT)
    ep = ExecutePreprocessor(timeout=60, kernel_name='python3')
    ep.preprocess(nb, {'metadata': {'path': str(filename.parent.absolute())}})

    result_as_stream = StringIO()
    nbformat.write(nb, result_as_stream)
    return result_as_stream.getvalue() 

def append_scrapbook_commands(input_nb_path, output_nb_path, scrap_specs):
    notebook = nbf.read(input_nb_path, as_version=nbf.NO_CONVERT)

    scrapbook_cells = []
    # Always need to import nteract-scrapbook
    scrapbook_cells.append(nbf.v4.new_code_cell(source="import scrapbook as sb"))

    # Create a cell to store each key and value in the scrapbook
    for k, v in scrap_specs.items():
        source = "sb.glue(\"{0}\", {1})".format(k, v.code)
        scrapbook_cells.append(nbf.v4.new_code_cell(source=source))

    # Append the cells to the notebook
    [notebook['cells'].append(c) for c in scrapbook_cells]

    # Write out the new notebook
    nbf.write(notebook, output_nb_path) 

def apply_preprocessors(preprocessors, nbname):
    notebooks_path = os.path.join(os.path.split(__file__)[0], 'notebooks')
    with open(os.path.join(notebooks_path, nbname)) as f:
        nb = nbformat.read(f, nbformat.NO_CONVERT)
        exporter = nbconvert.PythonExporter()
        for preprocessor in preprocessors:
            exporter.register_preprocessor(preprocessor)
        source, meta = exporter.from_notebook_node(nb)
    return source 

def convert(self, notebook_file, output_file, sargs=None,
                unknown_args=None):
        notebook = nbformat.read(notebook_file, nbformat.NO_CONVERT)
        kernel_name = notebook['metadata']['kernelspec']['name']

        nb = None
        # what are we supposed to do?
        if kernel_name == 'sos' and sargs.kernel and sargs.kernel not in (
                'sos', 'SoS'):
            # sos => nonSoS
            if sargs.execute:
                notebook = execute_sos_notebook(notebook_file)
            nb = self.SoS_to_nonSoS_notebook(notebook, sargs)
        elif kernel_name == 'sos' and not sargs.kernel:
            if sargs.execute:
                if output_file and notebook_file != output_file:
                    execute_sos_notebook(notebook_file, output_file)
                    env.logger.info(f'Jupyter notebook saved to {output_file}')
                    return
                else:
                    nb = execute_sos_notebook(notebook_file)
            # sos => sos
        elif kernel_name != 'sos' and sargs.kernel in ('sos', 'SoS', None):
            nb = self.nonSoS_to_SoS_notebook(notebook, sargs)
            if sargs.execute:
                nb = execute_sos_notebook(nb)

        if nb is None:
            # nothing to do (e.g. sos -> sos) without --execute
            return

        if sargs.inplace:
            with open(notebook_file, 'w') as new_nb:
                nbformat.write(nb, new_nb, 4)
            env.logger.info(f'Jupyter notebook saved to {notebook_file}')
        elif not output_file:
            nbformat.write(nb, sys.stdout, 4)
        else:
            with open(output_file, 'w') as new_nb:
                nbformat.write(nb, new_nb, 4)
            env.logger.info(f'Jupyter notebook saved to {output_file}') 

def export_to_python(filename=None,
         preprocessors=[OptsMagicProcessor(),
                        OutputMagicProcessor(),
                        StripMagicsProcessor()]):

    filename = filename if filename else sys.argv[1]
    with open(filename) as f:
        nb = nbformat.read(f, nbformat.NO_CONVERT)
        exporter = nbconvert.PythonExporter()
        for preprocessor in preprocessors:
            exporter.register_preprocessor(preprocessor)
        source, meta = exporter.from_notebook_node(nb)
        return source 

def notebook_to_rst(npth, rpth, rdir, cr=None):
    """
    Convert notebook at `npth` to rst document at `rpth`, in directory
    `rdir`. Parameter `cr` is a CrossReferenceLookup object.
    """

    # Read the notebook file
    ntbk = nbformat.read(npth, nbformat.NO_CONVERT)
    # Convert notebook object to rstpth
    notebook_object_to_rst(ntbk, rpth, rdir, cr) 

def __init__(self):
        with open('/opt/ml/input/data/iotanalytics/params') as params_file:
            params = json.load(params_file)
        self.variables = params['Variables']
        context = params['Context']
        notebook_path = context.get('Analysis', os.environ.get('NOTEBOOK_PATH'))
        with io.open(notebook_path, encoding='utf-8') as f:
            self.nb = nbformat.read(f, as_version=nbformat.NO_CONVERT)
        self.notebook_dir  = os.path.dirname(notebook_path)
        self.variables = params['Variables']
        output_uris = context['OutputUris']
        self.output_ipynb_s3_uri = output_uris['ipynb']
        self.output_html_s3_uri = output_uris['html']
        self.kernel_name = os.environ.get('KERNEL_NAME', _get_default_kernel_name())
        self.s3 = boto3.resource('s3') 

def notebook_to_rst(npth, rpth, rdir, cr=None):
    """
    Convert notebook at `npth` to rst document at `rpth`, in directory
    `rdir`. Parameter `cr` is a CrossReferenceLookup object.
    """

    # Read the notebook file
    ntbk = nbformat.read(npth, nbformat.NO_CONVERT)
    # Convert notebook object to rstpth
    notebook_object_to_rst(ntbk, rpth, rdir, cr) 

def notebook_node_from_string_list(string_list):
    """
    Reads a notebook from a string list and returns the NotebookNode
    object.

    :param string_list: The notebook file contents as list of strings
                        (linewise).
    :return:            The notebook as NotebookNode.
    """
    return nbformat.reads(''.join(string_list), nbformat.NO_CONVERT) 

def notebook_node_to_string_list(notebook_node):
    """
    Writes a NotebookNode to a list of strings.

    :param notebook_node: The notebook as NotebookNode to write.
    :return:              The notebook as list of strings (linewise).
    """
    return nbformat.writes(notebook_node, nbformat.NO_CONVERT).splitlines(True) 

def _jupytext_writes(fmt):
    def _writes(nbk, version=nbformat.NO_CONVERT, **kwargs):
        return writes(nbk, fmt, version=version, **kwargs)

    return _writes 

def reads(text, fmt, as_version=nbformat.NO_CONVERT, **kwargs):
    """
    Read a notebook from a string

    :param text: the text representation of the notebook
    :param fmt: (optional) the jupytext format like `md`, `py:percent`, ...
    :param as_version: see nbformat.reads
    :param kwargs: (not used) additional parameters for nbformat.reads
    :return: the notebook
    """
    fmt = copy(fmt) if fmt else divine_format(text)
    fmt = long_form_one_format(fmt)
    ext = fmt["extension"]

    if ext == ".ipynb":
        return nbformat.reads(text, as_version, **kwargs)

    format_name = read_format_from_metadata(text, ext) or fmt.get("format_name")

    if format_name:
        format_options = {}
    else:
        format_name, format_options = guess_format(text, ext)

    if format_name:
        fmt["format_name"] = format_name

    fmt.update(format_options)
    reader = TextNotebookConverter(fmt)
    notebook = reader.reads(text, **kwargs)
    rearrange_jupytext_metadata(notebook.metadata)

    if format_name and insert_or_test_version_number():
        notebook.metadata.setdefault("jupytext", {}).setdefault(
            "text_representation", {}
        ).update({"extension": ext, "format_name": format_name})

    return notebook 

def read(fp, as_version=nbformat.NO_CONVERT, fmt=None, **kwargs):
    """"
    Read a notebook from a file name or a file object

    :param fp: a file name or a file object
    :param as_version: see nbformat.read
    :param fmt: (optional) the jupytext format like `md`, `py:percent`, ...
    :param kwargs: (not used) additional parameters for nbformat.read
    :return: the notebook
    """
    if as_version != nbformat.NO_CONVERT and not isinstance(as_version, int):
        raise TypeError(
            "Second argument 'as_version' should be either nbformat.NO_CONVERT, or an integer."
        )

    if fp == "-":
        text = sys.stdin.read()
        return reads(text, fmt)

    if not hasattr(fp, "read"):
        # Treat fp as a file name
        fp = str(fp)
        _, ext = os.path.splitext(fp)
        fmt = copy(fmt or {})
        if not isinstance(fmt, dict):
            fmt = long_form_one_format(fmt)
        fmt.update({"extension": ext})
        with io.open(fp, encoding="utf-8") as stream:
            return read(stream, as_version=as_version, fmt=fmt, **kwargs)

    if fmt is not None:
        fmt = long_form_one_format(fmt)
        if fmt["extension"] == ".ipynb":
            notebook = nbformat.read(fp, as_version, **kwargs)
            rearrange_jupytext_metadata(notebook.metadata)
            return notebook

    return reads(fp.read(), fmt, **kwargs) 

def parse_ipynb(file: Path) -> str:
    with open(file, "r") as nb_file:
        nb_str = nb_file.read()
    nb = nbformat.reads(nb_str, nbformat.NO_CONVERT)
    exporter = PythonExporter()
    script, _ = exporter.from_notebook_node(nb)
    return script 

def extract_workflow(notebook):
    '''Extract workflow from a notebook file or notebook JSON instance'''
    if isinstance(notebook, str):
        nb = nbformat.read(notebook, nbformat.NO_CONVERT)
    else:
        nb = notebook
    cells = nb.cells
    content = '#!/usr/bin/env sos-runner\n#fileformat=SOS1.0\n\n'
    for cell in cells:
        if cell.cell_type != "code":
            continue
        # Non-sos code cells are also ignored
        if 'kernel' in cell.metadata and cell.metadata['kernel'] not in ('sos',
                                                                         'SoS',
                                                                         None):
            continue
        lines = cell.source.split('\n')
        valid_cell = False
        for idx, line in enumerate(lines):
            if valid_cell or (line.startswith('%include') or
                              line.startswith('%from')):
                content += line + '\n'
            elif SOS_SECTION_HEADER.match(line):
                valid_cell = True
                # look retrospectively for comments
                c = idx - 1
                comment = ''
                while c >= 0 and lines[c].startswith('#'):
                    comment = lines[c] + '\n' + comment
                    c -= 1
                content += comment + line + '\n'
        if valid_cell:
            content += '\n'
    return content 

def clear_notebook(old_ipynb, new_ipynb):
    with io.open(old_ipynb, 'r') as f:
        nb = nbformat.read(f, nbformat.NO_CONVERT)

    remove_outputs(nb)

    with io.open(new_ipynb, 'w', encoding='utf8') as f:
        nbformat.write(nb, f, nbformat.NO_CONVERT) 

def _save_notebook(self, path, nb):
        """Save a notebook to an os_path."""
        s = nbformat.writes(nb, version=nbformat.NO_CONVERT)
        self._pyfilesystem_instance.writetext(path, s) 

def clear_notebook(old_ipynb, new_ipynb):
    with io.open(old_ipynb, 'r') as f:
        nb = nbformat.read(f, nbformat.NO_CONVERT)

    remove_outputs(nb)

    with io.open(new_ipynb, 'w', encoding='utf8') as f:
        nbformat.write(nb, f, nbformat.NO_CONVERT) 

def test_can_run_examples_jupyter_notebook(path):
    notebook = nbformat.read(path, nbformat.NO_CONVERT)
    state = {}  # type: Dict[str, Any]

    for cell in notebook.cells:
        if cell.cell_type == 'code' and not is_matplotlib_cell(cell):
            try:
                exec(strip_magics_and_shows(cell.source), state)
            # coverage: ignore
            except:
                print('Failed to run {}.'.format(path))
                raise 

def load_notebook(filename):
    try:
        return nbformat.read(filename, nbformat.NO_CONVERT)
    except FileNotFoundError:
        raise Exception(f"{filename} Not found in current directory") 

def _save_notebook(self, path, nb):
        """
        Uploads notebook to GCS.
        :param path: blob path.
        :param nb: :class:`nbformat.notebooknode.NotebookNode` instance.
        :return: created :class:`google.cloud.storage.Blob`.
        """
        bucket_name, bucket_path = self._parse_path(path)
        bucket = self._get_bucket(bucket_name, throw=True)
        data = nbformat.writes(nb, version=nbformat.NO_CONVERT)
        blob = bucket.blob(bucket_path)
        blob.upload_from_string(data, "application/x-ipynb+json")
        return blob 

def get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack, data_args):
    model_string = inspect.getsource(model)
    model_string = remove_imports(model_string)

    if notebook_name:
        notebook_path = os.getcwd() + "/{}.ipynb".format(notebook_name)
        with open(notebook_path, 'r') as f:
            notebook = nbformat.reads(f.read(), nbformat.NO_CONVERT)
            exporter = PythonExporter()
            source, _ = exporter.from_notebook_node(notebook)
    else:
        calling_script_file = os.path.abspath(inspect.stack()[stack][1])
        with open(calling_script_file, 'r') as f:
            source = f.read()

    cleaned_source = remove_all_comments(source)
    imports = extract_imports(cleaned_source, verbose)

    parts = hyperparameter_names(model_string)
    aug_parts = augmented_names(parts)

    hyperopt_params = get_hyperparameters(model_string)
    space = get_hyperopt_space(parts, hyperopt_params, verbose)

    functions_string = retrieve_function_string(functions, verbose)
    data_string = retrieve_data_string(data, verbose, data_args)
    model = hyperopt_keras_model(model_string, parts, aug_parts, verbose)

    temp_str = temp_string(imports, model, data_string, functions_string, space)
    return temp_str 

def __init__(self,
                 source_notebook_path: str,
                 notebook_metadata_overrides: dict = None,
                 debug: bool = False,
                 auto_snapshot: bool = False):
        self.auto_snapshot = auto_snapshot
        self.source_path = str(source_notebook_path)
        if not os.path.exists(self.source_path):
            raise ValueError("Path {} does not exist".format(self.source_path))

        # read notebook
        self.notebook = nb.read(self.source_path,
                                as_version=nb.NO_CONVERT)

        # read Kale notebook metadata.
        # In case it is not specified get an empty dict
        notebook_metadata = self.notebook.metadata.get(
            KALE_NOTEBOOK_METADATA_KEY, dict())
        # override notebook metadata with provided arguments
        if notebook_metadata_overrides:
            notebook_metadata.update(notebook_metadata_overrides)

        # validate metadata and apply transformations when needed
        self.pipeline_metadata = parse_metadata(notebook_metadata)

        # used to set container step working dir same as current environment
        abs_working_dir = utils.get_abs_working_dir(self.source_path)
        self.pipeline_metadata['abs_working_dir'] = abs_working_dir
        self.detect_environment()

        # set up logging
        level = logging.DEBUG if debug else logging.INFO
        log_path = os.path.join(".", "kale.log")
        self.logger = get_or_create_logger(module=__name__, level=level,
                                           log_path=log_path)

        # mute other loggers
        logging.getLogger('urllib3.connectionpool').setLevel(logging.CRITICAL)