Python nbconvert.preprocessors.ExecutePreprocessor() Examples

def test_timeseries_controls(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def run_notebook(notebook_path):
    nb_name, _ = os.path.splitext(os.path.basename(notebook_path))

    with open(notebook_path) as f:
        nb = nbformat.read(f, as_version=4)

    proc = ExecutePreprocessor(timeout=600, kernel_name='python3')
    proc.allow_errors = True
    proc.preprocess(nb)

    for num, cell in enumerate(nb.cells):
        if 'outputs' in cell:
            for output in cell['outputs']:
                if output.output_type == 'error':
                    return cell.execution_count, output.traceback

    return None

# 7-bit C1 ANSI sequences 

def run_notebook(notebook_path):
    nb_name, _ = os.path.splitext(os.path.basename(notebook_path))
    dirname = os.path.dirname(notebook_path)

    with open(notebook_path) as f:
        nb = nbformat.read(f, as_version=4)

    nb['cells'] = [cell for cell in nb['cells'] if not cell['source'].startswith('!')]

    proc = ExecutePreprocessor(timeout=600, kernel_name='python3')
    proc.allow_errors = True

    proc.preprocess(nb, {'metadata': {'path': 'examples/'}})
    output_path = os.path.join(dirname, '_test_{}.ipynb'.format(nb_name))

    with open(output_path, mode='wt') as f:
        nbformat.write(nb, f)
    errors = []
    for cell in nb.cells:
        if 'outputs' in cell:
            for output in cell['outputs']:
                if output.output_type == 'error':
                    errors.append(output)
    return nb, errors 

def test_reexecute_result_notebook():
    with exec_for_test('define_hello_world_pipeline') as result:
        assert result.success

        materialization_events = [
            x for x in result.step_event_list if x.event_type_value == 'STEP_MATERIALIZATION'
        ]
        for materialization_event in materialization_events:
            result_path = get_path(materialization_event)

        if result_path.endswith('.ipynb'):
            with open(result_path) as fd:
                nb = nbformat.read(fd, as_version=4)
            ep = ExecutePreprocessor()
            ep.preprocess(nb, {})
            with open(result_path) as fd:
                assert nbformat.read(fd, as_version=4) == nb 

def test_invalid_notebooks(
    invalid_notebook_path, cell_location, error_name, error_value, error_output_type
):
    notebook_filename = script_relative_path(invalid_notebook_path)
    with open(notebook_filename) as f:
        nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name='python3')

        try:
            ep.preprocess(
                nb,
                {
                    'metadata': {
                        'path': script_relative_path(
                            notebook_filename[: notebook_filename.rfind('/')]
                        )
                    }
                },
            )
        except CellExecutionError:
            error_message = get_dict_value(nb, cell_location)
            assert error_message.ename == error_name
            assert bool(re.search(error_value, error_message.evalue))
            assert error_message.output_type == error_output_type 

def execute_notebook(ramp_kit_dir='.'):
    import nbformat
    from nbconvert.preprocessors import ExecutePreprocessor

    problem_name = os.path.basename(os.path.abspath(ramp_kit_dir))
    print('Testing if the notebook can be executed')
    notebook_filename = os.path.join(
        os.path.abspath(ramp_kit_dir),
        '{}_starting_kit.ipynb'.format(problem_name))
    kernel_name = 'python{}'.format(sys.version_info.major)

    with open(notebook_filename) as f:
        nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name)
        ep.preprocess(nb, {'metadata':
                           {'path': os.path.abspath(ramp_kit_dir)}}) 

def convert(nbfile):
    basename, _ = os.path.splitext(nbfile)

    meta = {'metadata': {'path': '.'}}
    with open(nbfile, 'r', encoding='utf-8') as nbf:
        nbdata = nbformat.read(nbf, as_version=4, encoding='utf-8')

    runner = ExecutePreprocessor(timeout=600, kernel_name='probscale')
    runner.preprocess(nbdata, meta)

    img_folder = basename + '_files'
    body_raw, images = RSTExporter().from_notebook_node(nbdata)
    body_final = body_raw.replace('.. image:: ', '.. image:: {}/'.format(img_folder))

    with open(basename + '.rst', 'w', encoding='utf-8') as rst_out:
        rst_out.write(body_final)

    for img_name, img_data in images['outputs'].items():
        img_path = os.path.join(img_folder, img_name)
        with open(img_path, 'wb') as img:
            img.write(img_data) 

def run_notebook(filename):

    run_path = os.path.split(filename)[0]

    with open(filename) as f:
        nb = nbformat.read(f, as_version=4)

    try:
        ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
        ep.preprocess(nb, {'metadata': {'path': run_path}})

        # FIXME: use tempfile and mv to avoid interruption
        # better split the source code of the notebook and the compiled targets.
        with open(filename, 'wt') as f:
            nbformat.write(nb, f)

    except Exception as e:
        print('processing', filename, e) 

def metadata_notebook(tmpdir, request):
    nb = nbformat.v4.new_notebook()

    ep = ExecutePreprocessor()
    ep.preprocess(nb, {'metadata': {'path': tmpdir}})

    nb['metadata'].update(
        {
            'authors': [{'name': 'author1'}, {'name': 'author2'}],
            'subtitle': 'subtitle',
            'date': '2019-05-11',
            'path': f'{tmpdir}',
        }
    )
    if request.param:
        nb['metadata']['title'] = 'title'

    return nb 

def test_notebooks(nbdir='docs/tutorials/'):
    """
    Run though notebook tutorials
    """

    nbfiles = sorted(glob(os.path.join(nbdir, '*.ipynb')))
    for nbfile in nbfiles:
        print(nbfile)
        with open(nbfile) as f:
            nb = nbformat.read(f, as_version=4)

        if sys.version_info[0] < 3:
            ep = ExecutePreprocessor(timeout=900, kernel_name='python2')
        else:
            ep = ExecutePreprocessor(timeout=900, kernel_name='python3')

        ep.preprocess(nb, {'metadata': {'path': nbdir}}) 

def render_notebook(filename: Path, replacements: Dict[str, str]) -> str:
    """ Takes path to the notebook, returns executed and rendered version
    :param filename: notebook
    :param replacements: dictionary with text replacements done before executing
    :return: notebook, rendered as string
    """
    with filename.open('r') as f:
        nb_as_str = f.read()
    for original, replacement in replacements.items():
        nb_as_str = nb_as_str.replace(original, replacement)

    nb = nbformat.read(StringIO(nb_as_str), nbformat.NO_CONVERT)
    ep = ExecutePreprocessor(timeout=60, kernel_name='python3')
    ep.preprocess(nb, {'metadata': {'path': str(filename.parent.absolute())}})

    result_as_stream = StringIO()
    nbformat.write(nb, result_as_stream)
    return result_as_stream.getvalue() 

def test_widgets_notebook(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def test_geoip_notebook(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()

        with open(nb_path, "rb") as f:
            nb_bytes = f.read()
        nb_text = nb_bytes.decode("utf-8")
        nb = nbformat.reads(nb_text, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def run_notebook(path, workdir=None, timeout=_DEFAULT_TIMEOUT):
    resources = {
        'metadata': {
            'path': workdir
        }
    } if workdir is not None else None

    with open(path, 'r') as f:
        nb = nbformat.read(f, as_version=_DEFAULT_IPYNB_VERSION)
    ep = ExecutePreprocessor(timeout=timeout)

    # Some notebooks may generate and automatically open reports in a web browser.
    # This is inconvenient in an automated test suite, so let's disable it.
    # Overwriting $BROWSER with a dummy command will keep the notebook
    # kernel from being able to open a web browser on Linux.
    # TODO find platform-neutral solution to suppress webbrowser.open
    browser = os.environ.get('BROWSER', None)
    os.environ['BROWSER'] = 'echo %s'
    ep.preprocess(nb, resources=resources)
    os.environ['BROWSER'] = browser 

def test_clustering_nbdisplay_notebook(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def test_process_tree_notebook():
    nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
    abs_path = Path(_NB_FOLDER).absolute()
    with open(nb_path) as f:
        nb = nbformat.read(f, as_version=4)
    ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

    try:
        ep.preprocess(nb, {"metadata": {"path": abs_path}})
    except CellExecutionError:
        nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
        msg = f"Error executing the notebook '{nb_path}'.\n"
        msg += f"See notebook '{nb_err}' for the traceback."
        print(msg)
        with open(nb_err, mode="w", encoding="utf-8") as f:
            nbformat.write(nb, f)
        raise 

def test_timeline_controls(self):
        nb_path = Path(_NB_FOLDER).joinpath(_NB_NAME)
        abs_path = Path(_NB_FOLDER).absolute()
        with open(nb_path) as f:
            nb = nbformat.read(f, as_version=4)
        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

        try:
            ep.preprocess(nb, {"metadata": {"path": abs_path}})
        except CellExecutionError:
            nb_err = str(nb_path).replace(".ipynb", "-err.ipynb")
            msg = f"Error executing the notebook '{nb_path}'.\n"
            msg += f"See notebook '{nb_err}' for the traceback."
            print(msg)
            with open(nb_err, mode="w", encoding="utf-8") as f:
                nbformat.write(nb, f)
            raise 

def execute_test(file_path, result_path):
  """Executes a single notebook.

  Args:
    file_path: Path to the notebook to execute.
    result_path: Path to store the resulting notebook.

  Returns:
    bool: True if the notebook does not have any errors, False otherwise.

  Raises:
    Exception if an unexpected error occurs executing the notebook.
  """
  try:
    with open(file_path, 'r') as f:
      filedata = f.read()
      if FLAGS.override_pip_install_agents:
        filedata = filedata.replace('pip install tf-agents', 'pip --version')
      nb = nbformat.reads(filedata, as_version=4)

      ep = ExecutePreprocessor(timeout=3600, kernel_name='python3')
      try:
        ep.preprocess(nb, {'metadata': {'path': FLAGS.output_dir}})
      except CellExecutionError as cex:
        logging.error('ERROR executing:%s', file_path)
        logging.error(cex)
        return False
    with open(result_path, 'w', encoding='utf-8') as fo:
      nbformat.write(nb, fo)
    return True
  except Exception as e:  # pylint: disable=W0703
    logging.error('Unexpected ERROR: in %s', file_path)
    logging.error(e) 

def execute_notebook(npth, dpth, timeout=600, kernel='python3'):
    """
    Execute the notebook at `npth` using `dpth` as the execution directory.
    The execution timeout and kernel are `timeout` and `kernel`
    respectively.
    """

    ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
    nb = nbformat.read(npth, as_version=4)
    t0 = timer()
    ep.preprocess(nb, {'metadata': {'path': dpth}})
    t1 = timer()
    with open(npth, 'wt') as f:
        nbformat.write(nb, f)
    return t1-t0 

def execution_cases(self, builderSelf, directory, allow_errors, subdirectory, language, futures, nb, filename, full_path):
        ## function to handle the cases of execution for coverage reports or html conversion pipeline
        if subdirectory != '':
            builderSelf.executed_notebook_dir = directory + "/" + subdirectory
        else:
            builderSelf.executed_notebook_dir = directory

        ## ensure that executed notebook directory
        ensuredir(builderSelf.executed_notebook_dir)
        ## specifying kernels
        if language == 'python':
            if (sys.version_info > (3, 0)):
                # Python 3 code in this block
                ep = ExecutePreprocessor(timeout=-1, allow_errors=allow_errors, kernel_name='python3')
            else:
                # Python 2 code in this block
                ep = ExecutePreprocessor(timeout=-1, allow_errors=allow_errors, kernel_name='python2')
        elif language == 'julia':
            ep = ExecutePreprocessor(timeout=-1, allow_errors=allow_errors)

        ### calling this function before starting work to ensure it starts recording
        if (self.startFlag == 0):
            self.startFlag = 1
            builderSelf.client.get_task_stream()


        future = builderSelf.client.submit(ep.preprocess, nb, {"metadata": {"path": builderSelf.executed_notebook_dir, "filename": filename, "filename_with_path": full_path}})

        ### dictionary to store info for errors in future
        future_dict = { "filename": full_path, "filename_with_path": full_path, "language_info": nb['metadata']['kernelspec']}
        builderSelf.futuresInfo[future.key] = future_dict

        futures.append(future) 

def test_notebooks(tmpdir, filename):
    """ Runs a given notebook in a tmpdir """
    import pylada
    from os.path import join, dirname
    from nbformat import read
    from nbconvert.preprocessors import ExecutePreprocessor
    from sys import version_info
    directory = dirname(__file__)
    with open(join(directory, filename + ".ipynb")) as notebook_file:
        notebook = read(notebook_file, as_version=4)
    preprocessor = ExecutePreprocessor(kernel_name='python%i' %
                                       version_info.major)
    preprocessor.preprocess(notebook, {'metadata': {'path': str(tmpdir)}}) 

def execute_notebook(path):
    notebook = nbformat.read(path, as_version=4)
    notebook.cells.extend([unittest_cell(), doctest_cell()])
    processor = ExecutePreprocessor(timeout=-1, kernel_name='python3')
    processor.preprocess(notebook, metadata(path))
    return parse_test_result(notebook.cells) 

def execute_notebook(npth, dpth, timeout=1800, kernel='python3'):
    """
    Execute the notebook at `npth` using `dpth` as the execution
    directory.  The execution timeout and kernel are `timeout` and
    `kernel` respectively.
    """

    ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
    nb = nbformat.read(npth, as_version=4)
    t0 = timer()
    ep.preprocess(nb, {'metadata': {'path': dpth}})
    t1 = timer()
    with open(npth, 'wt') as f:
        nbformat.write(nb, f)
    return t1 - t0 

def run_notebook(self, notebook_filename=None):

        if not notebook_filename:
            notebook_filename = self._output_stem + '.ipynb'

        dir_ = os.getcwd()
        with open(notebook_filename) as f:
            nb = nbformat.read(f, as_version=4)

        ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
        ep.preprocess(nb, {'metadata': {'path': dir_}})

        with open(notebook_filename, 'wt') as f:
            nbformat.write(nb, f) 

def run(infile, outfile, timeout=2400):
    print("Executing %s..." % infile)

    # Open the notebook
    with open(infile, "r") as f:
        nb = nbformat.read(f, as_version=4)

    # Execute the notebook
    if nb["metadata"].get("nbsphinx_execute", True):
        try:
            ep = ExecutePreprocessor(timeout=timeout, kernel_name="python3")
            ep.preprocess(
                nb,
                {
                    "metadata": {
                        "path": os.path.dirname(os.path.abspath(infile))
                    }
                },
            )
        except Exception as error:
            if "[allow-failures]" in os.getenv("COMMIT", ""):
                pass
            else:
                raise error

    # Replace input in certain cells
    for cell in nb.get("cells", []):
        replace_input_with = cell["metadata"].get("replace_input_with", None)
        if replace_input_with is not None:
            cell["source"] = replace_input_with

    # Write it back
    with open(outfile, "w", encoding="utf-8") as f:
        nbformat.write(nb, f) 

def _runTest(self):
        kernel = "python%d" % sys.version_info[0]
        cur_dir = os.path.dirname(self.nbfile)

        with open(self.nbfile) as f:
            nb = nbformat.read(f, as_version=4)
            if self.cov:
                covdict = {
                    "cell_type": "code",
                    "execution_count": 1,
                    "metadata": {"collapsed": True},
                    "outputs": [],
                    "nbsphinx": "hidden",
                    "source": "import coverage\n"
                    "coverage.process_startup()\n"
                    "import sys\n"
                    'sys.path.append("{0}")\n'.format(cur_dir),
                }
                nb["cells"].insert(0, nbformat.from_dict(covdict))

            exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600)

            try:
                run_dir = os.getenv("WRADLIB_BUILD_DIR", cur_dir)
                exproc.preprocess(nb, {"metadata": {"path": run_dir}})
            except CellExecutionError as e:
                raise e

        if self.cov:
            nb["cells"].pop(0)

        with io.open(self.nbfile, "wt") as f:
            nbformat.write(nb, f)

        self.assertTrue(True) 

def _run_notebook(filename):
        # Create the preprocessor.
        execute_preprocessor = ExecutePreprocessor(timeout=6000, kernel_name='python3')

        # Open the notebook.
        file_path = os.path.dirname(os.path.abspath(filename))
        with open(filename) as file_:
            notebook = nbformat.read(file_, as_version=4)

        with warnings.catch_warnings():
            # Silence some spurious warnings.
            warnings.filterwarnings('ignore', category=DeprecationWarning)
            # Finally, run the notebook.
            execute_preprocessor.preprocess(notebook,
                                            {'metadata': {'path': file_path}}) 

def test_notebook_runs_without_errors(name):
    with open(os.path.join(notebook_directory, name)) as f:
        nb = nbformat.read(f, as_version=4)
    ep = ExecutePreprocessor(timeout=120)
    ep.preprocess(nb, {'metadata': {'path': notebook_directory}}) 

def replace_definitions(nb, values, execute=None, execute_resources=None,
                        lang=None):
    """Return a copy of nb with the first code cell defining the given parameters.

    values should be a list of Parameter objects (as returned by extract_parameters),
    with their .value attribute set to the desired value.

    If execute is True, the notebook is executed with the new values.
    execute_resources is passed to nbconvert.ExecutePreprocessor; it's a dict,
    and if possible should contain a 'path' key for the working directory in
    which to run the notebook.

    lang may be used to override the kernel name embedded in the notebook. For
    now, nbparameterise only handles 'python3' and 'python2'.
    """
    if execute is None:
        warn(
            "Default execute=True for replace_definitions will change in a "
            "future version of nbparameterise. Pass execute=True if you need "
            "execution.", stacklevel=2
        )
        execute = True

    nb = copy.deepcopy(nb)
    drv = get_driver_module(nb, override=lang)
    first_code_cell(nb).source = drv.build_definitions(values)
    if execute:
        resources = execute_resources or {}
        nb, resources = ExecutePreprocessor().preprocess(nb, resources)
    return nb 

def runTutorialNotebook():
    import nbformat
    from nbconvert.preprocessors import ExecutePreprocessor

    with open("data_model.ipynb") as f:
        nb = nbformat.read(f, as_version=4)
    ep = ExecutePreprocessor(timeout=600, kernel_name="python3")
    ep.preprocess(nb, {})