Python multiprocessing.html() Examples

def __init__(self, hocr_file_name, dpi):
        self.rect = namedtuple('Rect', ['x1', 'y1', 'x2', 'y2'])
        self.dpi = dpi
        self.boxPattern = re.compile(r'bbox((\s+\d+){4})')
        self.hocr = ElementTree.parse(hocr_file_name)
        # if the hOCR file has a namespace, ElementTree requires its use to
        # find elements
        matches = re.match(r'({.*})html', self.hocr.getroot().tag)
        self.xmlns = ''
        if matches:
            self.xmlns = matches.group(1)
        # get dimension in pt (not pixel!!!!) of the OCRed image
        self.width, self.height = None, None
        for div in self.hocr.findall(
                ".//%sdiv[@class='ocr_page']" % (self.xmlns)):
            coords = self.element_coordinates(div)
            pt_coords = self.pt_from_pixel(coords)
            self.width = pt_coords.x2 - pt_coords.x1
            self.height = pt_coords.y2 - pt_coords.y1
            # there shouldn't be more than one, and if there is, we don't want it
            break
        if self.width is None or self.height is None:
            raise HocrTransformError("hocr file is missing page dimensions") 

def safe_tempfile_path():
    # This gets a valid temporary file path in the safest possible way, although there is still no
    # guarantee that another process will not create a file at this path. The NamedTemporaryFile is
    # deleted when the context manager exits and the file object is closed.
    #
    # This is preferable to using NamedTemporaryFile as a context manager and passing the name
    # attribute of the file object around because NamedTemporaryFiles cannot be opened a second time
    # if already open on Windows NT or later:
    # https://docs.python.org/3.8/library/tempfile.html#tempfile.NamedTemporaryFile
    # https://github.com/dagster-io/dagster/issues/1582
    with tempfile.NamedTemporaryFile() as fd:
        path = fd.name

    try:
        yield Path(path).as_posix()

    finally:
        if os.path.exists(path):
            os.unlink(path) 

def calculate_pool_chunksize(num_checkers, num_jobs):
    """Determine the chunksize for the multiprocessing Pool.

    - For chunksize, see: https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.imap  # noqa
    - This formula, while not perfect, aims to give each worker two batches of
      work.
    - See: https://gitlab.com/pycqa/flake8/merge_requests/156#note_18878876
    - See: https://gitlab.com/pycqa/flake8/issues/265
    """
    return max(num_checkers // (num_jobs * 2), 1) 

def safe_isfile(path):
    '''"Backport of Python 3.8 os.path.isfile behavior.

    This is intended to backport https://docs.python.org/dev/whatsnew/3.8.html#os-path. I'm not
    sure that there are other ways to provoke this behavior on Unix other than the null byte,
    but there are certainly other ways to do it on Windows. Afaict, we won't mask other
    ValueErrors, and the behavior in the status quo ante is rough because we risk throwing an
    unexpected, uncaught ValueError from very deep in our logic.
    '''
    try:
        return os.path.isfile(path)
    except ValueError:
        return False 

def __readonly__(self, *args, **kwargs):
        raise RuntimeError("Cannot modify ReadOnlyDict")

    # https://docs.python.org/3/library/pickle.html#object.__reduce__
    #
    # For a dict, the default behavior for pickle is to iteratively call __setitem__ (see 5th item
    #  in __reduce__ tuple). Since we want to disable __setitem__ and still inherit dict, we
    # override this behavior by defining __reduce__. We return the 3rd item in the tuple, which is
    # passed to __setstate__, allowing us to restore the frozendict. 

def get_multiprocessing_context():
    # Set execution method to spawn, to avoid fork and to have same behavior between platforms.
    # Older versions are stuck with whatever is the default on their platform (fork on
    # Unix-like and spawn on windows)
    #
    # https://docs.python.org/3/library/multiprocessing.html#multiprocessing.get_context
    if hasattr(multiprocessing, 'get_context'):
        return multiprocessing.get_context('spawn')
    else:
        return multiprocessing 

def calculate_pool_chunksize(num_checkers, num_jobs):
    """Determine the chunksize for the multiprocessing Pool.

    - For chunksize, see: https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.imap  # noqa
    - This formula, while not perfect, aims to give each worker two batches of
      work.
    - See: https://gitlab.com/pycqa/flake8/merge_requests/156#note_18878876
    - See: https://gitlab.com/pycqa/flake8/issues/265
    """
    return max(num_checkers // (num_jobs * 2), 1) 

def check_external_tools(self):
        """Check if external tools are available, aborting or warning in case of any error."""
        self.path_tesseract = shutil.which(self.cmd_tesseract)
        if self.path_tesseract is None:
            eprint("tesseract not found. Aborting...")
            exit(1)
        #
        self.tesseract_can_textonly_pdf = self.test_tesseract_textonly_pdf()
        self.tesseract_version = self.get_tesseract_version()
        #
        self.path_cuneiform = shutil.which(self.cmd_cuneiform)
        if self.path_cuneiform is None:
            self.debug("cuneiform not available")
        #
        # Try to avoid errors on Windows with native OS "convert" command
        # http://savage.net.au/ImageMagick/html/install-convert.html
        # https://www.imagemagick.org/script/magick.php
        self.path_convert = shutil.which(self.cmd_convert)
        if not self.test_convert():
            self.path_convert = shutil.which(self.cmd_magick)
        if self.path_convert is None:
            eprint("convert/magick from ImageMagick not found. Aborting...")
            exit(1)
        #
        self.path_mogrify = shutil.which(self.cmd_mogrify)
        if self.path_mogrify is None:
            eprint("mogrify from ImageMagick not found. Aborting...")
            exit(1)
        #
        self.path_file = shutil.which(self.cmd_file)
        if self.path_file is None:
            eprint("file not found. Aborting...")
            exit(1)
        #
        self.path_pdftoppm = shutil.which(self.cmd_pdftoppm)
        if self.path_pdftoppm is None:
            eprint("pdftoppm (poppler) not found. Aborting...")
            exit(1)
        #
        self.path_pdffonts = shutil.which(self.cmd_pdffonts)
        if self.path_pdffonts is None:
            eprint("pdffonts (poppler) not found. Aborting...")
            exit(1)
        #
        self.path_ps2pdf = shutil.which(self.cmd_ps2pdf)
        self.path_pdf2ps = shutil.which(self.cmd_pdf2ps)
        if self.path_ps2pdf is None or self.path_pdf2ps is None:
            eprint("ps2pdf or pdf2ps (ghostscript) not found. File repair will not work...")
        #
        self.path_qpdf = shutil.which(self.cmd_qpdf)
        if self.path_qpdf is None:
            self.log("External tool 'qpdf' not available. Merge can be slow")
        #