Python process spider exception

60 Python code examples are found related to " process spider exception". You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
Example 1
Source File: middlewares.py    From scrapy_spider with MIT License 5 votes vote down vote up 
def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass
Example 2
Source File: middlewares.py    From py-scraping-analysis-book with MIT License 5 votes vote down vote up 
def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass
Example 3
Source File: scrapy_pagestorage.py    From scrapy-pagestorage with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
def process_spider_exception(self, response, exception, spider):
        if (self.on_error_enabled and
                not isinstance(exception, IgnoreRequest) and
                self.counters['error'] < self.limits['error']):
            self.counters['error'] += 1
            self.save_response(response, spider)
Example 4
Source File: middlewares.py    From notes with Apache License 2.0 5 votes vote down vote up 
def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request, dict
        # or Item objects.
        pass
Example 5
Source File: spidermiddlewares.py    From PyFeeds with GNU Affero General Public License v3.0 5 votes vote down vote up 
def process_spider_exception(self, response, exception, spider):
        if isinstance(exception, HttpError):
            if response.status >= 500:
                # Transient errors usually caused by overloaded sites, updates, short
                # downtimes, etc.
                lgr = logger.info
            else:
                lgr = logger.warning
            lgr(
                "Ignoring response %(response)r: HTTP status code is not "
                "handled or not allowed",
                {"response": response},
                extra={"spider": spider},
            )
            return []
Example 6
Source File: middlewares.py    From OpenScraper with MIT License 5 votes vote down vote up 
def process_spider_exception(self, response, exception, spider):
		# Called when a spider or process_spider_input() method
		# (from other spider middleware) raises an exception.

		# Should return either None or an iterable of Response, dict
		# or Item objects.
		pass
Example 7
Source File: middlewares.py    From freshonions-torscraper with GNU Affero General Public License v3.0 5 votes vote down vote up 
def process_spider_exception(response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass
Example 8
Source File: middlewares.py    From AntSpider with MIT License 5 votes vote down vote up 
def process_spider_exception(self, response, exception, spider):
        print(f'#return exception reason：{type(exception)}')
        #if isinstance(exception,TimeoutError):
        #    spider.logger.info("Request TimeoutError.")
            #return request
        #spider.logger.info('exception: %s' % spider.name)
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass