Python process spider exception
60 Python code examples are found related to "
process spider exception".
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
Source File: middlewares.py From scrapy_spider with MIT License | 5 votes |
def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass
Example 2
Source File: middlewares.py From py-scraping-analysis-book with MIT License | 5 votes |
def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass
Example 3
Source File: scrapy_pagestorage.py From scrapy-pagestorage with BSD 3-Clause "New" or "Revised" License | 5 votes |
def process_spider_exception(self, response, exception, spider): if (self.on_error_enabled and not isinstance(exception, IgnoreRequest) and self.counters['error'] < self.limits['error']): self.counters['error'] += 1 self.save_response(response, spider)
Example 4
Source File: middlewares.py From notes with Apache License 2.0 | 5 votes |
def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Request, dict # or Item objects. pass
Example 5
Source File: spidermiddlewares.py From PyFeeds with GNU Affero General Public License v3.0 | 5 votes |
def process_spider_exception(self, response, exception, spider): if isinstance(exception, HttpError): if response.status >= 500: # Transient errors usually caused by overloaded sites, updates, short # downtimes, etc. lgr = logger.info else: lgr = logger.warning lgr( "Ignoring response %(response)r: HTTP status code is not " "handled or not allowed", {"response": response}, extra={"spider": spider}, ) return []
Example 6
Source File: middlewares.py From OpenScraper with MIT License | 5 votes |
def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass
Example 7
Source File: middlewares.py From freshonions-torscraper with GNU Affero General Public License v3.0 | 5 votes |
def process_spider_exception(response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass
Example 8
Source File: middlewares.py From AntSpider with MIT License | 5 votes |
def process_spider_exception(self, response, exception, spider): print(f'#return exception reason:{type(exception)}') #if isinstance(exception,TimeoutError): # spider.logger.info("Request TimeoutError.") #return request #spider.logger.info('exception: %s' % spider.name) # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass