Python scrapy.Field() Examples
The following are 11
code examples of scrapy.Field().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
scrapy
, or try the search function
.
.
Example #1
| Source File: pipelines.py From spidermon with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
def _add_errors_to_item(self, item, errors):
try:
if self.errors_field not in item.__class__.fields:
item.__class__.fields[self.errors_field] = Field()
if self.errors_field not in item._values:
item[self.errors_field] = defaultdict(list)
except AttributeError:
# The item is just a dict object instead of a Scrapy.Item object
if self.errors_field not in item:
item[self.errors_field] = defaultdict(list)
for field_name, messages in errors.items():
item[self.errors_field][field_name] += messages
Example #2
| Source File: event.py From In2ItChicago with GNU General Public License v3.0 | 5 votes |
|
def custom_field():
return scrapy.Field(input_processor=MapCompose(DataUtils.remove_html), output_processor=Join())
Example #3
| Source File: event.py From In2ItChicago with GNU General Public License v3.0 | 5 votes |
|
def numeric_field():
return scrapy.Field(input_processor=MapCompose(DataUtils.remove_html), output_processor=TakeFirst())
Example #4
| Source File: event.py From In2ItChicago with GNU General Public License v3.0 | 5 votes |
|
def price_field():
return scrapy.Field(input_processor=MapCompose(
lambda value: value.replace('$', '') if type(value) == str else value,
DataUtils.remove_html, float),
output_processor=TakeFirst())
Example #5
| Source File: event.py From In2ItChicago with GNU General Public License v3.0 | 5 votes |
|
def url_field():
return scrapy.Field(input_processor=MapCompose(DataUtils.remove_html,
lambda value: value \
.replace('//', '/') \
.replace('https:/', 'https://') \
.replace('http:/', 'http://') \
.rstrip('/')),
output_processor=Join())
Example #6
| Source File: event.py From In2ItChicago with GNU General Public License v3.0 | 5 votes |
|
def category_field():
return scrapy.Field(output_processor=Join())
Example #7
| Source File: event.py From In2ItChicago with GNU General Public License v3.0 | 5 votes |
|
def date_field():
def parse_date(value):
date_format = value['date_format']
time_utils = TimeUtils(date_format=date_format)
date_obj = {**create_time_data(), **value}
start_timestamp, end_timestamp = time_utils.get_timestamps(date_obj)
return {
'start_timestamp': start_timestamp,
'end_timestamp': end_timestamp
}
return scrapy.Field(input_processor=MapCompose(DataUtils.remove_html, parse_date), output_processor=TakeFirst())
Example #8
| Source File: lagou_Item.py From FunpySpiderSearchEngine with Apache License 2.0 | 5 votes |
|
def help_fields(self):
for field in self.field_list:
print(field, "= scrapy.Field()")
Example #9
| Source File: zhihu_item.py From FunpySpiderSearchEngine with Apache License 2.0 | 5 votes |
|
def help_fields(self):
for field in self.fields:
print(field, "= scrapy.Field()")
Example #10
| Source File: zhihu_item.py From FunpySpiderSearchEngine with Apache License 2.0 | 5 votes |
|
def help_fields(self):
for field in self.field_list:
print(field, "= scrapy.Field()")
Example #11
| Source File: jobbole_Item.py From FunpySpiderSearchEngine with Apache License 2.0 | 5 votes |
|
def help_fields(self):
for field in self.field_list:
print(field, "= scrapy.Field()")
