How to send data to scrapy pipeline to mongodb

# In your pipeline

class EPGD_pipeline(object):
    def __init__(self):
        self.collections = {
            spider_name: self.setup_db_connection(dj_mongo_database_url.parse(url))
            for spider_name, url in settings['MONGODB_PIPELINE_SETTINGS'].iterItems()
        )
    }

    def process_item(self, item, spider):
        collection = self.collections[spider.name]
        ...


# In settings.py

MONGODB_PIPELINE_SETTINGS = {
    "GenDis": "mongodb://myhost:29297/test_db/collection",
    "EPGD": "mongodb://myhost:29297/test_db/collection2",
}

View another examples Add Own solution

3.25

Justin Hammar 60 points

                                        from pymongo import MongoClient
    from scrapy.conf import settings
    from scrapy.exceptions import DropItem
    from scrapy import log

    class MongoDBPipeline(object):
        collection_name= 'members'
        def __init__(self, mongo_uri, mongo_db):
            self.mongo_uri = mongo_uri
            self.mongo_db = mongo_db
        @classmethod
        def from_crawler(cls, crawler):
            return cls(
                mongo_uri=crawler.settings.get('MONGO_URI')
                mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
            )
        def open_spider(self,spider):
            self.client = pymongo.MongoClient(self.mongo_uri)
            self.db = self.client[self.mongo_db]
        def close_spider(self, spider):
            self.client.close()
        def process_item(self, item, spider):
            self.db[self.collection_name].insert(dict(item))
            return item

Thank you! 4

3.25 (4 Votes)

Blitheringidiots 85 points

                                        BOT_NAME = 'capstone'

    SPIDER_MODULES = ['capstone.spiders']
    NEWSPIDER_MODULE = 'capstone.spiders'

    ITEM_PIPLINES = {'capstone.pipelines.MongoDBPipeline': 300,}
    MONGO_URI = 'mongodb://localhost:27017'
    MONGO_DATABASE = 'congress'
    ROBOTSTXT_OBEY = True
    DOWNLOAD_DELAY = 10

Thank you! 2

Are there any code examples left?

Find Add Code snippet

New code examples in category Python

Python 2023-04-11 03:04:20
Python 2022-03-27 22:40:04 pycharm no module named
Python 2022-03-27 22:25:05 assign multiple variablesin one line
Python 2022-03-27 22:20:02 levenshtein distance
Python 2022-03-27 21:35:09 get text from url python last slash
Python 2022-03-27 21:30:30 df concatenate df
Python 2022-03-27 21:25:09 python odd or even
Python 2022-03-27 21:15:32 python include function from another file
Python 2022-03-27 21:10:01 color module python
Python 2022-03-27 21:00:27 python tkinter cursor types

Create a Free Account

Unlock the power of data and AI by diving into Python, ChatGPT, SQL, Power BI, and beyond.

Develop soft skills on BrainApps

Complete the IQ Test

Relative searches

pipeline scrapy mongodb send scrapy data to mongodb scrapy mongodb pipeline scrapy database how to save scrapy in mongoDB mongo scrapy field name from response mongo scrapy name collection from response python crawling mongodb from scrapy crawl to mongodb scrapy mongodb how to not scrape data which is already present in mongodb in scrapy connect scrapy to mongodb How to send data to scrapy pipeline to mongodb

How to send data to scrapy pipeline to mongodb

Welcome Back!

Create a Free Account