Answer the question
In order to leave comments, you need to log in
How to write data to MySQL BD in scrapy?
Hello. Please help me write the data to the database, I just started working with scrapy.
I scan all links from the sitemap, like this
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
name = 'jobs'
sitemap_urls = ['http://example.com/sitemap.xml']
item = []
def parse(self, response):
item["title"] = response.css("title ::text").extract_first()
item["description"] = response.css("meta[name=\"description\"] ::attr(content)").extract_first()
item["url"] = response.url
item["status"] = response.status
return item
ITEM_PIPELINES = {
'craigslist.pipelines.CraigslistPipeline': 300,
}
DB_SETTINGS = {
'db': 'db',
'user': 'user',
'passwd': 'passwd',
'host': 'host',
}
class CraigslistPipeline(object):
def process_item(self, item, spider):
query = ("INSERT INTO table1 (title, description, code, url) VALUES (%s, %s, %s, %s)")
self.cursor.execute(query, (
item["title"],
item["description"],
item["code"],
item["url"],
)
)
self.conn.commit()
return item
Answer the question
In order to leave comments, you need to log in
Didn't find what you were looking for?
Ask your questionAsk a Question
731 491 924 answers to any question