Search on blog:

Python: How to scrape with scrapy

It is example code to scrape it:

#!/usr/bin/env python3

# date: 2019.07.29

import scrapy

class MySpider(scrapy.Spider):

    name = 'myspider'

    start_urls = [

    def parse(self, response):

        for link in response.css('.default .er').xpath('@href').extract():
             url = response.url
             path = response.css('ol.breadcrumb li a::text').extract()
             next_link = response.urljoin(link)
             yield scrapy.Request(next_link, callback=self.parse_det, meta={'url': url, 'path': path})

    def parse_det(self, response):

        def extract_with_css(query):
            return response.css(query).get(default='').strip()

        yield {
            'file_urls': [extract_with_css('a.btn.btn-primary::attr(href)')],

from scrapy.crawler import CrawlerProcess

c = CrawlerProcess({
    'USER_AGENT': 'Mozilla/5.0',

    # save in file as CSV, JSON or XML
    'FEED_FORMAT': 'csv',     # csv, json, xml
    'FEED_URI': 'output.csv', # 

    # download files to `FILES_STORE/full`
    # it needs `yield {'file_urls': [url]}` in `parse()`
    'ITEM_PIPELINES': {'scrapy.pipelines.files.FilesPipeline': 1},
    'FILES_STORE': '.',
If you like it
Buy a Coffee