selenium:socket.error: [Errno 61] Connection refused selenium:socket.error: [Errno 61] Connection refused selenium selenium

selenium:socket.error: [Errno 61] Connection refused


The problem is that you are closing the driver too early.

You should close it only when the spider finishes it work, listen to spider_closed signal:

from scrapy import signalsfrom scrapy.xlib.pydispatch import dispatcherfrom selenium import webdriverfrom scrapy.spider import Spiderfrom ta.items import TaItemfrom scrapy.http.request import Requestclass ProductSpider(Spider):    name = "spider2"      start_urls = ['http://www.test.com/']    def __init__(self):        self.driver = webdriver.Firefox()        dispatcher.connect(self.spider_closed, signals.spider_closed)    def parse(self, response):        self.driver.get(response.url)        self.driver.implicitly_wait(20)          next = self.driver.find_elements_by_css_selector("div.body .heading a")        for a in next:            item = TaItem()                item['link'] =  a.get_attribute("href")                 yield Request(url=item['link'], meta={'item': item}, callback=self.parse_detail)      def parse_detail(self,response):        item = response.meta['item']        yield item    def spider_closed(self, spider):        self.driver.close()

See also: scrapy: Call a function when a spider quits.