selenium:socket.error: [Errno 61] Connection refused
The problem is that you are closing the driver too early.
You should close it only when the spider finishes it work, listen to spider_closed
signal:
from scrapy import signalsfrom scrapy.xlib.pydispatch import dispatcherfrom selenium import webdriverfrom scrapy.spider import Spiderfrom ta.items import TaItemfrom scrapy.http.request import Requestclass ProductSpider(Spider): name = "spider2" start_urls = ['http://www.test.com/'] def __init__(self): self.driver = webdriver.Firefox() dispatcher.connect(self.spider_closed, signals.spider_closed) def parse(self, response): self.driver.get(response.url) self.driver.implicitly_wait(20) next = self.driver.find_elements_by_css_selector("div.body .heading a") for a in next: item = TaItem() item['link'] = a.get_attribute("href") yield Request(url=item['link'], meta={'item': item}, callback=self.parse_detail) def parse_detail(self,response): item = response.meta['item'] yield item def spider_closed(self, spider): self.driver.close()
See also: scrapy: Call a function when a spider quits.