import scrapy

class PoliticoSpider(scrapy.Spider):
    name = 'politicospider'
    start_urls = ['http://www.politico.com/search?start=11%2F01%2F2015&end=11%2F30%2F2015&s=newest']

    def __init__(self):
      self.download_delay = 1

    def parse(self, response):
      for url in response.xpath('//h3/a/@href').extract():
        if "/story/2015/11/" in url:
          yield scrapy.Request(response.urljoin(url), self.parse_story_page)

      for url in response.xpath('//a[contains(.,"Next page")]/@href').extract():
          yield scrapy.Request(response.urljoin(url))

    def parse_story_page(self, response):
      filename=response.url.split('/')[-1]
      f=open(filename,'w')
      print >>f, response.body
      f.close()