Q:

how to extract certain string from URL

I am trying to extract certain strings from the below mentioned URL :
sample URL :
http://www.ladyblush.com/buy-sarees-online.html?p=1
http://www.ladyblush.com/buy-ladies-suits-online.html?p=1
http://www.ladyblush.com/buy-women-fashion-accessories.html?p=1

i want to extract :
productCategory = "sarees" productSubCategory = ""
productCategory = "ladies" productSubCategory = "suits"
productCategory = "women" productSubCategory = "fashion-accessories"

And so on. Actually i am writing a spider and i need to extract productCategory and productSubCategory from URL's like above mentioned..so i am trying to extract these fields inside parse method from response.url. Can someone help me out please
My code :
import re
from scrapy.http import Request
from eScraper.items import EscraperItem
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider

#------------------------------------------------------------------------------ 

class ESpider(CrawlSpider):

    name = "ladyblushSpider"    
    allowed_domains = ["ladyblush.com"]    
    URLSList = []

    for n in range (1,100):

        URLSList.append('http://www.ladyblush.com/buy-sarees-online.html?p=' + str(n))
        URLSList.append('http://www.ladyblush.com/buy-ladies-suits-online.html?p=' + str(n))
        URLSList.append('http://www.ladyblush.com/buy-women-fashion-accessories.html?p=' + str(n))
        URLSList.append('http://www.ladyblush.com/buy-nightwear-lingerie-online.html?p=' + str(n))
        URLSList.append('http://www.ladyblush.com/buy-women-dress-online-skirts-suits-kurtis-tops.html?p=' + str(n))
        URLSList.append('http://www.ladyblush.com/buy-decor-online-wallclock-bedsheets-cushions-bedcovers.html?p=' + str(n))
        URLSList.append('http://www.ladyblush.com/buy-cosmetics-online-massage-oils-aromatherapy-perfumes-soaps.html?p=' + str(n))
        URLSList.append('http://www.ladyblush.com/buy-jewelery-online-art-fashion-semi-precious-antique-junk-jewellery.html?p=' + str(n))

    start_urls = URLSList

    def parse(self, response):

        item = EscraperItem()
        hxs = HtmlXPathSelector(response)
        sites = hxs.select('//div[@class="third thumbnailSpillLarge"]')
        items = []

        for site in sites:

            item = EscraperItem()

            item['currency'] = 'INR'
            item['productCategory'] = [""]
            item['productSubCategory'] = [""]
            item['productSite'] = ["http://ladyblush.com/"]
            item['productImage'] = site.select('./a/div/img/@src').extract()
            item['productTitle'] = site.select('./a/div/img/@title').extract()                         
            item['productURL'] = [site.select('./a/@href').extract()[0].replace(" ","%20")]
            productMRP = site.select('.//div[@class="salePrice"]//div[@class="price-box"]//p[@class="old-price"]//span[@class="price"]/text()').extract()
            productPrice = site.select('.//div[@class="salePrice"]//div[@class="price-box"]//p[@class="special-price"]//span[@class="price"]/text()').extract()

            if productMRP and productPrice:
                price = [productMRP[1].strip()] + [productPrice[1].strip()]  
            else:
                price = site.select('.//div[@class="salePrice"]//div[@class="price-box"]//span[@class="regular-price"]//span[@class="price"]/text()').extract()    
            item['productPrice'] = price

            items.append(item)            
            secondURL = item['productURL'][0] 
            request = Request(secondURL,callback=self.parsePage2)
            request.meta['item'] = item

            yield request

    def parsePage2(self, response):

        hxs = HtmlXPathSelector(response)

        item = response.meta['item']

        if  hxs.select('//div[@class="addtocart-container"]/div/text()').extract():                   
            item['availability'] = False            
        else:                   
            item['availability'] = True

        if hxs.select('//label[@class="required"]/text()').extract():
            item['hasVariants'] = True
        else:         
            item['hasVariants'] = False
        item['image_urls'] = list(set(item['productImage']))
        item['productDesc'] = [" ".join([re.sub(r'[\t\n\r]',"",i.strip()) for i in hxs.select('//div[@class="std"]/text()').extract()])] 
        item['productImage'] = item['productImage'] + hxs.select('//div[@class="more-views"]/ul/li/a/img/@src').extract() + hxs.select('//div[@class="more-views"]/ul/li/a/@href').extract()  

        return item

#------------------------------------------------------------------------------ 

A:

you can get the url from 
response.url in the parse method.  You could then parse that to just get the url path
import os
test = 'buy-women-fashion-accessories.html?p=1'
parts = os.path.splitext(test)
# ('buy-women-fashion-accessories', '.html?p=1')
parts[0].split('-')[1:]
# ['women', 'fashion', 'accessories']

This is rather flimsy solution though.  Are you sure the data is not stored somewhere in the page's html that your are parsing, instead of looking at the url?

