<commit_before>from parser import Parser
from scrapy import log
from scrapy.http import Request
from scrapy.selector import Selector
from FourmiCrawler.items import Result

"""
This parser will manage searching for chemicals through the ChemsSpider API,
and parsing the resulting ChemSpider page. 
The token required for the API should be in a configuration file somewhere.
"""
class ChemSpider(Parser):
    
    website = "http://www.chemspider.com/*"

    search = "Search.asmx/SimpleSearch?query=%s&token=052bfd06-5ce4-43d6-bf12-89<PASSWORD>338"
    structure = "Chemical-Structure.%s.html"

    ignore_list = []

    def parse(self, response):
        sel = Selector(response)
        requests = []
        requests_synonyms = self.parse_synonyms(sel)
        requests.extend(requests_synonyms)
        for wiki_url in sel.xpath('.//a[@title="Wiki"]/@href').extract():
            requests.append( Request(url=wiki_url) )

        return requests

    def parse_synonyms(self, sel):
        requests = []
        synonyms = []
        for syn in sel.xpath('//p[@class="syn"]/strong/text()').extract():
            synonyms.append( self.new_synonym( syn, 'high' ) )
        for syn in sel.xpath('//p[@class="syn"]/span[@class="synonym_confirmed"]/text()').extract():
            synonyms.append( self.new_synonym( syn, 'medium' ) )
        for syn in sel.xpath('//p[@class="syn"]/span[@class=""]/text()').extract():
            synonyms.append( self.new_synonym( syn, 'low' ) )

        return requests

    def new_synonym(self, name, reliability):
        log.msg('CS synonym: %s (%s)' % (name, reliability), level=log.WARNING)
        self.ignore_list.append(name)
        synonym = Result()
        synonym['attribute'] = 'synonym'
        synonym['value'] = name
        synonym['source'] = 'ChemSpider'
        synonym['reliability'] = reliability
        synonym['conditions'] = None
        return synonym


    def parse_searchrequest(self, response):
        sel = Selector(response)
        log.msg('chemspider parse_searchrequest', level=log.WARNING)
        sel.register_namespace('cs', 'http://www.chemspider.com/')
        csid = sel.xpath('.//cs:int/text()').extract()[0]
        #TODO: handle multiple csids in case of vague search term
        structure_url = self.website[:-1] + self.structure % csid
        log.msg('chemspider URL: %s' % structure_url, level=log.WARNING)
        return Request(structure_url, callback=self.parse)
    
    def new_compound_request(self,compound):
        if compound in self.ignore_list: #TODO: add regular expression
            return None
        searchurl = self.website[:-1] + self.search % compound
        log.msg('chemspider compound', level=log.WARNING)
        return Request(url=searchurl, callback=self.parse_searchrequest)
<commit_msg>chemspider parser now makes new synonym requests with the scraped synonyms<commit_after>from parser import Parser
from scrapy import log
from scrapy.http import Request
from scrapy.selector import Selector
from FourmiCrawler.items import Result

"""
This parser will manage searching for chemicals through the ChemsSpider API,
and parsing the resulting ChemSpider page. 
The token required for the API should be in a configuration file somewhere.
"""
class ChemSpider(Parser):
    
    website = "http://www.chemspider.com/*"

    search = "Search.asmx/SimpleSearch?query=%s&token=052bfd06-5ce4-43d6-bf12-89eabefd2338"
    structure = "Chemical-Structure.%s.html"

    ignore_list = []

    def parse(self, response):
        sel = Selector(response)
        requests = []
        requests_synonyms = self.parse_synonyms(sel)
        requests.extend(requests_synonyms)
        for wiki_url in sel.xpath('.//a[@title="Wiki"]/@href').extract():
            requests.append( Request(url=wiki_url) )

        return requests

    def parse_synonyms(self, sel):
        requests = []
        synonyms = []
        for syn in sel.xpath('//p[@class="syn"]/strong/text()').extract():
            synonyms.append( self.new_synonym( syn, 'high' ) )
        for syn in sel.xpath('//p[@class="syn"]/span[@class="synonym_confirmed"]/text()').extract():
            synonyms.append( self.new_synonym( syn, 'medium' ) )
        for syn in sel.xpath('//p[@class="syn"]/span[@class=""]/text()').extract():
            synonyms.append( self.new_synonym( syn, 'low' ) )

        for synonym in synonyms:
            if synonym['reliability'] == 'high':
                self._Parser__spider.get_synonym_requests(synonym['value'])

        return requests

    def new_synonym(self, name, reliability):
        log.msg('CS synonym: %s (%s)' % (name, reliability), level=log.WARNING)
        self.ignore_list.append(name)
        synonym = Result()
        synonym['attribute'] = 'synonym'
        synonym['value'] = name
        synonym['source'] = 'ChemSpider'
        synonym['reliability'] = reliability
        synonym['conditions'] = None
        return synonym


    def parse_searchrequest(self, response):
        sel = Selector(response)
        log.msg('chemspider parse_searchrequest', level=log.WARNING)
        sel.register_namespace('cs', 'http://www.chemspider.com/')
        csid = sel.xpath('.//cs:int/text()').extract()[0]
        #TODO: handle multiple csids in case of vague search term
        structure_url = self.website[:-1] + self.structure % csid
        log.msg('chemspider URL: %s' % structure_url, level=log.WARNING)
        return Request(structure_url, callback=self.parse)
    
    def new_compound_request(self,compound):
        if compound in self.ignore_list: #TODO: add regular expression
            return None
        searchurl = self.website[:-1] + self.search % compound
        log.msg('chemspider compound', level=log.WARNING)
        return Request(url=searchurl, callback=self.parse_searchrequest)

{'fraction_non_alphanumeric': 0.13038256990907532, 'fraction_numerical': 0.007033796534568536, 'mean_word_length': 3.045801526717557, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '651402', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1742, 'n_words': 396}