<commit_before>require 'hpricot'

module Seek
  
  module ModelBuilder
    BUILDER_URL_BASE = "http://jjj.mib.ac.uk/webMathematica/Examples/JWSconstructor_panels"
    
    def self.builder_url
      "#{BUILDER_URL_BASE}/DatFileReader.jsp"
    end
    
    def self.upload_dat_url
      self.builder_url+"?datFilePosted=true"
    end
    
    def self.get_validate_content model
      filepath=model.content_blob.filepath
      
      part=Multipart.new({:uploadedDatFile=>filepath})
      
      response = part.post(upload_dat_url)
      
      if response.instance_of?(Net::HTTPInternalServerError)       
        puts response.to_s
        raise Exception.new(response.body.gsub(/<head\>.*<\/head>/,""))
      end
      
      process_response_body(response.body)
      
    end
    
    def self.get_content       
      uri=URI.parse(builder_url)      
      http=Net::HTTP.new(uri.host,uri.port)

      http.use_ssl=true if uri.scheme=="https"
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE

      req=Net::HTTP::Get.new(uri.path)      
            
      process_response_body http.request(req).body      
    end
    
    def self.process_response_body body            
      
      doc = Hpricot(body)
      
      div_block = find_the_boxes_div(doc).join("\n")
      div_block = div_block.gsub("window.open('/webMathematica","window.open('http://jjj.mib.ac.uk/webMathematica/")
      
      return process_scripts_and_styles(doc).join("\n"),div_block
    end
    
    def self.process_scripts_and_styles doc
      
      ss = []
      doc.search("//script").each do |script|
        src=script.attributes['src']
        if src
          src=BUILDER_URL_BASE+"/"+src
          script.attributes['src'] = src  
        end        
        ss << script.to_s
        break if ss.size == 6
      end
      
      doc.search("//link[@rel='stylesheet'").each do |link|
        href=link.attributes['href']
        
        if href
          href=BUILDER_URL_BASE+"/"+href
          link.attributes['href'] = href  
        end        
        ss << link.to_html
      end
      
      return ss
    end
    <commit_msg>fixed xpath to select stylesheet links. now correctly strips out favicon<commit_after>require 'hpricot'

module Seek
  
  module ModelBuilder
    BUILDER_URL_BASE = "http://jjj.mib.ac.uk/webMathematica/Examples/JWSconstructor_panels"
    
    def self.builder_url
      "#{BUILDER_URL_BASE}/DatFileReader.jsp"
    end
    
    def self.upload_dat_url
      self.builder_url+"?datFilePosted=true"
    end
    
    def self.get_validate_content model
      filepath=model.content_blob.filepath
      
      part=Multipart.new({:uploadedDatFile=>filepath})
      
      response = part.post(upload_dat_url)
      
      if response.instance_of?(Net::HTTPInternalServerError)       
        puts response.to_s
        raise Exception.new(response.body.gsub(/<head\>.*<\/head>/,""))
      end
      
      process_response_body(response.body)
      
    end
    
    def self.get_content       
      uri=URI.parse(builder_url)      
      http=Net::HTTP.new(uri.host,uri.port)

      http.use_ssl=true if uri.scheme=="https"
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE

      req=Net::HTTP::Get.new(uri.path)      
            
      process_response_body http.request(req).body      
    end
    
    def self.process_response_body body            
      
      doc = Hpricot(body)
      
      scripts_and_stylesheets = process_scripts_and_styles(doc).join("\n")
      div_block = find_the_boxes_div(doc).join("\n")
      div_block = div_block.gsub("window.open('/webMathematica","window.open('http://jjj.mib.ac.uk/webMathematica/")
      
      return scripts_and_stylesheets,div_block
    end
    
    def self.process_scripts_and_styles doc
      
      ss = []
      doc.search("//script").each do |script|
        src=script.attributes['src']
        if src
          src=BUILDER_URL_BASE+"/"+src
          script.attributes['src'] = src  
        end        
        ss << script.to_s
        break if ss.size == 6
      end
      doc.search("//link[@rel='stylesheet']")
      doc.search("//link[@rel='stylesheet']").each do |link|
        href=link.attributes['href']
        
        if href
          href=BUILDER_URL_BASE+"/"+href
          link.attributes['href'] = href  
        end        
        ss << link.to_html
      end            
      
      return ss
    end
    
{'fraction_non_alphanumeric': 0.11224258882100023, 'fraction_numerical': 0.00045259108395564606, 'mean_word_length': 2.0273972602739727, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '781919', 'n_tokens_mistral': 1417, 'n_tokens_neox': 1409, 'n_words': 201}