<commit_before>require 'tempfile'

module PhantomJSProxy
	class PhantomJS
		attr_accessor :dom
		attr_accessor :image
    attr_accessor :ready
	
		def initialize()
			@ready = 503
		end
		
		def getUrl(url, pictureOnly=true, loadIFrames=true)
			puts("PhantomJS: "+url)
			@ready = 503
			
			pictureFile = nil
			picture = "none"
      
      loadFrames = "false"
      
      if loadIFrames
        loadFrames = "true"
      end
			
			if pictureOnly
				if !File.directory?("/tmp/phantom_proxy")
					Dir.mkdir("/tmp/phantom_proxy")
				end
				pictureFile = Tempfile.new(["phantom_proxy/page", ".png"])
				picture = pictureFile.path
			end
			
			url_args = ""
			url_args_ = []
      
			if /\?/.match(url)
				url_args = url.split('?')[1]
				url = url.split('?')[0]
				
				if url_args
					url_args_ = url_args.split('&')
					url_args = url_args_.join(' ')
				end
			end
			
			@dom = invokePhantomJS(SCRIPT, [picture, loadFrames, "\""+url+"\"", url_args_.length, url_args])
			
			puts("Opened page: "+ /Open page: (.*?) END/.match(@dom)[1])
			
			@ready = 503
			dom_text = "Failed to load page"
			
			if /DONE_LOADING_URL/.match(@dom)
        puts("LOAD_DOM_TEXT")
				dom_text = getDOMText @dom
        puts("LOAD_DOM_TEXT_DONE")
				if pictureOnly && File.exist?(picture) 
					puts("File is there")
					@image = IO::File.open(picture, "rb") {|f| f.read }
					pictureFile.close!
				else
					puts("No file to load at: "+picture)
					@image = ""
				end
				@ready = 200
			end
			if /URL_ERROR_CODE/.match(@dom)
        puts("LOAD_ERROR_CODE")
			  @ready = getHTTPCode @dom
        puts("LOAD_ERROR_CODE_DONE")
			end
			@dom = dom_text
			return @dom
		end
		
		def getDOMText data
		  tmp = data.split('PHANTOMJS_DOMDATA_WRITE:')[1];
		  tmp = tmp.split('PHANTOMJS_DOMDATA_END')[0]
		  tmp
		end
		
		def getHTTPCode data
		  tmp = data.split('URL_ERROR_CODE: ')[1];
      tmp = tmp.split('URL_ERROR_CODE_END')[0]
		  #tmp = /FAILED_LOADING_URL: (.*?)FAILED_LOADING_URL_END/.match(data)[1]
		  tmp.to_i
		end
		
		def getAsImageResponse(type='png')
			return "HTTP/1.0 200 OK\r\nConnection: close\r\nContent-Type: image/"+type+"\r\n\r\n"+@image;
		end
	
		def invokePhantomJS(script, args)
			argString = " "+args.join(" ")
			puts("Call phantomJS with: "+argString)
			out = ""

			IO.popen(PHANTOMJS_BIN+" --cookies-file=/tmp/phantom_proxy/cookies.txt --ignore-ssl-errors=yes "+script+argString) {|io|
			  out = io.readlines.join
			}
      #puts("PHANTOMJS_OUT: "+out)
			return out
		end
	end
end<commit_msg>dont use cookies file anymore<commit_after>require 'tempfile'

module PhantomJSProxy
	class PhantomJS
		attr_accessor :dom
		attr_accessor :image
    attr_accessor :ready
	
		def initialize()
			@ready = 503
		end
		
		def getUrl(url, pictureOnly=true, loadIFrames=true)
			puts("PhantomJS: "+url)
			@ready = 503
			
			pictureFile = nil
			picture = "none"
      
      loadFrames = "false"
      
      if loadIFrames
        loadFrames = "true"
      end
			
			if pictureOnly
				if !File.directory?("/tmp/phantom_proxy")
					Dir.mkdir("/tmp/phantom_proxy")
				end
				pictureFile = Tempfile.new(["phantom_proxy/page", ".png"])
				picture = pictureFile.path
			end
			
			url_args = ""
			url_args_ = []
      
			if /\?/.match(url)
				url_args = url.split('?')[1]
				url = url.split('?')[0]
				
				if url_args
					url_args_ = url_args.split('&')
					url_args = url_args_.join(' ')
				end
			end
			
			@dom = invokePhantomJS(SCRIPT, [picture, loadFrames, "\""+url+"\"", url_args_.length, url_args])
			
			puts("Opened page: "+ /Open page: (.*?) END/.match(@dom)[1])
			
			@ready = 503
			dom_text = "Failed to load page"
			
			if /DONE_LOADING_URL/.match(@dom)
        puts("LOAD_DOM_TEXT")
				dom_text = getDOMText @dom
        puts("LOAD_DOM_TEXT_DONE")
				if pictureOnly && File.exist?(picture) 
					puts("File is there")
					@image = IO::File.open(picture, "rb") {|f| f.read }
					pictureFile.close!
				else
					puts("No file to load at: "+picture)
					@image = ""
				end
				@ready = 200
			end
			if /URL_ERROR_CODE/.match(@dom)
        puts("LOAD_ERROR_CODE")
			  @ready = getHTTPCode @dom
        puts("LOAD_ERROR_CODE_DONE")
			end
			@dom = dom_text
			return @dom
		end
		
		def getDOMText data
		  tmp = data.split('PHANTOMJS_DOMDATA_WRITE:')[1];
		  tmp = tmp.split('PHANTOMJS_DOMDATA_END')[0]
		  tmp
		end
		
		def getHTTPCode data
		  tmp = data.split('URL_ERROR_CODE: ')[1];
      tmp = tmp.split('URL_ERROR_CODE_END')[0]
		  #tmp = /FAILED_LOADING_URL: (.*?)FAILED_LOADING_URL_END/.match(data)[1]
		  tmp.to_i
		end
		
		def getAsImageResponse(type='png')
			return "HTTP/1.0 200 OK\r\nConnection: close\r\nContent-Type: image/"+type+"\r\n\r\n"+@image;
		end
	
		def invokePhantomJS(script, args)
			argString = " "+args.join(" ")
			puts("Call phantomJS with: "+argString)
			out = ""

			IO.popen(PHANTOMJS_BIN+" --ignore-ssl-errors=yes "+script+argString) {|io|
			  out = io.readlines.join
			}
      #puts("PHANTOMJS_OUT: "+out)
			return out
		end
	end
end
{'fraction_non_alphanumeric': 0.14461172741679873, 'fraction_numerical': 0.009904912836767036, 'mean_word_length': 2.709772226304188, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1184845', 'n_tokens_mistral': 2312, 'n_tokens_neox': 1871, 'n_words': 366}