<commit_before>package com.davebsoft.sctw.twitter


import java.io.{InputStreamReader, BufferedInputStream, BufferedReader, OutputStreamWriter, InputStream}
import java.net.URL
import org.apache.commons.codec.binary.Base64
import org.apache.commons.httpclient.auth.AuthScope
import org.apache.commons.httpclient.cookie.CookiePolicy
import org.apache.commons.httpclient.methods.{PostMethod, GetMethod}
import org.apache.commons.httpclient.{UsernamePasswordCredentials, HttpMethodBase, HttpMethod, HttpClient}

object StreamUtil {
  def streamToString(stream: InputStream): String = {
    val buf = new StringBuilder
    val br = new BufferedReader(new InputStreamReader(stream))
    var line: String = null
    var eof = false
    while (! eof) {
      line = br.readLine()
      if (line == null) eof = true else 
      buf.append(line).append("\n")
    }
    buf.toString
  }  
}

/**
 * Generalises Http handling.
 * @author <NAME>  
 */
protected trait HttpHandler {

  val httpClient = new HttpClient()
  var username: String = _
  var password: String = _
  
  def doGet(urlString: String) = {
    val url = new URL(urlString)
    val conn = url.openConnection
    conn.addRequestProperty("Authorization", 
      "Basic " + new String(new Base64().encode((username + ":" + password).getBytes)))
    
    val headers = conn.getHeaderFields
    //TODO get actual response code
    //val status1 = headers.get("Status").get(0)
    //val result = Integer.parseInt(status1.split(" ")(0))
    (200, StreamUtil.streamToString(conn.getInputStream))
  }
  
  def doPost(url: String) = {
    val method = new PostMethod(url)
    handleCommonMethodSetup(method)<commit_msg>Check the status code from HTTP get<commit_after>package com.davebsoft.sctw.twitter


import java.io.{InputStreamReader, BufferedInputStream, BufferedReader, OutputStreamWriter, InputStream}
import java.net.URL
import org.apache.commons.codec.binary.Base64
import org.apache.commons.httpclient.auth.AuthScope
import org.apache.commons.httpclient.cookie.CookiePolicy
import org.apache.commons.httpclient.methods.{PostMethod, GetMethod}
import org.apache.commons.httpclient.{UsernamePasswordCredentials, HttpMethodBase, HttpMethod, HttpClient}

object StreamUtil {
  def streamToString(stream: InputStream): String = {
    val buf = new StringBuilder
    val br = new BufferedReader(new InputStreamReader(stream))
    var line: String = null
    var eof = false
    while (! eof) {
      line = br.readLine()
      if (line == null) eof = true else 
      buf.append(line).append("\n")
    }
    buf.toString
  }  
}

/**
 * Generalises Http handling.
 * @author <NAME>  
 */
protected trait HttpHandler {

  val httpClient = new HttpClient()
  var username: String = _
  var password: String = _
  
  def doGet(urlString: String) = {
    val url = new URL(urlString)
    val conn = url.openConnection
    conn.addRequestProperty("Authorization", 
      "Basic " + new String(new Base64().encode((username + ":" + password).getBytes)))
    
    val headers = conn.getHeaderFields
    //TODO test or get this working on Google App Engine. Things may be different.
    val status = headers.get("Status") // Looking for "200 OK"
    val statusCode = if (status == null || status.size == 0) 200 else 
      Integer.parseInt(status.get(0).split(" ")(0))
    (statusCode, StreamUtil.streamToString(conn.getInputStream))
  }
  
  def doPost(url: String) = {
    val method = new PostMethod(url)
    handleCommonMethodSetup(method)
{'fraction_non_alphanumeric': 0.11152522935779817, 'fraction_numerical': 0.006880733944954129, 'mean_word_length': 3.7729138166894667, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '978421', 'n_tokens_mistral': 1042, 'n_tokens_neox': 1025, 'n_words': 259}