<commit_before>// Source code repository: https://github.com/frabcus/code-scraper-in-browser-tool/

var editor
var editorDirty = false
var editorShare
var editorSpinner
var output
var status = 'nothing' // currently belief about script execution status: 'running' or 'nothing'
var changing ='' // for starting/stopping states
var stateShare // various things shared with share.js, including group consideration of the running status
var saveTimeout // store handle to timeout so can clear it
var connected = false // whether sharejs is connected - measured by sharejs
var online = true // whether browser is online - measured by errors from calling exec endpoint

// This is an arbitary number we tack onto the end of the document id in ShareJS.
// Incrementing it forces the code in the browser tool to use a new ShareJS
// document (and recover the data from the code/scraper file to initialise it)
var shareJSCode = '039'


// Called when we either load from the box filesystem, or get data from 
//ShareJS, upon first loading of the page
var done_initial_load = function() {
  // set syntax highlighting a tick later, when we have initial data
  setTimeout(function() {
    //console.log('editor version', editorShare.version, editor.getValue())
    set_editor_mode(editor.getValue())
    editor.setReadOnly(false)
    editor.moveCursorTo(0, 0)
    editor.focus()
    editorSpinner.stop()
  }, 1)
}

// Wire up shared document on the connection
var made_editor_connection = function(error, doc) {
  if (error) {
    console.log("sharing doc error", error)
    scraperwiki.alert("Trouble setting up pair editing!", error, true)
    return
  }
  console.log("shared doc")

  editorShare = doc
  editorShare.attach_ace(editor)

  if (editorShare.created) {
    console.log("first time this editor document exists on ShareJS server, loading from filesystem")
    load_code_from_file()
  } else {
    done_initial_load()
  }

  editorShare.on('error', function(error) {
    console.log("editorShare later error:", error)
    // if ShareJS server has restarted, we get document missing as an error
    // just redo everything in this case
    if (error == "Document does not exist")
      location.reload()
    else
      scraperwiki.alert("Editor sharing error!", error, true)
  })
}

// Wire up shared state on the connection
var made_state_connection = function(error, doc) {
  if (error) {
    console.log("sharing state error", error)
    scraperwiki.alert("Trouble setting up pair state!", error, true)
    return
  }
  console.log("shared state")

  stateShare = doc
  if (stateShare.created) {
    console.log("first time this state document exists on ShareJS server, initialising with 'nothing'")
    stateShare.submitOp([{p:[],od:null,oi:{status:'nothing'}}])
  }
  stateShare.on('change', function (op) {
    shared_state_update(op)
  })
  stateShare.on('error', function (error) {
    console.log("stateShare later error:", error)
    // if ShareJS server has restarted, we get document missing as an error
    // just redo everything in this case
    if (error == "Document does not exist")
      location.reload()
    else
      scraperwiki.alert("State sharing error!", error, true)
  })
}

// Used to initialise what is in the ShareJS server from the filesystem the
// first time run for the instantiation of the ShareJS server.
var load_code_from_file = function() {
  console.log("loading...")
  scraperwiki.exec('mkdir -p code && touch code/scraper && cat code/scraper && echo -n swinternalGOTCODEOFSCRAPER', function(data) {
    // scraperwiki.sql.meta(function(meta) {
      if (data.indexOf("swinternalGOTCODEOFSCRAPER") == -1) {
        scraperwiki.alert("Trouble loading code!", data, true)
        return
      }
      data = data.replace("swinternalGOTCODEOFSCRAPER", "")
      online = true

      // If nothing there, set some default content to get people going
      if (data.match(/^\s*$/)) {
        data = "#!/usr/bin/python\n" + 
               "\n" + 
               "import scraperwiki\n" + 
               "\n" + 
               "# scraperwiki.sql.save([unique keys], { data })"
        settings = scraperwiki.readSettings()
        //console.log(settings)
        // If we've been added as a view - XXX experimental, to try code in
        // browser for writing code that generates static views
        /* if (settings.target) {
          var sql_url = "" + settings.target.url + "/sql/"
          console.log("sql_url", sql_url)
          tables = _.keys(meta.table)
          table = "unknown"
          if (tables.length > 0)
            table = tables[0]
          data =  "#!/usr/bin/python\n"+
                  "\n"+
                  "import scraperwiki\n" +
                  "import requests\n" +
                  "import json\n" +
                  "\n" +
                  "# Query the database this view is attached to \n" +
                  "sql_url = '" + sql_url + "'\n" +
                  "query = 'select * from " + table + " limit 10'\n" +
                  "response = requests.get(sql_url, params = { 'q': query })\n" +
                  "response.raise_for_status()\n" + 
                  "\n" +
                  "# Loop through the response\n" +
                  "rows = json.loads(response.text)\n" +
                  "for row in rows:\n" +
                  "    print row\n"
        } */
      }
      console.log("...loaded")

      clear_alerts()
      set_editor_mode(data)
      editor.setValue(data) // XXX this overrides what is in filesystem on top of what is in sharejs
      update_dirty(false)
      done_initial_load()
    // }, handle_exec_error)
  }, handle_exec_error)
}

// Handle error
var handle_exec_error = function(jqXHR, textStatus, errorThrown) {
    console.log("got an error:", errorThrown, jqXHR, textStatus)
    // Handle special case of the browser not being connected to the net
    // http://stackoverflow.com/questions/10026204/dialog-box-if-there-is-no-internet-connection-using-jquery-or-ajax
    if (jqXHR.status == 0) { 
      // Wait half a second before error - otherwise they show on page refresh,
      // we only want to show if the browser is disconneted from the network.
      setTimeout(function() {
        clear_alerts()
        online = false
        refresh_saving_message()
      } , 500)
    } else {
      scraperwiki.alert(errorThrown, $(jqXHR.responseText).text(), "error")
    }
}

// Display whether we have unsaved edits,
var update_dirty = function(value) {
  editorDirty = value
  refresh_saving_message()
}

// Refresh the saving and so on text
var refresh_saving_message = function() {
  clearTimeout(saveTimeout)

  if (!online) {
    $("#saved").text("")
    $("#offline").text("Offline, not connected to the Internet")
    return
  }
  if (!connected) {
    $("#saved").text("")
    $("#offline").text("Offline, can't connect to the sharing server")
    return
  }

  $("#offline").text("")
  if (editorDirty) {
    // Wait three seconds and then save. If we get another change
    // in those three seconds, reset that timer to avoid excess saves.
    saveTimeout = setTimeout(save_code, 3000)
    $("#saved").text("Saving...")
  } else {
    $("#saved").text("All changes saved")
  } 
}


// Prevent navigation away if not saved
// XXX this doesn't seem to work for close events in chrome in the frame
$(window).on('beforeunload', function() {
  if (editorDirty) {
    return "You've made changes that haven't been saved yet."
  }
})

// Work out language we're in from the shebang line<commit_msg>Let's try not showing any saving menu<commit_after>// Source code repository: https://github.com/frabcus/code-scraper-in-browser-tool/

var editor
var editorDirty = false
var editorShare
var editorSpinner
var output
var status = 'nothing' // currently belief about script execution status: 'running' or 'nothing'
var changing ='' // for starting/stopping states
var stateShare // various things shared with share.js, including group consideration of the running status
var saveTimeout // store handle to timeout so can clear it
var connected = false // whether sharejs is connected - measured by sharejs
var online = true // whether browser is online - measured by errors from calling exec endpoint

// This is an arbitary number we tack onto the end of the document id in ShareJS.
// Incrementing it forces the code in the browser tool to use a new ShareJS
// document (and recover the data from the code/scraper file to initialise it)
var shareJSCode = '039'


// Called when we either load from the box filesystem, or get data from 
//ShareJS, upon first loading of the page
var done_initial_load = function() {
  // set syntax highlighting a tick later, when we have initial data
  setTimeout(function() {
    //console.log('editor version', editorShare.version, editor.getValue())
    set_editor_mode(editor.getValue())
    editor.setReadOnly(false)
    editor.moveCursorTo(0, 0)
    editor.focus()
    editorSpinner.stop()
  }, 1)
}

// Wire up shared document on the connection
var made_editor_connection = function(error, doc) {
  if (error) {
    console.log("sharing doc error", error)
    scraperwiki.alert("Trouble setting up pair editing!", error, true)
    return
  }
  console.log("shared doc")

  editorShare = doc
  editorShare.attach_ace(editor)

  if (editorShare.created) {
    console.log("first time this editor document exists on ShareJS server, loading from filesystem")
    load_code_from_file()
  } else {
    done_initial_load()
  }

  editorShare.on('error', function(error) {
    console.log("editorShare later error:", error)
    // if ShareJS server has restarted, we get document missing as an error
    // just redo everything in this case
    if (error == "Document does not exist")
      location.reload()
    else
      scraperwiki.alert("Editor sharing error!", error, true)
  })
}

// Wire up shared state on the connection
var made_state_connection = function(error, doc) {
  if (error) {
    console.log("sharing state error", error)
    scraperwiki.alert("Trouble setting up pair state!", error, true)
    return
  }
  console.log("shared state")

  stateShare = doc
  if (stateShare.created) {
    console.log("first time this state document exists on ShareJS server, initialising with 'nothing'")
    stateShare.submitOp([{p:[],od:null,oi:{status:'nothing'}}])
  }
  stateShare.on('change', function (op) {
    shared_state_update(op)
  })
  stateShare.on('error', function (error) {
    console.log("stateShare later error:", error)
    // if ShareJS server has restarted, we get document missing as an error
    // just redo everything in this case
    if (error == "Document does not exist")
      location.reload()
    else
      scraperwiki.alert("State sharing error!", error, true)
  })
}

// Used to initialise what is in the ShareJS server from the filesystem the
// first time run for the instantiation of the ShareJS server.
var load_code_from_file = function() {
  console.log("loading...")
  scraperwiki.exec('mkdir -p code && touch code/scraper && cat code/scraper && echo -n swinternalGOTCODEOFSCRAPER', function(data) {
    // scraperwiki.sql.meta(function(meta) {
      if (data.indexOf("swinternalGOTCODEOFSCRAPER") == -1) {
        scraperwiki.alert("Trouble loading code!", data, true)
        return
      }
      data = data.replace("swinternalGOTCODEOFSCRAPER", "")
      online = true

      // If nothing there, set some default content to get people going
      if (data.match(/^\s*$/)) {
        data = "#!/usr/bin/python\n" + 
               "\n" + 
               "import scraperwiki\n" + 
               "\n" + 
               "# scraperwiki.sql.save([unique keys], { data })"
        settings = scraperwiki.readSettings()
        //console.log(settings)
        // If we've been added as a view - XXX experimental, to try code in
        // browser for writing code that generates static views
        /* if (settings.target) {
          var sql_url = "" + settings.target.url + "/sql/"
          console.log("sql_url", sql_url)
          tables = _.keys(meta.table)
          table = "unknown"
          if (tables.length > 0)
            table = tables[0]
          data =  "#!/usr/bin/python\n"+
                  "\n"+
                  "import scraperwiki\n" +
                  "import requests\n" +
                  "import json\n" +
                  "\n" +
                  "# Query the database this view is attached to \n" +
                  "sql_url = '" + sql_url + "'\n" +
                  "query = 'select * from " + table + " limit 10'\n" +
                  "response = requests.get(sql_url, params = { 'q': query })\n" +
                  "response.raise_for_status()\n" + 
                  "\n" +
                  "# Loop through the response\n" +
                  "rows = json.loads(response.text)\n" +
                  "for row in rows:\n" +
                  "    print row\n"
        } */
      }
      console.log("...loaded")

      clear_alerts()
      set_editor_mode(data)
      editor.setValue(data) // XXX this overrides what is in filesystem on top of what is in sharejs
      update_dirty(false)
      done_initial_load()
    // }, handle_exec_error)
  }, handle_exec_error)
}

// Handle error
var handle_exec_error = function(jqXHR, textStatus, errorThrown) {
    console.log("got an error:", errorThrown, jqXHR, textStatus)
    // Handle special case of the browser not being connected to the net
    // http://stackoverflow.com/questions/10026204/dialog-box-if-there-is-no-internet-connection-using-jquery-or-ajax
    if (jqXHR.status == 0) { 
      // Wait half a second before error - otherwise they show on page refresh,
      // we only want to show if the browser is disconneted from the network.
      setTimeout(function() {
        clear_alerts()
        online = false
        refresh_saving_message()
      } , 500)
    } else {
      scraperwiki.alert(errorThrown, $(jqXHR.responseText).text(), "error")
    }
}

// Display whether we have unsaved edits,
var update_dirty = function(value) {
  editorDirty = value
  refresh_saving_message()
}

// Refresh the saving and so on text
var refresh_saving_message = function() {
  clearTimeout(saveTimeout)

  if (!online) {
    $("#saved").text("")
    $("#offline").text("Offline, not connected to the Internet")
    return
  }
  if (!connected) {
    $("#saved").text("")
    $("#offline").text("Offline, can't connect to the sharing server")
    return
  }

  $("#offline").text("")
  if (editorDirty) {
    // Wait three seconds and then save. If we get another change
    // in those three seconds, reset that timer to avoid excess saves.
    saveTimeout = setTimeout(save_code, 3000)
    //$("#saved").text("Saving...")
  } else {
    //$("#saved").text("All changes saved")
  } 
}


// Prevent navigation away if not saved
// XXX this doesn't seem to work for close events in chrome in the frame
$(window).on('beforeunload', function() {
  if (editorDirty) {
    return "You've made changes that haven't been saved yet."
  }
})

// Work out language we're in from the shebang line
{'fraction_non_alphanumeric': 0.11655181553140737, 'fraction_numerical': 0.0035780545984627618, 'mean_word_length': 2.6571359340925613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2426564', 'n_tokens_mistral': 4453, 'n_tokens_neox': 4327, 'n_words': 1425}