<commit_before>dofile("urlcode.lua")
dofile("table_show.lua")

local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')

local downloaded = {}
local addedtolist = {}

for ignore in io.open("ignore-list", "r"):lines() do
  downloaded[ignore] = true
end

read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]
  local html = urlpos["link_expect_html"]
  
  if downloaded[url] == true or addedtolist[url] == true then
    return false
  end
  
  if downloaded[url] ~= true and addedtolist[url] ~= true then
    if (string.match(url, "^https?://[^/]*home%.online%.no") or string.match(url, "^https?://[^/]*home%.frisurf%.no") or string.match(url, "^https?://[^/]*148%.122%.161%.133")) and not string.match(url, "////") then
      addedtolist[url] = true
      return true
    else
      return false
    end
  end
end


wget.callbacks.get_urls = function(file, url, is_css, iri)
  local urls = {}
  local html = nil

  downloaded[url] = true
  
  local function check(urla)
    local url = string.match(urla, "^([^#]+)")
    if (downloaded[url] ~= true and addedtolist[url] ~= true) and (string.match(url, "^https?://[^/]*home%.online%.no") or string.match(url, "^https?://[^/]*home%.frisurf%.no") or string.match(url, "^https?://[^/]*148%.122%.161%.133")) and not (string.match(url, "////") or string.match(url, "/mailto:")) then
--  io.stdout:write(url .. ".  \n")
--  io.stdout:flush()
      if string.match(url, "&amp;") then
        table.insert(urls, { url=string.gsub(url, "&amp;", "&") })
        addedtolist[url] = true
        addedtolist[string.gsub(url, "&amp;", "&")] = true
      else
        table.insert(urls, { url=url })
        addedtolist[url] = true
      end
    end
  end

  local function checknewurl(newurl)
    if string.match(newurl, "^https?://") then
      check(newurl)
    elseif string.match(newurl, "^//") then
      check("http:"..newurl)
    elseif string.match(newurl, "^/") then
      check(string.match(url, "^(https?://[^/]+)")..newurl)
    elseif string.match(newurl, "%.jpg$") or string.match(newurl, "%.gif$") then
      check(string.match(url, "^(https?://.+/)")..newurl)
    end
  end
  
  if (string.match(url, "^https?://[^/]*home%.online%.no") or string.match(url, "^https?://[^/]*home%.frisurf%.no") or string.match(url, "^https?://[^/]*148%.122%.161%.133")) and not (string.match(url, "%.mp4$") or string.match(url, "%.mp3$") or string.match(url, "%.jpg$") or string.match(url, "%.gif$") or string.match(url, "%.avi$") or string.match(url, "%.flv$") or string.match(url, "%.pdf$") or string.match(url, "%.rm$") or string.match(url, "%.ra$") or string.match(url, "%.wmv$") or string.match(url, "%.jpeg$") or string.match(url, "%.swf$")) then
    html = read_file(file)
    check(string.match(url, "^(https?://.+/)"))
    check(string.match(url, "^(https?://.+)/"))
    check(string.match(url, "^(https?://[^%?]+)"))
    check(string.match(url, "^(https?://[^&]+)"))
    for newurl in string.gmatch(html, '([^"]+)') do
      checknewurl(newurl)
    end
    for newurl in string.gmatch(html, "([^']+)") do
      checknewurl(newurl)
    end
    for newurl in string.gmatch(html, ">([^<]+)") do
      checknewurl(newurl)
    end
    for newurl in string.gmatch(html, "location='([^']+)'") do
      if not (string.match(newurl, "^/") or string.match(newurl, "^https?://")) then
        check(string.match(url, "^(https?://.+/)")..newurl)
      end
    end
    for newurl in string.gmatch(html, 'href="([^"]+)"') do
      if not (string.match(newurl, "^/") or string.match(newurl, "^https?://")) then
        check(string.match(url, "^(https?://.+/)")..newurl)
      end
    end
    if string.match(url, "^https?://home%.online%.no/~[^/]+.*") then
--  io.stdout:write("YESssssssss.  \n")
--  io.stdout:flush()
--  io.stdout:write("http://" .. string.match(url, "^https?://home%.online%.no/~([^/]+).*") .. ".home.online.no" .. string.match(url, "^https?://home%.online%.no/~[^/]+(.*)") .. "YESssssssss.  \n")
--  io.stdout:flush()
      check("http://" .. string.match(url, "^https?://home%.online%.no/~([^/]+).*") .. ".home.online.no" .. string.match(url, "^https?://home%.online%.no/~[^/]+(.*)"))
    end
    if string.match(url, "^https?://home%.frisurf%.no/~[^/]+.*") then
      check("http://" .. string.match(url, "^https?://home%.frisurf%.no/~([^/]+).*") .. ".home.frisurf.no" .. string.match(url, "^https?://home%.frisurf%.no/~[^/]+(.*)"))
    end
    if string.match(url, "^https?://[^%.]+%.home%.frisurf%.no/.*") then
      check("http://home.frisurf.no/~" .. string.match(url, "^https?://([^%.]+)%.home%.frisurf%.no/.*") .. "/" .. string.match(url, "^https?://[^%.]+%.home%.frisurf%.no/(.*)"))
    end
    if string.match(url, "^https?://[^%.]+%.home%.online%.no/.*") then
--  io.stdout:write("YsdasfdasdafsdasES.  \n")
--  io.stdout:flush()
--  io.stdout:write("http://home.online.no/~" .. string.match(url, "^https?://([^%.]+)%.home%.online%.no/.*") .. "/" .. string.match(url, "^https?://[^%.]+%.home%.online%.no/(.*)") .. "YsdasfdasdafsdasES.  \n")
--  io.stdout:flush()
      check("http://home.online.no/~" .. string.match(url, "^https?://([^%.]+)%.home%.online%.no/.*") .. "/" .. string.match(url, "^https?://[^%.]+%.home%.online%.no/(.*)"))
    end
    if string.match(url, "^https?://.+/[^/]+/") then
      check(string.match(url, "^(https?://.+/)[^/]+/"))
      check(string.match(url, "^(https?://.+)/[^/]+/"))
    end
  end
  
  return urls
end
  

wget.callbacks.httploop_result = function(url, err, http_stat)
  -- NEW for 2014: Slightly more verbose messages because people keep
  -- complaining that it's not moving or not working
  status_code = http_stat["statcode"]
  
  url_count = url_count + 1
  io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. ".  \n")
  io.stdout:flush()

  if string.match(url["url"], "^https?://[^%.]+%.home%.frisurf%.no/.*") then
    if downloaded["http://home.frisurf.no/~" .. string.match(url["url"], "^https?://([^%.]+)%.home%.frisurf%.no/.*") .. "/" .. string.match(url["url"], "^https?://[^%.]+%.home%.frisurf%.no/(.*)")] == true then
      return wget.actions.EXIT
    end
  end
  if string.match(url["url"], "^https?://[^%.]+%.home%.online%.no/.*") then
--  io.stdout:write("YES.  \n")
--  io.stdout:flush()
--  io.stdout:write("http://home.online.no/~" .. string.match(url["url"], "^https?://([^%.]+)%.home%.online%.no/.*") .. "/" .. string.match(url["url"], "^https?://[^%.]+%.home%.online%.no/(.*)") .. ".  \n")
--  io.stdout:flush()
    if downloaded["http://home.online.no/~" .. string.match(url["url"], "^https?://([^%.]+)%.home%.online%.no/.*") .. "/" .. string.match(url["url"], "^https?://[^%.]+%.home%.online%.no/(.*)")] == true then
      return wget.actions.EXIT
    end
  end

  if downloaded[url["url"]] == true then
    return wget.actions.EXIT
  end

  if (status_code >= 200 and status_code <= 399) then
    if string.match(url.url, "https://") then
      local newurl = string.gsub(url.url, "https://", "http://")
      downloaded[newurl] = true
    else
      downloaded[url.url] = true
    end
  end
  
  if status_code >= 500 or
    (status_code >= 400 and status_code ~= 404 and status_code ~= 403 and status_code ~= 400 and status_code ~= 414) then
    io.stdout:write("\nServer returned "..http_stat.statcode.." ("..err.."). Sleeping.\n")
    io.stdout:flush()
    if err == "AUTHFAILED" then
      return wget.actions.EXIT
    end
    os.execute("sleep 1")
    tries = tries + 1
    if tries >= 5 then<commit_msg>Remove debug messages.<commit_after>dofile("urlcode.lua")
dofile("table_show.lua")

local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')

local downloaded = {}
local addedtolist = {}

for ignore in io.open("ignore-list", "r"):lines() do
  downloaded[ignore] = true
end

read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]
  local html = urlpos["link_expect_html"]
  
  if downloaded[url] == true or addedtolist[url] == true then
    return false
  end
  
  if downloaded[url] ~= true and addedtolist[url] ~= true then
    if (string.match(url, "^https?://[^/]*home%.online%.no") or string.match(url, "^https?://[^/]*home%.frisurf%.no") or string.match(url, "^https?://[^/]*148%.122%.161%.133")) and not string.match(url, "////") then
      addedtolist[url] = true
      return true
    else
      return false
    end
  end
end


wget.callbacks.get_urls = function(file, url, is_css, iri)
  local urls = {}
  local html = nil

  downloaded[url] = true
  
  local function check(urla)
    local url = string.match(urla, "^([^#]+)")
    if (downloaded[url] ~= true and addedtolist[url] ~= true) and (string.match(url, "^https?://[^/]*home%.online%.no") or string.match(url, "^https?://[^/]*home%.frisurf%.no") or string.match(url, "^https?://[^/]*148%.122%.161%.133")) and not (string.match(url, "////") or string.match(url, "/mailto:")) then
      if string.match(url, "&amp;") then
        table.insert(urls, { url=string.gsub(url, "&amp;", "&") })
        addedtolist[url] = true
        addedtolist[string.gsub(url, "&amp;", "&")] = true
      else
        table.insert(urls, { url=url })
        addedtolist[url] = true
      end
    end
  end

  local function checknewurl(newurl)
    if string.match(newurl, "^https?://") then
      check(newurl)
    elseif string.match(newurl, "^//") then
      check("http:"..newurl)
    elseif string.match(newurl, "^/") then
      check(string.match(url, "^(https?://[^/]+)")..newurl)
    elseif string.match(newurl, "%.jpg$") or string.match(newurl, "%.gif$") then
      check(string.match(url, "^(https?://.+/)")..newurl)
    end
  end
  
  if (string.match(url, "^https?://[^/]*home%.online%.no") or string.match(url, "^https?://[^/]*home%.frisurf%.no") or string.match(url, "^https?://[^/]*148%.122%.161%.133")) and not (string.match(url, "%.mp4$") or string.match(url, "%.mp3$") or string.match(url, "%.jpg$") or string.match(url, "%.gif$") or string.match(url, "%.avi$") or string.match(url, "%.flv$") or string.match(url, "%.pdf$") or string.match(url, "%.rm$") or string.match(url, "%.ra$") or string.match(url, "%.wmv$") or string.match(url, "%.jpeg$") or string.match(url, "%.swf$")) then
    html = read_file(file)
    check(string.match(url, "^(https?://.+/)"))
    check(string.match(url, "^(https?://.+)/"))
    check(string.match(url, "^(https?://[^%?]+)"))
    check(string.match(url, "^(https?://[^&]+)"))
    for newurl in string.gmatch(html, '([^"]+)') do
      checknewurl(newurl)
    end
    for newurl in string.gmatch(html, "([^']+)") do
      checknewurl(newurl)
    end
    for newurl in string.gmatch(html, ">([^<]+)") do
      checknewurl(newurl)
    end
    for newurl in string.gmatch(html, "location='([^']+)'") do
      if not (string.match(newurl, "^/") or string.match(newurl, "^https?://")) then
        check(string.match(url, "^(https?://.+/)")..newurl)
      end
    end
    for newurl in string.gmatch(html, 'href="([^"]+)"') do
      if not (string.match(newurl, "^/") or string.match(newurl, "^https?://")) then
        check(string.match(url, "^(https?://.+/)")..newurl)
      end
    end
    if string.match(url, "^https?://home%.online%.no/~[^/]+.*") then
      check("http://" .. string.match(url, "^https?://home%.online%.no/~([^/]+).*") .. ".home.online.no" .. string.match(url, "^https?://home%.online%.no/~[^/]+(.*)"))
    end
    if string.match(url, "^https?://home%.frisurf%.no/~[^/]+.*") then
      check("http://" .. string.match(url, "^https?://home%.frisurf%.no/~([^/]+).*") .. ".home.frisurf.no" .. string.match(url, "^https?://home%.frisurf%.no/~[^/]+(.*)"))
    end
    if string.match(url, "^https?://[^%.]+%.home%.frisurf%.no/.*") then
      check("http://home.frisurf.no/~" .. string.match(url, "^https?://([^%.]+)%.home%.frisurf%.no/.*") .. "/" .. string.match(url, "^https?://[^%.]+%.home%.frisurf%.no/(.*)"))
    end
    if string.match(url, "^https?://[^%.]+%.home%.online%.no/.*") then
      check("http://home.online.no/~" .. string.match(url, "^https?://([^%.]+)%.home%.online%.no/.*") .. "/" .. string.match(url, "^https?://[^%.]+%.home%.online%.no/(.*)"))
    end
    if string.match(url, "^https?://.+/[^/]+/") then
      check(string.match(url, "^(https?://.+/)[^/]+/"))
      check(string.match(url, "^(https?://.+)/[^/]+/"))
    end
  end
  
  return urls
end
  

wget.callbacks.httploop_result = function(url, err, http_stat)
  -- NEW for 2014: Slightly more verbose messages because people keep
  -- complaining that it's not moving or not working
  status_code = http_stat["statcode"]
  
  url_count = url_count + 1
  io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. ".  \n")
  io.stdout:flush()

  if string.match(url["url"], "^https?://[^%.]+%.home%.frisurf%.no/.*") then
    if downloaded["http://home.frisurf.no/~" .. string.match(url["url"], "^https?://([^%.]+)%.home%.frisurf%.no/.*") .. "/" .. string.match(url["url"], "^https?://[^%.]+%.home%.frisurf%.no/(.*)")] == true then
      return wget.actions.EXIT
    end
  end
  if string.match(url["url"], "^https?://[^%.]+%.home%.online%.no/.*") then
    if downloaded["http://home.online.no/~" .. string.match(url["url"], "^https?://([^%.]+)%.home%.online%.no/.*") .. "/" .. string.match(url["url"], "^https?://[^%.]+%.home%.online%.no/(.*)")] == true then
      return wget.actions.EXIT
    end
  end

  if downloaded[url["url"]] == true then
    return wget.actions.EXIT
  end

  if (status_code >= 200 and status_code <= 399) then
    if string.match(url.url, "https://") then
      local newurl = string.gsub(url.url, "https://", "http://")
      downloaded[newurl] = true
    else
      downloaded[url.url] = true
    end
  end
  
  if status_code >= 500 or
    (status_code >= 400 and status_code ~= 404 and status_code ~= 403 and status_code ~= 400 and status_code ~= 414) then
    io.stdout:write("\nServer returned "..http_stat.statcode.." ("..err.."). Sleeping.\n")
    io.stdout:flush()
    if err == "AUTHFAILED" then
      return wget.actions.EXIT
    end
    os.execute("sleep 1")
    tries = tries + 1
    if tries >= 5 then
{'fraction_non_alphanumeric': 0.26061723337430015, 'fraction_numerical': 0.009832036050798854, 'mean_word_length': 3.1434229137199434, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1242298', 'n_tokens_mistral': 5721, 'n_tokens_neox': 5412, 'n_words': 1015}