<commit_before>local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')


read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]

  -- Skip redirect from home.swipnet.se
  if url == "http://www.tele2.se/" or
    url == "http://tele2.se/" then
    return false
  elseif string.match(url, "cgi%.swipnet%.se/") then
    return false
  elseif string.match(url, "www%.swipnet%.se/") then
    return false
  elseif string.match(url, "irc%.swipnet%.se/") then
    return false
  elseif string.match(url, "//////////") then
    return false
  elseif string.match(url, "swipnet%.se/([^/]+)/") then
    local directory_name = string.match(url, "swipnet%.se/([^/]+)/")
    directory_name = string.gsub(directory_name, '%%7E', '~')
    
    if directory_name ~= item_value then
      -- do not want someone else's homepage
       -- io.stdout:write("\n Reject " .. url .. " " .. directory_name .. "\n")
       -- io.stdout:flush()
      return false
    else
      return verdict
    end
  else
    return verdict
  end
end

wget.callbacks.httploop_result = function(url, err, http_stat)
  -- NEW for 2014: Slightly more verbose messages because people keep
  -- complaining that it's not moving or not working
  local status_code = http_stat["statcode"]

  url_count = url_count + 1
  io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. ".  \r")
  io.stdout:flush()

  if status_code >= 500 or
    (status_code >= 400 and status_code ~= 404 and status_code ~= 403) then
    io.stdout:write("\nServer returned "..http_stat.statcode..". Sleeping.\n")
    io.stdout:flush()

    os.execute("sleep 10")

    tries = tries + 1

    if tries >= 5 then
      io.stdout:write("\nI give up...\n")
      io.stdout:flush()
      return wget.actions.NOTHING
    else
      return wget.actions.CONTINUE
    end
  else
    return wget.actions.NOTHING
  end

  tries = 0

  -- We're okay; sleep a bit (if we have to) and continue
  -- local sleep_time = 0.1 * (math.random(75, 1000) / 100.0)
  local sleep_time = 0

  --  if string.match(url["host"], "cdn") or string.match(url["host"], "media") then
  --    -- We should be able to go fast on images since that's what a web browser does
  --    sleep_time = 0
  --  end

  if sleep_time > 0.001 then<commit_msg>tele2.lua: updated for the tele2 project<commit_after>local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')


read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]

  -- Skip redirect from home.swipnet.se
  if string.match(url, "www%.tele2%.se/") then
    return false
  elseif string.match(url, "cgi%.tele2%.se/") then
    return false
  elseif string.match(url, "www%.tele2%.se/") then
    return false
  elseif string.match(url, "irc%.tele2%.se/") then
    return false
  elseif string.match(url, "//////////") then
    return false
  elseif string.match(url, "tele2%.se/([^/]+)/") then
    local directory_name = string.match(url, "tele2%.se/([^/]+)/")
    directory_name = string.gsub(directory_name, '%%7E', '~')
    
    if directory_name ~= item_value then
      -- do not want someone else's homepage
       -- io.stdout:write("\n Reject " .. url .. " " .. directory_name .. "\n")
       -- io.stdout:flush()
      return false
    else
      return verdict
    end
  else
    return verdict
  end
end

wget.callbacks.httploop_result = function(url, err, http_stat)
  -- NEW for 2014: Slightly more verbose messages because people keep
  -- complaining that it's not moving or not working
  local status_code = http_stat["statcode"]

  url_count = url_count + 1
  io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. ".  \r")
  io.stdout:flush()

  if status_code >= 500 or
    (status_code >= 400 and status_code ~= 404 and status_code ~= 403) then
    io.stdout:write("\nServer returned "..http_stat.statcode..". Sleeping.\n")
    io.stdout:flush()

    os.execute("sleep 10")

    tries = tries + 1

    if tries >= 5 then
      io.stdout:write("\nI give up...\n")
      io.stdout:flush()
      return wget.actions.ABORT
    else
      return wget.actions.CONTINUE
    end
  elseif status_code == 0 then
    io.stdout:write("\nServer returned "..http_stat.statcode..". Sleeping.\n")
    io.stdout:flush()
    return wget.actions.ABORT
  else
    return wget.actions.NOTHING
  end

  tries = 0

  -- We're okay; sleep a bit (if we have to) and continue
  -- local sleep_time = 0.1 * (math.random(75, 1000) / 100.0)
  local sleep_time = 0

  --  if string.match(url["host"], "cdn") or string.match(url["host"], "media") then
  --    -- We should be able to go fast on images since that's what a web browser does
  --    sleep_time = 0
  --  end

  if sleep_time > 0.001 then
{'fraction_non_alphanumeric': 0.1460885956644675, 'fraction_numerical': 0.01828463713477851, 'mean_word_length': 2.7577903682719547, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4344738', 'n_tokens_mistral': 1883, 'n_tokens_neox': 1778, 'n_words': 494}