<commit_before>dofile("urlcode.lua")
dofile("table_show.lua")
JSON = (loadfile "JSON.lua")()

local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')

local downloaded = {}
local addedtolist = {}

load_json_file = function(file)
  if file then
    local f = io.open(file)
    local data = f:read("*all")
    f:close()
    return JSON:decode(data)
  else
    return nil
  end
end

read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]
  local html = urlpos["link_expect_html"]
  local parenturl = parent["url"]
  local html = nil
  
  if downloaded[url] == true or addedtolist[url] == true then
    return false
  end
  
  if item_type == "halo3file" then
    if string.match(url, "[^0-9]"..item_value.."[0-9][0-9]")
      or string.match(url, "/javascript/")
      or string.match(url, "/base_css/")
      or string.match(url, "/images/")
      or string.match(url, "/Silverlight/")
      or string.match(url, "/Stats/")
      or string.match(url, "/App_Themes/")
      or string.match(url, "/WebResource%.axd")
      or string.match(url, "/bprovideo/")
      or string.match(url, "/ScriptResource%.axd")
      or string.match(url, "/Screenshot%.ashx")
      or string.match(url, "silverlight%.dlservice%.microsoft%.com")
      or string.match(url, "/GameStatsHalo3%.aspx") then
      if not string.match(url, "[0-9]?"..item_value.."[0-9][0-9][0-9]")
        and not string.match(url, "PlayerStatsHalo3%.aspx%?player=")
        and not string.match(url, "/Stats/Halo3/Default%.aspx%?player=") then
        return true
      else
        return false
      end
    else
      return false
    end
  end
  
end


wget.callbacks.get_urls = function(file, url, is_css, iri)
  local urls = {}
  local html = nil
        
  if string.match(url, "/Stats/GameStatsHalo3%.aspx%?gameguid=[0-9]+") then
    local customurl = string.gsub(url, "/Stats/GameStatsHalo3%.aspx%?gameguid=", "/Stats/GameFiles%.aspx%?guid=")
    if downloaded[customurl] ~= true and addedtolist[customurl] ~= true then
      table.insert(urls, { url=customurl })
    end
  end
  
  if item_type == "halo3file" then
    if string.match(url, "[^0-9]"..item_value.."[0-9][0-9]") then
      if not string.match(url, "[0-9]?"..item_value.."[0-9][0-9][0-9]") then
        html = read_file(html)
        
        for customurl in string.gmatch(html, '"(http[s]?://[^"]+)"') do
          if string.match(customurl, "[^0-9]"..item_value.."[0-9][0-9]")
            or string.match(customurl, "/javascript/")
            or string.match(customurl, "/base_css/")
            or string.match(customurl, "/images/")
            or string.match(customurl, "/Silverlight/")
            or string.match(customurl, "/Stats/")
            or string.match(customurl, "/App_Themes/")
            or string.match(customurl, "/WebResource%.axd")
            or string.match(customurl, "/bprovideo/")
            or string.match(customurl, "/ScriptResource%.axd")
            or string.match(customurl, "/Screenshot%.ashx")
            or string.match(customurl, "silverlight%.dlservice%.microsoft%.com")
            or string.match(customurl, "/GameStatsHalo3%.aspx") then
            if not string.match(customurl, "[0-9]?"..item_value.."[0-9][0-9][0-9]")
              and not string.match(customurl, "PlayerStatsHalo3%.aspx%?player=")
              and not string.match(customurl, "/Stats/Halo3/Default%.aspx%?player=") then
              if downloaded[customurl] ~= true and addedtolist[customurl] ~= true then
                table.insert(urls, { url=customurl })
              end
            end
          end
        end
        for customurlnf in string.gmatch(html, '"(/[^"]+)"') do
          if string.match(customurlnf, "[^0-9]"..item_value.."[0-9][0-9]")
            or string.match(customurlnf, "/javascript/")
            or string.match(customurlnf, "/base_css/")
            or string.match(customurlnf, "/images/")
            or string.match(customurlnf, "/Silverlight/")<commit_msg>halo.lua: add support for url between ( and )<commit_after>dofile("urlcode.lua")
dofile("table_show.lua")
JSON = (loadfile "JSON.lua")()

local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')

local downloaded = {}
local addedtolist = {}

load_json_file = function(file)
  if file then
    local f = io.open(file)
    local data = f:read("*all")
    f:close()
    return JSON:decode(data)
  else
    return nil
  end
end

read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]
  local html = urlpos["link_expect_html"]
  local parenturl = parent["url"]
  local html = nil
  
  if downloaded[url] == true or addedtolist[url] == true then
    return false
  end
  
  if item_type == "halo3file" then
    if string.match(url, "[^0-9]"..item_value.."[0-9][0-9]")
      or string.match(url, "/javascript/")
      or string.match(url, "/base_css/")
      or string.match(url, "/images/")
      or string.match(url, "/Silverlight/")
      or string.match(url, "/Stats/")
      or string.match(url, "/App_Themes/")
      or string.match(url, "/WebResource%.axd")
      or string.match(url, "/bprovideo/")
      or string.match(url, "/ScriptResource%.axd")
      or string.match(url, "/Screenshot%.ashx")
      or string.match(url, "silverlight%.dlservice%.microsoft%.com")
      or string.match(url, "/GameStatsHalo3%.aspx") then
      if not string.match(url, "[0-9]?"..item_value.."[0-9][0-9][0-9]")
        and not string.match(url, "PlayerStatsHalo3%.aspx%?player=")
        and not string.match(url, "/Stats/Halo3/Default%.aspx%?player=") then
        return true
      else
        return false
      end
    else
      return false
    end
  end
  
end


wget.callbacks.get_urls = function(file, url, is_css, iri)
  local urls = {}
  local html = nil
        
  if string.match(url, "/Stats/GameStatsHalo3%.aspx%?gameguid=[0-9]+") then
    local customurl = string.gsub(url, "/Stats/GameStatsHalo3%.aspx%?gameguid=", "/Stats/GameFiles%.aspx%?guid=")
    if downloaded[customurl] ~= true and addedtolist[customurl] ~= true then
      table.insert(urls, { url=customurl })
    end
  end
  
  if item_type == "halo3file" then
    if string.match(url, "[^0-9]"..item_value.."[0-9][0-9]") then
      if not string.match(url, "[0-9]?"..item_value.."[0-9][0-9][0-9]") then
        html = read_file(html)
        
        for customurl in string.gmatch(html, '"(http[s]?://[^"]+)"') do
          if string.match(customurl, "[^0-9]"..item_value.."[0-9][0-9]")
            or string.match(customurl, "/javascript/")
            or string.match(customurl, "/base_css/")
            or string.match(customurl, "/images/")
            or string.match(customurl, "/Silverlight/")
            or string.match(customurl, "/Stats/")
            or string.match(customurl, "/App_Themes/")
            or string.match(customurl, "/WebResource%.axd")
            or string.match(customurl, "/bprovideo/")
            or string.match(customurl, "/ScriptResource%.axd")
            or string.match(customurl, "/Screenshot%.ashx")
            or string.match(customurl, "silverlight%.dlservice%.microsoft%.com")
            or string.match(customurl, "/GameStatsHalo3%.aspx") then
            if not string.match(customurl, "[0-9]?"..item_value.."[0-9][0-9][0-9]")
              and not string.match(customurl, "PlayerStatsHalo3%.aspx%?player=")
              and not string.match(customurl, "/Stats/Halo3/Default%.aspx%?player=") then
              if downloaded[customurl] ~= true and addedtolist[customurl] ~= true then
                table.insert(urls, { url=customurl })
              end
            end
          end
        end
        for customurl in string.gmatch(html, '%((http[s]?://[^%)]+)%)') do
          if string.match(customurl, "[^0-9]"..item_value.."[0-9][0-9]")
            or string.match(customurl, "/javascript/")
            or string.match(customurl, "/base_css/")
            or string.match(customurl, "/images/")
            or string.match(customurl, "/Silverlight/")
            or string.match(customurl, "/Stats/")
            or string.match(customurl, "/App_Themes/")
            or string.match(customurl, "/WebResource%.axd")
            or string.match(customurl, "/bprovideo/")
            or string.match(customurl, "/ScriptResource%.axd")
            or string.match(customurl, "/Screenshot%.ashx")
            or string.match(customurl, "silverlight%.dlservice%.microsoft%.com")
            or string.match(customurl, "/GameStatsHalo3%.aspx") then
            if not string.match(customurl, "[0-9]?"..item_value.."[0-9][0-9][0-9]")
              and not string.match(customurl, "PlayerStatsHalo3%.aspx%?player=")
              and not string.match(customurl, "/Stats/Halo3/Default%.aspx%?player=") then
              if downloaded[customurl] ~= true and addedtolist[customurl] ~= true then
                table.insert(urls, { url=customurl })
              end
            end
          end
        end
        for customurlnf in string.gmatch(html, '"(/[^"]+)"') do
          if string.match(customurlnf, "[^0-9]"..item_value.."[0-9][0-9]")
            or string.match(customurlnf, "/javascript/")
            or string.match(customurlnf, "/base_css/")
            or string.match(customurlnf, "/images/")
            or string.match(customurlnf, "/Silverlight/")
{'fraction_non_alphanumeric': 0.1649620253164557, 'fraction_numerical': 0.014278481012658228, 'mean_word_length': 2.597814207650273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1927775', 'n_tokens_mistral': 3367, 'n_tokens_neox': 3302, 'n_words': 666}