<commit_before>----TODO----
--Deeper url scraping for mundiasurnames
--DONE Add support for cgi urls for genforum
--Add support for myfamily


dofile("urlcode.lua")
dofile("table_show.lua")
JSON = (loadfile "JSON.lua")()

local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')

local downloaded = {}

load_json_file = function(file)
  if file then
    local f = io.open(file)
    local data = f:read("*all")
    f:close()
    return JSON:decode(data)
  else
    return nil
  end
end

read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]
  local html = urlpos["link_expect_html"]
  local parenturl = parent["url"]
  local html = nil
  
  if downloaded[url] == true then
    return false
  else
  
  if item_type == "genealogy" then
    if string.match(url, "www%.familyorigins%.com") then
      return false
    elseif string.match(url, "familytreemaker%.genealogy%.com") then
      return false
    else
      return verdict
    end
  elseif item_type == "familytreemaker" then
    if string.match(url, "www%.familyorigins%.com") then
      return false
    elseif string.match(url, "www%.genealogy%.com") then
      return false
    else
      return verdict
    end
  elseif item_type == "familyorigins" then
    if string.match(url, "www%.genealogy%.com") then
      return false
    elseif string.match(url, "familytreemaker%.genealogy%.com") then
      return false
    else
      return verdict
    end
  elseif item_type == "genforum" then
    return verdict
  end
  
  if item_type == "genealogy" or
    item_type == "familytreemaker" or
    item_type == "familyorigins" then
    local url_kind = os.getenv('url_kind')
    local url_first = os.getenv('url_first')
    local url_second = os.getenv('url_second')
    local url_third = os.getenv('url_third')
    local url_name = os.getenv('url_name')
    if (string.match(url, "%.genealogy%.com/genealogy/") or string.match(url, "%.familyorigins%.com/genealogy/")) and
      html == 1 then
      --example url: http://www.genealogy.com/genealogy/users/s/c/h/Aaron-D-Scholl/
      local url_kind_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/([^/]+)/[^/]+/[^/]+/[^/]+/[^/]+/")
      local url_first_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/[^/]+/([^/]+)/[^/]+/[^/]+/[^/]+/")
      local url_second_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/[^/]+/[^/]+/([^/]+)/[^/]+/[^/]+/")
      local url_third_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/[^/]+/[^/]+/[^/]+/([^/]+)/[^/]+/")
      local url_name_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/[^/]+/[^/]+/[^/]+/[^/]+/([^/]+)/")
      if url_kind_url ~= url_kind or
        url_first_url ~= url_first or
        url_second_url ~= url_second or
        url_third_url ~= url_third or
        url_name_url ~= url_name then
        return false
      else
        return verdict
      end
    elseif (string.match(url, "%.genealogy%.com") or string.match(url, "%.familyorigins%.com"))
      and html == 1 then
      --example url: http://www.genealogy.com/users/s/c/h/Aaron-D-Scholl/
      local url_kind_url = string.match(url, "[a-z]+%.[a-z]+%.com/([^/]+)/[^/]+/[^/]+/[^/]+/[^/]+/")
      local url_first_url = string.match(url, "[a-z]+%.[a-z]+%.com/[^/]+/([^/]+)/[^/]+/[^/]+/[^/]+/")
      local url_second_url = string.match(url, "[a-z]+%.[a-z]+%.com/[^/]+/[^/]+/([^/]+)/[^/]+/[^/]+/")
      local url_third_url = string.match(url, "[a-z]+%.[a-z]+%.com/[^/]+/[^/]+/[^/]+/([^/]+)/[^/]+/")
      local url_name_url = string.match(url, "[a-z]+%.[a-z]+%.com/[^/]+/[^/]+/[^/]+/[^/]+/([^/]+)/")
      if url_kind_url ~= url_kind or
        url_first_url ~= url_first or
        url_second_url ~= url_second or
        url_third_url ~= url_third or
        url_name_url ~= url_name then
        return false
      else
        return verdict
      end
    elseif (string.match(parenturl, "%.genealogy%.com/") or string.match(parenturl, "%.familyorigins%.com"))
      and html == 1 then
      return true
    elseif html == 0 then
      return true
    else
      return false
    end
  elseif item_type == "mundiasurnames" then
    if string.match(url, "c%.muncn%.com") then
      return true
    elseif string.match(url, "mediasvc%.ancestry%.com") then
      return true
    elseif string.match(url, "ecn%.dev%.virtualearth%.net") then
      return true
    elseif string.match(url, "myfamily2%.[0-9]+%.[a-z0-9]+%.net") then
      return true
    elseif string.match(url, "tiles%.virtualearth%.net") then
      return true
    elseif string.match(url, "dev%.virtualearth%.net") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/surnames/[a-z0-9A-Z]+") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/Search/Results%?surname=[^&]+&birthPlace=") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/Person/[^/]+/.+") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/Tree/Family/[^/]+/.+") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/Messages%?sendMessageTo=[^&]+&subject=") then
      return true
    elseif string.match(url, "/media/") then
      return true
    elseif string.match(url, ".jpg") or string.match(url, ".png") or string.match(url, ".gif") then
      return true
    elseif html == 0 then
      return true
    else
      return false
    end
  elseif item_type == "genforum" then
    if string.match(url, "genforum%.genealogy%.com/([^/]+)/") and
      html == 1 then
      local item_value_url = string.match(url, "genforum%.genealogy%.com/([^/])/")
      if item_value_url ~= item_value then
        return false
      else
        return verdict
      end
    elseif string.match(url, "genforum%.com/([^/]+)/") and
      html == 1 then
      local item_value_url = string.match(url, "genforum%.com/([^/])/")
      if item_value_url ~= item_value then
        return false
      else
        return verdict
      end
    elseif string.match(url, "genforum%.genealogy%.com[.]+"..item_value) or
      string.match(url, "genforum%.com[.]+"..item_value) then
      if string.match(url, ":::") then
        if not html then
          html = read_file(file)
        end
        if string.match(html, '<FONT FACE=[^>]+><B><[^<]+</A>[^<]+<A[^<]+</A></B></FONT><BR>[^<]+<UL>[^<]+</UL>[^<]+<[^<]+<B><[^<]+</A>[^<]+<[^<]+</A></B></font><BR>') then
          return false
        else
          return true
        end
      else
        return true
      end
    elseif html == 0 then
      return true
    else
      return false
    end
  else
    return false
  end
  end
end


wget.callbacks.get_urls = function(file, url, is_css, iri)
  local urls = {}
  local html = nil
  
  if item_type == "mundiasurnames" then
    --example url: http://www.mundia.com/us/surnames/aleo
    if string.match(url, "%.mundia%.com/[^/]+/surnames/[^/]+") then
      local surname_lower = string.match(url, "%.mundia%.com/[^/]+/surnames/([^/]+)")
      local surname_upper = string.upper(surname_lower)
      local country_code = string.match(url, "%.mundia%.com/([^/]+)/surnames/[^/]+")
      --chfoo - is it alright is I add all these urls for all countries?
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Afghanistan" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Albania" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Algeria" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Argentina" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Armenia" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Australia" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Austria" })<commit_msg>Revert "ancestry.lua"<commit_after>----TODO----
--Deeper url scraping for mundiasurnames
--DONE Add support for cgi urls for genforum
--Add support for myfamily


dofile("urlcode.lua")
dofile("table_show.lua")
JSON = (loadfile "JSON.lua")()

local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')

local downloaded = {}

load_json_file = function(file)
  if file then
    local f = io.open(file)
    local data = f:read("*all")
    f:close()
    return JSON:decode(data)
  else
    return nil
  end
end

read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]
  local html = urlpos["link_expect_html"]
  local parenturl = parent["url"]
  local html = nil
  
  if downloaded[url] == true then
    return false
  end
  
  if item_type == "genealogy" then
    if string.match(url, "www%.familyorigins%.com") then
      return false
    elseif string.match(url, "familytreemaker%.genealogy%.com") then
      return false
    else
      return verdict
    end
  elseif item_type == "familytreemaker" then
    if string.match(url, "www%.familyorigins%.com") then
      return false
    elseif string.match(url, "www%.genealogy%.com") then
      return false
    else
      return verdict
    end
  elseif item_type == "familyorigins" then
    if string.match(url, "www%.genealogy%.com") then
      return false
    elseif string.match(url, "familytreemaker%.genealogy%.com") then
      return false
    else
      return verdict
    end
  elseif item_type == "genforum" then
    return verdict
  end
  
  if item_type == "genealogy" or
    item_type == "familytreemaker" or
    item_type == "familyorigins" then
    local url_kind = os.getenv('url_kind')
    local url_first = os.getenv('url_first')
    local url_second = os.getenv('url_second')
    local url_third = os.getenv('url_third')
    local url_name = os.getenv('url_name')
    if (string.match(url, "%.genealogy%.com/genealogy/") or string.match(url, "%.familyorigins%.com/genealogy/")) and
      html == 1 then
      --example url: http://www.genealogy.com/genealogy/users/s/c/h/Aaron-D-Scholl/
      local url_kind_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/([^/]+)/[^/]+/[^/]+/[^/]+/[^/]+/")
      local url_first_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/[^/]+/([^/]+)/[^/]+/[^/]+/[^/]+/")
      local url_second_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/[^/]+/[^/]+/([^/]+)/[^/]+/[^/]+/")
      local url_third_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/[^/]+/[^/]+/[^/]+/([^/]+)/[^/]+/")
      local url_name_url = string.match(url, "[a-z]+%.[a-z]+%.com/genealogy/[^/]+/[^/]+/[^/]+/[^/]+/([^/]+)/")
      if url_kind_url ~= url_kind or
        url_first_url ~= url_first or
        url_second_url ~= url_second or
        url_third_url ~= url_third or
        url_name_url ~= url_name then
        return false
      else
        return verdict
      end
    elseif (string.match(url, "%.genealogy%.com") or string.match(url, "%.familyorigins%.com"))
      and html == 1 then
      --example url: http://www.genealogy.com/users/s/c/h/Aaron-D-Scholl/
      local url_kind_url = string.match(url, "[a-z]+%.[a-z]+%.com/([^/]+)/[^/]+/[^/]+/[^/]+/[^/]+/")
      local url_first_url = string.match(url, "[a-z]+%.[a-z]+%.com/[^/]+/([^/]+)/[^/]+/[^/]+/[^/]+/")
      local url_second_url = string.match(url, "[a-z]+%.[a-z]+%.com/[^/]+/[^/]+/([^/]+)/[^/]+/[^/]+/")
      local url_third_url = string.match(url, "[a-z]+%.[a-z]+%.com/[^/]+/[^/]+/[^/]+/([^/]+)/[^/]+/")
      local url_name_url = string.match(url, "[a-z]+%.[a-z]+%.com/[^/]+/[^/]+/[^/]+/[^/]+/([^/]+)/")
      if url_kind_url ~= url_kind or
        url_first_url ~= url_first or
        url_second_url ~= url_second or
        url_third_url ~= url_third or
        url_name_url ~= url_name then
        return false
      else
        return verdict
      end
    elseif (string.match(parenturl, "%.genealogy%.com/") or string.match(parenturl, "%.familyorigins%.com"))
      and html == 1 then
      return true
    elseif html == 0 then
      return true
    else
      return false
    end
  elseif item_type == "mundiasurnames" then
    if string.match(url, "c%.muncn%.com") then
      return true
    elseif string.match(url, "mediasvc%.ancestry%.com") then
      return true
    elseif string.match(url, "ecn%.dev%.virtualearth%.net") then
      return true
    elseif string.match(url, "myfamily2%.[0-9]+%.[a-z0-9]+%.net") then
      return true
    elseif string.match(url, "tiles%.virtualearth%.net") then
      return true
    elseif string.match(url, "dev%.virtualearth%.net") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/surnames/[a-z0-9A-Z]+") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/Search/Results%?surname=[^&]+&birthPlace=") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/Person/[^/]+/.+") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/Tree/Family/[^/]+/.+") then
      return true
    elseif string.match(url, "mundia%.com/[^/]+/Messages%?sendMessageTo=[^&]+&subject=") then
      return true
    elseif string.match(url, "/media/") then
      return true
    elseif string.match(url, ".jpg") or string.match(url, ".png") or string.match(url, ".gif") then
      return true
    elseif html == 0 then
      return true
    else
      return false
    end
  elseif item_type == "genforum" then
    if string.match(url, "genforum%.genealogy%.com/([^/]+)/") and
      html == 1 then
      local item_value_url = string.match(url, "genforum%.genealogy%.com/([^/])/")
      if item_value_url ~= item_value then
        return false
      else
        return verdict
      end
    elseif string.match(url, "genforum%.com/([^/]+)/") and
      html == 1 then
      local item_value_url = string.match(url, "genforum%.com/([^/])/")
      if item_value_url ~= item_value then
        return false
      else
        return verdict
      end
    elseif string.match(url, "genforum%.genealogy%.com[.]+"..item_value) or
      string.match(url, "genforum%.com[.]+"..item_value) then
      if string.match(url, ":::") then
        if not html then
          html = read_file(file)
        end
        if string.match(html, '<FONT FACE=[^>]+><B><[^<]+</A>[^<]+<A[^<]+</A></B></FONT><BR>[^<]+<UL>[^<]+</UL>[^<]+<[^<]+<B><[^<]+</A>[^<]+<[^<]+</A></B></font><BR>') then
          return false
        else
          return true
        end
      else
        return true
      end
    elseif html == 0 then
      return true
    else
      return false
    end
  else
    return false
  end
end


wget.callbacks.get_urls = function(file, url, is_css, iri)
  local urls = {}
  local html = nil
  
  if item_type == "mundiasurnames" then
    --example url: http://www.mundia.com/us/surnames/aleo
    if string.match(url, "%.mundia%.com/[^/]+/surnames/[^/]+") then
      local surname_lower = string.match(url, "%.mundia%.com/[^/]+/surnames/([^/]+)")
      local surname_upper = string.upper(surname_lower)
      local country_code = string.match(url, "%.mundia%.com/([^/]+)/surnames/[^/]+")
      --chfoo - is it alright is I add all these urls for all countries?
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Afghanistan" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Albania" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Algeria" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Argentina" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Armenia" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Australia" })
      table.insert(urls, { url="http://www.mundia.com/"..country_code.."/Search/Results?surname="..surname_upper.."&birthPlace=Austria" })
{'fraction_non_alphanumeric': 0.20272754224725764, 'fraction_numerical': 0.0020160094871034687, 'mean_word_length': 2.882596685082873, 'pattern_counts': {'":': 2, '<': 63, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 28, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '446590', 'n_tokens_mistral': 6335, 'n_tokens_neox': 6053, 'n_words': 1152}