<commit_before>dofile("urlcode.lua")
dofile("table_show.lua")

local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')
local item_dir = os.getenv('item_dir')
local warc_file_base = os.getenv('warc_file_base')

local profilepic = nil

local downloaded = {}
local addedtolist = {}
local profiles = {}

for ignore in io.open("ignore-list", "r"):lines() do
  downloaded[ignore] = true
end

read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]
  local html = urlpos["link_expect_html"]
  
  if string.match(url, "^https?://[^/]*fotolog%.com/[^/]+") and not string.match(url, "^https?://sp[0-9a-zA-Z]*%.fotolog%.com") then
    profiles[string.match(url, "^https?://[^/]*fotolog%.com/([^/]+)")] = true
  end

  if (downloaded[url] ~= true and addedtolist[url] ~= true) and (string.match(url, "^https?://[^/]*fotolog%.com") and string.match(url, item_value) or html == 0) and not (string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/twitter_register") or string.match(url, "^https?://[^/]*fotolog%.com%/login/%?redirect=") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/lang_switch") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/friend_add") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/share_overlay") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/report_overlay") or string.match(url, "^https?://[^/]*fotolog%.com/account/photos/edit") or string.match(url, "<a class=$")) then
    if string.match(url, "^https?://sp[0-9a-zA-Z]*%.fotolog%.com") and string.match(url, item_value) then<commit_msg>Regex fix for commit 3<PASSWORD>: grab any character that isn't a forward slash.<commit_after>dofile("urlcode.lua")
dofile("table_show.lua")

local url_count = 0
local tries = 0
local item_type = os.getenv('item_type')
local item_value = os.getenv('item_value')
local item_dir = os.getenv('item_dir')
local warc_file_base = os.getenv('warc_file_base')

local profilepic = nil

local downloaded = {}
local addedtolist = {}
local profiles = {}

for ignore in io.open("ignore-list", "r"):lines() do
  downloaded[ignore] = true
end

read_file = function(file)
  if file then
    local f = assert(io.open(file))
    local data = f:read("*all")
    f:close()
    return data
  else
    return ""
  end
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  local url = urlpos["url"]["url"]
  local html = urlpos["link_expect_html"]
  
  if string.match(url, "^https?://[^/]*fotolog%.com/[^/]+") and not string.match(url, "^https?://sp[0-9a-zA-Z]*%.fotolog%.com") then
    profiles[string.match(url, "^https?://[^/]*fotolog%.com/([^/]+)")] = true
  end

  if (downloaded[url] ~= true and addedtolist[url] ~= true) and (string.match(url, "^https?://[^/]*fotolog%.com") and string.match(url, item_value) or html == 0) and not (string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/twitter_register") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/login/%?redirect=") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/lang_switch") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/friend_add") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/share_overlay") or string.match(url, "^https?://[^/]*fotolog%.com[^/]*/act/report_overlay") or string.match(url, "^https?://[^/]*fotolog%.com/account/photos/edit") or string.match(url, "<a class=$")) then
    if string.match(url, "^https?://sp[0-9a-zA-Z]*%.fotolog%.com") and string.match(url, item_value) then
{'fraction_non_alphanumeric': 0.22604358099238644, 'fraction_numerical': 0.0039380414807035965, 'mean_word_length': 4.906976744186046, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4504548', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1417, 'n_words': 273}