<commit_before>from bs4 import BeautifulSoup as BS
import requests

liste=[]
#Get for each H (topic)
for x in range(1,12):
  liste.append("http://www.statistik-bw.de/SRDB/home.asp?H="+str(x)+"&E=GE")

liste2=[]
for x in liste:
  soup = BS(requests.get(x).text)
  for y in soup.find(attrs={"name":"U"}).children:
    soup2 = BS(requests.get(x + "&U=" + y.get(value)).text)
#    for z in soup.find(attrs={"name":"T"}).children:
#      liste2=(y + "&T=" + z.get(value))<commit_msg>fetch and print out a list of table URLs<commit_after>from bs4 import BeautifulSoup as BS
import requests

liste=[]
#Get for each H (topic)
for x in range(1,12):
  liste.append("http://www.statistik-bw.de/SRDB/home.asp?H="+str(x)+"&E=GE")

liste2=[]
found =[]
for x in liste:
  soup = BS(requests.get(x).text)
  for y in soup.find(attrs={"name":"U"}).children:
    if y == u"\n":
      continue
    soup2 = BS(requests.get(x + "&U=" + y.get("value")).text)
    res = soup2.find(attrs={"name":"T"})
    if not res:
        print "Warning, did not find any tables for:" + x + "&U=" + y.get("value")
        continue
    for z in res.children:
      if z == u"\n":
        continue
      liste2.append(x + "&U=" + y.get("value") + "&T=" + z.get("value"))

print liste2
{'fraction_non_alphanumeric': 0.20434432823813356, 'fraction_numerical': 0.012872083668543845, 'mean_word_length': 3.5072463768115942, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4841230', 'n_tokens_mistral': 496, 'n_tokens_neox': 492, 'n_words': 119}