<commit_before># -*- coding: utf-8 -*-

'''
Created on 17/12/2011

@author: nicolas
'''

import urllib2
import re
import xml.dom.minidom


base = 'http://www.cuentalo.com.uy'

url_sitemap = [base + '/contacto']
url_no_sitemap = []

f = file('/home/cuentalo/cuentalo.com.uy/src/Mondel/CuentaloBundle/Resources/tools/sitemap_generator/sedlog.txt', 'r')
data = [l.strip() for l in f.readlines()]

for d in data:
    if 'robots.txt' in d:
        url_no_sitemap.append(d)
        robots_html = urllib2.urlopen(d).read()
        patt = 'Disallow:\s*(.*)'
        for i in re.findall(patt, robots_html):
            url_no_sitemap.append(base + i)

for d in data:
    agregar = False
    if d not in url_no_sitemap:
        for u in url_no_sitemap:
            if u not in d and d not in url_sitemap:
                agregar = True
            else:
                agregar = False
                break
    if agregar:
        url_sitemap.append(d)

print url_no_sitemap
print len(url_sitemap)
<commit_msg>ahora saca todas las url? sigue siendo chancho pero por ahora...<commit_after># -*- coding: utf-8 -*-

'''
Created on 17/12/2011

@author: nicolas
'''

import urllib2
import re
import xml.dom.minidom


base = 'http://www.cuentalo.com.uy'

url_sitemap = [base + '/contacto']
url_no_sitemap = []

f = file('/home/cuentalo/cuentalo.com.uy/src/Mondel/CuentaloBundle/Resources/tools/sitemap_generator/sedlog.txt', 'r')
data = [l.strip() for l in f.readlines()]

for d in data:
    if 'robots.txt' in d:
        url_no_sitemap.append(d)
        robots_html = urllib2.urlopen(d).read()
        patt = 'Disallow:\s*(.*)'
        for i in re.findall(patt, robots_html):
            url_no_sitemap.append(base + i)

for d in data:
    agregar = False
    if d not in url_no_sitemap:
        for u in url_no_sitemap:
            if u not in d and d not in url_sitemap:
                agregar = True
            else:
                agregar = False
                break
    if agregar:
        url_sitemap.append(d)

url_sitemap_aux = []
idmax = 0

for url in url_sitemap:
    pattern = 'cuentalo.com.uy/contenido/(\d+)'
    m = re.search(url)
    if m is not None:
        if int(m.group(1)) > idmax:
            idmax = int(m.group(1))
        
    print idmax
    
    for i in range(idmax):
        url_aux = 'http://www.cuentalo.com.uy/contenido/%s' % i
        try:
            resp = urllib2.urlopen(url_aux)
            if url_aux not in url_sitemap:
                url_sitemap.append(url_aux)
        except:
            pass

{'fraction_non_alphanumeric': 0.12025316455696203, 'fraction_numerical': 0.010284810126582278, 'mean_word_length': 2.1183723797780516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '262300', 'n_tokens_mistral': 982, 'n_tokens_neox': 955, 'n_words': 221}