<commit_before>import lxml.html
import itertools
import requests
import urllib.parse
import threading
import sys
import traceback

from termcolor import colored

from util.files import *
from util.lists import *
from util.queue import *
from util.nhash import *
from util.urls import *

def find_links(tree, url):
    paths = ['//@href', '//@action', '//@src']

    for path in paths:
        for href in tree.xpath(path):
            if urllib.parse.urlparse(href).scheme == '':
                href = urllib.parse.urljoin(url, href)
            yield urlnorm(href)

class Scanner(object):
    def __init__(self, base):
        self.urls = UniqueQueue()
        self.urls.norm = urlnorm
        self.base = base

        self.session = requests.session()
        self.crawled = {}

        self.urls.put(base)

    def on_request(self, url):
        sys.stderr.write('%s\r%s\r' % (' '*64, trim(url, 64, '...')))

    def on_response(self, response, chash, unique):

        if unique:
            color = 'green'
        else:
            color = 'yellow'
            
        print((colored('[%08x]', color) + ' %s') % (chash, response.url))

    def add_crawled(self, chash, url):
        if chash not in self.crawled:
            self.crawled[chash] = set()

        self.crawled[chash].add(url)

    def was_crawled(self, chash):
        return chash in self.crawled

    def scan(self, url=None, words=None):

        if url != None:
            self.urls.put(url)

        if words == None:
            words = []
        
        while not self.urls.empty():
            
            try:
                url = self.urls.get()
                self.on_request(url)

                respn = self.session.get(url)
                chash = simhash(respn.content)

                if respn.ok:
                    self.on_response(response=respn, chash=chash, unique=(not self.was_crawled(chash)))
                    self.add_crawled(chash, respn.url)

                    ctype = respn.headers.get('content-type')
                    links = []

                    if 'html' in ctype or 'xml' in ctype:
                        xtree = lxml.html.fromstring(respn.content)
                        links = itertools.chain(links, find_links(xtree, url))

                    links = itertools.chain(links, urlprefixes(url))

                    for link in links:
                        if urlbase(self.base, link):
                            self.urls.put(link)

            except Exception as ex:
                # traceback.print_exc()
                pass

        # do bruteforce passes
        m = len(wordlist)
        n = 0<commit_msg>[BUG] Use resp.text instead of content in lxml.<commit_after>import lxml.html
import itertools
import requests
import urllib.parse
import threading
import sys
import traceback

from termcolor import colored

from util.files import *
from util.lists import *
from util.queue import *
from util.nhash import *
from util.urls import *

def find_links(tree, url):
    paths = ['//@href', '//@action', '//@src']

    for path in paths:
        for href in tree.xpath(path):
            if urllib.parse.urlparse(href).scheme == '':
                href = urllib.parse.urljoin(url, href)
            yield urlnorm(href)

class Scanner(object):
    def __init__(self, base):
        self.urls = UniqueQueue()
        self.urls.norm = urlnorm
        self.base = base

        self.session = requests.session()
        self.crawled = {}

        self.urls.put(base)

    def on_request(self, url):
        sys.stderr.write('%s\r%s\r' % (' '*64, trim(url, 64, '...')))

    def on_response(self, response, chash, unique):

        if unique:
            color = 'green'
        else:
            color = 'yellow'
            
        print((colored('[%08x]', color) + ' %s') % (chash, response.url))

    def add_crawled(self, chash, url):
        if chash not in self.crawled:
            self.crawled[chash] = set()

        self.crawled[chash].add(url)

    def was_crawled(self, chash):
        return chash in self.crawled

    def scan(self, url=None, words=None):

        if url != None:
            self.urls.put(url)

        if words == None:
            words = []
        
        while not self.urls.empty():
            
            try:
                url = self.urls.get()
                self.on_request(url)

                respn = self.session.get(url)
                chash = simhash(respn.content)

                if respn.ok:
                    self.on_response(response=respn, chash=chash, unique=(not self.was_crawled(chash)))
                    self.add_crawled(chash, respn.url)

                    ctype = respn.headers.get('content-type')
                    links = []

                    if 'html' in ctype or 'xml' in ctype:
                        xtree = lxml.html.fromstring(respn.text)
                        links = itertools.chain(links, find_links(xtree, url))

                    links = itertools.chain(links, urlprefixes(url))

                    for link in links:
                        if urlbase(self.base, link):
                            self.urls.put(link)

            except Exception as ex:
                # traceback.print_exc()
                pass

        # do bruteforce passes
        m = len(wordlist)
        n = 0
{'fraction_non_alphanumeric': 0.11342723004694835, 'fraction_numerical': 0.002629107981220657, 'mean_word_length': 1.598048780487805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1612329', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1581, 'n_words': 376}