<commit_before>#! /usr/bin/python

import urllib2
from validator import fullPath
from bs4 import BeautifulSoup
from argparse import ArgumentParser
from collections import deque
from writer import writer
import sys


def qtl(queue):
  '''Converts depth queue to list of urls
     [(url1, 0), (url2, 1)] -> [url1, url2]
  '''
  return map(lambda x: x[0], queue)

def run(options):
  output = writer(options)
  maxDepth = options.depth
  queue = deque([(options.target, 0)])
  info = {}
  visited = []

  while queue:
    url, depth = queue.popleft()
    if options.verbose == True:
      output(depth + " Visiting: " + url + '\n')
    visited.append(url);
    info[url] = {}
    try:
      response = urllib2.urlopen(url)
    except urllib2.HTTPError as e:
      if options.verbose == True:
        output("({0}): {1}".format(e.errno, e.strerror) + '\n')
      continue

    html = BeautifulSoup(response.read())

    if depth >= maxDepth:
      continue
    info[url]['src'] = html
    info[url]['a'] = html.find_all('a')
    info[url]['img'] = html.find_all('img')
    for link in info[url]['a']:
      if not link.get('href'):
        continue
      href = fullPath(url, link.get('href'));
      if options.extract == 'a':
        output(href + '\n')
      if href == None:
        continue
      if href not in visited and href not in qtl(queue):
        queue.append((href, depth));

def parse():
  parser = ArgumentParser(description='Website mapper')
  parser.add_argument('-t', '--target', help='Starting address', action='store', dest='target')
  parser.add_argument('-d', '--depth', help='How deep do you want to dig?', action='store', dest='depth', type=int, default=0)
  parser.add_argument('-v', '--verbose', help='Displays detailed ouput, i.e. things like depth, current base link...', action='store_true', default=False)
  parser.add_argument('-e', '--extract', choices=['a', 'img'], help="what to extract")
  parser.add_argument('-w', '--write', help='Write to file', default=False)


  options = parser.parse_args()
  if options.target == None:
    print "Invalid usage, try -h for help"<commit_msg>images can now be extracted<commit_after>#! /usr/bin/python

import urllib2
from validator import fullPath
from bs4 import BeautifulSoup
from argparse import ArgumentParser
from collections import deque
from writer import writer
import sys


def qtl(queue):
  '''Converts depth queue to list of urls
     [(url1, 0), (url2, 1)] -> [url1, url2]
  '''
  return map(lambda x: x[0], queue)

def run(options):
  output = writer(options)
  maxDepth = options.depth
  queue = deque([(options.target, 0)])
  info = {}
  visited = []

  while queue:
    url, depth = queue.popleft()
    if options.verbose == True:
      output(depth + " Visiting: " + url + '\n')
    visited.append(url);
    info[url] = {}
    try:
      response = urllib2.urlopen(url)
    except urllib2.HTTPError as e:
      if options.verbose == True:
        output("({0}): {1}".format(e.errno, e.strerror) + '\n')
      continue

    html = BeautifulSoup(response.read())

    if depth >= maxDepth:
      continue
    info[url]['src'] = html
    info[url]['a'] = html.find_all('a')
    info[url]['img'] = html.find_all('img')

    for image in info[url]['img']:
      href = fullPath(url, image['src'])
      output(href + '\n')

    for link in info[url]['a']:
      if not link.get('href'):
        continue
      href = fullPath(url, link.get('href'));
      if options.extract == 'a':
        output(href + '\n')
      if href == None:
        continue
      if href not in visited and href not in qtl(queue):
        queue.append((href, depth));

def parse():
  parser = ArgumentParser(description='Website mapper')
  parser.add_argument('-t', '--target', help='Starting address', action='store', dest='target')
  parser.add_argument('-d', '--depth', help='How deep do you want to dig?', action='store', dest='depth', type=int, default=0)
  parser.add_argument('-v', '--verbose', help='Displays detailed ouput, i.e. things like depth, current base link...', action='store_true', default=False)
  parser.add_argument('-e', '--extract', choices=['a', 'img'], help="what to extract")
  parser.add_argument('-w', '--write', help='Write to file', default=False)


  options = parser.parse_args()
  if options.target == None:
    print "Invalid usage, try -h for help"
{'fraction_non_alphanumeric': 0.16635987114588127, 'fraction_numerical': 0.006902899217671422, 'mean_word_length': 3.377643504531722, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '171900', 'n_tokens_mistral': 1423, 'n_tokens_neox': 1402, 'n_words': 398}