<commit_before>import os
import urllib
import re
# import time

class ImgDownloader4chan:

    def __init__(self):
        self.image_url = 'http://images.4chan.org'
        # self.rescan_time = 10

        self.raw_url = raw_input('Please enter the URL of the thread:\n').split('#')[0]
        print ''
        self.board = self.raw_url.split('/')[3]
        self.thread = self.raw_url.split('/')[5]

        self.url = self.create_url()
        self.directory = self.create_directory()

        self.loop()

    def loop(self):
        self.image_urls = self.find_image_urls()

        while len(self.image_urls) > self.remove_duplicates():
            print ''
            self.download_images()
            # time.sleep(self.rescan_time)

            print ''

            self.image_urls = self.find_image_urls()

        print '\nFinished.'

    def create_url(self):

        template = 'http://boards.4chan.org/{board}/res/{thread}'
        template = template.replace('{board}', self.board)
        template = template.replace('{thread}', self.thread)
        return template

    def create_directory(self):
        """
        Creates a directory called 'downloads' with the subdirectories for the images.
        e.g.: ./downloads/{board}/{thread}
        """
        directory = os.path.join(os.getcwd(), 'downloads', self.board, self.thread)

        if not os.path.exists(directory):
            os.makedirs(directory)

        return directory

    def find_image_urls(self):
        """
        Should find all images which should be downloaded from 4chan. Yes, 'should'.
        """
        img_urls = []
        html = urllib.urlopen(self.url).read()
        tmp_urls = re.findall('(/[A-Za-z]+/src/\\d+\\.)(jpeg|jpg|png|gif)', html)

        for img in tmp_urls:
            img_urls.append(self.image_url + img[0] + img[1])

        print str(len(img_urls)/2) + ' images found.' # note: tmp_urls containts every url twice

        return list(set(img_urls))

    def remove_duplicates(self):
        """
        Removes the image-URLs which are downloaded previously.
        """
        duplicates = []

        for file in self.image_urls:<commit_msg>Minor update<commit_after>import os
import re
import urllib
# import time

class ImgDownloader4chan:

    def __init__(self):
        # self.rescan_time = 10

        self.raw_url = raw_input('Please enter the URL of the thread:\n').split('#')[0]
        print ''
        self.board = self.raw_url.split('/')[3]
        self.thread = self.raw_url.split('/')[5]

        self.url = self.create_url()
        self.directory = self.create_directory()

        self.loop()

    def loop(self):
        self.image_urls = self.find_image_urls()

        while len(self.image_urls) > self.remove_duplicates():
            print ''
            self.download_images()
            # time.sleep(self.rescan_time)

            print ''
            self.image_urls = self.find_image_urls()

        print '\nFinished.'

    def create_url(self):

        template = 'http://boards.4chan.org/{board}/res/{thread}'
        template = template.replace('{board}', self.board)
        template = template.replace('{thread}', self.thread)
        return template

    def create_directory(self):
        """
        Creates a directory called 'downloads' with the subdirectories for the images.
        e.g.: ./downloads/{board}/{thread}
        """
        directory = os.path.join(os.getcwd(), 'downloads', self.board, self.thread)

        if not os.path.exists(directory):
            os.makedirs(directory)

        return directory

    def find_image_urls(self):
        """
        Should find all images which should be downloaded from 4chan. Yes, 'should'.
        """
        img_urls = []
        html = urllib.urlopen(self.url).read()
        tmp_urls = re.findall('(/[A-Za-z]+/src/\\d+\\.)(jpeg|jpg|png|gif)', html)

        for img in tmp_urls:
            img_urls.append('http://images.4chan.org' + img[0] + img[1])

        print str(len(img_urls)/2) + ' images found.' # note: img_urls containts every url twice

        return list(set(img_urls)) # removes duplicates in img_urls

    def remove_duplicates(self):
        """
        Removes the image-URLs which are downloaded previously.
        """
        duplicates = []

        for file in self.image_urls:
{'fraction_non_alphanumeric': 0.13281069863961265, 'fraction_numerical': 0.00553377910998386, 'mean_word_length': 2.19205298013245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2195356', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1330, 'n_words': 282}