<commit_before>import config
import feedparser
import json
import requests
import os
import subprocess
import time

class RSStoTGbot(object):
    """docstring for RSStoTGbot"""
    def __init__(self):
        super(RSStoTGbot, self).__init__()
        self.subreddits = ""
        for sr in config.subreddits:
            self.subreddits += sr + "+"
        self.feedurl = "http://www.reddit.com/r/" + self.subreddits + "/.rss"
        self.refreshdownloads()
        if not os.path.exists("downloads"):
            os.makedirs("downloads")

    def refreshdownloads(self):
        if not os.path.isfile("downloaded"):
            with open("downloaded", 'w') as downloaded:
                downloaded.write("[]")
            self.downloads = []
        else:
            with open("downloaded","r") as downloaded:
                self.downloads = json.load(downloaded)

    def indownloads(self, link):
        if link in self.downloads:
            return True
        else:
            return False

    def addtodownloads(self, link):
        self.downloads.append(link)

    def writedownloads(self):
        with open("downloaded","w") as downloaded:
            json.dump(self.downloads, downloaded)

    def findlink(self, description):
        prelink = description.split('">[link]')[0]
        thelink = "http" + prelink.split("http")[-1]
        return thelink

    def directimage(self, link):
        for a in ["jpg", "jpeg", "gif", "png"]:
            if link.endswith(a):
                return True
        return False

    def download(self, link):
        filename = config.togroup + "-" + link.split('/')[-1]
        r = requests.get(link, headers=header)
        with open("downloads/"+filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)
                    f.flush()
        return

    def sendtotg(self):
        subprocess.call(["./sender.sh", config.tgdir])
<commit_msg>header not required. we are not abusing api<commit_after>import config
import feedparser
import json
import requests
import os
import subprocess
import time

class RSStoTGbot(object):
    """docstring for RSStoTGbot"""
    def __init__(self):
        super(RSStoTGbot, self).__init__()
        self.subreddits = ""
        for sr in config.subreddits:
            self.subreddits += sr + "+"
        self.feedurl = "http://www.reddit.com/r/" + self.subreddits + "/.rss"
        self.refreshdownloads()
        if not os.path.exists("downloads"):
            os.makedirs("downloads")

    def refreshdownloads(self):
        if not os.path.isfile("downloaded"):
            with open("downloaded", 'w') as downloaded:
                downloaded.write("[]")
            self.downloads = []
        else:
            with open("downloaded","r") as downloaded:
                self.downloads = json.load(downloaded)

    def indownloads(self, link):
        if link in self.downloads:
            return True
        else:
            return False

    def addtodownloads(self, link):
        self.downloads.append(link)

    def writedownloads(self):
        with open("downloaded","w") as downloaded:
            json.dump(self.downloads, downloaded)

    def findlink(self, description):
        prelink = description.split('">[link]')[0]
        thelink = "http" + prelink.split("http")[-1]
        return thelink

    def directimage(self, link):
        for a in ["jpg", "jpeg", "gif", "png"]:
            if link.endswith(a):
                return True
        return False

    def download(self, link):
        filename = config.togroup + "-" + link.split('/')[-1]
        r = requests.get(link)
        with open("downloads/"+filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)
                    f.flush()
        return

    def sendtotg(self):
        subprocess.call(["./sender.sh", config.tgdir])

{'fraction_non_alphanumeric': 0.1272590361445783, 'fraction_numerical': 0.0035140562248995983, 'mean_word_length': 2.0419847328244276, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7499749', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1169, 'n_words': 276}