<commit_before>#encoding:utf-8

import importlib
import logging
import random
import time

import yaml
import praw

import utils
from reporting_stuff import report_error


@report_error
def supply(submodule_name, config, is_test=False):
    if not is_test:
        time.sleep(random.randrange(0, 40))
    submodule = importlib.import_module('channels.{}.app'.format(submodule_name))
    reddit = praw.Reddit(
        user_agent=config['reddit']['user_agent'],
        client_id=config['reddit']['client_id'],
        client_secret=config['reddit']['client_secret'],
        username=config['reddit']['username'],
        password=config['reddit']['password']
    )
    submissions_ranking = getattr(submodule, 'submissions_ranking', 'hot')
    submissions_limit = getattr(submodule, 'submissions_limit', 100)
    if submissions_ranking == 'top':
        submissions = reddit.subreddit(submodule.subreddit).top(limit=submissions_limit)
    elif submissions_ranking == 'hot':
        submissions = reddit.subreddit(submodule.subreddit).hot(limit=submissions_limit)
    elif submissions_ranking == 'new':
        submissions = reddit.subreddit(submodule.subreddit).new(limit=submissions_limit)
    else:
        logging.error('Unknown submissions_ranking. {}'.format(submissions_ranking))
    channel_to_post = submodule.t_channel if not is_test else '@r_channels_test'
    r2t = utils.Reddit2TelegramSender(channel_to_post, config)
    success = False
    for submission in submissions:
        link = submission.shortlink
        if r2t.was_before(link):
            continue
        if r2t.too_much_errors(link):
            continue
        success = submodule.send_post(submission, r2t)
        if success == utils.SupplyResult.SUCCESSFULLY:
            # Every thing is ok, post was sent
            r2t.mark_as_was_before(link, sent=True)
            break
        elif success == utils.SupplyResult.DO_NOT_WANT_THIS_SUBMISSION:
            # Do not want to send this post
            r2t.mark_as_was_before(link, sent=False)
            continue
        elif success == utils.SupplyResult.SKIP_FOR_NOW:
            # Do not want to send now
            continue
        elif success == utils.SupplyResult.STOP_THIS_SUPPLY:
            # If None — do not want to send anything this time
            break
        else:
            logging.error('Unknown SupplyResult. {}'.format(success))
    if success is False:
        logging.info('Nothing to post from {sub} to {channel}.'.format(
                    sub=submodule.subreddit, channel=submodule.t_channel))


def main(config_filename, sub, is_test=False):
    with open(config_filename) as config_file:
        config = yaml.load(config_file.read())
        supply(sub, config, is_test)


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', default='configs/prod.yml')
    parser.add_argument('--test', action='store_true')
    parser.add_argument('--sub')
    args = parser.parse_args()
    main(args.config, args.sub, args.test)<commit_msg>Moving submissioning into another function.<commit_after>#encoding:utf-8

import importlib
import logging
import random
import time

import yaml
import praw

import utils
from reporting_stuff import report_error


def send_to_channel_from_subreddit(how_to_post, channel_to_post, subreddit, submissions_ranking, submissions_limit, config):
    reddit = praw.Reddit(
        user_agent=config['reddit']['user_agent'],
        client_id=config['reddit']['client_id'],
        client_secret=config['reddit']['client_secret'],
        username=config['reddit']['username'],
        password=config['reddit']['password']
    )
    if submissions_ranking == 'top':
        submissions = reddit.subreddit(subreddit).top(limit=submissions_limit)
    elif submissions_ranking == 'hot':
        submissions = reddit.subreddit(subreddit).hot(limit=submissions_limit)
    elif submissions_ranking == 'new':
        submissions = reddit.subreddit(subreddit).new(limit=submissions_limit)
    else:
        logging.error('Unknown submissions_ranking. {}'.format(submissions_ranking))
    r2t = utils.Reddit2TelegramSender(channel_to_post, config)
    success = False
    for submission in submissions:
        link = submission.shortlink
        if r2t.was_before(link):
            continue
        if r2t.too_much_errors(link):
            continue
        success = how_to_post(submission, r2t)
        if success == utils.SupplyResult.SUCCESSFULLY:
            # Every thing is ok, post was sent
            r2t.mark_as_was_before(link, sent=True)
            break
        elif success == utils.SupplyResult.DO_NOT_WANT_THIS_SUBMISSION:
            # Do not want to send this post
            r2t.mark_as_was_before(link, sent=False)
            continue
        elif success == utils.SupplyResult.SKIP_FOR_NOW:
            # Do not want to send now
            continue
        elif success == utils.SupplyResult.STOP_THIS_SUPPLY:
            # If None — do not want to send anything this time
            break
        else:
            logging.error('Unknown SupplyResult. {}'.format(success))


@report_error
def supply(submodule_name, config, is_test=False):
    if not is_test:
        time.sleep(random.randrange(0, 40))
    submodule = importlib.import_module('channels.{}.app'.format(submodule_name))
    submissions_ranking = getattr(submodule, 'submissions_ranking', 'hot')
    submissions_limit = getattr(submodule, 'submissions_limit', 100)
    channel_to_post = submodule.t_channel if not is_test else '@r_channels_test'
    success = send_to_channel_from_subreddit(submodule.send_post, channel_to_post, submodule.subreddit, submissions_ranking, submissions_limit, config)
    if success is False:
        logging.info('Nothing to post from {sub} to {channel}.'.format(
                    sub=submodule.subreddit, channel=submodule.t_channel))


def main(config_filename, sub, is_test=False):
    with open(config_filename) as config_file:
        config = yaml.load(config_file.read())
        supply(sub, config, is_test)


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', default='configs/prod.yml')
    parser.add_argument('--test', action='store_true')
    parser.add_argument('--sub')
    args = parser.parse_args()
    main(args.config, args.sub, args.test)
{'fraction_non_alphanumeric': 0.10279207611917017, 'fraction_numerical': 0.004367493370768991, 'mean_word_length': 3.3062458025520485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1976608', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1903, 'n_words': 397}