<commit_before>"""Parse generic posts from a csv."""

import csv
import datetime
from dateutil import parser
import io
import uuid

import mediawords.util.url
import mediawords.util.web.user_agent

from mediawords.util.log import create_logger

from topics_base.posts import get_mock_data, filter_posts_for_date_range
from topics_mine.posts import AbstractPostFetcher

log = create_logger(__name__)


class McPostsGenericDataException(Exception):
    """exception indicating an error in the data for generic posts."""
    pass


class CSVStaticPostFetcher(AbstractPostFetcher):

    @staticmethod
    def _get_csv_string_from_dicts(dicts: list) -> str:
        """Given a list of dicts, return a representative csv string."""
        if len(dicts) < 1:
            return ''

        csv_io = io.StringIO()

        csv_writer = csv.DictWriter(csv_io, fieldnames=dicts[0].keys())

        csv_writer.writeheader()
        [csv_writer.writerow(d) for d in dicts]

        return csv_io.getvalue()

    @staticmethod
    def _get_dicts_from_csv_string(csv_string: str) -> list:
        """Given a csv string, return a list of dicts."""
        if len(csv_string) < 1:
            return []

        csv_io = io.StringIO(csv_string)

        return list(csv.DictReader(csv_io))

    def fetch_posts_from_api(self, query: str, start_date: datetime, end_date: datetime) -> list:
        """Return posts from a csv that are within the given date range."""
        if self.mock_enabled:
            query = self._get_csv_string_from_dicts(get_mock_data())

        if mediawords.util.url.is_http_url(query):
            ua = mediawords.util.web.user_agent.UserAgent()
            ua.set_max_size(1024 * 1024 * 1024)
            response = ua.get(query)
            if not response.is_success():
                raise McPostsGenericDataException("Unable to fetch query url: %s" % response.as_string())

            query = response.decoded_content()

        all_posts = self._get_dicts_from_csv_string(query)

        required_fields = ['content', 'author', 'publish_date']
        for post in all_posts:
            for field in required_fields:
                if field not in post:
                    raise McPostsGenericDataException(f"Missing required field: {field}")<commit_msg>add caching to csv_generic http fetch<commit_after>"""Parse generic posts from a csv."""

import csv
import datetime
from dateutil import parser
import io
import shelve
import uuid

import mediawords.util.url
import mediawords.util.web.user_agent

from mediawords.util.log import create_logger

from topics_base.posts import get_mock_data, filter_posts_for_date_range
from topics_mine.posts import AbstractPostFetcher

log = create_logger(__name__)

# http request cache
SHELVE_CACHE_FILE="/tmp/csv_generic_shelf_cache.dbm"


class McPostsGenericDataException(Exception):
    """exception indicating an error in the data for generic posts."""
    pass


class CSVStaticPostFetcher(AbstractPostFetcher):

    @staticmethod
    def _get_csv_string_from_dicts(dicts: list) -> str:
        """Given a list of dicts, return a representative csv string."""
        if len(dicts) < 1:
            return ''

        csv_io = io.StringIO()

        csv_writer = csv.DictWriter(csv_io, fieldnames=dicts[0].keys())

        csv_writer.writeheader()
        [csv_writer.writerow(d) for d in dicts]

        return csv_io.getvalue()

    @staticmethod
    def _get_dicts_from_csv_string(csv_string: str) -> list:
        """Given a csv string, return a list of dicts."""
        if len(csv_string) < 1:
            return []

        csv_io = io.StringIO(csv_string)

        return list(csv.DictReader(csv_io))

    def _fetch_csv_from_url_query(self, query: str) -> str:
        """If the query is a url, fetch it and use that as the url.

        Cache the url fetch so that we don't refetch the same url for many days.
        """
        if not  mediawords.util.url.is_http_url(query):
            return query 

        url = query
        cache = shelve.open(filename=SHELVE_CACHE_FILE, writeback=True)
        if url in cache:
            log.warning('CACHE HIT')
            query = cache[url]
        else:
            log.warning('CACHE MISS')
            ua = mediawords.util.web.user_agent.UserAgent()
            ua.set_max_size(1024 * 1024 * 1024)
            response = ua.get(url)
            if not response.is_success():
                raise McPostsGenericDataException("Unable to fetch query url: %s" % response.as_string())

            query = response.decoded_content()

            cache[url] = query
            cache.sync()
            cache.close()

        return query

    def fetch_posts_from_api(self, query: str, start_date: datetime, end_date: datetime) -> list:
        """Return posts from a csv that are within the given date range."""
        if self.mock_enabled:
            query = self._get_csv_string_from_dicts(get_mock_data())

        query = self._fetch_csv_from_url_query(query)

        all_posts = self._get_dicts_from_csv_string(query)

        required_fields = ['content', 'author', 'publish_date']
        for post in all_posts:
            for field in required_fields:
                if field not in post:
                    raise McPostsGenericDataException(f"Missing required field: {field}")
{'fraction_non_alphanumeric': 0.08796992481203007, 'fraction_numerical': 0.005639097744360902, 'mean_word_length': 2.734035087719298, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '255255', 'n_tokens_mistral': 1632, 'n_tokens_neox': 1529, 'n_words': 425}