<commit_before>from abc import ABCMeta, abstractmethod
from pathlib import Path
import json


class Stash(metaclass=ABCMeta):
    """Abstract base class for tweetstash backends."""
    @abstractmethod
    def is_stashed(self, tweet_id):
        pass

    @abstractmethod
    def stash(self, tweet):
        pass

    def stash_many(self, tweets):
        for tweet in tweets:
            self.stash(tweet)

    @abstractmethod
    def unstash(self, tweet_id):
        pass

    def unstash_many(self, tweet_ids):
        for tweet_id in tweet_ids:
            yield self.unstash(tweet_id)

    def unstash_all(self):
        yield from self.unstash_many(self.all_ids())

    @abstractmethod
    def all_ids(self):
        pass


class FileStash(Stash):
    """"Filesystem tweetstash backend.
    Tweets are stored as separate JSON files, with the id as filename.

    """
    def __init__(self, base_dir=None):
        self.base_dir = Path('.') if base_dir is None else Path(base_dir)

    def tweet_path(self, tweet_id):
        return self.base_dir / (tweet_id + '.json')

    def is_stashed(self, tweet_id):
        return self.tweet_path(tweet_id).exists()<commit_msg>check base_dir exists<commit_after>from abc import ABCMeta, abstractmethod
from pathlib import Path
import json


class Stash(metaclass=ABCMeta):
    """Abstract base class for tweetstash backends."""
    @abstractmethod
    def is_stashed(self, tweet_id):
        pass

    @abstractmethod
    def stash(self, tweet):
        pass

    def stash_many(self, tweets):
        for tweet in tweets:
            self.stash(tweet)

    @abstractmethod
    def unstash(self, tweet_id):
        pass

    def unstash_many(self, tweet_ids):
        for tweet_id in tweet_ids:
            yield self.unstash(tweet_id)

    def unstash_all(self):
        yield from self.unstash_many(self.all_ids())

    @abstractmethod
    def all_ids(self):
        pass


class FileStash(Stash):
    """"Filesystem tweetstash backend.
    Tweets are stored as separate JSON files, with the id as filename.

    """
    def __init__(self, base_dir=None, create_dir=False):
        self.base_dir = Path('.') if base_dir is None else Path(base_dir)
        if create_dir:
            self.base_dir.mkdir(parents=True, exist_ok=True)
        if not self.base_dir.exists():
            raise FileNotFoundError(base_dir)

    def tweet_path(self, tweet_id):
        return self.base_dir / (tweet_id + '.json')

    def is_stashed(self, tweet_id):
        return self.tweet_path(tweet_id).exists()
{'fraction_non_alphanumeric': 0.0917684127609295, 'fraction_numerical': 0.0, 'mean_word_length': 2.796711509715994, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1324218', 'n_tokens_mistral': 848, 'n_tokens_neox': 804, 'n_words': 211}