<commit_before>META_MAGIC = 'B'

import re
import os
import errno
import logging
from time import time
from .config import get_config


class Meta(object):

    def __init__(self):
        self.config = get_config()
        self.logger = logging.getLogger('katana.meta')

    def get(self, cache):
        try:
            with open('%s.meta' % cache, 'r') as cache_meta:
                splitted = cache_meta.read().split('|')
                if splitted[0] == META_MAGIC:
                    magic, timestamp, expires, last_modified, etag = splitted
                    return {
                        'timestamp': int(timestamp),
                        'expires': int(expires),
                        'last_modified': last_modified,
                        'etag': etag,
                    }
                else:
                    self.logger.error('Meta.get wrong magic [%s] for %s', splitted[0], cache)
                    os.remove('%s.meta' % cache)
        except (IOError, OSError) as exc:
            if exc.errno != errno.ENOENT:
                self.logger.error('Meta.get failed for %s: %s', cache, exc)
        return {}

    def set(self, cache, headers):
        try:
            with open('%s.meta' % cache, 'w') as cache_meta:
                timestamp = int(time())
                etag = headers.get('etag', '')
                last_modified = headers.get('last-modified', '')
                expires = self.config['cache_default_expires']
                if not self.config['cache_force_expires']:
                    m = re.match('.*max-age=(\d+).*', headers.get('cache-control', ''))
                    if m:
                        expires = int(m.group(1))
                cache_meta.write('%s|%s|%s|%s|%s' % (META_MAGIC, timestamp, expires, last_modified, etag))
                return {
                    'timestamp': timestamp,
                    'expires': expires,
                    'last_modified': last_modified,
                    'etag': etag,
                }
        except (IOError, OSError) as exc:
            self.logger.error('Meta.set failed for %s: %s', cache, exc)
        return {}
<commit_msg>use uppercase for meta files<commit_after>META_MAGIC = 'B'

import re
import os
import errno
import logging
from time import time
from .config import get_config


class Meta(object):

    def __init__(self):
        self.config = get_config()
        self.logger = logging.getLogger('katana.meta')

    def get(self, cache):
        try:
            with open('%s.META' % cache, 'r') as cache_meta:
                splitted = cache_meta.read().split('|')
                if splitted[0] == META_MAGIC:
                    magic, timestamp, expires, last_modified, etag = splitted
                    return {
                        'timestamp': int(timestamp),
                        'expires': int(expires),
                        'last_modified': last_modified,
                        'etag': etag,
                    }
                else:
                    self.logger.error('Meta.get wrong magic [%s] for %s', splitted[0], cache)
                    os.remove('%s.META' % cache)
        except (IOError, OSError) as exc:
            if exc.errno != errno.ENOENT:
                self.logger.error('Meta.get failed for %s: %s', cache, exc)
        return {}

    def set(self, cache, headers):
        try:
            with open('%s.META' % cache, 'w') as cache_meta:
                timestamp = int(time())
                etag = headers.get('etag', '')
                last_modified = headers.get('last-modified', '')
                expires = self.config['cache_default_expires']
                if not self.config['cache_force_expires']:
                    m = re.match('.*max-age=(\d+).*', headers.get('cache-control', ''))
                    if m:
                        expires = int(m.group(1))
                cache_meta.write('%s|%s|%s|%s|%s' % (META_MAGIC, timestamp, expires, last_modified, etag))
                return {
                    'timestamp': timestamp,
                    'expires': expires,
                    'last_modified': last_modified,
                    'etag': etag,
                }
        except (IOError, OSError) as exc:
            self.logger.error('Meta.set failed for %s: %s', cache, exc)
        return {}

{'fraction_non_alphanumeric': 0.12630117973629423, 'fraction_numerical': 0.0013879250520471894, 'mean_word_length': 1.4950952106174265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5953749', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1186, 'n_words': 284}