<commit_before>from __future__ import with_statement

import base64
import datetime
import pickle

import pytz
import rdflib
import redis

from django.conf import settings

from humfrey.update.transform.base import Transform
from humfrey.update.longliving.uploader import Uploader
from humfrey.sparql.endpoint import Endpoint
from humfrey.utils.namespaces import NS

class Upload(Transform):
    formats = {
        'rdf': 'xml',
        'n3': 'n3',
        'ttl': 'n3',
        'nt': 'nt',
    }

    created_query = """
        SELECT ?date WHERE {
          GRAPH %(graph)s {
            %(graph)s dcterms:created ?date
          }
        }
    """

    site_timezone = pytz.timezone(settings.TIME_ZONE)

    def __init__(self, graph_name, method='PUT', store=None, stores=None):
        self.graph_name = rdflib.URIRef(graph_name)
        self.method = method
        self.stores = stores or []
        if not stores:
            self.stores.append(store)

    def execute(self, transform_manager, input):
        for store in self.stores:
            if store:
                store = self.get_store(transform_manager, store, update=True)
                endpoint_query = store.query_endpoint
            else:
                endpoint_query = settings.ENDPOINT_QUERY

        transform_manager.start(self, [input])

        transform_manager.logger.debug("Starting upload of %r", input)

        client = redis.client.Redis(**settings.REDIS_PARAMS)

        extension = input.rsplit('.', 1)[-1]<commit_msg>Tidied up getting a redis client in upload transform.<commit_after>from __future__ import with_statement

import base64
import datetime
import pickle

import pytz
import rdflib
import redis

from django.conf import settings

from humfrey.update.transform.base import Transform
from humfrey.update.longliving.uploader import Uploader
from humfrey.sparql.endpoint import Endpoint
from humfrey.utils.namespaces import NS

class Upload(Transform):
    formats = {
        'rdf': 'xml',
        'n3': 'n3',
        'ttl': 'n3',
        'nt': 'nt',
    }

    created_query = """
        SELECT ?date WHERE {
          GRAPH %(graph)s {
            %(graph)s dcterms:created ?date
          }
        }
    """

    site_timezone = pytz.timezone(settings.TIME_ZONE)

    def __init__(self, graph_name, method='PUT', store=None, stores=None):
        self.graph_name = rdflib.URIRef(graph_name)
        self.method = method
        self.stores = stores or []
        if not stores:
            self.stores.append(store)

    def execute(self, transform_manager, input):
        for store in self.stores:
            if store:
                store = self.get_store(transform_manager, store, update=True)
                endpoint_query = store.query_endpoint
            else:
                endpoint_query = settings.ENDPOINT_QUERY

        transform_manager.start(self, [input])

        transform_manager.logger.debug("Starting upload of %r", input)

        client = self.get_redis_client()

        extension = input.rsplit('.', 1)[-1]
{'fraction_non_alphanumeric': 0.09750492449113592, 'fraction_numerical': 0.004596191726854892, 'mean_word_length': 2.478310502283105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2995568', 'n_tokens_mistral': 943, 'n_tokens_neox': 893, 'n_words': 219}