<commit_before>/*
 * Made with all the love in the world
 * by scireum in Remshalden, Germany
 *
 * Copyright by scireum GmbH
 * http://www.scireum.de - <EMAIL>
 */

package ninja;

import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponseStatus;
import sirius.kernel.async.CallContext;
import sirius.kernel.commons.Strings;
import sirius.kernel.commons.Value;
import sirius.kernel.di.std.Part;
import sirius.kernel.di.std.Register;
import sirius.kernel.health.Exceptions;
import sirius.kernel.health.HandledException;
import sirius.web.controller.Controller;
import sirius.web.controller.Routed;
import sirius.web.controller.UserContext;
import sirius.web.http.Response;
import sirius.web.http.WebContext;

import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Handles calls to the S3 API.
 *
 * @author <NAME> (<EMAIL>)
 * @since 2013/08
 */
@Register
public class S3Controller implements Controller {

    @Override
    public void onError(WebContext ctx, HandledException error) {
        signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, error.getMessage());
    }

    @Part
    private Storage storage;

    @Part
    private APILog log;

    private Map<String, ReentrantLock> locks = Maps.newConcurrentMap();

    /*
     * Computes the expected hash for the given request.
     */
    private String computeHash(WebContext ctx) {
        try {
            StringBuilder stringToSign = new StringBuilder(ctx.getRequest().getMethod().name());
            stringToSign.append("\n");
            stringToSign.append(ctx.getHeaderValue("Content-MD5").asString(""));
            stringToSign.append("\n");
            stringToSign.append(ctx.getHeaderValue("Content-Type").asString(""));
            stringToSign.append("\n");
            stringToSign.append(ctx.get("Expires")
                    .asString(ctx.getHeaderValue("x-amz-date")
                            .asString(ctx.getHeaderValue("Date").asString(""))));
            stringToSign.append("\n");

            List<String> headers = Lists.newArrayList();
            for (String name : ctx.getRequest().headers().names()) {
                if (name.toLowerCase().startsWith("x-amz-") && !"x-amz-date".equals(name.toLowerCase())) {
                    StringBuilder headerBuilder = new StringBuilder(name.toLowerCase().trim());
                    headerBuilder.append(":");
                    headerBuilder.append(Strings.join(ctx.getRequest().headers().getAll(name), ",").trim());
                    headers.add(headerBuilder.toString());
                }
            }
            Collections.sort(headers);
            for (String header : headers) {
                stringToSign.append(header);
                stringToSign.append("\n");
            }

            stringToSign.append(ctx.getRequestedURI().substring(3));

            SecretKeySpec keySpec = new SecretKeySpec(storage.getAwsSecretKey().getBytes(), "HmacSHA1");

            Mac mac = Mac.getInstance("HmacSHA1");
            mac.init(keySpec);
            byte[] result = mac.doFinal(stringToSign.toString().getBytes(Charsets.UTF_8.name()));
            return BaseEncoding.base64().encode(result);
        } catch (Throwable e) {
            throw Exceptions.handle(UserContext.LOG, e);
        }
    }

    private static final Pattern AWS_AUTH_PATTERN = Pattern.compile("AWS ([^:]+):(.*)");

    /*
     * Extracts the given hash from the given request. Returns null if no hash was given.
     */
    private String getAuthHash(WebContext ctx) {
        Value auth = ctx.getHeaderValue(HttpHeaders.Names.AUTHORIZATION);
        if (!auth.isFilled()) {
            return ctx.get("Signature").getString();
        }
        Matcher m = AWS_AUTH_PATTERN.matcher(auth.getString());
        if (m.matches()) {
            return m.group(2);
        }

        return null;
    }

    /*
     * Writes an API error to the log
     */
    private void signalObjectError(WebContext ctx, HttpResponseStatus status, String message) {
        ctx.respondWith().error(status, message);
        log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                message + " - " + ctx.getRequestedURI(),
                APILog.Result.ERROR,
                CallContext.getCurrent().getWatch());
    }

    /*
     * Writes an API success entry to the log
     */
    private void signalObjectSuccess(WebContext ctx) {
        log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                ctx.getRequestedURI(),
                APILog.Result.OK,
                CallContext.getCurrent().getWatch());
    }

    /**
     * Dispatching method handling all object specific calls.
     *
     * @param ctx        the context describing the current request
     * @param bucketName name of the bucket which contains the object (must exist)
     * @param id         name of the object ob interest
     * @throws Exception in case of IO errors and there like
     */
    @Routed("/s3/:1/:2")
    public void object(WebContext ctx, String bucketName, String id) throws Exception {
        Bucket bucket = storage.getBucket(bucketName);
        if (!bucket.exists()) {
            if (storage.isAutocreateBuckets()) {
                bucket.create();
            } else {
                signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Bucket does not exist");
                return;
            }
        }
        String hash = getAuthHash(ctx);
        if (hash != null) {
            if (!computeHash(ctx).equals(hash)) {
                ctx.respondWith().error(HttpResponseStatus.UNAUTHORIZED, "Invalid Hash");
                log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                        ctx.getRequestedURI(),
                        APILog.Result.REJECTED,
                        CallContext.getCurrent().getWatch());
                return;
            }
        }
        if (bucket.isPrivate() && !ctx.get("noAuth").isFilled() && hash == null) {
            ctx.respondWith().error(HttpResponseStatus.UNAUTHORIZED, "Authentication required");
            log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                    ctx.getRequestedURI(),
                    APILog.Result.REJECTED,
                    CallContext.getCurrent().getWatch());
            return;
        }
        if (ctx.getRequest().getMethod() == HttpMethod.GET) {
            getObject(ctx, bucket, id, true);
        } else if (ctx.getRequest().getMethod() == HttpMethod.PUT) {
            Value copy = ctx.getHeaderValue("x-amz-copy-source");
            if (copy.isFilled()) {
                copyObject(ctx, bucket, id, copy.asString());
            } else {
                putObject(ctx, bucket, id);
            }
        } else if (ctx.getRequest().getMethod() == HttpMethod.DELETE) {
            deleteObject(ctx, bucket, id);
        } else if (ctx.getRequest().getMethod() == HttpMethod.HEAD) {
            getObject(ctx, bucket, id, false);
        } else {
            throw new IllegalArgumentException(ctx.getRequest().getMethod().name());
        }
    }

    /**
     * Handles DELETE /bucket/id
     *
     * @param ctx    the context describing the current request
     * @param bucket the bucket containing the object to delete
     * @param id     name of the object to delete
     */

    private void deleteObject(WebContext ctx, Bucket bucket, String id) {
        StoredObject object = bucket.getObject(id);
        object.delete();

        ctx.respondWith().status(HttpResponseStatus.OK);
        signalObjectSuccess(ctx);
    }

    /**
     * Handles PUT /bucket/id
     *
     * @param ctx    the context describing the current request
     * @param bucket the bucket containing the object to upload
     * @param id     name of the object to upload
     */
    private void putObject(WebContext ctx, Bucket bucket, String id) throws Exception {
        StoredObject object = bucket.getObject(id);
        InputStream inputStream = ctx.getContent();
        if (inputStream == null) {
            signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, "No content posted");
            return;
        }
        try {
            FileOutputStream out = new FileOutputStream(object.getFile());
            try {
                ByteStreams.copy(inputStream, out);
            } finally {
                out.close();
            }
        } finally {
            inputStream.close();
        }

        Map<String, String> properties = Maps.newTreeMap();
        for (String name : ctx.getRequest().headers().names()) {
            String nameLower = name.toLowerCase();
            if (nameLower.startsWith("x-amz-meta-") || nameLower.equals("content-md5") || nameLower.equals(
                    "content-type") || nameLower.equals("x-amz-acl")) {
                properties.put(name, ctx.getHeader(name));
            }
        }
        HashCode hash = Files.hash(object.getFile(), Hashing.md5());
        String md5 = BaseEncoding.base64().encode(hash.asBytes());
        if (properties.containsKey("Content-MD5")) {
            if (!md5.equals(properties.get("Content-MD5"))) {
                object.delete();
                signalObjectError(ctx,
                        HttpResponseStatus.BAD_REQUEST,
                        Strings.apply("Invalid MD5 checksum (Input: %s, Expected: %s)",
                                properties.get("Content-MD5"),
                                md5));
                return;
            }
        }

        object.storeProperties(properties);
        String etag = hash.toString();
        ctx.respondWith().addHeader(HttpHeaders.Names.ETAG, etag).status(HttpResponseStatus.OK);
        signalObjectSuccess(ctx);
    }

    /**
     * Handles GET /bucket/id with an <tt>x-amz-copy-source</tt> header.
     *
     * @param ctx    the context describing the current request
     * @param bucket the bucket containing the object to use as destination
     * @param id     name of the object to use as destination
     */
    private void copyObject(WebContext ctx, Bucket bucket, String id, String copy) throws IOException {
        StoredObject object = bucket.getObject(id);
        if (!object.exists()) {
            signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Object does not exist");
            return;
        }
        if (!copy.contains("/")) {
            signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, "Source must contain '/'");
            return;
        }
        String srcBucketName = copy.substring(0, copy.lastIndexOf("/"));
        String srcId = copy.substring(copy.lastIndexOf("/"));
        Bucket srcBucket = storage.getBucket(srcBucketName);
        if (!srcBucket.exists()) {
            signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, "Source bucket does not exist");
            return;
        }
        StoredObject src = srcBucket.getObject(srcId);
        if (!src.exists()) {
            signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, "Source object does not exist");
            return;
        }
        Files.copy(src.getFile(), object.getFile());
        if (src.getPropertiesFile().exists()) {
            Files.copy(src.getPropertiesFile(), object.getPropertiesFile());
        }
        ctx.respondWith().status(HttpResponseStatus.OK);
        signalObjectSuccess(ctx);
    }

    /**
     * Handles GET /bucket/id
     *
     * @param ctx    the context describing the current request
     * @param bucket the bucket containing the object to download
     * @param id     name of the object to use as download
     */
    private void getObject(WebContext ctx, Bucket bucket, String id, boolean sendFile) throws Exception {
        StoredObject object = bucket.getObject(id);
        if (!object.exists()) {
            signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Object does not exist");
            return;
        }
        Response response = ctx.respondWith();
        for (Map.Entry<Object, Object> entry : object.getProperties()) {
            response.addHeader(entry.getKey().toString(), entry.getValue().toString());
        }
        if (sendFile) {
            response.file(object.getFile());
        } else {
            response.status(HttpResponseStatus.OK);
        }
        signalObjectSuccess(ctx);
    }
}
<commit_msg>the current version of the s3 client from amazon sends path names in a different way than the version we used before. When you set the base-url to http://localhost:9444/s3 the url used by the client to sign a request will not be /<bucketname> but /s3/<bucketname> which broke the old way of hash computation. This commit fixes that behaviour by allowing both variants when the hash is computed in the s3ninja.<commit_after>/*
 * Made with all the love in the world
 * by scireum in Remshalden, Germany
 *
 * Copyright by scireum GmbH
 * http://www.scireum.de - <EMAIL>
 */

package ninja;

import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponseStatus;
import sirius.kernel.async.CallContext;
import sirius.kernel.commons.Strings;
import sirius.kernel.commons.Value;
import sirius.kernel.di.std.Part;
import sirius.kernel.di.std.Register;
import sirius.kernel.health.Exceptions;
import sirius.kernel.health.HandledException;
import sirius.web.controller.Controller;
import sirius.web.controller.Routed;
import sirius.web.controller.UserContext;
import sirius.web.http.Response;
import sirius.web.http.WebContext;

import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Handles calls to the S3 API.
 *
 * @author <NAME> (<EMAIL>.de)
 * @since 2013/08
 */
@Register
public class S3Controller implements Controller {

    @Override
    public void onError(WebContext ctx, HandledException error) {
        signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, error.getMessage());
    }

    @Part
    private Storage storage;

    @Part
    private APILog log;

    private Map<String, ReentrantLock> locks = Maps.newConcurrentMap();

    /*
     * Computes the expected hash for the given request.
     */
    private String computeHash(WebContext ctx, String pathPrefix) {
        try {
            StringBuilder stringToSign = new StringBuilder(ctx.getRequest().getMethod().name());
            stringToSign.append("\n");
            stringToSign.append(ctx.getHeaderValue("Content-MD5").asString(""));
            stringToSign.append("\n");
            stringToSign.append(ctx.getHeaderValue("Content-Type").asString(""));
            stringToSign.append("\n");
            stringToSign.append(ctx.get("Expires")
                    .asString(ctx.getHeaderValue("x-amz-date")
                            .asString(ctx.getHeaderValue("Date").asString(""))));
            stringToSign.append("\n");

            List<String> headers = Lists.newArrayList();
            for (String name : ctx.getRequest().headers().names()) {
                if (name.toLowerCase().startsWith("x-amz-") && !"x-amz-date".equals(name.toLowerCase())) {
                    StringBuilder headerBuilder = new StringBuilder(name.toLowerCase().trim());
                    headerBuilder.append(":");
                    headerBuilder.append(Strings.join(ctx.getRequest().headers().getAll(name), ",").trim());
                    headers.add(headerBuilder.toString());
                }
            }
            Collections.sort(headers);
            for (String header : headers) {
                stringToSign.append(header);
                stringToSign.append("\n");
            }

            stringToSign.append(pathPrefix).append(ctx.getRequestedURI().substring(3));

            SecretKeySpec keySpec = new SecretKeySpec(storage.getAwsSecretKey().getBytes(), "HmacSHA1");

            Mac mac = Mac.getInstance("HmacSHA1");
            mac.init(keySpec);
            byte[] result = mac.doFinal(stringToSign.toString().getBytes(Charsets.UTF_8.name()));
            return BaseEncoding.base64().encode(result);
        } catch (Throwable e) {
            throw Exceptions.handle(UserContext.LOG, e);
        }
    }

    private static final Pattern AWS_AUTH_PATTERN = Pattern.compile("AWS ([^:]+):(.*)");

    /*
     * Extracts the given hash from the given request. Returns null if no hash was given.
     */
    private String getAuthHash(WebContext ctx) {
        Value auth = ctx.getHeaderValue(HttpHeaders.Names.AUTHORIZATION);
        if (!auth.isFilled()) {
            return ctx.get("Signature").getString();
        }
        Matcher m = AWS_AUTH_PATTERN.matcher(auth.getString());
        if (m.matches()) {
            return m.group(2);
        }

        return null;
    }

    /*
     * Writes an API error to the log
     */
    private void signalObjectError(WebContext ctx, HttpResponseStatus status, String message) {
        ctx.respondWith().error(status, message);
        log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                message + " - " + ctx.getRequestedURI(),
                APILog.Result.ERROR,
                CallContext.getCurrent().getWatch());
    }

    /*
     * Writes an API success entry to the log
     */
    private void signalObjectSuccess(WebContext ctx) {
        log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                ctx.getRequestedURI(),
                APILog.Result.OK,
                CallContext.getCurrent().getWatch());
    }

    /**
     * Dispatching method handling all object specific calls.
     *
     * @param ctx        the context describing the current request
     * @param bucketName name of the bucket which contains the object (must exist)
     * @param id         name of the object ob interest
     * @throws Exception in case of IO errors and there like
     */
    @Routed("/s3/:1/:2")
    public void object(WebContext ctx, String bucketName, String id) throws Exception {
        Bucket bucket = storage.getBucket(bucketName);
        if (!bucket.exists()) {
            if (storage.isAutocreateBuckets()) {
                bucket.create();
            } else {
                signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Bucket does not exist");
                return;
            }
        }
        String hash = getAuthHash(ctx);
        if (hash != null) {
            if (!computeHash(ctx, "/s3").equals(hash) && !computeHash(ctx, "").equals(hash)) {
                ctx.respondWith().error(HttpResponseStatus.UNAUTHORIZED, "Invalid Hash");
                log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                        ctx.getRequestedURI(),
                        APILog.Result.REJECTED,
                        CallContext.getCurrent().getWatch());
                return;
            }
        }
        if (bucket.isPrivate() && !ctx.get("noAuth").isFilled() && hash == null) {
            ctx.respondWith().error(HttpResponseStatus.UNAUTHORIZED, "Authentication required");
            log.log("OBJECT " + ctx.getRequest().getMethod().name(),
                    ctx.getRequestedURI(),
                    APILog.Result.REJECTED,
                    CallContext.getCurrent().getWatch());
            return;
        }
        if (ctx.getRequest().getMethod() == HttpMethod.GET) {
            getObject(ctx, bucket, id, true);
        } else if (ctx.getRequest().getMethod() == HttpMethod.PUT) {
            Value copy = ctx.getHeaderValue("x-amz-copy-source");
            if (copy.isFilled()) {
                copyObject(ctx, bucket, id, copy.asString());
            } else {
                putObject(ctx, bucket, id);
            }
        } else if (ctx.getRequest().getMethod() == HttpMethod.DELETE) {
            deleteObject(ctx, bucket, id);
        } else if (ctx.getRequest().getMethod() == HttpMethod.HEAD) {
            getObject(ctx, bucket, id, false);
        } else {
            throw new IllegalArgumentException(ctx.getRequest().getMethod().name());
        }
    }

    /**
     * Handles DELETE /bucket/id
     *
     * @param ctx    the context describing the current request
     * @param bucket the bucket containing the object to delete
     * @param id     name of the object to delete
     */

    private void deleteObject(WebContext ctx, Bucket bucket, String id) {
        StoredObject object = bucket.getObject(id);
        object.delete();

        ctx.respondWith().status(HttpResponseStatus.OK);
        signalObjectSuccess(ctx);
    }

    /**
     * Handles PUT /bucket/id
     *
     * @param ctx    the context describing the current request
     * @param bucket the bucket containing the object to upload
     * @param id     name of the object to upload
     */
    private void putObject(WebContext ctx, Bucket bucket, String id) throws Exception {
        StoredObject object = bucket.getObject(id);
        InputStream inputStream = ctx.getContent();
        if (inputStream == null) {
            signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, "No content posted");
            return;
        }
        try {
            FileOutputStream out = new FileOutputStream(object.getFile());
            try {
                ByteStreams.copy(inputStream, out);
            } finally {
                out.close();
            }
        } finally {
            inputStream.close();
        }

        Map<String, String> properties = Maps.newTreeMap();
        for (String name : ctx.getRequest().headers().names()) {
            String nameLower = name.toLowerCase();
            if (nameLower.startsWith("x-amz-meta-") || nameLower.equals("content-md5") || nameLower.equals(
                    "content-type") || nameLower.equals("x-amz-acl")) {
                properties.put(name, ctx.getHeader(name));
            }
        }
        HashCode hash = Files.hash(object.getFile(), Hashing.md5());
        String md5 = BaseEncoding.base64().encode(hash.asBytes());
        if (properties.containsKey("Content-MD5")) {
            if (!md5.equals(properties.get("Content-MD5"))) {
                object.delete();
                signalObjectError(ctx,
                        HttpResponseStatus.BAD_REQUEST,
                        Strings.apply("Invalid MD5 checksum (Input: %s, Expected: %s)",
                                properties.get("Content-MD5"),
                                md5));
                return;
            }
        }

        object.storeProperties(properties);
        String etag = hash.toString();
        ctx.respondWith().addHeader(HttpHeaders.Names.ETAG, etag).status(HttpResponseStatus.OK);
        signalObjectSuccess(ctx);
    }

    /**
     * Handles GET /bucket/id with an <tt>x-amz-copy-source</tt> header.
     *
     * @param ctx    the context describing the current request
     * @param bucket the bucket containing the object to use as destination
     * @param id     name of the object to use as destination
     */
    private void copyObject(WebContext ctx, Bucket bucket, String id, String copy) throws IOException {
        StoredObject object = bucket.getObject(id);
        if (!object.exists()) {
            signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Object does not exist");
            return;
        }
        if (!copy.contains("/")) {
            signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, "Source must contain '/'");
            return;
        }
        String srcBucketName = copy.substring(0, copy.lastIndexOf("/"));
        String srcId = copy.substring(copy.lastIndexOf("/"));
        Bucket srcBucket = storage.getBucket(srcBucketName);
        if (!srcBucket.exists()) {
            signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, "Source bucket does not exist");
            return;
        }
        StoredObject src = srcBucket.getObject(srcId);
        if (!src.exists()) {
            signalObjectError(ctx, HttpResponseStatus.BAD_REQUEST, "Source object does not exist");
            return;
        }
        Files.copy(src.getFile(), object.getFile());
        if (src.getPropertiesFile().exists()) {
            Files.copy(src.getPropertiesFile(), object.getPropertiesFile());
        }
        ctx.respondWith().status(HttpResponseStatus.OK);
        signalObjectSuccess(ctx);
    }

    /**
     * Handles GET /bucket/id
     *
     * @param ctx    the context describing the current request
     * @param bucket the bucket containing the object to download
     * @param id     name of the object to use as download
     */
    private void getObject(WebContext ctx, Bucket bucket, String id, boolean sendFile) throws Exception {
        StoredObject object = bucket.getObject(id);
        if (!object.exists()) {
            signalObjectError(ctx, HttpResponseStatus.NOT_FOUND, "Object does not exist");
            return;
        }
        Response response = ctx.respondWith();
        for (Map.Entry<Object, Object> entry : object.getProperties()) {
            response.addHeader(entry.getKey().toString(), entry.getValue().toString());
        }
        if (sendFile) {
            response.file(object.getFile());
        } else {
            response.status(HttpResponseStatus.OK);
        }
        signalObjectSuccess(ctx);
    }
}

{'fraction_non_alphanumeric': 0.12014535543947309, 'fraction_numerical': 0.002687561511090923, 'mean_word_length': 2.2575832305795314, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5188224', 'n_tokens_mistral': 7153, 'n_tokens_neox': 7011, 'n_words': 1604}