<commit_before>#!/usr/bin/env python3

# Copyright 2015, MIT license, github.com/tedder42.
# You know what the MIT license is, follow it.

# todo:
# - encrypt and "stream" file to s3/boto3, no intermediate file.
# - ensure python2 compatability (if anyone cares)
# - integrate into boto3

import base64
import json
from Crypto.Cipher import AES # pycryptodome
from Crypto import Random
import boto3
import struct
import sys
import os

if len(sys.argv) != 5:
  print("usage: put.py local_file bucket s3_key kms_arn")
  sys.exit(-1)

infile = sys.argv[1]
bucket_name = sys.argv[2]
key_name = sys.argv[3]
kms_arn = sys.argv[4]

# generating this "encrypt and put" code by:
# (a) reversing the decrypt, which I know works
# (b) black-boxing output (metadata) to match the Java SDK
# (c) using this: https://github.com/aws/aws-sdk-ruby/blob/master/aws-sdk-resources/lib/aws-sdk-resources/services/s3/encryption/kms_cipher_provider.rb#L16

# decrypt_file method from: http://eli.thegreenplace.net/2010/06/25/aes-encryption-of-files-in-python-with-pycrypto
# via: https://github.com/boto/boto3/issues/38#issuecomment-174106849
def encrypt_file(key, in_filename, iv, original_size, out_filename, chunksize=16*1024):
    with open(in_filename, 'rb') as infile:
        cipher = AES.new(key, AES.MODE_CBC, iv)

        with open(out_filename, 'wb') as outfile:
            while True:
                chunk = infile.read(chunksize)
                if len(chunk) == 0:
                    break
                elif len(chunk) % 16 != 0:
                    length = 16 - (len(chunk) % 16)
                    # not py2 compatible
                    #chunk += bytes([length])*length
                    chunk += struct.pack('B', length)*length
                outfile.write(cipher.encrypt(chunk))

def put_file(ciphertext_blob, new_iv, encrypt_ctx, upload_filename, unencrypted_file_size, bucket_name, key_name):

    matdesc_string = json.dumps(encrypt_ctx)
    metadata = {
        'x-amz-key-v2': base64.b64encode(ciphertext_blob).decode('utf-8'),
        'x-amz-iv': base64.b64encode(new_iv).decode('utf-8'),
        'x-amz-cek-alg': 'AES/CBC/PKCS5Padding',
        'x-amz-wrap-alg': 'kms',
        'x-amz-matdesc': matdesc_string,
        'x-amz-unencrypted-content-length': str(unencrypted_file_size)
    }

    s3client = boto3.client('s3')<commit_msg>"fix encryption padding on % 16 files<commit_after>#!/usr/bin/env python3

# Copyright 2015, MIT license, github.com/tedder42.
# You know what the MIT license is, follow it.

# todo:
# - encrypt and "stream" file to s3/boto3, no intermediate file.
# - ensure python2 compatability (if anyone cares)
# - integrate into boto3

import base64
import json
from Crypto.Cipher import AES # pycryptodome
from Crypto import Random
import boto3
import struct
import sys
import os

if len(sys.argv) != 5:
  print("usage: put.py local_file bucket s3_key kms_arn")
  sys.exit(-1)

infile = sys.argv[1]
bucket_name = sys.argv[2]
key_name = sys.argv[3]
kms_arn = sys.argv[4]

# generating this "encrypt and put" code by:
# (a) reversing the decrypt, which I know works
# (b) black-boxing output (metadata) to match the Java SDK
# (c) using this: https://github.com/aws/aws-sdk-ruby/blob/master/aws-sdk-resources/lib/aws-sdk-resources/services/s3/encryption/kms_cipher_provider.rb#L16

# decrypt_file method from: http://eli.thegreenplace.net/2010/06/25/aes-encryption-of-files-in-python-with-pycrypto
# via: https://github.com/boto/boto3/issues/38#issuecomment-174106849
def encrypt_file(key, in_filename, iv, original_size, out_filename, chunksize=16*1024):
    with open(in_filename, 'rb') as infile:
        cipher = AES.new(key, AES.MODE_CBC, iv)

        # 3 cases here for padding at the end of file:
        # - we get a full chunk of 16. pass it through.
        # - we get a partial chunk at EOF. we pad it up to 16. generally speaking each byte is the byte number, so if we have 7 bytes, the following nine are "07 07 07 07 07 07 07 07 07".
        # - we get a zero-byte chunk at EOF. This means the file was a perfect multiple of 16, but padding means the end of the file should be padded because IDK why but that's how it's done. See url below:
        #
        # the extra padding at zero-byte EOF: http://security.stackexchange.com/a/29997
        #   "The above problem is solved by knowing that you always pad your data, no matter the length."
        with open(out_filename, 'wb') as outfile:
            last_chunk_length = 0
            while True:
                chunk = infile.read(chunksize)
                last_chunk_length = len(chunk)
                if last_chunk_length == 0 or last_chunk_length < chunksize:
                  break
                outfile.write(cipher.encrypt(chunk))

            # write the final padding
            length_to_pad = 16 - (last_chunk_length % 16)
            # not py2 compatible
            # chunk += bytes([length])*length
            chunk += struct.pack('B', length_to_pad) * length_to_pad
            outfile.write(cipher.encrypt(chunk))


def put_file(ciphertext_blob, new_iv, encrypt_ctx, upload_filename, unencrypted_file_size, bucket_name, key_name):

    matdesc_string = json.dumps(encrypt_ctx)
    metadata = {
        'x-amz-key-v2': base64.b64encode(ciphertext_blob).decode('utf-8'),
        'x-amz-iv': base64.b64encode(new_iv).decode('utf-8'),
        'x-amz-cek-alg': 'AES/CBC/PKCS5Padding',
        'x-amz-wrap-alg': 'kms',
        'x-amz-matdesc': matdesc_string,
        'x-amz-unencrypted-content-length': str(unencrypted_file_size)
    }

    s3client = boto3.client('s3')
{'fraction_non_alphanumeric': 0.11543959971408149, 'fraction_numerical': 0.031987133666904935, 'mean_word_length': 2.952683615819209, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3963726', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1849, 'n_words': 494}