<commit_before>#!/usr/bin/python
#
# Copyright 2014 Rackspace Australia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""
Utility to upload folders to swift using the form post middleware
credentials provided by zuul
"""

import argparse
import magic
import os
import requests
import tempfile


def generate_log_index(file_list, logserver_prefix, swift_destination_prefix):
    """Create an index of logfiles and links to them"""

    output = '<html><head><title>Index of results</title></head><body>'
    output += '<ul>'
    for f in file_list:
        file_url = os.path.join(logserver_prefix, swift_destination_prefix,
                                f['filename'])

        output += '<li>'
        output += '<a href="%s">%s</a>' % (file_url, f['filename'])
        output += '</li>'

    output += '</ul>'
    output += '</body></html>'
    return output


def make_index_file(file_list, logserver_prefix, swift_destination_prefix,
                    index_filename='index.html'):
    """Writes an index into a file for pushing"""

    index_content = generate_log_index(file_list, logserver_prefix,
                                       swift_destination_prefix)
    tempdir = tempfile.mkdtemp()
    fd = open(os.path.join(tempdir, index_filename), 'w')
    fd.write(index_content)
    return os.path.join(tempdir, index_filename)


def get_file_mime(file_path):
    """Get the file mime using libmagic"""

    if not os.path.isfile(file_path):
        return None

    if hasattr(magic, 'from_file'):
        return magic.from_file(file_path, mime=True)
    else:
        # no magic.from_file, we might be using the libmagic bindings
        m = magic.open(magic.MIME)
        m.load()
        return m.file(file_path).split(';')[0]


def swift_form_post_submit(file_list, url, hmac_body, signature):
    """Send the files to swift via the FormPost middleware"""

    # We are uploading the file_list as an HTTP POST multipart encoded.
    # First grab out the information we need to send back from the hmac_body
    payload = {}

    (object_prefix,
     payload['redirect'],
     payload['max_file_size'],
     payload['max_file_count'],
     payload['expires']) = hmac_body.split('\n')
    payload['signature'] = signature

    if len(file_list) > payload['max_file_count']:
        # We can't upload this many files! We'll do what we can but the job
        # should be reconfigured
        file_list = file_list[:payload['max_file_count']]

    files = {}

    for i, f in enumerate(file_list):
        files['file%d' % (i + 1)] = (f['filename'], open(f['path'], 'rb'),
                                     get_file_mime(f['path']))

    requests.post(url, data=payload, files=files)


def zuul_swift_upload(file_path, swift_url, swift_hmac_body, swift_signature,
                      logserver_prefix, swift_destination_prefix):<commit_msg>Ensure there is a slash in the object prefix<commit_after>#!/usr/bin/python
#
# Copyright 2014 Rackspace Australia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""
Utility to upload folders to swift using the form post middleware
credentials provided by zuul
"""

import argparse
import magic
import os
import requests
import tempfile


def generate_log_index(file_list, logserver_prefix, swift_destination_prefix):
    """Create an index of logfiles and links to them"""

    output = '<html><head><title>Index of results</title></head><body>'
    output += '<ul>'
    for f in file_list:
        file_url = os.path.join(logserver_prefix, swift_destination_prefix,
                                f['filename'])

        output += '<li>'
        output += '<a href="%s">%s</a>' % (file_url, f['filename'])
        output += '</li>'

    output += '</ul>'
    output += '</body></html>'
    return output


def make_index_file(file_list, logserver_prefix, swift_destination_prefix,
                    index_filename='index.html'):
    """Writes an index into a file for pushing"""

    index_content = generate_log_index(file_list, logserver_prefix,
                                       swift_destination_prefix)
    tempdir = tempfile.mkdtemp()
    fd = open(os.path.join(tempdir, index_filename), 'w')
    fd.write(index_content)
    return os.path.join(tempdir, index_filename)


def get_file_mime(file_path):
    """Get the file mime using libmagic"""

    if not os.path.isfile(file_path):
        return None

    if hasattr(magic, 'from_file'):
        return magic.from_file(file_path, mime=True)
    else:
        # no magic.from_file, we might be using the libmagic bindings
        m = magic.open(magic.MIME)
        m.load()
        return m.file(file_path).split(';')[0]


def swift_form_post_submit(file_list, url, hmac_body, signature):
    """Send the files to swift via the FormPost middleware"""

    # We are uploading the file_list as an HTTP POST multipart encoded.
    # First grab out the information we need to send back from the hmac_body
    payload = {}

    (object_prefix,
     payload['redirect'],
     payload['max_file_size'],
     payload['max_file_count'],
     payload['expires']) = hmac_body.split('\n')
    payload['signature'] = signature

    if len(file_list) > payload['max_file_count']:
        # We can't upload this many files! We'll do what we can but the job
        # should be reconfigured
        file_list = file_list[:payload['max_file_count']]

    files = {}

    # Zuul's log path is generated without a tailing slash. As such the
    # object prefix does not contain a slash and the files would be
    # uploaded as 'prefix' + 'filename'. Assume we want the destination
    # url to look like a folder and make sure there's a slash between.
    filename_prefix = '/' if url[-1] != '/' else ''
    for i, f in enumerate(file_list):
        files['file%d' % (i + 1)] = (filename_prefix + f['filename'],
                                     open(f['path'], 'rb'),
                                     get_file_mime(f['path']))

    requests.post(url, data=payload, files=files)


def zuul_swift_upload(file_path, swift_url, swift_hmac_body, swift_signature,
                      logserver_prefix, swift_destination_prefix):
{'fraction_non_alphanumeric': 0.10780877611103322, 'fraction_numerical': 0.002944062806673209, 'mean_word_length': 3.0905963302752295, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4940144', 'n_tokens_mistral': 2112, 'n_tokens_neox': 2023, 'n_words': 664}