<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

# This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

import cPickle
import getpass
import optparse
import os
import re
import subprocess
import sys
import tempfile
import time

# archivematicaCommon
from custom_handlers import get_script_logger
logger = get_script_logger("archivematica.upload.qubit",
                              logfile="/var/log/archivematica/MCPClient/atom_upload.log")

# externals
import requests

import django
django.setup()
# dashboard
import main.models as models

PREFIX = "[uploadDIP]"

# Colorize output
def hilite(string, status=True):
    if not os.isatty(sys.stdout.fileno()):
        return string
    attr = []
    if status:
        attr.append('32')
    else:
        attr.append('31')
    return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), string)

# Print to stdout
def log(message, access=None):
    logger.error("%s %s" % (PREFIX, hilite(message)))
    if access:
        access.status = message
        access.save()

# Print to stderr and exit
def error(message, code=1):
    print >>sys.stderr, "%s %s" % (PREFIX, hilite(message, False))
    sys.exit(1)

# Make sure that archivematica user is executing this script
user = getpass.getuser()
if "archivematica" != user:
    error('This user is required to be executed as "archivematica" user but you are using %s.' % user)

def start(data):
    # Make sure we are working with an existing SIP record
    try:
      sip = models.SIP.objects.get(pk=data.uuid)
    except:
      error("UUID not recognized")

    # Get directory
    jobs = models.Job.objects.filter(sipuuid=data.uuid, jobtype="Upload DIP")
    if jobs.count():
        directory = jobs[0].directory.rstrip('/').replace('%sharedPath%', '/var/archivematica/sharedDirectory/')
    else:
        error("Directory not found: %s" % directory)

    # Check if exists
    if os.path.exists(directory) is False:
        log("Directory not found: %s" % directory)

        # Trying with uploadedDIPs
        log("Looking up uploadedDIPs/")
        directory = directory.replace('uploadDIP', 'uploadedDIPs')

        if os.path.exists(directory) is False:
          error("Directory not found: %s" % directory)

    try:
        # This upload was called before, restore Access record
        access = models.Access.objects.get(sipuuid=data.uuid)
    except:
        # First time this job is called, create new Access record
        access = models.Access(sipuuid=data.uuid)
        access.save()

    # The target columns contents a serialized Python dictionary
    # - target is the permalink string
    try:
        target = cPickle.loads(str(access.target))
        log("Target: %s" % (target['target']))
    except:
        error("No target was selected")

    # Rsync if data.rsync_target option was passed to this script
    if data.rsync_target:
        """ Build command (rsync)
          -a =
            -r = recursive
            -l = recreate symlinks on destination
            -p = set same permissions
            -t = transfer modification times
            -g = set same group owner on destination
            -o = set same user owner on destination (if possible, super-user)
            --devices = transfer character and block device files (only super-user)
            --specials = transfer special files like sockets and fifos
          -z = compress
          -P = --partial + --stats
        """
        # Using rsync -rltzP
        command = ["rsync", "--protect-args", "-rltz", "-P", "--chmod=ugo=rwX", directory, data.rsync_target]

        # Add -e if data.rsync_command was passed to this script
        if data.rsync_command:
            # Insert in second position. Example: rsync -e "ssh -i key" ...
            command.insert(1, "-e \"%s\"" % data.rsync_command)

        log(' '.join(command))

        # Getting around of rsync output buffering by outputting to a temporary file
        pipe_output, file_name = tempfile.mkstemp()
        log("Rsync output is being saved in %s" % file_name)

        # Call Rsync
        process = subprocess.Popen(command, stdout=pipe_output, stderr=pipe_output)

        # poll() returns None while the process is still running
        while process.poll() is None:
            time.sleep(1)
            last_line = open(file_name).readlines()

            # It's possible that it hasn't output yet, so continue
            if len(last_line) == 0:
                continue
            last_line = last_line[-1]

            # Matching to "[bytes downloaded]  number%  [speed] number:number:number"
            match = re.match(".* ([0-9]*)%.* ([0-9]*:[0-9]*:[0-9]*).*", last_line)

            if not match:
                continue

            # Update upload status
            # - percentage in match.group(1)
            # - ETA in match.group(2)
            access.status = "Sending... %s (ETA: %s)" % (match.group(1), match.group(2))<commit_msg>AtoM DIP upload --rsync-command fix ref. #9095<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

# This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

import cPickle
import getpass
import optparse
import os
import re
import subprocess
import sys
import tempfile
import time

# archivematicaCommon
from custom_handlers import get_script_logger
logger = get_script_logger("archivematica.upload.qubit",
                              logfile="/var/log/archivematica/MCPClient/atom_upload.log")

# externals
import requests

import django
django.setup()
# dashboard
import main.models as models

PREFIX = "[uploadDIP]"

# Colorize output
def hilite(string, status=True):
    if not os.isatty(sys.stdout.fileno()):
        return string
    attr = []
    if status:
        attr.append('32')
    else:
        attr.append('31')
    return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), string)

# Print to stdout
def log(message, access=None):
    logger.error("%s %s" % (PREFIX, hilite(message)))
    if access:
        access.status = message
        access.save()

# Print to stderr and exit
def error(message, code=1):
    print >>sys.stderr, "%s %s" % (PREFIX, hilite(message, False))
    sys.exit(1)

# Make sure that archivematica user is executing this script
user = getpass.getuser()
if "archivematica" != user:
    error('This user is required to be executed as "archivematica" user but you are using %s.' % user)

def start(data):
    # Make sure we are working with an existing SIP record
    try:
      sip = models.SIP.objects.get(pk=data.uuid)
    except:
      error("UUID not recognized")

    # Get directory
    jobs = models.Job.objects.filter(sipuuid=data.uuid, jobtype="Upload DIP")
    if jobs.count():
        directory = jobs[0].directory.rstrip('/').replace('%sharedPath%', '/var/archivematica/sharedDirectory/')
    else:
        error("Directory not found: %s" % directory)

    # Check if exists
    if os.path.exists(directory) is False:
        log("Directory not found: %s" % directory)

        # Trying with uploadedDIPs
        log("Looking up uploadedDIPs/")
        directory = directory.replace('uploadDIP', 'uploadedDIPs')

        if os.path.exists(directory) is False:
          error("Directory not found: %s" % directory)

    try:
        # This upload was called before, restore Access record
        access = models.Access.objects.get(sipuuid=data.uuid)
    except:
        # First time this job is called, create new Access record
        access = models.Access(sipuuid=data.uuid)
        access.save()

    # The target columns contents a serialized Python dictionary
    # - target is the permalink string
    try:
        target = cPickle.loads(str(access.target))
        log("Target: %s" % (target['target']))
    except:
        error("No target was selected")

    # Rsync if data.rsync_target option was passed to this script
    if data.rsync_target:
        """ Build command (rsync)
          -a =
            -r = recursive
            -l = recreate symlinks on destination
            -p = set same permissions
            -t = transfer modification times
            -g = set same group owner on destination
            -o = set same user owner on destination (if possible, super-user)
            --devices = transfer character and block device files (only super-user)
            --specials = transfer special files like sockets and fifos
          -z = compress
          -P = --partial + --stats
        """
        # Using rsync -rltzP
        command = ["rsync", "--protect-args", "-rltz", "-P", "--chmod=ugo=rwX", directory, data.rsync_target]

        # Add -e if data.rsync_command was passed to this script
        if data.rsync_command:
            # Insert in second position. Example: rsync -e "ssh -i key" ...
            command.insert(1, "-e %s" % data.rsync_command)

        log(' '.join(command))

        # Getting around of rsync output buffering by outputting to a temporary file
        pipe_output, file_name = tempfile.mkstemp()
        log("Rsync output is being saved in %s" % file_name)

        # Call Rsync
        process = subprocess.Popen(command, stdout=pipe_output, stderr=pipe_output)

        # poll() returns None while the process is still running
        while process.poll() is None:
            time.sleep(1)
            last_line = open(file_name).readlines()

            # It's possible that it hasn't output yet, so continue
            if len(last_line) == 0:
                continue
            last_line = last_line[-1]

            # Matching to "[bytes downloaded]  number%  [speed] number:number:number"
            match = re.match(".* ([0-9]*)%.* ([0-9]*:[0-9]*:[0-9]*).*", last_line)

            if not match:
                continue

            # Update upload status
            # - percentage in match.group(1)
            # - ETA in match.group(2)
            access.status = "Sending... %s (ETA: %s)" % (match.group(1), match.group(2))
{'fraction_non_alphanumeric': 0.10863213811420983, 'fraction_numerical': 0.006905710491367862, 'mean_word_length': 2.5533186536646744, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5034465', 'n_tokens_mistral': 3329, 'n_tokens_neox': 3236, 'n_words': 1099}