<commit_before>from datapackage_pipelines.wrapper import ingest, spew
import logging, sh


parameters, datapackage, resources = ingest()


source = parameters['source']
target = parameters['target']


logging.info('uploading {} --> {}'.format(source, target))
cmd = sh.Command('python2')
rsync_args = ['/gsutil/gsutil', '-qm', 'rsync', '-a', 'public-read', '-r', source, target]
ls_args = ['/gsutil/gsutil', 'ls', '-l', target]
for line in cmd(*rsync_args, _iter=True):<commit_msg>remove -m flag from gsutil sync processor<commit_after>from datapackage_pipelines.wrapper import ingest, spew
import logging, sh


parameters, datapackage, resources = ingest()


source = parameters['source']
target = parameters['target']


logging.info('uploading {} --> {}'.format(source, target))
cmd = sh.Command('python2')
rsync_args = ['/gsutil/gsutil', '-q', 'rsync', '-a', 'public-read', '-r', source, target]
ls_args = ['/gsutil/gsutil', 'ls', '-l', target]
for line in cmd(*rsync_args, _iter=True):
{'fraction_non_alphanumeric': 0.1870576339737108, 'fraction_numerical': 0.0020222446916076846, 'mean_word_length': 5.875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6432246', 'n_tokens_mistral': 337, 'n_tokens_neox': 324, 'n_words': 85}