<commit_before>'''
This script copies all resource files from a local FileStore directory
to a remote S3 bucket.

**It will not work for group images**

It requires SQLalchemy and Boto.

Please update the configuration details, all keys are mandatory except
AWS_STORAGE_PATH.

'''

import os
from sqlalchemy import create_engine
from sqlalchemy.sql import text
from boto.s3.connection import S3Connection
from boto.s3.key import Key


# Configuration

BASE_PATH = '/var/lib/ckan/default/resources'
SQLALCHEMY_URL = 'postgresql://user:pass@localhost/db'
AWS_ACCESS_KEY_ID = '<KEY>'
AWS_SECRET_ACCESS_KEY = <KEY>'
AWS_BUCKET_NAME = 'my-bucket'
AWS_STORAGE_PATH = 'some-path'


resource_ids_and_paths = {}

for root, dirs, files in os.walk(BASE_PATH):
    if files:
        resource_id = root.split('/')[-2] + root.split('/')[-1] + files[0]
        resource_ids_and_paths[resource_id] = os.path.join(root, files[0])

print 'Found {0} resource files in the file system'.format(
    len(resource_ids_and_paths.keys()))

engine = create_engine(SQLALCHEMY_URL)
connection = engine.connect()

resource_ids_and_names = {}

try:
    for resource_id, file_path in resource_ids_and_paths.iteritems():
        resource = connection.execute(text('''
            SELECT id, url, url_type
            FROM resource
            WHERE id = :id
        '''), id=resource_id)
        if resource.rowcount:
            _id, url, _type = resource.first()
            if _type == 'upload' and url:
                file_name = url.split('/')[-1] if '/' in url else url
                resource_ids_and_names[_id] = file_name
finally:
    connection.close()
    engine.dispose()

print '{0} resources matched on the database'.format(<commit_msg>Change the resource file names to lower case<commit_after>'''
This script copies all resource files from a local FileStore directory
to a remote S3 bucket.

**It will not work for group images**

It requires SQLalchemy and Boto.

Please update the configuration details, all keys are mandatory except
AWS_STORAGE_PATH.

'''

import os
from sqlalchemy import create_engine
from sqlalchemy.sql import text
from boto.s3.connection import S3Connection
from boto.s3.key import Key


# Configuration

BASE_PATH = '/var/lib/ckan/default/resources'
SQLALCHEMY_URL = 'postgresql://user:pass@localhost/db'
AWS_ACCESS_KEY_ID = '<KEY>'
AWS_SECRET_ACCESS_KEY = <KEY>'
AWS_BUCKET_NAME = 'my-bucket'
AWS_STORAGE_PATH = 'some-path'


resource_ids_and_paths = {}

for root, dirs, files in os.walk(BASE_PATH):
    if files:
        resource_id = root.split('/')[-2] + root.split('/')[-1] + files[0]
        resource_ids_and_paths[resource_id] = os.path.join(root, files[0])

print 'Found {0} resource files in the file system'.format(
    len(resource_ids_and_paths.keys()))

engine = create_engine(SQLALCHEMY_URL)
connection = engine.connect()

resource_ids_and_names = {}

try:
    for resource_id, file_path in resource_ids_and_paths.iteritems():
        resource = connection.execute(text('''
            SELECT id, url, url_type
            FROM resource
            WHERE id = :id
        '''), id=resource_id)
        if resource.rowcount:
            _id, url, _type = resource.first()
            if _type == 'upload' and url:
                file_name = url.split('/')[-1] if '/' in url else url
                resource_ids_and_names[_id] = file_name.lower()
finally:
    connection.close()
    engine.dispose()

print '{0} resources matched on the database'.format(
{'fraction_non_alphanumeric': 0.10893935038804255, 'fraction_numerical': 0.006323656223052601, 'mean_word_length': 3.4787644787644787, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2992777', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1105, 'n_words': 306}