<commit_before>#! /usr/bin/env python

from __future__ import print_function, absolute_import, division

'''
This ia a program designed to calibrate COS rawfiles to create CSUMs.

It also sets permissions and group ids appropriately as well
as zipping up any unzipped files to save space.
'''

__author__ = "<NAME>"
__date__ = "04-13-2016"
__maintainer__ = "<NAME>"

# Import necessary packages.
import os
import stat
import subprocess
from datetime import datetime as dt
from collections import defaultdict
import sys

from cos_monitoring.retrieval.retrieval_info import BASE_DIR
from cos_monitoring.retrieval.manualabor import parallelize
from cos_monitoring.retrieval.find_new_cos_data import tally_cs, check_proprietary_status

PERM_755 = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
PERM_872 = stat.S_ISVTX | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP

#------------------------------------------------------------------------------#

def set_user_permissions(perm, mydir=BASE_DIR, prl=True):
    print("Opening permissions of {0}..".format(mydir))
    all_files_dirs = glob.glob(os.path.join(mydir, "*")) + \
                     glob.glob(os.path.join(mydir, "*", "*"))
    if prl:
        parallelize(chmod, all_files_dirs, perm)
    else:
        chmod(all_files_dirs, perm)

#------------------------------------------------------------------------------#

def set_grpid(mydir=BASE_DIR, prl=True):
    """
    Given a base directory mydir, determine which datasets are proprietary, 
    and should have the group permissions set to 'cosgo' while all else
    should have the group permissions set to 'cosstis'.
    
    Parameters:
    -----------
        mydir : str
            The base directory to set group permissions on.

    Returns:
    --------
        Nothing
    """
    priv_id = 65545
    pub_id = 6045
            
    existing, existing_filenames, existing_root = tally_cs(mydir, uniq_roots=False)
    propr_status, sql_roots = check_proprietary_status(list(set(existing_root)))
    propr_d = dict(zip(sql_roots, propr_status))

    propr_status = []
    for i in range(len(existing)):
        rootname = existing_root[i].upper()
        try:
            propr_status.append(propr_d[rootname])
        except:
            propr_status.append(pub_id)

    if prl:
        parallelize(chgrp, existing, propr_status)
    else:
        chgrp(existing, propr_status)

    
#------------------------------------------------------------------------------#
#------------------------------------------------------------------------------#

def chgrp(files, gid):
    user_id = 5026 # jotaylor's user ID
    
    if isinstance(files, str):
        files = [files]
    else:
        if isinstance(gid, str):
            gid = [gid for x in range(len(files))]

    if isinstance(gid, str):
        gid = [gid]
    assert len(files)==len(gid), "Length of files and gid must be equal"

    for i in range(len(files)):<commit_msg>Updated references to parallelize to support new argument order<commit_after>#! /usr/bin/env python

from __future__ import print_function, absolute_import, division

'''
This ia a program designed to calibrate COS rawfiles to create CSUMs.

It also sets permissions and group ids appropriately as well
as zipping up any unzipped files to save space.
'''

__author__ = "<NAME>"
__date__ = "04-13-2016"
__maintainer__ = "<NAME>"

# Import necessary packages.
import os
import stat
import subprocess
from datetime import datetime as dt
from collections import defaultdict
import sys

from cos_monitoring.retrieval.retrieval_info import BASE_DIR
from cos_monitoring.retrieval.manualabor import parallelize
from cos_monitoring.retrieval.find_new_cos_data import tally_cs, check_proprietary_status

PERM_755 = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
PERM_872 = stat.S_ISVTX | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP

#------------------------------------------------------------------------------#

def set_user_permissions(perm, mydir=BASE_DIR, prl=True):
    print("Opening permissions of {0}..".format(mydir))
    all_files_dirs = glob.glob(os.path.join(mydir, "*")) + \
                     glob.glob(os.path.join(mydir, "*", "*"))
    if prl:
        parallelize("smart", "check_usage", chmod, all_files_dirs, perm)
    else:
        chmod(all_files_dirs, perm)

#------------------------------------------------------------------------------#

def set_grpid(mydir=BASE_DIR, prl=True):
    """
    Given a base directory mydir, determine which datasets are proprietary, 
    and should have the group permissions set to 'cosgo' while all else
    should have the group permissions set to 'cosstis'.
    
    Parameters:
    -----------
        mydir : str
            The base directory to set group permissions on.

    Returns:
    --------
        Nothing
    """
    priv_id = 65545
    pub_id = 6045
            
    existing, existing_filenames, existing_root = tally_cs(mydir, uniq_roots=False)
    propr_status, sql_roots = check_proprietary_status(list(set(existing_root)))
    propr_d = dict(zip(sql_roots, propr_status))

    propr_status = []
    for i in range(len(existing)):
        rootname = existing_root[i].upper()
        try:
            propr_status.append(propr_d[rootname])
        except:
            propr_status.append(pub_id)

    if prl:
        parallelize("smart", "check_usage", chgrp, existing, propr_status)
    else:
        chgrp(existing, propr_status)

    
#------------------------------------------------------------------------------#
#------------------------------------------------------------------------------#

def chgrp(files, gid):
    user_id = 5026 # jotaylor's user ID
    
    if isinstance(files, str):
        files = [files]
    else:
        if isinstance(gid, str):
            gid = [gid for x in range(len(files))]

    if isinstance(gid, str):
        gid = [gid]
    assert len(files)==len(gid), "Length of files and gid must be equal"

    for i in range(len(files)):
{'fraction_non_alphanumeric': 0.19419716452357402, 'fraction_numerical': 0.009231783712495879, 'mean_word_length': 2.1065028161802357, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5206219', 'n_tokens_mistral': 1888, 'n_tokens_neox': 1785, 'n_words': 477}