<commit_before>'''
Created on Apr 27, 2014

@author: crusherofheads

Get stats about the workspace. Object counts by pub/priv/type, unique users
who have saved data, pub/priv workspaces, total size of data (note this
double counts copied data and data with the same MD5)

Don't run this during high loads - runs through every object in the DB
Hasn't been optimized much either
'''
from __future__ import print_function
from configobj import ConfigObj
from pymongo import MongoClient
import time
import sys
from collections import defaultdict

# where to get credentials (don't check these into git, idiot)
CREDS_FILE = 'ws_mongo_creds'
CREDS_SECTION = 'WorkspaceMongoCreds'
USER = 'user'
PWD = '<PASSWORD>'
# mongo db location
WS_MONGO_HOST = 'localhost'
WS_MONGO_PORT = 49990
WS_MONGO_DB = 'workspace'
#collection names
COL_WS = 'workspaces'
COL_ACLS = 'workspaceACLs'
COL_OBJ = 'workspaceObjects'
COL_VERS = 'workspaceObjVersions'

PUBLIC = 'pub'
PRIVATE = 'priv'

LIMIT = 10000
OR_QUERY_SIZE = 100
MAX_WS = 358  # for testing, set to < 1 for all ws


def chunkiter(iterable, size):
    """Iterates over an iterable in chunks of size size. Returns an iterator
  that in turn returns iterators over the iterable that each iterate through
  size objects in the iterable.
  Note that since the inner and outer loops are pulling values from the same
  iterator, continue and break don't necessarily behave exactly as one would
  expect. In the outer loop of the iteration, continue effectively does
  nothing, but break works normally. In the inner loop, break has no real
  effect but continue works normally. For the latter issue, wrapping the inner
  iterator in a tuple will cause break to skip the remaining items in the
  iterator. Alternatively, one can set a flag and exhaust the inner iterator.
  """
    def inneriter(first, iterator, size):
        yield first
        for _ in xrange(size - 1):
            yield iterator.next()
    it = iter(iterable)
    while True:
        yield inneriter(it.next(), it, size)


def process_objects(objs, unique_users, types, workspaces):
    objsproc = 0
    size = 0
    # note all objects are from the same workspace<commit_msg>Looks like a batch size of 100 was a good guess<commit_after>'''
Created on Apr 27, 2014

@author: crusherofheads

Get stats about the workspace. Object counts by pub/priv/type, unique users
who have saved data, pub/priv workspaces, total size of data (note this
double counts copied data and data with the same MD5)

Don't run this during high loads - runs through every object in the DB
Hasn't been optimized much either
'''
from __future__ import print_function
from configobj import ConfigObj
from pymongo import MongoClient
import time
import sys
from collections import defaultdict

# where to get credentials (don't check these into git, idiot)
CREDS_FILE = 'ws_mongo_creds'
CREDS_SECTION = 'WorkspaceMongoCreds'
USER = 'user'
PWD = '<PASSWORD>'
# mongo db location
WS_MONGO_HOST = 'localhost'
WS_MONGO_PORT = 49990
WS_MONGO_DB = 'workspace'
#collection names
COL_WS = 'workspaces'
COL_ACLS = 'workspaceACLs'
COL_OBJ = 'workspaceObjects'
COL_VERS = 'workspaceObjVersions'

PUBLIC = 'pub'
PRIVATE = 'priv'

LIMIT = 10000
OR_QUERY_SIZE = 100  # 75 was slower, 150 was slower
MAX_WS = -1  # for testing, set to < 1 for all ws


def chunkiter(iterable, size):
    """Iterates over an iterable in chunks of size size. Returns an iterator
  that in turn returns iterators over the iterable that each iterate through
  size objects in the iterable.
  Note that since the inner and outer loops are pulling values from the same
  iterator, continue and break don't necessarily behave exactly as one would
  expect. In the outer loop of the iteration, continue effectively does
  nothing, but break works normally. In the inner loop, break has no real
  effect but continue works normally. For the latter issue, wrapping the inner
  iterator in a tuple will cause break to skip the remaining items in the
  iterator. Alternatively, one can set a flag and exhaust the inner iterator.
  """
    def inneriter(first, iterator, size):
        yield first
        for _ in xrange(size - 1):
            yield iterator.next()
    it = iter(iterable)
    while True:
        yield inneriter(it.next(), it, size)


def process_objects(objs, unique_users, types, workspaces):
    objsproc = 0
    size = 0
    # note all objects are from the same workspace
{'fraction_non_alphanumeric': 0.060379918588873815, 'fraction_numerical': 0.013568521031207599, 'mean_word_length': 4.049086757990867, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6534586', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1244, 'n_words': 595}