<commit_before>#!/bin/bash

# Pull the latest EdX Forum dump from deploy.prod.class.stanford.edu to datastage. 
# Untar, and load into datastage Edx.Forum****

scp deploy.prod.class.stanford.edu:/data/dump/forum-latest.tar.gz \
    ~dataman/Data/FullDumps/EdxForum
cd ~dataman/Data/FullDumps/EdxForum/
tar -zxvf forum-latest.tar.gz

# Now we have at least one subdirectory of the form
# forum-20140405. There may also be older ones. We
# want to symlink the latest of them to forum-latest
#
# The pipe below this comment does that linking. The pieces:
#
#      ls -t -r -d forum-[0-9]*
#
# list, sorted by modification time (-t) in
# reverse order, i.e. latest first (-r) all
# directories (-d) whose names start with 
# 'forum-' and are are followed by nothing but 
# digits (forum-[0-9]*)
#
# Grab the first line (head -1). Now we have
# just the directory name that resulted from the
# tar -x above.
#
# Symbolically link the result to forum-latest:
# ln -s using backticks.

 ln -s `ls -t -r -d forum-[0-9]* | head -1` forum-latest

# Anonymize and load:
#python extractor.py
<commit_msg>In cronRefreshEdxForum.sh added removal of existing symlink to latest forum. Also started to include run of extractor.py<commit_after>#!/bin/bash

# Pull the latest EdX Forum dump from deploy.prod.class.stanford.edu to datastage. 
# Untar, and load into datastage EdxForum.contents. Two options:
# 

scp deploy.prod.class.stanford.edu:/data/dump/forum-latest.tar.gz \
    ~dataman/Data/FullDumps/EdxForum
cd ~dataman/Data/FullDumps/EdxForum/
tar -zxvf forum-latest.tar.gz

# Now we have at least one subdirectory of the form
# forum-20140405. There may also be older ones. We
# want to symlink the latest of them to forum-latest
#
# The pipe below this comment does that linking. The pieces:
#
#      ls -t -r -d forum-[0-9]*
#
# list, sorted by modification time (-t) in
# reverse order, i.e. latest first (-r) all
# directories (-d) whose names start with 
# 'forum-' and are are followed by nothing but 
# digits (forum-[0-9]*)
#
# Grab the first line (tail -1). Now we have
# just the directory name that resulted from the
# tar -x above.
#
# Symbolically link the result to forum-latest:
# ln -s using backticks.

 rm forum-latest
 ln -s `ls -t -r -d forum-[0-9]* | tail -1` forum-latest

# Anonymize and load:
python extractor.py --anonymize $PWD/forum-latest/app*/contents.bson

{'fraction_non_alphanumeric': 0.11400925536390408, 'fraction_numerical': 0.013462347496844763, 'mean_word_length': 3.4365671641791047, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5300279', 'n_tokens_mistral': 850, 'n_tokens_neox': 803, 'n_words': 306}