<commit_before>#!/bin/sh
#
# Create the periodic database dump files
#

# Directory where the script must run
RUNDIR=/home/dds/src/github-mirror

# Time to start dumping from
if [ -r lastrun ]
then
	timeStart=`cat lastrun`
else
	timeStart=0
fi

# Time to end dumping
if [ "$1" = "" ]
then
	timeEnd=`date +%s`
else
	timeEnd=`date -d "$1" +%s` || exit 1
fi

# Name used for the files
dateName=`date -d @$timeEnd -u +'%Y-%m-%dT%H%M%SZ'`

# _id example:
# 4f208c3e08d69a1835000077
# 000102030405060708091011
# |      ||    ||  ||    |
# time    mach  pid count

endId=`printf '%08x0000000000000000' $timeEnd`
startId=`printf '%08x0000000000000000' $timeStart`

echo "Dumping database from `date -d @$timeStart` to `date -d @$timeEnd`"

# Dump events and commits
rm -rf dump
mongodump --db github --collection events -q '{"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }' || exit 1
mongodump --db github --collection commits -q '{"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }' || exit 1

# Report the metadata for the given database
meta()
{
	echo -n "Number of $1: "
	mongo --quiet --eval 'db.'$1'.find({"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }).count() + 0' github
	echo -n "Uncompressed size of $1: "
	wc -c dump/github/$1.bson | awk '{printf "%d bytes ", $1}'
	du -h dump/github/$1.bson | awk '{print " (" $1 ")" }'
}

(
	echo "Start date: `date -u -d @$timeStart +'%Y-%m-%dT%H:%M:%SZ'`"
	echo "End date: `date -u -d @$timeEnd +'%Y-%m-%dT%H:%M:%SZ'`"
	meta commits
	meta events
) |
tee README.$dateName.txt >dump/github/README.txt

# Create an archive of the dumped files
mv dump/github github-dump.$dateName
tar -cf - github-dump.$dateName | bzip2 -c >github-dump.$dateName.tar.bz2

# Update last run info
echo $timeEnd >lastrun<commit_msg>Automatically dump all collections<commit_after>#!/bin/sh
#
# Create the periodic database dump files
#

# Directory where the script must run
RUNDIR=/home/dds/src/github-mirror

# Time to start dumping from
if [ -r lastrun ]
then
	timeStart=`cat lastrun`
else
	timeStart=0
fi

# Time to end dumping
if [ "$1" = "" ]
then
	timeEnd=`date +%s`
else
	timeEnd=`date -d "$1" +%s` || exit 1
fi

# Name used for the files
dateName=`date -d @$timeEnd -u +'%Y-%m-%dT%H%M%SZ'`

# _id example:
# 4f208c3e08d69a1835000077
# 000102030405060708091011
# |      ||    ||  ||    |
# time    mach  pid count

endId=`printf '%08x0000000000000000' $timeEnd`
startId=`printf '%08x0000000000000000' $timeStart`

echo "Dumping database from `date -d @$timeStart` to `date -d @$timeEnd`"

collections=`echo "show collections"|mongo --quiet github|egrep -v "system|bye"`
	
rm -rf dump
for col in $collections; do

	echo "Dumping $col"
	mongodump --db github --collection $col -q '{"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }' || exit 1
done

# Report the metadata for the given database
meta()
{
	echo -n "Number of $1: "
	mongo --quiet --eval 'db.'$1'.find({"_id" : {"$gte" : ObjectId("'$startId'"), "$lt"  : ObjectId("'$endId'")} }).count() + 0' github
	echo -n "Uncompressed size of $1: "
	wc -c dump/github/$1.bson | awk '{printf "%d bytes ", $1}'
	du -h dump/github/$1.bson | awk '{print " (" $1 ")" }'
}

for col in $collections; do
(
	echo "Start date: `date -u -d @$timeStart +'%Y-%m-%dT%H:%M:%SZ'`"
	echo "End date: `date -u -d @$timeEnd +'%Y-%m-%dT%H:%M:%SZ'`"
	meta $col 
) 
done|
tee README.$dateName.txt >dump/github/README.txt

# Create an archive of the dumped files
mv dump/github github-dump.$dateName
tar -cf - github-dump.$dateName | bzip2 -c >github-dump.$dateName.tar.bz2

# Update last run info
echo $timeEnd >lastrun
{'fraction_non_alphanumeric': 0.1899836690255852, 'fraction_numerical': 0.05144256940664126, 'mean_word_length': 3.9196787148594376, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5956563', 'n_tokens_mistral': 1620, 'n_tokens_neox': 1437, 'n_words': 411}