<commit_before>#!/bin/bash

set -e

: ${ES_HOST:=es}
: ${CIRRUS_DUMP_URL:=https://dumps.wikimedia.org/other/cirrussearch/20170109/simplewiki-20170109-cirrussearch-content.json.gz}
: ${CITOLYTICS_DUMP_URL:=http://citolytics-demo.wmflabs.org/dumps/citolytics_simplewiki.json.gz}
: ${BATCH_SIZE:=1000}


rm -fr $MEDIAWIKI_DIR/data
mkdir $MEDIAWIKI_DIR/data

# Download CirrusSearch dump, split the dump in chunks and send the data to Elasticsearch:

wget $CIRRUS_DUMP_URL -O $MEDIAWIKI_DIR/data/cirrus.json.gz
zcat $MEDIAWIKI_DIR/data/cirrus.json.gz > $MEDIAWIKI_DIR/data/cirrus.json
mkdir $MEDIAWIKI_DIR/data/cirrus.splits.d
split -l $BATCH_SIZE $MEDIAWIKI_DIR/data/cirrus.json $MEDIAWIKI_DIR/data/cirrus.splits.d/
for f in $MEDIAWIKI_DIR/data/cirrus.splits.d/{.,}*; do curl -XPOST $ES_HOST:9200/mediawiki_content_first/page/_bulk?pretty --data-binary @$f; done

# Popupate Citolytics data to ES
<commit_msg>updated chmod<commit_after>#!/bin/bash

set -e

: ${ES_HOST:=es}
: ${CIRRUS_DUMP_URL:=https://dumps.wikimedia.org/other/cirrussearch/20170605/simplewiki-20170109-cirrussearch-content.json.gz}
: ${CITOLYTICS_DUMP_URL:=http://citolytics-demo.wmflabs.org/dumps/citolytics_simplewiki.json.gz}
: ${BATCH_SIZE:=1000}


rm -fr $MEDIAWIKI_DIR/data
mkdir $MEDIAWIKI_DIR/data

# Download CirrusSearch dump, split the dump in chunks and send the data to Elasticsearch:

wget $CIRRUS_DUMP_URL -O $MEDIAWIKI_DIR/data/cirrus.json.gz
zcat $MEDIAWIKI_DIR/data/cirrus.json.gz > $MEDIAWIKI_DIR/data/cirrus.json
mkdir $MEDIAWIKI_DIR/data/cirrus.splits.d
split -l $BATCH_SIZE $MEDIAWIKI_DIR/data/cirrus.json $MEDIAWIKI_DIR/data/cirrus.splits.d/
for f in $MEDIAWIKI_DIR/data/cirrus.splits.d/{.,}*; do curl -XPOST $ES_HOST:9200/mediawiki_content_first/page/_bulk?pretty --data-binary @$f; done

# Popupate Citolytics data to ES

{'fraction_non_alphanumeric': 0.1368653421633554, 'fraction_numerical': 0.026490066225165563, 'mean_word_length': 5.616788321167883, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '455561', 'n_tokens_mistral': 825, 'n_tokens_neox': 774, 'n_words': 113}