<commit_before>#!/bin/sh

#  load_data.sh
#  
#
#  Created by <NAME> on 11/18/13.
#



# Enforce JSON format
echo "Before continuing, make sure your files in xml format! Also make sure that there is no directory called solr_files if you are planning on converting files first"

# read in file name
echo "Which dataset do you wish to load? Specify: Australia / DUC"
read dataset_name


# test input
flag=true
while [ $flag ]; do
    if [ "$dataset_name" != "Australia" ] && [ "$dataset_name" != "DUC" ]; then
        echo "Please enter either Australia or DUC"
        read dataset_name
    else
        break
    fi
done

# configure path to json files
# NOTE: DUC PATH NOT CONFIGURED
if [ "$dataset_name" == "Australia" ]; then
    data_path="corpus/fulltext/*"

else
    echo "DUC set is unavailable at the moment!"
    break
fi

# INDEXING:
# Parse XML for SOLR Format
if [ -d "solr_files" ]; then
    echo "skipping conversion..."
else
    mkdir "solr_files"
    echo "Converting files into a format compatible with SOLR"
    for f in $data_path
    do
        filename=$(basename $f)
    if [ "$dataset_name" == "Australia" ]; then
        python australia_xml_parser.py "$f" "solr_files/$filename"
    fi
    done
fi
# PUT data from path onto SOLR
echo "Posting files to SOLR"
curl "http://localhost:8983/solr/admin/cores?action=CREATE&name=AustralianDataset&instanceDir=/home/vagrant/source/SEDA-ML-Summarization/Data_Processing/solr_config/collection1"
java -Durl=http://localhost:8983/solr/AustralianDataset/update -jar post.jar "solr_files/*.xml"
<commit_msg>fixed load data script to create new core. copies configuration files to vagrant home. standardized script file for linux general environment<commit_after>#!/bin/sh

#  load_data.sh
#  
#
#  Created by <NAME> on 11/18/13.
#



# Enforce JSON format
echo "Before continuing, make sure your files in xml format! Also make sure that there is no directory called solr_files if you are planning on converting files first"

# read in file name
echo "Which dataset do you wish to load? Specify: Australia / DUC"
read dataset_name


# test input
flag=true
while [ ${flag} ]; do
    if [ ${dataset_name} != "Australia" ] && [ ${dataset_name} != "DUC" ]; then
        echo "Please enter either Australia or DUC"
        read dataset_name
    else
        break
    fi
done

# configure path to json files
# NOTE: DUC PATH NOT CONFIGURED
if [ ${dataset_name} = "Australia" ]; then
    data_path="./corpus/fulltext/*"

else
    echo "DUC set is unavailable at the moment!"
    exit
fi

# INDEXING:
# Parse XML for SOLR Format
if [ -d "solr_files" ]; then
    echo "skipping conversion..."
else
    mkdir "solr_files"
    echo "Converting files into a format compatible with SOLR"
    for f in ${data_path}
    do
        filename=$(basename $f)
    if [ "${dataset_name}" = "Australia" ]; then
        python australia_xml_parser.py "${f}" "solr_files/${filename}"
    fi
    done
fi
# PUT data from path onto SOLR
echo "Posting files to SOLR"
cp -a "/home/vagrant/source/SEDA-ML-Summarization/Data_Processing/solr_config" "/home/vagrant"
curl "http://localhost:8983/solr/admin/cores?action=CREATE&name=AustralianDataset&instanceDir=/home/vagrant/solr_config/collection1"
java -Durl=http://localhost:8983/solr/AustralianDataset/update -jar post.jar "solr_files/*.xml"

{'fraction_non_alphanumeric': 0.10586466165413534, 'fraction_numerical': 0.009022556390977444, 'mean_word_length': 3.56241426611797, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '787749', 'n_tokens_mistral': 1108, 'n_tokens_neox': 1056, 'n_words': 358}