<commit_before>#!/bin/bash
#
#3> <> prov:specializationOf <https://github.com/timrdf/csv2rdf4lod-automation/blob/master/bin/util/cache-queries.sh>;
#3>    rdfs:seeAlso          <https://github.com/timrdf/csv2rdf4lod-automation/wiki/Script:-cache-queries.sh>;
#3> .
#
#   Copyright 2012 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

outputTypes="sparql xml"

if [ $# -lt 1 ]; then
   echo "usage: `basename $0` <endpoint> [-p {output,format}] [-o {sparql,gvds,xml,exhibit,csv}+] [-q a.sparql b.sparql ...]* [--limit-offset] [-od path/to/output/dir]"
   echo
   echo "    Executes SPARQL queries against an endpoint requesting the given output formats."
   echo
   echo "            -p  : the URL parameter name used to request a different output/format from <endpoint>."
   echo "    default -p  : 'output'"
   echo "            -o  : the URL parameter value(s) to request."
   echo "    default -o  : $outputTypes"
   echo "    default -q  : *.sparql *.rq"
   echo " --limit-offset : iterate with LIMIT / OFFSET until no more useful results."
   echo "            -od : output directory"
   echo "    default -od : results/"
   exit 1
fi

# http://dbpedia.org/sparql
# format = 
# "text/html"                       HTML
# "application/vnd.ms-excel"        Spreadsheet
# "application/sparql-results+xml"  XML
# "application/sparql-results+json" JSON           &format=application%2Frdf%2Bxml
# "application/javascript"          Javascript
# "text/plain"                      NTriples
# "application/rdf+xml"             RDF/XML
# "text/csv"                        CSV

endpoint="http://logd.tw.rpi.edu/sparql"
endpoint="http://dbpedia.org/sparql"
endpoint="$1"
shift

outputVarName="output"
if [ "$1" == "-p" ]; then
   shift
   outputVarName="$1"
   shift
fi
#echo "`basename $0` using results format param: $outputVarName"

if [ "$1" == "-o" ]; then
   shift
   outputTypes=""
   while [ "$1" != "-q" -a $# -gt 0 ]; do
      outputTypes="$outputTypes $1"
      shift
   done
fi
#echo "`basename $0` using results format value: $outputTypes"

queryFiles=""
if [ $# -gt 0 -a "$1" == "-q" ]; then
   shift
   while [[ $# -gt 0 && ( "$1" != '-od' && "$1" != '--limit-offset' ) ]]; do
      queryFiles="$queryFiles $1"
      shift 
   done
else
   for sparql in `ls *.sparql *.rq 2> /dev/null`; do
      queryFiles="$queryFiles $sparql"
   done
fi

limit_offset=''
if [[ "$1" == '--limit-offset' ]]; then
   limit_offset='yes' 
   if [[ "$2" =~ -* ]]; then
      echo "`basename $0` will use default LIMIT, or the LIMIT defined in the file." >&2
   else
      limit_offset="$2" # An actual number.
      shift
   fi
   shift
fi

echo "limit_offset: $limit_offset" >&2

results="results"
if [ "$1" == "-od" -a $# -gt 1 ]; then
   shift
   results="$1"
fi
if [ ! -d $results ]; then
   mkdir -p $results
fi

for sparql in $queryFiles; do
   echo $sparql

   # limit_offset is either: '' (no), 'yes', or a number e.g. '100000'
   if [[ -n "$limit_offset" ]]; then
      limit=`cat $sparql | grep -i '^limit' | awk '{print $2}' | head -1`
      if [[ "$limit" =~ [0-9]+ ]]; then
         echo "Found limit in $sparql: $limit" >&2
      fi
      if [[ "$limit_offset" =~ [0-9]+ ]]; then<commit_msg>--limit-offset argument for cache-queries.sh<commit_after>#!/bin/bash
#
#3> <> prov:specializationOf <https://github.com/timrdf/csv2rdf4lod-automation/blob/master/bin/util/cache-queries.sh>;
#3>    rdfs:seeAlso          <https://github.com/timrdf/csv2rdf4lod-automation/wiki/Script:-cache-queries.sh>;
#3> .
#
#   Copyright 2012 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

outputTypes="sparql xml"

if [ $# -lt 1 ]; then
   echo "usage: `basename $0` <endpoint> [-p {output,format}] [-o {sparql,gvds,xml,exhibit,csv}+] [-q a.sparql b.sparql ...]* [--limit-offset] [-od path/to/output/dir]"
   echo
   echo "    Executes SPARQL queries against an endpoint requesting the given output formats."
   echo
   echo "            -p  : the URL parameter name used to request a different output/format from <endpoint>."
   echo "    default -p  : 'output'"
   echo "            -o  : the URL parameter value(s) to request."
   echo "    default -o  : $outputTypes"
   echo "    default -q  : *.sparql *.rq"
   echo " --limit-offset : iterate with LIMIT / OFFSET until no more useful results."
   echo "            -od : output directory"
   echo "    default -od : results/"
   exit 1
fi

# http://dbpedia.org/sparql
# format = 
# "text/html"                       HTML
# "application/vnd.ms-excel"        Spreadsheet
# "application/sparql-results+xml"  XML
# "application/sparql-results+json" JSON           &format=application%2Frdf%2Bxml
# "application/javascript"          Javascript
# "text/plain"                      NTriples
# "application/rdf+xml"             RDF/XML
# "text/csv"                        CSV

endpoint="http://logd.tw.rpi.edu/sparql"
endpoint="http://dbpedia.org/sparql"
endpoint="$1"
shift

outputVarName="output"
if [ "$1" == "-p" ]; then
   shift
   outputVarName="$1"
   shift
fi
#echo "`basename $0` using results format param: $outputVarName"

if [ "$1" == "-o" ]; then
   shift
   outputTypes=""
   while [ "$1" != "-q" -a $# -gt 0 ]; do
      outputTypes="$outputTypes $1"
      shift
   done
fi
#echo "`basename $0` using results format value: $outputTypes"

queryFiles=""
if [ $# -gt 0 -a "$1" == "-q" ]; then
   shift
   while [[ $# -gt 0 && ( "$1" != '-od' && "$1" != '--limit-offset' ) ]]; do
      queryFiles="$queryFiles $1"
      shift 
   done
else
   for sparql in `ls *.sparql *.rq 2> /dev/null`; do
      queryFiles="$queryFiles $sparql"
   done
fi

limit_offset=''
if [[ "$1" == '--limit-offset' ]]; then
   limit_offset='yes' 
   if [[ "$2" =~ -* ]]; then
      echo "`basename $0` will use default LIMIT, or the LIMIT defined in the file." >&2
   else
      echo "`basename $0` accepting LIMIT override: $2" >&2
      limit_offset="$2" # An actual number.
      shift
   fi
   shift
fi

echo "limit_offset: $limit_offset" >&2

results="results"
if [ "$1" == "-od" -a $# -gt 1 ]; then
   shift
   results="$1"
fi
if [ ! -d $results ]; then
   mkdir -p $results
fi

for sparql in $queryFiles; do
   echo $sparql

   # limit_offset is either: '' (no), 'yes', or a number e.g. '100000'
   if [[ -n "$limit_offset" ]]; then
      limit=`cat $sparql | grep -i '^limit' | awk '{print $2}' | head -1`
      if [[ "$limit" =~ [0-9]+ ]]; then
         echo "Found limit in $sparql: $limit" >&2
      fi
      if [[ "$limit_offset" =~ [0-9]+ ]]; then
{'fraction_non_alphanumeric': 0.15423480920090413, 'fraction_numerical': 0.01582236404733413, 'mean_word_length': 2.6496846191169334, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 30, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '890327', 'n_tokens_mistral': 2628, 'n_tokens_neox': 2547, 'n_words': 780}