<commit_before>#!/bin/bash
#
#3> <> a conversion:RetrievalTrigger;
#3>    prov:specializationOf <https://github.com/timrdf/csv2rdf4lod-automation/blob/master/bin/cr-publish-isdefinedby-to-endpoint.sh>;
#3>    rdfs:seeAlso          <https://github.com/timrdf/csv2rdf4lod-automation/wiki/Aggregating-subsets-of-converted-datasets> .
#
# Environment variables used:
#
#    CSV2RDF4LOD_PUBLISH_OUR_SOURCE_ID - to know which source-id to place the dataset into.
#                                        (not needed if invoked within a cr:source directory)
#
#    CSV2RDF4LOD_BASE_URI - to know the URI of the dataset created.
#
#    CSV2RDF4LOD_PUBLISH_SPARQL_ENDPOINT - the SPARQL endpoint to look for vocabulary terms used.
#
#    CSV2RDF4LOD_PUBLISH_VIRTUOSO_SPARQL_ENDPOINT - fallback, in case ^^ is not defined.
#
#    (see https://github.com/timrdf/csv2rdf4lod-automation/wiki/CSV2RDF4LOD-environment-variables)
#
# Usage:
#
# Example usage:
# 

[ -n "`readlink $0`" ] && this=`readlink $0` || this=$0
HOME=$(cd ${this%/*/*/*} && pwd)
export PATH=$PATH`$HOME/bin/util/cr-situate-paths.sh`
export CLASSPATH=$CLASSPATH`$HOME/bin/util/cr-situate-classpaths.sh`

# cr:data-root cr:source cr:directory-of-datasets cr:dataset cr:directory-of-versions cr:conversion-cockpit
ACCEPTABLE_PWDs="cr:data-root cr:source cr:dataset cr:directory-of-versions cr:conversion-cockpit"
if [ `is-pwd-a.sh $ACCEPTABLE_PWDs` != "yes" ]; then
   pwd-not-a.sh $ACCEPTABLE_PWDs
   exit 1
fi

TEMP="_"`basename $0``date +%s`_$$.tmp

sourceID=$CSV2RDF4LOD_PUBLISH_OUR_SOURCE_ID
datasetID=`basename $0 | sed -e 's/-publish//' -e 's/-to-endpoint//' -e 's/.sh$//'` # e.g. cr-publish-void-to-endpoint.sh -> cr-void
versionID=`date +%Y-%b-%d`

if [[ "$1" == "--help" ]]; then
   echo "usage: `basename $0` [version-identifier] [URL]"
   echo "   version-identifier: conversion:version_identifier for the VersionedDataset to create. Can be '', 'cr:auto', 'cr:today', 'cr:force'."
   echo "   URL               : URL to use during retrieval."
   exit 1
fi

n=''
dryRun="false"
if [ "$1" == "-n" ]; then
   n="-n"
   dryRun="true"
   dryrun.sh $dryrun beginning
   shift 
fi

if [[ `is-pwd-a.sh                                                                                       cr:conversion-cockpit` == "yes" ]]; then

   versionID=`cr-version-id.sh`
   pushd ../ &> /dev/null
      $this $n $versionID 
   popd &> /dev/null

elif [[ `is-pwd-a.sh                                                              cr:directory-of-versions` == "yes" ]]; then

   endpoint="$CSV2RDF4LOD_PUBLISH_SPARQL_ENDPOINT"
   if [ ${#endpoint} -eq 0 ]; then
      endpoint="$CSV2RDF4LOD_PUBLISH_VIRTUOSO_SPARQL_ENDPOINT"
   fi
   if [ ${#endpoint} -eq 0 ]; then
      echo "ERROR: no endpoint defined. Define CSV2RDF4LOD_PUBLISH_SPARQL_ENDPOINT"
      exit 1
   fi

   if [ -e $versionID ]; then
      echo "[INFO] `basename $0`: version $versionID already exists; skipping."
      exit 1
   fi

   cockpit="$versionID"
   if [ ! -d $cockpit/automatic ]; then
      mkdir -p $cockpit/automatic # TODO: pull down as csv, convert with eparams.
   fi
   rm -rf $cockpit/source/* # opposite of "if exists, quit" above.

   echo "[INFO] python $CSV2RDF4LOD_HOME/bin/cr-publish-isdefinedby-to-endpoint.py $endpoint"
   if [ "$dryRun" != "true" ]; then
      python $CSV2RDF4LOD_HOME/bin/cr-publish-isdefinedby-to-endpoint.py $endpoint > $cockpit/automatic/isdefinedby.nt
      pushd $cockpit &> /dev/null
         aggregate-source-rdf.sh --link-as-latest automatic/* <commit_msg>moving isdefinedby into enabled dataset<commit_after>#!/bin/bash
#
#3> <> a conversion:RetrievalTrigger;
#3>    prov:specializationOf <https://github.com/timrdf/csv2rdf4lod-automation/blob/master/bin/cr-publish-isdefinedby-to-endpoint.sh>;
#3>    rdfs:seeAlso          <https://github.com/timrdf/csv2rdf4lod-automation/wiki/Aggregating-subsets-of-converted-datasets> .
#
# Environment variables used:
#
#    CSV2RDF4LOD_PUBLISH_OUR_SOURCE_ID - to know which source-id to place the dataset into.
#                                        (not needed if invoked within a cr:source directory)
#
#    CSV2RDF4LOD_BASE_URI - to know the URI of the dataset created.
#
#    CSV2RDF4LOD_PUBLISH_SPARQL_ENDPOINT - the SPARQL endpoint to look for vocabulary terms used.
#
#    CSV2RDF4LOD_PUBLISH_VIRTUOSO_SPARQL_ENDPOINT - fallback, in case ^^ is not defined.
#
#    (see https://github.com/timrdf/csv2rdf4lod-automation/wiki/CSV2RDF4LOD-environment-variables)
#
# Usage:
#
# Example usage:
# 

[ -n "`readlink $0`" ] && this=`readlink $0` || this=$0
HOME=$(cd ${this%/*/*/*} && pwd)
export PATH=$PATH`$HOME/bin/util/cr-situate-paths.sh`
export CLASSPATH=$CLASSPATH`$HOME/bin/util/cr-situate-classpaths.sh`

# cr:data-root cr:source cr:directory-of-datasets cr:dataset cr:directory-of-versions cr:conversion-cockpit
ACCEPTABLE_PWDs="cr:data-root cr:source cr:dataset cr:directory-of-versions cr:conversion-cockpit"
if [ `is-pwd-a.sh $ACCEPTABLE_PWDs` != "yes" ]; then
   pwd-not-a.sh $ACCEPTABLE_PWDs
   exit 1
fi

TEMP="_"`basename $0``date +%s`_$$.tmp

sourceID=$CSV2RDF4LOD_PUBLISH_OUR_SOURCE_ID
datasetID=`basename $0 | sed -e 's/-publish//' -e 's/-to-endpoint//' -e 's/.sh$//'` # e.g. cr-publish-void-to-endpoint.sh -> cr-void
versionID=`date +%Y-%b-%d`

if [[ "$1" == "--help" ]]; then
   echo "usage: `basename $0` [version-identifier] [URL]"
   echo "   version-identifier: conversion:version_identifier for the VersionedDataset to create. Can be '', 'cr:auto', 'cr:today', 'cr:force'."
   echo "   URL               : URL to use during retrieval."
   exit 1
fi

n=''
dryRun="false"
if [ "$1" == "-n" ]; then
   n="-n"
   dryRun="true"
   dryrun.sh $dryrun beginning
   shift 
fi

if [[ `is-pwd-a.sh                                                                                       cr:conversion-cockpit` == "yes" ]]; then

   versionID=`cr-version-id.sh`
   pushd ../ &> /dev/null
      $this $n $versionID 
   popd &> /dev/null

elif [[ `is-pwd-a.sh                                                              cr:directory-of-versions` == "yes" ]]; then

   endpoint="$CSV2RDF4LOD_PUBLISH_SPARQL_ENDPOINT"
   if [ ${#endpoint} -eq 0 ]; then
      endpoint="$CSV2RDF4LOD_PUBLISH_VIRTUOSO_SPARQL_ENDPOINT"
   fi
   if [ ${#endpoint} -eq 0 ]; then
      echo "ERROR: no endpoint defined. Define CSV2RDF4LOD_PUBLISH_SPARQL_ENDPOINT"
      exit 1
   fi

   if [ -e $versionID ]; then
      echo "[INFO] `basename $0`: version $versionID already exists; skipping."
      exit 1
   fi

   cockpit="$versionID"
   if [ ! -d $cockpit/automatic ]; then
      mkdir -p $cockpit/automatic # TODO: pull down as csv, convert with eparams.
   fi
   rm -rf $cockpit/source/* # opposite of "if exists, quit" above.

   echo "[INFO] python ../src/cr-isdefinedby.py $endpoint"
   if [ "$dryRun" != "true" ]; then
      python ../src/cr-isdefinedby.py $endpoint > $cockpit/automatic/isdefinedby.nt
      pushd $cockpit &> /dev/null
         aggregate-source-rdf.sh --link-as-latest automatic/* 
{'fraction_non_alphanumeric': 0.14710082833476149, 'fraction_numerical': 0.012567837760639817, 'mean_word_length': 2.862658576944291, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 25, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '639858', 'n_tokens_mistral': 2641, 'n_tokens_neox': 2511, 'n_words': 528}