<commit_before># <> prov:specializationOf <https://raw.github.com/timrdf/csv2rdf4lod-automation/master/bin/convert-aggregate.sh> .
#
#   Copyright 2012 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

# Aggregate conversion outputs into publishable datafiles.
#
# Up to this point, all processing was done using filenames provided by the dataset source organization.
# Put all data into $sourceID-$datasetID-$versionID dump files and construct command appropriate
# for lod-materialization.
#
#
# This script is ___NOT___ to be called directly:
# Called by conversion trigger (e.g. convert-DATASET.sh)
# The conversion trigger is created by manually invoking cr-create-conversion-trigger.sh)
# This script can also be invoked by running publish/bin/publish.sh (created by the conversion trigger)

CSV2RDF4LOD_HOME=${CSV2RDF4LOD_HOME:?"not set; source csv2rdf4lod/source-me.sh or see https://github.com/timrdf/csv2rdf4lod-automation/wiki/CSV2RDF4LOD-not-set"}

if [ "$CSV2RDF4LOD_CONVERT_DEBUG_LEVEL" == "fine" ]; then
   # The following variables are needed by this script
   # They are set by publish/bin/publish.sh before calling this script.

   echo $sourceID  `cr-source-id.sh`
   echo $datasetID `cr-dataset-id.sh`
   echo $versionID `cr-version-id.sh`

   echo $eID TODO

   echo $graph `cr-dataset-uri.sh --uri`

   CSV2RDF4LOD_FORCE_PUBLISH="true"
fi

# These directory names have become canonical.
# The flexibility to change them is no longer desirable.
convertDir=${convertDir:-automatic}   # Could be renamed to 'converted'?
#publishDir=${publishDir:-publish}    # hard coded now, as it should be.
# # # # # # # # # # # # # # # # # # # #


# These variables are embedded in the directory conventions.
baseURI=${CSV2RDF4LOD_BASE_URI_OVERRIDE:-$CSV2RDF4LOD_BASE_URI}
 sourceID=`cr-source-id.sh`     #
datasetID=`cr-dataset-id.sh`    #
versionID=`cr-version-id.sh`    #
# see https://github.com/timrdf/csv2rdf4lod-automation/wiki/Directory-Conventions

# NOTE: eID is provided by conversion trigger, but the case where it is not provided should be handled
# (e.g. when cr-publish-cockpit.sh calls this script)

graph=`cr-dataset-uri.sh --uri`

if [ "$CSV2RDF4LOD_FORCE_PUBLISH" == "true" ]; then
   echo "convert-aggregate.sh publishing raw and enhancements (forced)." | tee -a $CSV2RDF4LOD_LOG
   echo "===========================================================================================" | tee -a $CSV2RDF4LOD_LOG
else
   if [ "$CSV2RDF4LOD_PUBLISH" == "false" ]; then
         echo "convert-aggregate.sh not publishing b/c \$CSV2RDF4LOD_PUBLISH=false."                        | tee -a $CSV2RDF4LOD_LOG
         echo "===========================================================================================" | tee -a $CSV2RDF4LOD_LOG
         rm $CSV2RDF4LOD_LOG
         CSV2RDF4LOD_LOG=""
         exit 1
   fi
   rawdumps="no"; for dump in `find $convertDir -name "*.raw.ttl"`;   do rawdumps="yes"; done
   edumps="no";   for dump in `find $convertDir -name "*.e$eID.ttl"`; do edumps="yes";   done
   if [[ "$CSV2RDF4LOD_CONVERT_OMIT_RAW_LAYER" == "true" && "$CSV2RDF4LOD_PUBLISH_DELAY_UNTIL_ENHANCED" == "false" && "$edumps" == "no" ]]; then
      echo "convert-aggregate.sh delaying publishing until an enhancement is available."                                                  | tee -a $CSV2RDF4LOD_LOG
   fi
   if [ "$CSV2RDF4LOD_PUBLISH_DELAY_UNTIL_ENHANCED" == "true" ]; then
      #if [[ $runEnhancement == "yes" && ( `ls $convertDir/*.e$eID.ttl 2> /dev/null | wc -l` > 0 || ${CSV2RDF4LOD_CONVERT_EXAMPLE_SUBSET_ONLY-"."} == "true" ) ]]; then
      if [[ $runEnhancement == "yes" && ( $edumps == "yes" || "$CSV2RDF4LOD_CONVERT_EXAMPLE_SUBSET_ONLY" == "true" ) ]]; then
         echo "convert-aggregate.sh publishing raw and enhancements."                                                                     | tee -a $CSV2RDF4LOD_LOG
      else
         # NOTE: If multiple files to convert and the LAST file is not enhanced,
         #       the runEnhancement flag will be "no" and convert-aggregate.sh will not aggregate.
         # To overcome this bug, manually run publish/bin/publish.sh to force the aggregation.
         sourceme="my-csv2rdf4lod-source-me.sh"
         echo "convert-aggregate.sh delaying publishing until an enhancement is available."                                               | tee -a $CSV2RDF4LOD_LOG
         echo "  To publish with only raw, set CSV2RDF4LOD_PUBLISH_DELAY_UNTIL_ENHANCED=\"false\" in \$CSV2RDF4LOD_HOME/$sourceme.sh."    | tee -a $CSV2RDF4LOD_LOG
         echo "  To publish raw with enhanced, add enhancement to $eParamsDir/$datafile.e$eID.params.ttl and rerun convert-$datasetID.sh" | tee -a $CSV2RDF4LOD_LOG
         echo "  To force publishing now, run publish/bin/publish.sh"                                                                     | tee -a $CSV2RDF4LOD_LOG
         echo "==========================================================================================="                               | tee -a $CSV2RDF4LOD_LOG
         rm $CSV2RDF4LOD_LOG
         CSV2RDF4LOD_LOG=""
         exit 1
      fi
   fi
   newest_publication="publish/`ls -lt publish/ | grep -v "total" | grep -v "bin" | head -1 | awk '{print $NF}'`"
   if [ -e "$newest_publication" ]; then
      newest_automatic="automatic/`find automatic -newer $newest_publication -and -not -name "*params*" | head -1`"
      if [ "$newest_automatic" == "automatic/" ]; then
         echo "INFO `basename $0` skipping aggregation into publish/ because nothing in automatic/ is newer."
         exit
      fi
   fi
fi

if [ ! -e publish/bin ]; then
   mkdir -p publish/bin
fi
touch publish

myMD5=`md5.sh $0`

if [ ${#versionID} -le 0 ]; then
   # For legacy compatibility. datasetVersion was renamed to versionID.
   versionID=$datasetVersion
fi
    S_D_V=$sourceID-$datasetID-$versionID
     pSDV=publish/$sourceID-$datasetID-$versionID
   allRaw=publish/$sourceID-$datasetID-$versionID.raw.ttl
    allEX=publish/$sourceID-$datasetID-$versionID.e$eID.ttl # only the current enhancement.
   allTTL=publish/$sourceID-$datasetID-$versionID.ttl
    allNT=publish/$sourceID-$datasetID-$versionID.nt
allRDFXML=publish/$sourceID-$datasetID-$versionID.rdf
  allVOID=publish/$sourceID-$datasetID-$versionID.void.ttl
allVOIDNT=publish/$sourceID-$datasetID-$versionID.void.nt
   allPML=publish/$sourceID-$datasetID-$versionID.pml.ttl
allSAMEAS=publish/$sourceID-$datasetID-$versionID.sameas.nt
rawSAMPLE=publish/$sourceID-$datasetID-$versionID.raw.sample.ttl
versionedDatasetURI="$baseURI/source/$sourceID/dataset/$datasetID/version/$versionID"
     rawSampleGraph="$baseURI/source/$sourceID/dataset/$datasetID/version/$versionID/conversion/raw/subset/sample"
         http_allNT="$baseURI/source/${sourceID}/file/${datasetID}/version/${versionID}/conversion/${sourceID}-${datasetID}-${versionID}.nt"
        http_allTTL="$baseURI/source/${sourceID}/file/${datasetID}/version/${versionID}/conversion/${sourceID}-${datasetID}-${versionID}.ttl"
     http_allRDFXML="$baseURI/source/${sourceID}/file/${datasetID}/version/${versionID}/conversion/${sourceID}-${datasetID}-${versionID}.rdf"

# Special case needs:
    allNT_L=$sourceID-$datasetID-$versionID.nt        # L: Local name (not including directory; for use when pushd'ing to cHuNk)
allSAMEAS_L=$sourceID-$datasetID-$versionID.sameas.nt

filesToCompress=""

# TODO: align with CSV2RDF4LOD_CONVERT_DUMP_FILE_EXTENSIONS                 "cr:auto" => ttl.gz
# bin/util/dump-file-extensions.sh defaults to gz
zip="gz"
if [ ! `which rapper` ]; then
   # check if rapper is on path, if not, report error.
   echo "NOTE: rapper not found. Some serializations will probably be empty." | tee -a $CSV2RDF4LOD_LOG
fi

or_see_github="or see https://github.com/timrdf/csv2rdf4lod-automation/wiki/CSV2RDF4LOD-not-set"

#
# Raw ttl
#
conversionIDs="raw"
layerSlugs="raw"
convertedRaw="no"; for raw in `find automatic -name "*.raw.ttl"`; do convertedRaw="yes"; done
if [ $convertedRaw == "yes" ]; then
   echo $allRaw | tee -a $CSV2RDF4LOD_LOG
   cat automatic/*.raw.ttl > $allRaw
   filesToCompress="$allRaw"
else
   echo "$allRaw - omitted" | tee -a $CSV2RDF4LOD_LOG
fi

#
# Sample of raw (enhanced samples done below)
#
convertedRawSamples="no"; for raw in `find automatic -name "*.raw.sample.ttl"`; do convertedRaw="yes"; done
if [ $convertedRawSamples == "yes" ]; then
   echo $pSDV.raw.sample.ttl | tee -a $CSV2RDF4LOD_LOG
   cat automatic/*.raw.sample.ttl > $pSDV.raw.sample.ttl
else
   echo "INFO: no automatic/*raw.sample.ttl found." 
fi

#
# Individual enhancement ttl (any that are not aggregated)
#
# Got messed up when added sample.ttl:
#     enhancementLevels=`ls $convertDir/*.e*.ttl 2> /dev/null | grep -v void.ttl | sed -e 's/^.*\.e\([^.]*\).ttl/\1/' | sort -u`
# This works, but moved to script:
#     enhancementLevels=`find $convertDir -name "*.e[!.].ttl" | sed -e 's/^.*\.e\([^.]*\).ttl/\1/' | sort -u` # WARNING: only handles e1 through e9<commit_msg>publish stopped premature<commit_after># <> prov:specializationOf <https://raw.github.com/timrdf/csv2rdf4lod-automation/master/bin/convert-aggregate.sh> .
#
#   Copyright 2012 <NAME>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

# Aggregate conversion outputs into publishable datafiles.
#
# Up to this point, all processing was done using filenames provided by the dataset source organization.
# Put all data into $sourceID-$datasetID-$versionID dump files and construct command appropriate
# for lod-materialization.
#
#
# This script is ___NOT___ to be called directly:
# Called by conversion trigger (e.g. convert-DATASET.sh)
# The conversion trigger is created by manually invoking cr-create-conversion-trigger.sh)
# This script can also be invoked by running publish/bin/publish.sh (created by the conversion trigger)

CSV2RDF4LOD_HOME=${CSV2RDF4LOD_HOME:?"not set; source csv2rdf4lod/source-me.sh or see https://github.com/timrdf/csv2rdf4lod-automation/wiki/CSV2RDF4LOD-not-set"}

if [ "$CSV2RDF4LOD_CONVERT_DEBUG_LEVEL" == "fine" ]; then
   # The following variables are needed by this script
   # They are set by publish/bin/publish.sh before calling this script.

   echo $sourceID  `cr-source-id.sh`
   echo $datasetID `cr-dataset-id.sh`
   echo $versionID `cr-version-id.sh`

   echo $eID TODO

   echo $graph `cr-dataset-uri.sh --uri`

   CSV2RDF4LOD_FORCE_PUBLISH="true"
fi

# These directory names have become canonical.
# The flexibility to change them is no longer desirable.
convertDir=${convertDir:-automatic}   # Could be renamed to 'converted'?
#publishDir=${publishDir:-publish}    # hard coded now, as it should be.
# # # # # # # # # # # # # # # # # # # #


# These variables are embedded in the directory conventions.
baseURI=${CSV2RDF4LOD_BASE_URI_OVERRIDE:-$CSV2RDF4LOD_BASE_URI}
 sourceID=`cr-source-id.sh`     #
datasetID=`cr-dataset-id.sh`    #
versionID=`cr-version-id.sh`    #
# see https://github.com/timrdf/csv2rdf4lod-automation/wiki/Directory-Conventions

# NOTE: eID is provided by conversion trigger, but the case where it is not provided should be handled
# (e.g. when cr-publish-cockpit.sh calls this script)

graph=`cr-dataset-uri.sh --uri`

if [ "$CSV2RDF4LOD_FORCE_PUBLISH" == "true" ]; then
   echo "convert-aggregate.sh publishing raw and enhancements (forced)." | tee -a $CSV2RDF4LOD_LOG
   echo "===========================================================================================" | tee -a $CSV2RDF4LOD_LOG
else
   if [ "$CSV2RDF4LOD_PUBLISH" == "false" ]; then
         echo "convert-aggregate.sh not publishing b/c \$CSV2RDF4LOD_PUBLISH=false."                        | tee -a $CSV2RDF4LOD_LOG
         echo "===========================================================================================" | tee -a $CSV2RDF4LOD_LOG
         rm $CSV2RDF4LOD_LOG
         CSV2RDF4LOD_LOG=""
         exit 1
   fi
   rawdumps="no"; for dump in `find $convertDir -name "*.raw.ttl"`;   do rawdumps="yes"; done
   edumps="no";   for dump in `find $convertDir -name "*.e$eID.ttl"`; do edumps="yes";   done
   if [[ "$CSV2RDF4LOD_CONVERT_OMIT_RAW_LAYER" == "true" && "$CSV2RDF4LOD_PUBLISH_DELAY_UNTIL_ENHANCED" == "false" && "$edumps" == "no" ]]; then
      echo "convert-aggregate.sh delaying publishing until an enhancement is available."                                                  | tee -a $CSV2RDF4LOD_LOG
   fi
   if [ "$CSV2RDF4LOD_PUBLISH_DELAY_UNTIL_ENHANCED" == "true" ]; then
      #if [[ $runEnhancement == "yes" && ( `ls $convertDir/*.e$eID.ttl 2> /dev/null | wc -l` > 0 || ${CSV2RDF4LOD_CONVERT_EXAMPLE_SUBSET_ONLY-"."} == "true" ) ]]; then
      if [[ $runEnhancement == "yes" && ( $edumps == "yes" || "$CSV2RDF4LOD_CONVERT_EXAMPLE_SUBSET_ONLY" == "true" ) ]]; then
         echo "convert-aggregate.sh publishing raw and enhancements."                                                                     | tee -a $CSV2RDF4LOD_LOG
      else
         # NOTE: If multiple files to convert and the LAST file is not enhanced,
         #       the runEnhancement flag will be "no" and convert-aggregate.sh will not aggregate.
         # To overcome this bug, manually run publish/bin/publish.sh to force the aggregation.
         sourceme="my-csv2rdf4lod-source-me.sh"
         echo "convert-aggregate.sh delaying publishing until an enhancement is available."                                               | tee -a $CSV2RDF4LOD_LOG
         echo "  To publish with only raw, set CSV2RDF4LOD_PUBLISH_DELAY_UNTIL_ENHANCED=\"false\" in \$CSV2RDF4LOD_HOME/$sourceme.sh."    | tee -a $CSV2RDF4LOD_LOG
         echo "  To publish raw with enhanced, add enhancement to $eParamsDir/$datafile.e$eID.params.ttl and rerun convert-$datasetID.sh" | tee -a $CSV2RDF4LOD_LOG
         echo "  To force publishing now, run publish/bin/publish.sh"                                                                     | tee -a $CSV2RDF4LOD_LOG
         echo "==========================================================================================="                               | tee -a $CSV2RDF4LOD_LOG
         rm $CSV2RDF4LOD_LOG
         CSV2RDF4LOD_LOG=""
         exit 1
      fi
   fi
   newest_publication="publish/`ls -lt publish/ | grep -v "total" | grep -v "bin" | head -1 | awk '{print $NF}'`"
   if [ -e "$newest_publication" ]; then
      newest_automatic="automatic/`find automatic -newer $newest_publication -and -not -name "*params*" | head -1`"
      if [ "$newest_automatic" == "automatic/" ]; then
         echo "INFO `basename $0` skipping aggregation into publish/ because nothing in automatic/ is newer."
         exit
      fi
   fi
fi

if [ ! -e publish/bin ]; then
   mkdir -p publish/bin
fi
touch publish

myMD5=`md5.sh $0`

if [ ${#versionID} -le 0 ]; then
   # For legacy compatibility. datasetVersion was renamed to versionID.
   versionID=$datasetVersion
fi
    S_D_V=$sourceID-$datasetID-$versionID
     pSDV=publish/$sourceID-$datasetID-$versionID
   allRaw=publish/$sourceID-$datasetID-$versionID.raw.ttl
    allEX=publish/$sourceID-$datasetID-$versionID.e$eID.ttl # only the current enhancement.
   allTTL=publish/$sourceID-$datasetID-$versionID.ttl
    allNT=publish/$sourceID-$datasetID-$versionID.nt
allRDFXML=publish/$sourceID-$datasetID-$versionID.rdf
  allVOID=publish/$sourceID-$datasetID-$versionID.void.ttl
allVOIDNT=publish/$sourceID-$datasetID-$versionID.void.nt
   allPML=publish/$sourceID-$datasetID-$versionID.pml.ttl
allSAMEAS=publish/$sourceID-$datasetID-$versionID.sameas.nt
rawSAMPLE=publish/$sourceID-$datasetID-$versionID.raw.sample.ttl
versionedDatasetURI="$baseURI/source/$sourceID/dataset/$datasetID/version/$versionID"
     rawSampleGraph="$baseURI/source/$sourceID/dataset/$datasetID/version/$versionID/conversion/raw/subset/sample"
         http_allNT="$baseURI/source/${sourceID}/file/${datasetID}/version/${versionID}/conversion/${sourceID}-${datasetID}-${versionID}.nt"
        http_allTTL="$baseURI/source/${sourceID}/file/${datasetID}/version/${versionID}/conversion/${sourceID}-${datasetID}-${versionID}.ttl"
     http_allRDFXML="$baseURI/source/${sourceID}/file/${datasetID}/version/${versionID}/conversion/${sourceID}-${datasetID}-${versionID}.rdf"

# Special case needs:
    allNT_L=$sourceID-$datasetID-$versionID.nt        # L: Local name (not including directory; for use when pushd'ing to cHuNk)
allSAMEAS_L=$sourceID-$datasetID-$versionID.sameas.nt

filesToCompress=""

# TODO: align with CSV2RDF4LOD_CONVERT_DUMP_FILE_EXTENSIONS                 "cr:auto" => ttl.gz
# bin/util/dump-file-extensions.sh defaults to gz
zip="gz"
if [ ! `which rapper` ]; then
   # check if rapper is on path, if not, report error.
   echo "NOTE: rapper not found. Some serializations will probably be empty." | tee -a $CSV2RDF4LOD_LOG
fi

or_see_github="or see https://github.com/timrdf/csv2rdf4lod-automation/wiki/CSV2RDF4LOD-not-set"

#
# Raw ttl
#
conversionIDs="raw"
layerSlugs="raw"
convertedRaw="no"; for raw in `find automatic -name "*.raw.ttl"`; do convertedRaw="yes"; done
if [ $convertedRaw == "yes" ]; then
   echo $allRaw | tee -a $CSV2RDF4LOD_LOG
   cat automatic/*.raw.ttl > $allRaw
   filesToCompress="$allRaw"
else
   echo "$allRaw - omitted" | tee -a $CSV2RDF4LOD_LOG
fi

#
# Sample of raw (enhanced samples done below)
#
convertedRawSamples="no"; for raw in `find automatic -name "*.raw.sample.ttl"`; do convertedRawSamples="yes"; done
if [ $convertedRawSamples == "yes" ]; then
   echo $pSDV.raw.sample.ttl | tee -a $CSV2RDF4LOD_LOG
   cat automatic/*.raw.sample.ttl > $pSDV.raw.sample.ttl
else
   echo "INFO: no automatic/*raw.sample.ttl found." 
fi

#
# Individual enhancement ttl (any that are not aggregated)
#
# Got messed up when added sample.ttl:
#     enhancementLevels=`ls $convertDir/*.e*.ttl 2> /dev/null | grep -v void.ttl | sed -e 's/^.*\.e\([^.]*\).ttl/\1/' | sort -u`
# This works, but moved to script:
#     enhancementLevels=`find $convertDir -name "*.e[!.].ttl" | sed -e 's/^.*\.e\([^.]*\).ttl/\1/' | sort -u` # WARNING: only handles e1 through e9
{'fraction_non_alphanumeric': 0.15308433861876417, 'fraction_numerical': 0.011800031607227519, 'mean_word_length': 3.465772759350741, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708828', 'n_tokens_mistral': 6438, 'n_tokens_neox': 6187, 'n_words': 1537}