<commit_before>#!/bin/sh
#################################################
# 
# Purpose: Create a CSV table of licenses for OSGeo-Live documentation
# Author:  <NAME>
#
#################################################
# Copyright (c) 2011 Open Source Geospatial Foundation (OSGeo)
# Copyright (c) 2011 LISAsoft
#
# Licensed under the GNU LGPL.
# 
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 2.1 of the License,
# or any later version.  This library is distributed in the hope that
# it will be useful, but WITHOUT ANY WARRANTY, without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details, either
# in the "LICENSE.LGPL.txt" file distributed with this software or at
# web page "http://www.fsf.org/licenses/lgpl.html".
##################################################

# Store the script root directory for later
cd `dirname "$0"`
BUILD_DIR=`pwd`

# Title
#echo '"Document","Authors(s)", Reviewer(s),"License(s)"'

for DIR in  ../doc/en/overview ../doc/en/quickstart ;  do
  cd "$BUILD_DIR/$DIR"

  for FILE in *.rst ;  do
    # Document
    echo -n "\""
    echo -n "$FILE" | sed -e 's/.rst$//'
    echo -n "\",\""

    # Authors
    awk '/^:Author:/{printf("%s;", $0)}' "$FILE" | \
      sed -e 's/:Author:\s*//g'

    # Reviewers
    echo -n "\",\""
    awk '/^:Reviewer:/{printf("%s;", $0)}' "$FILE" | \
      sed -e 's/:Reviewer:\s*//g'

    # License(s)
    echo -n "\",\""
    awk '/^:License:/{printf("%s", $0)}' "$FILE" | \
      sed -e 's/:License:\s*//g'
    echo "\""

  done
done
<commit_msg>cosmetic cleanup, skip index pages<commit_after>#!/bin/sh
#################################################
# 
# Purpose: Create a CSV table of licenses for OSGeo-Live documentation
# Author:  <NAME>
#
#################################################
# Copyright (c) 2011 Open Source Geospatial Foundation (OSGeo)
# Copyright (c) 2011 LISAsoft
#
# Licensed under the GNU LGPL.
# 
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 2.1 of the License,
# or any later version.  This library is distributed in the hope that
# it will be useful, but WITHOUT ANY WARRANTY, without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details, either
# in the "LICENSE.LGPL.txt" file distributed with this software or at
# web page "http://www.fsf.org/licenses/lgpl.html".
##################################################

# Store the script root directory for later
cd `dirname "$0"`
BUILD_DIR=`pwd`

# Title
#echo '"Document","Authors(s)", Reviewer(s),"License(s)"'

for DIR in  ../doc/en/overview ../doc/en/quickstart ;  do
  cd "$BUILD_DIR/$DIR"

  for FILE in *.rst ;  do
    if [ "$FILE" = "overview.rst" ] ||  [ "$FILE" = "quickstart.rst" ] ; then
        continue
    fi

    # Document
    echo -n '"'
    echo -n "$FILE" | sed -e 's/.rst$//'
    echo -n '","'

    # Authors
    awk '/^:Author:/{printf("%s; ", $0)}' "$FILE" | \
      sed -e 's/:Author:\s*//g' -e 's/; $//'

    # Reviewers
    echo -n '","'
    awk '/^:Reviewer:/{printf("%s; ", $0)}' "$FILE" | \
      sed -e 's/:Reviewer:\s*//g' -e 's/; $//'

    # License(s)
    echo -n '","'
    awk '/^:License:/{printf("%s", $0)}' "$FILE" | \
      sed -e 's/:License:\s*//g'
    echo "\""

  done
done

{'fraction_non_alphanumeric': 0.22992600712523978, 'fraction_numerical': 0.007673335160317895, 'mean_word_length': 3.334916864608076, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7184749', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1132, 'n_words': 400}