<commit_before>#!/bin/bash

# Page scanner - Scans the given domain for href-tags

function help { # Might connect to actual command later, focus on other scripts now
  echo "Script1 is a page scanner that downloads the given url, searches for hrefs-tags and present their values in a structured fashion."
  echo "Usage: ./script1.bash <DOMAIN> <URL>"
  echo "Ex. ./script1.bash mechani.se /"
  echo ""
  echo "Commands:"
  echo "-h,--help                                             Displays helptext"
}

DOMAIN=$1

URL=$2

SOURCE=$(curl -s $DOMAIN$URL) # Get the source from our given parameters, keep it silent to avoid annoying output

STR=""

for i in $(echo $SOURCE | grep -Po '(?<=href=")[^"]*' ); do # Get all the href-tags value and loop through
  if [[ $i == *"$DOMAIN"* ]] || [[ $i != "http"* ]]; then # Only save the records that is within the domain
    TARGETDOMAIN=$DOMAIN
    if [[ $i == "/"* ]]; then
      TARGETURL=$i
    else
      TARGETURL="/$i"
    fi
    STR="$STR$DOMAIN $URL $TARGETDOMAIN $TARGETURL\n"
  fi
done

echo -e $STR<commit_msg>Added comment for revisiting<commit_after>#!/bin/bash

# Page scanner - Scans the given domain for href-tags

function help { # Might connect to actual command later, focus on other scripts now
  echo "Script1 is a page scanner that downloads the given url, searches for hrefs-tags and present their values in a structured fashion."
  echo "Usage: ./script1.bash <DOMAIN> <URL>"
  echo "Ex. ./script1.bash mechani.se /"
  echo ""
  echo "Commands:"
  echo "-h,--help                                             Displays helptext"
}

DOMAIN=$1

URL=$2

SOURCE=$(curl -s $DOMAIN$URL) # Get the source from our given parameters, keep it silent to avoid annoying output

STR=""

for i in $(echo $SOURCE | grep -Po '(?<=href=")[^"]*' ); do # Get all the href-tags value and loop through
  if [[ $i == *"$DOMAIN"* ]] || [[ $i != "http"* ]]; then # Only save the records that is within the domain
    TARGETDOMAIN=$DOMAIN
    if [[ $i == "/"* ]]; then # ------------------------------------------------------------------------------ May need to revisit this since DOMAIN is accepted as a href-value
      TARGETURL=$i
    else
      TARGETURL="/$i"
    fi
    STR="$STR$DOMAIN $URL $TARGETDOMAIN $TARGETURL\n"
  fi
done

echo -e $STR
{'fraction_non_alphanumeric': 0.1615720524017467, 'fraction_numerical': 0.004366812227074236, 'mean_word_length': 2.5300462249614792, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3996819', 'n_tokens_mistral': 708, 'n_tokens_neox': 677, 'n_words': 272}