<commit_before>#!/usr/bin/env bash

set -e

SQL_FILE="$1"

maindomain="$2"

localdomain="$3"

# @link https://stackoverflow.com/questions/29902647/sed-match-replace-url-and-update-serialized-array-count
# @link https://serverfault.com/questions/1114188/php-serialize-awk-command-speed-up/1114191#1114191
sed 's/;s:/;\ns:/g' "$SQL_FILE" | \
  awk -F'"' '/s:.+'$maindomain'/ {sub("'$maindomain'", "'$localdomain'"); n=length($2)-1; sub(/:[[:digit:]]+:/, ":" n ":")} 1' | \
  sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/ /g' | \
  sed "s/$maindomain/$localdomain/g" > "$SQL_FILE.txt"

rm "$SQL_FILE"

mv "$SQL_FILE.txt" "$SQL_FILE"<commit_msg>updating migration, adding old sql back 2 the mix<commit_after>#!/usr/bin/env bash

set -e

SQL_FILE="$1"

maindomain="$2"

localdomain="$3"

cp "$SQL_FILE" "$SQL_FILE.old.sql"

# @link https://stackoverflow.com/questions/29902647/sed-match-replace-url-and-update-serialized-array-count
# @link https://serverfault.com/questions/1114188/php-serialize-awk-command-speed-up/1114191#1114191
sed 's/;s:/;\ns:/g' "$SQL_FILE" | \
  awk -F'"' '/s:.+'$maindomain'/ {sub("'$maindomain'", "'$localdomain'"); n=length($2)-1; sub(/:[[:digit:]]+:/, ":" n ":")} 1' | \
  sed "s/$maindomain/$localdomain/g" > "$SQL_FILE.txt"

mv "$SQL_FILE.txt" "$SQL_FILE"
{'fraction_non_alphanumeric': 0.2549019607843137, 'fraction_numerical': 0.05568627450980392, 'mean_word_length': 4.523809523809524, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6258346', 'n_tokens_mistral': 583, 'n_tokens_neox': 526, 'n_words': 80}