<commit_before>#!/bin/bash

mongourl=localhost:27017

collectionName=$1
fileName=$2

if [ -z "$collectionName" ]
  then echo "Collection name required."
   exit 1
fi
if [ -z "$fileName" ]
  then echo "File name required."
   exit 1
fi

tail -n 3 "$fileName" | wc -c | xargs -I {} truncate "$fileName" -s -{}

mongoimport --host $mongourl --db sa --collection $collectionName --ignoreBlanks --type csv --headerline $fileName

mongo $mongourl/sa --eval "
var count = 0;
db.$collectionName.find({RET_TAXPAYER_REFERENCE: {\$exists: true}}).forEach(function(x) {
    x.RET_TAXPAYER_REFERENCE = '' + x.RET_TAXPAYER_REFERENCE;
    db.$collectionName.save(x);
    count = count + 1;
    print('RET_TAXPAYER_REFERENCE converted to String for ' + count + ' documents.');
});
'RET_TAXPAYER_REFERENCE converted to String for ' + count + ' documents.';"<commit_msg>TSV-823 Improved import-to-mongo.sh log statement<commit_after>#!/bin/bash

mongourl=localhost:27017

collectionName=$1
fileName=$2

if [ -z "$collectionName" ]
  then echo "Collection name required."
   exit 1
fi
if [ -z "$fileName" ]
  then echo "File name required."
   exit 1
fi

tail -n 3 "$fileName" | wc -c | xargs -I {} truncate "$fileName" -s -{}

mongoimport --host $mongourl --db sa --collection $collectionName --ignoreBlanks --type csv --headerline $fileName

mongo $mongourl/sa --eval "
var count = 0;
db.$collectionName.find({RET_TAXPAYER_REFERENCE: {\$exists: true}}).forEach(function(x) {
    x.RET_TAXPAYER_REFERENCE = '' + x.RET_TAXPAYER_REFERENCE;
    db.$collectionName.save(x);
    count = count + 1;
    print('RET_TAXPAYER_REFERENCE converted to String for ' + count + ' documents.');
});
'RET_TAXPAYER_REFERENCE converted to String for ' + count + ' documents in total.';"
{'fraction_non_alphanumeric': 0.14636935391652373, 'fraction_numerical': 0.015437392795883362, 'mean_word_length': 3.9857549857549857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '502355', 'n_tokens_mistral': 644, 'n_tokens_neox': 630, 'n_words': 161}