<commit_before>#!/bin/bash

#####
# Note about extracting files using `cat`:
# On some devices (e.g. Moto X 2014) this outputs newline characters. We remove these using `sed`.
#####

function notice {
    echo -e "\033[1m$1\033[0m"
}

function success {
    echo -e "\033[1;32m$1\033[0m"
}

function error {
    echo -e "\033[1;31m$1\033[0m"
}

function fatal {
    error "$1"
    echo ""
    exit
}

notice "`basename $0` v1.2.0"

function dump_db {
    pkg=$1
    filename=$2
    notice "Dumping $pkg/$filename to dumps/$pkg/$filename..."
    mode="$(adb shell run-as $pkg ls -al /data/data/$pkg/$filename | awk '{k=0;for(i=0;i<=8;i++)k+=((substr($1,i+2,1)~/[rwx]/)*2^(8-i));if(k)printf("%0o ",k)}')"
    # make the file world-readable
    adb shell run-as $pkg chmod 777 /data/data/$pkg/$filename 1>/dev/null
    # check if the file exists
    adb shell run-as $pkg ls /data/data/$pkg/$filename | grep "No such file" 2>/dev/null
    if [ $? != 0 ]; then
        # prepare a directory
        mkdir -p `dirname dumps/$pkg/$filename` 2>/dev/null
        # attempt to pull the file
        adb pull /data/data/$pkg/$filename dumps/$pkg/$filename 2>/dev/null
        if [ $? == 0 ]; then
            success "Success!"
        else
            # couldn't pull the file; stream its contents instead, removing any end-of-line character returns
            adb shell run-as $pkg cat /data/data/$pkg/$filename | sed 's/\r$//' > dumps/$pkg/$filename
            if [ $? == 0 ]; then
                success "Success!"
            else
                error "Failed; found database, but could not pull it"
            fi
        fi
    else
        error "Failed; not installed?"
        list_files
    fi
    # restore permission on file
    adb shell run-as $pkg chmod $mode /data/data/$pkg/$filename 1>/dev/null
    if [ $? != 0 ]; then<commit_msg>attempt to resolve line feed matching on OS X's broken version of sed<commit_after>#!/bin/bash

#####
# Note about extracting files using `cat`:
# On some devices (e.g. Moto X 2014) this outputs newline characters. We remove these using `sed`.
#####

function notice {
    echo -e "\033[1m$1\033[0m"
}

function success {
    echo -e "\033[1;32m$1\033[0m"
}

function error {
    echo -e "\033[1;31m$1\033[0m"
}

function fatal {
    error "$1"
    echo ""
    exit
}

notice "`basename $0` v1.2.0"

function dump_db {
    pkg=$1
    filename=$2
    notice "Dumping $pkg/$filename to dumps/$pkg/$filename..."
    mode="$(adb shell run-as $pkg ls -al /data/data/$pkg/$filename | awk '{k=0;for(i=0;i<=8;i++)k+=((substr($1,i+2,1)~/[rwx]/)*2^(8-i));if(k)printf("%0o ",k)}')"
    # make the file world-readable
    adb shell run-as $pkg chmod 777 /data/data/$pkg/$filename 1>/dev/null
    # check if the file exists
    adb shell run-as $pkg ls /data/data/$pkg/$filename | grep "No such file" 2>/dev/null
    if [ $? != 0 ]; then
        # prepare a directory
        mkdir -p `dirname dumps/$pkg/$filename` 2>/dev/null
        # attempt to pull the file
        adb pull /data/data/$pkg/$filename dumps/$pkg/$filename 2>/dev/null
        if [ $? == 0 ]; then
            success "Success!"
        else
            # couldn't pull the file; stream its contents instead, removing any end-of-line character returns
            adb shell run-as $pkg cat /data/data/$pkg/$filename | sed $'s/\r$//' > dumps/$pkg/$filename
            if [ $? == 0 ]; then
                success "Success!"
            else
                error "Failed; found database, but could not pull it"
            fi
        fi
    else
        error "Failed; not installed?"
        list_files
    fi
    # restore permission on file
    adb shell run-as $pkg chmod $mode /data/data/$pkg/$filename 1>/dev/null
    if [ $? != 0 ]; then
{'fraction_non_alphanumeric': 0.1554072096128171, 'fraction_numerical': 0.03364485981308411, 'mean_word_length': 2.2773403324584427, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1390500', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1228, 'n_words': 377}