<jupyter_start><jupyter_text>Downloading Dogs vs Cats data, and adding more samples to sample directory<jupyter_code>%cd ./fastai_courses/deeplearning1
%cd nbs
%ls
!mkdir data
% cd data<jupyter_output>/gpfs/global_fs01/sym_shared/YPProdSpark/user/s7c0-1e5ad2b4d891fd-5791133ac135/notebook/work/fastai_courses/deeplearning1/nbs/data<jupyter_text>Download and unzip data<jupyter_code>!wget --no-check http://files.fast.ai/data/dogscats.zip
!unzip -q dogscats.zip
!ls dogscats
rm dogscats.zip
cd dogscats
!ls -1 train/dogs | wc -l
!ls -1 train/cats | wc -l
!ls -1 valid/cats | wc -l
!ls -1 test1 | wc -l
%cd ./fastai_courses/deeplearning1/nbs/data
%ls -p
!ls -1 sample/train/cats | wc -l
!ls -1 sample/train/dogs | wc -l
!ls -1 sample/valid/cats | wc -l
!ls -1 sample/valid/dogs | wc -l<jupyter_output>4
4<jupyter_text>Add more samples to sample directory<jupyter_code>!rm -r sample
!mkdir sample
!mkdir sample/train
!mkdir sample/train/dogs
!mkdir sample/train/cats
!mkdir sample/valid
!mkdir sample/valid/dogs
!mkdir sample/valid/cats
!mkdir sample/test
%pwd<jupyter_output><empty_output><jupyter_text>The following copy (100, 100, 50, 50, 16) random imagesfrom directories: train/dogs, train/cats, valid/dogs, valid/cats, test  to directories: sample/train/dogs, sample/train/cats, sample/valid/dogs, sample/valid/cats, sample/test<jupyter_code>%cd train/dogs
%%bash

sourceDir="./"
targetDir="../../sample/train/dogs"
number="100"

## Collect all file and directory names. The globstar
## bash option lets ** match all files and directories
## recursively
shopt -s globstar
dirsAndFiles=( ** )

## Get $num random numbers from 0 until
## the number of files and dirs found. This
## will let us take a random selection.
limit=$((${#dirsAndFiles[@]}-1))  
numbers=$(shuf -i 0-"$limit" -n "$number")

for num in $numbers; do
    cp -rv "${dirsAndFiles[$num]}" "$targetDir"
done
!ls -1 ../../sample/train/dogs | wc -l
%cd ..
%cd cats
%%bash

sourceDir="./"
targetDir="../../sample/train/cats"
number="100"

## Collect all file and directory names. The globstar
## bash option lets ** match all files and directories
## recursively
shopt -s globstar
dirsAndFiles=( ** )

## Get $num random numbers from 0 until
## the number of files and dirs found. This
## will let us take a random selection.
limit=$((${#dirsAndFiles[@]}-1))  
numbers=$(shuf -i 0-"$limit" -n "$number")

for num in $numbers; do
    cp -rv "${dirsAndFiles[$num]}" "$targetDir"
done
%ls -1 ../../sample/train/cats | wc -l
%cd ../../
%ls
%cd valid/dogs
%%bash

sourceDir="./"
targetDir="../../sample/valid/dogs"
number="50"

## Collect all file and directory names. The globstar
## bash option lets ** match all files and directories
## recursively
shopt -s globstar
dirsAndFiles=( ** )

## Get $num random numbers from 0 until
## the number of files and dirs found. This
## will let us take a random selection.
limit=$((${#dirsAndFiles[@]}-1))  
numbers=$(shuf -i 0-"$limit" -n "$number")

for num in $numbers; do
    cp -rv "${dirsAndFiles[$num]}" "$targetDir"
done
!ls -1 ../../sample/valid/dogs | wc -l
%cd ../cats
%%bash

sourceDir="./"
targetDir="../../sample/valid/cats"
number="50"

## Collect all file and directory names. The globstar
## bash option lets ** match all files and directories
## recursively
shopt -s globstar
dirsAndFiles=( ** )

## Get $num random numbers from 0 until
## the number of files and dirs found. This
## will let us take a random selection.
limit=$((${#dirsAndFiles[@]}-1))  
numbers=$(shuf -i 0-"$limit" -n "$number")

for num in $numbers; do
    cp -rv "${dirsAndFiles[$num]}" "$targetDir"
done
!ls -1 ../../sample/valid/cats | wc -l
%cd ../../test1
%%bash

sourceDir="./"
targetDir="../sample/test"
number="16"

## Collect all file and directory names. The globstar
## bash option lets ** match all files and directories
## recursively
shopt -s globstar
dirsAndFiles=( ** )

## Get $num random numbers from 0 until
## the number of files and dirs found. This
## will let us take a random selection.
limit=$((${#dirsAndFiles[@]}-1))  
numbers=$(shuf -i 0-"$limit" -n "$number")

for num in $numbers; do
    cp -rv "${dirsAndFiles[$num]}" "$targetDir"
done
!ls -1 ../sample/test | wc -l<jupyter_output>16<jupyter_text>Now download dogs vs cats redux from Kaggle and prepare data similar to dogs vs cats<jupyter_code>%%bash
mkdir ~/bin
cp randomCopy ~/bin
cp randomMove ~/bin
chmod a+x ~/bin/randomCopy
chmod a+x ~/bin/randomMove

cd ~/fastai_courses/deeplearning1/nbs/data
mkdir -p dogscatsredux/test
cd dogscatsredux

pip install kaggle-cli
kg config -g -u `ohassan` -p `1etmeinBaby` -c `dogs-vs-cats-redux-kernels-edition`
kg download

sudo apt-get -update
sudo apt install unzip
unzip train.zip
unzip test.zip -d ./test

mkdir train/dogs
mkdir train/cats
mv train/dog.* train/dogs
mv train/cat.* train/cats
%%bash
mkdir -p valid/dogs
mkdir valid/cats
randomMove train/dogs valid/dogs 1000
randomMove train/cats valid/cats 1000

mkdir -p sample/train/dogs
mkdir -p sample/train/cats
mkdir -p sample/valid/dogs
mkdir -p sample/valid/cats
mkdir -p sample/test/images

randomCopy train/dogs sample/train/dogs 1000
randomCopy train/cats sample/train/cats 1000
randomCopy valid/dogs sample/valid/dogs 100
randomCopy valid/cats sample/valid/cats 100
randomCopy test/test sample/test/images 16<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14702011656326378, 'fraction_numerical': 0.02030456852791878, 'mean_word_length': 4.135135135135135, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '645078', 'license': 'Apache-2.0', 'n_tokens_mistral': 2031, 'n_tokens_neox': 1877, 'n_words': 599, 'path': 'deeplearning1/nbs/prepare/prepare_lesson1.ipynb', 'repo_name': 'ohassan00/fastai_courses', 'size': '5319'}