<jupyter_start><jupyter_text>Copyright (C) Microsoft Corporation.  see license file for details  Builds docker image for training (experimentation)Runs basic tests and pushes image to dockerhub<jupyter_code># Allow multiple displays per cell
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
%load_ext dotenv
dotenv_file_path='./../not_shared/general.env'
import os
!pwd
!ls -l ./
%dotenv $dotenv_file_path

# Your docker login and image repository name
docker_login = os.getenv('docker_login') 
image_tag = os.getenv('image_tag')
image_version = os.getenv('image_version')

docker_login
image_tag
image_version
%dotenv
os.getenv('image_version')
dockerfile_name = 'dockerfile'+ '_' + os.getenv('image_version')
%%writefile ./../docker/$dockerfile_name

FROM nvidia/cuda:10.0-cudnn7-devel
# FROM nvidia/cuda:9.2-cudnn7-devel
#FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04

MAINTAINER <NAME> <<EMAIL>>
#based on https://github.com/keras-team/keras/blob/master/docker/Dockerfile

# Install system packages
RUN apt-get update && apt-get install -y --no-install-recommends \
      apt-utils \
      bzip2 \
      g++ \
      git \
      graphviz \
      libgl1-mesa-glx \
      libhdf5-dev \
      openmpi-bin \
      wget \
      ssh \
      rsync && \
    rm -rf /var/lib/apt/lists/*

# Install conda
ENV CONDA_DIR /opt/conda
ENV PATH $CONDA_DIR/bin:$PATH

RUN wget --quiet --no-check-certificate https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \
    echo "c59b3dd3cad550ac7596e0d599b91e75d88826db132e4146030ef471bb434e9a *Miniconda3-4.2.12-Linux-x86_64.sh" | sha256sum -c - && \
    /bin/bash /Miniconda3-4.2.12-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
    rm Miniconda3-4.2.12-Linux-x86_64.sh && \
    echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh        

# Install AzCopy        
RUN mkdir -p /tmp/azcopy && mkdir -p /azcopy10 && \
    wget -O /tmp/azcopy/azcopyv10.tar.gz https://aka.ms/downloadazcopy-v10-linux &&  \
    tar -xf /tmp/azcopy/azcopyv10.tar.gz -C /azcopy10 
RUN rm -rf /tmp/azcopy
ENV PATH=/azcopy10/azcopy_linux_amd64_10.3.2:$PATH

# Install Python packages and keras
# ENV NB_USER keras
# ENV NB_UID 1000

# RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
#     chown $NB_USER $CONDA_DIR -R && \
#     mkdir -p /src && \
#     chown $NB_USER /src

# USER $NB_USER
RUN mkdir -p /sr

ARG python_version=3.6

RUN conda config --append channels conda-forge
RUN conda install -y python=${python_version} && \
    pip install --upgrade pip && \
    pip install \
      sklearn_pandas \
      tensorflow-gpu==1.14 \
      python-dotenv \
      papermill[azure] &&  \
    conda install \
      bcolz \
      h5py \
      imageio \
      matplotlib \
      mkl \
      numpy \
      nose \
      notebook \
      opencv \
      Pillow \
      pandas \
      pydot \
      pygpu \
      pyyaml \
      scikit-learn \
      scikit-image \
      six \
      theano \
      mkdocs \
      tqdm \
      && \
    conda install pytorch torchvision cudatoolkit=9.2 -c pytorch && \
    git clone git://github.com/keras-team/keras.git /src && pip install -e /src[tests] && \
    pip install git+git://github.com/keras-team/keras.git && \
    pip install git+https://github.com/aleju/imgaug && \
    conda clean -yt

# TF 1.13:cuda 10.0, cudnn 7 
# RUN conda install -y python=${python_version} && \
#     pip install --upgrade pip && \
#     pip install \
#       sklearn_pandas \
#       scikit-image \
#       tensorflow-gpu==1.14 \
#       tensorflow-tensorboard \
#       matplotlib \
#       numpy && \
#     conda install \
#       bcolz \
#       h5py \
#       matplotlib \
#       mkl \
#       nose \
#       notebook \
#       opencv \
#       Pillow \
#       pandas \
#       pydot \
#       pygpu \
#       pyyaml \
#       requests \
#       scikit-learn \
#       six \
#       tqdm \
#       pytorch torchvision cudatoolkit=9.0 -c pytorch && \
#     conda install -c conda-forge imageio && \
#     git clone git://github.com/keras-team/keras.git /src && pip install -e /src[tests] && \
#     pip install git+git://github.com/keras-team/keras.git && \
#     pip install git+https://github.com/aleju/imgaug && \
#     pip install git+https://www.github.com/keras-team/keras-contrib.git && \
#     conda clean -yt     

#ADD theanorc /home/keras/.theanorc

ENV PYTHONPATH='/src/:$PYTHONPATH'

WORKDIR /src

EXPOSE 8888

# CMD jupyter notebook --port=8888 --ip=0.0.0.0

%dotenv
docker_image_name = os.getenv('docker_login') + os.getenv('image_tag') + ':' + os.getenv('image_version')
docker_file_location = os.path.join(*(['.', '..','docker', dockerfile_name]))
working_path = '.'

docker_image_name
docker_file_location
!mkdir -p ./docker_history
!cp $docker_file_location ./docker_history/
!echo docker build -t $docker_image_name -f $docker_file_location $working_path
!docker build -t $docker_image_name -f $docker_file_location $working_path
cli_command_inside_container=' /bin/bash -c "conda env list; pwd; ls -l /workspace ; ' + \
    'pip3 list | grep tensorflow;' + \
    'python -c \'import tensorflow as tf; ' +\
    'from tensorflow.python.client import device_lib; ' +\
    'print(tf.__version__);' +\
    'gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1);' +\
    'sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options));' +\
    'local_device_protos = device_lib.list_local_devices(); ' +\
    'print( [x.name for x in local_device_protos if x.device_type == \\"GPU\\"]); ' +\
    'sess.close() ' +\
    '\';' +\
    '"'



cli_command_inside_container

cli_command='docker run --gpus all -it --rm  --name chestxray-no-aml-gpu_docker_container ' + \
'-v $(pwd):/workspace:rw ' + \
docker_image_name + \
cli_command_inside_container

cli_command
!$cli_command
cli_command = 'docker login  --username '+ os.getenv('docker_login') +' --password '+os.getenv('docker_password')
# cli_command
!$cli_command
!docker push $docker_image_name
!jupyter nbconvert --to html 010_create_training_docker_image<jupyter_output>[NbConvertApp] Converting notebook 010_create_training_docker_image.ipynb to html
[NbConvertApp] Writing 290232 bytes to 010_create_training_docker_image.html
{'fraction_non_alphanumeric': 0.13605005615273544, 'fraction_numerical': 0.027274185785336115, 'mean_word_length': 2.728468899521531, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '267770', 'license': 'MIT', 'n_tokens_mistral': 2479, 'n_tokens_neox': 2306, 'n_words': 462, 'path': 'code/010_create_training_docker_image.ipynb', 'repo_name': 'georgeAccnt-GH/AzureChestXRayNoAML', 'size': '6259'}