From ff8079a18b7988efb03bf0694933fb771ceb607d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Ruales?= Date: Tue, 30 Oct 2018 23:36:39 -0700 Subject: [PATCH 0001/1210] Fix broken link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 655c3977..453a4ca8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![example script](http://i.imgur.com/yrWycNA.png) -This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://https://gcr.io/kaggle-images/python) the Docker image on Google Container Registry. +This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://gcr.io/kaggle-images/python) the Docker image on Google Container Registry. ## Getting started From 8fcae6112175c2a1eef8f1c7246cb7b43fffadeb Mon Sep 17 00:00:00 2001 From: Wendy Kan Date: Mon, 19 Nov 2018 14:10:35 -0800 Subject: [PATCH 0002/1210] adding kaggle api --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 71f236c3..a8212c59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -475,6 +475,7 @@ RUN pip install flashtext && \ pip install PDPbox && \ pip install ggplot && \ pip install cesium && \ + pip install kaggle && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From 40499323b7c83503ccfe849120e8cf1e1cbf78bd Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 20 Nov 2018 09:38:46 -0800 Subject: [PATCH 0003/1210] Update ImageMagick binary url --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a8212c59..abda466c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN pip install --upgrade seaborn python-dateutil dask && \ libpaper-utils libpaper1 libpixman-1-0 libpng16-16 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ cd /usr/local/src && \ - wget http://transloadit.imagemagick.org/download/ImageMagick.tar.gz && \ + wget https://imagemagick.org/download/ImageMagick.tar.gz && \ tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files From 922c5a9030748ebab4da23d7dbdd4cb89a7b86a1 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 21 Nov 2018 09:06:11 -0800 Subject: [PATCH 0004/1210] Remove the kaggle api package This breaks the twosigma competition because the code for both lives in a python package called `kaggle`. The code for the twosigma competition under `kaggle.competitions` should probably be move. Additionally, the kaggle api client package requires an API secret key. We don't offer a way to manage secrets for now and we don't want to encourage people adding their secret in the code (easy to leak if ever made public). --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index abda466c..0aff17f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -475,7 +475,6 @@ RUN pip install flashtext && \ pip install PDPbox && \ pip install ggplot && \ pip install cesium && \ - pip install kaggle && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From 5976f3c19b36060790863af361b769786643a72c Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 18 Sep 2018 22:22:43 +0000 Subject: [PATCH 0005/1210] Create process to prebuild tensorflow wheels --- tensorflow-whl/CHANGELOG.md | 1 + tensorflow-whl/Dockerfile | 100 ++++++++++++++++++++++++++++++++++++ tensorflow-whl/README.md | 22 ++++++++ tensorflow-whl/build | 11 ++++ tensorflow-whl/push | 25 +++++++++ 5 files changed, 159 insertions(+) create mode 100644 tensorflow-whl/CHANGELOG.md create mode 100644 tensorflow-whl/Dockerfile create mode 100644 tensorflow-whl/README.md create mode 100755 tensorflow-whl/build create mode 100755 tensorflow-whl/push diff --git a/tensorflow-whl/CHANGELOG.md b/tensorflow-whl/CHANGELOG.md new file mode 100644 index 00000000..52ab3b13 --- /dev/null +++ b/tensorflow-whl/CHANGELOG.md @@ -0,0 +1 @@ +1.11.0-py36: Tensorflow 1.11.0 wheels built with python 3.6 diff --git a/tensorflow-whl/Dockerfile b/tensorflow-whl/Dockerfile new file mode 100644 index 00000000..db5bc8df --- /dev/null +++ b/tensorflow-whl/Dockerfile @@ -0,0 +1,100 @@ +FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 AS nvidia +FROM continuumio/anaconda3:5.0.1 + +# Avoid interactive configuration prompts/dialogs during apt-get. +ENV DEBIAN_FRONTEND=noninteractive + +# This is necessary to for apt to access HTTPS sources +RUN apt-get update && \ + apt-get install apt-transport-https + +# Cuda support +COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ +COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ +COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg + +ENV CUDA_VERSION=9.1.85 +ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 +LABEL com.nvidia.volumes.needed="nvidia_driver" +LABEL com.nvidia.cuda.version="${CUDA_VERSION}" +ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +# The stub is useful to us both for built-time linking and run-time linking, on CPU-only systems. +# When intended to be used with actual GPUs, make sure to (besides providing access to the host +# CUDA user libraries, either manually or through the use of nvidia-docker) exclude them. One +# convenient way to do so is to obscure its contents by a bind mount: +# docker run .... -v /non-existing-directory:/usr/local/cuda/lib64/stubs:ro ... +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs" +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility +ENV NVIDIA_REQUIRE_CUDA="cuda>=9.0" +RUN apt-get update && apt-get install -y --no-install-recommends \ + cuda-cudart-$CUDA_PKG_VERSION \ + cuda-libraries-$CUDA_PKG_VERSION \ + cuda-libraries-dev-$CUDA_PKG_VERSION \ + cuda-nvml-dev-$CUDA_PKG_VERSION \ + cuda-minimal-build-$CUDA_PKG_VERSION \ + cuda-command-line-tools-$CUDA_PKG_VERSION \ + libcudnn7=7.0.5.15-1+cuda9.1 \ + libcudnn7-dev=7.0.5.15-1+cuda9.1 \ + libnccl2=2.2.12-1+cuda9.1 \ + libnccl-dev=2.2.12-1+cuda9.1 && \ + ln -s /usr/local/cuda-9.1 /usr/local/cuda && \ + ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ + rm -rf /var/lib/apt/lists/* + +# Install bazel +RUN apt-get update && apt-get install -y python-software-properties zip && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ + apt-get update && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get install -y oracle-java8-installer && \ + echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ + curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ + apt-get update && apt-get install -y bazel && \ + apt-get upgrade -y bazel + +# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517 +RUN conda install -y python=3.6.6 && \ + # Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518 + pip install keras_applications==1.0.4 --no-deps && \ + pip install keras_preprocessing==1.0.2 --no-deps + +# Fetch tensorflow +RUN cd /usr/local/src && \ + git clone https://github.com/tensorflow/tensorflow && \ + cd tensorflow && \ + git checkout r1.11 + +# Create a tensorflow wheel for CPU +RUN cd /usr/local/src/tensorflow && \ + cat /dev/null | ./configure && \ + bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_cpu && \ + bazel clean + +# Create a tensorflow wheel for GPU/cuda +ENV TF_NEED_CUDA=1 +ENV TF_CUDA_VERSION=9.1 +ENV TF_CUDA_COMPUTE_CAPABILITIES=3.7,6.0 +ENV TF_CUDNN_VERSION=7 +ENV TF_NCCL_VERSION=2 +ENV NCCL_INSTALL_PATH=/usr/ + +RUN cd /usr/local/src/tensorflow && \ + # TF_NCCL_INSTALL_PATH is used for both libnccl.so.2 and libnccl.h. Make sure they are both accessible from the same directory. + ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/lib/ && \ + cat /dev/null | ./configure && \ + echo "/usr/local/cuda-${TF_CUDA_VERSION}/targets/x86_64-linux/lib/stubs" > /etc/ld.so.conf.d/cuda-stubs.conf && ldconfig && \ + bazel build --config=opt \ + --config=cuda \ + --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ + //tensorflow/tools/pip_package:build_pip_package && \ + rm /etc/ld.so.conf.d/cuda-stubs.conf && ldconfig && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_gpu && \ + bazel clean + +# Print out the built .whl files +RUN ls -R /tmp/tensorflow* diff --git a/tensorflow-whl/README.md b/tensorflow-whl/README.md new file mode 100644 index 00000000..ba0e0003 --- /dev/null +++ b/tensorflow-whl/README.md @@ -0,0 +1,22 @@ +# Build new Tensorflow wheels + +``` +./build +``` + +# Push the new wheels + +1. Add an entry in the [CHANGELOG](CHANGELOG.md) with an appropriate `LABEL`. +2. Push the new image using the `LABEL` you picked above. + + ``` + ./push LABEL + ``` + +# Use the new wheels + +Update the line below in the [CPU Dockerfile](../Dockerfile) and the [GPU Dockerfile](../gpu.Dockerfile) to use the new `LABEL`. + +``` +FROM gcr.io/kaggle-images/python-tensorflow-whl: as tensorflow_whl +``` diff --git a/tensorflow-whl/build b/tensorflow-whl/build new file mode 100755 index 00000000..b5678526 --- /dev/null +++ b/tensorflow-whl/build @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +set -x + +# Default behavior is to do everything from scratch. +# The --use-cache option is useful if you're iterating on a broken build. +if [[ "$1" == "--use-cache" ]]; then + docker build --rm -t kaggle/python-tensorflow-whl . +else + docker build --pull --rm --no-cache -t kaggle/python-tensorflow-whl . +fi diff --git a/tensorflow-whl/push b/tensorflow-whl/push new file mode 100755 index 00000000..a27de8a1 --- /dev/null +++ b/tensorflow-whl/push @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Push a newly-built image with the given label to gcr.io and DockerHub. +# +# Usage: +# ./push LABEL +# +# Description: +# LABEL: Image label. See CHANGELOG.md +# +set -e +set -x + +SOURCE_IMAGE="kaggle/python-tensorflow-whl" +TARGET_IMAGE="gcr.io/kaggle-images/python-tensorflow-whl" + +LABEL=$1 + +if [[ -z "$LABEL" ]]; then + echo "You must provide a label for the image" + exit 1 +fi + +docker tag $SOURCE_IMAGE:latest $TARGET_IMAGE:$LABEL +gcloud docker -- push $TARGET_IMAGE:$LABEL From 377387914cd323bda8163c1800e81c627d1474e0 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 27 Nov 2018 02:39:59 +0000 Subject: [PATCH 0006/1210] Add TODOs and improve build/push script --- gpu.Dockerfile | 2 ++ tensorflow-whl/Dockerfile | 8 +++++-- tensorflow-whl/build | 47 ++++++++++++++++++++++++++++++++------- tensorflow-whl/push | 45 ++++++++++++++++++++++++++++--------- 4 files changed, 81 insertions(+), 21 deletions(-) diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 247c7dd4..164b0d25 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -7,6 +7,8 @@ COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg +# Ensure the cuda libraries are compatible with the custom Tensorflow wheels. +# TODO(b/120050292): Use templating to keep in sync. ENV CUDA_VERSION=9.1.85 ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 LABEL com.nvidia.volumes.needed="nvidia_driver" diff --git a/tensorflow-whl/Dockerfile b/tensorflow-whl/Dockerfile index db5bc8df..5b7b421b 100644 --- a/tensorflow-whl/Dockerfile +++ b/tensorflow-whl/Dockerfile @@ -13,6 +13,8 @@ COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg +# Ensure the cuda libraries are compatible with the GPU image. +# TODO(b/120050292): Use templating to keep in sync. ENV CUDA_VERSION=9.1.85 ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 LABEL com.nvidia.volumes.needed="nvidia_driver" @@ -28,14 +30,16 @@ ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility ENV NVIDIA_REQUIRE_CUDA="cuda>=9.0" RUN apt-get update && apt-get install -y --no-install-recommends \ + cuda-cupti-$CUDA_PKG_VERSION \ cuda-cudart-$CUDA_PKG_VERSION \ + cuda-cudart-dev-$CUDA_PKG_VERSION \ cuda-libraries-$CUDA_PKG_VERSION \ cuda-libraries-dev-$CUDA_PKG_VERSION \ cuda-nvml-dev-$CUDA_PKG_VERSION \ cuda-minimal-build-$CUDA_PKG_VERSION \ cuda-command-line-tools-$CUDA_PKG_VERSION \ - libcudnn7=7.0.5.15-1+cuda9.1 \ - libcudnn7-dev=7.0.5.15-1+cuda9.1 \ + libcudnn7=7.2.1.38-1+cuda9.0 \ + libcudnn7-dev=7.2.1.38-1+cuda9.0 \ libnccl2=2.2.12-1+cuda9.1 \ libnccl-dev=2.2.12-1+cuda9.1 && \ ln -s /usr/local/cuda-9.1 /usr/local/cuda && \ diff --git a/tensorflow-whl/build b/tensorflow-whl/build index b5678526..dc705f17 100755 --- a/tensorflow-whl/build +++ b/tensorflow-whl/build @@ -1,11 +1,42 @@ #!/bin/bash set -e -set -x -# Default behavior is to do everything from scratch. -# The --use-cache option is useful if you're iterating on a broken build. -if [[ "$1" == "--use-cache" ]]; then - docker build --rm -t kaggle/python-tensorflow-whl . -else - docker build --pull --rm --no-cache -t kaggle/python-tensorflow-whl . -fi +usage() { +cat << EOF +Usage: $0 [OPTIONS] +Build new Tensorflow wheels for use in the Kaggle Docker Python base images. + +Options: + -c, --use-cache Use layer cache when building a new image. +EOF +} + +CACHE_FLAG="--no-cache" +IMAGE_TAG="kaggle/python-build" + +while :; do + case "$1" in + -h|--help) + usage + exit + ;; + -c|--use-cache) + CACHE_FLAG="" + ;; + -?*) + usage + printf 'ERROR: Unknown option: %s\n' "$1" >&2 + exit + ;; + *) + break + esac + + shift +done + +readonly CACHE_FLAG +readonly IMAGE_TAG + +set -x +docker build --rm --pull $CACHE_FLAG -t "$IMAGE_TAG" . diff --git a/tensorflow-whl/push b/tensorflow-whl/push index a27de8a1..dfb64ec8 100755 --- a/tensorflow-whl/push +++ b/tensorflow-whl/push @@ -1,19 +1,37 @@ #!/bin/bash -# -# Push a newly-built image with the given label to gcr.io and DockerHub. -# -# Usage: -# ./push LABEL -# -# Description: -# LABEL: Image label. See CHANGELOG.md -# set -e + +usage() { +cat << EOF +Usage: $0 [LABEL] +Push a newly-built image with the given LABEL to gcr.io and DockerHub. +See CHANGELOG.md file for LABEL naming convention. +EOF +} + set -x SOURCE_IMAGE="kaggle/python-tensorflow-whl" TARGET_IMAGE="gcr.io/kaggle-images/python-tensorflow-whl" +while :; do + case "$1" in + -h|--help) + usage + exit + ;; + -?*) + usage + printf 'ERROR: Unknown option: %s\n' "$1" >&2 + exit + ;; + *) + break + esac + + shift +done + LABEL=$1 if [[ -z "$LABEL" ]]; then @@ -21,5 +39,10 @@ if [[ -z "$LABEL" ]]; then exit 1 fi -docker tag $SOURCE_IMAGE:latest $TARGET_IMAGE:$LABEL -gcloud docker -- push $TARGET_IMAGE:$LABEL +readonly SOURCE_IMAGE +readonly TARGET_IMAGE +readonly LABEL + +set -x +docker tag "$SOURCE_IMAGE:latest" "$TARGET_IMAGE:$LABEL" +gcloud docker -- push "$TARGET_IMAGE:$LABEL" From 52b051e44b53a816f62080ed17228d699787b1fb Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 27 Nov 2018 02:45:17 +0000 Subject: [PATCH 0007/1210] fix typo --- tensorflow-whl/README.md | 2 +- tensorflow-whl/build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow-whl/README.md b/tensorflow-whl/README.md index ba0e0003..509591a2 100644 --- a/tensorflow-whl/README.md +++ b/tensorflow-whl/README.md @@ -18,5 +18,5 @@ Update the line below in the [CPU Dockerfile](../Dockerfile) and the [GPU Dockerfile](../gpu.Dockerfile) to use the new `LABEL`. ``` -FROM gcr.io/kaggle-images/python-tensorflow-whl: as tensorflow_whl +FROM gcr.io/kaggle-images/python-tensorflow-whl: