From 987863d34740b65715c71cdebc5a038609a03dff Mon Sep 17 00:00:00 2001
From: Johnny Chavez <64660690+calderjo@users.noreply.github.com>
Date: Wed, 25 Jun 2025 17:26:35 +0000
Subject: [PATCH 01/54] Clean up lightgbm install and fix build (#1488)
Bumping up the base image version to the latest.
Good news, Colab installs lightgbm on both CPU and GPU,
Bad news seems like the way they installed the package seems to give
issues.
I assume they install lightgbm in a similar manner to us, build the
wheel and install in a tmp folder.
UV PIP doesn't like this and errors at install requirement file step:
error: Distribution not found at:
file:///tmp/lightgbm/LightGBM/dist/lightgbm-4.5.0-py3-none-linux_x86_64.whl
Removing our pin on the package fixes the build.
I could also introduce a pin for lightgbm, but this could be
problematic, as we'll continuously have to update the pining.
---
Dockerfile.tmpl | 23 ++---------------------
Jenkinsfile | 23 -----------------------
config.txt | 3 +--
packages/lightgbm.Dockerfile | 30 ------------------------------
4 files changed, 3 insertions(+), 76 deletions(-)
delete mode 100644 packages/lightgbm.Dockerfile
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index 9ca776a9..1dc4c0fa 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -1,20 +1,12 @@
ARG BASE_IMAGE \
- BASE_IMAGE_TAG \
- LIGHTGBM_VERSION
+ BASE_IMAGE_TAG
-{{ if eq .Accelerator "gpu" }}
-FROM gcr.io/kaggle-images/python-lightgbm-whl:${BASE_IMAGE_TAG}-${LIGHTGBM_VERSION} AS lightgbm_whl
-{{ end }}
FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG}
-#b/415358342: UV reports missing requirements files https://github.com/googlecolab/colabtools/issues/5237
-ENV UV_CONSTRAINT= \
- UV_BUILD_CONSTRAINT=
-
ADD kaggle_requirements.txt /kaggle_requirements.txt
# Freeze existing requirements from base image for critical packages:
-RUN pip freeze | grep -E 'tensorflow|keras|torch|jax|lightgbm' > /colab_requirements.txt
+RUN pip freeze | grep -E 'tensorflow|keras|torch|jax' > /colab_requirements.txt
# Merge requirements files:
RUN cat /colab_requirements.txt >> /requirements.txt
@@ -67,17 +59,6 @@ ARG PACKAGE_PATH=/usr/local/lib/python3.11/dist-packages
# Install GPU-specific non-pip packages.
{{ if eq .Accelerator "gpu" }}
RUN uv pip install --system "pycuda"
-
-# b/381256047 Remove once installed in Colabs base image.
-# Install LightGBM
-COPY --from=lightgbm_whl /tmp/whl/*.whl /tmp/lightgbm/
-# Install OpenCL (required by LightGBM GPU version)
-RUN apt-get install -y ocl-icd-libopencl1 clinfo && \
- mkdir -p /etc/OpenCL/vendors && \
- echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd && \
- uv pip install --system /tmp/lightgbm/*.whl && \
- rm -rf /tmp/lightgbm && \
- /tmp/clean-layer.sh
{{ end }}
diff --git a/Jenkinsfile b/Jenkinsfile
index ea2cb9be..906e0464 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -21,29 +21,6 @@ pipeline {
}
stages {
- stage('Pre-build Packages from Source') {
- stages {
- stage('lightgbm') {
- options {
- timeout(time: 10, unit: 'MINUTES')
- }
- steps {
- sh '''#!/bin/bash
- set -exo pipefail
- source config.txt
- cd packages/
- ./build_package --base-image $BASE_IMAGE:$BASE_IMAGE_TAG \
- --package lightgbm \
- --version $LIGHTGBM_VERSION \
- --build-arg CUDA_MAJOR_VERSION=$CUDA_MAJOR_VERSION \
- --build-arg CUDA_MINOR_VERSION=$CUDA_MINOR_VERSION \
- --push
- '''
- }
- }
- }
- }
-
stage('Build/Test/Diff') {
parallel {
stage('CPU') {
diff --git a/config.txt b/config.txt
index cfe8026a..d3829bcf 100644
--- a/config.txt
+++ b/config.txt
@@ -1,5 +1,4 @@
BASE_IMAGE=us-docker.pkg.dev/colab-images/public/runtime
-BASE_IMAGE_TAG=release-colab_20250404-060113_RC00
-LIGHTGBM_VERSION=4.6.0
+BASE_IMAGE_TAG=release-colab_20250603-060055_RC00
CUDA_MAJOR_VERSION=12
CUDA_MINOR_VERSION=5
diff --git a/packages/lightgbm.Dockerfile b/packages/lightgbm.Dockerfile
deleted file mode 100644
index 376eaaef..00000000
--- a/packages/lightgbm.Dockerfile
+++ /dev/null
@@ -1,30 +0,0 @@
-ARG BASE_IMAGE
-
-FROM ${BASE_IMAGE} AS builder
-
-ARG PACKAGE_VERSION
-ARG CUDA_MAJOR_VERSION
-ARG CUDA_MINOR_VERSION
-
-# Make sure we are on the right version of CUDA
-RUN update-alternatives --set cuda /usr/local/cuda-$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION
-
-# Build instructions: https://lightgbm.readthedocs.io/en/latest/GPU-Tutorial.html#build-lightgbm
-RUN apt-get update && \
- apt-get install -y build-essential cmake libboost-dev libboost-system-dev libboost-filesystem-dev clinfo nvidia-opencl-dev opencl-headers
-
-RUN cd /usr/local/src && \
- git clone --recursive https://github.com/microsoft/LightGBM && \
- cd LightGBM && \
- git checkout tags/v$PACKAGE_VERSION && \
- ./build-python.sh bdist_wheel --gpu --opencl-library=/usr/local/cuda/lib64/libOpenCL.so --opencl-include-dir=/usr/local/cuda/include/
-
-# Using multi-stage builds to ensure the output image is very small
-# See: https://docs.docker.com/develop/develop-images/multistage-build/
-FROM alpine:latest
-
-RUN mkdir -p /tmp/whl/
-COPY --from=builder /usr/local/src/LightGBM/dist/*.whl /tmp/whl
-
-# Print out the built .whl file.
-RUN ls -lh /tmp/whl/
\ No newline at end of file
From f6db354afe8a5395a98f35113814f0eaaa7c3fbe Mon Sep 17 00:00:00 2001
From: Johnny Chavez <64660690+calderjo@users.noreply.github.com>
Date: Fri, 27 Jun 2025 00:45:37 +0000
Subject: [PATCH 02/54] Fix torch tune, keras, tensorflow tests (#1489)
Looks like torch tune changed the output of the --help command, this
cause issues with our smoke tests.
Keras, along with other package had issues with existing issues with
cudnn downgrading due to torch requirements, we pinned relevant tests.
---
Dockerfile.tmpl | 7 +++++--
kaggle_requirements.txt | 5 ++---
tests/test_torchtune.py | 12 +++++++++---
3 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index 1dc4c0fa..8f698612 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -35,7 +35,10 @@ RUN uv pip install --no-build-isolation --system "git+https://github.com/Kaggle/
# b/408281617: Torch is adamant that it can not install cudnn 9.3.x, only 9.1.x, but Tensorflow can only support 9.3.x.
# This conflict causes a number of package downgrades, which are handled in this command
RUN uv pip install --system --force-reinstall --extra-index-url https://pypi.nvidia.com "cuml-cu12==25.2.1" \
- "nvidia-cudnn-cu12==9.3.0.75"
+ "nvidia-cudnn-cu12==9.3.0.75" "nvidia-cublas-cu12==12.5.3.2" "nvidia-cusolver-cu12==11.6.3.83" \
+ "nvidia-cuda-cupti-cu12==12.5.82" "nvidia-cuda-nvrtc-cu12==12.5.82" "nvidia-cuda-runtime-cu12==12.5.82" \
+ "nvidia-cufft-cu12==11.2.3.61" "nvidia-curand-cu12==10.3.6.82" "nvidia-cusparse-cu12==12.5.1.3" \
+ "nvidia-nvjitlink-cu12==12.5.82"
RUN uv pip install --system --force-reinstall "pynvjitlink-cu12==0.5.2"
# b/385145217 Latest Colab lacks mkl numpy, install it.
@@ -46,7 +49,7 @@ RUN uv pip install --system "tbb>=2022" "libpysal==4.9.2"
# b/404590350: Ray and torchtune have conflicting tune cli, we will prioritize torchtune.
# b/415358158: Gensim removed from Colab image to upgrade scipy
-RUN uv pip install --system --force-reinstall --no-deps torchtune gensim
+RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3"
# Adding non-package dependencies:
ADD clean-layer.sh /tmp/clean-layer.sh
diff --git a/kaggle_requirements.txt b/kaggle_requirements.txt
index 22b26470..cc43b8c2 100644
--- a/kaggle_requirements.txt
+++ b/kaggle_requirements.txt
@@ -121,6 +121,7 @@ qtconsole
ray
rgf-python
s3fs
+# b/302136621: Fix eli5 import for learntools
scikit-learn==1.2.2
# Scikit-learn accelerated library for x86
scikit-learn-intelex>=2023.0.1
@@ -128,12 +129,10 @@ scikit-multilearn
scikit-optimize
scikit-plot
scikit-surprise
-# b/415358158: Gensim removed from Colab image to upgrade scipy to 1.14.1
-scipy==1.15.1
# Also pinning seaborn for learntools
seaborn==0.12.2
git+https://github.com/facebookresearch/segment-anything.git
-# b/329869023 shap 0.45.0 breaks learntools
+# b/329869023: shap 0.45.0 breaks learntools
shap==0.44.1
squarify
tensorflow-cloud
diff --git a/tests/test_torchtune.py b/tests/test_torchtune.py
index aab17442..c4a702fd 100644
--- a/tests/test_torchtune.py
+++ b/tests/test_torchtune.py
@@ -3,8 +3,14 @@
class TestTorchtune(unittest.TestCase):
def test_help(self):
- result = subprocess.run(["tune", "--help"], stdout=subprocess.PIPE)
+ result = subprocess.run(
+ ["tune", "--help"],
+ capture_output=True,
+ text=True
+ )
self.assertEqual(0, result.returncode)
- self.assertIsNone(result.stderr)
- self.assertIn("Download a model from the Hugging Face Hub or Kaggle Model Hub.", result.stdout.decode("utf-8"))
+ self.assertIn(
+ "Download a model from the Hugging Face Hub or Kaggle",
+ result.stdout
+ )
From 0eb38ee5b80e4003101469c19d6da1f4353d5960 Mon Sep 17 00:00:00 2001
From: Johnny Chavez <64660690+calderjo@users.noreply.github.com>
Date: Tue, 1 Jul 2025 14:38:01 -0700
Subject: [PATCH 03/54] Bump Colab Base Image - July (#1490)
let's bump the image, this was released a couple days ago. while we were
planning a release with the prior Base image, let's save us a release
and get our self to the latest
---
config.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/config.txt b/config.txt
index d3829bcf..61395f5e 100644
--- a/config.txt
+++ b/config.txt
@@ -1,4 +1,4 @@
BASE_IMAGE=us-docker.pkg.dev/colab-images/public/runtime
-BASE_IMAGE_TAG=release-colab_20250603-060055_RC00
+BASE_IMAGE_TAG=release-colab_20250626-060053_RC00
CUDA_MAJOR_VERSION=12
CUDA_MINOR_VERSION=5
From 4d494f144d2915ad83b198a526c4623d3e087fce Mon Sep 17 00:00:00 2001
From: Eric Johnson <65414824+metrizable@users.noreply.github.com>
Date: Wed, 17 Sep 2025 20:42:58 -0700
Subject: [PATCH 04/54] Upgrade TPU image to Python 3.11.Tpupy311 (#1493)
Python 3.10 is entering its [last year of
support](https://devguide.python.org/versions/) before end-of-life and
many packages, including [NumPy](https://devguide.python.org/versions/),
have dropped support for it altogether.
Included in this change:
* Upgrade the TPU docker image to derive from `python:3.11`.
* Upgrade `tensorflow` to 2.20.0.
* Upgrade `jax` to >=0.5.2. For a compatible dep closure, this installs
`jax` 0.7.2 re: `tensorflow-tpu` dep on`libtpu`.
* Upgrade `torch` (and ecosystem) to 2.8.0. Of note, there is no wheel
with a `+libtpu` label.
* Remove unneeded environment variable.
Tested:
Locally, by invoking `./tpu/build`:
>
Also invoked other back-end testing.
---
tpu/Dockerfile | 1 -
tpu/config.txt | 14 +++++++-------
tpu/requirements.txt | 4 ++--
3 files changed, 9 insertions(+), 10 deletions(-)
diff --git a/tpu/Dockerfile b/tpu/Dockerfile
index ce68baf1..fd0c0684 100644
--- a/tpu/Dockerfile
+++ b/tpu/Dockerfile
@@ -55,7 +55,6 @@ RUN sed -i '/from tensorflow_hub import uncompressed_module_resolver/a from tens
RUN sed -i '/_install_default_resolvers()/a \ \ registry.resolver.add_implementation(kaggle_module_resolver.KaggleFileResolver())' /usr/local/lib/${PYTHON_VERSION_PATH}/site-packages/tensorflow_hub/config.py
# Set these env vars so that they don't produce errs calling the metadata server to load them:
-ENV TPU_ACCELERATOR_TYPE=v3-8
ENV TPU_PROCESS_ADDRESSES=local
# Metadata
diff --git a/tpu/config.txt b/tpu/config.txt
index 645d2faf..b399a592 100644
--- a/tpu/config.txt
+++ b/tpu/config.txt
@@ -1,12 +1,12 @@
-BASE_IMAGE=python:3.10
-PYTHON_WHEEL_VERSION=cp310
-PYTHON_VERSION_PATH=python3.10
-TENSORFLOW_VERSION=2.18.0
+BASE_IMAGE=python:3.11
+PYTHON_WHEEL_VERSION=cp311
+PYTHON_VERSION_PATH=python3.11
+TENSORFLOW_VERSION=2.20.0
# gsutil ls gs://pytorch-xla-releases/wheels/tpuvm/* | grep libtpu | grep torch_xla | grep -v -E ".*rc[0-9].*" | sed 's/.*torch_xla-\(.*\)+libtpu.*/\1/' | sort -rV
# Supports nightly
-TORCH_VERSION=2.6.0
+TORCH_VERSION=2.8.0
# https://github.com/pytorch/audio supports nightly
-TORCHAUDIO_VERSION=2.6.0
+TORCHAUDIO_VERSION=2.8.0
# https://github.com/pytorch/vision supports nightly
-TORCHVISION_VERSION=0.21.0
+TORCHVISION_VERSION=0.23.0
TORCH_LINUX_WHEEL_VERSION=manylinux_2_28_x86_64
diff --git a/tpu/requirements.txt b/tpu/requirements.txt
index ba3a1bdf..c4b0cc58 100644
--- a/tpu/requirements.txt
+++ b/tpu/requirements.txt
@@ -8,11 +8,11 @@ tensorflow-io
tensorflow-probability
# Torch packages
torch==${TORCH_VERSION}
-https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}+libtpu-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl
+https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl
torchaudio==${TORCHAUDIO_VERSION}
torchvision==${TORCHVISION_VERSION}
# Jax packages
-jax[tpu]>=0.4.34
+jax[tpu]>=0.5.2
--find-links https://storage.googleapis.com/jax-releases/libtpu_releases.html
distrax
flax
From da65765599c2f8d6d65a0ceb665270d3035adf8d Mon Sep 17 00:00:00 2001
From: Dustin H
Date: Thu, 18 Sep 2025 00:34:16 -0400
Subject: [PATCH 05/54] Add tunix to tpu (#1494)
---
tpu/requirements.txt | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/tpu/requirements.txt b/tpu/requirements.txt
index c4b0cc58..f31a2e51 100644
--- a/tpu/requirements.txt
+++ b/tpu/requirements.txt
@@ -6,6 +6,7 @@ tensorflow-tpu==${TENSORFLOW_VERSION}
tensorflow_hub
tensorflow-io
tensorflow-probability
+tensorflow_datasets
# Torch packages
torch==${TORCH_VERSION}
https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl
@@ -20,6 +21,10 @@ git+https://github.com/deepmind/dm-haiku
jraph
optax
trax
+# Tunix GRPO
+git+https://github.com/google/tunix
+git+https://github.com/google/qwix
+grain
# Jupyter packages
jupyter-lsp==1.5.1
jupyterlab
@@ -38,6 +43,7 @@ albumentations
diffusers
einops
fastparquet
+ipywidgets
matplotlib
opencv-python
opencv-python-headless
From d019ef74b14fa4281074e1e8709f077e8fbc78c7 Mon Sep 17 00:00:00 2001
From: Eric Johnson <65414824+metrizable@users.noreply.github.com>
Date: Thu, 18 Sep 2025 13:24:17 -0700
Subject: [PATCH 06/54] Include the default PyPI for missing libucx-cu12
package version. (#1495)
The CPU/GPU container image builds are currently broken due to
dependency resolution failures:
It appears, this version was removed from the Nvidia index since the
last build.
We ensure that a compatible package version `libucx-cu12==1.18.1` is
available by including the default PyPI and specifying an appropriate
[index
strategy](https://docs.astral.sh/uv/concepts/indexes/#searching-across-multiple-indexes)
to `uv`.
In the near future, we may want to consider upgrading `cuml-cu12` and
ecosystem to 25.6 or later.
---
Dockerfile.tmpl | 6 +++--
kaggle_requirements.txt | 9 +++++--
tests/test_fastai.py | 59 +++++++++++++++++++++--------------------
3 files changed, 41 insertions(+), 33 deletions(-)
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index 8f698612..1b63837d 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -34,7 +34,9 @@ RUN uv pip install --no-build-isolation --system "git+https://github.com/Kaggle/
# b/408281617: Torch is adamant that it can not install cudnn 9.3.x, only 9.1.x, but Tensorflow can only support 9.3.x.
# This conflict causes a number of package downgrades, which are handled in this command
-RUN uv pip install --system --force-reinstall --extra-index-url https://pypi.nvidia.com "cuml-cu12==25.2.1" \
+RUN uv pip install \
+ --index-url https://pypi.nvidia.com --extra-index-url https://pypi.org/simple/ --index-strategy unsafe-first-match \
+ --system --force-reinstall "cuml-cu12==25.2.1" \
"nvidia-cudnn-cu12==9.3.0.75" "nvidia-cublas-cu12==12.5.3.2" "nvidia-cusolver-cu12==11.6.3.83" \
"nvidia-cuda-cupti-cu12==12.5.82" "nvidia-cuda-nvrtc-cu12==12.5.82" "nvidia-cuda-runtime-cu12==12.5.82" \
"nvidia-cufft-cu12==11.2.3.61" "nvidia-curand-cu12==10.3.6.82" "nvidia-cusparse-cu12==12.5.1.3" \
@@ -171,7 +173,7 @@ ENV PYTHONUSERBASE="/root/.local"
ADD patches/kaggle_gcp.py \
patches/kaggle_secrets.py \
patches/kaggle_session.py \
- patches/kaggle_web_client.py \
+ patches/kaggle_web_client.py \
patches/kaggle_datasets.py \
patches/log.py \
$PACKAGE_PATH/
diff --git a/kaggle_requirements.txt b/kaggle_requirements.txt
index cc43b8c2..03c489b4 100644
--- a/kaggle_requirements.txt
+++ b/kaggle_requirements.txt
@@ -35,7 +35,10 @@ easyocr
# b/302136621: Fix eli5 import for learntools
eli5
emoji
-fastcore>=1.7.20
+fastcore
+# b/445960030: Requires a newer version of fastai than the currently used base image.
+# Remove when relying on a newer base image.
+fastai>=2.8.4
fasttext
featuretools
fiona
@@ -89,7 +92,9 @@ nbconvert==6.4.5
nbdev
nilearn
olefile
-onnx
+# b/445960030: Broken in 1.19.0. See https://github.com/onnx/onnx/issues/7249.
+# Fixed with https://github.com/onnx/onnx/pull/7254. Upgrade when version with fix is published.
+onnx==1.18.0
openslide-bin
openslide-python
optuna
diff --git a/tests/test_fastai.py b/tests/test_fastai.py
index 49bce0ac..33a436a5 100644
--- a/tests/test_fastai.py
+++ b/tests/test_fastai.py
@@ -1,35 +1,36 @@
import unittest
import fastai
-
from fastai.tabular.all import *
+
class TestFastAI(unittest.TestCase):
- # Basic import
- def test_basic(self):
- import fastai
- import fastcore
- import fastprogress
- import fastdownload
-
- def test_has_version(self):
- self.assertGreater(len(fastai.__version__), 2)
-
- # based on https://github.com/fastai/fastai/blob/master/tests/test_torch_core.py#L17
- def test_torch_tensor(self):
- a = tensor([1, 2, 3])
- b = torch.tensor([1, 2, 3])
-
- self.assertTrue(torch.all(a == b))
-
- def test_tabular(self):
- dls = TabularDataLoaders.from_csv(
- "/input/tests/data/train.csv",
- cont_names=["pixel"+str(i) for i in range(784)],
- y_names='label',
- procs=[FillMissing, Categorify, Normalize])
- learn = tabular_learner(dls, layers=[200, 100])
- with learn.no_bar():
- learn.fit_one_cycle(n_epoch=1)
-
- self.assertGreater(learn.smooth_loss, 0)
+ # Basic import
+ def test_basic(self):
+ import fastai
+ import fastcore
+ import fastprogress
+ import fastdownload
+
+ def test_has_version(self):
+ self.assertGreater(len(fastai.__version__), 2)
+
+ # based on https://github.com/fastai/fastai/blob/master/tests/test_torch_core.py#L17
+ def test_torch_tensor(self):
+ a = tensor([1, 2, 3])
+ b = torch.tensor([1, 2, 3])
+
+ self.assertTrue(torch.all(a == b))
+
+ def test_tabular(self):
+ dls = TabularDataLoaders.from_csv(
+ "/input/tests/data/train.csv",
+ cont_names=["pixel" + str(i) for i in range(784)],
+ y_names="label",
+ procs=[FillMissing, Categorify, Normalize],
+ )
+ learn = tabular_learner(dls, layers=[200, 100])
+ with learn.no_bar():
+ learn.fit_one_cycle(n_epoch=1)
+
+ self.assertGreater(learn.smooth_loss, 0)
From 506c34b7805b1f545c544daf75fd5feb4c3c10ac Mon Sep 17 00:00:00 2001
From: Dustin H
Date: Mon, 22 Sep 2025 10:24:06 -0400
Subject: [PATCH 07/54] Python 3.12 bump (#1496)
---
tpu/config.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tpu/config.txt b/tpu/config.txt
index b399a592..ab933ba7 100644
--- a/tpu/config.txt
+++ b/tpu/config.txt
@@ -1,6 +1,6 @@
-BASE_IMAGE=python:3.11
-PYTHON_WHEEL_VERSION=cp311
-PYTHON_VERSION_PATH=python3.11
+BASE_IMAGE=python:3.12
+PYTHON_WHEEL_VERSION=cp312
+PYTHON_VERSION_PATH=python3.12
TENSORFLOW_VERSION=2.20.0
# gsutil ls gs://pytorch-xla-releases/wheels/tpuvm/* | grep libtpu | grep torch_xla | grep -v -E ".*rc[0-9].*" | sed 's/.*torch_xla-\(.*\)+libtpu.*/\1/' | sort -rV
# Supports nightly
From acb8bccefc274f76aa51effdeacf69e2e1aa2dd7 Mon Sep 17 00:00:00 2001
From: Johnny Chavez <64660690+calderjo@users.noreply.github.com>
Date: Thu, 25 Sep 2025 09:56:21 -0700
Subject: [PATCH 08/54] Upgrade base image to colab_20250725-060057_RC00,
install google-adk (#1498)
A couple of things here:
we upgraded to our base to latest images that is still
- on py3.11
- uses keras 3.8.x and tf 2.18.x
we encounter some issues with upgrading to py3.12, so we'll punt that
for later.
in addition, we will install google-adk and pyngrok
we will add a more involved test in a later PR.
also we diffed new base image and removed packages that is now installed
in their image
@jplotts please lmk if pyngrok is needed for b/443054743 , we can always
remove this at a later point
---
Dockerfile.tmpl | 4 ++--
config.txt | 2 +-
kaggle_requirements.txt | 13 ++++---------
3 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index 1b63837d..7561a700 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -25,8 +25,8 @@ RUN uv pip uninstall --system google-cloud-bigquery-storage
# b/394382016: sigstore (dependency of kagglehub) requires a prerelease packages, installing separate.
# b/408284143: google-cloud-automl 2.0.0 introduced incompatible API changes, need to pin to 1.0.1,
# installed outside of kaggle_requirements.txt due to requiring an incompatibile version of protobuf.
-RUN uv pip install --system --force-reinstall --prerelease=allow kagglehub[pandas-datasets,hf-datasets,signing]>=0.3.12 \
- google-cloud-automl==1.0.1
+RUN uv pip install --system --force-reinstall --prerelease=allow "kagglehub[pandas-datasets,hf-datasets,signing]>=0.3.12" \
+ "google-cloud-automl==1.0.1"
# uv cannot install this in requirements.txt without --no-build-isolation
# to avoid affecting the larger build, we'll post-install it.
diff --git a/config.txt b/config.txt
index 61395f5e..af541652 100644
--- a/config.txt
+++ b/config.txt
@@ -1,4 +1,4 @@
BASE_IMAGE=us-docker.pkg.dev/colab-images/public/runtime
-BASE_IMAGE_TAG=release-colab_20250626-060053_RC00
+BASE_IMAGE_TAG=release-colab_20250725-060057_RC00
CUDA_MAJOR_VERSION=12
CUDA_MINOR_VERSION=5
diff --git a/kaggle_requirements.txt b/kaggle_requirements.txt
index 03c489b4..c711869e 100644
--- a/kaggle_requirements.txt
+++ b/kaggle_requirements.txt
@@ -1,6 +1,4 @@
# Please keep this in alphabetical order
-Altair>=5.4.0
-Babel
Boruta
Cartopy
ImageHash
@@ -24,7 +22,6 @@ category-encoders
cesium
comm
cytoolz
-dask-expr
# Older versions of datasets fail with "Loading a dataset cached in a LocalFileSystem is not supported"
# https://stackoverflow.com/questions/77433096/notimplementederror-loading-a-dataset-cached-in-a-localfilesystem-is-not-suppor
datasets>=2.14.6
@@ -48,6 +45,8 @@ geojson
# geopandas > v0.14.4 breaks learn tools
geopandas==v0.14.4
gensim
+# b/443054743
+google-adk
google-cloud-aiplatform
# b/315753846: Unpin translate package.
google-cloud-translate==3.12.1
@@ -111,7 +110,6 @@ pyLDAvis
pycryptodome
pydegensac
pydicom
-pydub
pyemd
pyexcel-ods
pymc3
@@ -144,18 +142,15 @@ tensorflow-cloud
tensorflow-io
tensorflow-text
tensorflow_decision_forests
-timm
-torchao
torchinfo
torchmetrics
torchtune
transformers>=4.51.0
-triton
-tsfresh
vtk
-wandb
wavio
# b/350573866: xgboost v2.1.0 breaks learntools
xgboost==2.0.3
xvfbwrapper
ydata-profiling
+# b/443054743: pinned as newer versions requires protobuf > 3.20.3
+ydf==0.9.0
From 3e031bac8b3c6d484215bdd6de6e33c0c51cd4fb Mon Sep 17 00:00:00 2001
From: Eric Johnson <65414824+metrizable@users.noreply.github.com>
Date: Fri, 26 Sep 2025 11:18:38 -0700
Subject: [PATCH 09/54] Fix libtpu version for torch and do not pre-install
tensorflow-tpu on TPU. (#1499)
We install a libtpu version compatible with both jax 0.7.2 and torch
2.8.0. Why? tunix latest -> flax 0.12 -> jax 0.7.2 -> libtpu 0.0.23, and
that libtpu version causes pjrt api errors for torch 2.8.0:
```
pjrt_c_api_helpers.cc:258] Unexpected error status Unexpected PJRT_Plugin_Attributes_Args size: expe
cted 32, got 24. The plugin is likely built with a later version than the framework. This plugin is built with PJRT API version 0.75.
```
*
https://github.com/pytorch/xla/blob/d517649bdef6ab0519c30c704bde8779c8216502/setup.py#L111
*
https://github.com/jax-ml/jax/blob/3489529b38d1f11d1e5caf4540775aadd5f2cdda/setup.py#L26
Of particular note, we no longer pre-install `tensorflow-tpu` as the
newer libtpu causes issues finding the TPUs
```
external/local_xla/xla/stream_executor/tpu/tpu_platform_interface.cc:78] No TPU platform found. Platform manager status: OK
```
We also update how we install Python packages via `uv` for consistency
and reproducibility. From a `requirements.in` file, we first generate a
consistent dependency closure via `uv pip compile`, and then `uv pip
install` the packages from the generated `requirements.txt`.
---
tpu/Dockerfile | 33 ++++++++++++++++++-----
tpu/{requirements.txt => requirements.in} | 7 +++--
2 files changed, 29 insertions(+), 11 deletions(-)
rename tpu/{requirements.txt => requirements.in} (82%)
diff --git a/tpu/Dockerfile b/tpu/Dockerfile
index fd0c0684..343443ae 100644
--- a/tpu/Dockerfile
+++ b/tpu/Dockerfile
@@ -34,20 +34,39 @@ RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y
# Additional useful packages should be added in the requirements.txt
# Bring in the requirements.txt and replace variables in it:
RUN apt-get install -y gettext
-ADD tpu/requirements.txt /kaggle_requirements.txt
-RUN envsubst < /kaggle_requirements.txt > /requirements.txt
+ADD tpu/requirements.in /kaggle_requirements.in
+RUN envsubst < /kaggle_requirements.in > /requirements.in
# Install uv and then install the requirements:
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
-RUN export PATH="${HOME}/.local/bin:${PATH}" && uv pip install --system -r /requirements.txt --prerelease=allow --force-reinstall && \
+RUN export PATH="${HOME}/.local/bin:${PATH}" && \
+ uv pip compile --system --prerelease=allow \
+ --verbose \
+ --upgrade \
+ --find-links=https://storage.googleapis.com/jax-releases/libtpu_releases.html \
+ --find-links=https://storage.googleapis.com/libtpu-releases/index.html \
+ --find-links=https://storage.googleapis.com/libtpu-wheels/index.html \
+ --find-links=https://download.pytorch.org/whl/torch_stable.html \
+ --emit-find-links \
+ --no-emit-package pip \
+ --no-emit-package setuptools \
+ --output-file /requirements.txt \
+ /requirements.in && \
+ uv pip install --system --prerelease=allow --force-reinstall \
+ -r /requirements.txt && \
+ uv cache clean && \
/tmp/clean-layer.sh
ENV PATH="~/.local/bin:${PATH}"
-# Try to force tensorflow to reliably install without breaking other installed deps
+# We install a libtpu version compatible with both jax 0.7.2 and torch 2.8.0.
+# Why? tunix latest -> flax 0.12 -> jax 0.7.2 -> libtpu 0.0.23. However, that
+# libtpu causes pjrt api errors for torch 2.8.0. screenshot/5heUtdyaJ4MmR3D
+# https://github.com/pytorch/xla/blob/d517649bdef6ab0519c30c704bde8779c8216502/setup.py#L111
+# https://github.com/jax-ml/jax/blob/3489529b38d1f11d1e5caf4540775aadd5f2cdda/setup.py#L26
RUN export PATH="${HOME}/.local/bin:${PATH}" && \
- uv pip freeze --system > /tmp/constraints.txt && \
- uv pip install --system -c /tmp/constraints.txt tensorflow-tpu -f https://storage.googleapis.com/libtpu-tf-releases/index.html --force-reinstall && \
- rm /tmp/constraints.txt
+ uv pip install --system --force-reinstall libtpu==0.0.17 && \
+ uv cache clean && \
+ /tmp/clean-layer.sh
# Kaggle Model Hub patches:
ADD patches/kaggle_module_resolver.py /usr/local/lib/${PYTHON_VERSION_PATH}/site-packages/tensorflow_hub/kaggle_module_resolver.py
diff --git a/tpu/requirements.txt b/tpu/requirements.in
similarity index 82%
rename from tpu/requirements.txt
rename to tpu/requirements.in
index f31a2e51..3991c7d3 100644
--- a/tpu/requirements.txt
+++ b/tpu/requirements.in
@@ -1,8 +1,8 @@
# TPU Utils
tpu-info
# Tensorflow packages
-tensorflow-tpu==${TENSORFLOW_VERSION}
---find-links https://storage.googleapis.com/libtpu-tf-releases/index.html
+# TODO: b/447621961 - re-enable tensorflow-tpu when a compatible libtpu can be found.
+tensorflow-cpu==${TENSORFLOW_VERSION}
tensorflow_hub
tensorflow-io
tensorflow-probability
@@ -13,8 +13,7 @@ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TOR
torchaudio==${TORCHAUDIO_VERSION}
torchvision==${TORCHVISION_VERSION}
# Jax packages
-jax[tpu]>=0.5.2
---find-links https://storage.googleapis.com/jax-releases/libtpu_releases.html
+jax[tpu]
distrax
flax
git+https://github.com/deepmind/dm-haiku
From 0898ca48ac0b34bf2966e77585e6c41ab5d7bb7a Mon Sep 17 00:00:00 2001
From: Dustin Herbison
Date: Wed, 1 Oct 2025 20:51:36 +0000
Subject: [PATCH 10/54] Switch to newer libtpu/tunix and cpu-based torch for
tpu.
http://b/436838265
---
tpu/Dockerfile | 10 ----------
tpu/requirements.in | 6 +++---
2 files changed, 3 insertions(+), 13 deletions(-)
diff --git a/tpu/Dockerfile b/tpu/Dockerfile
index 343443ae..fe2d065d 100644
--- a/tpu/Dockerfile
+++ b/tpu/Dockerfile
@@ -58,16 +58,6 @@ RUN export PATH="${HOME}/.local/bin:${PATH}" && \
/tmp/clean-layer.sh
ENV PATH="~/.local/bin:${PATH}"
-# We install a libtpu version compatible with both jax 0.7.2 and torch 2.8.0.
-# Why? tunix latest -> flax 0.12 -> jax 0.7.2 -> libtpu 0.0.23. However, that
-# libtpu causes pjrt api errors for torch 2.8.0. screenshot/5heUtdyaJ4MmR3D
-# https://github.com/pytorch/xla/blob/d517649bdef6ab0519c30c704bde8779c8216502/setup.py#L111
-# https://github.com/jax-ml/jax/blob/3489529b38d1f11d1e5caf4540775aadd5f2cdda/setup.py#L26
-RUN export PATH="${HOME}/.local/bin:${PATH}" && \
- uv pip install --system --force-reinstall libtpu==0.0.17 && \
- uv cache clean && \
- /tmp/clean-layer.sh
-
# Kaggle Model Hub patches:
ADD patches/kaggle_module_resolver.py /usr/local/lib/${PYTHON_VERSION_PATH}/site-packages/tensorflow_hub/kaggle_module_resolver.py
RUN sed -i '/from tensorflow_hub import uncompressed_module_resolver/a from tensorflow_hub import kaggle_module_resolver' /usr/local/lib/${PYTHON_VERSION_PATH}/site-packages/tensorflow_hub/config.py
diff --git a/tpu/requirements.in b/tpu/requirements.in
index 3991c7d3..1fceeebb 100644
--- a/tpu/requirements.in
+++ b/tpu/requirements.in
@@ -8,10 +8,10 @@ tensorflow-io
tensorflow-probability
tensorflow_datasets
# Torch packages
-torch==${TORCH_VERSION}
+https://download.pytorch.org/whl/cpu/torch-${TORCH_VERSION}%2Bcpu-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl
+https://download.pytorch.org/whl/cpu/torchaudio-${TORCHAUDIO_VERSION}%2Bcpu-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl
+https://download.pytorch.org/whl/cpu/torchvision-${TORCHVISION_VERSION}%2Bcpu-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl
https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}-${PYTHON_WHEEL_VERSION}-${PYTHON_WHEEL_VERSION}-${TORCH_LINUX_WHEEL_VERSION}.whl
-torchaudio==${TORCHAUDIO_VERSION}
-torchvision==${TORCHVISION_VERSION}
# Jax packages
jax[tpu]
distrax
From 221ec49fe771668afc2026f4e0ae9a85b817a777 Mon Sep 17 00:00:00 2001
From: Dustin Herbison
Date: Wed, 1 Oct 2025 21:54:01 +0000
Subject: [PATCH 11/54] Re-add the libtpu pin to make torch and jax work
together again...
http://b/436838265
---
tpu/Dockerfile | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/tpu/Dockerfile b/tpu/Dockerfile
index fe2d065d..343443ae 100644
--- a/tpu/Dockerfile
+++ b/tpu/Dockerfile
@@ -58,6 +58,16 @@ RUN export PATH="${HOME}/.local/bin:${PATH}" && \
/tmp/clean-layer.sh
ENV PATH="~/.local/bin:${PATH}"
+# We install a libtpu version compatible with both jax 0.7.2 and torch 2.8.0.
+# Why? tunix latest -> flax 0.12 -> jax 0.7.2 -> libtpu 0.0.23. However, that
+# libtpu causes pjrt api errors for torch 2.8.0. screenshot/5heUtdyaJ4MmR3D
+# https://github.com/pytorch/xla/blob/d517649bdef6ab0519c30c704bde8779c8216502/setup.py#L111
+# https://github.com/jax-ml/jax/blob/3489529b38d1f11d1e5caf4540775aadd5f2cdda/setup.py#L26
+RUN export PATH="${HOME}/.local/bin:${PATH}" && \
+ uv pip install --system --force-reinstall libtpu==0.0.17 && \
+ uv cache clean && \
+ /tmp/clean-layer.sh
+
# Kaggle Model Hub patches:
ADD patches/kaggle_module_resolver.py /usr/local/lib/${PYTHON_VERSION_PATH}/site-packages/tensorflow_hub/kaggle_module_resolver.py
RUN sed -i '/from tensorflow_hub import uncompressed_module_resolver/a from tensorflow_hub import kaggle_module_resolver' /usr/local/lib/${PYTHON_VERSION_PATH}/site-packages/tensorflow_hub/config.py
From f972e95de3e7eeec59f77b5be435bbed8ebab3a7 Mon Sep 17 00:00:00 2001
From: Sohier Dane
Date: Wed, 29 Oct 2025 11:19:33 -0700
Subject: [PATCH 12/54] Update Dockerfile.tmpl (#1508)
Drop support for BQ Helper, which is obsolete now that the official
BigQuery library is more feature complete.
[Context](https://chat.kaggle.net/kaggle/pl/egqzrknaz7dcfydjfrn1xiwame).
---
Dockerfile.tmpl | 10 ----------
1 file changed, 10 deletions(-)
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index 7561a700..b0d0d111 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -146,16 +146,6 @@ RUN mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py
mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \
/tmp/clean-layer.sh
-# Fix to import bq_helper library without downgrading setuptools and upgrading protobuf
-RUN mkdir -p ~/src && git clone https://github.com/SohierDane/BigQuery_Helper ~/src/BigQuery_Helper && \
- mkdir -p ~/src/BigQuery_Helper/bq_helper && \
- mv ~/src/BigQuery_Helper/bq_helper.py ~/src/BigQuery_Helper/bq_helper/__init__.py && \
- mv ~/src/BigQuery_Helper/test_helper.py ~/src/BigQuery_Helper/bq_helper/ && \
- sed -i 's/)/packages=["bq_helper"])/g' ~/src/BigQuery_Helper/setup.py && \
- uv pip install --system -e ~/src/BigQuery_Helper "protobuf<3.21"&& \
- /tmp/clean-layer.sh
-
-
# install imagemagick for wand
# https://docs.wand-py.org/en/latest/guide/install.html#install-imagemagick-on-debian-ubuntu
RUN apt-get install libmagickwand-dev
From dd0d0b39f079eca46f304fa480dc0c307f2a13e1 Mon Sep 17 00:00:00 2001
From: Jim Plotts
Date: Wed, 29 Oct 2025 14:30:09 -0400
Subject: [PATCH 13/54] Changes to support agents (#1507)
Removes automl implementation, which requires old protobuf (3.20.3).
This enables it to be upgraded to a modern version, which is required by
`google-adk[a2a]`.
Also removes old unused `BigQuery_Helper`.
Adds ADK `eval` and `a2a` as dependencies.
http://b/455550872
---
Dockerfile.tmpl | 5 +----
kaggle_requirements.txt | 7 +++----
patches/kaggle_gcp.py | 32 --------------------------------
3 files changed, 4 insertions(+), 40 deletions(-)
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index b0d0d111..320d1dea 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -23,10 +23,7 @@ RUN uv pip install --system -r /requirements.txt
RUN uv pip uninstall --system google-cloud-bigquery-storage
# b/394382016: sigstore (dependency of kagglehub) requires a prerelease packages, installing separate.
-# b/408284143: google-cloud-automl 2.0.0 introduced incompatible API changes, need to pin to 1.0.1,
-# installed outside of kaggle_requirements.txt due to requiring an incompatibile version of protobuf.
-RUN uv pip install --system --force-reinstall --prerelease=allow "kagglehub[pandas-datasets,hf-datasets,signing]>=0.3.12" \
- "google-cloud-automl==1.0.1"
+RUN uv pip install --system --force-reinstall --prerelease=allow "kagglehub[pandas-datasets,hf-datasets,signing]>=0.3.12"
# uv cannot install this in requirements.txt without --no-build-isolation
# to avoid affecting the larger build, we'll post-install it.
diff --git a/kaggle_requirements.txt b/kaggle_requirements.txt
index c711869e..9bb07412 100644
--- a/kaggle_requirements.txt
+++ b/kaggle_requirements.txt
@@ -45,8 +45,8 @@ geojson
# geopandas > v0.14.4 breaks learn tools
geopandas==v0.14.4
gensim
-# b/443054743
-google-adk
+# b/443054743,b/455550872
+google-adk[a2a,eval]
google-cloud-aiplatform
# b/315753846: Unpin translate package.
google-cloud-translate==3.12.1
@@ -152,5 +152,4 @@ wavio
xgboost==2.0.3
xvfbwrapper
ydata-profiling
-# b/443054743: pinned as newer versions requires protobuf > 3.20.3
-ydf==0.9.0
+ydf
diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py
index 2c8b64cc..64a4611f 100644
--- a/patches/kaggle_gcp.py
+++ b/patches/kaggle_gcp.py
@@ -253,37 +253,6 @@ def init_gcs():
KaggleKernelCredentials(target=GcpTarget.GCS))
return storage
-def init_automl():
- from google.cloud import automl, automl_v1beta1
- if not is_user_secrets_token_set():
- return
-
- from kaggle_gcp import get_integrations
- if not get_integrations().has_cloudai():
- return
-
- from kaggle_secrets import GcpTarget
- from kaggle_gcp import KaggleKernelCredentials
- kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
-
- # Patch the 2 GA clients: AutoMlClient and PreditionServiceClient
- monkeypatch_client(automl.AutoMlClient, kaggle_kernel_credentials)
- monkeypatch_client(automl.PredictionServiceClient, kaggle_kernel_credentials)
-
- # The AutoML client library exposes 3 different client classes (AutoMlClient,
- # TablesClient, PredictionServiceClient), so patch each of them.
- # The same KaggleKernelCredentials are passed to all of them.
- # The GcsClient class is only used internally by TablesClient.
-
- # The beta version of the clients that are now GA are included here for now.
- # They are deprecated and will be removed by 1 May 2020.
- monkeypatch_client(automl_v1beta1.AutoMlClient, kaggle_kernel_credentials)
- monkeypatch_client(automl_v1beta1.PredictionServiceClient, kaggle_kernel_credentials)
-
- # The TablesClient is still in beta, so this will not be deprecated until
- # the TablesClient is GA.
- monkeypatch_client(automl_v1beta1.TablesClient, kaggle_kernel_credentials)
-
def init_translation_v2():
from google.cloud import translate_v2
if not is_user_secrets_token_set():
@@ -379,7 +348,6 @@ def init_vision():
def init():
init_bigquery()
init_gcs()
- init_automl()
init_translation_v2()
init_translation_v3()
init_natural_language()
From 0d57d8ddae4d4a695d4866d4cade6d9d2078bd52 Mon Sep 17 00:00:00 2001
From: Dustin H
Date: Wed, 29 Oct 2025 16:14:21 -0400
Subject: [PATCH 14/54] Add jupyter_server_proxy to requirements
---
kaggle_requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/kaggle_requirements.txt b/kaggle_requirements.txt
index 9bb07412..c91dc073 100644
--- a/kaggle_requirements.txt
+++ b/kaggle_requirements.txt
@@ -66,6 +66,7 @@ jedi
jupyter-lsp==1.5.1
# b/333854354: pin jupyter-server to version 2.12.5; later versions break LSP (b/333854354)
jupyter_server==2.12.5
+jupyter_server_proxy
jupyterlab
jupyterlab-lsp
# b/409363708: Ensure we have the update version, we can consider removing it once
From 523ec52937102fd6375273179f0d87f25a83760f Mon Sep 17 00:00:00 2001
From: Johnny Chavez <64660690+calderjo@users.noreply.github.com>
Date: Wed, 29 Oct 2025 14:35:45 -0700
Subject: [PATCH 15/54] Pin huggingface-hub (#1510)
pytorch-lighting and transformer are throwing errors due to
huggingface-hub requirements.
let's pin huggingface-hub for now
---
Dockerfile.tmpl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index 320d1dea..ba164474 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -48,7 +48,8 @@ RUN uv pip install --system "tbb>=2022" "libpysal==4.9.2"
# b/404590350: Ray and torchtune have conflicting tune cli, we will prioritize torchtune.
# b/415358158: Gensim removed from Colab image to upgrade scipy
-RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3"
+# b/456239669: remove huggingface-hub pin when pytorch-lighting and transformer are compatible
+RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3" "huggingface-hub==0.36.0"
# Adding non-package dependencies:
ADD clean-layer.sh /tmp/clean-layer.sh
From 20bbb5c1e080dcc5c4790d55d348e1f2474ff077 Mon Sep 17 00:00:00 2001
From: Jim Plotts
Date: Wed, 29 Oct 2025 18:56:47 -0400
Subject: [PATCH 16/54] Clean up missed references to AutoML (#1509)
---
patches/sitecustomize.py | 1 -
tests/test_automl.py | 139 ---------------------------------------
2 files changed, 140 deletions(-)
delete mode 100644 tests/test_automl.py
diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py
index e8afb361..4caa5d28 100644
--- a/patches/sitecustomize.py
+++ b/patches/sitecustomize.py
@@ -56,7 +56,6 @@ def create_module(self, spec):
_LOADERS = {
'google.cloud.bigquery': kaggle_gcp.init_bigquery,
'google.cloud.storage': kaggle_gcp.init_gcs,
- 'google.cloud.automl_v1beta1': kaggle_gcp.init_automl,
'google.cloud.translate': kaggle_gcp.init_translation_v3,
'google.cloud.translate_v2': kaggle_gcp.init_translation_v2,
'google.cloud.translate_v3': kaggle_gcp.init_translation_v3,
diff --git a/tests/test_automl.py b/tests/test_automl.py
deleted file mode 100644
index 9a048b14..00000000
--- a/tests/test_automl.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import unittest
-
-from unittest.mock import Mock, patch
-
-from kaggle_gcp import KaggleKernelCredentials, init_automl
-from test.support.os_helper import EnvironmentVarGuard
-from google.cloud import storage, automl_v1beta1, automl
-
-def _make_credentials():
- import google.auth.credentials
- credentials = Mock(spec=google.auth.credentials.Credentials)
- credentials.universe_domain = 'googleapis.com'
- return credentials
-
-class TestAutoMl(unittest.TestCase):
-
- class FakeClient:
- def __init__(self, credentials=None, client_info=None, **kwargs):
- self.credentials = credentials
-
- class FakeConnection():
- def __init__(self, user_agent):
- self.user_agent = user_agent
- if (client_info is not None):
- self._connection = FakeConnection(client_info.user_agent)
-
- @patch("google.cloud.automl.AutoMlClient", new=FakeClient)
- def test_user_provided_credentials(self):
- credentials = _make_credentials()
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
- with env:
- init_automl()
- client = automl.AutoMlClient(credentials=credentials)
- self.assertNotIsInstance(client.credentials, KaggleKernelCredentials)
- self.assertIsNotNone(client.credentials)
-
- def test_tables_gcs_client(self):
- # The GcsClient can't currently be monkeypatched for default
- # credentials because it requires a project which can't be set.
- # Verify that creating an automl_v1beta1.GcsClient given an actual
- # storage.Client sets the client properly.
- gcs_client = storage.Client(project="xyz", credentials=_make_credentials())
- tables_gcs_client = automl_v1beta1.GcsClient(client=gcs_client)
- self.assertIs(tables_gcs_client.client, gcs_client)
-
- @patch("google.cloud.automl_v1beta1.gapic.auto_ml_client.AutoMlClient", new=FakeClient)
- def test_tables_client_credentials(self):
- credentials = _make_credentials()
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
- with env:
- init_automl()
- tables_client = automl_v1beta1.TablesClient(credentials=credentials)
- self.assertEqual(tables_client.auto_ml_client.credentials, credentials)
-
- @patch("google.cloud.automl.AutoMlClient", new=FakeClient)
- def test_default_credentials_automl_client(self):
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
- with env:
- init_automl()
- automl_client = automl.AutoMlClient()
- self.assertIsNotNone(automl_client.credentials)
- self.assertIsInstance(automl_client.credentials, KaggleKernelCredentials)
- self.assertTrue(automl_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))
-
- @patch("google.cloud.automl_v1beta1.AutoMlClient", new=FakeClient)
- def test_default_credentials_automl_v1beta1_client(self):
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
- with env:
- init_automl()
- automl_client = automl_v1beta1.AutoMlClient()
- self.assertIsNotNone(automl_client.credentials)
- self.assertIsInstance(automl_client.credentials, KaggleKernelCredentials)
- self.assertTrue(automl_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))
-
- @patch("google.cloud.automl_v1beta1.TablesClient", new=FakeClient)
- def test_default_credentials_tables_client(self):
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
- with env:
- init_automl()
- tables_client = automl_v1beta1.TablesClient()
- self.assertIsNotNone(tables_client.credentials)
- self.assertIsInstance(tables_client.credentials, KaggleKernelCredentials)
- self.assertTrue(tables_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))
-
- @patch("google.cloud.automl.PredictionServiceClient", new=FakeClient)
- def test_default_credentials_prediction_client(self):
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
- with env:
- prediction_client = automl.PredictionServiceClient()
- self.assertIsNotNone(prediction_client.credentials)
- self.assertIsInstance(prediction_client.credentials, KaggleKernelCredentials)
- self.assertTrue(prediction_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))
-
- @patch("google.cloud.automl_v1beta1.PredictionServiceClient", new=FakeClient)
- def test_default_credentials_prediction_v1beta1_client(self):
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
- with env:
- prediction_client = automl_v1beta1.PredictionServiceClient()
- self.assertIsNotNone(prediction_client.credentials)
- self.assertIsInstance(prediction_client.credentials, KaggleKernelCredentials)
- self.assertTrue(prediction_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))
-
- def test_monkeypatching_idempotent(self):
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
- with env:
- client1 = automl.AutoMlClient.__init__
- init_automl()
- client2 = automl.AutoMlClient.__init__
- self.assertEqual(client1, client2)
-
- @patch("google.cloud.automl_v1beta1.PredictionServiceClient", new=FakeClient)
- def test_legacy_AUTOML_variable_v1beta1_client(self):
- """
- Tests previous KAGGLE_KERNEL_INTEGRATIONS="AUTOML" environment setting
- """
- env = EnvironmentVarGuard()
- env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
- env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
- with env:
- prediction_client = automl_v1beta1.PredictionServiceClient()
- self.assertIsNotNone(prediction_client.credentials)
- self.assertIsInstance(prediction_client.credentials, KaggleKernelCredentials)
- self.assertTrue(prediction_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))
\ No newline at end of file
From 0cb222723ba71fa79c9cf0694297586f53840f49 Mon Sep 17 00:00:00 2001
From: Johnny Chavez <64660690+calderjo@users.noreply.github.com>
Date: Wed, 29 Oct 2025 16:14:48 -0700
Subject: [PATCH 17/54] Remove even more references to AutoMl and Bq (#1511)
---
patches/sitecustomize.py | 1 -
tests/test_imports.py | 1 -
2 files changed, 2 deletions(-)
diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py
index 4caa5d28..b8ae0692 100644
--- a/patches/sitecustomize.py
+++ b/patches/sitecustomize.py
@@ -13,7 +13,6 @@ class GcpModuleFinder(importlib.abc.MetaPathFinder):
_MODULES = [
'google.cloud.bigquery',
'google.cloud.storage',
- 'google.cloud.automl_v1beta1',
'google.cloud.translate',
'google.cloud.translate_v2',
'google.cloud.translate_v3',
diff --git a/tests/test_imports.py b/tests/test_imports.py
index b22ebe7a..6c429516 100644
--- a/tests/test_imports.py
+++ b/tests/test_imports.py
@@ -3,6 +3,5 @@
class TestImport(unittest.TestCase):
# Basic import tests for packages without any.
def test_basic(self):
- import bq_helper
import tensorflow_datasets
import segment_anything
From 708186076f37107ff322fa2b31ca8e34e6bdacea Mon Sep 17 00:00:00 2001
From: Johnny Chavez <64660690+calderjo@users.noreply.github.com>
Date: Fri, 31 Oct 2025 02:35:32 -0700
Subject: [PATCH 18/54] move cloud-translate (#1512)
---
Dockerfile.tmpl | 3 ++-
kaggle_requirements.txt | 2 --
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index ba164474..20acac5c 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -49,7 +49,8 @@ RUN uv pip install --system "tbb>=2022" "libpysal==4.9.2"
# b/404590350: Ray and torchtune have conflicting tune cli, we will prioritize torchtune.
# b/415358158: Gensim removed from Colab image to upgrade scipy
# b/456239669: remove huggingface-hub pin when pytorch-lighting and transformer are compatible
-RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3" "huggingface-hub==0.36.0"
+# b/315753846: Unpin translate package, currently conflicts with adk 1.17.0
+RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3" "huggingface-hub==0.36.0" "google-cloud-translate==3.12.1"
# Adding non-package dependencies:
ADD clean-layer.sh /tmp/clean-layer.sh
diff --git a/kaggle_requirements.txt b/kaggle_requirements.txt
index c91dc073..503d37f9 100644
--- a/kaggle_requirements.txt
+++ b/kaggle_requirements.txt
@@ -48,8 +48,6 @@ gensim
# b/443054743,b/455550872
google-adk[a2a,eval]
google-cloud-aiplatform
-# b/315753846: Unpin translate package.
-google-cloud-translate==3.12.1
google-cloud-videointelligence
google-cloud-vision
google-genai
From e756e929cb98b6c8ef6e016c0c0f917536c4c871 Mon Sep 17 00:00:00 2001
From: Jim Plotts
Date: Mon, 3 Nov 2025 12:12:52 -0500
Subject: [PATCH 19/54] Remove custom logging. (#1514)
This is very old and was probably mostly useful while the GCP
integrations were in development. However, it interferes with the user's
logging in a way that's peculiar to Kaggle, so let's remove it to make
Kaggle more similar to other platforms.
http://b/455836683
---
Dockerfile.tmpl | 1 -
patches/kaggle_gcp.py | 33 +++++-----
patches/log.py | 133 ---------------------------------------
patches/sitecustomize.py | 3 +-
4 files changed, 17 insertions(+), 153 deletions(-)
delete mode 100644 patches/log.py
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index 20acac5c..ba382531 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -164,7 +164,6 @@ ADD patches/kaggle_gcp.py \
patches/kaggle_session.py \
patches/kaggle_web_client.py \
patches/kaggle_datasets.py \
- patches/log.py \
$PACKAGE_PATH/
# Figure out why this is in a different place?
diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py
index 64a4611f..4cb98858 100644
--- a/patches/kaggle_gcp.py
+++ b/patches/kaggle_gcp.py
@@ -1,5 +1,6 @@
import os
import inspect
+import logging
from google.auth import credentials, environment_vars
from google.auth.exceptions import RefreshError
from google.api_core.gapic_v1.client_info import ClientInfo
@@ -8,8 +9,6 @@
from google.cloud.bigquery._http import Connection
from kaggle_secrets import GcpTarget, UserSecretsClient
-from log import Log
-
KAGGLE_GCP_CLIENT_USER_AGENT="kaggle-gcp-client/1.0"
def get_integrations():
@@ -22,7 +21,7 @@ def get_integrations():
target = GcpTarget[integration.upper()]
kernel_integrations.add_integration(target)
except KeyError as e:
- Log.error(f"Unknown integration target: {integration.upper()}")
+ logging.debug(f"Unknown integration target: {integration.upper()}")
return kernel_integrations
@@ -66,14 +65,14 @@ def refresh(self, request):
elif self.target == GcpTarget.CLOUDAI:
self.token, self.expiry = client._get_cloudai_access_token()
except ConnectionError as e:
- Log.error(f"Connection error trying to refresh access token: {e}")
+ logging.error(f"Connection error trying to refresh access token: {e}")
print("There was a connection error trying to fetch the access token. "
f"Please ensure internet is on in order to use the {self.target.service} Integration.")
raise RefreshError('Unable to refresh access token due to connection error.') from e
except Exception as e:
- Log.error(f"Error trying to refresh access token: {e}")
+ logging.error(f"Error trying to refresh access token: {e}")
if (not get_integrations().has_integration(self.target)):
- Log.error(f"No {self.target.service} integration found.")
+ logging.error(f"No {self.target.service} integration found.")
print(
f"Please ensure you have selected a {self.target.service} account in the Notebook Add-ons menu.")
raise RefreshError('Unable to refresh access token.') from e
@@ -102,7 +101,7 @@ def api_request(self, *args, **kwargs):
msg = ("Permission denied using Kaggle's public BigQuery integration. "
"Did you mean to select a BigQuery account in the Notebook Add-ons menu?")
print(msg)
- Log.info(msg)
+ logging.info(msg)
raise e
@@ -156,23 +155,23 @@ def monkeypatch_bq(bq_client, *args, **kwargs):
# Remove these two lines once this is resolved:
# https://github.com/googleapis/google-cloud-python/issues/8108
if explicit_project_id:
- Log.info(f"Explicit project set to {explicit_project_id}")
+ logging.info(f"Explicit project set to {explicit_project_id}")
kwargs['project'] = explicit_project_id
if explicit_project_id is None and specified_credentials is None and not has_bigquery:
msg = "Using Kaggle's public dataset BigQuery integration."
- Log.info(msg)
+ logging.info(msg)
print(msg)
return PublicBigqueryClient(*args, **kwargs)
else:
if specified_credentials is None:
- Log.info("No credentials specified, using KaggleKernelCredentials.")
+ logging.info("No credentials specified, using KaggleKernelCredentials.")
kwargs['credentials'] = KaggleKernelCredentials()
if (not has_bigquery):
- Log.info("No bigquery integration found, creating client anyways.")
+ logging.info("No bigquery integration found, creating client anyways.")
print('Please ensure you have selected a BigQuery '
'account in the Notebook Add-ons menu.')
if explicit_project_id is None:
- Log.info("No project specified while using the unmodified client.")
+ logging.info("No project specified while using the unmodified client.")
print('Please ensure you specify a project id when creating the client'
' in order to use your BigQuery account.')
kwargs['client_info'] = set_kaggle_user_agent(kwargs.get('client_info'))
@@ -196,20 +195,20 @@ def monkeypatch_aiplatform_init(aiplatform_klass, kaggle_kernel_credentials):
def patched_init(*args, **kwargs):
specified_credentials = kwargs.get('credentials')
if specified_credentials is None:
- Log.info("No credentials specified, using KaggleKernelCredentials.")
+ logging.info("No credentials specified, using KaggleKernelCredentials.")
kwargs['credentials'] = kaggle_kernel_credentials
return aiplatform_init(*args, **kwargs)
if (not has_been_monkeypatched(aiplatform_klass.init)):
aiplatform_klass.init = patched_init
- Log.info("aiplatform.init patched")
+ logging.info("aiplatform.init patched")
def monkeypatch_client(client_klass, kaggle_kernel_credentials):
client_init = client_klass.__init__
def patched_init(self, *args, **kwargs):
specified_credentials = kwargs.get('credentials')
if specified_credentials is None:
- Log.info("No credentials specified, using KaggleKernelCredentials.")
+ logging.info("No credentials specified, using KaggleKernelCredentials.")
# Some GCP services demand the billing and target project must be the same.
# To avoid using default service account based credential as caller credential
# user need to provide ClientOptions with quota_project_id:
@@ -227,7 +226,7 @@ def patched_init(self, *args, **kwargs):
if (not has_been_monkeypatched(client_klass.__init__)):
client_klass.__init__ = patched_init
- Log.info(f"Client patched: {client_klass}")
+ logging.info(f"Client patched: {client_klass}")
def set_kaggle_user_agent(client_info: ClientInfo):
# Add kaggle client user agent in order to attribute usage.
@@ -360,4 +359,4 @@ def init():
# google.cloud.* and kaggle_gcp. By calling init here, we guarantee
# that regardless of the original import that caused google.cloud.* to be
# loaded, the monkeypatching will be done.
-init()
+init()
\ No newline at end of file
diff --git a/patches/log.py b/patches/log.py
deleted file mode 100644
index 59a07c8c..00000000
--- a/patches/log.py
+++ /dev/null
@@ -1,133 +0,0 @@
-import io
-import logging
-import os
-
-import google.auth
-
-
-_LOG_TO_FILE_ENV = os.getenv("KAGGLE_LOG_TO_FILE")
-
-
-class _LogFormatter(logging.Formatter):
- """A logging formatter which truncates long messages."""
-
- _MAX_LOG_LENGTH = 10000 # Be generous, not to truncate long backtraces.
-
- def format(self, record):
- msg = super(_LogFormatter, self).format(record)
- return msg[:_LogFormatter._MAX_LOG_LENGTH] if msg else msg
-
-# TODO(vimota): Clean this up once we're using python 3.8 and can use
-# (https://github.com/python/cpython/commit/dde9fdbe453925279ac3d2a6a72102f6f9ef247c)
-# Right now, making the logging module display the intended frame's information
-# when the logging calls (info, warn, ...) are wrapped (as is the case in our
-# Log class) involves fragile logic.
-class _Logger(logging.Logger):
-
- # This is a copy of logging.Logger.findCaller with the filename ignore
- # set expanded to include the current filename (".../log.py").
- # Copyright 2001-2015 by Vinay Sajip. All Rights Reserved.
- # License: https://github.com/python/cpython/blob/ce9e62544571e7ade7186697d5dd065fb4c5243f/LICENSE
- def findCaller(self, stack_info=False, stacklevel=1):
- f = logging.currentframe()
- f = f.f_back
- rv = "(unknown file)", 0, "(unknown function)", None
- while hasattr(f, "f_code"):
- co = f.f_code
- filename = os.path.normcase(co.co_filename)
- if filename in _ignore_srcfiles:
- f = f.f_back
- continue
- sinfo = None
- if stack_info:
- sio = io.StringIO()
- sio.write('Stack (most recent call last):\n')
- traceback.print_stack(f, file=sio)
- sinfo = sio.getvalue()
- if sinfo[-1] == '\n':
- sinfo = sinfo[:-1]
- sio.close()
- rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
- break
- return rv
-
-
-_srcfile = os.path.normcase(_Logger.findCaller.__code__.co_filename)
-_ignore_srcfiles = (_srcfile, logging._srcfile)
-
-class Log:
- """ Helper aggregate for all things related to logging activity. """
-
- _GLOBAL_LOG = logging.getLogger("")
- _initialized = False
-
- # These are convenience helpers. For performance, consider saving Log.get_logger() and using that
- @staticmethod
- def critical(msg, *args, **kwargs):
- Log._GLOBAL_LOG.critical(msg, *args, **kwargs)
-
- @staticmethod
- def fatal(msg, *args, **kwargs):
- Log._GLOBAL_LOG.fatal(msg, *args, **kwargs)
-
- @staticmethod
- def exception(msg, *args, **kwargs):
- Log._GLOBAL_LOG.exception(msg, *args, **kwargs)
-
- @staticmethod
- def error(msg, *args, **kwargs):
- Log._GLOBAL_LOG.error(msg, *args, **kwargs)
-
- @staticmethod
- def warn(msg, *args, **kwargs):
- Log._GLOBAL_LOG.warn(msg, *args, **kwargs)
-
- @staticmethod
- def warning(msg, *args, **kwargs):
- Log._GLOBAL_LOG.warning(msg, *args, **kwargs)
-
- @staticmethod
- def debug(msg, *args, **kwargs):
- Log._GLOBAL_LOG.debug(msg, *args, **kwargs)
-
- @staticmethod
- def info(msg, *args, **kwargs):
- Log._GLOBAL_LOG.info(msg, *args, **kwargs)
-
- @staticmethod
- def set_level(loglevel):
- if isinstance(loglevel, int):
- Log._GLOBAL_LOG.setLevel(loglevel)
- return
- elif isinstance(loglevel, str):
- # idea from https://docs.python.org/3.5/howto/logging.html#logging-to-a-file
- numeric_level = getattr(logging, loglevel.upper(), None)
- if isinstance(numeric_level, int):
- Log._GLOBAL_LOG.setLevel(numeric_level)
- return
-
- raise ValueError('Invalid log level: %s' % loglevel)
-
- @staticmethod
- def _static_init():
- if Log._initialized:
- return
-
- logging.setLoggerClass(_Logger)
- # The root logger's type is unfortunately (and surprisingly) not affected by
- # `setLoggerClass`. Monkey patch it instead. TODO(vimota): Remove this, see the TODO
- # associated with _Logger.
- logging.RootLogger.findCaller = _Logger.findCaller
- log_to_file = _LOG_TO_FILE_ENV.lower() in ("yes", "true", "t", "1") if _LOG_TO_FILE_ENV is not None else True
- if log_to_file:
- handler = logging.FileHandler(filename='/tmp/kaggle.log', mode='w')
- else:
- handler = logging.StreamHandler()
-
- # ".1s" is for the first letter: http://stackoverflow.com/a/27453084/1869.
- format_string = "%(asctime)s %(levelname).1s %(process)d %(filename)s:%(lineno)d] %(message)s"
- handler.setFormatter(_LogFormatter(format_string))
- logging.basicConfig(level=logging.INFO, handlers=[handler])
- Log._initialized = True
-
-Log._static_init()
diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py
index b8ae0692..1bb8a1b6 100644
--- a/patches/sitecustomize.py
+++ b/patches/sitecustomize.py
@@ -1,7 +1,6 @@
+import logging
import os
-from log import Log
-
import sys
import importlib.abc
import importlib
From 4ae73d1e828496efae0eb930b968627399b8c995 Mon Sep 17 00:00:00 2001
From: Johnny Chavez <64660690+calderjo@users.noreply.github.com>
Date: Fri, 12 Dec 2025 12:27:14 -0800
Subject: [PATCH 20/54] Upgrade to Python 3.12 (#1516)
we're bumping our image to Python 3.12
which required the following:
remove numpy-mkl: unfortunately we were not able to find/install a
compatible version, we opted to remove it based on previous conv we had
on this topic. we will instead use the default installed in colab.
remove cuml installation hack:
thankfully are able to use the pre-installed base image version without
build errors.
unpinned package due to learn:
Learn is no longer dependent on this build, we can freely unpinned many
packages
-seaborn, scikit-learn, matplotlib, geopandas, TPOT, shapely, tfdf, ydf,
etc
remove incompatible packages:
Some of these are no longer support and cause build issues
-pydegensac, pymc3, eli5, etc
remove preinstalled package:
where applicable we removed packages that are already installed in colab
base image
https://b.corp.google.com/issues/468103319
---
Dockerfile.tmpl | 28 ++++-----------
config.txt | 2 +-
kaggle_requirements.txt | 34 ++++--------------
.../bert_tiny_en_uncased/2/metadata.json | 6 ----
.../bert_tiny_en_uncased/2/tokenizer.json | 21 -----------
.../{2 => 3}/assets/tokenizer/vocabulary.txt | 0
.../bert_tiny_en_uncased/{2 => 3}/config.json | 6 ++--
.../bert_tiny_en_uncased/3/metadata.json | 10 ++++++
.../bert_tiny_en_uncased/3/model.weights.h5 | Bin 0 -> 17632080 bytes
.../bert_tiny_en_uncased/3/tokenizer.json | 27 ++++++++++++++
tests/test_geopandas.py | 16 ---------
tests/test_matplotlib.py | 4 ---
tests/test_numpy.py | 18 ----------
tests/test_pydegensac.py | 18 ----------
tests/test_xgboost.py | 3 +-
15 files changed, 55 insertions(+), 138 deletions(-)
delete mode 100755 tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/metadata.json
delete mode 100755 tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/tokenizer.json
rename tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/{2 => 3}/assets/tokenizer/vocabulary.txt (100%)
mode change 100755 => 100644
rename tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/{2 => 3}/config.json (68%)
create mode 100755 tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/metadata.json
create mode 100755 tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/model.weights.h5
create mode 100755 tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/tokenizer.json
delete mode 100644 tests/test_geopandas.py
delete mode 100644 tests/test_pydegensac.py
diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl
index ba382531..4f7f05b1 100644
--- a/Dockerfile.tmpl
+++ b/Dockerfile.tmpl
@@ -12,9 +12,6 @@ RUN pip freeze | grep -E 'tensorflow|keras|torch|jax' > /colab_requirements.txt
RUN cat /colab_requirements.txt >> /requirements.txt
RUN cat /kaggle_requirements.txt >> /requirements.txt
-# TODO: GPU requirements.txt
-# TODO: merge them better (override matching ones).
-
# Install Kaggle packages
RUN uv pip install --system -r /requirements.txt
@@ -29,20 +26,6 @@ RUN uv pip install --system --force-reinstall --prerelease=allow "kagglehub[pand
# to avoid affecting the larger build, we'll post-install it.
RUN uv pip install --no-build-isolation --system "git+https://github.com/Kaggle/learntools"
-# b/408281617: Torch is adamant that it can not install cudnn 9.3.x, only 9.1.x, but Tensorflow can only support 9.3.x.
-# This conflict causes a number of package downgrades, which are handled in this command
-RUN uv pip install \
- --index-url https://pypi.nvidia.com --extra-index-url https://pypi.org/simple/ --index-strategy unsafe-first-match \
- --system --force-reinstall "cuml-cu12==25.2.1" \
- "nvidia-cudnn-cu12==9.3.0.75" "nvidia-cublas-cu12==12.5.3.2" "nvidia-cusolver-cu12==11.6.3.83" \
- "nvidia-cuda-cupti-cu12==12.5.82" "nvidia-cuda-nvrtc-cu12==12.5.82" "nvidia-cuda-runtime-cu12==12.5.82" \
- "nvidia-cufft-cu12==11.2.3.61" "nvidia-curand-cu12==10.3.6.82" "nvidia-cusparse-cu12==12.5.1.3" \
- "nvidia-nvjitlink-cu12==12.5.82"
-RUN uv pip install --system --force-reinstall "pynvjitlink-cu12==0.5.2"
-
-# b/385145217 Latest Colab lacks mkl numpy, install it.
-RUN uv pip install --system --force-reinstall -i https://pypi.anaconda.org/intel/simple numpy
-
# newer daal4py requires tbb>=2022, but libpysal is downgrading it for some reason
RUN uv pip install --system "tbb>=2022" "libpysal==4.9.2"
@@ -50,15 +33,18 @@ RUN uv pip install --system "tbb>=2022" "libpysal==4.9.2"
# b/415358158: Gensim removed from Colab image to upgrade scipy
# b/456239669: remove huggingface-hub pin when pytorch-lighting and transformer are compatible
# b/315753846: Unpin translate package, currently conflicts with adk 1.17.0
-RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3" "huggingface-hub==0.36.0" "google-cloud-translate==3.12.1"
+# b/468379293: Unpin Pandas once cuml/cudf are compatible, version 3.0 causes issues
+# b/468383498: numpy will auto-upgrade to 2.4.x, which causes issues with numerous packages
+# b/468367647: Unpin protobuf, version greater than v5.29.5 causes issues with numerous packages
+RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3" "huggingface-hub==0.36.0" "google-cloud-translate==3.12.1" "numpy==2.0.2" "pandas==2.2.2"
+RUN uv pip install --system --force-reinstall "protobuf==5.29.5"
# Adding non-package dependencies:
ADD clean-layer.sh /tmp/clean-layer.sh
ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl
ADD patches/template_conf.json /opt/kaggle/conf.json
-# /opt/conda/lib/python3.11/site-packages
-ARG PACKAGE_PATH=/usr/local/lib/python3.11/dist-packages
+ARG PACKAGE_PATH=/usr/local/lib/python3.12/dist-packages
# Install GPU-specific non-pip packages.
{{ if eq .Accelerator "gpu" }}
@@ -168,7 +154,7 @@ ADD patches/kaggle_gcp.py \
# Figure out why this is in a different place?
# Found by doing a export PYTHONVERBOSE=1 and then running python and checking for where it looked for it.
-ADD patches/sitecustomize.py /usr/lib/python3.11/sitecustomize.py
+ADD patches/sitecustomize.py /usr/lib/python3.12/sitecustomize.py
ARG GIT_COMMIT=unknown \
BUILD_DATE=unknown
diff --git a/config.txt b/config.txt
index af541652..9546b47a 100644
--- a/config.txt
+++ b/config.txt
@@ -1,4 +1,4 @@
BASE_IMAGE=us-docker.pkg.dev/colab-images/public/runtime
-BASE_IMAGE_TAG=release-colab_20250725-060057_RC00
+BASE_IMAGE_TAG=release-colab-external_20251024-060052_RC00
CUDA_MAJOR_VERSION=12
CUDA_MINOR_VERSION=5
diff --git a/kaggle_requirements.txt b/kaggle_requirements.txt
index 503d37f9..7ff2f74e 100644
--- a/kaggle_requirements.txt
+++ b/kaggle_requirements.txt
@@ -7,11 +7,9 @@ PyArabic
PyUpSet
Pympler
Rtree
-shapely<2
+shapely
SimpleITK
-# b/302136621: Fix eli5 import for learntools, newer version require scikit-learn > 1.3
-TPOT==0.12.1
-Theano
+TPOT
Wand
annoy
arrow
@@ -29,21 +27,14 @@ deap
dipy
docker
easyocr
-# b/302136621: Fix eli5 import for learntools
-eli5
emoji
fastcore
-# b/445960030: Requires a newer version of fastai than the currently used base image.
-# Remove when relying on a newer base image.
-fastai>=2.8.4
fasttext
featuretools
fiona
fury
fuzzywuzzy
geojson
-# geopandas > v0.14.4 breaks learn tools
-geopandas==v0.14.4
gensim
# b/443054743,b/455550872
google-adk[a2a,eval]
@@ -81,7 +72,7 @@ libpysal<=4.9.2
lime
line_profiler
mamba
-matplotlib<3.8
+matplotlib
mlcrate
mne
mpld3
@@ -90,9 +81,7 @@ nbconvert==6.4.5
nbdev
nilearn
olefile
-# b/445960030: Broken in 1.19.0. See https://github.com/onnx/onnx/issues/7249.
-# Fixed with https://github.com/onnx/onnx/pull/7254. Upgrade when version with fix is published.
-onnx==1.18.0
+onnx
openslide-bin
openslide-python
optuna
@@ -107,11 +96,9 @@ preprocessing
pudb
pyLDAvis
pycryptodome
-pydegensac
pydicom
pyemd
pyexcel-ods
-pymc3
pymongo
pypdf
pytesseract
@@ -123,32 +110,25 @@ qtconsole
ray
rgf-python
s3fs
-# b/302136621: Fix eli5 import for learntools
-scikit-learn==1.2.2
+scikit-learn
# Scikit-learn accelerated library for x86
scikit-learn-intelex>=2023.0.1
scikit-multilearn
scikit-optimize
scikit-plot
scikit-surprise
-# Also pinning seaborn for learntools
-seaborn==0.12.2
+seaborn
git+https://github.com/facebookresearch/segment-anything.git
-# b/329869023: shap 0.45.0 breaks learntools
-shap==0.44.1
+shap
squarify
tensorflow-cloud
tensorflow-io
tensorflow-text
-tensorflow_decision_forests
torchinfo
torchmetrics
torchtune
transformers>=4.51.0
vtk
wavio
-# b/350573866: xgboost v2.1.0 breaks learntools
-xgboost==2.0.3
xvfbwrapper
ydata-profiling
-ydf
diff --git a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/metadata.json b/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/metadata.json
deleted file mode 100755
index e6beacde..00000000
--- a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/metadata.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "keras_version": "3.0.1",
- "keras_nlp_version": "0.7.0",
- "parameter_count": 4385920,
- "date_saved": "2023-12-27@02:02:24"
-}
\ No newline at end of file
diff --git a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/tokenizer.json b/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/tokenizer.json
deleted file mode 100755
index 48d99632..00000000
--- a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/tokenizer.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "module": "keras_nlp.src.models.bert.bert_tokenizer",
- "class_name": "BertTokenizer",
- "config": {
- "name": "bert_tokenizer",
- "trainable": true,
- "dtype": "int32",
- "vocabulary": null,
- "sequence_length": null,
- "lowercase": true,
- "strip_accents": false,
- "split": true,
- "suffix_indicator": "##",
- "oov_token": "[UNK]"
- },
- "registered_name": "keras_nlp>BertTokenizer",
- "assets": [
- "assets/tokenizer/vocabulary.txt"
- ],
- "weights": null
-}
\ No newline at end of file
diff --git a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/assets/tokenizer/vocabulary.txt b/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/assets/tokenizer/vocabulary.txt
old mode 100755
new mode 100644
similarity index 100%
rename from tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/assets/tokenizer/vocabulary.txt
rename to tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/assets/tokenizer/vocabulary.txt
diff --git a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/config.json b/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/config.json
similarity index 68%
rename from tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/config.json
rename to tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/config.json
index 3afddd31..94aa0b65 100755
--- a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/config.json
+++ b/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/config.json
@@ -1,5 +1,5 @@
{
- "module": "keras_nlp.src.models.bert.bert_backbone",
+ "module": "keras_hub.src.models.bert.bert_backbone",
"class_name": "BertBackbone",
"config": {
"name": "bert_backbone",
@@ -13,7 +13,5 @@
"max_sequence_length": 512,
"num_segments": 2
},
- "registered_name": "keras_nlp>BertBackbone",
- "assets": [],
- "weights": "model.weights.h5"
+ "registered_name": "keras_hub>BertBackbone"
}
\ No newline at end of file
diff --git a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/metadata.json b/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/metadata.json
new file mode 100755
index 00000000..db25ecad
--- /dev/null
+++ b/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/metadata.json
@@ -0,0 +1,10 @@
+{
+ "keras_version": "3.7.0",
+ "keras_hub_version": "0.19.0",
+ "parameter_count": 4385920,
+ "date_saved": "2024-12-20@19:42:50",
+ "tasks": [
+ "MaskedLM",
+ "TextClassifier"
+ ]
+}
\ No newline at end of file
diff --git a/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/model.weights.h5 b/tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/model.weights.h5
new file mode 100755
index 0000000000000000000000000000000000000000..2951f93dcf2fd427558da222ca840236cf0681ce
GIT binary patch
literal 17632080
zcmeEt2{hKcinsckF}g>fA{&G{r#MM_Bnf>6ISyVNQf$m
z^8D>5EX*UoBmMX6pMO7p|Lnyic>h-Sr~Ieg`FG~;jidSC-{o|^Um0GWzy0y>DEzLM
z`OW{Ybjz1oSn!Ne{^9>8{rwovYr$VR;Xf7sO#eqoz{+B-{oe+>`uC{+;OYKvoc$5T
zz71Zx|CV>(;O+9a+#g(;{UP{wrvC`PR{Xyhx&FrGFZ}-R`SDl${yV$qOR@WB##q3}D`CZ;6@~i&OdrkU{?>}^Z
z#ryxoEB%*z^dIQ~`@hKVmmc8zhwE6n_`jAjPt}LNSLFYfl*&I}`rj*oKYGCV_r!Yk
zn}Kx0FC6?Mob5XQt34v^|4f&j{j2@{jbG`X_xneDrGHs|Jo>+J`ES`_*}u)hKYHR1
z|9__cGbHc_zklqoKf49{-}w0>g+F%KzpNjB8vcXnuXRk2$MyH~`M=kzKm7lS``X`h
zlX%7dPV$G&>UY_d|5ttRZ@L4;f9BP%ij?2wJEgzs|BRO>`Crp%|1Mjl{;K~oFW&v-
z{r)=7es}x-Jo!g@Ktb}?s~G?Op#QXI|DS)F|56?PRS*2Z{pvpo;rD3%%sc!4q|yD-
z1H3$c`sF@Q?8O`cJ>VartlIw>GTz@2~hz{Qmh;|FhD6gx^WO@$*L-
z`oA-O))}im4gbOPuj^R;Z`}Uj|5qIU2fO)i{}?Zi;IA{@--6@i(f<3GdHH@-u=f8F
zf!}XQ{I$$K>Y4Z(rOYJNQQybkev^3Qg?Z$U{5{Vz%3b*H?Rzxl5_XT^#l>40p7i>{gVlSG#{$V6kH`S!O@&$a~-vqPA
zjDhg%%7XE_9pLdW3j~(<;N9LQ@TAilUnVBOqea{3c!e7#Crt&w?Y05Us}O`2_g;f*
zWguxa*^h5d6*2XjPLs)k_3&(p8Qm}b14bUl!y$td7>t-mHft!s*a`j6HBp$pm=S<7
zEEZFoX-wse;+PA)ePp?;u&L6M(+p7wg^^nYn5FcId6+AMxlbjif`Tf{sp3U9)fzZp
z9Y)q}mEkV4jsT7P8bNAdi&r#M+L5e`Kfkdd>kL~U^t
zCe#a(6&jhSbs3N)(*^bUl}tq6cq)330~@ErgO>q6tz51KPS2Lm5?y&1cw&k?0y6Mb
zWj7wq)F~7fNq`4(dr5kPLG?hN&H@7O!vhJloFTfRld92EE&gKLD>lj!!-(7wHoF*@{<
zqZ+Y;*}3i#nRa6pM*I3PGe@?7@2ha=ei#TvA_;idbr0DVk%po&CN%wABwZn8#MOB>
zOzwYN3+pur=^Q~&lBHlUI~Kp}zYkHz*&LBxMXZRjrbD_KxVgceERi`x$~A2<(4~2!d;xv4Pe#BHYk?lL;^^9vp~x2LVDUm#g|99Ltb5Nd02V3%4F4B!>09U4m?
zKgnTCrJBIuP7|!Y5{d8Xl5wABEn|FQHy&Tl04HFF$)fAka8@1g*|K3A|EvWrUwZ`C
zbNRVnOmWSq9=X|`iw!v%T!%9|nc)FmdblsVA3NcIO3I
z=KK}xzH{MuS_s)&!w+%IQK09kg!6b6=%!<1OhIIj*k(|8aeWoW-V$Z{mp(+ZF(UM2
zZY1mcW;5(N>knzK=fJLXCzAyN0T{U01pbw
zjTMbpzmr9_)VYG?-o+r$cnT(lABLTkBD81U64+$K;@P82)%-Fl!}9H?GBfCv(BA)rqOLKLbN;Q_*m6B9kQgfEZd`fwsouwC~s^
zh~F$g6_Xb-z1k7*#Bd2bxX9wZePCGVm3RW)7q=pv@`gFQsnKM|QhV-jZV={-lPZcj
z=t;KCP^5QU`#|3~8`sJRQ0G)W>g{=p*`Aw&LaXec_y-G=cddbAyoOXOJ`JlU$brF6
zXL|P3RdR4nEP83&gKK9Wkr1C^5D0Gs4gE71Y^g@R>}x0I21nqIu@p6X8ehQ6o=80d
zcf*^f2GGiPhpDx8p^5w4!9Qy(lwS`6T^l2M(mf1^%-4ZH$8+}gNg4&0Pha7DP`kr=
zTw{p0&C4)rn9`rK%UGSxgM`nyn{~i42*y1rMsEQnX4k?1$g6oz-fIlQbccO#!FM_Y
ze&FF|K8=RO;}+65**%QG^lI>|x(avNTNqYpC{%tQM}<|3N$ATUi0)beM+0x*h@C#Y
zv*Zj%_kcO+7x@hK$&JKca)hMhTeD@Y>S4ZtI|xOIGC%1|`ZX^B)22DnOx+a7EM#*V
z?gO4uVKY~&oM6ZH_t5&$6*9_Sqf?h1HAO9`6g1}UERrMBT<)RD!X6SH*aVYQBQP`v
zA;$kT9>2C8FP{vBwQFSQ$3@?9>C#kS?p$X%OneP!L57Xhfbr1>D7=QyMlQD=J
z3Bk9ok+6%a=z#h#ti0V!42-f#PeCsn>wS*K!QZ)eX$Q2Iq@iKX7`XOOA6z7wSdjtF
zU~kJ`G_&P1@w}W0TI!nAcYG_uKf{PVDjG*M-}B}(C8>0m1{`ZDf!>jD
zX3iluSk-Dny`>|Fe_aBWE;1Ji}&Xq~w!
zIB)l%_sj)pQ1K94e`1AHPL}rP+<~Ltb}`f61Q4e|7Wmuffr(@?^o9hTBujW~A!Kr4kvyJzvk=JB@wE6fgEK7V!oJXAvU%