From 9397c232b8bc3d8ba91b9c82dc03c8c639e85e15 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 9 Aug 2016 14:57:39 +1000 Subject: [PATCH 0001/1660] Add TrueSkill --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2e94de8d..7fc50f1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/tflearn/tflearn.git && \ pip install fitter && \ pip install langid && \ + pip install trueskill && \ # Delorean. Useful for dealing with datetime pip install delorean From 1a86017247a5cefcc2010b6f00239cd69343ef9f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 17 Aug 2016 07:35:58 +1000 Subject: [PATCH 0002/1660] Add heamy package --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 342af278..8cc82bcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,5 +31,6 @@ RUN pip install --upgrade mpld3 && \ # tflean. Deep learning library featuring a higher-level API for TensorFlow. http://tflearn.org pip install git+https://github.com/tflearn/tflearn.git && \ pip install fitter && \ - pip install langid + pip install langid && \ + pip install heamy From 5c2a255d9fc4af461726c204d30b8c8868b62c56 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 19 Aug 2016 13:00:00 +1000 Subject: [PATCH 0003/1660] Add vida library --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9fcac86d..3b2728ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,8 @@ RUN pip install --upgrade mpld3 && \ # Delorean. Useful for dealing with datetime pip install delorean && \ pip install trueskill && \ - pip install heamy + pip install heamy && \ + pip install vida + From 1aa4706a9a2fd9093db516bc9148ee2906f3026d Mon Sep 17 00:00:00 2001 From: Florian Geigl Date: Fri, 19 Aug 2016 18:22:08 -0700 Subject: [PATCH 0004/1660] Reduce the size of the final docker image deleting various temporary/source files and clean up cache --- Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b2728ab..b3bc80c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,11 @@ RUN cd /usr/local/src && git clone https://github.com/scikit-learn/scikit-learn. # HDF5 support conda install h5py && \ # https://github.com/biopython/biopython - pip install biopython + pip install biopython && \ + # clean up + rm -rf /usr/local/src/* && \ + conda clean -i -l -t -y && \ + rm -rf /root/.cache/pip/* ########### # @@ -37,7 +41,9 @@ RUN pip install --upgrade mpld3 && \ pip install delorean && \ pip install trueskill && \ pip install heamy && \ - pip install vida + pip install vida && \ + # clean up pip cache + rm -rf /root/.cache/pip/* From 30bea5e5874c9d7c17ce3e95227cb3c91ff93931 Mon Sep 17 00:00:00 2001 From: Yassine Alouini Date: Sun, 21 Aug 2016 08:31:17 +0200 Subject: [PATCH 0005/1660] Add two Python packages for data exploration * The `missingno` package is nice to explore missing data. Example: ``` import missingno as msno msno.matrix(your_df) ``` * The `pandas_profiling` package is nice for generating data reports (% of missing values, correlated variables, types of variables, histograms...) Example: ``` import pandas_profiling as pdp pdp.ProfileReport(your_df) ``` --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b2728ab..9da95310 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,9 @@ RUN pip install --upgrade mpld3 && \ pip install delorean && \ pip install trueskill && \ pip install heamy && \ - pip install vida - + pip install vida && \ + # Useful data exploration libraries (for missing data and generating reports) + pip install missingno && \ + pip install pandas-profiling From 1d946276fa8fa72e27c5adb86d51c02111261760 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 29 Aug 2016 14:12:20 +1000 Subject: [PATCH 0006/1660] Add s2sphere --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9da95310..c881731e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,8 @@ RUN pip install --upgrade mpld3 && \ pip install vida && \ # Useful data exploration libraries (for missing data and generating reports) pip install missingno && \ - pip install pandas-profiling + pip install pandas-profiling && \ + pip install s2sphere + From de08b18441ec51dd865720e286263953826e1382 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 8 Sep 2016 20:45:43 +1000 Subject: [PATCH 0007/1660] Merge python container into one build stage --- Dockerfile | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 190 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c881731e..c7f3bd8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,199 @@ -FROM kaggle/python3:latest +FROM continuumio/anaconda3:latest +RUN apt-get update && apt-get install -y build-essential && \ + cd /usr/local/src && \ + # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 + # Why does this work, when `pip install tensorflow` fails? It is a mystery + wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl && \ + pip install tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl && \ + # Vowpal Rabbit + apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ + cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ + ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ + pip install vowpalwabbit && \ + pip install seaborn python-dateutil spacy dask pytagcloud pyyaml ggplot joblib \ + husl geopy ml_metrics mne pyshp gensim && \ + # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source + apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ + libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ + libcups2 libcupsfilters1 libcupsimage2 libdatrie1 libdbus-1-3 libdjvulibre-text libdjvulibre21 libfftw3-double3 libfontconfig1 \ + libfreetype6 libgdk-pixbuf2.0-0 libgdk-pixbuf2.0-common libgomp1 libgraphite2-3 libgs9 libgs9-common libharfbuzz0b libijs-0.35 \ + libilmbase6 libjasper1 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1-0 libltdl7 libmagickcore-6.q16-2 \ + libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ + libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ + libxcb-render0 libxcb-shm0 netpbm poppler-data && \ + wget http://www.imagemagick.org/download/ImageMagick-7.0.3-0.tar.gz && \ + tar xzf ImageMagick-7.0.3-0.tar.gz && cd ImageMagick-7.0.3-0 && ./configure && \ + make -j $(nproc) && make install && \ + # clean up ImageMagick source files + cd ../ && rm -rf ImageMagick-7.0.3* && \ + apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ + apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \ + apt-get -y install cmake && \ + cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \ + cd opencv && \ + mkdir build && cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.5m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.5m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.5m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.5m/ -D BUILD_PNG=TRUE .. && \ + make -j $(nproc) && make install && \ + echo "/usr/local/lib/python3.5/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ + cp /usr/local/lib/python3.5/site-packages/cv2.cpython-35m-x86_64-linux-gnu.so /opt/conda/lib/python3.5/site-packages/ && \ + # Clean up install cruft + rm -rf /usr/local/src/opencv && \ + rm /usr/local/src/tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl && \ + rm -rf /root/.cache/pip/* && \ + apt-get autoremove -y && apt-get clean + +RUN apt-get install -y libfreetype6-dev && \ + apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ + # textblob + pip install textblob && \ + #word cloud + pip install git+git://github.com/amueller/word_cloud.git && \ + #igraph + pip install python-igraph && \ + #xgboost + cd /usr/local/src && mkdir xgboost && cd xgboost && \ + git clone --depth 1 --recursive https://github.com/dmlc/xgboost.git && cd xgboost && \ + make && cd python-package && python setup.py install && \ + #lasagne + cd /usr/local/src && mkdir Lasagne && cd Lasagne && \ + git clone --depth 1 https://github.com/Lasagne/Lasagne.git && cd Lasagne && \ + pip install -r requirements.txt && python setup.py install && \ + #keras + cd /usr/local/src && mkdir keras && cd keras && \ + git clone --depth 1 https://github.com/fchollet/keras.git && \ + cd keras && python setup.py install && \ + #neon + cd /usr/local/src && \ + git clone --depth 1 https://github.com/NervanaSystems/neon.git && \ + cd neon && pip install -e . && \ + #nolearn + cd /usr/local/src && mkdir nolearn && cd nolearn && \ + git clone --depth 1 https://github.com/dnouri/nolearn.git && cd nolearn && \ + echo "x" > README.rst && echo "x" > CHANGES.rst && \ + python setup.py install && \ + # Dev branch of Theano + pip install git+git://github.com/Theano/Theano.git --upgrade --no-deps && \ + # put theano compiledir inside /tmp (it needs to be in writable dir) + printf "[global]\nbase_compiledir = /tmp/.theano\n" > /.theanorc && \ + cd /usr/local/src && git clone --depth 1 https://github.com/pybrain/pybrain && \ + cd pybrain && python setup.py install && \ + # Base ATLAS plus tSNE + apt-get install -y libatlas-base-dev && \ + # NOTE: we provide the tsne package, but sklearn.manifold.TSNE now does the same + # job + cd /usr/local/src && git clone --depth 1 https://github.com/danielfrg/tsne.git && \ + cd tsne && python setup.py install && \ + cd /usr/local/src && git clone --depth 1 https://github.com/ztane/python-Levenshtein && \ + cd python-Levenshtein && python setup.py install && \ + cd /usr/local/src && git clone --depth 1 https://github.com/arogozhnikov/hep_ml.git && \ + cd hep_ml && pip install . && \ + # chainer + pip install chainer && \ + # NLTK Project datasets + mkdir -p /usr/share/nltk_data && \ + # NLTK Downloader no longer continues smoothly after an error, so we explicitly list + # the corpuses that work + python -m nltk.downloader -d /usr/share/nltk_data abc alpino \ + averaged_perceptron_tagger basque_grammars biocreative_ppi bllip_wsj_no_aux \ +book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \ +comparative_sentences comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ +europarl_raw floresta framenet_v15 gazetteers genesis gutenberg hmm_treebank_pos_tagger \ +ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho machado \ +masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ +mte_teip5 names nps_chat omw opinion_lexicon panlex_swadesh paradigms \ +pil pl196x ppattach problem_reports product_reviews_1 product_reviews_2 propbank \ +pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor senseval sentence_polarity \ +sentiwordnet shakespeare sinica_treebank smultron snowball_data spanish_grammars \ +state_union stopwords subjectivity swadesh switchboard tagsets timit toolbox treebank \ +twitter_samples udhr2 udhr unicode_samples universal_tagset universal_treebanks_v20 \ +verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ + # Stop-words + pip install stop-words && \ + # clean up + find /usr/share/nltk_data/ -name *.zip | xargs -n1 -I@ rm @ && \ + rm -rf /root/.cache/pip/* && \ + apt-get autoremove -y && apt-get clean && \ + rm -rf /usr/local/src/* + + # Install OpenCV-3 with Python support +RUN apt-get update && \ + cd /usr/local/src/opencv && \ + mkdir build && cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.5m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.5m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.5m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.5m/ -D BUILD_PNG=TRUE .. && \ + make && make install && \ + echo "/usr/local/lib/python3.5/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ + cp /usr/local/lib/python3.5/site-packages/cv2.cpython-35m-x86_64-linux-gnu.so /opt/conda/lib/python3.5/site-packages/ && \ + apt-get -y install libgeos-dev && \ + # pyshp and pyproj are now external dependencies of Basemap + pip install pyshp pyproj && \ + cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \ + export GEOS_DIR=/usr/local && \ + cd basemap && python setup.py install && \ + # Pillow (PIL) + apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev && \ + pip install Pillow && \ + cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep.git && \ + cd opendeep && python setup.py develop && \ + # sasl is apparently an ibis dependency + apt-get -y install libsasl2-dev && \ + pip install ibis-framework && \ + # Cartopy plus dependencies + yes | conda install proj4 && \ + pip install packaging && \ + cd /usr/local/src && git clone https://github.com/Toblerity/Shapely.git && \ + cd Shapely && python setup.py install && \ + cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ + cd cartopy && python setup.py install && \ + # MXNet + cd /usr/local/src && git clone --recursive https://github.com/dmlc/mxnet && \ + cd /usr/local/src/mxnet && cp make/config.mk . && \ + sed -i 's/ADD_LDFLAGS =/ADD_LDFLAGS = -lstdc++/' config.mk && \ + make && cd python && python setup.py install && \ + # set backend for matplotlib to Agg + matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ + sed -i 's/^backend : Qt4Agg/backend : Agg/' $matplotlibrc_path && \ + # Stop jupyter nbconvert trying to rewrite its folder hierarchy + mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ + mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ + # Stop Matplotlib printing junk to the console on first load + sed -i "s/^.*Matplotlib is building the font cache using fc-list.*$/# Warning removed by Kaggle/g" /opt/conda/lib/python3.5/site-packages/matplotlib/font_manager.py && \ + # Make matplotlib output in Jupyter notebooks display correctly + mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \ + # h2o + # This requires python-software-properties and Java. + apt-get install -y python-software-properties zip && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ + apt-get update && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get install -y oracle-java8-installer && \ + cd /usr/local/src && mkdir h2o && cd h2o && \ + wget http://h2o-release.s3.amazonaws.com/h2o/latest_stable -O latest && \ + wget --no-check-certificate -i latest -O h2o.zip && rm latest && \ + unzip h2o.zip && rm h2o.zip && cp h2o-*/h2o.jar . && \ + pip install `find . -name "*whl"` && \ + # Keras setup + # Keras likes to add a config file in a custom directory when it's + # first imported. This doesn't work with our read-only filesystem, so we + # have it done now + python -c "from keras.models import Sequential" && \ + # Switch to TF backend + sed -i 's/theano/tensorflow/' /root/.keras/keras.json && \ + # Re-run it to flush any more disk writes + python -c "from keras.models import Sequential; from keras import backend; print(backend._BACKEND)" && \ + # Keras reverts to /tmp from ~ when it detects a read-only file system + mkdir -p /tmp/.keras && cp /root/.keras/keras.json /tmp/.keras && \ # Scikit-Learn nightly build -RUN cd /usr/local/src && git clone https://github.com/scikit-learn/scikit-learn.git && \ + cd /usr/local/src && git clone https://github.com/scikit-learn/scikit-learn.git && \ cd scikit-learn && python setup.py build && python setup.py install && \ # HDF5 support conda install h5py && \ # https://github.com/biopython/biopython - pip install biopython + pip install biopython && \ + rm -rf /root/.cache/pip/* && \ + apt-get autoremove -y && apt-get clean && \ + rm -rf /usr/local/src/* ########### # @@ -42,6 +229,3 @@ RUN pip install --upgrade mpld3 && \ pip install missingno && \ pip install pandas-profiling && \ pip install s2sphere - - - From 8043e96dc4affa369a40240313e615b3e5649421 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 8 Sep 2016 21:10:29 +1000 Subject: [PATCH 0008/1660] Remove broken corpus --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4c990d4a..c29d8a10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,7 +101,7 @@ comparative_sentences comtrans conll2000 conll2002 conll2007 crubadan dependency europarl_raw floresta framenet_v15 gazetteers genesis gutenberg hmm_treebank_pos_tagger \ ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho machado \ masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ -mte_teip5 names nps_chat omw opinion_lexicon panlex_swadesh paradigms \ +mte_teip5 names nps_chat omw opinion_lexicon paradigms \ pil pl196x ppattach problem_reports product_reviews_1 product_reviews_2 propbank \ pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor senseval sentence_polarity \ sentiwordnet shakespeare sinica_treebank smultron snowball_data spanish_grammars \ From efd7c6d20ab18a0f70eb8501bfb19d96b5429144 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 8 Sep 2016 21:23:02 +1000 Subject: [PATCH 0009/1660] Remove duplicated OpenCV lines --- Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c29d8a10..841651c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -118,12 +118,6 @@ verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Install OpenCV-3 with Python support RUN apt-get update && \ - cd /usr/local/src/opencv && \ - mkdir build && cd build && \ - cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.5m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.5m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.5m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.5m/ -D BUILD_PNG=TRUE .. && \ - make && make install && \ - echo "/usr/local/lib/python3.5/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ - cp /usr/local/lib/python3.5/site-packages/cv2.cpython-35m-x86_64-linux-gnu.so /opt/conda/lib/python3.5/site-packages/ && \ apt-get -y install libgeos-dev && \ # pyshp and pyproj are now external dependencies of Basemap pip install pyshp pyproj && \ From f2b4d46b8f0979b766cba05dddfe129768c06d62 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 12 Sep 2016 09:26:57 +1000 Subject: [PATCH 0010/1660] Add BayesianOptimization --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 841651c9..3d96faec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -224,5 +224,7 @@ RUN pip install --upgrade mpld3 && \ pip install missingno && \ pip install pandas-profiling && \ pip install s2sphere && \ + pip install git+https://github.com/fmfn/BayesianOptimization.git && \ + ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From c178409a4c91a30eb4ada69a69601acf64d516e2 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 12 Sep 2016 09:27:51 +1000 Subject: [PATCH 0011/1660] Add matplotlib-venn --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 3d96faec..ddc3a81a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -225,6 +225,7 @@ RUN pip install --upgrade mpld3 && \ pip install pandas-profiling && \ pip install s2sphere && \ pip install git+https://github.com/fmfn/BayesianOptimization.git && \ + pip install matplotlib-venn && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 60bf275ab9b913663078d5098ce3754c7d76c23a Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 12 Sep 2016 09:33:46 +1000 Subject: [PATCH 0012/1660] Add test script to build repo --- test_build.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 test_build.py diff --git a/test_build.py b/test_build.py new file mode 100644 index 00000000..6e88137d --- /dev/null +++ b/test_build.py @@ -0,0 +1,55 @@ +# This script should run without errors whenever we update the +# kaggle/python container. It checks that all our most popular packages can +# be loaded and used without errors. + +import numpy as np +print("Numpy imported ok") +print("Your lucky number is: " + str(np.random.randint(100))) + +import pandas as pd +print("Pandas imported ok") + +from sklearn import datasets +print("sklearn imported ok") +iris = datasets.load_iris() +X, y = iris.data, iris.target + +from sklearn.ensemble import RandomForestClassifier +rf1 = RandomForestClassifier() +rf1.fit(X,y) +print("sklearn RandomForestClassifier: ok") + +from xgboost import XGBClassifier +xgb1 = XGBClassifier(n_estimators=3) +xgb1.fit(X[0:70],y[0:70]) +print("xgboost XGBClassifier: ok") + +import matplotlib.pyplot as plt +plt.plot(np.linspace(0,1,50), np.random.rand(50)) +plt.savefig("plot1.png") +print("matplotlib.pyplot ok") + +from mpl_toolkits.basemap import Basemap +print("Basemap ok") + +import theano +print("Theano ok") + +from keras.models import Sequential +from keras.layers.core import Dense, Dropout, Activation, Flatten +from keras.layers.convolutional import Convolution2D, MaxPooling2D +from keras.optimizers import SGD +print("keras ok") + +import nltk +from nltk.stem import WordNetLemmatizer +print("nltk ok") + +import tensorflow as tf +hello = tf.constant('TensorFlow ok') +sess = tf.Session() +print(sess.run(hello)) + +import cv2 +img = cv2.imread('plot1.png',0) +print("OpenCV ok") \ No newline at end of file From 021696fa45b465da88d0fcf08a16fc43f84ae378 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 20 Sep 2016 10:40:45 +1000 Subject: [PATCH 0013/1660] Update Readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 12257a87..1f188531 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ ![example script](http://i.imgur.com/GrZ7diw.png) -This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://registry.hub.docker.com/u/kaggle/python/) the Docker image on Dockerhub +This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://registry.hub.docker.com/u/kaggle/python/) the Docker image on Dockerhub. -Because the build process of this image has become quite lengthy, we've split it into three. The base images are at [here](https://github.com/Kaggle/docker-python0) and [here](https://github.com/Kaggle/docker-python1). +For a guide to using the container, see [our blog](http://blog.kaggle.com/2016/02/05/how-to-get-started-with-data-science-in-containers). **We welcome pull requests** if there are any packages you'd like to add! From 0968f2cdcb9004f658ca010136c6e37311af4c25 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 22 Sep 2016 09:33:59 +1000 Subject: [PATCH 0014/1660] Add mlxtend --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index ddc3a81a..6d869e89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -226,6 +226,7 @@ RUN pip install --upgrade mpld3 && \ pip install s2sphere && \ pip install git+https://github.com/fmfn/BayesianOptimization.git && \ pip install matplotlib-venn && \ + pip install git+git://github.com/rasbt/mlxtend.git#egg=mlxtend && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 85b9c68be08fae148e920d3d2dcc7d25d24a6675 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 22 Sep 2016 09:50:18 +1000 Subject: [PATCH 0015/1660] Update Imagemagick version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6d869e89..1b4ef7c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ RUN apt-get update && apt-get install -y build-essential && \ libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data && \ - wget http://www.imagemagick.org/download/ImageMagick-7.0.3-0.tar.gz && \ - tar xzf ImageMagick-7.0.3-0.tar.gz && cd ImageMagick-7.0.3-0 && ./configure && \ + wget http://www.imagemagick.org/download/ImageMagick-7.0.3-1.tar.gz && \ + tar xzf ImageMagick-7.0.3-1.tar.gz && cd ImageMagick-7.0.3-1 && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick-7.0.3* && \ From 8b8a21564a261600575162d4df002d9cab34a91a Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 26 Sep 2016 06:29:38 +1000 Subject: [PATCH 0016/1660] Add automated build script --- build | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 build diff --git a/build b/build new file mode 100755 index 00000000..7ac5b4a1 --- /dev/null +++ b/build @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +docker pull continuumio/anaconda3:latest +docker build --rm --no-cache -t kaggle/python-build . + +rm -rf /tmp/python-build +mkdir -p /tmp/python-build/tmp +mkdir -p /tmp/python-build/devshm +mkdir -p /tmp/python-build/working +docker run --rm -it --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' + + +# The test_build.py script creates a plot called plot1.png; check that it exists +[ -s /tmp/python-build/working/plot1.png ] || exit 1 + From f54b413bd828bb4550dd6979c79a427c80c66701 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 26 Sep 2016 06:29:49 +1000 Subject: [PATCH 0017/1660] Add rgf_python --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1b4ef7c7..658d0564 100644 --- a/Dockerfile +++ b/Dockerfile @@ -184,7 +184,9 @@ RUN apt-get update && \ # HDF5 support conda install h5py && \ # https://github.com/biopython/biopython - pip install biopython && \ + pip install biopython && \ + cd /usr/local/src && git clone https://github.com/fukatani/rgf_python.git && \ + cd rgf_python && python setup.py install && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ conda clean -i -l -t -y && \ From 65101d38d0aa8a36cee604cfba56437001f6ad3d Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 26 Sep 2016 06:34:05 +1000 Subject: [PATCH 0018/1660] Add docker push --- build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build b/build index 7ac5b4a1..8ceaacd9 100755 --- a/build +++ b/build @@ -10,7 +10,8 @@ mkdir -p /tmp/python-build/devshm mkdir -p /tmp/python-build/working docker run --rm -it --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' - # The test_build.py script creates a plot called plot1.png; check that it exists [ -s /tmp/python-build/working/plot1.png ] || exit 1 +docker tag kaggle/python-build:latest kaggle/python:latest +docker push kaggle/python:latest From beeaf98148d1cb1fa4b3c5425ea14a3485d9e3d5 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 28 Sep 2016 06:03:18 +1000 Subject: [PATCH 0019/1660] Add RGF binary --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 658d0564..848bf80f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -185,8 +185,12 @@ RUN apt-get update && \ conda install h5py && \ # https://github.com/biopython/biopython pip install biopython && \ + # Regularized Greedy Forests + cd /usr/local/src && wget http://stat.rutgers.edu/home/tzhang/software/rgf/rgf1.2.zip && \ + unzip rgf1.2.zip && cd rgf1.2 && make && mv bin/rgf /usr/local/bin && \ cd /usr/local/src && git clone https://github.com/fukatani/rgf_python.git && \ - cd rgf_python && python setup.py install && \ + cd rgf_python && sed -i 's/\/opt\/rgf1.2\/bin\/rgf/\/usr\/local\/bin\/rgf/' rgf/rgf.py && \ + python setup.py install && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ conda clean -i -l -t -y && \ From f1a0119f30b05ae8711a8d2c41e7324d6c9cc47d Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 28 Sep 2016 06:12:39 +1000 Subject: [PATCH 0020/1660] Add PUDB --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 848bf80f..9127c289 100644 --- a/Dockerfile +++ b/Dockerfile @@ -191,6 +191,8 @@ RUN apt-get update && \ cd /usr/local/src && git clone https://github.com/fukatani/rgf_python.git && \ cd rgf_python && sed -i 's/\/opt\/rgf1.2\/bin\/rgf/\/usr\/local\/bin\/rgf/' rgf/rgf.py && \ python setup.py install && \ + # PUDB, for local debugging convenience + pip install pudb && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ conda clean -i -l -t -y && \ From 88d73c5807feacf390a45ed42d05311a140f49b1 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 28 Sep 2016 06:36:31 +1000 Subject: [PATCH 0021/1660] Remove defective NLTK corpus --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9127c289..bf768e67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,7 +103,7 @@ ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho m masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ mte_teip5 names nps_chat omw opinion_lexicon paradigms \ pil pl196x ppattach problem_reports product_reviews_1 product_reviews_2 propbank \ -pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor senseval sentence_polarity \ +pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor sentence_polarity \ sentiwordnet shakespeare sinica_treebank smultron snowball_data spanish_grammars \ state_union stopwords subjectivity swadesh switchboard tagsets timit toolbox treebank \ twitter_samples udhr2 udhr unicode_samples universal_tagset universal_treebanks_v20 \ From 21084495629bf0b16c37a4aa9e74b606278fdd7f Mon Sep 17 00:00:00 2001 From: Michele Lacchia Date: Sat, 8 Oct 2016 09:34:11 +0200 Subject: [PATCH 0022/1660] Add Altair package --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bf768e67..3a90ff87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -235,6 +235,9 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/fmfn/BayesianOptimization.git && \ pip install matplotlib-venn && \ pip install git+git://github.com/rasbt/mlxtend.git#egg=mlxtend && \ + pip install altair && \ ##### ^^^^ Add new contributions above here # clean up pip cache - rm -rf /root/.cache/pip/* + rm -rf /root/.cache/pip/* && \ + # Required to display Altair charts in Jupyter notebook + jupyter nbextension install --py vega From 04466032b43f0365253375a5960399b2457f3404 Mon Sep 17 00:00:00 2001 From: Michele Lacchia Date: Sat, 8 Oct 2016 09:40:23 +0200 Subject: [PATCH 0023/1660] Add the --user option to nbextension install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3a90ff87..8bc8689c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -240,4 +240,4 @@ RUN pip install --upgrade mpld3 && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ # Required to display Altair charts in Jupyter notebook - jupyter nbextension install --py vega + jupyter nbextension install --user --py vega From e0f86f14110c128fd590f8336fb0ce1d7cecb719 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 11 Oct 2016 21:18:29 +1100 Subject: [PATCH 0024/1660] Add Spacy download --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index bf768e67..d01766d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y build-essential && \ pip install vowpalwabbit && \ pip install seaborn python-dateutil spacy dask pytagcloud pyyaml ggplot joblib \ husl geopy ml_metrics mne pyshp gensim && \ + python -m spacy.en.download && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ From e8cb716a7ed0dbcf1365d2c3680210144649b505 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 11 Oct 2016 21:32:15 +1100 Subject: [PATCH 0025/1660] Update ImageMagick version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d01766d0..fa27b788 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,8 @@ RUN apt-get update && apt-get install -y build-essential && \ libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data && \ - wget http://www.imagemagick.org/download/ImageMagick-7.0.3-1.tar.gz && \ - tar xzf ImageMagick-7.0.3-1.tar.gz && cd ImageMagick-7.0.3-1 && ./configure && \ + wget http://www.imagemagick.org/download/ImageMagick-7.0.3-4.tar.gz && \ + tar xzf ImageMagick-7.0.3-4.tar.gz && cd ImageMagick-7.0.3-4 && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick-7.0.3* && \ From 621925e9e6747ed5a2efb718717bdd1561842b26 Mon Sep 17 00:00:00 2001 From: Ben Hamner Date: Mon, 17 Oct 2016 09:16:45 -0700 Subject: [PATCH 0026/1660] removing spacy/shapely/cartopy for the moment --- Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index fa27b788..21d50f99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,9 @@ RUN apt-get update && apt-get install -y build-essential && \ cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ pip install vowpalwabbit && \ - pip install seaborn python-dateutil spacy dask pytagcloud pyyaml ggplot joblib \ + pip install seaborn python-dateutil dask pytagcloud pyyaml ggplot joblib \ husl geopy ml_metrics mne pyshp gensim && \ - python -m spacy.en.download && \ + #python -m spacy.en.download && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ @@ -136,10 +136,11 @@ RUN apt-get update && \ # Cartopy plus dependencies yes | conda install proj4 && \ pip install packaging && \ - cd /usr/local/src && git clone https://github.com/Toblerity/Shapely.git && \ - cd Shapely && python setup.py install && \ - cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ - cd cartopy && python setup.py install && \ + # Removing Shapely for now + #cd /usr/local/src && git clone https://github.com/Toblerity/Shapely.git && \ + #cd Shapely && python setup.py install && \ + #cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ + #cd cartopy && python setup.py install && \ # MXNet cd /usr/local/src && git clone --recursive https://github.com/dmlc/mxnet && \ cd /usr/local/src/mxnet && cp make/config.mk . && \ From b2db240657bdbeb3a70aefe0db86ba9a2a6d98be Mon Sep 17 00:00:00 2001 From: Sara Collins Date: Tue, 18 Oct 2016 16:38:07 -0500 Subject: [PATCH 0027/1660] Added pip installation of pyLDAvis Added pyLDAvis, from https://github.com/bmabey/pyLDAvis --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 21d50f99..221c5667 100644 --- a/Dockerfile +++ b/Dockerfile @@ -236,6 +236,7 @@ RUN pip install --upgrade mpld3 && \ pip install s2sphere && \ pip install git+https://github.com/fmfn/BayesianOptimization.git && \ pip install matplotlib-venn && \ + pip install pyldavis && \ pip install git+git://github.com/rasbt/mlxtend.git#egg=mlxtend && \ ##### ^^^^ Add new contributions above here # clean up pip cache From 12b8c4f499c17bf86a644db3ea0113893c94224e Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 19 Oct 2016 10:28:32 +1100 Subject: [PATCH 0028/1660] Support building from cache --- build | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build b/build index 8ceaacd9..86ae8570 100755 --- a/build +++ b/build @@ -1,8 +1,16 @@ #!/bin/bash set -e -docker pull continuumio/anaconda3:latest -docker build --rm --no-cache -t kaggle/python-build . +# Default behavior is to do everything from scratch. +# The --use-cache option is useful if you're iterating on a broken build. +if [[ "$1" == "--use-cache" ]]; then + docker build --rm -t kaggle/python-build . +else + docker pull continuumio/anaconda3:latest + docker build --rm --no-cache -t kaggle/python-build . +fi + + rm -rf /tmp/python-build mkdir -p /tmp/python-build/tmp From a5c748f916fcdf6ccecd7eb6f3be9d42b05aa6ef Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 19 Oct 2016 11:30:30 +1100 Subject: [PATCH 0029/1660] Update Conda GCC version --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 21d50f99..63ba2f6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM continuumio/anaconda3:latest RUN apt-get update && apt-get install -y build-essential && \ + # Anaconda's gcc is out of date, which causes linking problems; this helps a bit + conda install -y gcc && \ cd /usr/local/src && \ # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery From 21b3aac423e61faaaa138bd97e9999a3579db23a Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 19 Oct 2016 11:45:15 +1100 Subject: [PATCH 0030/1660] Move GCC installation order --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 63ba2f6d..196ba808 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ FROM continuumio/anaconda3:latest -RUN apt-get update && apt-get install -y build-essential && \ +RUN apt-get update && \ # Anaconda's gcc is out of date, which causes linking problems; this helps a bit conda install -y gcc && \ + apt-get install -y build-essential && \ cd /usr/local/src && \ # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery From 5a6e5a1b7f2dd077f4619c57b2160127f666a696 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 19 Oct 2016 16:47:51 +1100 Subject: [PATCH 0031/1660] Reinstate Shapely etc; monkey-patch Conda's gcc --- Dockerfile | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 196ba808..eb176a46 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,6 @@ FROM continuumio/anaconda3:latest -RUN apt-get update && \ - # Anaconda's gcc is out of date, which causes linking problems; this helps a bit - conda install -y gcc && \ - apt-get install -y build-essential && \ +RUN apt-get update && apt-get install -y build-essential && \ cd /usr/local/src && \ # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery @@ -140,10 +137,10 @@ RUN apt-get update && \ yes | conda install proj4 && \ pip install packaging && \ # Removing Shapely for now - #cd /usr/local/src && git clone https://github.com/Toblerity/Shapely.git && \ - #cd Shapely && python setup.py install && \ - #cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ - #cd cartopy && python setup.py install && \ + cd /usr/local/src && git clone https://github.com/Toblerity/Shapely.git && \ + cd Shapely && python setup.py install && \ + cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ + cd cartopy && python setup.py install && \ # MXNet cd /usr/local/src && git clone --recursive https://github.com/dmlc/mxnet && \ cd /usr/local/src/mxnet && cp make/config.mk . && \ @@ -198,6 +195,12 @@ RUN apt-get update && \ python setup.py install && \ # PUDB, for local debugging convenience pip install pudb && \ + # Anaconda's build of gcc is way out of date; monkey-patch some linking problems that affect + # packages like xgboost and Shapely + rm /opt/conda/lib/libstdc++* && rm /opt/conda/lib/libgomp.* && \ + ln -s /usr/lib/x86_64-linux-gnu/libgomp.so.1 /opt/conda/lib/libgomp.so.1 && \ + ln -s /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6 && \ + # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ conda clean -i -l -t -y && \ From a5537d975279ec8ab4b3bd3a754762d1f041fa32 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 19 Oct 2016 17:51:01 +1100 Subject: [PATCH 0032/1660] Fix Shapely linking issue --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index eb176a46..185bddf8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,6 +119,12 @@ verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Install OpenCV-3 with Python support RUN apt-get update && \ + # Anaconda's build of gcc is way out of date; monkey-patch some linking problems that affect + # packages like xgboost and Shapely + rm /opt/conda/lib/libstdc++* && rm /opt/conda/lib/libgomp.* && \ + ln -s /usr/lib/x86_64-linux-gnu/libgomp.so.1 /opt/conda/lib/libgomp.so.1 && \ + ln -s /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6 && \ + # Libgeos, for mapping libraries apt-get -y install libgeos-dev && \ # pyshp and pyproj are now external dependencies of Basemap pip install pyshp pyproj && \ @@ -136,7 +142,6 @@ RUN apt-get update && \ # Cartopy plus dependencies yes | conda install proj4 && \ pip install packaging && \ - # Removing Shapely for now cd /usr/local/src && git clone https://github.com/Toblerity/Shapely.git && \ cd Shapely && python setup.py install && \ cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ @@ -195,11 +200,6 @@ RUN apt-get update && \ python setup.py install && \ # PUDB, for local debugging convenience pip install pudb && \ - # Anaconda's build of gcc is way out of date; monkey-patch some linking problems that affect - # packages like xgboost and Shapely - rm /opt/conda/lib/libstdc++* && rm /opt/conda/lib/libgomp.* && \ - ln -s /usr/lib/x86_64-linux-gnu/libgomp.so.1 /opt/conda/lib/libgomp.so.1 && \ - ln -s /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6 && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 54d8068366708100a2152cac58bac0bbb8f7c95e Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 19 Oct 2016 21:23:59 +1100 Subject: [PATCH 0033/1660] Update Matplotlib config --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 185bddf8..af346231 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,7 +153,7 @@ RUN apt-get update && \ make && cd python && python setup.py install && \ # set backend for matplotlib to Agg matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ - sed -i 's/^backend : Qt4Agg/backend : Agg/' $matplotlibrc_path && \ + sed -i 's/^backend : Qt5Agg/backend : Agg/' $matplotlibrc_path && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ From 7512a823752786edeca30637bce8fd3843b6e31e Mon Sep 17 00:00:00 2001 From: Sara Collins Date: Sat, 22 Oct 2016 15:56:17 -0500 Subject: [PATCH 0034/1660] Update Dockerfile to add 'pattern' python package Updated Dockerfile to add 'pattern' python package via pip --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 221c5667..e38665ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -237,6 +237,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/fmfn/BayesianOptimization.git && \ pip install matplotlib-venn && \ pip install pyldavis && \ + pip install pattern && \ pip install git+git://github.com/rasbt/mlxtend.git#egg=mlxtend && \ ##### ^^^^ Add new contributions above here # clean up pip cache From ce0761ec91e98755eaf696ab9e485870bd7aa2d2 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 24 Oct 2016 09:21:06 +1100 Subject: [PATCH 0035/1660] Remove STDIN mapping from test exec --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 86ae8570..d2720b3f 100755 --- a/build +++ b/build @@ -16,7 +16,7 @@ rm -rf /tmp/python-build mkdir -p /tmp/python-build/tmp mkdir -p /tmp/python-build/devshm mkdir -p /tmp/python-build/working -docker run --rm -it --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' +docker run --rm -t --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' # The test_build.py script creates a plot called plot1.png; check that it exists [ -s /tmp/python-build/working/plot1.png ] || exit 1 From 4f362d0f818f5f83704ef9e0e289da24d9894b5b Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Oct 2016 06:51:43 +1100 Subject: [PATCH 0036/1660] Add vader_lexicon to NLTK --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index af346231..f65aea98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,7 +108,7 @@ pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor sentence_polarity sentiwordnet shakespeare sinica_treebank smultron snowball_data spanish_grammars \ state_union stopwords subjectivity swadesh switchboard tagsets timit toolbox treebank \ twitter_samples udhr2 udhr unicode_samples universal_tagset universal_treebanks_v20 \ -verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ +vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Stop-words pip install stop-words && \ # clean up From 796f97c512bdc127846784d29abd110dc0cab315 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Oct 2016 09:30:15 +1100 Subject: [PATCH 0037/1660] Add imbalanced-learn --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index f65aea98..32277b7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -200,6 +200,9 @@ RUN apt-get update && \ python setup.py install && \ # PUDB, for local debugging convenience pip install pudb && \ + # Imbalanced-learn + cd /usr/local/src && git clone https://github.com/scikit-learn-contrib/imbalanced-learn.git && \ + cd imbalanced-learn && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 89ae959d0ab2df2429a61b91bf5e32479f9b4f06 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Oct 2016 10:43:46 +1100 Subject: [PATCH 0038/1660] Temporarily remove Pattern until Py3 compatible --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 998e15b5..4fe917e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -246,7 +246,8 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/fmfn/BayesianOptimization.git && \ pip install matplotlib-venn && \ pip install pyldavis && \ - pip install pattern && \ + # Pattern not yet Py3 compatible... + # pip install pattern && \ pip install git+git://github.com/rasbt/mlxtend.git#egg=mlxtend && \ pip install altair && \ ##### ^^^^ Add new contributions above here From 398d6558c32b31ddbc2f54e4214e8e6a667ea1e0 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Oct 2016 14:16:31 +1100 Subject: [PATCH 0039/1660] Spare vader lexicon when cleaning NLTK zip files --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4fe917e3..a5888788 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,7 +112,7 @@ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Stop-words pip install stop-words && \ # clean up - find /usr/share/nltk_data/ -name *.zip | xargs -n1 -I@ rm @ && \ + find /usr/share/nltk_data/ -name *.zip | grep -v vader_lexicon | xargs -n1 -I@ rm @ && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ rm -rf /usr/local/src/* From e5fe6cd580d5aad612d82fc23db187881ee95cb0 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 26 Oct 2016 07:37:54 +1100 Subject: [PATCH 0040/1660] Stop removing NLTK corpora zips --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a5888788..85bb53ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,7 +112,6 @@ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Stop-words pip install stop-words && \ # clean up - find /usr/share/nltk_data/ -name *.zip | grep -v vader_lexicon | xargs -n1 -I@ rm @ && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ rm -rf /usr/local/src/* From 952d5e75dc57175b6367f4ed8745e20fa6cef67b Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 27 Oct 2016 12:20:28 +1100 Subject: [PATCH 0041/1660] Add cvxopt and other utilities --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 85bb53ab..37498e12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -202,6 +202,12 @@ RUN apt-get update && \ # Imbalanced-learn cd /usr/local/src && git clone https://github.com/scikit-learn-contrib/imbalanced-learn.git && \ cd imbalanced-learn && python setup.py install && \ + # Convex Optimization library + pip install cvxopt && \ + # Profiling and other utilities + pip install line_profiler && \ + pip install orderedmultidict && \ + pip install smhasher && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 9560713510cc93b8c8b964c883a0d1ec866361f5 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 27 Oct 2016 16:00:02 +1100 Subject: [PATCH 0042/1660] Update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1f188531..97f95c71 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -**Kaggle Scripts** allow users to run scripts against our competition data sets without having to download data or set up their environment. Here's [an example](https://www.kaggle.com/users/213536/vasco/predict-west-nile-virus/west-nile-heatmap): +**Kaggle Kernels** allow users to run scripts against our competitions and datasets without having to download data or set up their environment. Here's [an example](https://www.kaggle.com/devinanzelmo/d/devinanzelmo/dota-2-matches/setting-up-a-prediction-problem-dota-2): -![example script](http://i.imgur.com/GrZ7diw.png) +![example script](http://i.imgur.com/yrWycNA.png) This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://registry.hub.docker.com/u/kaggle/python/) the Docker image on Dockerhub. -For a guide to using the container, see [our blog](http://blog.kaggle.com/2016/02/05/how-to-get-started-with-data-science-in-containers). +To get started with this image, read our (guide)[http://blog.kaggle.com/2016/02/05/how-to-get-started-with-data-science-in-containers/] to using it yourself, or browse [Kaggle Kernels](https://www.kaggle.com/kernels) for ideas. **We welcome pull requests** if there are any packages you'd like to add! From c69b782dc0c60199e2095d6937afbf76b5af1ed7 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 4 Nov 2016 11:38:53 +1100 Subject: [PATCH 0043/1660] Update readme --- README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/README.md b/README.md index 97f95c71..f4bd4270 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,44 @@ This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://registry.hub.docker.com/u/kaggle/python/) the Docker image on Dockerhub. +## Getting started + To get started with this image, read our (guide)[http://blog.kaggle.com/2016/02/05/how-to-get-started-with-data-science-in-containers/] to using it yourself, or browse [Kaggle Kernels](https://www.kaggle.com/kernels) for ideas. +## Requesting new features + **We welcome pull requests** if there are any packages you'd like to add! + +We can merge your request quickly if you check that it builds correctly. Here's how to do that. + +Start by running this image on your system: + +``` +me@my-computer:/home$ docker run --rm -it kaggle/python +root@d72b81a003e1:/# +``` + +Then follow the package's installation instructions for a Linux system. It could be as simple as installing via Pip: + +``` +root@d72b81a003e1:/# pip install coolpackage +Collecking coolpackage +[...etc...] +``` + +Once that's done, check that you can import it correctly. (Sometimes, if a package is missing a dependency, it throws an error when you try to import it.) + +``` +root@d72b81a003e1:/# python +Python 3.5.2 |Anaconda 4.2.0 (64-bit)| (default, Jul 2 2016, 17:53:06) +[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import coolpackage +>>> +``` + +Once that's working, add the necessary lines to our `Dockerfile`. (In this case, that would mean adding `pip install coolpackage` to the end section of the Dockerfile.) Then submit your pull request, and you're all set! + + + + From bbeea98afd4c3dbcebc7e53423ade73a5b18231a Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 4 Nov 2016 11:41:12 +1100 Subject: [PATCH 0044/1660] Update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4bd4270..9772b51d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is the Dockerfile (etc.) used for building the image that runs python scrip ## Getting started -To get started with this image, read our (guide)[http://blog.kaggle.com/2016/02/05/how-to-get-started-with-data-science-in-containers/] to using it yourself, or browse [Kaggle Kernels](https://www.kaggle.com/kernels) for ideas. +To get started with this image, read our [guide](http://blog.kaggle.com/2016/02/05/how-to-get-started-with-data-science-in-containers/) to using it yourself, or browse [Kaggle Kernels](https://www.kaggle.com/kernels) for ideas. ## Requesting new features From e2cd140c43f9cbd292199d83ba9db421ff085e27 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 4 Nov 2016 11:43:05 +1100 Subject: [PATCH 0045/1660] Correct typo, update link --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9772b51d..5809c317 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Then follow the package's installation instructions for a Linux system. It could ``` root@d72b81a003e1:/# pip install coolpackage -Collecking coolpackage +Collecting coolpackage [...etc...] ``` @@ -40,7 +40,7 @@ Type "help", "copyright", "credits" or "license" for more information. >>> ``` -Once that's working, add the necessary lines to our `Dockerfile`. (In this case, that would mean adding `pip install coolpackage` to the end section of the Dockerfile.) Then submit your pull request, and you're all set! +Once that's working, add the necessary lines to our [Dockerfile](https://github.com/Kaggle/docker-python/blob/master/Dockerfile). (In this case, that would mean adding `pip install coolpackage` to the last section.) Then submit your pull request, and you're all set! From c4e6b6ac08f91e07cf3ad6e5cecd568e69f25091 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 15 Nov 2016 16:41:55 +1100 Subject: [PATCH 0046/1660] Add PyStan --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 37498e12..885944d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -255,6 +255,7 @@ RUN pip install --upgrade mpld3 && \ # pip install pattern && \ pip install git+git://github.com/rasbt/mlxtend.git#egg=mlxtend && \ pip install altair && \ + pip install pystan && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 9642dfaf0e9b73886578b39f7d32ea719a92fedd Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 15 Nov 2016 17:43:55 +1100 Subject: [PATCH 0047/1660] Update ImageMagick version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 885944d6..150b67a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,8 @@ RUN apt-get update && apt-get install -y build-essential && \ libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data && \ - wget http://www.imagemagick.org/download/ImageMagick-7.0.3-4.tar.gz && \ - tar xzf ImageMagick-7.0.3-4.tar.gz && cd ImageMagick-7.0.3-4 && ./configure && \ + wget http://www.imagemagick.org/download/ImageMagick-7.0.3-6.tar.gz && \ + tar xzf ImageMagick-7.0.3-6.tar.gz && cd ImageMagick-7.0.3-6 && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick-7.0.3* && \ From 3bedbf0b6b90582d77d5609083fc931a96d48d6a Mon Sep 17 00:00:00 2001 From: Tomasz Bartczak Date: Thu, 17 Nov 2016 09:33:34 +0100 Subject: [PATCH 0048/1660] ImageHash library adding https://pypi.python.org/pypi/ImageHash checked that after pip install in the image one can import imagehash --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 150b67a1..93dfe2c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -256,6 +256,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+git://github.com/rasbt/mlxtend.git#egg=mlxtend && \ pip install altair && \ pip install pystan && \ + pip install ImageHash && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 56252157fa8aa7122c9426a286f68f4424f1fe0f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 18 Nov 2016 06:05:35 +1100 Subject: [PATCH 0049/1660] Update ImageMagick version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 150b67a1..5c0f924a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,8 @@ RUN apt-get update && apt-get install -y build-essential && \ libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data && \ - wget http://www.imagemagick.org/download/ImageMagick-7.0.3-6.tar.gz && \ - tar xzf ImageMagick-7.0.3-6.tar.gz && cd ImageMagick-7.0.3-6 && ./configure && \ + wget http://www.imagemagick.org/download/ImageMagick-7.0.3-7.tar.gz && \ + tar xzf ImageMagick-7.0.3-7.tar.gz && cd ImageMagick-7.0.3-7 && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick-7.0.3* && \ From a58fe337ebac88ee9c9742772d0fff937e3e6aef Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 18 Nov 2016 07:16:10 +1100 Subject: [PATCH 0050/1660] Add datashader --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 6f5124bd..5c969c37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -208,6 +208,7 @@ RUN apt-get update && \ pip install line_profiler && \ pip install orderedmultidict && \ pip install smhasher && \ + conda install -y -c bokeh datashader && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From ecb20659bf6626d1beda13858a3df9fd136b8dfb Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Sat, 26 Nov 2016 05:55:55 +1100 Subject: [PATCH 0051/1660] Install Pymc3 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5c969c37..97864539 100644 --- a/Dockerfile +++ b/Dockerfile @@ -258,6 +258,7 @@ RUN pip install --upgrade mpld3 && \ pip install altair && \ pip install pystan && \ pip install ImageHash && \ + pip install git+https://github.com/pymc-devs/pymc3 && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From dcc1dc53dbccca8caa74728cff4fdc2dfc9a51b4 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Sat, 26 Nov 2016 07:13:12 +1100 Subject: [PATCH 0052/1660] Use fukatani's mirror of missing RGF source --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 97864539..b31f5321 100644 --- a/Dockerfile +++ b/Dockerfile @@ -192,7 +192,7 @@ RUN apt-get update && \ # https://github.com/biopython/biopython pip install biopython && \ # Regularized Greedy Forests - cd /usr/local/src && wget http://stat.rutgers.edu/home/tzhang/software/rgf/rgf1.2.zip && \ + cd /usr/local/src && wget https://github.com/fukatani/rgf_python/releases/download/0.2.0/rgf1.2.zip && \ unzip rgf1.2.zip && cd rgf1.2 && make && mv bin/rgf /usr/local/bin && \ cd /usr/local/src && git clone https://github.com/fukatani/rgf_python.git && \ cd rgf_python && sed -i 's/\/opt\/rgf1.2\/bin\/rgf/\/usr\/local\/bin\/rgf/' rgf/rgf.py && \ From fadd693b15b4eb228cbe577490eb18db5b4fee34 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Sat, 26 Nov 2016 07:35:03 +1100 Subject: [PATCH 0053/1660] Update ImageMagick --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b31f5321..db507d0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,8 @@ RUN apt-get update && apt-get install -y build-essential && \ libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data && \ - wget http://www.imagemagick.org/download/ImageMagick-7.0.3-7.tar.gz && \ - tar xzf ImageMagick-7.0.3-7.tar.gz && cd ImageMagick-7.0.3-7 && ./configure && \ + wget http://www.imagemagick.org/download/ImageMagick-7.0.3-8.tar.gz && \ + tar xzf ImageMagick-7.0.3-8.tar.gz && cd ImageMagick-7.0.3-8 && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick-7.0.3* && \ From 4fdcb60e3927f12fa5a585b3bf6ac702735e218a Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 29 Nov 2016 12:34:29 +1100 Subject: [PATCH 0054/1660] Add libpq-dev prereq for ibis-framework --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index db507d0a..e401cea5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -137,6 +137,8 @@ RUN apt-get update && \ cd opendeep && python setup.py develop && \ # sasl is apparently an ibis dependency apt-get -y install libsasl2-dev && \ + # ...as is psycopg2 + apt-get install -y libpq-dev && \ pip install ibis-framework && \ # Cartopy plus dependencies yes | conda install proj4 && \ From 9e21703690a3020462767e0c4eb64e6b9c0f6f63 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 29 Nov 2016 16:59:27 +1100 Subject: [PATCH 0055/1660] Add Boruta --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index e401cea5..69217ac8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -211,6 +211,12 @@ RUN apt-get update && \ pip install orderedmultidict && \ pip install smhasher && \ conda install -y -c bokeh datashader && \ + # Boruta (python implementation) + cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ + cd boruta_py && + # boruta_py's dependencies want to uninstall the nightlies of sklearn etc + sed -i "s/'numpy==1.10.4',//" setup.py && sed -i "s/'scikit-learn==0.17.1',//" setup.py && \ + sed -i "s/'scipy==0.17.0'//" setup.py && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From f62cecae0b706c45947bc371519afabcf2f9b358 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 29 Nov 2016 19:16:25 +1100 Subject: [PATCH 0056/1660] Add missing line continuator --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 69217ac8..34db9927 100644 --- a/Dockerfile +++ b/Dockerfile @@ -213,7 +213,7 @@ RUN apt-get update && \ conda install -y -c bokeh datashader && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ - cd boruta_py && + cd boruta_py && \ # boruta_py's dependencies want to uninstall the nightlies of sklearn etc sed -i "s/'numpy==1.10.4',//" setup.py && sed -i "s/'scikit-learn==0.17.1',//" setup.py && \ sed -i "s/'scipy==0.17.0'//" setup.py && python setup.py install && \ From 39b71a1d61cc4f504e13838e8c0e9bcb9b77f719 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 30 Nov 2016 06:09:23 +1100 Subject: [PATCH 0057/1660] Switch to mbernico's updated version of Boruta --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 34db9927..595cc7bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -212,11 +212,8 @@ RUN apt-get update && \ pip install smhasher && \ conda install -y -c bokeh datashader && \ # Boruta (python implementation) - cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ - cd boruta_py && \ - # boruta_py's dependencies want to uninstall the nightlies of sklearn etc - sed -i "s/'numpy==1.10.4',//" setup.py && sed -i "s/'scikit-learn==0.17.1',//" setup.py && \ - sed -i "s/'scipy==0.17.0'//" setup.py && python setup.py install && \ + cd /usr/local/src && git clone https://github.com/mbernico/boruta_py.git && \ + cd boruta_py && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 48bce074b55978dad5742a494bdcb40d77485f4d Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 30 Nov 2016 07:06:22 +1100 Subject: [PATCH 0058/1660] Revert to master branch of Boruta --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 595cc7bf..a15809c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -212,7 +212,7 @@ RUN apt-get update && \ pip install smhasher && \ conda install -y -c bokeh datashader && \ # Boruta (python implementation) - cd /usr/local/src && git clone https://github.com/mbernico/boruta_py.git && \ + cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ From b1e8d0df17342447ed723613f7386bbcb403b61c Mon Sep 17 00:00:00 2001 From: gidler Date: Thu, 1 Dec 2016 19:23:03 -0700 Subject: [PATCH 0059/1660] Add keras-rl package --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index a15809c6..cc2484cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,10 @@ RUN apt-get install -y libfreetype6-dev && \ cd /usr/local/src && mkdir keras && cd keras && \ git clone --depth 1 https://github.com/fchollet/keras.git && \ cd keras && python setup.py install && \ + #keras-rl + cd /usr/local/src && mkdir keras-rl && cd keras-rl && \ + git clone --depth 1 https://github.com/matthiasplappert/keras-rl.git && \ + cd keras-rl && python setup.py install && \ #neon cd /usr/local/src && \ git clone --depth 1 https://github.com/NervanaSystems/neon.git && \ From acd91ee55b6e3552c0fb521a9c9624c8775a59a0 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 2 Dec 2016 13:37:51 +1100 Subject: [PATCH 0060/1660] Upgrade TensorFlow to 0.12 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a15809c6..19f60a6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ RUN apt-get update && apt-get install -y build-essential && \ cd /usr/local/src && \ # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery - wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl && \ - pip install tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl && \ + wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.0rc0-cp35-cp35m-linux_x86_64.whl && \ + pip install tensorflow-0.12.0rc0-cp35-cp35m-linux_x86_64.whl && \ # Vowpal Rabbit apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ From 9174981d991e5ae233cb1efa4f9588d7630cd6fd Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 2 Dec 2016 15:28:44 +1100 Subject: [PATCH 0061/1660] Update TF version in cleanup --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 204a837c..d58b126e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN apt-get update && apt-get install -y build-essential && \ cp /usr/local/lib/python3.5/site-packages/cv2.cpython-35m-x86_64-linux-gnu.so /opt/conda/lib/python3.5/site-packages/ && \ # Clean up install cruft rm -rf /usr/local/src/opencv && \ - rm /usr/local/src/tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl && \ + rm /usr/local/src/tensorflow-*.whl && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean From c1b8fbed6d439b23ca543d57a4015f9b3981efad Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 2 Dec 2016 17:45:25 +1100 Subject: [PATCH 0062/1660] Temporarily remove cvxopt until suitesparse dependency is fixed --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d58b126e..fcc1cc57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -209,7 +209,9 @@ RUN apt-get update && \ cd /usr/local/src && git clone https://github.com/scikit-learn-contrib/imbalanced-learn.git && \ cd imbalanced-learn && python setup.py install && \ # Convex Optimization library - pip install cvxopt && \ + # Latest version fails to install, see https://github.com/cvxopt/cvxopt/issues/77 + # and https://github.com/cvxopt/cvxopt/issues/80 + # pip install cvxopt && \ # Profiling and other utilities pip install line_profiler && \ pip install orderedmultidict && \ From 92abd8a1f6313ee053e60edc17fb17919bdc8638 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 5 Dec 2016 07:52:09 +1100 Subject: [PATCH 0063/1660] Remove explicit ImageMagick version pin --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fcc1cc57..3efb6f68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,8 @@ RUN apt-get update && apt-get install -y build-essential && \ libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data && \ - wget http://www.imagemagick.org/download/ImageMagick-7.0.3-8.tar.gz && \ - tar xzf ImageMagick-7.0.3-8.tar.gz && cd ImageMagick-7.0.3-8 && ./configure && \ + wget http://www.imagemagick.org/download/ImageMagick.tar.gz && \ + tar xzf ImageMagick*.tar.gz && cd `ls -d ImageMagick-*` && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick-7.0.3* && \ From e0e0a168a8b3c118696709e8f7f6a63a8e199686 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 13 Dec 2016 12:15:48 +1100 Subject: [PATCH 0064/1660] Tidy ImageMagick cleanup, add verbose diagnostics --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3efb6f68..72b28ddc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,10 +24,10 @@ RUN apt-get update && apt-get install -y build-essential && \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data && \ wget http://www.imagemagick.org/download/ImageMagick.tar.gz && \ - tar xzf ImageMagick*.tar.gz && cd `ls -d ImageMagick-*` && ./configure && \ + tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files - cd ../ && rm -rf ImageMagick-7.0.3* && \ + cd ../ && rm -rf ImageMagick* && \ apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \ apt-get -y install cmake && \ From 7ed610afcfb35ded438125bceb53f16bdf26767f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 13 Dec 2016 13:27:50 +1100 Subject: [PATCH 0065/1660] Change path before building ImageMagick --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 72b28ddc..d698710b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y build-essential && \ libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data && \ + cd /usr/local/src && \ wget http://www.imagemagick.org/download/ImageMagick.tar.gz && \ tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ make -j $(nproc) && make install && \ From 99318f63c5bf9e07c09818f48a3656822693fac3 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 13 Dec 2016 14:31:52 +1100 Subject: [PATCH 0066/1660] Limit build to 2 cores --- build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build b/build index d2720b3f..31a702f7 100755 --- a/build +++ b/build @@ -4,10 +4,10 @@ set -e # Default behavior is to do everything from scratch. # The --use-cache option is useful if you're iterating on a broken build. if [[ "$1" == "--use-cache" ]]; then - docker build --rm -t kaggle/python-build . + docker build --rm --cpu-period=100000 --cpu-quota=200000 -t kaggle/python-build . else docker pull continuumio/anaconda3:latest - docker build --rm --no-cache -t kaggle/python-build . + docker build --rm --no-cache --cpu-period=100000 --cpu-quota=200000 -t kaggle/python-build . fi From d4d0b203613517911f6cecf433d297d1b6492a34 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 13 Dec 2016 22:08:53 +1100 Subject: [PATCH 0067/1660] Add LightGBM --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d698710b..5c329dea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -221,6 +221,8 @@ RUN apt-get update && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ + cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ + cd LightGBM && mkdir build && cd build && cmake .. && make -j $(nproc) && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 885e55fb1f565a91613595e5226eeedce789536e Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 14 Dec 2016 06:03:59 +1100 Subject: [PATCH 0068/1660] Install LightGBM python wrapper --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5c329dea..f3b5d117 100644 --- a/Dockerfile +++ b/Dockerfile @@ -223,6 +223,7 @@ RUN apt-get update && \ cd boruta_py && python setup.py install && \ cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ cd LightGBM && mkdir build && cd build && cmake .. && make -j $(nproc) && \ + cd python-package && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From c3ad4129fa9f2d677770895cb2572f33e67afb7c Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 14 Dec 2016 06:56:59 +1100 Subject: [PATCH 0069/1660] Remove malfunctioning framenet_v15 download --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f3b5d117..f8d58d49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,7 +104,7 @@ RUN apt-get install -y libfreetype6-dev && \ averaged_perceptron_tagger basque_grammars biocreative_ppi bllip_wsj_no_aux \ book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \ comparative_sentences comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ -europarl_raw floresta framenet_v15 gazetteers genesis gutenberg hmm_treebank_pos_tagger \ +europarl_raw floresta gazetteers genesis gutenberg hmm_treebank_pos_tagger \ ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho machado \ masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ mte_teip5 names nps_chat omw opinion_lexicon paradigms \ From 0975663fffde4090a894cae9551ab62b7112658b Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 14 Dec 2016 09:21:06 +1100 Subject: [PATCH 0070/1660] Correct path for LightGBM python package --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f8d58d49..d7d22e30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -223,7 +223,7 @@ RUN apt-get update && \ cd boruta_py && python setup.py install && \ cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ cd LightGBM && mkdir build && cd build && cmake .. && make -j $(nproc) && \ - cd python-package && python setup.py install && \ + cd /usr/local/src/LightGBM/python-package && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 5d2c709b12c30c49d3f27b70cc051fbc69de7af5 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 16 Dec 2016 06:58:21 +1100 Subject: [PATCH 0071/1660] Add pyeconometrics (thanks ljx0520) --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d7d22e30..42a1f605 100644 --- a/Dockerfile +++ b/Dockerfile @@ -224,6 +224,8 @@ RUN apt-get update && \ cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ cd LightGBM && mkdir build && cd build && cmake .. && make -j $(nproc) && \ cd /usr/local/src/LightGBM/python-package && python setup.py install && \ + cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ + cd pyeconometrics && python setup.py install # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 5385a91a6742e73c60cfeea1e9d9c262388d34da Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 16 Dec 2016 09:46:08 +1100 Subject: [PATCH 0072/1660] Circumvent intermittent apt-get failures --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 42a1f605..88161c76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ FROM continuumio/anaconda3:latest -RUN apt-get update && apt-get install -y build-essential && \ + # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, + # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 +RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \ + apt-get update && apt-get install -y build-essential && \ cd /usr/local/src && \ # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery From 2711acf53d220cfd547374096cc9e02c4def84b6 Mon Sep 17 00:00:00 2001 From: wjpjet Date: Fri, 16 Dec 2016 01:16:34 -0500 Subject: [PATCH 0073/1660] Added missing line continuation --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 88161c76..1a61ca8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -228,7 +228,7 @@ RUN apt-get update && \ cd LightGBM && mkdir build && cd build && cmake .. && make -j $(nproc) && \ cd /usr/local/src/LightGBM/python-package && python setup.py install && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ - cd pyeconometrics && python setup.py install + cd pyeconometrics && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 2525608dd92e4a341654fea3971825e8e1e05ca2 Mon Sep 17 00:00:00 2001 From: Iceman121 Date: Sat, 17 Dec 2016 09:05:01 +0800 Subject: [PATCH 0074/1660] Update Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 1a61ca8a..05aa953a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -278,6 +278,10 @@ RUN pip install --upgrade mpld3 && \ pip install altair && \ pip install pystan && \ pip install ImageHash && \ + + #Installation of Imputation module + pip install fancyimput && \ + pip install git+https://github.com/pymc-devs/pymc3 && \ ##### ^^^^ Add new contributions above here # clean up pip cache From d684910b00877fd91fcd0899c79b28f047ae0814 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 19 Dec 2016 22:30:25 -0500 Subject: [PATCH 0075/1660] pip install tifffile and descartes --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1a61ca8a..a0354170 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & pip install tensorflow-0.12.0rc0-cp35-cp35m-linux_x86_64.whl && \ # Vowpal Rabbit apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ - cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ + cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ pip install vowpalwabbit && \ pip install seaborn python-dateutil dask pytagcloud pyyaml ggplot joblib \ @@ -279,6 +279,8 @@ RUN pip install --upgrade mpld3 && \ pip install pystan && \ pip install ImageHash && \ pip install git+https://github.com/pymc-devs/pymc3 && \ + pip install tifffile && \ + pip install descartes && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From e4df3c682d499629fe9654eb007e8bd43a28dd7e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 20 Dec 2016 16:03:53 -0500 Subject: [PATCH 0076/1660] tacked on the geojson module as well --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a0354170..c9ab8c73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -281,6 +281,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/pymc-devs/pymc3 && \ pip install tifffile && \ pip install descartes && \ + pip install geojson && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 05e5089c6dd32697cf787b732e826b454695ac1f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 21 Dec 2016 09:21:37 +1100 Subject: [PATCH 0077/1660] Correct installation of Fancyimpute --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c7d4464c..fb28f1d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -278,10 +278,7 @@ RUN pip install --upgrade mpld3 && \ pip install altair && \ pip install pystan && \ pip install ImageHash && \ - - #Installation of Imputation module - pip install fancyimput && \ - + pip install git+https://github.com/hammerlab/fancyimpute.git && \ pip install git+https://github.com/pymc-devs/pymc3 && \ pip install tifffile && \ pip install descartes && \ From 24229d33ff88f5c6f0c409ffef21526ab856e3dc Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 22 Dec 2016 14:01:53 +1100 Subject: [PATCH 0078/1660] Add pysal (thanks ljx0520) --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index fb28f1d0..ff6ee401 100644 --- a/Dockerfile +++ b/Dockerfile @@ -283,6 +283,7 @@ RUN pip install --upgrade mpld3 && \ pip install tifffile && \ pip install descartes && \ pip install geojson && \ + pip install pysal && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From a80b1dc8efa12a1433bc8087990765b7fdcfd76c Mon Sep 17 00:00:00 2001 From: mithrillion Date: Thu, 22 Dec 2016 20:57:09 +1100 Subject: [PATCH 0079/1660] add GDAL --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index ff6ee401..90a0940a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -284,6 +284,7 @@ RUN pip install --upgrade mpld3 && \ pip install descartes && \ pip install geojson && \ pip install pysal && \ + conda install -y gdal && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 2dae20a923ac6fac85c1f56279ed69469dbcb993 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Jan 2017 07:47:15 +1100 Subject: [PATCH 0080/1660] Add PyFlux (thanks matsavage!) --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 90a0940a..817a8b97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -285,6 +285,7 @@ RUN pip install --upgrade mpld3 && \ pip install geojson && \ pip install pysal && \ conda install -y gdal && \ + pip install pyflux && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From e1497d77e1d7aa0ddd4e09b9c4558dad7a913710 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Jan 2017 07:58:43 +1100 Subject: [PATCH 0081/1660] Add terminalplot --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 817a8b97..99cc7eee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -286,6 +286,7 @@ RUN pip install --upgrade mpld3 && \ pip install pysal && \ conda install -y gdal && \ pip install pyflux && \ + pip install terminalplot && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From d82eb65030df39c65ac777143b0f579e4f784fa5 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 10 Jan 2017 07:33:16 +1100 Subject: [PATCH 0082/1660] Add graphviz --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 99cc7eee..87505891 100644 --- a/Dockerfile +++ b/Dockerfile @@ -229,6 +229,7 @@ RUN apt-get update && \ cd /usr/local/src/LightGBM/python-package && python setup.py install && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ + apt-get install -y graphviz && pip install graphviz && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From e1e5bec8700e88b8c463e5bb4c8890fee615a402 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 10 Jan 2017 12:12:52 +1100 Subject: [PATCH 0083/1660] Add geopandas --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 87505891..a763c8dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -230,6 +230,7 @@ RUN apt-get update && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ apt-get install -y graphviz && pip install graphviz && \ + apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From 33e6c1c124ec2901553b044bc804063c57e668de Mon Sep 17 00:00:00 2001 From: wcuk Date: Tue, 10 Jan 2017 15:29:52 -0500 Subject: [PATCH 0084/1660] Adding pydicom --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a763c8dc..572ade96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -289,6 +289,7 @@ RUN pip install --upgrade mpld3 && \ conda install -y gdal && \ pip install pyflux && \ pip install terminalplot && \ + pip install pydicom && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 3c18ded2daf24448e1e5a102a3f48609c67b43c1 Mon Sep 17 00:00:00 2001 From: Megan Risdal Date: Wed, 11 Jan 2017 18:04:45 -0500 Subject: [PATCH 0085/1660] Add wavio to read wav files using numpy arrays --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 572ade96..80483efd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -290,6 +290,7 @@ RUN pip install --upgrade mpld3 && \ pip install pyflux && \ pip install terminalplot && \ pip install pydicom && \ + pip install wavio && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 3601dfaad9ac3cacb472f201e704af0732eaebb7 Mon Sep 17 00:00:00 2001 From: Kevin Mader Date: Fri, 13 Jan 2017 18:41:05 +0100 Subject: [PATCH 0086/1660] Update Dockerfile adding SimpleITK for processing and handling DICOM volumes --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 572ade96..597993ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -290,6 +290,7 @@ RUN pip install --upgrade mpld3 && \ pip install pyflux && \ pip install terminalplot && \ pip install pydicom && \ + conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From e1ea5926291ef6ad4339130a2015669215da447b Mon Sep 17 00:00:00 2001 From: Kevin Mader Date: Fri, 13 Jan 2017 18:50:05 +0100 Subject: [PATCH 0087/1660] Update Dockerfile Adding p7zip-full for extracting files from 7z archives (needed for DSB2017) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 572ade96..34485d68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & libilmbase6 libjasper1 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1-0 libltdl7 libmagickcore-6.q16-2 \ libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ - libxcb-render0 libxcb-shm0 netpbm poppler-data && \ + libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ cd /usr/local/src && \ wget http://www.imagemagick.org/download/ImageMagick.tar.gz && \ tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ From 2902fcac85839ba611db7d43f1da77d63874eb20 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 16 Jan 2017 18:45:45 +1100 Subject: [PATCH 0088/1660] Add Raccoon --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 572ade96..c616b1eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -290,6 +290,7 @@ RUN pip install --upgrade mpld3 && \ pip install pyflux && \ pip install terminalplot && \ pip install pydicom && \ + pip install raccoon && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 58ba5bc4bd7d1984817c3ad08ef8fc63a007834e Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 16 Jan 2017 20:08:28 +1100 Subject: [PATCH 0089/1660] Remove averaged_perceptron_tagger (network error) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2b6eb6b3..c5c71170 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,7 +104,7 @@ RUN apt-get install -y libfreetype6-dev && \ # NLTK Downloader no longer continues smoothly after an error, so we explicitly list # the corpuses that work python -m nltk.downloader -d /usr/share/nltk_data abc alpino \ - averaged_perceptron_tagger basque_grammars biocreative_ppi bllip_wsj_no_aux \ + basque_grammars biocreative_ppi bllip_wsj_no_aux \ book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \ comparative_sentences comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ europarl_raw floresta gazetteers genesis gutenberg hmm_treebank_pos_tagger \ From a2c7af3c736c41919bf44e84704254058f22c824 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 18 Jan 2017 07:18:48 +1100 Subject: [PATCH 0090/1660] Reinstate problematic NLTK datasets --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c5c71170..b5da6422 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,7 +103,7 @@ RUN apt-get install -y libfreetype6-dev && \ mkdir -p /usr/share/nltk_data && \ # NLTK Downloader no longer continues smoothly after an error, so we explicitly list # the corpuses that work - python -m nltk.downloader -d /usr/share/nltk_data abc alpino \ + python -m nltk.downloader -d /usr/share/nltk_data abc alpino averaged_perceptron_tagger \ basque_grammars biocreative_ppi bllip_wsj_no_aux \ book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \ comparative_sentences comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ @@ -111,8 +111,8 @@ europarl_raw floresta gazetteers genesis gutenberg hmm_treebank_pos_tagger \ ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho machado \ masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ mte_teip5 names nps_chat omw opinion_lexicon paradigms \ -pil pl196x ppattach problem_reports product_reviews_1 product_reviews_2 propbank \ -pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor sentence_polarity \ +pil pl196x porter_test ppattach problem_reports product_reviews_1 product_reviews_2 propbank \ +pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor senseval sentence_polarity \ sentiwordnet shakespeare sinica_treebank smultron snowball_data spanish_grammars \ state_union stopwords subjectivity swadesh switchboard tagsets timit toolbox treebank \ twitter_samples udhr2 udhr unicode_samples universal_tagset universal_treebanks_v20 \ From e7052d555503c214d56152770ead0d44ad31f54d Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 19 Jan 2017 08:40:27 +1100 Subject: [PATCH 0091/1660] Reinstate wavio --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b5da6422..f6086c29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -291,6 +291,7 @@ RUN pip install --upgrade mpld3 && \ pip install terminalplot && \ pip install raccoon && \ pip install pydicom && \ + pip install wavio && \ conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ ##### ^^^^ Add new contributions above here # clean up pip cache From 2fafd0f4d5a50ba05e47baf015e6e1f6c1b04ed4 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 20 Jan 2017 06:22:34 +1100 Subject: [PATCH 0092/1660] Add hmmlearn --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f6086c29..eb469597 100644 --- a/Dockerfile +++ b/Dockerfile @@ -293,6 +293,7 @@ RUN pip install --upgrade mpld3 && \ pip install pydicom && \ pip install wavio && \ conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ + pip install -U --user hmmlearn && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 6c0de2f01a20b23f93b38d12a0760742653b7cbe Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 23 Jan 2017 13:25:08 +1100 Subject: [PATCH 0093/1660] Adjust hmmlearn install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eb469597..16b05020 100644 --- a/Dockerfile +++ b/Dockerfile @@ -293,7 +293,7 @@ RUN pip install --upgrade mpld3 && \ pip install pydicom && \ pip install wavio && \ conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ - pip install -U --user hmmlearn && \ + pip install hmmlearn && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 4c8faa6fb5f12cf017813800209f2114b8b07f54 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 27 Jan 2017 15:01:36 +1100 Subject: [PATCH 0094/1660] Update Pandas to most recent version --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 16b05020..2422165a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,8 @@ FROM continuumio/anaconda3:latest # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \ apt-get update && apt-get install -y build-essential && \ + # Newer version of Pandas than comes with continuumio/anaconda3 + conda uninstall -y pandas && pip install pandas && \ cd /usr/local/src && \ # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery From 8613e38914920486c49bdf06777348bfc2aaa00a Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 30 Jan 2017 15:51:11 +1100 Subject: [PATCH 0095/1660] Add PyEarth --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 2422165a..c9743613 100644 --- a/Dockerfile +++ b/Dockerfile @@ -233,6 +233,8 @@ RUN apt-get update && \ cd pyeconometrics && python setup.py install && \ apt-get install -y graphviz && pip install graphviz && \ apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ + cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \ + cd py-earth && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From c522dfd90c82583ef72f828fc4b0035d4d73f307 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 30 Jan 2017 15:55:21 +1100 Subject: [PATCH 0096/1660] Add BayesPy --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c9743613..aa5ab787 100644 --- a/Dockerfile +++ b/Dockerfile @@ -298,6 +298,7 @@ RUN pip install --upgrade mpld3 && \ pip install wavio && \ conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ pip install hmmlearn && \ + pip install bayespy && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 5250ef4625b4ea5660c44aaf3eb3f882c8747778 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 31 Jan 2017 11:04:07 +1100 Subject: [PATCH 0097/1660] Add gplearn --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index aa5ab787..83037ef0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -299,6 +299,7 @@ RUN pip install --upgrade mpld3 && \ conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ pip install hmmlearn && \ pip install bayespy && \ + pip install gplearn && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 65b94fe088f61a16eb8cff9435ea3f9fd9d13a75 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 31 Jan 2017 11:21:13 +1100 Subject: [PATCH 0098/1660] Add PyAstronomy --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 83037ef0..ceca1e95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -300,6 +300,7 @@ RUN pip install --upgrade mpld3 && \ pip install hmmlearn && \ pip install bayespy && \ pip install gplearn && \ + pip install PyAstronomy && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 55e1a165838edc5b4c513296335852bcf504791c Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 1 Feb 2017 15:42:32 +1100 Subject: [PATCH 0099/1660] Cache keras weights for transfer learning --- Dockerfile | 3 +++ cache_keras_weights.py | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 cache_keras_weights.py diff --git a/Dockerfile b/Dockerfile index ceca1e95..896cd3b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM continuumio/anaconda3:latest +ADD cache_keras_weights.py + # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \ @@ -235,6 +237,7 @@ RUN apt-get update && \ apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \ cd py-earth && python setup.py install && \ + cd && python cache_keras_weights.py && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ diff --git a/cache_keras_weights.py b/cache_keras_weights.py new file mode 100644 index 00000000..b20529f9 --- /dev/null +++ b/cache_keras_weights.py @@ -0,0 +1,9 @@ +from keras.applications.resnet50 import ResNet50 +from keras.applications.vgg16 import VGG16 +from keras.applications.vgg19 import VGG19 +from keras.applications.inception_v3 import InceptionV3 + +resnet = ResNet50(weights='imagenet') +vgg16 = VGG16(weights='imagenet') +vgg19 = VGG19(weights='imagenet') +inception = InceptionV3(weights='imagenet') From b4f7fcd84465e1dea1e89e8b115b8dcca796c969 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 1 Feb 2017 15:46:51 +1100 Subject: [PATCH 0100/1660] Correct syntax on Docker add --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 896cd3b6..c314f9ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM continuumio/anaconda3:latest -ADD cache_keras_weights.py +ADD cache_keras_weights.py /usr/local/src/cache_keras_weights.py # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 @@ -123,6 +123,7 @@ twitter_samples udhr2 udhr unicode_samples universal_tagset universal_treebanks_ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Stop-words pip install stop-words && \ + cd /usr/local/src && python cache_keras_weights.py && \ # clean up rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ @@ -237,7 +238,6 @@ RUN apt-get update && \ apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \ cd py-earth && python setup.py install && \ - cd && python cache_keras_weights.py && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From a47f228c5bd1e2cb38d551f92a38977d33d6725a Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 1 Feb 2017 16:42:42 +1100 Subject: [PATCH 0101/1660] Remove broken comparative_sentences corpus --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c314f9ce..555576f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -110,7 +110,7 @@ RUN apt-get install -y libfreetype6-dev && \ python -m nltk.downloader -d /usr/share/nltk_data abc alpino averaged_perceptron_tagger \ basque_grammars biocreative_ppi bllip_wsj_no_aux \ book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \ -comparative_sentences comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ +comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ europarl_raw floresta gazetteers genesis gutenberg hmm_treebank_pos_tagger \ ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho machado \ masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ From 00e0205fde7b5fca2a7345a09e5e4a163fc252f1 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Feb 2017 10:45:33 +1100 Subject: [PATCH 0102/1660] Add pytorch --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 555576f7..8494ff70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -304,6 +304,7 @@ RUN pip install --upgrade mpld3 && \ pip install bayespy && \ pip install gplearn && \ pip install PyAstronomy && \ + conda install pytorch torchvision -c soumith && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 99412cbaf388cee722271fdee366acb394ad456e Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Feb 2017 10:46:09 +1100 Subject: [PATCH 0103/1660] Add squarify --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 8494ff70..c6f946fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -305,6 +305,7 @@ RUN pip install --upgrade mpld3 && \ pip install gplearn && \ pip install PyAstronomy && \ conda install pytorch torchvision -c soumith && \ + pip install squarify && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 8b4fc00e7a5ac1d416d54952cbb6d09ef328a9c3 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Feb 2017 10:50:21 +1100 Subject: [PATCH 0104/1660] Add Xception to keras cache --- cache_keras_weights.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cache_keras_weights.py b/cache_keras_weights.py index b20529f9..a7318768 100644 --- a/cache_keras_weights.py +++ b/cache_keras_weights.py @@ -2,8 +2,10 @@ from keras.applications.vgg16 import VGG16 from keras.applications.vgg19 import VGG19 from keras.applications.inception_v3 import InceptionV3 +from keras.applications.xception import Xception resnet = ResNet50(weights='imagenet') vgg16 = VGG16(weights='imagenet') vgg19 = VGG19(weights='imagenet') inception = InceptionV3(weights='imagenet') +xception = Xception(weights='imagenet') From a31626cfe4549864fd30c0a3b7b4145efc87d803 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Feb 2017 11:28:24 +1100 Subject: [PATCH 0105/1660] Update TensorFlow to 0.12.1 --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6f946fb..bef54856 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,9 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & cd /usr/local/src && \ # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery - wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.0rc0-cp35-cp35m-linux_x86_64.whl && \ - pip install tensorflow-0.12.0rc0-cp35-cp35m-linux_x86_64.whl && \ + wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl + && \ + pip install tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl && \ # Vowpal Rabbit apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ From f69a15fad463fce097fe9e5038a26e3485bf2e8f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Feb 2017 11:36:44 +1100 Subject: [PATCH 0106/1660] Remove stray linebreak --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index bef54856..cc72e60d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,8 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & cd /usr/local/src && \ # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery - wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl - && \ - pip install tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl && \ + wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl && \ + pip install tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl && \ # Vowpal Rabbit apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ From 5b0164aafabbccfc36e91ebf032232c5e6282288 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Feb 2017 16:46:54 +1100 Subject: [PATCH 0107/1660] Cheeky patch to make TF accept Python 3.6 --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cc72e60d..d0a37916 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,8 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 # Why does this work, when `pip install tensorflow` fails? It is a mystery wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl && \ - pip install tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl && \ + mv tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl tensorflow-0.12.1-cp36-cp36m-linux_x86_64.whl && \ + pip install tensorflow-0.12.1-cp36-cp36m-linux_x86_64.whl && \ # Vowpal Rabbit apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ From e055a16635f3ef65307ea24af88edaae001e0f8b Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Feb 2017 18:00:01 +1100 Subject: [PATCH 0108/1660] Update build links for Python 3.6 --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d0a37916..c54af5ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,10 +43,10 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \ cd opencv && \ mkdir build && cd build && \ - cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.5m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.5m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.5m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.5m/ -D BUILD_PNG=TRUE .. && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D BUILD_PNG=TRUE .. && \ make -j $(nproc) && make install && \ - echo "/usr/local/lib/python3.5/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ - cp /usr/local/lib/python3.5/site-packages/cv2.cpython-35m-x86_64-linux-gnu.so /opt/conda/lib/python3.5/site-packages/ && \ + echo "/usr/local/lib/python3.6/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ + cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \ # Clean up install cruft rm -rf /usr/local/src/opencv && \ rm /usr/local/src/tensorflow-*.whl && \ @@ -173,7 +173,7 @@ RUN apt-get update && \ mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ # Stop Matplotlib printing junk to the console on first load - sed -i "s/^.*Matplotlib is building the font cache using fc-list.*$/# Warning removed by Kaggle/g" /opt/conda/lib/python3.5/site-packages/matplotlib/font_manager.py && \ + sed -i "s/^.*Matplotlib is building the font cache using fc-list.*$/# Warning removed by Kaggle/g" /opt/conda/lib/python3.6/site-packages/matplotlib/font_manager.py && \ # Make matplotlib output in Jupyter notebooks display correctly mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \ # h2o From 2627937621f64c29dcb3edc68438d8396887c105 Mon Sep 17 00:00:00 2001 From: Thomas Vanesse Date: Tue, 7 Feb 2017 11:48:28 +0100 Subject: [PATCH 0109/1660] Added a new dependency to `pyexcel-ods` This package provides some tools to read from `.ods` files (LibreOffice/OpenOffice). --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c54af5ca..27842b5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -307,6 +307,7 @@ RUN pip install --upgrade mpld3 && \ pip install PyAstronomy && \ conda install pytorch torchvision -c soumith && \ pip install squarify && \ + pip install pyexcel-ods && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 1b626c2b48fb3956dad87b337551f236f9ef1671 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 9 Feb 2017 10:50:19 +1100 Subject: [PATCH 0110/1660] pip install tensorflow now works, hooray --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index c54af5ca..01be7ea3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,11 +9,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & # Newer version of Pandas than comes with continuumio/anaconda3 conda uninstall -y pandas && pip install pandas && \ cd /usr/local/src && \ - # https://github.com/tensorflow/tensorflow/issues/64#issuecomment-155270240 - # Why does this work, when `pip install tensorflow` fails? It is a mystery - wget https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl && \ - mv tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl tensorflow-0.12.1-cp36-cp36m-linux_x86_64.whl && \ - pip install tensorflow-0.12.1-cp36-cp36m-linux_x86_64.whl && \ + pip install tensorflow && \ # Vowpal Rabbit apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ From 761ef6e7e86ef69db10aeca11ddf7f57ee79c9e3 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 9 Feb 2017 10:51:32 +1100 Subject: [PATCH 0111/1660] Add fuzzywuzzy --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 01be7ea3..26faf9ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -303,6 +303,7 @@ RUN pip install --upgrade mpld3 && \ pip install PyAstronomy && \ conda install pytorch torchvision -c soumith && \ pip install squarify && \ + pip install fuzzywuzzy && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 71cd6c5ee9315235dac6bd7171d06b979aff5967 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 9 Feb 2017 10:52:15 +1100 Subject: [PATCH 0112/1660] Add python-louvain --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 26faf9ec..284f16a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -304,6 +304,7 @@ RUN pip install --upgrade mpld3 && \ conda install pytorch torchvision -c soumith && \ pip install squarify && \ pip install fuzzywuzzy && \ + pip install python-louvain && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 9113a2d88994bb24a19ae62e160b70d1a52ed88f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 9 Feb 2017 11:26:12 +1100 Subject: [PATCH 0113/1660] Tidy up tensorflow tidy-up --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 66f177c1..c0fc4f1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,6 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \ # Clean up install cruft rm -rf /usr/local/src/opencv && \ - rm /usr/local/src/tensorflow-*.whl && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean From 8be5be6bab5aeb7186240727ec4c371f6d42d4a2 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 9 Feb 2017 13:27:27 +1100 Subject: [PATCH 0114/1660] Temporarily remove gdal (conda package not updated for py3.6 yet) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c0fc4f1d..93a4d4d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -289,7 +289,7 @@ RUN pip install --upgrade mpld3 && \ pip install descartes && \ pip install geojson && \ pip install pysal && \ - conda install -y gdal && \ + #conda install -y gdal && \ pip install pyflux && \ pip install terminalplot && \ pip install raccoon && \ From e4b77a9683df98e527b60d76374586a9aabb5045 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 9 Feb 2017 16:19:06 +1100 Subject: [PATCH 0115/1660] Temporarily revert SimpleITK due to Py3.6 conflict --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 93a4d4d9..ed2a4cfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -295,7 +295,7 @@ RUN pip install --upgrade mpld3 && \ pip install raccoon && \ pip install pydicom && \ pip install wavio && \ - conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ + #conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ pip install hmmlearn && \ pip install bayespy && \ pip install gplearn && \ From af97a4aec9fee5a0e2c0dbc39d1d68966cc8e4e2 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 16 Feb 2017 06:27:21 +1100 Subject: [PATCH 0116/1660] Add sklearn-pandas --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index ed2a4cfd..299d2132 100644 --- a/Dockerfile +++ b/Dockerfile @@ -305,6 +305,7 @@ RUN pip install --upgrade mpld3 && \ pip install fuzzywuzzy && \ pip install python-louvain && \ pip install pyexcel-ods && \ + pip install sklearn-pandas && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 31c3dcf899a6df04445c8c4842b564f8c01e3650 Mon Sep 17 00:00:00 2001 From: Michael Boker Date: Sat, 18 Feb 2017 20:08:38 -0500 Subject: [PATCH 0117/1660] Add the stemming library to the docker file Stemming is a useful tool for text preprocessing, and would be nice to have available in the kernels notebook --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 299d2132..ccaaac95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -306,6 +306,7 @@ RUN pip install --upgrade mpld3 && \ pip install python-louvain && \ pip install pyexcel-ods && \ pip install sklearn-pandas && \ + pip install stemming && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 4ce677c7893487b080434133876fe55df8e581e6 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 24 Feb 2017 17:31:00 +1100 Subject: [PATCH 0118/1660] Add holoviews, geoviews --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 299d2132..2f417311 100644 --- a/Dockerfile +++ b/Dockerfile @@ -306,6 +306,7 @@ RUN pip install --upgrade mpld3 && \ pip install python-louvain && \ pip install pyexcel-ods && \ pip install sklearn-pandas && \ + conda install -c conda-forge -c ioam holoviews>=1.6.2 geoviews && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From c69eda758653387a49d476faff0b20572d411787 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 24 Feb 2017 21:34:29 +1100 Subject: [PATCH 0119/1660] Install geoviews from git --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2f417311..bc23c88b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -306,7 +306,8 @@ RUN pip install --upgrade mpld3 && \ pip install python-louvain && \ pip install pyexcel-ods && \ pip install sklearn-pandas && \ - conda install -c conda-forge -c ioam holoviews>=1.6.2 geoviews && \ + conda install -y -c conda-forge -c ioam holoviews && \ + pip install git+https://github.com/ioam/geoviews.git && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 3146c2c764ae4293252fa9a6cc8ab5337b392bab Mon Sep 17 00:00:00 2001 From: Zachary Thomas Date: Mon, 27 Feb 2017 10:04:28 -0800 Subject: [PATCH 0120/1660] Update Dockerfile with Facebook Prophet Add `fbprophet` as package installed in Kaggle Docker image. More info on Prophet, which was open-sourced last week: https://facebookincubator.github.io/prophet/ --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index bc23c88b..6bb0f0e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -306,6 +306,7 @@ RUN pip install --upgrade mpld3 && \ pip install python-louvain && \ pip install pyexcel-ods && \ pip install sklearn-pandas && \ + pip install fbprophet && \ conda install -y -c conda-forge -c ioam holoviews && \ pip install git+https://github.com/ioam/geoviews.git && \ ##### ^^^^ Add new contributions above here From 4d962096e4e8400d4793a9067236207ac0e2adcf Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 10 Mar 2017 07:26:30 +1100 Subject: [PATCH 0121/1660] Add hypertools --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 398c52f2..95499ab8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -310,6 +310,7 @@ RUN pip install --upgrade mpld3 && \ pip install fbprophet && \ conda install -y -c conda-forge -c ioam holoviews && \ pip install git+https://github.com/ioam/geoviews.git && \ + pip install hypertools && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 5f8702f5984a862e80737c69869b73740b8aee0a Mon Sep 17 00:00:00 2001 From: Mathew Savage Date: Mon, 17 Apr 2017 21:54:23 +0100 Subject: [PATCH 0122/1660] Add nxvis, package for visualising networkx graphs --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 95499ab8..f078775d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -311,6 +311,7 @@ RUN pip install --upgrade mpld3 && \ conda install -y -c conda-forge -c ioam holoviews && \ pip install git+https://github.com/ioam/geoviews.git && \ pip install hypertools && \ + pip install nxviz && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 2d25f08d64a4609b9b5dc76abb6d6225d3cd40c2 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 20 Apr 2017 07:06:54 +1000 Subject: [PATCH 0123/1660] Add SimpleITK (py36 conflict now resolved) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95499ab8..f284520b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -295,7 +295,7 @@ RUN pip install --upgrade mpld3 && \ pip install raccoon && \ pip install pydicom && \ pip install wavio && \ - #conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0' && \ + pip install SimpleITK && \ pip install hmmlearn && \ pip install bayespy && \ pip install gplearn && \ From 96a79815d427676596a8f9253dca85ed34bf0f02 Mon Sep 17 00:00:00 2001 From: Kevin Mader Date: Wed, 26 Apr 2017 14:06:04 +0200 Subject: [PATCH 0124/1660] Update Dockerfile adding nibabel for reading nifti images (https://github.com/Kaggle/docker-python/issues/76) --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1f4cf38c..a87fbf81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -312,6 +312,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/ioam/geoviews.git && \ pip install hypertools && \ pip install nxviz && \ + pip install nibabel && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 8c82c6a573f9c8456551148f055df8d51aafab69 Mon Sep 17 00:00:00 2001 From: behzadg Date: Tue, 2 May 2017 14:09:37 -0700 Subject: [PATCH 0125/1660] Add the py_stringsimjoin package The package is well-tested and allows for fast joins based on string similarity. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1f4cf38c..75a0b45d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -312,6 +312,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/ioam/geoviews.git && \ pip install hypertools && \ pip install nxviz && \ + pip install py_stringsimjoin && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From cbaa804b9e922b1df0902909393315e8411c3354 Mon Sep 17 00:00:00 2001 From: Robin Kraft Date: Thu, 4 May 2017 16:48:42 -0700 Subject: [PATCH 0126/1660] add spectral library for multi/hyper spectral images --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1f4cf38c..82420bd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -286,6 +286,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/hammerlab/fancyimpute.git && \ pip install git+https://github.com/pymc-devs/pymc3 && \ pip install tifffile && \ + pip install spectral && \ pip install descartes && \ pip install geojson && \ pip install pysal && \ From d6d5722b360bba099ab7446d4d63d4f82edb3e3c Mon Sep 17 00:00:00 2001 From: Manav Sehgal Date: Sun, 7 May 2017 21:36:41 +0530 Subject: [PATCH 0127/1660] Add speedml package Speedml package (speedml.com) provides quick start wrappers for popular packages in an easy, linear, intuitive machine learning workflow including components for EDA, feature engineering, model evaluation, hyper-parameters tuning, and feature selection. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1f4cf38c..d9ef03ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -312,6 +312,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/ioam/geoviews.git && \ pip install hypertools && \ pip install nxviz && \ + pip install speedml && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From d9220c7bbd4a97cfea0d53b2b44c76f775cb79de Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 8 May 2017 12:43:42 +1000 Subject: [PATCH 0128/1660] Revert Pandas version to anaconda's --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 10bafd04..eb4589cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,8 @@ ADD cache_keras_weights.py /usr/local/src/cache_keras_weights.py # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \ apt-get update && apt-get install -y build-essential && \ - # Newer version of Pandas than comes with continuumio/anaconda3 - conda uninstall -y pandas && pip install pandas && \ + ## Newer version of Pandas than comes with continuumio/anaconda3 + #conda uninstall -y pandas && pip install pandas && \ cd /usr/local/src && \ pip install tensorflow && \ # Vowpal Rabbit From b6157859c918392d4241f70443b55bc97988867c Mon Sep 17 00:00:00 2001 From: Sebastian Flennerhag Date: Mon, 29 May 2017 19:40:18 +0200 Subject: [PATCH 0129/1660] install mlens ML-Ensemble is a library for ensemble learning with a Scikit-learn API. http://mlens.readthedocs.io/en/latest/ --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index eb4589cf..bf7e3ca7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -316,6 +316,7 @@ RUN pip install --upgrade mpld3 && \ pip install py_stringsimjoin && \ pip install speedml && \ pip install nibabel && \ + pip install mlens && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From fd27937274f1f33ed19d6580c26371beb74c3581 Mon Sep 17 00:00:00 2001 From: Sebastian Flennerhag Date: Mon, 29 May 2017 19:43:31 +0200 Subject: [PATCH 0130/1660] added empty line after installs --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index bf7e3ca7..11ca9f1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -317,6 +317,7 @@ RUN pip install --upgrade mpld3 && \ pip install speedml && \ pip install nibabel && \ pip install mlens && \ + ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 2afe260ebc8fd9da549d5b1d2efce0950a020a4c Mon Sep 17 00:00:00 2001 From: "Jiaping(Vincent) Zhang" Date: Thu, 8 Jun 2017 21:19:08 -0700 Subject: [PATCH 0131/1660] Add scikit-multilearn for multilabel ML --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 11ca9f1f..1edb6290 100644 --- a/Dockerfile +++ b/Dockerfile @@ -317,6 +317,7 @@ RUN pip install --upgrade mpld3 && \ pip install speedml && \ pip install nibabel && \ pip install mlens && \ + pip install scikit-multilearn && \ ##### ^^^^ Add new contributions above here # clean up pip cache From 3da21b7dc2acdd1297f6677b375badc56272568d Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 19 Jun 2017 08:55:22 +1000 Subject: [PATCH 0132/1660] Update RGF installation step --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1edb6290..bab2b767 100644 --- a/Dockerfile +++ b/Dockerfile @@ -206,7 +206,7 @@ RUN apt-get update && \ cd /usr/local/src && wget https://github.com/fukatani/rgf_python/releases/download/0.2.0/rgf1.2.zip && \ unzip rgf1.2.zip && cd rgf1.2 && make && mv bin/rgf /usr/local/bin && \ cd /usr/local/src && git clone https://github.com/fukatani/rgf_python.git && \ - cd rgf_python && sed -i 's/\/opt\/rgf1.2\/bin\/rgf/\/usr\/local\/bin\/rgf/' rgf/rgf.py && \ + cd rgf_python && sed -i 's/\/opt\/rgf1.2\/bin\/rgf/\/usr\/local\/bin\/rgf/' rgf/sklearn.py && \ python setup.py install && \ # PUDB, for local debugging convenience pip install pudb && \ From 84ee008b06726baa29a99efeeae4444d9cd2f61b Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 19 Jun 2017 21:32:29 +1000 Subject: [PATCH 0133/1660] Smoke-test jupyter notebook --- build | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build b/build index 31a702f7..c9e7ad59 100755 --- a/build +++ b/build @@ -13,9 +13,14 @@ fi rm -rf /tmp/python-build +docker rm jupyter_test || true mkdir -p /tmp/python-build/tmp mkdir -p /tmp/python-build/devshm mkdir -p /tmp/python-build/working +# Check that Jupyter server can run; if it dies on startup, the `docker kill` command will throw an error +docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root +sleep 3 +docker kill jupyter_test && docker rm jupyter_test docker run --rm -t --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' # The test_build.py script creates a plot called plot1.png; check that it exists From 3a7f931085952934edde15d9232a34671472cbfc Mon Sep 17 00:00:00 2001 From: Guolin Ke Date: Tue, 20 Jun 2017 13:39:04 +0800 Subject: [PATCH 0134/1660] update lightgbm --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bab2b767..063e20c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -226,7 +226,6 @@ RUN apt-get update && \ cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ - cd LightGBM && mkdir build && cd build && cmake .. && make -j $(nproc) && \ cd /usr/local/src/LightGBM/python-package && python setup.py install && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ From f2c9bd1800e6fbf83e1989dfe6391f8b3f97adfa Mon Sep 17 00:00:00 2001 From: Ben Hamner Date: Fri, 23 Jun 2017 14:56:54 -0700 Subject: [PATCH 0135/1660] added cleverhans library --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index bab2b767..68140087 100644 --- a/Dockerfile +++ b/Dockerfile @@ -318,6 +318,7 @@ RUN pip install --upgrade mpld3 && \ pip install nibabel && \ pip install mlens && \ pip install scikit-multilearn && \ + pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ ##### ^^^^ Add new contributions above here # clean up pip cache From 5a54eee98d0c9d935236739847b810b491f57012 Mon Sep 17 00:00:00 2001 From: Sebastien Boving Date: Wed, 28 Jun 2017 17:37:31 -0700 Subject: [PATCH 0136/1660] Patch tornado for .NET websocket compatibility. This adds a generic patching mechanism to `kaggle/python` and a single patch to apply the equivalent of https://github.com/tornadoweb/tornado/pull/2098 so we don't have to wait for the upstream change to trickle down. Resulting image has been verified to prevent kaggle websockets to be broken off every 100s. --- Dockerfile | 7 ++++++- patches/tornado-fix-issue-2097.patch | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 patches/tornado-fix-issue-2097.patch diff --git a/Dockerfile b/Dockerfile index 68140087..38e64403 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM continuumio/anaconda3:latest ADD cache_keras_weights.py /usr/local/src/cache_keras_weights.py +ADD patches/ /tmp/patches/ # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 @@ -319,9 +320,13 @@ RUN pip install --upgrade mpld3 && \ pip install mlens && \ pip install scikit-multilearn && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ - + ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ # Required to display Altair charts in Jupyter notebook jupyter nbextension install --user --py vega + +# Finally, apply any locally defined patches. +RUN /bin/bash -c \ + "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" diff --git a/patches/tornado-fix-issue-2097.patch b/patches/tornado-fix-issue-2097.patch new file mode 100644 index 00000000..f3c7a82f --- /dev/null +++ b/patches/tornado-fix-issue-2097.patch @@ -0,0 +1,27 @@ +diff -Naur /base/opt/conda/pkgs/tornado-4.5.1-py36_0/lib/python3.6/site-packages/tornado/http1connection.py /patched/opt.new/conda/lib/python3.6/site-packages/tornado/http1connection.py +--- /base/opt/conda/lib/python3.6/site-packages/tornado/http1connection.py 2017-04-16 23:47:57.000000000 +0000 ++++ /patched/opt.new/conda/lib/python3.6/site-packages/tornado/http1connection.py 2017-06-28 21:18:59.902402220 +0000 +@@ -349,6 +349,11 @@ + # self._request_start_line.version or + # start_line.version? + self._request_start_line.version == 'HTTP/1.1' and ++ # 1xx, 204 and 304 responses have no body (not even a zero-length ++ # body), and so should not have either Content-Length or ++ # Transfer-Encoding headers. ++ start_line.code not in (204, 304) and ++ (start_line.code < 100 or start_line.code >= 200) and + # 304 responses have no body (not even a zero-length body), and so + # should not have either Content-Length or Transfer-Encoding. + # headers. +diff -Naur /base/opt/conda/lib/python3.6/site-packages/tornado/web.py /patched/opt.new/conda/lib/python3.6/site-packages/tornado/web.py +--- /base/opt/conda/lib/python3.6/site-packages/tornado/web.py 2017-06-28 21:22:56.505186743 +0000 ++++ /patched/opt.new/conda/lib/python3.6/site-packages/tornado/web.py 2017-06-28 21:22:48.421228255 +0000 +@@ -977,7 +977,8 @@ + if self._status_code in (204, 304): + assert not self._write_buffer, "Cannot send body with %s" % self._status_code + self._clear_headers_for_304() +- elif "Content-Length" not in self._headers: ++ elif ("Content-Length" not in self._headers and ++ (self._status_code < 100 or self._status_code >= 200)): + content_length = sum(len(part) for part in self._write_buffer) + self.set_header("Content-Length", content_length) From 2aa8769630ffe355dec7dc799ebd91db300db271 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 7 Jul 2017 06:19:27 +1000 Subject: [PATCH 0137/1660] Add leven --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 38e64403..68f204d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -320,7 +320,7 @@ RUN pip install --upgrade mpld3 && \ pip install mlens && \ pip install scikit-multilearn && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ - + pip install leven && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 02b9980c9e4d0250a9477e82c65edca3acb5de05 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 13 Jul 2017 11:21:20 +1000 Subject: [PATCH 0138/1660] Use ImageMagick mirror --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 68f204d7..22c932f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ cd /usr/local/src && \ - wget http://www.imagemagick.org/download/ImageMagick.tar.gz && \ + wget http://transloadit.imagemagick.org/download/ImageMagick.tar.gz && \ tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files From f8bac42b84bcbd63109b2bdc728f9723fac0b83e Mon Sep 17 00:00:00 2001 From: pershinmr Date: Tue, 18 Jul 2017 15:26:27 +0300 Subject: [PATCH 0139/1660] added catboost package --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 22c932f3..d916c39a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -321,6 +321,7 @@ RUN pip install --upgrade mpld3 && \ pip install scikit-multilearn && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ pip install leven && \ + pip install catboost && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From bddf36243766c0f05bc1c4d05fdd89ae639be1c4 Mon Sep 17 00:00:00 2001 From: Axel Date: Fri, 21 Jul 2017 16:02:24 +0200 Subject: [PATCH 0140/1660] Add "MLBox" python package Hello ! I would like to add a package I have developed for auto-ML. I have already tested it on several Kaggle competitions and it works well :) --> github: https://github.com/AxeldeRomblay/MLBox --> my Kaggle profile: https://www.kaggle.com/axelderomblay --> PyPI : https://pypi.python.org/pypi/mlbox Then I would like to write a kernel tuto on this competition: https://www.kaggle.com/c/nyc-taxi-trip-duration/discussion Thank you ! --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d916c39a..cf67b15b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -322,6 +322,8 @@ RUN pip install --upgrade mpld3 && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ pip install leven && \ pip install catboost && \ + pip install mlbox && \ + ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 8c827a560f2360a0b4c334f48695997bfd7cb18f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 24 Jul 2017 08:23:02 +1000 Subject: [PATCH 0141/1660] Get lightgbm from pip --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a969296..9b3aa985 100644 --- a/Dockerfile +++ b/Dockerfile @@ -226,8 +226,7 @@ RUN apt-get update && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ - cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ - cd /usr/local/src/LightGBM/python-package && python setup.py install && \ + pip install lightgbm && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ apt-get install -y graphviz && pip install graphviz && \ From 6311b18bb288f47d0352a4f4b086aa45caa1c6ae Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 24 Jul 2017 10:57:28 +1000 Subject: [PATCH 0142/1660] Revert LightGBM to source install --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9b3aa985..cf67b15b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -226,7 +226,9 @@ RUN apt-get update && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ - pip install lightgbm && \ + cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ + cd LightGBM && mkdir build && cd build && cmake .. && make -j $(nproc) && \ + cd /usr/local/src/LightGBM/python-package && python setup.py install && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ apt-get install -y graphviz && pip install graphviz && \ From df23b3b7ec82544a36017819fe4a0b1d29c509bc Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 24 Jul 2017 11:42:20 +1000 Subject: [PATCH 0143/1660] Upgrade cmake, reinstate fast LightGBM install --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cf67b15b..f7aaa3c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -226,8 +226,10 @@ RUN apt-get update && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ + # LightGBM requires a newer cmake than apt-get currently provides + cd /usr/local/src && wget https://cmake.org/files/v3.8/cmake-3.8.2.tar.gz && tar xzf cmake-3.8.2.tar.gz && \ + cd cmake-3.8.2 && ./bootstrap && make -j2 && make install && \ cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ - cd LightGBM && mkdir build && cd build && cmake .. && make -j $(nproc) && \ cd /usr/local/src/LightGBM/python-package && python setup.py install && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ From fb4c8ba14a87bbdba3aeedb54c17a015c1ac26ff Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Jul 2017 06:12:50 +1000 Subject: [PATCH 0144/1660] Move LightGBM to pip --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f7aaa3c6..9b3aa985 100644 --- a/Dockerfile +++ b/Dockerfile @@ -226,11 +226,7 @@ RUN apt-get update && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ - # LightGBM requires a newer cmake than apt-get currently provides - cd /usr/local/src && wget https://cmake.org/files/v3.8/cmake-3.8.2.tar.gz && tar xzf cmake-3.8.2.tar.gz && \ - cd cmake-3.8.2 && ./bootstrap && make -j2 && make install && \ - cd /usr/local/src && git clone --recursive --depth 1 https://github.com/Microsoft/LightGBM && \ - cd /usr/local/src/LightGBM/python-package && python setup.py install && \ + pip install lightgbm && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ apt-get install -y graphviz && pip install graphviz && \ From dd424cb0af3fe9fd34fd055a30a3968554b8185e Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Jul 2017 09:28:29 +1000 Subject: [PATCH 0145/1660] Remove keras weights cache from Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9b3aa985..caa2d4b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM continuumio/anaconda3:latest -ADD cache_keras_weights.py /usr/local/src/cache_keras_weights.py ADD patches/ /tmp/patches/ # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, From 2eea1c1e50fa367876d92a1f764aea34097fb1c7 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Jul 2017 09:29:59 +1000 Subject: [PATCH 0146/1660] Rm keras weights cache (plz use datasets instead) --- cache_keras_weights.py | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 cache_keras_weights.py diff --git a/cache_keras_weights.py b/cache_keras_weights.py deleted file mode 100644 index a7318768..00000000 --- a/cache_keras_weights.py +++ /dev/null @@ -1,11 +0,0 @@ -from keras.applications.resnet50 import ResNet50 -from keras.applications.vgg16 import VGG16 -from keras.applications.vgg19 import VGG19 -from keras.applications.inception_v3 import InceptionV3 -from keras.applications.xception import Xception - -resnet = ResNet50(weights='imagenet') -vgg16 = VGG16(weights='imagenet') -vgg19 = VGG19(weights='imagenet') -inception = InceptionV3(weights='imagenet') -xception = Xception(weights='imagenet') From 75920d83bc55e884e23ec40e29cfb6425603f9c0 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Jul 2017 09:30:57 +1000 Subject: [PATCH 0147/1660] Move LightGBM earlier in the install queue --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index caa2d4b5..4d249021 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,6 +60,7 @@ RUN apt-get install -y libfreetype6-dev && \ cd /usr/local/src && mkdir xgboost && cd xgboost && \ git clone --depth 1 --recursive https://github.com/dmlc/xgboost.git && cd xgboost && \ make && cd python-package && python setup.py install && \ + pip install lightgbm && \ #lasagne cd /usr/local/src && mkdir Lasagne && cd Lasagne && \ git clone --depth 1 https://github.com/Lasagne/Lasagne.git && cd Lasagne && \ @@ -225,7 +226,6 @@ RUN apt-get update && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ - pip install lightgbm && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ apt-get install -y graphviz && pip install graphviz && \ From 7aee8a923c65b5d11c32f722efd2115cc32f9b67 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Jul 2017 10:12:18 +1000 Subject: [PATCH 0148/1660] Did you spot my deliberate mistake? --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4d249021..d4145bfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,7 +120,6 @@ twitter_samples udhr2 udhr unicode_samples universal_tagset universal_treebanks_ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Stop-words pip install stop-words && \ - cd /usr/local/src && python cache_keras_weights.py && \ # clean up rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From cff3c239c832bdf328a184e20a5fb6f38e42fac9 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Jul 2017 12:04:57 +1000 Subject: [PATCH 0149/1660] Temporarily revert mlbox --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d4145bfe..b869069c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -318,7 +318,6 @@ RUN pip install --upgrade mpld3 && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ pip install leven && \ pip install catboost && \ - pip install mlbox && \ ##### ^^^^ Add new contributions above here # clean up pip cache From 89ff2a6208b6735c1e716884772c8293623a1930 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 25 Jul 2017 22:12:37 +1000 Subject: [PATCH 0150/1660] Reinstate mlbox --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b869069c..d4145bfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -318,6 +318,7 @@ RUN pip install --upgrade mpld3 && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ pip install leven && \ pip install catboost && \ + pip install mlbox && \ ##### ^^^^ Add new contributions above here # clean up pip cache From ccf85f9dfd669596143daecb31aac41a54d90cb8 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 28 Jul 2017 06:52:25 +1000 Subject: [PATCH 0151/1660] Patch Geohash import error --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d4145bfe..8cdc1597 100644 --- a/Dockerfile +++ b/Dockerfile @@ -250,6 +250,8 @@ RUN pip install --upgrade mpld3 && \ pip install arrow && \ pip install sexmachine && \ pip install Geohash && \ + # https://github.com/vinsci/geohash/issues/4 + sed -i -- 's/geohash/.geohash/g' /opt/conda/lib/python3.6/site-packages/Geohash/__init__.py && \ pip install deap && \ pip install tpot && \ pip install haversine && \ From d066d0b773489288ef3be661ecfb3d33c622509f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 31 Jul 2017 11:40:27 +1000 Subject: [PATCH 0152/1660] Reinstate spaCy --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8cdc1597..8daf13f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & pip install vowpalwabbit && \ pip install seaborn python-dateutil dask pytagcloud pyyaml ggplot joblib \ husl geopy ml_metrics mne pyshp gensim && \ - #python -m spacy.en.download && \ + pip install -U spacy && python -m spacy download en && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ From 975314bbc8bc868faff76fe945a0d7226579e914 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 31 Jul 2017 12:35:48 +1000 Subject: [PATCH 0153/1660] Install spaCy using Conda --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8daf13f6..55abeeb8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & pip install vowpalwabbit && \ pip install seaborn python-dateutil dask pytagcloud pyyaml ggplot joblib \ husl geopy ml_metrics mne pyshp gensim && \ - pip install -U spacy && python -m spacy download en && \ + conda install -y -c conda-forge spacy && python -m spacy download en && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ From f2880e5423f9a203e5684f2f125dd0fc499056cc Mon Sep 17 00:00:00 2001 From: Atlas7 Date: Fri, 4 Aug 2017 22:23:53 +0100 Subject: [PATCH 0154/1660] add paramnb --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 55abeeb8..efff18c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -321,7 +321,8 @@ RUN pip install --upgrade mpld3 && \ pip install leven && \ pip install catboost && \ pip install mlbox && \ - + pip install paramnb && \ + ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 5181c37c5494aa657e5ebe647b66925a469bd233 Mon Sep 17 00:00:00 2001 From: Anirudh Kamalapuram Muralidhar Date: Sun, 6 Aug 2017 17:19:01 -0500 Subject: [PATCH 0155/1660] folium package for map visualization! --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 55abeeb8..8969e107 100644 --- a/Dockerfile +++ b/Dockerfile @@ -321,6 +321,7 @@ RUN pip install --upgrade mpld3 && \ pip install leven && \ pip install catboost && \ pip install mlbox && \ + pip install folium && \ ##### ^^^^ Add new contributions above here # clean up pip cache From ee97eb75ab6137d3e5add2578aeabed838314b43 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 9 Aug 2017 12:55:42 +1000 Subject: [PATCH 0156/1660] Upgrade numpy to patch MKL link error --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d0e6a8ac..684ff36f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,8 @@ ADD patches/ /tmp/patches/ # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \ apt-get update && apt-get install -y build-essential && \ - ## Newer version of Pandas than comes with continuumio/anaconda3 - #conda uninstall -y pandas && pip install pandas && \ + # https://github.com/ContinuumIO/anaconda-issues/issues/720 + conda install -f -y numpy && \ cd /usr/local/src && \ pip install tensorflow && \ # Vowpal Rabbit From d57cae736d26b3687bfd9fe764ed353a955714b9 Mon Sep 17 00:00:00 2001 From: Anirudh Kamalapuram Muralidhar Date: Tue, 8 Aug 2017 22:06:19 -0500 Subject: [PATCH 0157/1660] dipy package DIPY is a python package used for brain image analysis such as MRI data (2D and 3D). Would be really useful for many competitions. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 684ff36f..8b72f5ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -323,6 +323,7 @@ RUN pip install --upgrade mpld3 && \ pip install mlbox && \ pip install paramnb && \ pip install folium && \ + pip install dipy && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 026def616ddd79d122bcab3285ecc7c51bbde7d1 Mon Sep 17 00:00:00 2001 From: Reiichiro Nakano Date: Thu, 17 Aug 2017 01:54:01 +0800 Subject: [PATCH 0158/1660] Add scikit-plot --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 684ff36f..df91592e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -323,6 +323,7 @@ RUN pip install --upgrade mpld3 && \ pip install mlbox && \ pip install paramnb && \ pip install folium && \ + pip install scikit-plot && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From ee83a579ea243d7e6badec8153d40c171d730755 Mon Sep 17 00:00:00 2001 From: dvaida Date: Fri, 25 Aug 2017 10:39:20 +0300 Subject: [PATCH 0159/1660] Add hallucinate - a lightweight library for quick ML experimentation --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 91704604..377b04c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -325,6 +325,7 @@ RUN pip install --upgrade mpld3 && \ pip install folium && \ pip install scikit-plot && \ pip install dipy && \ + pip install git+https://github.com/dvaida/hallucinate.git && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 62a83b17a47b68bc40a298f0d741f956762fc257 Mon Sep 17 00:00:00 2001 From: sinia <4824267+sinia@users.noreply.github.com> Date: Wed, 30 Aug 2017 21:55:56 +0100 Subject: [PATCH 0160/1660] add plotnine package plotnine is an implementation of a grammar of graphics in Python, it is based on ggplot2 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 91704604..422d988d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -325,6 +325,7 @@ RUN pip install --upgrade mpld3 && \ pip install folium && \ pip install scikit-plot && \ pip install dipy && \ + pip install plotnine && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 3c38997760ba5f401537311392dc38c133ae93ef Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Thu, 31 Aug 2017 11:41:54 -0700 Subject: [PATCH 0161/1660] Add nbonvert template extension to support cell hidding --- Dockerfile | 1 + nbconvert-extensions.tpl | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 nbconvert-extensions.tpl diff --git a/Dockerfile b/Dockerfile index 91704604..91d01e33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM continuumio/anaconda3:latest ADD patches/ /tmp/patches/ +ADD nbconvert-extensions.tpl /tmp/nbconvert-extensions.tpl # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 diff --git a/nbconvert-extensions.tpl b/nbconvert-extensions.tpl new file mode 100644 index 00000000..b3de090d --- /dev/null +++ b/nbconvert-extensions.tpl @@ -0,0 +1,12 @@ +{# +Jinja template to inject notebook cell metadata to enhance generated HTML output +All cell metadata starting with '_kg_' will be included with its value ({key}-{value}) +as a class in the cell's DIV container +#} + +{% extends 'full.tpl'%} +{% block any_cell %} +
+ {{ super() }} +
+{% endblock any_cell %} \ No newline at end of file From c29e9dee7a4042726560cd6a8a38892aae23b15a Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Thu, 31 Aug 2017 11:58:54 -0700 Subject: [PATCH 0162/1660] Update template location per PR --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 91d01e33..7f14d838 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM continuumio/anaconda3:latest ADD patches/ /tmp/patches/ -ADD nbconvert-extensions.tpl /tmp/nbconvert-extensions.tpl +ADD nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 From fbce5af13180c7f61a67c7da326d171308db0fbd Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 4 Sep 2017 06:53:06 +1000 Subject: [PATCH 0163/1660] Install lightfm and fastfm --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7f14d838..4f2603bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -322,6 +322,8 @@ RUN pip install --upgrade mpld3 && \ pip install leven && \ pip install catboost && \ pip install mlbox && \ + pip install fastFM && \ + pip install lightfm && \ pip install paramnb && \ pip install folium && \ pip install scikit-plot && \ From 0cb6792a9733bdb9cffcd822e671f568e674a0cd Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 4 Sep 2017 07:03:01 +1000 Subject: [PATCH 0164/1660] Move numpy reinstallation to end of sequence --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d2455633..3e5af9a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,6 @@ ADD nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \ apt-get update && apt-get install -y build-essential && \ - # https://github.com/ContinuumIO/anaconda-issues/issues/720 - conda install -f -y numpy && \ cd /usr/local/src && \ pip install tensorflow && \ # Vowpal Rabbit @@ -333,6 +331,8 @@ RUN pip install --upgrade mpld3 && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ + # https://github.com/ContinuumIO/anaconda-issues/issues/720 + conda install -f -y numpy && \ # Required to display Altair charts in Jupyter notebook jupyter nbextension install --user --py vega From 5a5f8449bbe6d971fb08e34959140e573016d0a7 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 5 Sep 2017 06:17:53 +1000 Subject: [PATCH 0165/1660] Temporarily revert nxviz --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3e5af9a4..4623d87e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -310,7 +310,8 @@ RUN pip install --upgrade mpld3 && \ conda install -y -c conda-forge -c ioam holoviews && \ pip install git+https://github.com/ioam/geoviews.git && \ pip install hypertools && \ - pip install nxviz && \ + # Nxviz has been causing an installation issue by trying unsuccessfully to remove setuptools. + #pip install nxviz && \ pip install py_stringsimjoin && \ pip install speedml && \ pip install nibabel && \ From f546dde02b0e837af3496820737299c937fd91a9 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 11 Sep 2017 12:57:28 +1000 Subject: [PATCH 0166/1660] Install surprise --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 4623d87e..2b5a2ea3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -329,6 +329,7 @@ RUN pip install --upgrade mpld3 && \ pip install dipy && \ pip install plotnine && \ pip install git+https://github.com/dvaida/hallucinate.git && \ + pip install scikit-surprise && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From f4a51b1abafaad36fae8b3fae80207a00e141875 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 12 Sep 2017 00:13:46 +1000 Subject: [PATCH 0167/1660] Add pymongo --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2b5a2ea3..bf34f50e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -330,6 +330,7 @@ RUN pip install --upgrade mpld3 && \ pip install plotnine && \ pip install git+https://github.com/dvaida/hallucinate.git && \ pip install scikit-surprise && \ + pip install pymongo && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 5b7596056463059c93d25b893cc9db53bc86d4dd Mon Sep 17 00:00:00 2001 From: vfdev <2459423+vfdev-5@users.noreply.github.com> Date: Thu, 14 Sep 2017 12:58:15 +0200 Subject: [PATCH 0168/1660] Add Google PAIR-code Facets [Google PAIR-code Facets](https://github.com/pair-code/facets) --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index bf34f50e..0179fcb4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -331,6 +331,9 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/dvaida/hallucinate.git && \ pip install scikit-surprise && \ pip install pymongo && \ + # Add google PAIR-code Facets + cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ + export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From fcdbfe38ef8fb4f7c39d07576ce910911483ca70 Mon Sep 17 00:00:00 2001 From: vfdev-5 Date: Thu, 14 Sep 2017 13:17:18 +0200 Subject: [PATCH 0169/1660] * Update setuptools to the latest and add tensorpack framework --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index bf34f50e..1727024d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -331,6 +331,8 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/dvaida/hallucinate.git && \ pip install scikit-surprise && \ pip install pymongo && \ + # Update setuptools and add tensorpack + pip install --upgrade --ignore-installed setuptools && pip install --no-cache-dir git+git://github.com/ppwwyyxx/tensorpack && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 568663a17a0630099ba3f8605b4e2ee2b0958fe6 Mon Sep 17 00:00:00 2001 From: patientpremier Date: Thu, 14 Sep 2017 09:07:12 -0500 Subject: [PATCH 0170/1660] Add ethnicolor package for predicting ethnicity --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index bf34f50e..af4d4a8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -331,6 +331,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/dvaida/hallucinate.git && \ pip install scikit-surprise && \ pip install pymongo && \ + pip install ethnicolr && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 1d3cd531cc9b50bb73b307cae027c00afe2563f3 Mon Sep 17 00:00:00 2001 From: Aleksey Bilogur Date: Thu, 21 Sep 2017 20:14:39 -0400 Subject: [PATCH 0171/1660] Add geoplot --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0e4b2094..ba26e7f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -331,6 +331,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/dvaida/hallucinate.git && \ pip install scikit-surprise && \ pip install pymongo && \ + pip install geoplot && \ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ From 5fbd6e9819e1e263434e5c3878dd093ea4f46384 Mon Sep 17 00:00:00 2001 From: Ioannis Kourouklides Date: Thu, 28 Sep 2017 00:53:53 +0300 Subject: [PATCH 0172/1660] edward library http://edwardlib.org/ --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0e4b2094..5edc2f10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -331,6 +331,7 @@ RUN pip install --upgrade mpld3 && \ pip install git+https://github.com/dvaida/hallucinate.git && \ pip install scikit-surprise && \ pip install pymongo && \ + pip install edward # http://edwardlib.org/ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ From ea1554ee951f0840002957b5d0b07e417b0e7dc3 Mon Sep 17 00:00:00 2001 From: Fabiana Boldrin Date: Thu, 28 Sep 2017 14:03:29 -0300 Subject: [PATCH 0173/1660] Update test_build.py add some visualizations tools --- test_build.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_build.py b/test_build.py index 6e88137d..4d5b8713 100644 --- a/test_build.py +++ b/test_build.py @@ -32,6 +32,11 @@ from mpl_toolkits.basemap import Basemap print("Basemap ok") +import plotly.plotly as py +import plotly.graph_objs as go + +from ggplot import * + import theano print("Theano ok") @@ -52,4 +57,4 @@ import cv2 img = cv2.imread('plot1.png',0) -print("OpenCV ok") \ No newline at end of file +print("OpenCV ok") From d6831beb49edf2929fb9992992302134efc8f9d5 Mon Sep 17 00:00:00 2001 From: Fabiana Boldrin Date: Thu, 28 Sep 2017 14:04:12 -0300 Subject: [PATCH 0174/1660] Update test_build.py --- test_build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_build.py b/test_build.py index 4d5b8713..6fad5aa5 100644 --- a/test_build.py +++ b/test_build.py @@ -34,8 +34,10 @@ import plotly.plotly as py import plotly.graph_objs as go +print("plotly ok") from ggplot import * +print("ggplot ok") import theano print("Theano ok") From ab9a67ea3c53750b3c93f3ce6d753b69420b13fb Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Date: Sun, 8 Oct 2017 12:17:36 +0200 Subject: [PATCH 0175/1660] adding pycountry and iso3166 Two packages for handling countries' names, subdivisions, languages, currencies, and their different ISO codes. It would be quite useful, used in association with other packages for text analysis and map visualization. Use case example: extracting geographical named entities from a text, converting them to their ISO codes, and plotting them on a map. https://pypi.python.org/pypi/pycountry https://pypi.python.org/pypi/iso3166 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0e4b2094..340a39c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -337,6 +337,7 @@ RUN pip install --upgrade mpld3 && \ pip install --no-dependencies ethnicolr && \ # Update setuptools and add tensorpack pip install --upgrade --ignore-installed setuptools && pip install --no-cache-dir git+git://github.com/ppwwyyxx/tensorpack && \ + pip install pycountry && pip install iso3166 && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 84445f8de828d08ad57751e92e7d09f9163a5cc7 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Oct 2017 11:21:32 +1100 Subject: [PATCH 0176/1660] Install Essentia --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index da1da3c0..5196f820 100644 --- a/Dockerfile +++ b/Dockerfile @@ -230,6 +230,9 @@ RUN apt-get update && \ apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \ cd py-earth && python setup.py install && \ + cd /usr/local/src && git clone https://github.com/MTG/essentia.git && cd essentia && \ + ./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ + ./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From a8aeddb46cc62ec351c75928299a466e28efada1 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Oct 2017 11:21:59 +1100 Subject: [PATCH 0177/1660] Add Essentia to build test --- test_build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_build.py b/test_build.py index 6fad5aa5..beb34ae2 100644 --- a/test_build.py +++ b/test_build.py @@ -60,3 +60,6 @@ import cv2 img = cv2.imread('plot1.png',0) print("OpenCV ok") + +import essentia +print("Essentia ok") From 8fa45e6176f9a032750fe46772afd476aa891de6 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Oct 2017 12:53:29 +1100 Subject: [PATCH 0178/1660] Switch wordcloud to pypi --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5196f820..582cc198 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,7 +52,7 @@ RUN apt-get install -y libfreetype6-dev && \ # textblob pip install textblob && \ #word cloud - pip install git+git://github.com/amueller/word_cloud.git && \ + pip install wordcloud && \ #igraph pip install python-igraph && \ #xgboost From fddad81e9aee6774ae13ac3cbbe697440358d0c4 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Oct 2017 13:15:23 +1100 Subject: [PATCH 0179/1660] Did I say pypi? I meant conda --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 582cc198..fa673535 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,7 +52,7 @@ RUN apt-get install -y libfreetype6-dev && \ # textblob pip install textblob && \ #word cloud - pip install wordcloud && \ + conda install -y -c https://conda.anaconda.org/amueller wordcloud && \ #igraph pip install python-igraph && \ #xgboost From a96133e1c8e5345312996b4d6b017007fa0739a3 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Oct 2017 13:27:43 +1100 Subject: [PATCH 0180/1660] Move igraph to conda --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fa673535..b060ec09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,7 @@ RUN apt-get install -y libfreetype6-dev && \ #word cloud conda install -y -c https://conda.anaconda.org/amueller wordcloud && \ #igraph - pip install python-igraph && \ + conda install -y -c conda-forge python-igraph && \ #xgboost cd /usr/local/src && mkdir xgboost && cd xgboost && \ git clone --depth 1 --recursive https://github.com/dmlc/xgboost.git && cd xgboost && \ From caeb6e355ee0e2e4a67e64d42b66664ec0e336e8 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Oct 2017 13:34:22 +1100 Subject: [PATCH 0181/1660] Remove tsne (use sklearn.manifold.TSNE) --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index b060ec09..433c412f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,12 +87,8 @@ RUN apt-get install -y libfreetype6-dev && \ printf "[global]\nbase_compiledir = /tmp/.theano\n" > /.theanorc && \ cd /usr/local/src && git clone --depth 1 https://github.com/pybrain/pybrain && \ cd pybrain && python setup.py install && \ - # Base ATLAS plus tSNE + # Base ATLAS apt-get install -y libatlas-base-dev && \ - # NOTE: we provide the tsne package, but sklearn.manifold.TSNE now does the same - # job - cd /usr/local/src && git clone --depth 1 https://github.com/danielfrg/tsne.git && \ - cd tsne && python setup.py install && \ cd /usr/local/src && git clone --depth 1 https://github.com/ztane/python-Levenshtein && \ cd python-Levenshtein && python setup.py install && \ cd /usr/local/src && git clone --depth 1 https://github.com/arogozhnikov/hep_ml.git && \ From 36a6e7711d2494a81fb26ab2220cd2a1c147f79e Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Oct 2017 13:51:40 +1100 Subject: [PATCH 0182/1660] Fix conda compiler stompage --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 433c412f..2df894b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,9 @@ ADD nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \ apt-get update && apt-get install -y build-essential && \ - cd /usr/local/src && \ + # https://stackoverflow.com/a/46498173 + conda update -y conda && conda update -y python && \ + # Tensorflow pip install tensorflow && \ # Vowpal Rabbit apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ From 184f1df9c0d9d6bb855392b7cae7b1b651552ffb Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 9 Oct 2017 20:38:11 +1100 Subject: [PATCH 0183/1660] Temporarily revert essentia --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2df894b2..48c1badb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -228,9 +228,9 @@ RUN apt-get update && \ apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \ cd py-earth && python setup.py install && \ - cd /usr/local/src && git clone https://github.com/MTG/essentia.git && cd essentia && \ - ./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ - ./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ + #cd /usr/local/src && git clone https://github.com/MTG/essentia.git && cd essentia && \ + #./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ + #./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From a9a0ff9d435ab26cf7d975e93531675d80e2c38e Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 11 Oct 2017 12:26:07 +1100 Subject: [PATCH 0184/1660] Avoid pip install of mlbox --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 48c1badb..39714a0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -321,7 +321,7 @@ RUN pip install --upgrade mpld3 && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ pip install leven && \ pip install catboost && \ - pip install mlbox && \ + cd /usr/local/src && git clone --depth=1 https://github.com/AxeldeRomblay/MLBox && cd MLBox && python setup.py install && \ pip install fastFM && \ pip install lightfm && \ pip install paramnb && \ From f23c4e26f6e9b6d4328f370f8d2fbcd7840c659b Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 11 Oct 2017 13:56:30 +1100 Subject: [PATCH 0185/1660] Fix mlbox subdirectory path --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 39714a0b..4f7a7ae6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -321,7 +321,7 @@ RUN pip install --upgrade mpld3 && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ pip install leven && \ pip install catboost && \ - cd /usr/local/src && git clone --depth=1 https://github.com/AxeldeRomblay/MLBox && cd MLBox && python setup.py install && \ + cd /usr/local/src && git clone --depth=1 https://github.com/AxeldeRomblay/MLBox && cd MLBox/python-package && python setup.py install && \ pip install fastFM && \ pip install lightfm && \ pip install paramnb && \ From e9947b66e096c55eccd53f4e3cd01d7c6c3e1561 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 12 Oct 2017 07:54:20 +1100 Subject: [PATCH 0186/1660] Update matplotlib backend --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4f7a7ae6..8ca631f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -160,7 +160,7 @@ RUN apt-get update && \ make && cd python && python setup.py install && \ # set backend for matplotlib to Agg matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ - sed -i 's/^backend : Qt5Agg/backend : Agg/' $matplotlibrc_path && \ + sed -i 's/^backend : qt5agg/backend : Agg/' $matplotlibrc_path && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ From c3339d4b8870147ec753bf515d97df76c227d2f7 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 12 Oct 2017 10:09:07 +1100 Subject: [PATCH 0187/1660] Revert gcc monkeypatch --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ca631f8..e243eb58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -124,11 +124,6 @@ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Install OpenCV-3 with Python support RUN apt-get update && \ - # Anaconda's build of gcc is way out of date; monkey-patch some linking problems that affect - # packages like xgboost and Shapely - rm /opt/conda/lib/libstdc++* && rm /opt/conda/lib/libgomp.* && \ - ln -s /usr/lib/x86_64-linux-gnu/libgomp.so.1 /opt/conda/lib/libgomp.so.1 && \ - ln -s /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6 && \ # Libgeos, for mapping libraries apt-get -y install libgeos-dev && \ # pyshp and pyproj are now external dependencies of Basemap From 98e5d590f700b57bb162d0902e00f5b30cae6a3b Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 16 Oct 2017 19:44:31 +1100 Subject: [PATCH 0188/1660] Revert to Anaconda 4.4 https://github.com/ContinuumIO/docker-images/pull/69#issuecomment-336213874 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e243eb58..05d00e74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM continuumio/anaconda3:latest +FROM continuumio/anaconda3:4.4.0 ADD patches/ /tmp/patches/ ADD nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl From 247be71349f0cf6592bb4a9ef678659cdeba05a7 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 16 Oct 2017 19:51:05 +1100 Subject: [PATCH 0189/1660] Fix capitalization of matplotlib backend hotfix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 05d00e74..72a1b09f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -155,7 +155,7 @@ RUN apt-get update && \ make && cd python && python setup.py install && \ # set backend for matplotlib to Agg matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ - sed -i 's/^backend : qt5agg/backend : Agg/' $matplotlibrc_path && \ + sed -i 's/^backend : qt5agg/backend : agg/' $matplotlibrc_path && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ From da45845e611b3237cbc5be46f3502213866b3065 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 16 Oct 2017 21:34:22 +1100 Subject: [PATCH 0190/1660] Redo matplotlib backend --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 72a1b09f..7fd31672 100644 --- a/Dockerfile +++ b/Dockerfile @@ -155,7 +155,7 @@ RUN apt-get update && \ make && cd python && python setup.py install && \ # set backend for matplotlib to Agg matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ - sed -i 's/^backend : qt5agg/backend : agg/' $matplotlibrc_path && \ + sed -i 's/^backend : Qt5Agg/backend : agg/' $matplotlibrc_path && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ From 2b487bfe85d87dda6e7e89fd8f950d708db9fe52 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 19 Oct 2017 17:00:59 +1100 Subject: [PATCH 0191/1660] Revert Essentia from build test --- test_build.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test_build.py b/test_build.py index beb34ae2..6fad5aa5 100644 --- a/test_build.py +++ b/test_build.py @@ -60,6 +60,3 @@ import cv2 img = cv2.imread('plot1.png',0) print("OpenCV ok") - -import essentia -print("Essentia ok") From 3cb836d6046d63b03b7639f7d9c1256fe6658ae0 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 19 Oct 2017 17:01:30 +1100 Subject: [PATCH 0192/1660] Add LinearRegression to build test --- test_build.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_build.py b/test_build.py index 6fad5aa5..239c63ec 100644 --- a/test_build.py +++ b/test_build.py @@ -19,6 +19,13 @@ rf1.fit(X,y) print("sklearn RandomForestClassifier: ok") +from sklearn.linear_model import LinearRegression +boston = datasets.load_boston() +X, y = boston.data, boston.target +lr1 = LinearRegression() +lr1.fit(X,y) +print("sklearn LinearRegression: ok") + from xgboost import XGBClassifier xgb1 = XGBClassifier(n_estimators=3) xgb1.fit(X[0:70],y[0:70]) From a44c68cd1ca62cf2eb0dea4812a337be9dd352a7 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 19 Oct 2017 17:02:24 +1100 Subject: [PATCH 0193/1660] Add skimage to build test https://github.com/Kaggle/docker-python/issues/103 --- test_build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_build.py b/test_build.py index 239c63ec..8e97f4ae 100644 --- a/test_build.py +++ b/test_build.py @@ -67,3 +67,6 @@ import cv2 img = cv2.imread('plot1.png',0) print("OpenCV ok") + +from skimage.io import imread +print("skimage ok") From d32fda296ef8538764728309e5bdd16f61481a38 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 19 Oct 2017 18:01:11 +1100 Subject: [PATCH 0194/1660] Temporarily pin numpy to 1.13 --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7fd31672..8342a2e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -344,6 +344,10 @@ RUN pip install --upgrade mpld3 && \ # Required to display Altair charts in Jupyter notebook jupyter nbextension install --user --py vega +# Temporary fix for Anaconda linking error +# See https://github.com/Kaggle/docker-python/issues/103 +RUN conda install -f -y numpy==1.13.0 + # Finally, apply any locally defined patches. RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" From b9dfe3ee59e120e61f3b911ce9b116164a02abf5 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 23 Oct 2017 10:03:25 +1100 Subject: [PATCH 0195/1660] Update PYTHONPATH for Facets --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 8342a2e9..90069897 100644 --- a/Dockerfile +++ b/Dockerfile @@ -344,6 +344,9 @@ RUN pip install --upgrade mpld3 && \ # Required to display Altair charts in Jupyter notebook jupyter nbextension install --user --py vega +# For Facets +ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ + # Temporary fix for Anaconda linking error # See https://github.com/Kaggle/docker-python/issues/103 RUN conda install -f -y numpy==1.13.0 From 524be7cc9a7aaf71598f7db987187b7bb16e6755 Mon Sep 17 00:00:00 2001 From: Delaunay Arnaud Date: Thu, 26 Oct 2017 08:46:56 +0200 Subject: [PATCH 0196/1660] Add eli5 ELI5 is a cool package which helps to have some insights about "black boxes classifiers". It might be really interesting to use in some Kaggle Kernels ! https://eli5.readthedocs.io/en/latest/overview.html --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 90069897..2318bf54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -329,6 +329,7 @@ RUN pip install --upgrade mpld3 && \ pip install pymongo && \ pip install edward && \ pip install geoplot && \ + pip install eli5 && \ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ From 45940ac717bc61ea43a9dba83149d345493fdf2f Mon Sep 17 00:00:00 2001 From: spohngellert-o Date: Wed, 1 Nov 2017 13:10:28 -0400 Subject: [PATCH 0197/1660] Added pydash to docker file. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 90069897..6e1f29c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -329,6 +329,7 @@ RUN pip install --upgrade mpld3 && \ pip install pymongo && \ pip install edward && \ pip install geoplot && \ + pip install pydash && \ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ From 28e71b9bdd32d2a1c1066fd579acf43f3d42ef69 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Nov 2017 10:05:42 +1100 Subject: [PATCH 0198/1660] Fix Dask version --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 314f350a..57f2e239 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ pip install vowpalwabbit && \ - pip install seaborn python-dateutil dask pytagcloud pyyaml ggplot joblib \ + pip install seaborn python-dateutil pytagcloud pyyaml ggplot joblib \ husl geopy ml_metrics mne pyshp gensim && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source @@ -351,7 +351,9 @@ ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ # Temporary fix for Anaconda linking error # See https://github.com/Kaggle/docker-python/issues/103 -RUN conda install -f -y numpy==1.13.0 +RUN conda install -f -y numpy==1.13.0 && \ + # Temporary patch for Dask getting downgraded, which breaks Keras + pip install --upgrade dask # Finally, apply any locally defined patches. RUN /bin/bash -c \ From ddc72f6be124d9daa7583b14aa7ad4393975ae40 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 6 Nov 2017 10:10:15 +1100 Subject: [PATCH 0199/1660] Restore Dask to first image layer --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 57f2e239..a7ca25b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ pip install vowpalwabbit && \ - pip install seaborn python-dateutil pytagcloud pyyaml ggplot joblib \ + pip install seaborn python-dateutil dask pytagcloud pyyaml ggplot joblib \ husl geopy ml_metrics mne pyshp gensim && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source From ab5bc69d279ae8de663edf61d652378ea80e8d30 Mon Sep 17 00:00:00 2001 From: Felix Last Date: Wed, 8 Nov 2017 16:02:45 +0000 Subject: [PATCH 0200/1660] Add kmeans-smote --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a7ca25b5..bbc41f8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -330,6 +330,7 @@ RUN pip install --upgrade mpld3 && \ pip install edward && \ pip install geoplot && \ pip install eli5 && \ + pip install kmeans-smote && \ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ From 62dd9dfe811f1689c09d5fdf146b25865414af50 Mon Sep 17 00:00:00 2001 From: balaca Date: Fri, 10 Nov 2017 12:02:30 +0530 Subject: [PATCH 0201/1660] Add implicit Fast Python Collaborative Filtering for Implicit Datasets This project provides fast Python implementations of the algorithms described in the paper Collaborative Filtering for Implicit Feedback Datasets and in Applications of the Conjugate Gradient Method for Implicit Feedback Collaborative Filtering. http://implicit.readthedocs.io/en/latest/index.html --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a7ca25b5..8ab1bb54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -330,6 +330,7 @@ RUN pip install --upgrade mpld3 && \ pip install edward && \ pip install geoplot && \ pip install eli5 && \ + pip install implicit && \ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ From 73c7d5658adcc4ab7ad59418e1cd6d453cf866c4 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 13 Nov 2017 16:20:37 +1100 Subject: [PATCH 0202/1660] Update matplotlibrc backend hack --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index af8965ae..e69bc40a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -155,7 +155,7 @@ RUN apt-get update && \ make && cd python && python setup.py install && \ # set backend for matplotlib to Agg matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ - sed -i 's/^backend : Qt5Agg/backend : agg/' $matplotlibrc_path && \ + sed -i 's/^backend : TkAgg/backend : agg/' $matplotlibrc_path && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ From 1a56b47b216458b4f83582fe2265cffe375c3080 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 13 Nov 2017 18:55:00 +1100 Subject: [PATCH 0203/1660] Move matplotlib fixes to later in build --- Dockerfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index e69bc40a..75a01699 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,16 +153,6 @@ RUN apt-get update && \ cd /usr/local/src/mxnet && cp make/config.mk . && \ sed -i 's/ADD_LDFLAGS =/ADD_LDFLAGS = -lstdc++/' config.mk && \ make && cd python && python setup.py install && \ - # set backend for matplotlib to Agg - matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ - sed -i 's/^backend : TkAgg/backend : agg/' $matplotlibrc_path && \ - # Stop jupyter nbconvert trying to rewrite its folder hierarchy - mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ - mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ - # Stop Matplotlib printing junk to the console on first load - sed -i "s/^.*Matplotlib is building the font cache using fc-list.*$/# Warning removed by Kaggle/g" /opt/conda/lib/python3.6/site-packages/matplotlib/font_manager.py && \ - # Make matplotlib output in Jupyter notebooks display correctly - mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \ # h2o # This requires python-software-properties and Java. apt-get install -y python-software-properties zip && \ @@ -355,7 +345,18 @@ ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ # See https://github.com/Kaggle/docker-python/issues/103 RUN conda install -f -y numpy==1.13.0 && \ # Temporary patch for Dask getting downgraded, which breaks Keras - pip install --upgrade dask + pip install --upgrade dask && \ + # Matplotlib corrections: + # set backend for matplotlib to Agg + matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ + sed -i 's/^backend : TkAgg/backend : agg/' $matplotlibrc_path && \ + # Stop jupyter nbconvert trying to rewrite its folder hierarchy + mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ + mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ + # Stop Matplotlib printing junk to the console on first load + sed -i "s/^.*Matplotlib is building the font cache using fc-list.*$/# Warning removed by Kaggle/g" /opt/conda/lib/python3.6/site-packages/matplotlib/font_manager.py && \ + # Make matplotlib output in Jupyter notebooks display correctly + mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py # Finally, apply any locally defined patches. RUN /bin/bash -c \ From 8cc297ba226de87afc24c8b31c8c3d6fb05709f0 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 14 Nov 2017 11:28:11 +1100 Subject: [PATCH 0204/1660] Fix Jupyter test socket binding --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index c9e7ad59..3f7e3664 100755 --- a/build +++ b/build @@ -18,7 +18,7 @@ mkdir -p /tmp/python-build/tmp mkdir -p /tmp/python-build/devshm mkdir -p /tmp/python-build/working # Check that Jupyter server can run; if it dies on startup, the `docker kill` command will throw an error -docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root +docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root --ip="*" sleep 3 docker kill jupyter_test && docker rm jupyter_test docker run --rm -t --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' From 5e3fe7ba896986ba0fdb91551a6070d981f10af3 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 17 Nov 2017 06:03:53 +1100 Subject: [PATCH 0205/1660] Add kmodes and librosa --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 75a01699..e31e2a11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -330,6 +330,8 @@ RUN pip install --upgrade mpld3 && \ pip install --upgrade --ignore-installed setuptools && pip install --no-cache-dir git+git://github.com/ppwwyyxx/tensorpack && \ pip install pycountry && pip install iso3166 && \ pip install pydash && \ + pip install kmodes && \ + pip install librosa && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 064e9df2410bbddfc8c00fc75d14d853afe9687c Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 17 Nov 2017 06:28:05 +1100 Subject: [PATCH 0206/1660] Add PyTorch audio --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index e31e2a11..b82a3643 100644 --- a/Dockerfile +++ b/Dockerfile @@ -216,6 +216,10 @@ RUN apt-get update && \ #cd /usr/local/src && git clone https://github.com/MTG/essentia.git && cd essentia && \ #./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ #./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ + # PyTorch Audio + apt-get install -y sox libsox-dev libsox-fmt-all && \ + pip install cffi && \ + cd /usr/local/src && git clone https://github.com/pytorch/audio && cd audio && python setup.py install && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From c671751a4f7f48fa67d55c8562ae869633504dd6 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 17 Nov 2017 07:42:08 +1100 Subject: [PATCH 0207/1660] Adjust pytorch installation timing --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b82a3643..209ab407 100644 --- a/Dockerfile +++ b/Dockerfile @@ -216,6 +216,8 @@ RUN apt-get update && \ #cd /usr/local/src && git clone https://github.com/MTG/essentia.git && cd essentia && \ #./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ #./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ + # PyTorch + conda install pytorch torchvision -c soumith && \ # PyTorch Audio apt-get install -y sox libsox-dev libsox-fmt-all && \ pip install cffi && \ @@ -289,7 +291,6 @@ RUN pip install --upgrade mpld3 && \ pip install bayespy && \ pip install gplearn && \ pip install PyAstronomy && \ - conda install pytorch torchvision -c soumith && \ pip install squarify && \ pip install fuzzywuzzy && \ pip install python-louvain && \ From 1b7d06f26a7d022e71831e8b6d0c44b9a9c55913 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 21 Nov 2017 06:02:33 +1100 Subject: [PATCH 0208/1660] Temporarily downgrade Pandas --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 209ab407..62db1854 100644 --- a/Dockerfile +++ b/Dockerfile @@ -353,6 +353,8 @@ ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ RUN conda install -f -y numpy==1.13.0 && \ # Temporary patch for Dask getting downgraded, which breaks Keras pip install --upgrade dask && \ + # Temporary downgrade for Pandas to circumvent https://github.com/pandas-dev/pandas/issues/18186 + pip uninstall -y pandas && pip install pandas==0.20.3 && \ # Matplotlib corrections: # set backend for matplotlib to Agg matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ From f992da79bb7b3a5918607997cf0a5c36da655c2b Mon Sep 17 00:00:00 2001 From: Prince Grover Date: Fri, 24 Nov 2017 02:26:47 -0800 Subject: [PATCH 0209/1660] Update Dockerfile --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 62db1854..fe2ae5e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -370,3 +370,6 @@ RUN conda install -f -y numpy==1.13.0 && \ # Finally, apply any locally defined patches. RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" + +# importing fastai library for dl and ml +RUN pip install fastai From 43ab989f52f756e891535b8cfaff041b3d8bdf7e Mon Sep 17 00:00:00 2001 From: paultimothymooney <30876065+paultimothymooney@users.noreply.github.com> Date: Tue, 28 Nov 2017 10:36:10 -0700 Subject: [PATCH 0210/1660] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 62db1854..1a59b31f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -239,6 +239,7 @@ RUN pip install --upgrade mpld3 && \ pip install mplleaflet && \ pip install gpxpy && \ pip install arrow && \ + pip install vtk && \ pip install sexmachine && \ pip install Geohash && \ # https://github.com/vinsci/geohash/issues/4 From a5c0a628cf539bf0d43fd2370907d9b138b66987 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 29 Nov 2017 12:36:24 +1100 Subject: [PATCH 0211/1660] Add dask-ml --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 62db1854..c98c75ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -326,6 +326,7 @@ RUN pip install --upgrade mpld3 && \ pip install geoplot && \ pip install eli5 && \ pip install implicit && \ + pip install dask-ml[xgboost] && \ pip install kmeans-smote && \ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ From 6c4cac180ced8b0712e3cd3b34adac19f6daaa46 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 30 Nov 2017 11:37:40 +1100 Subject: [PATCH 0212/1660] Switch MXNet to pip install, per docs --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c98c75ba..19902475 100644 --- a/Dockerfile +++ b/Dockerfile @@ -149,10 +149,7 @@ RUN apt-get update && \ cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ cd cartopy && python setup.py install && \ # MXNet - cd /usr/local/src && git clone --recursive https://github.com/dmlc/mxnet && \ - cd /usr/local/src/mxnet && cp make/config.mk . && \ - sed -i 's/ADD_LDFLAGS =/ADD_LDFLAGS = -lstdc++/' config.mk && \ - make && cd python && python setup.py install && \ + pip install mxnet==0.11.0 && \ # h2o # This requires python-software-properties and Java. apt-get install -y python-software-properties zip && \ From ac42b270aa5da6eae52c8309e19d733e1727662d Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 30 Nov 2017 12:57:17 +1100 Subject: [PATCH 0213/1660] Add pandoc prior to deap --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 19902475..6249dee5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -240,6 +240,8 @@ RUN pip install --upgrade mpld3 && \ pip install Geohash && \ # https://github.com/vinsci/geohash/issues/4 sed -i -- 's/geohash/.geohash/g' /opt/conda/lib/python3.6/site-packages/Geohash/__init__.py && \ + # Pandoc is a dependency of deap + apt-get install pandoc && \ pip install deap && \ pip install tpot && \ pip install haversine && \ From 3f43c35e96cf19cc4c221c98c7c8363ca1242093 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 30 Nov 2017 12:58:32 +1100 Subject: [PATCH 0214/1660] Add water-drinking bird --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6249dee5..5d1eb9c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -241,7 +241,7 @@ RUN pip install --upgrade mpld3 && \ # https://github.com/vinsci/geohash/issues/4 sed -i -- 's/geohash/.geohash/g' /opt/conda/lib/python3.6/site-packages/Geohash/__init__.py && \ # Pandoc is a dependency of deap - apt-get install pandoc && \ + apt-get install -y pandoc && \ pip install deap && \ pip install tpot && \ pip install haversine && \ From 369820302c17b0017c0ed69c2c03d39f4f2358aa Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 1 Dec 2017 10:37:11 +1100 Subject: [PATCH 0215/1660] Move pip to user request block --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe2ae5e1..049f3d37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -337,6 +337,7 @@ RUN pip install --upgrade mpld3 && \ pip install pydash && \ pip install kmodes && \ pip install librosa && \ + pip install fastai && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ @@ -370,6 +371,3 @@ RUN conda install -f -y numpy==1.13.0 && \ # Finally, apply any locally defined patches. RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" - -# importing fastai library for dl and ml -RUN pip install fastai From 27f6a18242ca879ad679037deba38d0398fe0289 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Fri, 1 Dec 2017 14:33:10 +0000 Subject: [PATCH 0216/1660] adding the polyglot NLP library --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 49959cfc..2c8ab9e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -330,7 +330,7 @@ RUN pip install --upgrade mpld3 && \ pip install kmeans-smote && \ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ - export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ + export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ pip install --no-dependencies ethnicolr && \ # Update setuptools and add tensorpack pip install --upgrade --ignore-installed setuptools && pip install --no-cache-dir git+git://github.com/ppwwyyxx/tensorpack && \ @@ -339,6 +339,7 @@ RUN pip install --upgrade mpld3 && \ pip install kmodes && \ pip install librosa && \ pip install fastai && \ + pip install polyglot && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From cf2e8966d9b6e758132d5f40c55f7843bfc8af82 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 4 Dec 2017 07:48:48 +1100 Subject: [PATCH 0217/1660] Add mmh3, fbpca, sentencepiece, glmnet --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 49959cfc..2b8cf04c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -339,6 +339,10 @@ RUN pip install --upgrade mpld3 && \ pip install kmodes && \ pip install librosa && \ pip install fastai && \ + pip install mmh3 && \ + pip install fbpca && \ + pip install sentencepiece && \ + pip install glmnet_py && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From cef6c28f01e9aaf808a43ce3ab7cd66cacb2b086 Mon Sep 17 00:00:00 2001 From: emanceau Date: Sun, 3 Dec 2017 22:02:49 +0100 Subject: [PATCH 0218/1660] adding lime package --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 16b0414e..fb332bf0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -344,6 +344,7 @@ RUN pip install --upgrade mpld3 && \ pip install fbpca && \ pip install sentencepiece && \ pip install glmnet_py && \ + pip install lime && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 4a93b11a84651ad1f5a8e083dc9575eb310d267c Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 4 Dec 2017 08:05:02 +1100 Subject: [PATCH 0219/1660] Temporarily remove Vowpal Wabbit pending fix --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16b0414e..05d4df58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,10 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & # Tensorflow pip install tensorflow && \ # Vowpal Rabbit - apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ - cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ - ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ - pip install vowpalwabbit && \ + #apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ + #cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ + #ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ + #pip install vowpalwabbit && \ pip install seaborn python-dateutil dask pytagcloud pyyaml ggplot joblib \ husl geopy ml_metrics mne pyshp gensim && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ From dff6abb230619509d8e14c79b69f0d16f6d60d3c Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 4 Dec 2017 09:06:43 +1100 Subject: [PATCH 0220/1660] Drop MLBox (causing dependency downgrades) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 05d4df58..37cc6f05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -311,7 +311,7 @@ RUN pip install --upgrade mpld3 && \ pip install -e git+http://github.com/tensorflow/cleverhans.git#egg=cleverhans && \ pip install leven && \ pip install catboost && \ - cd /usr/local/src && git clone --depth=1 https://github.com/AxeldeRomblay/MLBox && cd MLBox/python-package && python setup.py install && \ + #cd /usr/local/src && git clone --depth=1 https://github.com/AxeldeRomblay/MLBox && cd MLBox/python-package && python setup.py install && \ pip install fastFM && \ pip install lightfm && \ pip install paramnb && \ From fbb0fbc9a82c876bbda5b4cfd49e7f6f659d3a82 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 6 Dec 2017 05:28:18 +1100 Subject: [PATCH 0221/1660] Move ggplot to github install --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 37cc6f05..aae051ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & #cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ #ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ #pip install vowpalwabbit && \ - pip install seaborn python-dateutil dask pytagcloud pyyaml ggplot joblib \ + pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ husl geopy ml_metrics mne pyshp gensim && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source @@ -219,6 +219,8 @@ RUN apt-get update && \ apt-get install -y sox libsox-dev libsox-fmt-all && \ pip install cffi && \ cd /usr/local/src && git clone https://github.com/pytorch/audio && cd audio && python setup.py install && \ + # ggpy / ggplot + pip install git+https://github.com/yhat/ggplot.git && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ From e417b3a625be6541b8b1b0540ca26bc85fa51833 Mon Sep 17 00:00:00 2001 From: Nicolau Werneck Date: Wed, 6 Dec 2017 21:50:59 +0100 Subject: [PATCH 0222/1660] Annoy Install Annoy, an approximate nearest-neighbor library. Properly tested in my local docker, no extra dependencies necessary... https://github.com/spotify/annoy --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index fdd8291f..7c51e726 100644 --- a/Dockerfile +++ b/Dockerfile @@ -347,6 +347,7 @@ RUN pip install --upgrade mpld3 && \ pip install sentencepiece && \ pip install glmnet_py && \ pip install lime && \ + pip install annoy && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From cf118b03fcf78ae448142208797175d7490ee303 Mon Sep 17 00:00:00 2001 From: Paul Mooney <30876065+paultimothymooney@users.noreply.github.com> Date: Wed, 6 Dec 2017 18:02:30 -0700 Subject: [PATCH 0223/1660] Update Dockerfile - install pronouncing and markovify pip install pronouncing && \ pip install markovify && \ both are useful for natural language processing applications --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index fdd8291f..c9c13947 100644 --- a/Dockerfile +++ b/Dockerfile @@ -239,6 +239,8 @@ RUN pip install --upgrade mpld3 && \ pip install gpxpy && \ pip install arrow && \ pip install vtk && \ + pip install pronouncing && \ + pip install markovify && \ pip install sexmachine && \ pip install Geohash && \ # https://github.com/vinsci/geohash/issues/4 From 9eef34dafab855fb3f812637abe74f87dafe0d96 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 8 Dec 2017 07:21:22 +1100 Subject: [PATCH 0224/1660] Smoother pytorch install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fdd8291f..32dff3c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -214,7 +214,7 @@ RUN apt-get update && \ #./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ #./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ # PyTorch - conda install pytorch torchvision -c soumith && \ + conda install -y pytorch torchvision -c soumith && \ # PyTorch Audio apt-get install -y sox libsox-dev libsox-fmt-all && \ pip install cffi && \ From 3b3251a3a2df477a9d2618c98f4051f57bb1aeb3 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 8 Dec 2017 07:23:01 +1100 Subject: [PATCH 0225/1660] Add memory_profiler --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 32dff3c8..f22a04f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -347,6 +347,7 @@ RUN pip install --upgrade mpld3 && \ pip install sentencepiece && \ pip install glmnet_py && \ pip install lime && \ + pip install memory_profiler && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 891e494743d247c8e939cba2abfa4fec9545234a Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 8 Dec 2017 07:25:00 +1100 Subject: [PATCH 0226/1660] Add Spacy en_core_web_lg --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f22a04f8..7a16a022 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ husl geopy ml_metrics mne pyshp gensim && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ + python -m spacy download en_core_web_lg && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ From 336c3108eb5e84a08fc20e76f21f80c2e6795617 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 8 Dec 2017 07:27:04 +1100 Subject: [PATCH 0227/1660] Add pyfasttext --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7a16a022..7e027977 100644 --- a/Dockerfile +++ b/Dockerfile @@ -349,6 +349,7 @@ RUN pip install --upgrade mpld3 && \ pip install glmnet_py && \ pip install lime && \ pip install memory_profiler && \ + pip install pyfasttext && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 6ef6474239688c2693d2a9bc5bdb0be2ffcf5184 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 11 Dec 2017 12:52:28 +1100 Subject: [PATCH 0228/1660] Add category_encoders --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index adc8ac93..79b7ef8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -353,6 +353,7 @@ RUN pip install --upgrade mpld3 && \ pip install memory_profiler && \ pip install pyfasttext && \ pip install annoy && \ + pip install category_encoders && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From b5be4f5702773c919313d6487b3b6051ad5f8c67 Mon Sep 17 00:00:00 2001 From: Paul Mooney <30876065+paultimothymooney@users.noreply.github.com> Date: Mon, 11 Dec 2017 11:02:01 -0700 Subject: [PATCH 0229/1660] pip install nibabel pip install nibabel the nibabel library supports reading and writing of neuroimaging data formats. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 79b7ef8b..5d0db4ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -240,6 +240,7 @@ RUN pip install --upgrade mpld3 && \ pip install gpxpy && \ pip install arrow && \ pip install vtk && \ + pip install nibabel && \ pip install pronouncing && \ pip install markovify && \ pip install sexmachine && \ From cbf5775ece3ad7b9e444c468ae0ca2c1286063d6 Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Tue, 12 Dec 2017 14:04:48 -0800 Subject: [PATCH 0230/1660] Add BigQuery client library and proxy overrides --- Dockerfile | 7 +++++- .../nbconvert-extensions.tpl | 0 patches/sitecustomize.py | 23 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) rename nbconvert-extensions.tpl => patches/nbconvert-extensions.tpl (100%) create mode 100644 patches/sitecustomize.py diff --git a/Dockerfile b/Dockerfile index 7e027977..4c8983f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM continuumio/anaconda3:4.4.0 ADD patches/ /tmp/patches/ -ADD nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl +ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 @@ -350,6 +350,7 @@ RUN pip install --upgrade mpld3 && \ pip install lime && \ pip install memory_profiler && \ pip install pyfasttext && \ + pip install google-cloud-bigquery && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ @@ -380,6 +381,10 @@ RUN conda install -f -y numpy==1.13.0 && \ # Make matplotlib output in Jupyter notebooks display correctly mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py +# Add BigQuery client proxy settings +ENV PYTHONUSERBASE "/root/.local" +ADD patches/sitecustomize.py /root/.local/lib/python3.6/site-packages/sitecustomize.py + # Finally, apply any locally defined patches. RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" diff --git a/nbconvert-extensions.tpl b/patches/nbconvert-extensions.tpl similarity index 100% rename from nbconvert-extensions.tpl rename to patches/nbconvert-extensions.tpl diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py new file mode 100644 index 00000000..4701a549 --- /dev/null +++ b/patches/sitecustomize.py @@ -0,0 +1,23 @@ +# Monkey patches BigQuery client creation to use data proxy +import os + +kaggle_proxy_data = os.getenv("KAGGLE_PROXY_DATA") +if kaggle_proxy_data: + from google.auth import credentials + from google.cloud import bigquery + from google.cloud.bigquery._http import Connection + from google.cloud import _http + from google.cloud.bigquery import __version__ + + Connection.API_BASE_URL = os.getenv("KAGGLE_PROXY_URL") + + _CLIENT_INFO = _http.CLIENT_INFO_TEMPLATE.format(__version__) + Connection._EXTRA_HEADERS = { + _http.CLIENT_INFO_HEADER: _CLIENT_INFO, + 'X-KAGGLE-PROXY-DATA': kaggle_proxy_data + } + + bq_client = bigquery.Client + bigquery.Client = lambda: bq_client( + credentials=credentials.AnonymousCredentials(), + project="bigquery-public-data") \ No newline at end of file From f2a43611a1c7ce1d24bb089e397312bff9d5fe0e Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Tue, 12 Dec 2017 14:48:01 -0800 Subject: [PATCH 0231/1660] Implement CR feedback --- patches/sitecustomize.py | 17 ++++++----------- test_build.py | 4 ++++ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 4701a549..1500c625 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -1,4 +1,4 @@ -# Monkey patches BigQuery client creation to use data proxy +# Monkey patches BigQuery client creation to use proxy. import os kaggle_proxy_data = os.getenv("KAGGLE_PROXY_DATA") @@ -6,18 +6,13 @@ from google.auth import credentials from google.cloud import bigquery from google.cloud.bigquery._http import Connection - from google.cloud import _http - from google.cloud.bigquery import __version__ Connection.API_BASE_URL = os.getenv("KAGGLE_PROXY_URL") - - _CLIENT_INFO = _http.CLIENT_INFO_TEMPLATE.format(__version__) - Connection._EXTRA_HEADERS = { - _http.CLIENT_INFO_HEADER: _CLIENT_INFO, - 'X-KAGGLE-PROXY-DATA': kaggle_proxy_data - } + Connection._EXTRA_HEADERS["X-KAGGLE-PROXY-DATA"] = kaggle_proxy_data bq_client = bigquery.Client - bigquery.Client = lambda: bq_client( + bigquery.Client = lambda *args, **kwargs: bq_client( + *args, credentials=credentials.AnonymousCredentials(), - project="bigquery-public-data") \ No newline at end of file + project="bigquery-public-data", + **kwargs) \ No newline at end of file diff --git a/test_build.py b/test_build.py index 8e97f4ae..0af541f4 100644 --- a/test_build.py +++ b/test_build.py @@ -70,3 +70,7 @@ from skimage.io import imread print("skimage ok") + +from google.cloud import bigquery +client = bigquery.Client() +print("bigquery ok") \ No newline at end of file From 2569da55ba6aec9bc6e4f90b2e274b17593ad0df Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Tue, 12 Dec 2017 22:01:07 -0800 Subject: [PATCH 0232/1660] Fix build and add BigQuery proxy test --- Dockerfile | 44 ++++++++++++++++---------------------------- test_build.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index 22f16c17..f5434e3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,31 +22,19 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & python -m spacy download en_core_web_lg && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ - libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ - libcups2 libcupsfilters1 libcupsimage2 libdatrie1 libdbus-1-3 libdjvulibre-text libdjvulibre21 libfftw3-double3 libfontconfig1 \ - libfreetype6 libgdk-pixbuf2.0-0 libgdk-pixbuf2.0-common libgomp1 libgraphite2-3 libgs9 libgs9-common libharfbuzz0b libijs-0.35 \ - libilmbase6 libjasper1 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1-0 libltdl7 libmagickcore-6.q16-2 \ - libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ - libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ - libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ - cd /usr/local/src && \ - wget http://transloadit.imagemagick.org/download/ImageMagick.tar.gz && \ - tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ - make -j $(nproc) && make install && \ - # clean up ImageMagick source files - cd ../ && rm -rf ImageMagick* && \ +libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ +libcups2 libcupsfilters1 libcupsimage2 libdatrie1 libdbus-1-3 libdjvulibre-text libdjvulibre21 libfftw3-double3 libfontconfig1 \ +libfreetype6 libgdk-pixbuf2.0-0 libgdk-pixbuf2.0-common libgomp1 libgraphite2-3 libgs9 libgs9-common libharfbuzz0b libijs-0.35 \ +libilmbase6 libjasper1 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1-0 libltdl7 libmagickcore-6.q16-2 \ +libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ +libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ +libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ + conda install -y -c conda-forge imagemagick && \ apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \ apt-get -y install cmake && \ - cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \ - cd opencv && \ - mkdir build && cd build && \ - cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D BUILD_PNG=TRUE .. && \ - make -j $(nproc) && make install && \ - echo "/usr/local/lib/python3.6/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ - cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \ + conda install -y -c conda-forge opencv && \ # Clean up install cruft - rm -rf /usr/local/src/opencv && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean @@ -123,7 +111,6 @@ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ apt-get autoremove -y && apt-get clean && \ rm -rf /usr/local/src/* - # Install OpenCV-3 with Python support RUN apt-get update && \ # Libgeos, for mapping libraries apt-get -y install libgeos-dev && \ @@ -209,6 +196,8 @@ RUN apt-get update && \ cd pyeconometrics && python setup.py install && \ apt-get install -y graphviz && pip install graphviz && \ apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ + # Pandoc is a dependency of deap + apt-get install -y pandoc && \ cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \ cd py-earth && python setup.py install && \ #cd /usr/local/src && git clone https://github.com/MTG/essentia.git && cd essentia && \ @@ -246,8 +235,6 @@ RUN pip install --upgrade mpld3 && \ pip install Geohash && \ # https://github.com/vinsci/geohash/issues/4 sed -i -- 's/geohash/.geohash/g' /opt/conda/lib/python3.6/site-packages/Geohash/__init__.py && \ - # Pandoc is a dependency of deap - apt-get install -y pandoc && \ pip install deap && \ pip install tpot && \ pip install haversine && \ @@ -278,7 +265,9 @@ RUN pip install --upgrade mpld3 && \ pip install altair && \ pip install pystan && \ pip install ImageHash && \ - pip install git+https://github.com/hammerlab/fancyimpute.git && \ + conda install -y ecos && \ + conda install -y CVXcanon && \ + pip install fancyimpute && \ pip install git+https://github.com/pymc-devs/pymc3 && \ pip install tifffile && \ pip install spectral && \ @@ -302,9 +291,8 @@ RUN pip install --upgrade mpld3 && \ pip install pyexcel-ods && \ pip install sklearn-pandas && \ pip install stemming && \ - pip install fbprophet && \ - conda install -y -c conda-forge -c ioam holoviews && \ - pip install git+https://github.com/ioam/geoviews.git && \ + conda install -y -c conda-forge fbprophet && \ + conda install -y -c conda-forge -c ioam holoviews geoviews && \ pip install hypertools && \ # Nxviz has been causing an installation issue by trying unsuccessfully to remove setuptools. #pip install nxviz && \ diff --git a/test_build.py b/test_build.py index 0af541f4..76e117e1 100644 --- a/test_build.py +++ b/test_build.py @@ -32,6 +32,7 @@ print("xgboost XGBClassifier: ok") import matplotlib.pyplot as plt +plt.switch_backend('agg') plt.plot(np.linspace(0,1,50), np.random.rand(50)) plt.savefig("plot1.png") print("matplotlib.pyplot ok") @@ -71,6 +72,39 @@ from skimage.io import imread print("skimage ok") +import os +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer from google.cloud import bigquery + +HOSTNAME = "127.0.0.1" +PORT = 8000 +URL = "http://%s:%s" % (HOSTNAME, PORT) + +header_found = False + +class HTTPHandler(BaseHTTPRequestHandler): + def do_HEAD(s): + s.send_response(200) + + def do_GET(s): + global header_found + header_found = any(k for k in s.headers if k == "X-KAGGLE-PROXY-DATA" and s.headers[k] == "test-key") + s.send_response(200) + +httpd = HTTPServer((HOSTNAME, PORT), HTTPHandler) +thread = threading.Thread(target=httpd.serve_forever).start() + client = bigquery.Client() -print("bigquery ok") \ No newline at end of file + +try: + for ds in client.list_datasets(): pass +except: + pass + +if header_found: + print("bigquery proxy ok") +else: + print("bigquery proxy failed") + +httpd.shutdown() From 9b418b2791bda23b32474c6cff53a65af691593c Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Tue, 12 Dec 2017 22:05:20 -0800 Subject: [PATCH 0233/1660] Add build test environment variables --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 3f7e3664..ce5db8d5 100755 --- a/build +++ b/build @@ -21,7 +21,7 @@ mkdir -p /tmp/python-build/working docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root --ip="*" sleep 3 docker kill jupyter_test && docker rm jupyter_test -docker run --rm -t --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' +docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_PROXY_DATA=test-key -e KAGGLE_PROXY_URL=http://127.0.0.1:8000 -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' # The test_build.py script creates a plot called plot1.png; check that it exists [ -s /tmp/python-build/working/plot1.png ] || exit 1 From 12b45f59f89944355e7762f26fcb086b33362120 Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Wed, 13 Dec 2017 09:25:08 -0800 Subject: [PATCH 0234/1660] Remove unused `thread` variable --- test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_build.py b/test_build.py index 76e117e1..33186dd3 100644 --- a/test_build.py +++ b/test_build.py @@ -93,7 +93,7 @@ def do_GET(s): s.send_response(200) httpd = HTTPServer((HOSTNAME, PORT), HTTPHandler) -thread = threading.Thread(target=httpd.serve_forever).start() +threading.Thread(target=httpd.serve_forever).start() client = bigquery.Client() From c8f45c5bf7e2e5e0d790c5602780df27c5867970 Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Wed, 13 Dec 2017 09:49:22 -0800 Subject: [PATCH 0235/1660] Add asserts and fake-called to BQ test --- test_build.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test_build.py b/test_build.py index 33186dd3..b6c50d20 100644 --- a/test_build.py +++ b/test_build.py @@ -81,15 +81,18 @@ PORT = 8000 URL = "http://%s:%s" % (HOSTNAME, PORT) -header_found = False +fake_bq_called = False +fake_bq_header_found = False class HTTPHandler(BaseHTTPRequestHandler): def do_HEAD(s): s.send_response(200) def do_GET(s): - global header_found - header_found = any(k for k in s.headers if k == "X-KAGGLE-PROXY-DATA" and s.headers[k] == "test-key") + global fake_bq_called + global fake_bq_header_found + fale_bq_called = True + fake_bq_header_found = any(k for k in s.headers if k == "X-KAGGLE-PROXY-DATA" and s.headers[k] == "test-key") s.send_response(200) httpd = HTTPServer((HOSTNAME, PORT), HTTPHandler) @@ -102,9 +105,10 @@ def do_GET(s): except: pass -if header_found: - print("bigquery proxy ok") -else: - print("bigquery proxy failed") - httpd.shutdown() + +assert fake_bq_called, "Fake server did not recieve a request from the BQ client." +assert fake_bq_header_found, "X-KAGGLE-PROXY-DATA header was missing from the BQ request." + +print("bigquery proxy ok") + From 627e7834e369f8ae388a4bcfab87913dd66e33c6 Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Wed, 13 Dec 2017 09:53:24 -0800 Subject: [PATCH 0236/1660] Fix typo --- test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_build.py b/test_build.py index b6c50d20..de5678b2 100644 --- a/test_build.py +++ b/test_build.py @@ -91,7 +91,7 @@ def do_HEAD(s): def do_GET(s): global fake_bq_called global fake_bq_header_found - fale_bq_called = True + fake_bq_called = True fake_bq_header_found = any(k for k in s.headers if k == "X-KAGGLE-PROXY-DATA" and s.headers[k] == "test-key") s.send_response(200) From 92544c3b826e6f6446c6417dfb853c0aec5567a7 Mon Sep 17 00:00:00 2001 From: "Jason S. Kessler" Date: Fri, 15 Dec 2017 15:00:07 -0800 Subject: [PATCH 0237/1660] Update Dockerfile to install Scattertext Scattertext produces interactive visualizations showing how BoW features differ among categorized documents. This can be very useful in exploring text classification tasks. It can also be used for visualizing categorical feature importance scores. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f5434e3d..88eeaf17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -343,6 +343,7 @@ RUN pip install --upgrade mpld3 && \ pip install annoy && \ pip install category_encoders && \ pip install google-cloud-bigquery && \ + pip install scattertext && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From cede7f34c4dbda75299a45855a8b6fe3bda631e8 Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Fri, 15 Dec 2017 23:31:01 -0800 Subject: [PATCH 0238/1660] Fix matplotlib backend --- Dockerfile | 7 +++---- test_build.py | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f5434e3d..c3a425f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -361,10 +361,6 @@ RUN conda install -f -y numpy==1.13.0 && \ pip install --upgrade dask && \ # Temporary downgrade for Pandas to circumvent https://github.com/pandas-dev/pandas/issues/18186 pip uninstall -y pandas && pip install pandas==0.20.3 && \ - # Matplotlib corrections: - # set backend for matplotlib to Agg - matplotlibrc_path=$(python -c "import site, os, fileinput; packages_dir = site.getsitepackages()[0]; print(os.path.join(packages_dir, 'matplotlib', 'mpl-data', 'matplotlibrc'))") && \ - sed -i 's/^backend : TkAgg/backend : agg/' $matplotlibrc_path && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ @@ -377,6 +373,9 @@ RUN conda install -f -y numpy==1.13.0 && \ ENV PYTHONUSERBASE "/root/.local" ADD patches/sitecustomize.py /root/.local/lib/python3.6/site-packages/sitecustomize.py +# Set backend for matplotlib +ENV MPLBACKEND "agg" + # Finally, apply any locally defined patches. RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" diff --git a/test_build.py b/test_build.py index de5678b2..4b0415c8 100644 --- a/test_build.py +++ b/test_build.py @@ -32,7 +32,6 @@ print("xgboost XGBClassifier: ok") import matplotlib.pyplot as plt -plt.switch_backend('agg') plt.plot(np.linspace(0,1,50), np.random.rand(50)) plt.savefig("plot1.png") print("matplotlib.pyplot ok") From 37d0a16a430a132e8b48454a5bcdd3356f43c179 Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Sat, 16 Dec 2017 18:52:27 -0800 Subject: [PATCH 0239/1660] Revert conda installation of `imagemagick` and `opencv` do to possible regression (changes in integration tests image output) --- Dockerfile | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c3a425f6..6afc2613 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,12 +29,24 @@ libilmbase6 libjasper1 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1 libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ - conda install -y -c conda-forge imagemagick && \ + cd /usr/local/src && \ + wget http://transloadit.imagemagick.org/download/ImageMagick.tar.gz && \ + tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ + make -j $(nproc) && make install && \ + # clean up ImageMagick source files + cd ../ && rm -rf ImageMagick* && \ apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \ apt-get -y install cmake && \ - conda install -y -c conda-forge opencv && \ + cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \ + cd opencv && \ + mkdir build && cd build && \ + cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D BUILD_PNG=TRUE .. && \ + make -j $(nproc) && make install && \ + echo "/usr/local/lib/python3.6/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ + cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \ # Clean up install cruft + rm -rf /usr/local/src/opencv && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean From ab26cf6d875c6d5efbe6485452a3c821d3bd43bf Mon Sep 17 00:00:00 2001 From: Kilian Batzner Date: Sun, 17 Dec 2017 15:34:52 +0100 Subject: [PATCH 0240/1660] Add pip install ortools in Dockerfile ortools is the Python wrapper for the Google Optimization Tools: https://developers.google.com/optimization/ --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 6afc2613..dd4e8b4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -355,6 +355,7 @@ RUN pip install --upgrade mpld3 && \ pip install annoy && \ pip install category_encoders && \ pip install google-cloud-bigquery && \ + pip install ortools && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 1d0e9d7275cf6de1eb53145a006f704116f5f736 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 19 Dec 2017 06:53:34 +1100 Subject: [PATCH 0241/1660] Add cufflinks --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index ffbc68bc..c466c5f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -349,6 +349,7 @@ RUN pip install --upgrade mpld3 && \ pip install mmh3 && \ pip install fbpca && \ pip install sentencepiece && \ + pip install cufflinks && \ pip install glmnet_py && \ pip install lime && \ pip install memory_profiler && \ From a6e3907244717c12bd77c8c61db9d6ae9b351a3b Mon Sep 17 00:00:00 2001 From: "Pradipta A. Suryadi" <116080050.pas@gmail.com> Date: Tue, 19 Dec 2017 15:50:03 +0700 Subject: [PATCH 0242/1660] Add Janome MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Janome (蛇の目) is a Japanese morphological analysis engine (tokenizer, pos-tagger) including built-in dictionary and language model. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index c466c5f9..947430d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -359,6 +359,8 @@ RUN pip install --upgrade mpld3 && \ pip install google-cloud-bigquery && \ pip install ortools && \ pip install scattertext && \ + # Add Japanese morphological analysis engine + pip install janome &&\ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From ae7a2bc7cb685c4a81248b35075460eb477e567a Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Tue, 19 Dec 2017 08:54:29 -0800 Subject: [PATCH 0243/1660] Fetch proxy project from env var --- build | 2 +- patches/sitecustomize.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build b/build index ce5db8d5..fce4dd99 100755 --- a/build +++ b/build @@ -21,7 +21,7 @@ mkdir -p /tmp/python-build/working docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root --ip="*" sleep 3 docker kill jupyter_test && docker rm jupyter_test -docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_PROXY_DATA=test-key -e KAGGLE_PROXY_URL=http://127.0.0.1:8000 -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' +docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_PROXY_TOKEN=test-key -e KAGGLE_PROXY_URL=http://127.0.0.1:8000 -e KAGGLE_PROXY_PROJECT=test -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' # The test_build.py script creates a plot called plot1.png; check that it exists [ -s /tmp/python-build/working/plot1.png ] || exit 1 diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 1500c625..42dad46a 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -1,18 +1,18 @@ # Monkey patches BigQuery client creation to use proxy. import os -kaggle_proxy_data = os.getenv("KAGGLE_PROXY_DATA") -if kaggle_proxy_data: +kaggle_proxy_token = os.getenv("KAGGLE_PROXY_TOKEN") +if kaggle_proxy_token: from google.auth import credentials from google.cloud import bigquery from google.cloud.bigquery._http import Connection Connection.API_BASE_URL = os.getenv("KAGGLE_PROXY_URL") - Connection._EXTRA_HEADERS["X-KAGGLE-PROXY-DATA"] = kaggle_proxy_data + Connection._EXTRA_HEADERS["X-KAGGLE-PROXY-DATA"] = kaggle_proxy_token bq_client = bigquery.Client bigquery.Client = lambda *args, **kwargs: bq_client( *args, credentials=credentials.AnonymousCredentials(), - project="bigquery-public-data", + project=os.getenv("KAGGLE_PROXY_PROJECT"), **kwargs) \ No newline at end of file From a5b8052078a873e02858a2985686cea60e2c8b81 Mon Sep 17 00:00:00 2001 From: "Pradipta A. Suryadi" <116080050.pas@gmail.com> Date: Wed, 20 Dec 2017 08:37:52 +0700 Subject: [PATCH 0244/1660] Add Space Adding space in EOL separator. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 947430d9..72ee9c4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -360,7 +360,7 @@ RUN pip install --upgrade mpld3 && \ pip install ortools && \ pip install scattertext && \ # Add Japanese morphological analysis engine - pip install janome &&\ + pip install janome && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From fd3ff2840f78906925be48f4772aeb114bc31561 Mon Sep 17 00:00:00 2001 From: bertcarremans Date: Wed, 20 Dec 2017 20:33:52 +0100 Subject: [PATCH 0245/1660] add emoji package --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c466c5f9..53f5aea3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -359,6 +359,7 @@ RUN pip install --upgrade mpld3 && \ pip install google-cloud-bigquery && \ pip install ortools && \ pip install scattertext && \ + pip install emoji && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 18d159e58ee4d7591882cf2ff3ffd868a35bc8fb Mon Sep 17 00:00:00 2001 From: Paul Mooney <30876065+paultimothymooney@users.noreply.github.com> Date: Wed, 20 Dec 2017 14:16:31 -0700 Subject: [PATCH 0246/1660] pip install nilearn Nilearn: Machine learning for NeuroImaging in Python --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c466c5f9..e531e442 100644 --- a/Dockerfile +++ b/Dockerfile @@ -241,6 +241,7 @@ RUN pip install --upgrade mpld3 && \ pip install gpxpy && \ pip install arrow && \ pip install vtk && \ + pip install nilearn && \ pip install nibabel && \ pip install pronouncing && \ pip install markovify && \ From 111c73344afa6dd984f39935b746b4eef593c89e Mon Sep 17 00:00:00 2001 From: Behzad Golshan Date: Wed, 20 Dec 2017 15:04:18 -0800 Subject: [PATCH 0247/1660] Adding the pykoko package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Koko is an information extraction tool (developed in Python 3) that allows users to query a text corpus and extract those entities that is of interest to them. A wide range of lexical, syntactic and semantic patterns can be specified using Koko’s query language which can be combined to efficiently extract and score relevant entities. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c466c5f9..57a2de79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -359,6 +359,7 @@ RUN pip install --upgrade mpld3 && \ pip install google-cloud-bigquery && \ pip install ortools && \ pip install scattertext && \ + pip install pykoko && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 7cc56bc7f3d184acb52be1b5cb19ac1d7bb826d4 Mon Sep 17 00:00:00 2001 From: Seth Rothschild Date: Thu, 21 Dec 2017 10:10:21 -0500 Subject: [PATCH 0248/1660] pip install featuretools --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c466c5f9..21d20895 100644 --- a/Dockerfile +++ b/Dockerfile @@ -359,6 +359,7 @@ RUN pip install --upgrade mpld3 && \ pip install google-cloud-bigquery && \ pip install ortools && \ pip install scattertext && \ + pip install featuretools && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From e1bc5afa633328594861e348377f0d210186882b Mon Sep 17 00:00:00 2001 From: Paul Mooney <30876065+paultimothymooney@users.noreply.github.com> Date: Mon, 25 Dec 2017 19:18:48 -0700 Subject: [PATCH 0249/1660] Update Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index c466c5f9..386dd4b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -244,6 +244,10 @@ RUN pip install --upgrade mpld3 && \ pip install nibabel && \ pip install pronouncing && \ pip install markovify && \ + pip install imgaug && \ + pip install preprocessing && \ + pip install Baker && \ + pip install path.py && \ pip install sexmachine && \ pip install Geohash && \ # https://github.com/vinsci/geohash/issues/4 From 469489bb63cd3217e6af109a447d5cfaf7319a44 Mon Sep 17 00:00:00 2001 From: Mihir Garg Date: Tue, 26 Dec 2017 23:17:30 +0530 Subject: [PATCH 0250/1660] Update Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index c466c5f9..bffa7223 100644 --- a/Dockerfile +++ b/Dockerfile @@ -395,3 +395,7 @@ ENV MPLBACKEND "agg" # Finally, apply any locally defined patches. RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" + +# Pandas data reader +RUN pip install pandas-datareader + From cb9399f5439ca4bb2ebdc754100ba34a58d454e2 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 9 Jan 2018 10:35:14 +1100 Subject: [PATCH 0251/1660] Remove missing hmm_treebank_pos_tagger dataset --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 907d1000..9b4a9d62 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ RUN apt-get install -y libfreetype6-dev && \ basque_grammars biocreative_ppi bllip_wsj_no_aux \ book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \ comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ -europarl_raw floresta gazetteers genesis gutenberg hmm_treebank_pos_tagger \ +europarl_raw floresta gazetteers genesis gutenberg \ ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho machado \ masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ mte_teip5 names nps_chat omw opinion_lexicon paradigms \ From 19dab85a3b254602da64a0ea79eaf77db6120dff Mon Sep 17 00:00:00 2001 From: Dan Becker Date: Mon, 8 Jan 2018 16:45:02 -0700 Subject: [PATCH 0252/1660] Change TensorFlow build to nightly --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9b4a9d62..8cebd7cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & # https://stackoverflow.com/a/46498173 conda update -y conda && conda update -y python && \ # Tensorflow - pip install tensorflow && \ + pip install tf-nightly && \ # Vowpal Rabbit #apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ #cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ From a3e8c5ad9b0fa62694b7eb08134e770c2ebe7940 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 9 Jan 2018 14:59:57 +1100 Subject: [PATCH 0253/1660] Removing RGF temporarily (GCC version issue) --- Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9b4a9d62..48353096 100644 --- a/Dockerfile +++ b/Dockerfile @@ -181,12 +181,6 @@ RUN apt-get update && \ conda install h5py && \ # https://github.com/biopython/biopython pip install biopython && \ - # Regularized Greedy Forests - cd /usr/local/src && wget https://github.com/fukatani/rgf_python/releases/download/0.2.0/rgf1.2.zip && \ - unzip rgf1.2.zip && cd rgf1.2 && make && mv bin/rgf /usr/local/bin && \ - cd /usr/local/src && git clone https://github.com/fukatani/rgf_python.git && \ - cd rgf_python && sed -i 's/\/opt\/rgf1.2\/bin\/rgf/\/usr\/local\/bin\/rgf/' rgf/sklearn.py && \ - python setup.py install && \ # PUDB, for local debugging convenience pip install pudb && \ # Imbalanced-learn From b7594b18c44242d37f215a939a711d711467ebd5 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 9 Jan 2018 15:33:07 +1100 Subject: [PATCH 0254/1660] Add various NLP packages --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 1d9fbdcd..63d173fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -360,6 +360,9 @@ RUN pip install --upgrade mpld3 && \ pip install scattertext && \ pip install pykoko && \ pip install featuretools && \ + pip install wordsegment && \ + pip install pyahocorasick && \ + pip install wordbatch && \ pip install emoji && \ # Add Japanese morphological analysis engine pip install janome && \ From e91643e6a1d2e82044321a1fd47ffdc1bea1f1ef Mon Sep 17 00:00:00 2001 From: Mihir Garg Date: Tue, 9 Jan 2018 12:37:55 +0530 Subject: [PATCH 0255/1660] changes done --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bffa7223..4e52e1a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -359,6 +359,8 @@ RUN pip install --upgrade mpld3 && \ pip install google-cloud-bigquery && \ pip install ortools && \ pip install scattertext && \ + # Pandas data reader + pip install pandas-datareader && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ @@ -396,6 +398,5 @@ ENV MPLBACKEND "agg" RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" -# Pandas data reader -RUN pip install pandas-datareader + From 1a4f6515484c8a3dd3424a44fa494ddfabc682d9 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 10 Jan 2018 08:14:13 +1100 Subject: [PATCH 0256/1660] Build TensorFlow from source --- Dockerfile | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63d173fa..020a9758 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,6 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & apt-get update && apt-get install -y build-essential && \ # https://stackoverflow.com/a/46498173 conda update -y conda && conda update -y python && \ - # Tensorflow - pip install tf-nightly && \ # Vowpal Rabbit #apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ #cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ @@ -50,6 +48,22 @@ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean +# Tensorflow source build +RUN apt-get install -y python-software-properties zip && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ + apt-get update && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get install -y oracle-java8-installer && \ + echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ + curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ + apt-get update && apt-get install -y bazel && apt-get upgrade -y bazel && \ + cd /usr/local/src && git clone https://github.com/tensorflow/tensorflow && \ + cd tensorflow && echo "\n" | ./configure && \ + bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ + pip install /tmp/tensorflow_pkg/tensorflow*.whl + RUN apt-get install -y libfreetype6-dev && \ apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ # textblob @@ -151,13 +165,7 @@ RUN apt-get update && \ # MXNet pip install mxnet==0.11.0 && \ # h2o - # This requires python-software-properties and Java. - apt-get install -y python-software-properties zip && \ - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ - apt-get update && \ - echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ - echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ - apt-get install -y oracle-java8-installer && \ + # This requires python-software-properties and Java, which were installed above. cd /usr/local/src && mkdir h2o && cd h2o && \ wget http://h2o-release.s3.amazonaws.com/h2o/latest_stable -O latest && \ wget --no-check-certificate -i latest -O h2o.zip && rm latest && \ From 72017a3bcde9c82526b3a2003148b7e91bef2af9 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Wed, 10 Jan 2018 10:53:01 +1100 Subject: [PATCH 0257/1660] Remove numpy version pin skimage and pytables now seem to require numpy 1.12. The symptoms in the original issue https://github.com/Kaggle/docker-python/issues/103 seem to have gone away. --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d1c68ffc..bf7feb59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -387,11 +387,9 @@ RUN pip install --upgrade mpld3 && \ # For Facets ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ -# Temporary fix for Anaconda linking error -# See https://github.com/Kaggle/docker-python/issues/103 -RUN conda install -f -y numpy==1.13.0 && \ +# Temporary fixes and patches # Temporary patch for Dask getting downgraded, which breaks Keras - pip install --upgrade dask && \ +RUN pip install --upgrade dask && \ # Temporary downgrade for Pandas to circumvent https://github.com/pandas-dev/pandas/issues/18186 pip uninstall -y pandas && pip install pandas==0.20.3 && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy From a3e3384b2fa427cafd53e2d3ae8450bf4b75bd4b Mon Sep 17 00:00:00 2001 From: Douglas Ashton Date: Thu, 11 Jan 2018 11:36:44 +0000 Subject: [PATCH 0258/1660] Add wfdb to pip install --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index bf7feb59..a00f2a0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -376,6 +376,7 @@ RUN pip install --upgrade mpld3 && \ pip install emoji && \ # Add Japanese morphological analysis engine pip install janome && \ + pip install wfdb && \ # Read data from physionet ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From c589a6842fe3ce383e64e19594cb5c6fcbb19534 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 15 Jan 2018 15:44:36 +1100 Subject: [PATCH 0259/1660] Update build script --- build | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build b/build index fce4dd99..5174478f 100755 --- a/build +++ b/build @@ -4,10 +4,10 @@ set -e # Default behavior is to do everything from scratch. # The --use-cache option is useful if you're iterating on a broken build. if [[ "$1" == "--use-cache" ]]; then - docker build --rm --cpu-period=100000 --cpu-quota=200000 -t kaggle/python-build . + docker build --rm -t kaggle/python-build . else docker pull continuumio/anaconda3:latest - docker build --rm --no-cache --cpu-period=100000 --cpu-quota=200000 -t kaggle/python-build . + docker build --rm --no-cache -t kaggle/python-build . fi @@ -26,5 +26,8 @@ docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_PROXY_TOKEN=tes # The test_build.py script creates a plot called plot1.png; check that it exists [ -s /tmp/python-build/working/plot1.png ] || exit 1 +docker tag kaggle/python-build gcr.io/kaggle-images/python +gcloud docker -- push gcr.io/kaggle-images/python + docker tag kaggle/python-build:latest kaggle/python:latest docker push kaggle/python:latest From 62ac46b153bb79508923f1c70b4ec67b364115db Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 15 Jan 2018 15:45:57 +1100 Subject: [PATCH 0260/1660] Update to Anaconda5, remove numpy pin --- Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index bf7feb59..6e4c264f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM continuumio/anaconda3:4.4.0 +FROM continuumio/anaconda3:5.0.1 ADD patches/ /tmp/patches/ ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl @@ -137,6 +137,9 @@ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ apt-get autoremove -y && apt-get clean && \ rm -rf /usr/local/src/* +# Make sure the dynamic linker finds the right libstdc++ +ENV LD_LIBRARY_PATH=/opt/conda/lib + RUN apt-get update && \ # Libgeos, for mapping libraries apt-get -y install libgeos-dev && \ @@ -379,13 +382,13 @@ RUN pip install --upgrade mpld3 && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ - # https://github.com/ContinuumIO/anaconda-issues/issues/720 - conda install -f -y numpy && \ # Required to display Altair charts in Jupyter notebook jupyter nbextension install --user --py vega # For Facets ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ +# For Theano with MKL +ENV MKL_THREADING_LAYER=GNU # Temporary fixes and patches # Temporary patch for Dask getting downgraded, which breaks Keras From 47b15de8d0500ac91e0c661a6e02bbc8fb9a69ee Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 15 Jan 2018 15:46:44 +1100 Subject: [PATCH 0261/1660] Add wordbatch and pyfasttext to tests --- test_build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_build.py b/test_build.py index 4b0415c8..5af53776 100644 --- a/test_build.py +++ b/test_build.py @@ -71,6 +71,12 @@ from skimage.io import imread print("skimage ok") +from wordbatch.extractors import WordBag +print("wordbatch ok") + +import pyfasttext +print("pyfasttext ok") + import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer From c79fa24aa8a183e24794325ea7f194f54b5d60c9 Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Wed, 17 Jan 2018 08:51:36 -0800 Subject: [PATCH 0262/1660] Suppress exception in call to `AnonymousCredentials.refresh` with NOP --- patches/sitecustomize.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 42dad46a..6293a3b2 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -1,18 +1,20 @@ # Monkey patches BigQuery client creation to use proxy. import os -kaggle_proxy_token = os.getenv("KAGGLE_PROXY_TOKEN") +kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN") if kaggle_proxy_token: from google.auth import credentials from google.cloud import bigquery from google.cloud.bigquery._http import Connection - Connection.API_BASE_URL = os.getenv("KAGGLE_PROXY_URL") + Connection.API_BASE_URL = os.getenv("KAGGLE_DATA_PROXY_URL") Connection._EXTRA_HEADERS["X-KAGGLE-PROXY-DATA"] = kaggle_proxy_token bq_client = bigquery.Client bigquery.Client = lambda *args, **kwargs: bq_client( *args, credentials=credentials.AnonymousCredentials(), - project=os.getenv("KAGGLE_PROXY_PROJECT"), - **kwargs) \ No newline at end of file + project=os.getenv("KAGGLE_DATA_PROXY_PROJECT"), + **kwargs) + + credentials.AnonymousCredentials.refresh = lambda *args: None From a4683b348be0bb48c37a6ea69d2bad44ff732421 Mon Sep 17 00:00:00 2001 From: Aurelio Agundez Date: Wed, 17 Jan 2018 10:41:34 -0800 Subject: [PATCH 0263/1660] Fix env vars for BQ client test --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 5174478f..781e96d6 100755 --- a/build +++ b/build @@ -21,7 +21,7 @@ mkdir -p /tmp/python-build/working docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root --ip="*" sleep 3 docker kill jupyter_test && docker rm jupyter_test -docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_PROXY_TOKEN=test-key -e KAGGLE_PROXY_URL=http://127.0.0.1:8000 -e KAGGLE_PROXY_PROJECT=test -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' +docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_DATA_PROXY_TOKEN=test-key -e KAGGLE_DATA_PROXY_URL=http://127.0.0.1:8000 -e KAGGLE_DATA_PROXY_PROJECT=test -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' # The test_build.py script creates a plot called plot1.png; check that it exists [ -s /tmp/python-build/working/plot1.png ] || exit 1 From 1d025228bd563e2703d6e23bfcdf6627456faa7c Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 22 Jan 2018 08:59:00 +1100 Subject: [PATCH 0264/1660] Add fasttext --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c5e853af..ca40d48d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -364,6 +364,7 @@ RUN pip install --upgrade mpld3 && \ pip install lime && \ pip install memory_profiler && \ pip install pyfasttext && \ + cd /usr/local/src && git clone --depth=1 https://github.com/facebookresearch/fastText.git && cd fastText && pip install . && \ pip install annoy && \ pip install category_encoders && \ pip install google-cloud-bigquery && \ From 49d830d8b6f01b35cb68adcf4418a06a919761db Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 22 Jan 2018 09:08:36 +1100 Subject: [PATCH 0265/1660] Add hunspell, vecstack, lightning --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index ca40d48d..62134119 100644 --- a/Dockerfile +++ b/Dockerfile @@ -365,6 +365,7 @@ RUN pip install --upgrade mpld3 && \ pip install memory_profiler && \ pip install pyfasttext && \ cd /usr/local/src && git clone --depth=1 https://github.com/facebookresearch/fastText.git && cd fastText && pip install . && \ + apt-get install -y libhunspell-dev && pip install hunspell && \ pip install annoy && \ pip install category_encoders && \ pip install google-cloud-bigquery && \ @@ -381,6 +382,8 @@ RUN pip install --upgrade mpld3 && \ # Add Japanese morphological analysis engine pip install janome && \ pip install wfdb && \ # Read data from physionet + pip install vecstack && \ + pip install sklearn-contrib-lightning && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 0244bd34474f225b084cebcbf87e6911f170da54 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Mon, 22 Jan 2018 09:10:45 +1100 Subject: [PATCH 0266/1660] Update Dockerfile syntax --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 62134119..bb71ab7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -381,7 +381,7 @@ RUN pip install --upgrade mpld3 && \ pip install emoji && \ # Add Japanese morphological analysis engine pip install janome && \ - pip install wfdb && \ # Read data from physionet + pip install wfdb && \ pip install vecstack && \ pip install sklearn-contrib-lightning && \ ##### ^^^^ Add new contributions above here From a35d17689dc40715a094c5b8742b4d8f56d18277 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 23 Jan 2018 08:05:54 +1100 Subject: [PATCH 0267/1660] Update TensorFlow to 1.5rc1 --- Dockerfile | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb71ab7e..5d57bc57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,23 +48,8 @@ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean -# Tensorflow source build -RUN apt-get install -y python-software-properties zip && \ - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ - apt-get update && \ - echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ - echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ - apt-get install -y oracle-java8-installer && \ - echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ - curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ - apt-get update && apt-get install -y bazel && apt-get upgrade -y bazel && \ - cd /usr/local/src && git clone https://github.com/tensorflow/tensorflow && \ - cd tensorflow && echo "\n" | ./configure && \ - bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ - bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ - pip install /tmp/tensorflow_pkg/tensorflow*.whl - -RUN apt-get install -y libfreetype6-dev && \ +RUN pip install tensorflow==1.5.0-rc1 && \ + apt-get install -y libfreetype6-dev && \ apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ # textblob pip install textblob && \ @@ -167,8 +152,14 @@ RUN apt-get update && \ cd cartopy && python setup.py install && \ # MXNet pip install mxnet==0.11.0 && \ + apt-get install -y python-software-properties zip && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ + apt-get update && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get install -y oracle-java8-installer && \ # h2o - # This requires python-software-properties and Java, which were installed above. + # This requires python-software-properties and Java. cd /usr/local/src && mkdir h2o && cd h2o && \ wget http://h2o-release.s3.amazonaws.com/h2o/latest_stable -O latest && \ wget --no-check-certificate -i latest -O h2o.zip && rm latest && \ From b88a88604d2bf33ef5f81c9cbb58f89026a4d070 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 23 Jan 2018 08:07:08 +1100 Subject: [PATCH 0268/1660] Update build tests --- test_build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_build.py b/test_build.py index 5af53776..68b176d6 100644 --- a/test_build.py +++ b/test_build.py @@ -77,6 +77,9 @@ import pyfasttext print("pyfasttext ok") +import fastText +print("fastText ok") + import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer From 7dedfe696699ff1d1e4f8369d276a3c47cef10db Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 23 Jan 2018 09:13:02 +1100 Subject: [PATCH 0269/1660] Revert "Update TensorFlow to 1.5rc1" This reverts commit a35d17689dc40715a094c5b8742b4d8f56d18277. --- Dockerfile | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d57bc57..bb71ab7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,8 +48,23 @@ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean -RUN pip install tensorflow==1.5.0-rc1 && \ - apt-get install -y libfreetype6-dev && \ +# Tensorflow source build +RUN apt-get install -y python-software-properties zip && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ + apt-get update && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get install -y oracle-java8-installer && \ + echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ + curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ + apt-get update && apt-get install -y bazel && apt-get upgrade -y bazel && \ + cd /usr/local/src && git clone https://github.com/tensorflow/tensorflow && \ + cd tensorflow && echo "\n" | ./configure && \ + bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ + pip install /tmp/tensorflow_pkg/tensorflow*.whl + +RUN apt-get install -y libfreetype6-dev && \ apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ # textblob pip install textblob && \ @@ -152,14 +167,8 @@ RUN apt-get update && \ cd cartopy && python setup.py install && \ # MXNet pip install mxnet==0.11.0 && \ - apt-get install -y python-software-properties zip && \ - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ - apt-get update && \ - echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ - echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ - apt-get install -y oracle-java8-installer && \ # h2o - # This requires python-software-properties and Java. + # This requires python-software-properties and Java, which were installed above. cd /usr/local/src && mkdir h2o && cd h2o && \ wget http://h2o-release.s3.amazonaws.com/h2o/latest_stable -O latest && \ wget --no-check-certificate -i latest -O h2o.zip && rm latest && \ From 369cd4dee455206805cb83d9b341c97e1713eca7 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Tue, 23 Jan 2018 11:06:05 +1100 Subject: [PATCH 0270/1660] Drop unfortunately-named rarely-used package --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bb71ab7e..2ccd8723 100644 --- a/Dockerfile +++ b/Dockerfile @@ -254,7 +254,6 @@ RUN pip install --upgrade mpld3 && \ pip install preprocessing && \ pip install Baker && \ pip install path.py && \ - pip install sexmachine && \ pip install Geohash && \ # https://github.com/vinsci/geohash/issues/4 sed -i -- 's/geohash/.geohash/g' /opt/conda/lib/python3.6/site-packages/Geohash/__init__.py && \ From b4f19de1f295cb92476a79f99c3462a3438fe019 Mon Sep 17 00:00:00 2001 From: Danielsen Date: Tue, 23 Jan 2018 11:34:01 -0500 Subject: [PATCH 0271/1660] add yellowbrick to dockerfile --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2ccd8723..c39f6f5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -383,6 +383,8 @@ RUN pip install --upgrade mpld3 && \ pip install wfdb && \ pip install vecstack && \ pip install sklearn-contrib-lightning && \ + # yellowbrick machine learning visualization library + pip install yellowbrick && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ @@ -417,6 +419,3 @@ ENV MPLBACKEND "agg" # Finally, apply any locally defined patches. RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" - - - From 430cb31cbb2b3adb413d8d572e5888499eee7580 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 25 Jan 2018 13:16:44 +1100 Subject: [PATCH 0272/1660] Test for MKL linkage --- test_build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_build.py b/test_build.py index 68b176d6..604c7e28 100644 --- a/test_build.py +++ b/test_build.py @@ -6,6 +6,12 @@ print("Numpy imported ok") print("Your lucky number is: " + str(np.random.randint(100))) +# Numpy must be linked to the MKL. (Occasionally, a third-party package will muck up the installation +# and numpy will be reinstalled with an OpenBLAS backing.) +from numpy.distutils.system_info import get_info +# This will throw an exception if the MKL is not linked correctly. +get_info("blas_mkl") + import pandas as pd print("Pandas imported ok") From a7e8dd82bf52e809ee62d2a4463fdb4a7df7e377 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Thu, 25 Jan 2018 13:17:46 +1100 Subject: [PATCH 0273/1660] Remove featuretools https://github.com/Kaggle/docker-python/pull/154#issuecomment-360323457 --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2ccd8723..f5d27f21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -373,7 +373,6 @@ RUN pip install --upgrade mpld3 && \ # Pandas data reader pip install pandas-datareader && \ pip install pykoko && \ - pip install featuretools && \ pip install wordsegment && \ pip install pyahocorasick && \ pip install wordbatch && \ From efbc01e8a5ace1b25f87ba45af1ca1684d422cae Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 24 Jan 2018 09:42:11 +0300 Subject: [PATCH 0274/1660] Unpin mxnet version This unpins mxnet version. It was build from source before, and then moved to pip install in 6c4cac180ced8b0712e3cd3b34adac19f6daaa46. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f5d27f21..f3d45682 100644 --- a/Dockerfile +++ b/Dockerfile @@ -166,7 +166,7 @@ RUN apt-get update && \ cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ cd cartopy && python setup.py install && \ # MXNet - pip install mxnet==0.11.0 && \ + pip install mxnet && \ # h2o # This requires python-software-properties and Java, which were installed above. cd /usr/local/src && mkdir h2o && cd h2o && \ From ea34e64b365d454ead5e2af29eb05b91a8b2b15e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 25 Jan 2018 22:41:25 +0300 Subject: [PATCH 0275/1660] Add mxnet and mxnet.gluon tests --- test_build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test_build.py b/test_build.py index 604c7e28..645f5488 100644 --- a/test_build.py +++ b/test_build.py @@ -86,6 +86,10 @@ import fastText print("fastText ok") +import mxnet +import mxnet.gluon +print("mxnet ok") + import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer From e0095d44a4a6bf5eee7dd7e53a63295795811832 Mon Sep 17 00:00:00 2001 From: Collin Donahue-Oponski Date: Fri, 26 Jan 2018 09:21:32 -0700 Subject: [PATCH 0276/1660] =?UTF-8?q?`conda=20install=20pytorch=20-c=20sou?= =?UTF-8?q?mith`=20=E2=80=94>=20`-c=20pytorch`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f5d27f21..5575a1c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -221,7 +221,7 @@ RUN apt-get update && \ #./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ #./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ # PyTorch - conda install -y pytorch torchvision -c soumith && \ + conda install -y pytorch torchvision -c pytorch && \ # PyTorch Audio apt-get install -y sox libsox-dev libsox-fmt-all && \ pip install cffi && \ From b1886270816088e35f303c905f70e3fa95efa177 Mon Sep 17 00:00:00 2001 From: Kevin Mader Date: Mon, 29 Jan 2018 16:49:33 +0100 Subject: [PATCH 0277/1660] Adding Keras RCNN adding code to install Keras RCNN (https://github.com/Kaggle/docker-python/issues/172) --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7c36beb3..414eeddf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -89,6 +89,8 @@ RUN apt-get install -y libfreetype6-dev && \ cd /usr/local/src && mkdir keras-rl && cd keras-rl && \ git clone --depth 1 https://github.com/matthiasplappert/keras-rl.git && \ cd keras-rl && python setup.py install && \ + #keras-rcnn + pip install git+https://github.com/broadinstitute/keras-rcnn && \ #neon cd /usr/local/src && \ git clone --depth 1 https://github.com/NervanaSystems/neon.git && \ From d6f58e133f060999a492ef9c943997bfa6e8b958 Mon Sep 17 00:00:00 2001 From: Mikel Bober-Irizar Date: Tue, 30 Jan 2018 21:16:25 +0000 Subject: [PATCH 0278/1660] Add mlcrate --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7c36beb3..79244728 100644 --- a/Dockerfile +++ b/Dockerfile @@ -384,6 +384,7 @@ RUN pip install --upgrade mpld3 && \ pip install sklearn-contrib-lightning && \ # yellowbrick machine learning visualization library pip install yellowbrick && \ + pip install mlcrate && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ From 95d0ce189c2a75601cac02103c57259be0d7d8a3 Mon Sep 17 00:00:00 2001 From: Paul Mooney <30876065+paultimothymooney@users.noreply.github.com> Date: Wed, 31 Jan 2018 22:46:46 -0700 Subject: [PATCH 0279/1660] pip install rf_perm_feat_import This class implements the permutation importance metric for random forests --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7c36beb3..f036a3bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -250,6 +250,7 @@ RUN pip install --upgrade mpld3 && \ pip install nibabel && \ pip install pronouncing && \ pip install markovify && \ + pip install rf_perm_feat_import && \ pip install imgaug && \ pip install preprocessing && \ pip install Baker && \ From 112057fdbc8ef76a5a876c73b7e6ee7967dcf44d Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 2 Feb 2018 10:37:53 +1100 Subject: [PATCH 0280/1660] Add flashtext, move contribs to new layer --- Dockerfile | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4636173d..28f635c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -236,13 +236,6 @@ RUN apt-get update && \ conda clean -i -l -t -y && \ rm -rf /usr/local/src/* - ########### - # - # NEW CONTRIBUTORS: - # Please add new pip/apt installs in this block. Don't forget a "&& \" at the end - # of all non-final lines. Thanks! - # - ########### RUN pip install --upgrade mpld3 && \ pip install mplleaflet && \ pip install gpxpy && \ @@ -388,12 +381,25 @@ RUN pip install --upgrade mpld3 && \ # yellowbrick machine learning visualization library pip install yellowbrick && \ pip install mlcrate && \ - ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* && \ # Required to display Altair charts in Jupyter notebook jupyter nbextension install --user --py vega + + ########### + # + # NEW CONTRIBUTORS: + # Please add new pip/apt installs in this block. Don't forget a "&& \" at the end + # of all non-final lines. Thanks! + # + ########### +RUN pip install flashtext && \ + ##### ^^^^ Add new contributions above here + # clean up pip cache + rm -rf /root/.cache/pip/* + + # For Facets ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ # For Theano with MKL From 8cad7e4b4e683cba452e1d0940e686b54450b473 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 2 Feb 2018 10:38:19 +1100 Subject: [PATCH 0281/1660] Add marisa-trie --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 28f635c6..700571ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -395,6 +395,7 @@ RUN pip install --upgrade mpld3 && \ # ########### RUN pip install flashtext && \ + pip install marisa-trie && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 176135896a715a55f8646b718efed5aee64f8101 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 2 Feb 2018 10:39:04 +1100 Subject: [PATCH 0282/1660] Add pyemd --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 700571ec..f5780f3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -396,6 +396,7 @@ RUN pip install --upgrade mpld3 && \ ########### RUN pip install flashtext && \ pip install marisa-trie && \ + pip install pyemd && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 5a2649de5a3418d9b4eb86d9c7171591319cdf75 Mon Sep 17 00:00:00 2001 From: Sohier Dane Date: Thu, 1 Feb 2018 16:38:23 -0800 Subject: [PATCH 0283/1660] Update Dockerfile Added pip install for BigQuery_Helper: https://github.com/SohierDane/BigQuery_Helper --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index f5780f3b..0a601d92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -397,6 +397,7 @@ RUN pip install --upgrade mpld3 && \ RUN pip install flashtext && \ pip install marisa-trie && \ pip install pyemd && \ + pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 796d7ea352782b531c1285b4a7b83edec45711b4 Mon Sep 17 00:00:00 2001 From: steadyfish Date: Sat, 3 Feb 2018 17:19:24 -0500 Subject: [PATCH 0284/1660] Added pyupset package Reference: https://pypi.python.org/pypi/pyupset/0.1.1.post1 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0a601d92..8ba83709 100644 --- a/Dockerfile +++ b/Dockerfile @@ -397,6 +397,7 @@ RUN pip install --upgrade mpld3 && \ RUN pip install flashtext && \ pip install marisa-trie && \ pip install pyemd && \ + pip install pyupset && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ ##### ^^^^ Add new contributions above here # clean up pip cache From 44ff0c2eb39546c3ddc73bb008b86c7353ae8672 Mon Sep 17 00:00:00 2001 From: Dan Becker Date: Sat, 3 Feb 2018 17:42:02 -0700 Subject: [PATCH 0285/1660] Add ktext for fast text processing with Keras --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0a601d92..51c8a3a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -359,6 +359,7 @@ RUN pip install --upgrade mpld3 && \ pip install lime && \ pip install memory_profiler && \ pip install pyfasttext && \ + pip install ktext && \ cd /usr/local/src && git clone --depth=1 https://github.com/facebookresearch/fastText.git && cd fastText && pip install . && \ apt-get install -y libhunspell-dev && pip install hunspell && \ pip install annoy && \ From a3eb1c4a0791c17c394dc7c1d951477ae0d39e14 Mon Sep 17 00:00:00 2001 From: Kevin Mader Date: Mon, 5 Feb 2018 09:30:13 +0100 Subject: [PATCH 0286/1660] Update Dockerfile adding scikit-optimize dependency for more intelligent parameter and hyperparameter optimization --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 05407e96..173ce41e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -255,6 +255,7 @@ RUN pip install --upgrade mpld3 && \ sed -i -- 's/geohash/.geohash/g' /opt/conda/lib/python3.6/site-packages/Geohash/__init__.py && \ pip install deap && \ pip install tpot && \ + pip install scikit-optimize && \ pip install haversine && \ pip install toolz cytoolz && \ pip install sacred && \ From 4b6a795408422b97159c54d82d27df16a5318822 Mon Sep 17 00:00:00 2001 From: Bri-Will Date: Mon, 5 Feb 2018 09:25:09 -0800 Subject: [PATCH 0287/1660] Add support for hyperopt-sklearn --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 05407e96..cce66d1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -400,6 +400,7 @@ RUN pip install flashtext && \ pip install pyemd && \ pip install pyupset && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ + cd /usr/local/src && git clone https://github.com/hyperopt/hyperopt-sklearn.git && cd hyperopt-sklearn && pip install -e . && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From b3c6992d2aa5bfd1318e13b8c787bc99a7acc739 Mon Sep 17 00:00:00 2001 From: Ahmed BESBES Date: Wed, 7 Feb 2018 18:18:14 +0100 Subject: [PATCH 0288/1660] Update Dockerfile adding keras-tqdm --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 05407e96..406e9e71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,6 +91,8 @@ RUN apt-get install -y libfreetype6-dev && \ cd keras-rl && python setup.py install && \ #keras-rcnn pip install git+https://github.com/broadinstitute/keras-rcnn && \ + #keras-tqdm + pip install keras-tqdm #neon cd /usr/local/src && \ git clone --depth 1 https://github.com/NervanaSystems/neon.git && \ From a071c8c027412ee89c7beab413ba3ce1d27f454b Mon Sep 17 00:00:00 2001 From: Seth Rothschild Date: Tue, 27 Feb 2018 12:05:31 -0500 Subject: [PATCH 0289/1660] pip install featuretools --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 05407e96..f9b36739 100644 --- a/Dockerfile +++ b/Dockerfile @@ -399,6 +399,9 @@ RUN pip install flashtext && \ pip install marisa-trie && \ pip install pyemd && \ pip install pyupset && \ + pip install pympler && \ + pip install s3fs && \ + pip install featuretools && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ ##### ^^^^ Add new contributions above here # clean up pip cache From 0073be5fff856a3ac939f53381d8a1897bd8549c Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 9 Mar 2018 10:03:26 +1100 Subject: [PATCH 0290/1660] Add bokeh and seaborn to build test --- test_build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_build.py b/test_build.py index 645f5488..0c143924 100644 --- a/test_build.py +++ b/test_build.py @@ -90,6 +90,12 @@ import mxnet.gluon print("mxnet ok") +import bokeh +print("bokeh ok") + +import seaborn +print("seaborn ok") + import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer From 5677cbed54db0eb22de7fdf872978a51260d2fe6 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 9 Mar 2018 11:09:40 +1100 Subject: [PATCH 0291/1660] Upgrade scipy; fix fast.ai installation --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 173ce41e..78693c81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,8 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & #cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ #ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ #pip install vowpalwabbit && \ + # Anaconda's scipy is currently behind the main release (1.0) + pip install scipy --upgrade && \ pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ husl geopy ml_metrics mne pyshp gensim && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ @@ -350,7 +352,6 @@ RUN pip install --upgrade mpld3 && \ pip install pydash && \ pip install kmodes && \ pip install librosa && \ - pip install fastai && \ pip install polyglot && \ pip install mmh3 && \ pip install fbpca && \ @@ -383,6 +384,10 @@ RUN pip install --upgrade mpld3 && \ # yellowbrick machine learning visualization library pip install yellowbrick && \ pip install mlcrate && \ + pip install opencv-python && \ + pip install pyarrow && \ + cd /usr/local/src && git clone --depth=1 https://github.com/fastai/fastai && \ + cd fastai && python setup.py install && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ # Required to display Altair charts in Jupyter notebook From 20adb40cea0928ff68c860efb2f295bccbcf9ae5 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 9 Mar 2018 12:08:21 +1100 Subject: [PATCH 0292/1660] Workaround for PyTorch issue 2575 --- patches/sitecustomize.py | 4 ++++ test_build.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 6293a3b2..9c6f566a 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -1,4 +1,8 @@ # Monkey patches BigQuery client creation to use proxy. + +# Import torch before anything else. This is a hacky workaround to an error on dlopen +# reporting a limit on static TLS, tracked in https://github.com/pytorch/pytorch/issues/2575 +import torch import os kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN") diff --git a/test_build.py b/test_build.py index 0c143924..bc70c2b0 100644 --- a/test_build.py +++ b/test_build.py @@ -96,6 +96,20 @@ import seaborn print("seaborn ok") +# PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html +import torch +import torch.nn as tnn +import torch.autograd as autograd +torch.manual_seed(31337) +linear_torch = tnn.Linear(5,3) +data_torch = autograd.Variable(torch.randn(2, 5)) +print(linear_torch(data_torch)) +print("PyTorch ok") + +import fastai +from fastai.io import get_data +print("fast.ai ok") + import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer From 29ff16c231420c71252eaaab0a6b635ec6f0a59f Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 9 Mar 2018 13:14:31 +1100 Subject: [PATCH 0293/1660] Fix wordcloud installation --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 78693c81..f2d32bf4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,7 +71,7 @@ RUN apt-get install -y libfreetype6-dev && \ # textblob pip install textblob && \ #word cloud - conda install -y -c https://conda.anaconda.org/amueller wordcloud && \ + pip install wordcloud && \ #igraph conda install -y -c conda-forge python-igraph && \ #xgboost From bf366b63c0ca146fcdb18b8b8e61e8b58cf05e73 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 9 Mar 2018 13:35:53 +1100 Subject: [PATCH 0294/1660] Upgrade conda numpy to match pip --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index f2d32bf4..6e1d6229 100644 --- a/Dockerfile +++ b/Dockerfile @@ -172,6 +172,8 @@ RUN apt-get update && \ # MXNet pip install mxnet && \ # h2o + # Temporary sync of conda's numpy with pip's, needed to avoid an install error + conda upgrade -y numpy && \ # This requires python-software-properties and Java, which were installed above. cd /usr/local/src && mkdir h2o && cd h2o && \ wget http://h2o-release.s3.amazonaws.com/h2o/latest_stable -O latest && \ From 0820a9e2305f45d1c548e85e322ac3c815995dfa Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 9 Mar 2018 18:58:57 +1100 Subject: [PATCH 0295/1660] Remove pandas downgrade --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e1d6229..6c871131 100644 --- a/Dockerfile +++ b/Dockerfile @@ -388,6 +388,8 @@ RUN pip install --upgrade mpld3 && \ pip install mlcrate && \ pip install opencv-python && \ pip install pyarrow && \ + # Ensure we have pandas >= 0.21 + pip install --upgrade pandas && \ cd /usr/local/src && git clone --depth=1 https://github.com/fastai/fastai && \ cd fastai && python setup.py install && \ # clean up pip cache @@ -421,8 +423,6 @@ ENV MKL_THREADING_LAYER=GNU # Temporary fixes and patches # Temporary patch for Dask getting downgraded, which breaks Keras RUN pip install --upgrade dask && \ - # Temporary downgrade for Pandas to circumvent https://github.com/pandas-dev/pandas/issues/18186 - pip uninstall -y pandas && pip install pandas==0.20.3 && \ # Stop jupyter nbconvert trying to rewrite its folder hierarchy mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \ mkdir -p /.jupyter && touch /.jupyter/jupyter_nbconvert_config.py && touch /.jupyter/migrated && \ From 5e484cc92b3b8bd93565897d9fb9964918323935 Mon Sep 17 00:00:00 2001 From: Jamie Hall Date: Fri, 9 Mar 2018 19:09:09 +1100 Subject: [PATCH 0296/1660] Move fastai to separate build stage --- Dockerfile | 67 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c871131..f14ded06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -386,18 +386,73 @@ RUN pip install --upgrade mpld3 && \ # yellowbrick machine learning visualization library pip install yellowbrick && \ pip install mlcrate && \ + # clean up pip cache + rm -rf /root/.cache/pip/* && \ + # Required to display Altair charts in Jupyter notebook + jupyter nbextension install --user --py vega + +# Fast.ai and dependencies +RUN pip install bcolz && \ + pip install bleach && \ + pip install certifi && \ + pip install cycler && \ + pip install decorator && \ + pip install entrypoints && \ + pip install html5lib && \ + pip install ipykernel && \ + pip install ipython && \ + pip install ipython-genutils && \ + pip install ipywidgets && \ + pip install isoweek && \ + pip install jedi && \ + pip install Jinja2 && \ + pip install jsonschema && \ + pip install jupyter && \ + pip install jupyter-client && \ + pip install jupyter-console && \ + pip install jupyter-core && \ + pip install MarkupSafe && \ + pip install matplotlib && \ + pip install mistune && \ + pip install nbconvert && \ + pip install nbformat && \ + pip install notebook && \ + pip install numpy && \ + pip install olefile && \ pip install opencv-python && \ - pip install pyarrow && \ - # Ensure we have pandas >= 0.21 pip install --upgrade pandas && \ + pip install pandas_summary && \ + pip install pandocfilters && \ + pip install pexpect && \ + pip install pickleshare && \ + pip install Pillow && \ + pip install prompt-toolkit && \ + pip install ptyprocess && \ + pip install Pygments && \ + pip install pyparsing && \ + pip install python-dateutil && \ + pip install pytz && \ + pip install PyYAML && \ + pip install pyzmq && \ + pip install qtconsole && \ + pip install scipy && \ + pip install seaborn && \ + pip install simplegeneric && \ + pip install six && \ + pip install terminado && \ + pip install testpath && \ + pip install tornado && \ + pip install tqdm && \ + pip install traitlets && \ + pip install wcwidth && \ + pip install webencodings && \ + pip install widgetsnbextension && \ cd /usr/local/src && git clone --depth=1 https://github.com/fastai/fastai && \ cd fastai && python setup.py install && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ - # Required to display Altair charts in Jupyter notebook - jupyter nbextension install --user --py vega - - + cd && rm -rf /usr/local/src/* + ########### # # NEW CONTRIBUTORS: From 42b12ae40befd0004ff4875d524553e5222fdcc5 Mon Sep 17 00:00:00 2001 From: Bri-Will Date: Mon, 12 Mar 2018 13:41:43 -0700 Subject: [PATCH 0297/1660] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cce66d1e..0a835296 100644 --- a/Dockerfile +++ b/Dockerfile @@ -400,7 +400,7 @@ RUN pip install flashtext && \ pip install pyemd && \ pip install pyupset && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ - cd /usr/local/src && git clone https://github.com/hyperopt/hyperopt-sklearn.git && cd hyperopt-sklearn && pip install -e . && \ + pip install hpsklearn && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 1be67f8a1429556db314aea14a6265d4032d31fe Mon Sep 17 00:00:00 2001 From: crawforc3 Date: Mon, 12 Mar 2018 13:59:07 -0700 Subject: [PATCH 0298/1660] Moved to "new contributors" block and added "&& \" --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 406e9e71..73aa7e5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,8 +91,6 @@ RUN apt-get install -y libfreetype6-dev && \ cd keras-rl && python setup.py install && \ #keras-rcnn pip install git+https://github.com/broadinstitute/keras-rcnn && \ - #keras-tqdm - pip install keras-tqdm #neon cd /usr/local/src && \ git clone --depth 1 https://github.com/NervanaSystems/neon.git && \ @@ -402,6 +400,7 @@ RUN pip install flashtext && \ pip install pyemd && \ pip install pyupset && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ + pip install keras-tqdm ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From d690a1be7debbf357988e390f2604b426708679d Mon Sep 17 00:00:00 2001 From: crawforc3 Date: Mon, 12 Mar 2018 14:11:46 -0700 Subject: [PATCH 0299/1660] Actually added "&& \" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 73aa7e5d..0f125734 100644 --- a/Dockerfile +++ b/Dockerfile @@ -400,7 +400,7 @@ RUN pip install flashtext && \ pip install pyemd && \ pip install pyupset && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ - pip install keras-tqdm + pip install keras-tqdm && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 6515ababe9b371b6dc65a8620399a4b07706f012 Mon Sep 17 00:00:00 2001 From: crawforc3 Date: Fri, 16 Mar 2018 08:47:22 -0700 Subject: [PATCH 0300/1660] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f14ded06..b3b8e409 100644 --- a/Dockerfile +++ b/Dockerfile @@ -430,7 +430,7 @@ RUN pip install bcolz && \ pip install ptyprocess && \ pip install Pygments && \ pip install pyparsing && \ - pip install python-dateutil && \ + pip install python-dateutil==2.6.0 && \ pip install pytz && \ pip install PyYAML && \ pip install pyzmq && \ From 64fb057eb65619730f7ac75d3f85072797023ae4 Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Mon, 19 Mar 2018 16:06:00 +0000 Subject: [PATCH 0301/1660] Split build script into build/test/push. The push will by default push a 'testing' labeled image. Use `./push prod` to push the production affecting image. --- build | 22 ---------------------- push | 11 +++++++++++ test | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 22 deletions(-) create mode 100755 push create mode 100755 test diff --git a/build b/build index 781e96d6..642cc268 100755 --- a/build +++ b/build @@ -9,25 +9,3 @@ else docker pull continuumio/anaconda3:latest docker build --rm --no-cache -t kaggle/python-build . fi - - - -rm -rf /tmp/python-build -docker rm jupyter_test || true -mkdir -p /tmp/python-build/tmp -mkdir -p /tmp/python-build/devshm -mkdir -p /tmp/python-build/working -# Check that Jupyter server can run; if it dies on startup, the `docker kill` command will throw an error -docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root --ip="*" -sleep 3 -docker kill jupyter_test && docker rm jupyter_test -docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_DATA_PROXY_TOKEN=test-key -e KAGGLE_DATA_PROXY_URL=http://127.0.0.1:8000 -e KAGGLE_DATA_PROXY_PROJECT=test -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' - -# The test_build.py script creates a plot called plot1.png; check that it exists -[ -s /tmp/python-build/working/plot1.png ] || exit 1 - -docker tag kaggle/python-build gcr.io/kaggle-images/python -gcloud docker -- push gcr.io/kaggle-images/python - -docker tag kaggle/python-build:latest kaggle/python:latest -docker push kaggle/python:latest diff --git a/push b/push new file mode 100755 index 00000000..ee01a621 --- /dev/null +++ b/push @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +set +x + +label=${1:-testing} + +docker tag kaggle/python-build:latest gcr.io/kaggle-images/python:${label} +gcloud docker -- push gcr.io/kaggle-images/python:${label} + +docker tag kaggle/python-build:latest kaggle/python:${label} +docker push kaggle/python:${label} diff --git a/test b/test new file mode 100755 index 00000000..5de44db7 --- /dev/null +++ b/test @@ -0,0 +1,17 @@ +#!/bin/bash +set -e +set +x + +rm -rf /tmp/python-build +docker rm jupyter_test || true +mkdir -p /tmp/python-build/tmp +mkdir -p /tmp/python-build/devshm +mkdir -p /tmp/python-build/working +# Check that Jupyter server can run; if it dies on startup, the `docker kill` command will throw an error +docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root --ip="*" +sleep 3 +docker kill jupyter_test && docker rm jupyter_test +docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_DATA_PROXY_TOKEN=test-key -e KAGGLE_DATA_PROXY_URL=http://127.0.0.1:8000 -e KAGGLE_DATA_PROXY_PROJECT=test -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' + +# The test_build.py script creates a plot called plot1.png; check that it exists +[ -s /tmp/python-build/working/plot1.png ] || exit 1 From 256676aedf2c0a47c079ac548592a26ae54828fc Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Mon, 19 Mar 2018 16:13:46 +0000 Subject: [PATCH 0302/1660] Build the python image using Jenkins. --- Jenkinsfile | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..35fc71f0 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,59 @@ +String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1-5" : "" + +pipeline { + agent { label 'linux' } + options { + disableConcurrentBuilds() + } + triggers { + cron(cron_string) + } + environment { + GIT_COMMIT_SHORT = sh(returnStdout: true, script:"git rev-parse --short=7 HEAD").trim() + GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim() + GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim() + GIT_COMMIT_SUMMARY = "${GIT_BRANCH} `` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" + SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"ci\" ]]; then echo \"#kernels\"; else echo \"#builds\"; fi").trim() + } + + stages { + stage('Docker Build') { + steps { + slackSend color: 'none', message: "*<${env.BUILD_URL}console|docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + sh '''#!/bin/bash + ./build + ''' + } + } + + stage('Test Image') { + steps { + slackSend color: 'none', message: "*<${env.BUILD_URL}console|test image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + sh '''#!/bin/bash + ./test + ''' + } + } + + stage('Push Image') { + steps { + slackSend color: 'none', message: "*<${env.BUILD_URL}console|pushing image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + sh '''#!/bin/bash + ./push staging + ''' + } + } + } + + post { + failure { + slackSend color: 'danger', message: "*<${env.BUILD_URL}console|failed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + } + success { + slackSend color: 'good', message: "*<${env.BUILD_URL}console|passed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + } + aborted { + slackSend color: 'warning', message: "*<${env.BUILD_URL}console|aborted>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + } + } +} From a65a7cfd4c232de596e1d6461367d361f283a948 Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Mon, 19 Mar 2018 21:18:32 +0000 Subject: [PATCH 0303/1660] Fix ci -> master for this repo. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 35fc71f0..c43b4ebb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ pipeline { GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim() GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim() GIT_COMMIT_SUMMARY = "${GIT_BRANCH} `` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" - SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"ci\" ]]; then echo \"#kernels\"; else echo \"#builds\"; fi").trim() + SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernels\"; else echo \"#builds\"; fi").trim() } stages { From c7b8d8ff80c3b28aa2b305056b1f578ae4f715ed Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Wed, 21 Mar 2018 19:27:43 +0000 Subject: [PATCH 0304/1660] Carry over nits from the cuda branch's Jenkinsfile. --- Jenkinsfile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c43b4ebb..b60442b8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1-5" : "" pipeline { - agent { label 'linux' } + agent { label 'linux && !gpu' } options { disableConcurrentBuilds() } @@ -21,7 +21,9 @@ pipeline { steps { slackSend color: 'none', message: "*<${env.BUILD_URL}console|docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash - ./build + set -exo pipefail + + ./build | ts ''' } } @@ -30,6 +32,9 @@ pipeline { steps { slackSend color: 'none', message: "*<${env.BUILD_URL}console|test image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash + set -exo pipefail + + date ./test ''' } @@ -39,6 +44,9 @@ pipeline { steps { slackSend color: 'none', message: "*<${env.BUILD_URL}console|pushing image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash + set -exo pipefail + + date ./push staging ''' } From a3296aa915de9cd89a6467fc612bf492faefc677 Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Wed, 28 Mar 2018 17:19:09 +0000 Subject: [PATCH 0305/1660] Only push the tagged image to docker.io. --- push | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/push b/push index ee01a621..ac3d2136 100755 --- a/push +++ b/push @@ -7,5 +7,7 @@ label=${1:-testing} docker tag kaggle/python-build:latest gcr.io/kaggle-images/python:${label} gcloud docker -- push gcr.io/kaggle-images/python:${label} -docker tag kaggle/python-build:latest kaggle/python:${label} -docker push kaggle/python:${label} +if [[ "$label" == "latest" ]]; then + docker tag kaggle/python-build:latest kaggle/python:${label} + docker push kaggle/python:${label} +fi From 7b110689a5bf92f7d1aa7d4cd1133ae75af3e6bc Mon Sep 17 00:00:00 2001 From: Dan Becker Date: Fri, 30 Mar 2018 14:13:53 -0600 Subject: [PATCH 0306/1660] Add learntools for use in learn exercises --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 31ea2a78..5591e1ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -452,7 +452,7 @@ RUN pip install bcolz && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ cd && rm -rf /usr/local/src/* - + ########### # # NEW CONTRIBUTORS: @@ -466,6 +466,7 @@ RUN pip install flashtext && \ pip install pyupset && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ pip install hpsklearn && \ + pip install git+https://github.com/Kaggle/learntools && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 062f975d92c5795feb530e3ea1914d3c7dd3a96b Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Tue, 3 Apr 2018 23:15:20 +0000 Subject: [PATCH 0307/1660] Support adding an /etc/host entry for the data proxy, if asked. --- patches/sitecustomize.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 9c6f566a..3a37d118 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -5,12 +5,27 @@ import torch import os +_HOST_FILE = "/etc/hosts" + kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN") if kaggle_proxy_token: from google.auth import credentials from google.cloud import bigquery from google.cloud.bigquery._http import Connection + host_entry = os.getenv("KAGGLE_DATA_PROXY_HOST_ENTRY") + if host_entry: + host_entry = host_entry.strip() + with open(_HOST_FILE, "r") as host_file: + for line in host_file: + if line.strip() == host_entry: + break + else: + with open(_HOST_FILE, "a") as host_file_append: + host_file_append.write("\n" # Just in case it wasn't newline terminated. + + host_entry + + "\n") + Connection.API_BASE_URL = os.getenv("KAGGLE_DATA_PROXY_URL") Connection._EXTRA_HEADERS["X-KAGGLE-PROXY-DATA"] = kaggle_proxy_token From 97894a171d1831838da28b757aabb352bc5ecfd9 Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Thu, 5 Apr 2018 00:36:06 +0000 Subject: [PATCH 0308/1660] Revert "Support adding an /etc/host entry for the data proxy, if asked." This reverts commit 062f975d92c5795feb530e3ea1914d3c7dd3a96b. There is no more need for this support in the docker image. It is fully externally handled through docker run's `--add-host` feature. --- patches/sitecustomize.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 3a37d118..9c6f566a 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -5,27 +5,12 @@ import torch import os -_HOST_FILE = "/etc/hosts" - kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN") if kaggle_proxy_token: from google.auth import credentials from google.cloud import bigquery from google.cloud.bigquery._http import Connection - host_entry = os.getenv("KAGGLE_DATA_PROXY_HOST_ENTRY") - if host_entry: - host_entry = host_entry.strip() - with open(_HOST_FILE, "r") as host_file: - for line in host_file: - if line.strip() == host_entry: - break - else: - with open(_HOST_FILE, "a") as host_file_append: - host_file_append.write("\n" # Just in case it wasn't newline terminated. - + host_entry - + "\n") - Connection.API_BASE_URL = os.getenv("KAGGLE_DATA_PROXY_URL") Connection._EXTRA_HEADERS["X-KAGGLE-PROXY-DATA"] = kaggle_proxy_token From a8cb9a1ce2377de2f9daa4292fa025e449fabd8a Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Fri, 6 Apr 2018 20:56:56 +0000 Subject: [PATCH 0309/1660] Apply new slack message format to docker-python's pipeline. --- Jenkinsfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b60442b8..b288763d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,14 +12,14 @@ pipeline { GIT_COMMIT_SHORT = sh(returnStdout: true, script:"git rev-parse --short=7 HEAD").trim() GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim() GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim() - GIT_COMMIT_SUMMARY = "${GIT_BRANCH} `` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" + GIT_COMMIT_SUMMARY = "`` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernels\"; else echo \"#builds\"; fi").trim() } stages { stage('Docker Build') { steps { - slackSend color: 'none', message: "*<${env.BUILD_URL}console|docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash set -exo pipefail @@ -30,7 +30,7 @@ pipeline { stage('Test Image') { steps { - slackSend color: 'none', message: "*<${env.BUILD_URL}console|test image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} test image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash set -exo pipefail @@ -42,7 +42,7 @@ pipeline { stage('Push Image') { steps { - slackSend color: 'none', message: "*<${env.BUILD_URL}console|pushing image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} pushing image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash set -exo pipefail @@ -55,13 +55,13 @@ pipeline { post { failure { - slackSend color: 'danger', message: "*<${env.BUILD_URL}console|failed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + slackSend color: 'danger', message: "*<${env.BUILD_URL}console|${JOB_NAME} failed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL } success { - slackSend color: 'good', message: "*<${env.BUILD_URL}console|passed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + slackSend color: 'good', message: "*<${env.BUILD_URL}console|${JOB_NAME} passed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL } aborted { - slackSend color: 'warning', message: "*<${env.BUILD_URL}console|aborted>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + slackSend color: 'warning', message: "*<${env.BUILD_URL}console|${JOB_NAME} aborted>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL } } } From ee2ac28961b96cfe945fc6348901af3a05b0d0b9 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 6 Apr 2018 17:46:36 -0700 Subject: [PATCH 0310/1660] Updating Dockerfile to install ray Adds support for `ray` which makes it easy to do parallel computation. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5591e1ba..0b909477 100644 --- a/Dockerfile +++ b/Dockerfile @@ -467,6 +467,7 @@ RUN pip install flashtext && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ pip install hpsklearn && \ pip install git+https://github.com/Kaggle/learntools && \ + pip install ray && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From e6beca0598c962e52ede533f9de4924daf852f6d Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Fri, 13 Apr 2018 19:06:05 +0000 Subject: [PATCH 0311/1660] Install openCV from pip instead of source --- Dockerfile | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5591e1ba..02507746 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,21 +34,24 @@ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files - cd ../ && rm -rf ImageMagick* && \ - apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ - apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \ - apt-get -y install cmake && \ - cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \ - cd opencv && \ - mkdir build && cd build && \ - cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D BUILD_PNG=TRUE .. && \ - make -j $(nproc) && make install && \ - echo "/usr/local/lib/python3.6/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ - cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \ + cd ../ && rm -rf ImageMagick* + +# OpenCV install (from pip or source) +RUN pip install opencv-python + #apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ + #apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \ + #apt-get -y install cmake && \ + #cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \ + #cd opencv && \ + #mkdir build && cd build && \ + #cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D BUILD_PNG=TRUE .. && \ + #make -j $(nproc) && make install && \ + #echo "/usr/local/lib/python3.6/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ + #cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \ # Clean up install cruft - rm -rf /usr/local/src/opencv && \ - rm -rf /root/.cache/pip/* && \ - apt-get autoremove -y && apt-get clean + #rm -rf /usr/local/src/opencv && \ + #rm -rf /root/.cache/pip/* && \ + #apt-get autoremove -y && apt-get clean # Tensorflow source build RUN apt-get install -y python-software-properties zip && \ From e7f7dc51562c3413236ae8e6ae97cd73389c794d Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Fri, 13 Apr 2018 19:52:46 +0000 Subject: [PATCH 0312/1660] Temporary fix for pyfasttext - cysignals 1.7.0 causes build to fail --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 02507746..e548e69e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -364,7 +364,10 @@ RUN pip install --upgrade mpld3 && \ pip install cufflinks && \ pip install glmnet_py && \ pip install lime && \ - pip install memory_profiler && \ + pip install memory_profiler + +# Temp fix for pyfasttext (cysignals 1.7.0 causes build to fail) +RUN pip install --upgrade cysignals==1.6.9 && \ pip install pyfasttext && \ pip install ktext && \ cd /usr/local/src && git clone --depth=1 https://github.com/facebookresearch/fastText.git && cd fastText && pip install . && \ From 28259c93e4bdbe2b449837ec4377a27042b9b34c Mon Sep 17 00:00:00 2001 From: Nathaniel Saul Date: Fri, 13 Apr 2018 15:16:58 -0700 Subject: [PATCH 0313/1660] Include Kepler Mapper --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5591e1ba..03227a93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -467,6 +467,7 @@ RUN pip install flashtext && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ pip install hpsklearn && \ pip install git+https://github.com/Kaggle/learntools && \ + pip install kmapper && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 955901aed9100911a182c3b52973d332a066fea5 Mon Sep 17 00:00:00 2001 From: Scott Lundberg Date: Fri, 13 Apr 2018 23:26:07 -0700 Subject: [PATCH 0314/1660] Add shap to Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5591e1ba..8626929e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -467,6 +467,7 @@ RUN pip install flashtext && \ pip install -e git+https://github.com/SohierDane/BigQuery_Helper#egg=bq_helper && \ pip install hpsklearn && \ pip install git+https://github.com/Kaggle/learntools && \ + pip install shap && \ ##### ^^^^ Add new contributions above here # clean up pip cache rm -rf /root/.cache/pip/* From 92d3aa1bba49b02f2b9806c824e9bdd42bc366d8 Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Tue, 17 Apr 2018 19:36:09 +0000 Subject: [PATCH 0315/1660] Fix pyfasttext opencv, tensorflow --- Dockerfile | 16 +++++++++------- test_build.py | 30 +++++++++++++----------------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index e548e69e..fde4d44c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,11 +63,12 @@ RUN apt-get install -y python-software-properties zip && \ echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ apt-get update && apt-get install -y bazel && apt-get upgrade -y bazel && \ - cd /usr/local/src && git clone https://github.com/tensorflow/tensorflow && \ - cd tensorflow && echo "\n" | ./configure && \ - bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ - bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ - pip install /tmp/tensorflow_pkg/tensorflow*.whl + pip install tensorflow + #cd /usr/local/src && git clone https://github.com/tensorflow/tensorflow && \ + #cd tensorflow && echo "\n" | ./configure && \ + #bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ + #bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ + #pip install /tmp/tensorflow_pkg/tensorflow*.whl RUN apt-get install -y libfreetype6-dev && \ apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ @@ -366,8 +367,9 @@ RUN pip install --upgrade mpld3 && \ pip install lime && \ pip install memory_profiler -# Temp fix for pyfasttext (cysignals 1.7.0 causes build to fail) -RUN pip install --upgrade cysignals==1.6.9 && \ +# install cython & cysignals before pyfasttext +RUN pip install --upgrade cython && \ + pip install --upgrade cysignals && \ pip install pyfasttext && \ pip install ktext && \ cd /usr/local/src && git clone --depth=1 https://github.com/facebookresearch/fastText.git && cd fastText && pip install . && \ diff --git a/test_build.py b/test_build.py index bc70c2b0..64830189 100644 --- a/test_build.py +++ b/test_build.py @@ -55,24 +55,27 @@ import theano print("Theano ok") -from keras.models import Sequential -from keras.layers.core import Dense, Dropout, Activation, Flatten -from keras.layers.convolutional import Convolution2D, MaxPooling2D -from keras.optimizers import SGD -print("keras ok") - import nltk from nltk.stem import WordNetLemmatizer print("nltk ok") +import cv2 +img = cv2.imread('plot1.png',0) +print("OpenCV ok") + import tensorflow as tf +print(tf.__version__) hello = tf.constant('TensorFlow ok') sess = tf.Session() print(sess.run(hello)) +print("Tensorflow ok") + +from keras.models import Sequential +from keras.layers.core import Dense, Dropout, Activation, Flatten +from keras.layers.convolutional import Convolution2D, MaxPooling2D +from keras.optimizers import SGD +print("Keras ok") -import cv2 -img = cv2.imread('plot1.png',0) -print("OpenCV ok") from skimage.io import imread print("skimage ok") @@ -110,18 +113,16 @@ from fastai.io import get_data print("fast.ai ok") +# Test BigQuery import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer from google.cloud import bigquery - HOSTNAME = "127.0.0.1" PORT = 8000 URL = "http://%s:%s" % (HOSTNAME, PORT) - fake_bq_called = False fake_bq_header_found = False - class HTTPHandler(BaseHTTPRequestHandler): def do_HEAD(s): s.send_response(200) @@ -135,18 +136,13 @@ def do_GET(s): httpd = HTTPServer((HOSTNAME, PORT), HTTPHandler) threading.Thread(target=httpd.serve_forever).start() - client = bigquery.Client() - try: for ds in client.list_datasets(): pass except: pass - httpd.shutdown() - assert fake_bq_called, "Fake server did not recieve a request from the BQ client." assert fake_bq_header_found, "X-KAGGLE-PROXY-DATA header was missing from the BQ request." - print("bigquery proxy ok") From bd070a7285ce8b1d4b489c6b730615a7b8a00b38 Mon Sep 17 00:00:00 2001 From: crawforc3 Date: Thu, 19 Apr 2018 14:45:28 -0700 Subject: [PATCH 0316/1660] Move Tensorflow and PyTorch to the beginning --- test_build.py | 56 +++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/test_build.py b/test_build.py index 64830189..6a08f588 100644 --- a/test_build.py +++ b/test_build.py @@ -2,6 +2,33 @@ # kaggle/python container. It checks that all our most popular packages can # be loaded and used without errors. +import tensorflow as tf +print(tf.__version__) +hello = tf.constant('TensorFlow ok') +sess = tf.Session() +print(sess.run(hello)) +print("Tensorflow ok") + +from keras.models import Sequential +from keras.layers.core import Dense, Dropout, Activation, Flatten +from keras.layers.convolutional import Convolution2D, MaxPooling2D +from keras.optimizers import SGD +print("Keras ok") + +# PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html +import torch +import torch.nn as tnn +import torch.autograd as autograd +torch.manual_seed(31337) +linear_torch = tnn.Linear(5,3) +data_torch = autograd.Variable(torch.randn(2, 5)) +print(linear_torch(data_torch)) +print("PyTorch ok") + +import fastai +from fastai.io import get_data +print("fast.ai ok") + import numpy as np print("Numpy imported ok") print("Your lucky number is: " + str(np.random.randint(100))) @@ -63,20 +90,6 @@ img = cv2.imread('plot1.png',0) print("OpenCV ok") -import tensorflow as tf -print(tf.__version__) -hello = tf.constant('TensorFlow ok') -sess = tf.Session() -print(sess.run(hello)) -print("Tensorflow ok") - -from keras.models import Sequential -from keras.layers.core import Dense, Dropout, Activation, Flatten -from keras.layers.convolutional import Convolution2D, MaxPooling2D -from keras.optimizers import SGD -print("Keras ok") - - from skimage.io import imread print("skimage ok") @@ -99,20 +112,6 @@ import seaborn print("seaborn ok") -# PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html -import torch -import torch.nn as tnn -import torch.autograd as autograd -torch.manual_seed(31337) -linear_torch = tnn.Linear(5,3) -data_torch = autograd.Variable(torch.randn(2, 5)) -print(linear_torch(data_torch)) -print("PyTorch ok") - -import fastai -from fastai.io import get_data -print("fast.ai ok") - # Test BigQuery import os import threading @@ -145,4 +144,3 @@ def do_GET(s): assert fake_bq_called, "Fake server did not recieve a request from the BQ client." assert fake_bq_header_found, "X-KAGGLE-PROXY-DATA header was missing from the BQ request." print("bigquery proxy ok") - From 78b9abeae2c8a821fb9e7943ba929a35f1838a5c Mon Sep 17 00:00:00 2001 From: crawforc3 Date: Thu, 19 Apr 2018 14:54:01 -0700 Subject: [PATCH 0317/1660] Update Dockerfile --- Dockerfile | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 754a450a..d36e47af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,22 +53,27 @@ RUN pip install opencv-python #rm -rf /root/.cache/pip/* && \ #apt-get autoremove -y && apt-get clean -# Tensorflow source build -RUN apt-get install -y python-software-properties zip && \ - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ +RUN apt-get update && apt-get install -y python-software-properties zip && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ apt-get update && \ echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ apt-get install -y oracle-java8-installer && \ echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ - apt-get update && apt-get install -y bazel && apt-get upgrade -y bazel && \ - pip install tensorflow - #cd /usr/local/src && git clone https://github.com/tensorflow/tensorflow && \ - #cd tensorflow && echo "\n" | ./configure && \ - #bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ - #bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ - #pip install /tmp/tensorflow_pkg/tensorflow*.whl + apt-get update && apt-get install -y bazel && \ + apt-get upgrade -y bazel + +# Tensorflow source build +RUN cd /usr/local/src && \ + git clone https://github.com/tensorflow/tensorflow && \ + cd tensorflow && \ + cat /dev/null | ./configure && \ + bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ + pip install /tmp/tensorflow_pkg/tensorflow*.whl RUN apt-get install -y libfreetype6-dev && \ apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ From 0cd7f84f94fede90dd2f1715076430dff77aa89b Mon Sep 17 00:00:00 2001 From: Scott Lundberg Date: Thu, 19 Apr 2018 15:14:22 -0700 Subject: [PATCH 0318/1660] Added import test --- test_build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_build.py b/test_build.py index 64830189..8a466b4c 100644 --- a/test_build.py +++ b/test_build.py @@ -146,3 +146,6 @@ def do_GET(s): assert fake_bq_header_found, "X-KAGGLE-PROXY-DATA header was missing from the BQ request." print("bigquery proxy ok") +import shap +print("shap ok") + From 9673bcc21fa9a73c69782864a5a07a307acd556e Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Thu, 26 Apr 2018 21:50:09 +0000 Subject: [PATCH 0319/1660] Add conda install bokeh --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index e63ed888..0887a25d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -220,6 +220,7 @@ RUN apt-get update && \ pip install line_profiler && \ pip install orderedmultidict && \ pip install smhasher && \ + conda install -y -c bokeh bokeh && \ conda install -y -c bokeh datashader && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ From 546da7d62071639e5d6814658133276e75d47eab Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Fri, 27 Apr 2018 22:17:36 +0000 Subject: [PATCH 0320/1660] Add exports to PyTorch --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0887a25d..a5bf1d4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -237,6 +237,8 @@ RUN apt-get update && \ #./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ #./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ # PyTorch + export CXXFLAGS="-std=c++11" && \ + export CFLAGS="-std=c99" && \ conda install -y pytorch torchvision -c pytorch && \ # PyTorch Audio apt-get install -y sox libsox-dev libsox-fmt-all && \ From b03166575df7bd8e8ef303a97858f8a1c7e39fe9 Mon Sep 17 00:00:00 2001 From: Nathaniel Saul Date: Sun, 29 Apr 2018 14:12:30 -0700 Subject: [PATCH 0321/1660] adding kmapper test in build --- test_build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_build.py b/test_build.py index a238bdce..e47961fb 100644 --- a/test_build.py +++ b/test_build.py @@ -148,3 +148,5 @@ def do_GET(s): import shap print("shap ok") +import kmapper +print("kmapper ok") \ No newline at end of file From a80dc5b11457407dfdf0e22a0f1c0027f225e49d Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Fri, 4 May 2018 22:16:22 +0000 Subject: [PATCH 0322/1660] s/#kernels/#kernelops/g --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b288763d..5db817d0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ pipeline { GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim() GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim() GIT_COMMIT_SUMMARY = "`` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" - SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernels\"; else echo \"#builds\"; fi").trim() + SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernelops\"; else echo \"#builds\"; fi").trim() } stages { From cbc396ff90e8f6ad2b533ba1d309f70f55aa42bb Mon Sep 17 00:00:00 2001 From: Dan Becker Date: Thu, 10 May 2018 13:50:03 -0600 Subject: [PATCH 0323/1660] Add gym and tensorforce --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fedb0cd0..192ee289 100644 --- a/Dockerfile +++ b/Dockerfile @@ -490,7 +490,8 @@ RUN pip install flashtext && \ pip install kmapper && \ pip install shap && \ pip install ray && \ - + pip install gym && \ + pip install tensorforce && \ ##### ^^^^ Add new contributions above here ^^^^ ##### From e270a76a212e52966c803e9b02211961aee18b83 Mon Sep 17 00:00:00 2001 From: Seb Boving Date: Fri, 11 May 2018 09:41:37 -0700 Subject: [PATCH 0324/1660] Update README.md with build/test instructions. --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5809c317..402eda9c 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,21 @@ Type "help", "copyright", "credits" or "license" for more information. >>> ``` -Once that's working, add the necessary lines to our [Dockerfile](https://github.com/Kaggle/docker-python/blob/master/Dockerfile). (In this case, that would mean adding `pip install coolpackage` to the last section.) Then submit your pull request, and you're all set! +Once that's working, add the necessary lines to our [Dockerfile](https://github.com/Kaggle/docker-python/blob/master/Dockerfile). (In this case, that would mean adding `pip install coolpackage` to the last section.) +Next run the build: + +``` +./build +``` + +Finally run the tests: + +``` +./tests +``` + +Then submit your pull request, and you're all set! From f23bb487e99e16bc6c76d03cf5c0abd87636c449 Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Thu, 17 May 2018 16:05:35 +0000 Subject: [PATCH 0325/1660] Upgrade pip, add --no-dependencies flag to kmeans-smote and kmodes --- Dockerfile | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index fedb0cd0..4192c776 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,13 +9,14 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & apt-get update && apt-get install -y build-essential && \ # https://stackoverflow.com/a/46498173 conda update -y conda && conda update -y python && \ + pip install --upgrade pip && \ # Vowpal Rabbit #apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ #cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ #ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ #pip install vowpalwabbit && \ # Anaconda's scipy is currently behind the main release (1.0) - pip install scipy --upgrade && \ + #pip install scipy --upgrade && \ pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ husl geopy ml_metrics mne pyshp gensim && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ @@ -199,8 +200,11 @@ RUN apt-get update && \ # Re-run it to flush any more disk writes python -c "from keras.models import Sequential; from keras import backend; print(backend._BACKEND)" && \ # Keras reverts to /tmp from ~ when it detects a read-only file system - mkdir -p /tmp/.keras && cp /root/.keras/keras.json /tmp/.keras && \ - # Scikit-Learn nightly build + mkdir -p /tmp/.keras && cp /root/.keras/keras.json /tmp/.keras + + # scikit-learn dependencies +RUN pip install scipy && \ + # Scikit-Learn pinned to 0.19.X until 0.20.0 (many packages break with scikitlearn 0.20.0dev) cd /usr/local/src && git clone https://github.com/scikit-learn/scikit-learn.git && \ cd scikit-learn && python setup.py build && python setup.py install && \ # HDF5 support @@ -354,8 +358,9 @@ RUN pip install --upgrade mpld3 && \ pip install geoplot && \ pip install eli5 && \ pip install implicit && \ - pip install dask-ml[xgboost] && \ - pip install kmeans-smote && \ + pip install dask-ml[xgboost] + +RUN pip install kmeans-smote --no-dependencies && \ # Add google PAIR-code Facets cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ @@ -364,7 +369,7 @@ RUN pip install --upgrade mpld3 && \ pip install --upgrade --ignore-installed setuptools && pip install --no-cache-dir git+git://github.com/ppwwyyxx/tensorpack && \ pip install pycountry && pip install iso3166 && \ pip install pydash && \ - pip install kmodes && \ + pip install kmodes --no-dependencies && \ pip install librosa && \ pip install polyglot && \ pip install mmh3 && \ @@ -402,10 +407,11 @@ RUN pip install --upgrade cython && \ # yellowbrick machine learning visualization library pip install yellowbrick && \ pip install mlcrate && \ - # clean up pip cache - rm -rf /root/.cache/pip/* && \ # Required to display Altair charts in Jupyter notebook - jupyter nbextension install --user --py vega + pip install vega3 && \ + jupyter nbextension install --sys-prefix --py vega3 && \ + # clean up pip cache + rm -rf /root/.cache/pip/* # Fast.ai and dependencies RUN pip install bcolz && \ @@ -451,7 +457,6 @@ RUN pip install bcolz && \ pip install PyYAML && \ pip install pyzmq && \ pip install qtconsole && \ - pip install scipy && \ pip install seaborn && \ pip install simplegeneric && \ pip install six && \ @@ -490,11 +495,7 @@ RUN pip install flashtext && \ pip install kmapper && \ pip install shap && \ pip install ray && \ - ##### ^^^^ Add new contributions above here ^^^^ ##### - - - # clean up pip cache rm -rf /root/.cache/pip/* From b3973c71751f66ab1ec04f2af00f983d71ef60cc Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Fri, 18 May 2018 00:16:38 +0000 Subject: [PATCH 0326/1660] install basemap from conda, fancyimpute from source, remove ggplot --- Dockerfile | 35 +++++++++++++++++++++-------------- test_build.py | 5 +---- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4192c776..4a74720b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & # https://stackoverflow.com/a/46498173 conda update -y conda && conda update -y python && \ pip install --upgrade pip && \ + apt-get -y install cmake && \ # Vowpal Rabbit #apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ #cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ @@ -41,7 +42,6 @@ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ RUN pip install opencv-python #apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ #apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \ - #apt-get -y install cmake && \ #cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \ #cd opencv && \ #mkdir build && cd build && \ @@ -154,18 +154,24 @@ vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Make sure the dynamic linker finds the right libstdc++ ENV LD_LIBRARY_PATH=/opt/conda/lib +# Install Basemap via conda temporarily RUN apt-get update && \ - # Libgeos, for mapping libraries - apt-get -y install libgeos-dev && \ - # pyshp and pyproj are now external dependencies of Basemap - pip install pyshp pyproj && \ - cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \ - export GEOS_DIR=/usr/local && \ - cd basemap && python setup.py install && \ + #apt-get -y install libgeos-dev && \ + #pip install matplotlib && \ + #pip install pyshp && \ + #pip install pyproj && \ + #cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \ + #cd basemap/geos-3.3.3 && \ + #export GEOS_DIR=/usr/local && \ + #./configure --prefix=$GEOS_DIR && \ + #make && make install && \ + #cd .. && python setup.py install && \ + conda install basemap && \ # Pillow (PIL) apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev && \ - pip install Pillow && \ - cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep.git && \ + pip install Pillow + +RUN cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep.git && \ cd opendeep && python setup.py develop && \ # sasl is apparently an ibis dependency apt-get -y install libsasl2-dev && \ @@ -248,8 +254,6 @@ RUN pip install scipy && \ apt-get install -y sox libsox-dev libsox-fmt-all && \ pip install cffi && \ cd /usr/local/src && git clone https://github.com/pytorch/audio && cd audio && python setup.py install && \ - # ggpy / ggplot - pip install git+https://github.com/yhat/ggplot.git && \ # ~~~~ CLEAN UP ~~~~ rm -rf /root/.cache/pip/* && \ apt-get autoremove -y && apt-get clean && \ @@ -305,8 +309,11 @@ RUN pip install --upgrade mpld3 && \ pip install pystan && \ pip install ImageHash && \ conda install -y ecos && \ - conda install -y CVXcanon && \ - pip install fancyimpute && \ + conda install -y CVXcanon + +RUN cd /usr/local/src && git clone https://github.com/iskandr/fancyimpute && \ + cd fancyimpute && \ + python setup.py install && \ pip install git+https://github.com/pymc-devs/pymc3 && \ pip install tifffile && \ pip install spectral && \ diff --git a/test_build.py b/test_build.py index e47961fb..86bb0a7d 100644 --- a/test_build.py +++ b/test_build.py @@ -76,9 +76,6 @@ import plotly.graph_objs as go print("plotly ok") -from ggplot import * -print("ggplot ok") - import theano print("Theano ok") @@ -149,4 +146,4 @@ def do_GET(s): print("shap ok") import kmapper -print("kmapper ok") \ No newline at end of file +print("kmapper ok") From cfb46faf9e8a49c4205e5b98ba2edcb21bda46ba Mon Sep 17 00:00:00 2001 From: crawforc3 Date: Fri, 18 May 2018 11:17:25 -0700 Subject: [PATCH 0327/1660] Add tensorflow to import before other pacakges --- patches/sitecustomize.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 9c6f566a..f0dfab45 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -1,7 +1,10 @@ # Monkey patches BigQuery client creation to use proxy. -# Import torch before anything else. This is a hacky workaround to an error on dlopen -# reporting a limit on static TLS, tracked in https://github.com/pytorch/pytorch/issues/2575 +# Import tensorflow and torch before anything else. This is a hacky workaround to an error on dlopen +# reporting a limit on static TLS, tracked in: +# tensorflow: https://github.com/tensorflow/tensorflow/issues/19010 +# torch: https://github.com/pytorch/pytorch/issues/2575 +import tensorflow import torch import os From bc6d76c674db19d62c0c8fb753d6be70cee48a42 Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Fri, 18 May 2018 21:50:42 +0000 Subject: [PATCH 0328/1660] Add pyarabic, conx, pandasql, trackml --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4a74720b..9e994426 100644 --- a/Dockerfile +++ b/Dockerfile @@ -502,6 +502,10 @@ RUN pip install flashtext && \ pip install kmapper && \ pip install shap && \ pip install ray && \ + pip install pyarabic && \ + pip install conx && \ + pip install pandasql && \ + pip install trackml && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From 2891925ede1b7796fb04f63b063ca52fc25836db Mon Sep 17 00:00:00 2001 From: mlisovyi Date: Mon, 28 May 2018 09:24:55 +0200 Subject: [PATCH 0329/1660] add vw build --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 9e994426..7987fd64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -506,6 +506,7 @@ RUN pip install flashtext && \ pip install conx && \ pip install pandasql && \ pip install trackml && \ + cd /usr/local/src && git clone https://github.com/JohnLangford/vowpal_wabbit.git && ./vowpal_wabbit/python/conda_install.sh && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From d92c5a0e351d443efee0d7b4931ff88033b13e33 Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Wed, 13 Jun 2018 20:04:54 +0000 Subject: [PATCH 0330/1660] Upgrade numpy with pip --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9e994426..c4a93a97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -189,7 +189,9 @@ RUN cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep. pip install mxnet && \ # h2o # Temporary sync of conda's numpy with pip's, needed to avoid an install error - conda upgrade -y numpy && \ + #conda upgrade -y numpy && \ + # Upgrade numpy with pip to avoid install errors + pip install --upgrade numpy && \ # This requires python-software-properties and Java, which were installed above. cd /usr/local/src && mkdir h2o && cd h2o && \ wget http://h2o-release.s3.amazonaws.com/h2o/latest_stable -O latest && \ From c8985b357b50a0c84fd8f04134ef13f9824bf26a Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Tue, 26 Jun 2018 18:19:15 +0000 Subject: [PATCH 0331/1660] add vowpalwabbit test --- test_build.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_build.py b/test_build.py index 86bb0a7d..db2ef2f8 100644 --- a/test_build.py +++ b/test_build.py @@ -147,3 +147,10 @@ def do_GET(s): import kmapper print("kmapper ok") + +from vowpalwabbit import pyvw +vw = pyvw.vw(quiet=True) +ex = vw.example('1 | a b c') +vw.learn(ex) +vw.predict(ex) +print('vowpalwabbit ok') From 04ade60108088f69e47fd49caca571b5532d471c Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Mon, 9 Jul 2018 19:44:00 +0000 Subject: [PATCH 0332/1660] Add Essentia and test --- Dockerfile | 4 +--- test_build.py | 6 +++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd635250..37e165a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -245,9 +245,7 @@ RUN pip install scipy && \ apt-get install -y pandoc && \ cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \ cd py-earth && python setup.py install && \ - #cd /usr/local/src && git clone https://github.com/MTG/essentia.git && cd essentia && \ - #./waf configure --mode=release --build-static --with-python --with-cpptests --with-examples --with-vamp && \ - #./waf && ./waf install && mv /usr/local/lib/python3.6/site-packages/essentia /opt/conda/lib/python3.6 && \ + pip install essentia && \ # PyTorch export CXXFLAGS="-std=c++11" && \ export CFLAGS="-std=c99" && \ diff --git a/test_build.py b/test_build.py index db2ef2f8..416e3c25 100644 --- a/test_build.py +++ b/test_build.py @@ -152,5 +152,9 @@ def do_GET(s): vw = pyvw.vw(quiet=True) ex = vw.example('1 | a b c') vw.learn(ex) -vw.predict(ex) +print(vw.predict(ex)) print('vowpalwabbit ok') + +import essentia +print(essentia.__version__) +print("Essentia ok") From 332d8703dd6d7cb19a46482b3f8e26a2fe0014d6 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 11 Jul 2018 11:27:15 -0700 Subject: [PATCH 0333/1660] Fix typo in github repo name --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5db817d0..c02627a8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ pipeline { GIT_COMMIT_SHORT = sh(returnStdout: true, script:"git rev-parse --short=7 HEAD").trim() GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim() GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim() - GIT_COMMIT_SUMMARY = "`` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" + GIT_COMMIT_SUMMARY = "`` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernelops\"; else echo \"#builds\"; fi").trim() } From bf0ba53d7d189a39a9407768174e6be40efff4e6 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 19 Jul 2018 16:03:20 -0700 Subject: [PATCH 0334/1660] Run jenkins pipeline on ephemeral worker --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c02627a8..a3bc0ea7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1-5" : "" pipeline { - agent { label 'linux && !gpu' } + agent { label 'ephemeral-linux && !gpu' } options { disableConcurrentBuilds() } From ef2b1a39a956653ef9ba9db591061153b7ef0d08 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 19 Jul 2018 17:59:12 -0700 Subject: [PATCH 0335/1660] remove not gpu tag --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a3bc0ea7..cef5a0b4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1-5" : "" pipeline { - agent { label 'ephemeral-linux && !gpu' } + agent { label 'ephemeral-linux' } options { disableConcurrentBuilds() } From ac7b78cdcc9e8799adeef950984f018326eb67b0 Mon Sep 17 00:00:00 2001 From: d1jang Date: Wed, 25 Jul 2018 12:22:12 -0700 Subject: [PATCH 0336/1660] Patch .bashrc to set up pip to enable pip install Create a directory for pip to install modules via users' "pip install" command. Set up PIP_CONFIG_FILE & PYTHONPATH to use those modeuls in a Kernel session. Currently this change is no-op until we populate KAGGLE_WORKING_DIR from the worker. --- Dockerfile | 5 +++++ patches/kaggle_bashrc | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 patches/kaggle_bashrc diff --git a/Dockerfile b/Dockerfile index 37e165a0..dc5118e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -535,6 +535,11 @@ ADD patches/sitecustomize.py /root/.local/lib/python3.6/site-packages/sitecustom # Set backend for matplotlib ENV MPLBACKEND "agg" +# Set up pip to enable pip install. +ADD patches/kaggle_bashrc /root/.bashrc +# Patch the system-wide bashrc file for non-root users. +RUN cat /root/.bashrc >> /etc/bash.bashrc + # Finally, apply any locally defined patches. RUN /bin/bash -c \ "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" diff --git a/patches/kaggle_bashrc b/patches/kaggle_bashrc new file mode 100644 index 00000000..b19e2748 --- /dev/null +++ b/patches/kaggle_bashrc @@ -0,0 +1,31 @@ + +# Kaggle-specific .bashrc script to be appended to /etc/bash.bashrc to apply it +# when any user session starts. +# This script sets up pip to enable a user to install and use python modules via +# pip intall. $KAGGLE_WORKING_DIR should be available for it to work. + +if [[ ! -z "${KAGGLE_WORKING_DIR}" ]]; then + PIP_INSTALL_PREFIX_DIR="${KAGGLE_WORKING_DIR}/pip" + PIP_CONFIG_FILE_PATH="${KAGGLE_WORKING_DIR}/config/pip/pip.conf" + + # Create a directory for pip to install modules. + mkdir -p ${PIP_INSTALL_PREFIX_DIR} + + # Create pip config file to use the prefix directory created above for + # installation. Also, ignore-installed is set to true to prevent pip + # from trying to remove existing modules from the read-only filesystem. + mkdir -p `dirname ${PIP_CONFIG_FILE_PATH}` + echo "[install]\nprefix=${PIP_INSTALL_PREFIX_DIR}\nignore-installed=true" > ${PIP_CONFIG_FILE_PATH} + # Instruct pip to use this config file. + export PIP_CONFIG_FILE=${PIP_CONFIG_FILE_PATH} + + # Set up PYTHONPATH correctly to include the user-installed library. + # Note that the pip prefix directory overrides the system default to enable + # a user to use his/her installed one. + # TODO(dsjang): Currently "lib/python3.6/site-packages" is hard-coded + # throughout Dockerfile. Parameterize it to avoid a version mismatch. + export PYTHONPATH=${PIP_INSTALL_PREFIX_DIR}/lib/python3.6/site-packages:${PYTHONPATH} + + # Include pip-installed binaries in PATH. + export PATH=${PATH}:${PIP_INSTALL_PREFIX_DIR}/bin +fi \ No newline at end of file From ba2ae6924ceeaac55c028414f97e6426bc62c2c6 Mon Sep 17 00:00:00 2001 From: d1jang Date: Wed, 25 Jul 2018 12:49:38 -0700 Subject: [PATCH 0337/1660] Remove root/.bashrc. --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc5118e0..9f241b5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -536,9 +536,10 @@ ADD patches/sitecustomize.py /root/.local/lib/python3.6/site-packages/sitecustom ENV MPLBACKEND "agg" # Set up pip to enable pip install. -ADD patches/kaggle_bashrc /root/.bashrc +ADD patches/kaggle_bashrc /root # Patch the system-wide bashrc file for non-root users. -RUN cat /root/.bashrc >> /etc/bash.bashrc +RUN cat /root/kaggle_bashrc >> /etc/bash.bashrc +RUN rm /root/kaggle_bashrc # Finally, apply any locally defined patches. RUN /bin/bash -c \ From 80fe4ad542feea446815f50041cc4d31e436fd04 Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Tue, 31 Jul 2018 22:35:51 +0000 Subject: [PATCH 0338/1660] fix geopandas, pin vowpalwabbit, python3.6.6 --- Dockerfile | 76 +++++++++++++++++++++++------------------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9f241b5d..cbbec9f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,27 +10,21 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & # https://stackoverflow.com/a/46498173 conda update -y conda && conda update -y python && \ pip install --upgrade pip && \ - apt-get -y install cmake && \ - # Vowpal Rabbit - #apt-get install -y libboost-program-options-dev zlib1g-dev libboost-python-dev && \ - #cd /usr/lib/x86_64-linux-gnu/ && rm -f libboost_python.a && rm -f libboost_python.so && \ - #ln -sf libboost_python-py34.so libboost_python.so && ln -sf libboost_python-py34.a libboost_python.a && \ - #pip install vowpalwabbit && \ - # Anaconda's scipy is currently behind the main release (1.0) - #pip install scipy --upgrade && \ - pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ + apt-get -y install cmake + +RUN pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ husl geopy ml_metrics mne pyshp gensim && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ python -m spacy download en_core_web_lg && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ -libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ -libcups2 libcupsfilters1 libcupsimage2 libdatrie1 libdbus-1-3 libdjvulibre-text libdjvulibre21 libfftw3-double3 libfontconfig1 \ -libfreetype6 libgdk-pixbuf2.0-0 libgdk-pixbuf2.0-common libgomp1 libgraphite2-3 libgs9 libgs9-common libharfbuzz0b libijs-0.35 \ -libilmbase6 libjasper1 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1-0 libltdl7 libmagickcore-6.q16-2 \ -libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ -libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ -libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ + libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ + libcups2 libcupsfilters1 libcupsimage2 libdatrie1 libdbus-1-3 libdjvulibre-text libdjvulibre21 libfftw3-double3 libfontconfig1 \ + libfreetype6 libgdk-pixbuf2.0-0 libgdk-pixbuf2.0-common libgomp1 libgraphite2-3 libgs9 libgs9-common libharfbuzz0b libijs-0.35 \ + libilmbase6 libjasper1 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1-0 libltdl7 libmagickcore-6.q16-2 \ + libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ + libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ + libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ cd /usr/local/src && \ wget http://transloadit.imagemagick.org/download/ImageMagick.tar.gz && \ tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ @@ -40,19 +34,6 @@ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ # OpenCV install (from pip or source) RUN pip install opencv-python - #apt-get -y install libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ - #apt-get -y install libtbb2 libtbb-dev libjpeg-dev libtiff-dev libjasper-dev && \ - #cd /usr/local/src && git clone --depth 1 https://github.com/Itseez/opencv.git && \ - #cd opencv && \ - #mkdir build && cd build && \ - #cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D WITH_TBB=ON -D WITH_FFMPEG=OFF -D WITH_V4L=ON -D WITH_QT=OFF -D WITH_OPENGL=ON -D PYTHON3_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON3_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D PYTHON_LIBRARY=/opt/conda/lib/libpython3.6m.so -D PYTHON_INCLUDE_DIR=/opt/conda/include/python3.6m/ -D BUILD_PNG=TRUE .. && \ - #make -j $(nproc) && make install && \ - #echo "/usr/local/lib/python3.6/site-packages" > /etc/ld.so.conf.d/opencv.conf && ldconfig && \ - #cp /usr/local/lib/python3.6/site-packages/cv2.cpython-36m-x86_64-linux-gnu.so /opt/conda/lib/python3.6/site-packages/ && \ - # Clean up install cruft - #rm -rf /usr/local/src/opencv && \ - #rm -rf /root/.cache/pip/* && \ - #apt-get autoremove -y && apt-get clean RUN apt-get update && apt-get install -y python-software-properties zip && \ echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ @@ -67,8 +48,11 @@ RUN apt-get update && apt-get install -y python-software-properties zip && \ apt-get update && apt-get install -y bazel && \ apt-get upgrade -y bazel -# Tensorflow source build -RUN cd /usr/local/src && \ +# Tensorflow +# Temp fixes: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2 +RUN conda install -y python=3.6.6 && \ + pip install pandas==0.23.2 && \ + cd /usr/local/src && \ git clone https://github.com/tensorflow/tensorflow && \ cd tensorflow && \ cat /dev/null | ./configure && \ @@ -132,18 +116,18 @@ RUN apt-get install -y libfreetype6-dev && \ # the corpuses that work python -m nltk.downloader -d /usr/share/nltk_data abc alpino averaged_perceptron_tagger \ basque_grammars biocreative_ppi bllip_wsj_no_aux \ -book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \ -comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ -europarl_raw floresta gazetteers genesis gutenberg \ -ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho machado \ -masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ -mte_teip5 names nps_chat omw opinion_lexicon paradigms \ -pil pl196x porter_test ppattach problem_reports product_reviews_1 product_reviews_2 propbank \ -pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor senseval sentence_polarity \ -sentiwordnet shakespeare sinica_treebank smultron snowball_data spanish_grammars \ -state_union stopwords subjectivity swadesh switchboard tagsets timit toolbox treebank \ -twitter_samples udhr2 udhr unicode_samples universal_tagset universal_treebanks_v20 \ -vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ + book_grammars brown brown_tei cess_cat cess_esp chat80 city_database cmudict \ + comtrans conll2000 conll2002 conll2007 crubadan dependency_treebank \ + europarl_raw floresta gazetteers genesis gutenberg \ + ieer inaugural indian jeita kimmo knbc large_grammars lin_thesaurus mac_morpho machado \ + masc_tagged maxent_ne_chunker maxent_treebank_pos_tagger moses_sample movie_reviews \ + mte_teip5 names nps_chat omw opinion_lexicon paradigms \ + pil pl196x porter_test ppattach problem_reports product_reviews_1 product_reviews_2 propbank \ + pros_cons ptb punkt qc reuters rslp rte sample_grammars semcor senseval sentence_polarity \ + sentiwordnet shakespeare sinica_treebank smultron snowball_data spanish_grammars \ + state_union stopwords subjectivity swadesh switchboard tagsets timit toolbox treebank \ + twitter_samples udhr2 udhr unicode_samples universal_tagset universal_treebanks_v20 \ + vader_lexicon verbnet webtext word2vec_sample wordnet wordnet_ic words ycoe && \ # Stop-words pip install stop-words && \ # clean up @@ -339,6 +323,8 @@ RUN cd /usr/local/src && git clone https://github.com/iskandr/fancyimpute && \ pip install stemming && \ conda install -y -c conda-forge fbprophet && \ conda install -y -c conda-forge -c ioam holoviews geoviews && \ +# Temp fix : Fiona is already installed by pip and conda installs another version as a dependency for holoviews + conda uninstall -y fiona && \ pip install hypertools && \ # Nxviz has been causing an installation issue by trying unsuccessfully to remove setuptools. #pip install nxviz && \ @@ -488,6 +474,7 @@ RUN pip install bcolz && \ # of all non-final lines. Thanks! # ########### + RUN pip install flashtext && \ pip install marisa-trie && \ pip install pyemd && \ @@ -506,11 +493,12 @@ RUN pip install flashtext && \ pip install conx && \ pip install pandasql && \ pip install trackml && \ - cd /usr/local/src && git clone https://github.com/JohnLangford/vowpal_wabbit.git && ./vowpal_wabbit/python/conda_install.sh && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* +# Pin Vowpal Wabbit v8.6.0 because 8.6.1 does not build or install successfully +RUN cd /usr/local/src && git clone -b 8.6.0 https://github.com/JohnLangford/vowpal_wabbit.git && ./vowpal_wabbit/python/conda_install.sh # For Facets ENV PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ From 460916383daa1b8c2bd94a48c0169bfc901febf8 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 1 Aug 2018 10:17:31 -0700 Subject: [PATCH 0339/1660] Workaround to build a new image with the latest learn tools --- README.md | 2 +- kaggle_tools_update.Dockerfile | 14 ++++++++++++++ test_build.py | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 kaggle_tools_update.Dockerfile diff --git a/README.md b/README.md index 402eda9c..90faac2e 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Next run the build: Finally run the tests: ``` -./tests +./test ``` Then submit your pull request, and you're all set! diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile new file mode 100644 index 00000000..375f5884 --- /dev/null +++ b/kaggle_tools_update.Dockerfile @@ -0,0 +1,14 @@ +# This Dockerfile is a temporary solution to unblock the +# community team by updating their library latest using +# the latest python image. +# This is needed until we resolved the issues with the +# main build. + +# Usage: +# docker build --rm -t kaggle/python-build -f kaggle_tools_update.Dockerfile . +# ./test +# ./push (if tests are passing) + +FROM gcr.io/kaggle-images/python:latest + +RUN pip install git+https://github.com/Kaggle/learntools diff --git a/test_build.py b/test_build.py index 416e3c25..44bb5861 100644 --- a/test_build.py +++ b/test_build.py @@ -15,6 +15,8 @@ from keras.optimizers import SGD print("Keras ok") +# Test learntools + # PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html import torch import torch.nn as tnn From 29bff4ff33181b447808bce1deecd31cade63d94 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 1 Aug 2018 10:30:36 -0700 Subject: [PATCH 0340/1660] add learntools test --- test_build.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test_build.py b/test_build.py index 44bb5861..5c57afde 100644 --- a/test_build.py +++ b/test_build.py @@ -15,7 +15,11 @@ from keras.optimizers import SGD print("Keras ok") -# Test learntools +# Test Kaggle learntools +from learntools.core import binder; binder.bind(globals()) +from learntools.python.ex1 import * +color="blue" +q0.check() # PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html import torch From 7ac2658b9d5abd7ea98002eb78b6f5080446ed8f Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 1 Aug 2018 10:34:43 -0700 Subject: [PATCH 0341/1660] Add print statement to test --- test_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_build.py b/test_build.py index 5c57afde..52d88f33 100644 --- a/test_build.py +++ b/test_build.py @@ -20,6 +20,7 @@ from learntools.python.ex1 import * color="blue" q0.check() +print("learntools ok") # PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html import torch From c65e1de5c1c17e25798f6495d5ffdce4905eba08 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 1 Aug 2018 11:26:21 -0700 Subject: [PATCH 0342/1660] Addressed comments --- kaggle_tools_update.Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index 375f5884..970fdf9d 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -1,14 +1,14 @@ # This Dockerfile is a temporary solution to unblock the -# community team by updating their library latest using +# Kaggle team by updating their library latest using # the latest python image. # This is needed until we resolved the issues with the # main build. # Usage: -# docker build --rm -t kaggle/python-build -f kaggle_tools_update.Dockerfile . +# docker build --pull --rm -t kaggle/python-build -f kaggle_tools_update.Dockerfile . # ./test # ./push (if tests are passing) FROM gcr.io/kaggle-images/python:latest -RUN pip install git+https://github.com/Kaggle/learntools +RUN pip install --upgrade git+https://github.com/Kaggle/learntools From 2c38297fff17e8331c831798690ca0b6c3efb835 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 1 Aug 2018 15:21:24 -0700 Subject: [PATCH 0343/1660] addressed feedback --- kaggle_tools_update.Dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index 970fdf9d..6dadda9a 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -1,11 +1,14 @@ -# This Dockerfile is a temporary solution to unblock the -# Kaggle team by updating their library latest using -# the latest python image. -# This is needed until we resolved the issues with the -# main build. +# This Dockerfile is a temporary solution until we +# resolved the broken main build. + +# This Dockerfile creates a new image based on our +# current published python image with the latest +# version of the LearnTools library to allow us +# to release new Learn content. # Usage: -# docker build --pull --rm -t kaggle/python-build -f kaggle_tools_update.Dockerfile . +# docker rmi gcr.io/kaggle-images/python:latest +# docker build --rm -t kaggle/python-build -f kaggle_tools_update.Dockerfile . # ./test # ./push (if tests are passing) From 4d4487e6b69a58b915ea0aa3876e5cf393b52328 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 1 Aug 2018 15:56:08 -0700 Subject: [PATCH 0344/1660] setup pip for internet access --- kaggle_tools_update.Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index 6dadda9a..b1bd79db 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -4,7 +4,9 @@ # This Dockerfile creates a new image based on our # current published python image with the latest # version of the LearnTools library to allow us -# to release new Learn content. +# to release new Learn content. It also configures +# pip to work out-of-the-box when internet access +# is enabled. # Usage: # docker rmi gcr.io/kaggle-images/python:latest @@ -15,3 +17,9 @@ FROM gcr.io/kaggle-images/python:latest RUN pip install --upgrade git+https://github.com/Kaggle/learntools + +# Set up pip to enable pip install. +ADD patches/kaggle_bashrc /root +# Patch the system-wide bashrc file for non-root users. +RUN cat /root/kaggle_bashrc >> /etc/bash.bashrc +RUN rm /root/kaggle_bashrc From cddee841c94fe5bb1d953422d91988046ee21590 Mon Sep 17 00:00:00 2001 From: d1jang Date: Thu, 2 Aug 2018 13:06:45 -0700 Subject: [PATCH 0345/1660] Add ENTRYPOINT script to initialize the image. --- Dockerfile | 10 +++++----- kaggle_tools_update.Dockerfile | 10 +++++----- patches/{kaggle_bashrc => entrypoint.sh} | 13 +++++++++---- 3 files changed, 19 insertions(+), 14 deletions(-) rename patches/{kaggle_bashrc => entrypoint.sh} (79%) diff --git a/Dockerfile b/Dockerfile index cbbec9f9..27a7fa89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -523,11 +523,11 @@ ADD patches/sitecustomize.py /root/.local/lib/python3.6/site-packages/sitecustom # Set backend for matplotlib ENV MPLBACKEND "agg" -# Set up pip to enable pip install. -ADD patches/kaggle_bashrc /root -# Patch the system-wide bashrc file for non-root users. -RUN cat /root/kaggle_bashrc >> /etc/bash.bashrc -RUN rm /root/kaggle_bashrc +# Set up an initialization script to be executed before any other commands initiated by the user. +ADD patches/entrypoint.sh /root/entrypoint.sh +RUN chmod +x /root/entrypoint.sh +# This script gets executed by "docker run " and it runs at the end of its execution. +ENTRYPOINT ["/root/entrypoint.sh"] # Finally, apply any locally defined patches. RUN /bin/bash -c \ diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index b1bd79db..79e18812 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -18,8 +18,8 @@ FROM gcr.io/kaggle-images/python:latest RUN pip install --upgrade git+https://github.com/Kaggle/learntools -# Set up pip to enable pip install. -ADD patches/kaggle_bashrc /root -# Patch the system-wide bashrc file for non-root users. -RUN cat /root/kaggle_bashrc >> /etc/bash.bashrc -RUN rm /root/kaggle_bashrc +# Set up an initialization script to be executed before any other commands initiated by the user. +ADD patches/entrypoint.sh /root/entrypoint.sh +RUN chmod +x /root/entrypoint.sh +# This script gets executed by "docker run " and it runs at the end of its execution. +ENTRYPOINT ["/root/entrypoint.sh"] \ No newline at end of file diff --git a/patches/kaggle_bashrc b/patches/entrypoint.sh similarity index 79% rename from patches/kaggle_bashrc rename to patches/entrypoint.sh index b19e2748..98d4d968 100644 --- a/patches/kaggle_bashrc +++ b/patches/entrypoint.sh @@ -1,9 +1,11 @@ +#!/usr/bin/env bash + +# This shell script is executed as an entrypoint file for the Kernels docker image. +# It sets up the execution environment before starting the Jypyter Notebook Server. +# Refer to https://docs.docker.com/engine/reference/builder/#entrypoint for details. -# Kaggle-specific .bashrc script to be appended to /etc/bash.bashrc to apply it -# when any user session starts. # This script sets up pip to enable a user to install and use python modules via # pip intall. $KAGGLE_WORKING_DIR should be available for it to work. - if [[ ! -z "${KAGGLE_WORKING_DIR}" ]]; then PIP_INSTALL_PREFIX_DIR="${KAGGLE_WORKING_DIR}/pip" PIP_CONFIG_FILE_PATH="${KAGGLE_WORKING_DIR}/config/pip/pip.conf" @@ -28,4 +30,7 @@ if [[ ! -z "${KAGGLE_WORKING_DIR}" ]]; then # Include pip-installed binaries in PATH. export PATH=${PATH}:${PIP_INSTALL_PREFIX_DIR}/bin -fi \ No newline at end of file +fi + +# Execute the command provided from "docker run" in the current process. +exec "$@" \ No newline at end of file From 810893186d9adafb0367ecbdb78e8d00042a02a6 Mon Sep 17 00:00:00 2001 From: d1jang Date: Thu, 2 Aug 2018 16:16:13 -0700 Subject: [PATCH 0346/1660] use echo -e to print newlines correctly. --- patches/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patches/entrypoint.sh b/patches/entrypoint.sh index 98d4d968..12c0305c 100644 --- a/patches/entrypoint.sh +++ b/patches/entrypoint.sh @@ -17,7 +17,7 @@ if [[ ! -z "${KAGGLE_WORKING_DIR}" ]]; then # installation. Also, ignore-installed is set to true to prevent pip # from trying to remove existing modules from the read-only filesystem. mkdir -p `dirname ${PIP_CONFIG_FILE_PATH}` - echo "[install]\nprefix=${PIP_INSTALL_PREFIX_DIR}\nignore-installed=true" > ${PIP_CONFIG_FILE_PATH} + echo -e "[install]\nprefix=${PIP_INSTALL_PREFIX_DIR}\nignore-installed=true" > ${PIP_CONFIG_FILE_PATH} # Instruct pip to use this config file. export PIP_CONFIG_FILE=${PIP_CONFIG_FILE_PATH} From b404a22087732795f486e8025ecfc589cc66ecf4 Mon Sep 17 00:00:00 2001 From: d1jang Date: Thu, 2 Aug 2018 16:20:36 -0700 Subject: [PATCH 0347/1660] add comment. --- kaggle_tools_update.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index 79e18812..752c5567 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -18,6 +18,8 @@ FROM gcr.io/kaggle-images/python:latest RUN pip install --upgrade git+https://github.com/Kaggle/learntools +# TODO(dsjang): Remove these lines once the docker image build turns green since they are copied from Dockerfile +# to apply on top of the last green. # Set up an initialization script to be executed before any other commands initiated by the user. ADD patches/entrypoint.sh /root/entrypoint.sh RUN chmod +x /root/entrypoint.sh From dfbe82ee4f83f76742bf0897ec932ea2246720a0 Mon Sep 17 00:00:00 2001 From: d1jang Date: Thu, 2 Aug 2018 17:41:33 -0700 Subject: [PATCH 0348/1660] Add a custom ENTRYPOINT on top of the existing ENTRYPOINT. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a docker run runs with this change, it's running under tini as follows: root@byod-test-x1qm:/home/kaggle_admin/tmp# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 955c97d95ebd kaggle/python-build "/usr/bin/tini -- /r…" 5 minutes ago Up 5 minutes hungry_ritchie root@byod-test-x1qm:/home/kaggle_admin/tmp# --- Dockerfile | 6 ++++-- kaggle_tools_update.Dockerfile | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 27a7fa89..afb36544 100644 --- a/Dockerfile +++ b/Dockerfile @@ -526,8 +526,10 @@ ENV MPLBACKEND "agg" # Set up an initialization script to be executed before any other commands initiated by the user. ADD patches/entrypoint.sh /root/entrypoint.sh RUN chmod +x /root/entrypoint.sh -# This script gets executed by "docker run " and it runs at the end of its execution. -ENTRYPOINT ["/root/entrypoint.sh"] +# This script gets executed by "docker run " and it runs at the end of its execution. +# NOTE: ENTRYPOINT set by "FROM " should preceed the our own custom entrypoint. +# Specifically, tini can be combined with another entrypoint (https://github.com/krallin/tini). +ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] # Finally, apply any locally defined patches. RUN /bin/bash -c \ diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index 752c5567..e6f6dabe 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -23,5 +23,7 @@ RUN pip install --upgrade git+https://github.com/Kaggle/learntools # Set up an initialization script to be executed before any other commands initiated by the user. ADD patches/entrypoint.sh /root/entrypoint.sh RUN chmod +x /root/entrypoint.sh -# This script gets executed by "docker run " and it runs at the end of its execution. -ENTRYPOINT ["/root/entrypoint.sh"] \ No newline at end of file +# This script gets executed by "docker run " and it runs at the end of its execution. +# NOTE: ENTRYPOINT set by "FROM " should preceed the our own custom entrypoint. +# Specifically, tini can be combined with another entrypoint (https://github.com/krallin/tini). +ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] \ No newline at end of file From 6f8d681dbbd2709af002de4d2e72d7f532709957 Mon Sep 17 00:00:00 2001 From: d1jang Date: Thu, 2 Aug 2018 17:50:29 -0700 Subject: [PATCH 0349/1660] Add a safety check for tini. --- Dockerfile | 6 ++++-- kaggle_tools_update.Dockerfile | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index afb36544..fdcb1d9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -526,9 +526,11 @@ ENV MPLBACKEND "agg" # Set up an initialization script to be executed before any other commands initiated by the user. ADD patches/entrypoint.sh /root/entrypoint.sh RUN chmod +x /root/entrypoint.sh -# This script gets executed by "docker run " and it runs at the end of its execution. # NOTE: ENTRYPOINT set by "FROM " should preceed the our own custom entrypoint. -# Specifically, tini can be combined with another entrypoint (https://github.com/krallin/tini). +# Currently tini is set as ENTRYPOINT for the base image, and it can be combined with our entrypoint (https://github.com/krallin/tini). +# ENTRYPOINT gets executed by "docker run " and it runs at the end of its execution. +# Make sure tini exists. +RUN /usr/bin/tini -h ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] # Finally, apply any locally defined patches. diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index e6f6dabe..46f143f4 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -23,7 +23,9 @@ RUN pip install --upgrade git+https://github.com/Kaggle/learntools # Set up an initialization script to be executed before any other commands initiated by the user. ADD patches/entrypoint.sh /root/entrypoint.sh RUN chmod +x /root/entrypoint.sh -# This script gets executed by "docker run " and it runs at the end of its execution. # NOTE: ENTRYPOINT set by "FROM " should preceed the our own custom entrypoint. -# Specifically, tini can be combined with another entrypoint (https://github.com/krallin/tini). +# Currently tini is set as ENTRYPOINT for the base image, and it can be combined with our entrypoint (https://github.com/krallin/tini). +# ENTRYPOINT gets executed by "docker run " and it runs at the end of its execution. +# Make sure tini exists. +RUN /usr/bin/tini -h ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] \ No newline at end of file From 1426423ed4fff17e5dbf31d9d98ee2230232f999 Mon Sep 17 00:00:00 2001 From: d1jang Date: Thu, 2 Aug 2018 17:54:38 -0700 Subject: [PATCH 0350/1660] suppress output from the check. --- Dockerfile | 2 +- kaggle_tools_update.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fdcb1d9c..ac7339b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -530,7 +530,7 @@ RUN chmod +x /root/entrypoint.sh # Currently tini is set as ENTRYPOINT for the base image, and it can be combined with our entrypoint (https://github.com/krallin/tini). # ENTRYPOINT gets executed by "docker run " and it runs at the end of its execution. # Make sure tini exists. -RUN /usr/bin/tini -h +RUN /usr/bin/tini -h > /dev/null ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] # Finally, apply any locally defined patches. diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index 46f143f4..4d9d2b86 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -27,5 +27,5 @@ RUN chmod +x /root/entrypoint.sh # Currently tini is set as ENTRYPOINT for the base image, and it can be combined with our entrypoint (https://github.com/krallin/tini). # ENTRYPOINT gets executed by "docker run " and it runs at the end of its execution. # Make sure tini exists. -RUN /usr/bin/tini -h +RUN /usr/bin/tini -h > /dev/null ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] \ No newline at end of file From a66bcee4c0ddf3415cfcb947d40c1b1485eae393 Mon Sep 17 00:00:00 2001 From: d1jang Date: Fri, 3 Aug 2018 12:19:01 -0700 Subject: [PATCH 0351/1660] Create a symbolic link to pip directory --- patches/entrypoint.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/patches/entrypoint.sh b/patches/entrypoint.sh index 12c0305c..f6f79765 100644 --- a/patches/entrypoint.sh +++ b/patches/entrypoint.sh @@ -8,6 +8,7 @@ # pip intall. $KAGGLE_WORKING_DIR should be available for it to work. if [[ ! -z "${KAGGLE_WORKING_DIR}" ]]; then PIP_INSTALL_PREFIX_DIR="${KAGGLE_WORKING_DIR}/pip" + PIP_INSTALLED_MODULE_DIR="${PIP_INSTALL_PREFIX_DIR}/pip_installed" PIP_CONFIG_FILE_PATH="${KAGGLE_WORKING_DIR}/config/pip/pip.conf" # Create a directory for pip to install modules. @@ -25,8 +26,15 @@ if [[ ! -z "${KAGGLE_WORKING_DIR}" ]]; then # Note that the pip prefix directory overrides the system default to enable # a user to use his/her installed one. # TODO(dsjang): Currently "lib/python3.6/site-packages" is hard-coded - # throughout Dockerfile. Parameterize it to avoid a version mismatch. - export PYTHONPATH=${PIP_INSTALL_PREFIX_DIR}/lib/python3.6/site-packages:${PYTHONPATH} + # throughout Dockerfile. Parameterize it to avoid a version mismatch. + # TODO(dsjang): This is a hack to sidestep a problem of Python ignoring modules + # in .../site-packages other than the system-wide and user-specific site packages. + mkdir -p "${PIP_INSTALL_PREFIX_DIR}/lib/python3.6/site-packages" + ln -s "${PIP_INSTALL_PREFIX_DIR}/lib/python3.6/site-packages" ${PIP_INSTALLED_MODULE_DIR} + export PYTHONPATH=${PIP_INSTALLED_MODULE_DIR}:${PYTHONPATH} + + # Create a symbolic link to site-packages. + # can't override conda installed package. # Include pip-installed binaries in PATH. export PATH=${PATH}:${PIP_INSTALL_PREFIX_DIR}/bin From ab7f1a3e5382720cbdf8d00c45cef685f94ff00e Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 3 Aug 2018 20:13:48 +0000 Subject: [PATCH 0352/1660] Remove tornado patch The patch was released in tornado v4.5.3. We are now pulling v5.0.2 which includes the patch already. --- patches/tornado-fix-issue-2097.patch | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 patches/tornado-fix-issue-2097.patch diff --git a/patches/tornado-fix-issue-2097.patch b/patches/tornado-fix-issue-2097.patch deleted file mode 100644 index f3c7a82f..00000000 --- a/patches/tornado-fix-issue-2097.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff -Naur /base/opt/conda/pkgs/tornado-4.5.1-py36_0/lib/python3.6/site-packages/tornado/http1connection.py /patched/opt.new/conda/lib/python3.6/site-packages/tornado/http1connection.py ---- /base/opt/conda/lib/python3.6/site-packages/tornado/http1connection.py 2017-04-16 23:47:57.000000000 +0000 -+++ /patched/opt.new/conda/lib/python3.6/site-packages/tornado/http1connection.py 2017-06-28 21:18:59.902402220 +0000 -@@ -349,6 +349,11 @@ - # self._request_start_line.version or - # start_line.version? - self._request_start_line.version == 'HTTP/1.1' and -+ # 1xx, 204 and 304 responses have no body (not even a zero-length -+ # body), and so should not have either Content-Length or -+ # Transfer-Encoding headers. -+ start_line.code not in (204, 304) and -+ (start_line.code < 100 or start_line.code >= 200) and - # 304 responses have no body (not even a zero-length body), and so - # should not have either Content-Length or Transfer-Encoding. - # headers. -diff -Naur /base/opt/conda/lib/python3.6/site-packages/tornado/web.py /patched/opt.new/conda/lib/python3.6/site-packages/tornado/web.py ---- /base/opt/conda/lib/python3.6/site-packages/tornado/web.py 2017-06-28 21:22:56.505186743 +0000 -+++ /patched/opt.new/conda/lib/python3.6/site-packages/tornado/web.py 2017-06-28 21:22:48.421228255 +0000 -@@ -977,7 +977,8 @@ - if self._status_code in (204, 304): - assert not self._write_buffer, "Cannot send body with %s" % self._status_code - self._clear_headers_for_304() -- elif "Content-Length" not in self._headers: -+ elif ("Content-Length" not in self._headers and -+ (self._status_code < 100 or self._status_code >= 200)): - content_length = sum(len(part) for part in self._write_buffer) - self.set_header("Content-Length", content_length) From b82e854111bca5316b8027eecbae502b4c078629 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 7 Aug 2018 13:00:44 -0700 Subject: [PATCH 0353/1660] remove patching mecanism --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ac7339b9..95d88360 100644 --- a/Dockerfile +++ b/Dockerfile @@ -532,7 +532,3 @@ RUN chmod +x /root/entrypoint.sh # Make sure tini exists. RUN /usr/bin/tini -h > /dev/null ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] - -# Finally, apply any locally defined patches. -RUN /bin/bash -c \ - "cd / && for p in $(ls /tmp/patches/*.patch); do echo '= Applying patch '\${p}; patch -p2 < \${p}; done" From fc690383de8b501135df21c043f3d4c184f61e75 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 8 Aug 2018 10:18:58 -0700 Subject: [PATCH 0354/1660] Simplify build script by using the --pull flag. With the --pull flag, it always attempt to pull a newer version of the image --- build | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build b/build index 642cc268..d3f34959 100755 --- a/build +++ b/build @@ -6,6 +6,5 @@ set -e if [[ "$1" == "--use-cache" ]]; then docker build --rm -t kaggle/python-build . else - docker pull continuumio/anaconda3:latest - docker build --rm --no-cache -t kaggle/python-build . + docker build --pull --rm --no-cache -t kaggle/python-build . fi From 3b64add9f7d5b29a53d36130b51f7a7502f6f987 Mon Sep 17 00:00:00 2001 From: d1jang Date: Fri, 10 Aug 2018 11:05:27 -0700 Subject: [PATCH 0355/1660] Delete pip install support script & use a manual tag for update dockerfile. --- Dockerfile | 12 +--------- kaggle_tools_update.Dockerfile | 19 ++++----------- patches/entrypoint.sh | 44 ---------------------------------- 3 files changed, 5 insertions(+), 70 deletions(-) delete mode 100644 patches/entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 95d88360..2c8a70b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -521,14 +521,4 @@ ENV PYTHONUSERBASE "/root/.local" ADD patches/sitecustomize.py /root/.local/lib/python3.6/site-packages/sitecustomize.py # Set backend for matplotlib -ENV MPLBACKEND "agg" - -# Set up an initialization script to be executed before any other commands initiated by the user. -ADD patches/entrypoint.sh /root/entrypoint.sh -RUN chmod +x /root/entrypoint.sh -# NOTE: ENTRYPOINT set by "FROM " should preceed the our own custom entrypoint. -# Currently tini is set as ENTRYPOINT for the base image, and it can be combined with our entrypoint (https://github.com/krallin/tini). -# ENTRYPOINT gets executed by "docker run " and it runs at the end of its execution. -# Make sure tini exists. -RUN /usr/bin/tini -h > /dev/null -ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] +ENV MPLBACKEND "agg" \ No newline at end of file diff --git a/kaggle_tools_update.Dockerfile b/kaggle_tools_update.Dockerfile index 4d9d2b86..8f7fac20 100644 --- a/kaggle_tools_update.Dockerfile +++ b/kaggle_tools_update.Dockerfile @@ -9,23 +9,12 @@ # is enabled. # Usage: -# docker rmi gcr.io/kaggle-images/python:latest +# docker rmi gcr.io/kaggle-images/python:pinned # docker build --rm -t kaggle/python-build -f kaggle_tools_update.Dockerfile . # ./test # ./push (if tests are passing) -FROM gcr.io/kaggle-images/python:latest +# Pull the last build manually tagged as "pinned". +FROM gcr.io/kaggle-images/python:pinned -RUN pip install --upgrade git+https://github.com/Kaggle/learntools - -# TODO(dsjang): Remove these lines once the docker image build turns green since they are copied from Dockerfile -# to apply on top of the last green. -# Set up an initialization script to be executed before any other commands initiated by the user. -ADD patches/entrypoint.sh /root/entrypoint.sh -RUN chmod +x /root/entrypoint.sh -# NOTE: ENTRYPOINT set by "FROM " should preceed the our own custom entrypoint. -# Currently tini is set as ENTRYPOINT for the base image, and it can be combined with our entrypoint (https://github.com/krallin/tini). -# ENTRYPOINT gets executed by "docker run " and it runs at the end of its execution. -# Make sure tini exists. -RUN /usr/bin/tini -h > /dev/null -ENTRYPOINT ["/usr/bin/tini", "--", "/root/entrypoint.sh"] \ No newline at end of file +RUN pip install --upgrade git+https://github.com/Kaggle/learntools \ No newline at end of file diff --git a/patches/entrypoint.sh b/patches/entrypoint.sh deleted file mode 100644 index f6f79765..00000000 --- a/patches/entrypoint.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# This shell script is executed as an entrypoint file for the Kernels docker image. -# It sets up the execution environment before starting the Jypyter Notebook Server. -# Refer to https://docs.docker.com/engine/reference/builder/#entrypoint for details. - -# This script sets up pip to enable a user to install and use python modules via -# pip intall. $KAGGLE_WORKING_DIR should be available for it to work. -if [[ ! -z "${KAGGLE_WORKING_DIR}" ]]; then - PIP_INSTALL_PREFIX_DIR="${KAGGLE_WORKING_DIR}/pip" - PIP_INSTALLED_MODULE_DIR="${PIP_INSTALL_PREFIX_DIR}/pip_installed" - PIP_CONFIG_FILE_PATH="${KAGGLE_WORKING_DIR}/config/pip/pip.conf" - - # Create a directory for pip to install modules. - mkdir -p ${PIP_INSTALL_PREFIX_DIR} - - # Create pip config file to use the prefix directory created above for - # installation. Also, ignore-installed is set to true to prevent pip - # from trying to remove existing modules from the read-only filesystem. - mkdir -p `dirname ${PIP_CONFIG_FILE_PATH}` - echo -e "[install]\nprefix=${PIP_INSTALL_PREFIX_DIR}\nignore-installed=true" > ${PIP_CONFIG_FILE_PATH} - # Instruct pip to use this config file. - export PIP_CONFIG_FILE=${PIP_CONFIG_FILE_PATH} - - # Set up PYTHONPATH correctly to include the user-installed library. - # Note that the pip prefix directory overrides the system default to enable - # a user to use his/her installed one. - # TODO(dsjang): Currently "lib/python3.6/site-packages" is hard-coded - # throughout Dockerfile. Parameterize it to avoid a version mismatch. - # TODO(dsjang): This is a hack to sidestep a problem of Python ignoring modules - # in .../site-packages other than the system-wide and user-specific site packages. - mkdir -p "${PIP_INSTALL_PREFIX_DIR}/lib/python3.6/site-packages" - ln -s "${PIP_INSTALL_PREFIX_DIR}/lib/python3.6/site-packages" ${PIP_INSTALLED_MODULE_DIR} - export PYTHONPATH=${PIP_INSTALLED_MODULE_DIR}:${PYTHONPATH} - - # Create a symbolic link to site-packages. - # can't override conda installed package. - - # Include pip-installed binaries in PATH. - export PATH=${PATH}:${PIP_INSTALL_PREFIX_DIR}/bin -fi - -# Execute the command provided from "docker run" in the current process. -exec "$@" \ No newline at end of file From dcb594f2fe32e101b010e540ea6299c89f753ee2 Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Fri, 10 Aug 2018 21:22:25 +0000 Subject: [PATCH 0356/1660] Install keras_applications before tensorflow --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95d88360..452e5d0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,9 +49,12 @@ RUN apt-get update && apt-get install -y python-software-properties zip && \ apt-get upgrade -y bazel # Tensorflow -# Temp fixes: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2 +# Fix to install tf 1.10:: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2 RUN conda install -y python=3.6.6 && \ pip install pandas==0.23.2 && \ +# Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518 + pip install keras_applications==1.0.4 --no-deps && \ + pip install keras_preprocessing==1.0.2 --no-deps && \ cd /usr/local/src && \ git clone https://github.com/tensorflow/tensorflow && \ cd tensorflow && \ From 58b7c82d809d37b9f8c66fec3142314f200ecf08 Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Fri, 10 Aug 2018 21:59:09 +0000 Subject: [PATCH 0357/1660] change install method for fancyimpute --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 452e5d0c..1e272503 100644 --- a/Dockerfile +++ b/Dockerfile @@ -298,9 +298,7 @@ RUN pip install --upgrade mpld3 && \ conda install -y ecos && \ conda install -y CVXcanon -RUN cd /usr/local/src && git clone https://github.com/iskandr/fancyimpute && \ - cd fancyimpute && \ - python setup.py install && \ +RUN pip install fancyimpute && \ pip install git+https://github.com/pymc-devs/pymc3 && \ pip install tifffile && \ pip install spectral && \ From 520b1c27fd706a7a04874c4e9e1b304317383c76 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Sat, 11 Aug 2018 00:43:55 +0000 Subject: [PATCH 0358/1660] Refactor test and improve coverage --- test | 13 ++- test_build.py | 167 ------------------------------------- tests/data/dot.png | Bin 0 -> 150 bytes tests/data/text.txt | 1 + tests/data/train.csv | 20 +++++ tests/test_bigquery.py | 38 +++++++++ tests/test_bokeh.py | 7 ++ tests/test_essentia.py | 7 ++ tests/test_fastai.py | 9 ++ tests/test_fasttext.py | 9 ++ tests/test_keras.py | 25 ++++++ tests/test_kmapper.py | 7 ++ tests/test_learntools.py | 9 ++ tests/test_matplotlib.py | 18 ++++ tests/test_nltk.py | 9 ++ tests/test_numpy.py | 16 ++++ tests/test_opencv.py | 9 ++ tests/test_pandas.py | 9 ++ tests/test_plotly.py | 9 ++ tests/test_pyfasttext.py | 9 ++ tests/test_pytorch.py | 13 +++ tests/test_seaborn.py | 7 ++ tests/test_shap.py | 7 ++ tests/test_skimage.py | 8 ++ tests/test_sklearn.py | 18 ++++ tests/test_tensorflow.py | 12 +++ tests/test_theano.py | 20 +++++ tests/test_vowpalwabbit.py | 10 +++ tests/test_wordbatch.py | 15 ++++ tests/test_xgboost.py | 12 +++ tests/text_mxnet.py | 9 ++ 31 files changed, 351 insertions(+), 171 deletions(-) delete mode 100644 test_build.py create mode 100644 tests/data/dot.png create mode 100644 tests/data/text.txt create mode 100644 tests/data/train.csv create mode 100644 tests/test_bigquery.py create mode 100644 tests/test_bokeh.py create mode 100644 tests/test_essentia.py create mode 100644 tests/test_fastai.py create mode 100644 tests/test_fasttext.py create mode 100644 tests/test_keras.py create mode 100644 tests/test_kmapper.py create mode 100644 tests/test_learntools.py create mode 100644 tests/test_matplotlib.py create mode 100644 tests/test_nltk.py create mode 100644 tests/test_numpy.py create mode 100644 tests/test_opencv.py create mode 100644 tests/test_pandas.py create mode 100644 tests/test_plotly.py create mode 100644 tests/test_pyfasttext.py create mode 100644 tests/test_pytorch.py create mode 100644 tests/test_seaborn.py create mode 100644 tests/test_shap.py create mode 100644 tests/test_skimage.py create mode 100644 tests/test_sklearn.py create mode 100644 tests/test_tensorflow.py create mode 100644 tests/test_theano.py create mode 100644 tests/test_vowpalwabbit.py create mode 100644 tests/test_wordbatch.py create mode 100644 tests/test_xgboost.py create mode 100644 tests/text_mxnet.py diff --git a/test b/test index 5de44db7..2528b6af 100755 --- a/test +++ b/test @@ -11,7 +11,12 @@ mkdir -p /tmp/python-build/working docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root --ip="*" sleep 3 docker kill jupyter_test && docker rm jupyter_test -docker run --rm -t --read-only --net=none -e HOME=/tmp -e KAGGLE_DATA_PROXY_TOKEN=test-key -e KAGGLE_DATA_PROXY_URL=http://127.0.0.1:8000 -e KAGGLE_DATA_PROXY_PROJECT=test -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build /bin/bash -c 'python /input/test_build.py' - -# The test_build.py script creates a plot called plot1.png; check that it exists -[ -s /tmp/python-build/working/plot1.png ] || exit 1 +docker run --rm -t --read-only --net=none \ + -e HOME=/tmp -e KAGGLE_DATA_PROXY_TOKEN=test-key \ + -e KAGGLE_DATA_PROXY_URL=http://127.0.0.1:8000 \ + -e KAGGLE_DATA_PROXY_PROJECT=test \ + -v $PWD:/input:ro -v /tmp/python-build/working:/working \ + -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm \ + -w=/working \ + kaggle/python-build \ + /bin/bash -c 'python -m unittest discover -s /input/tests' diff --git a/test_build.py b/test_build.py deleted file mode 100644 index 52d88f33..00000000 --- a/test_build.py +++ /dev/null @@ -1,167 +0,0 @@ -# This script should run without errors whenever we update the -# kaggle/python container. It checks that all our most popular packages can -# be loaded and used without errors. - -import tensorflow as tf -print(tf.__version__) -hello = tf.constant('TensorFlow ok') -sess = tf.Session() -print(sess.run(hello)) -print("Tensorflow ok") - -from keras.models import Sequential -from keras.layers.core import Dense, Dropout, Activation, Flatten -from keras.layers.convolutional import Convolution2D, MaxPooling2D -from keras.optimizers import SGD -print("Keras ok") - -# Test Kaggle learntools -from learntools.core import binder; binder.bind(globals()) -from learntools.python.ex1 import * -color="blue" -q0.check() -print("learntools ok") - -# PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html -import torch -import torch.nn as tnn -import torch.autograd as autograd -torch.manual_seed(31337) -linear_torch = tnn.Linear(5,3) -data_torch = autograd.Variable(torch.randn(2, 5)) -print(linear_torch(data_torch)) -print("PyTorch ok") - -import fastai -from fastai.io import get_data -print("fast.ai ok") - -import numpy as np -print("Numpy imported ok") -print("Your lucky number is: " + str(np.random.randint(100))) - -# Numpy must be linked to the MKL. (Occasionally, a third-party package will muck up the installation -# and numpy will be reinstalled with an OpenBLAS backing.) -from numpy.distutils.system_info import get_info -# This will throw an exception if the MKL is not linked correctly. -get_info("blas_mkl") - -import pandas as pd -print("Pandas imported ok") - -from sklearn import datasets -print("sklearn imported ok") -iris = datasets.load_iris() -X, y = iris.data, iris.target - -from sklearn.ensemble import RandomForestClassifier -rf1 = RandomForestClassifier() -rf1.fit(X,y) -print("sklearn RandomForestClassifier: ok") - -from sklearn.linear_model import LinearRegression -boston = datasets.load_boston() -X, y = boston.data, boston.target -lr1 = LinearRegression() -lr1.fit(X,y) -print("sklearn LinearRegression: ok") - -from xgboost import XGBClassifier -xgb1 = XGBClassifier(n_estimators=3) -xgb1.fit(X[0:70],y[0:70]) -print("xgboost XGBClassifier: ok") - -import matplotlib.pyplot as plt -plt.plot(np.linspace(0,1,50), np.random.rand(50)) -plt.savefig("plot1.png") -print("matplotlib.pyplot ok") - -from mpl_toolkits.basemap import Basemap -print("Basemap ok") - -import plotly.plotly as py -import plotly.graph_objs as go -print("plotly ok") - -import theano -print("Theano ok") - -import nltk -from nltk.stem import WordNetLemmatizer -print("nltk ok") - -import cv2 -img = cv2.imread('plot1.png',0) -print("OpenCV ok") - -from skimage.io import imread -print("skimage ok") - -from wordbatch.extractors import WordBag -print("wordbatch ok") - -import pyfasttext -print("pyfasttext ok") - -import fastText -print("fastText ok") - -import mxnet -import mxnet.gluon -print("mxnet ok") - -import bokeh -print("bokeh ok") - -import seaborn -print("seaborn ok") - -# Test BigQuery -import os -import threading -from http.server import BaseHTTPRequestHandler, HTTPServer -from google.cloud import bigquery -HOSTNAME = "127.0.0.1" -PORT = 8000 -URL = "http://%s:%s" % (HOSTNAME, PORT) -fake_bq_called = False -fake_bq_header_found = False -class HTTPHandler(BaseHTTPRequestHandler): - def do_HEAD(s): - s.send_response(200) - - def do_GET(s): - global fake_bq_called - global fake_bq_header_found - fake_bq_called = True - fake_bq_header_found = any(k for k in s.headers if k == "X-KAGGLE-PROXY-DATA" and s.headers[k] == "test-key") - s.send_response(200) - -httpd = HTTPServer((HOSTNAME, PORT), HTTPHandler) -threading.Thread(target=httpd.serve_forever).start() -client = bigquery.Client() -try: - for ds in client.list_datasets(): pass -except: - pass -httpd.shutdown() -assert fake_bq_called, "Fake server did not recieve a request from the BQ client." -assert fake_bq_header_found, "X-KAGGLE-PROXY-DATA header was missing from the BQ request." -print("bigquery proxy ok") - -import shap -print("shap ok") - -import kmapper -print("kmapper ok") - -from vowpalwabbit import pyvw -vw = pyvw.vw(quiet=True) -ex = vw.example('1 | a b c') -vw.learn(ex) -print(vw.predict(ex)) -print('vowpalwabbit ok') - -import essentia -print(essentia.__version__) -print("Essentia ok") diff --git a/tests/data/dot.png b/tests/data/dot.png new file mode 100644 index 0000000000000000000000000000000000000000..482df8ba39602ac729cb25528b0cc090001836e6 GIT binary patch literal 150 zcmeAS@N?(olHy`uVBq!ia0vp^j3CUx1SBVv2j2ryoCO|{#S9GG!XV7ZFl&wkP>{XE z)7O>#5eJu;Is3iXz?(oJ+02lL66gHf+|;}hAeVu`xhOTUBsE2$JhLQ2!QIn0AVn{g l9VpJ@>Eal|aXmR9Apyu^VPI4@Q3ttz!PC{xWt~$(699(VAFlua literal 0 HcmV?d00001 diff --git a/tests/data/text.txt b/tests/data/text.txt new file mode 100644 index 00000000..80aa862f --- /dev/null +++ b/tests/data/text.txt @@ -0,0 +1 @@ +Kaggle is a platform for predictive modelling and analytics competitions in which statisticians and data miners compete to produce the best models for predicting and describing the datasets uploaded by companies and users. diff --git a/tests/data/train.csv b/tests/data/train.csv new file mode 100644 index 00000000..229dbc7c --- /dev/null +++ b/tests/data/train.csv @@ -0,0 +1,20 @@ +label,pixel0,pixel1,pixel2,pixel3,pixel4,pixel5,pixel6,pixel7,pixel8,pixel9,pixel10,pixel11,pixel12,pixel13,pixel14,pixel15,pixel16,pixel17,pixel18,pixel19,pixel20,pixel21,pixel22,pixel23,pixel24,pixel25,pixel26,pixel27,pixel28,pixel29,pixel30,pixel31,pixel32,pixel33,pixel34,pixel35,pixel36,pixel37,pixel38,pixel39,pixel40,pixel41,pixel42,pixel43,pixel44,pixel45,pixel46,pixel47,pixel48,pixel49,pixel50,pixel51,pixel52,pixel53,pixel54,pixel55,pixel56,pixel57,pixel58,pixel59,pixel60,pixel61,pixel62,pixel63,pixel64,pixel65,pixel66,pixel67,pixel68,pixel69,pixel70,pixel71,pixel72,pixel73,pixel74,pixel75,pixel76,pixel77,pixel78,pixel79,pixel80,pixel81,pixel82,pixel83,pixel84,pixel85,pixel86,pixel87,pixel88,pixel89,pixel90,pixel91,pixel92,pixel93,pixel94,pixel95,pixel96,pixel97,pixel98,pixel99,pixel100,pixel101,pixel102,pixel103,pixel104,pixel105,pixel106,pixel107,pixel108,pixel109,pixel110,pixel111,pixel112,pixel113,pixel114,pixel115,pixel116,pixel117,pixel118,pixel119,pixel120,pixel121,pixel122,pixel123,pixel124,pixel125,pixel126,pixel127,pixel128,pixel129,pixel130,pixel131,pixel132,pixel133,pixel134,pixel135,pixel136,pixel137,pixel138,pixel139,pixel140,pixel141,pixel142,pixel143,pixel144,pixel145,pixel146,pixel147,pixel148,pixel149,pixel150,pixel151,pixel152,pixel153,pixel154,pixel155,pixel156,pixel157,pixel158,pixel159,pixel160,pixel161,pixel162,pixel163,pixel164,pixel165,pixel166,pixel167,pixel168,pixel169,pixel170,pixel171,pixel172,pixel173,pixel174,pixel175,pixel176,pixel177,pixel178,pixel179,pixel180,pixel181,pixel182,pixel183,pixel184,pixel185,pixel186,pixel187,pixel188,pixel189,pixel190,pixel191,pixel192,pixel193,pixel194,pixel195,pixel196,pixel197,pixel198,pixel199,pixel200,pixel201,pixel202,pixel203,pixel204,pixel205,pixel206,pixel207,pixel208,pixel209,pixel210,pixel211,pixel212,pixel213,pixel214,pixel215,pixel216,pixel217,pixel218,pixel219,pixel220,pixel221,pixel222,pixel223,pixel224,pixel225,pixel226,pixel227,pixel228,pixel229,pixel230,pixel231,pixel232,pixel233,pixel234,pixel235,pixel236,pixel237,pixel238,pixel239,pixel240,pixel241,pixel242,pixel243,pixel244,pixel245,pixel246,pixel247,pixel248,pixel249,pixel250,pixel251,pixel252,pixel253,pixel254,pixel255,pixel256,pixel257,pixel258,pixel259,pixel260,pixel261,pixel262,pixel263,pixel264,pixel265,pixel266,pixel267,pixel268,pixel269,pixel270,pixel271,pixel272,pixel273,pixel274,pixel275,pixel276,pixel277,pixel278,pixel279,pixel280,pixel281,pixel282,pixel283,pixel284,pixel285,pixel286,pixel287,pixel288,pixel289,pixel290,pixel291,pixel292,pixel293,pixel294,pixel295,pixel296,pixel297,pixel298,pixel299,pixel300,pixel301,pixel302,pixel303,pixel304,pixel305,pixel306,pixel307,pixel308,pixel309,pixel310,pixel311,pixel312,pixel313,pixel314,pixel315,pixel316,pixel317,pixel318,pixel319,pixel320,pixel321,pixel322,pixel323,pixel324,pixel325,pixel326,pixel327,pixel328,pixel329,pixel330,pixel331,pixel332,pixel333,pixel334,pixel335,pixel336,pixel337,pixel338,pixel339,pixel340,pixel341,pixel342,pixel343,pixel344,pixel345,pixel346,pixel347,pixel348,pixel349,pixel350,pixel351,pixel352,pixel353,pixel354,pixel355,pixel356,pixel357,pixel358,pixel359,pixel360,pixel361,pixel362,pixel363,pixel364,pixel365,pixel366,pixel367,pixel368,pixel369,pixel370,pixel371,pixel372,pixel373,pixel374,pixel375,pixel376,pixel377,pixel378,pixel379,pixel380,pixel381,pixel382,pixel383,pixel384,pixel385,pixel386,pixel387,pixel388,pixel389,pixel390,pixel391,pixel392,pixel393,pixel394,pixel395,pixel396,pixel397,pixel398,pixel399,pixel400,pixel401,pixel402,pixel403,pixel404,pixel405,pixel406,pixel407,pixel408,pixel409,pixel410,pixel411,pixel412,pixel413,pixel414,pixel415,pixel416,pixel417,pixel418,pixel419,pixel420,pixel421,pixel422,pixel423,pixel424,pixel425,pixel426,pixel427,pixel428,pixel429,pixel430,pixel431,pixel432,pixel433,pixel434,pixel435,pixel436,pixel437,pixel438,pixel439,pixel440,pixel441,pixel442,pixel443,pixel444,pixel445,pixel446,pixel447,pixel448,pixel449,pixel450,pixel451,pixel452,pixel453,pixel454,pixel455,pixel456,pixel457,pixel458,pixel459,pixel460,pixel461,pixel462,pixel463,pixel464,pixel465,pixel466,pixel467,pixel468,pixel469,pixel470,pixel471,pixel472,pixel473,pixel474,pixel475,pixel476,pixel477,pixel478,pixel479,pixel480,pixel481,pixel482,pixel483,pixel484,pixel485,pixel486,pixel487,pixel488,pixel489,pixel490,pixel491,pixel492,pixel493,pixel494,pixel495,pixel496,pixel497,pixel498,pixel499,pixel500,pixel501,pixel502,pixel503,pixel504,pixel505,pixel506,pixel507,pixel508,pixel509,pixel510,pixel511,pixel512,pixel513,pixel514,pixel515,pixel516,pixel517,pixel518,pixel519,pixel520,pixel521,pixel522,pixel523,pixel524,pixel525,pixel526,pixel527,pixel528,pixel529,pixel530,pixel531,pixel532,pixel533,pixel534,pixel535,pixel536,pixel537,pixel538,pixel539,pixel540,pixel541,pixel542,pixel543,pixel544,pixel545,pixel546,pixel547,pixel548,pixel549,pixel550,pixel551,pixel552,pixel553,pixel554,pixel555,pixel556,pixel557,pixel558,pixel559,pixel560,pixel561,pixel562,pixel563,pixel564,pixel565,pixel566,pixel567,pixel568,pixel569,pixel570,pixel571,pixel572,pixel573,pixel574,pixel575,pixel576,pixel577,pixel578,pixel579,pixel580,pixel581,pixel582,pixel583,pixel584,pixel585,pixel586,pixel587,pixel588,pixel589,pixel590,pixel591,pixel592,pixel593,pixel594,pixel595,pixel596,pixel597,pixel598,pixel599,pixel600,pixel601,pixel602,pixel603,pixel604,pixel605,pixel606,pixel607,pixel608,pixel609,pixel610,pixel611,pixel612,pixel613,pixel614,pixel615,pixel616,pixel617,pixel618,pixel619,pixel620,pixel621,pixel622,pixel623,pixel624,pixel625,pixel626,pixel627,pixel628,pixel629,pixel630,pixel631,pixel632,pixel633,pixel634,pixel635,pixel636,pixel637,pixel638,pixel639,pixel640,pixel641,pixel642,pixel643,pixel644,pixel645,pixel646,pixel647,pixel648,pixel649,pixel650,pixel651,pixel652,pixel653,pixel654,pixel655,pixel656,pixel657,pixel658,pixel659,pixel660,pixel661,pixel662,pixel663,pixel664,pixel665,pixel666,pixel667,pixel668,pixel669,pixel670,pixel671,pixel672,pixel673,pixel674,pixel675,pixel676,pixel677,pixel678,pixel679,pixel680,pixel681,pixel682,pixel683,pixel684,pixel685,pixel686,pixel687,pixel688,pixel689,pixel690,pixel691,pixel692,pixel693,pixel694,pixel695,pixel696,pixel697,pixel698,pixel699,pixel700,pixel701,pixel702,pixel703,pixel704,pixel705,pixel706,pixel707,pixel708,pixel709,pixel710,pixel711,pixel712,pixel713,pixel714,pixel715,pixel716,pixel717,pixel718,pixel719,pixel720,pixel721,pixel722,pixel723,pixel724,pixel725,pixel726,pixel727,pixel728,pixel729,pixel730,pixel731,pixel732,pixel733,pixel734,pixel735,pixel736,pixel737,pixel738,pixel739,pixel740,pixel741,pixel742,pixel743,pixel744,pixel745,pixel746,pixel747,pixel748,pixel749,pixel750,pixel751,pixel752,pixel753,pixel754,pixel755,pixel756,pixel757,pixel758,pixel759,pixel760,pixel761,pixel762,pixel763,pixel764,pixel765,pixel766,pixel767,pixel768,pixel769,pixel770,pixel771,pixel772,pixel773,pixel774,pixel775,pixel776,pixel777,pixel778,pixel779,pixel780,pixel781,pixel782,pixel783 +1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,188,255,94,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,191,250,253,93,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,123,248,253,167,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,80,247,253,208,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,29,207,253,235,77,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,54,209,253,253,88,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,93,254,253,238,170,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,23,210,254,253,159,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16,209,253,254,240,81,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,27,253,253,254,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,206,254,254,198,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,168,253,253,196,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,203,253,248,76,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,22,188,253,245,93,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,103,253,253,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,89,240,253,195,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,15,220,253,253,80,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,94,253,253,253,94,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,89,251,253,250,131,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,214,218,95,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18,30,137,137,192,86,72,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,86,250,254,254,254,254,217,246,151,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16,179,254,254,254,254,254,254,254,254,254,231,54,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,72,254,254,254,254,254,254,254,254,254,254,254,254,104,0,0,0,0,0,0,0,0,0,0,0,0,0,61,191,254,254,254,254,254,109,83,199,254,254,254,254,243,85,0,0,0,0,0,0,0,0,0,0,0,0,172,254,254,254,202,147,147,45,0,11,29,200,254,254,254,171,0,0,0,0,0,0,0,0,0,0,0,1,174,254,254,89,67,0,0,0,0,0,0,128,252,254,254,212,76,0,0,0,0,0,0,0,0,0,0,47,254,254,254,29,0,0,0,0,0,0,0,0,83,254,254,254,153,0,0,0,0,0,0,0,0,0,0,80,254,254,240,24,0,0,0,0,0,0,0,0,25,240,254,254,153,0,0,0,0,0,0,0,0,0,0,64,254,254,186,7,0,0,0,0,0,0,0,0,0,166,254,254,224,12,0,0,0,0,0,0,0,0,14,232,254,254,254,29,0,0,0,0,0,0,0,0,0,75,254,254,254,17,0,0,0,0,0,0,0,0,18,254,254,254,254,29,0,0,0,0,0,0,0,0,0,48,254,254,254,17,0,0,0,0,0,0,0,0,2,163,254,254,254,29,0,0,0,0,0,0,0,0,0,48,254,254,254,17,0,0,0,0,0,0,0,0,0,94,254,254,254,200,12,0,0,0,0,0,0,0,16,209,254,254,150,1,0,0,0,0,0,0,0,0,0,15,206,254,254,254,202,66,0,0,0,0,0,21,161,254,254,245,31,0,0,0,0,0,0,0,0,0,0,0,60,212,254,254,254,194,48,48,34,41,48,209,254,254,254,171,0,0,0,0,0,0,0,0,0,0,0,0,0,86,243,254,254,254,254,254,233,243,254,254,254,254,254,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,114,254,254,254,254,254,254,254,254,254,254,239,86,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,182,254,254,254,254,254,254,254,254,243,70,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,76,146,254,255,254,255,146,19,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,141,139,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,106,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,185,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,89,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,146,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,156,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,255,255,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,254,254,62,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,220,179,6,0,0,0,0,0,0,0,0,9,77,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,247,17,0,0,0,0,0,0,0,0,27,202,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,242,155,0,0,0,0,0,0,0,0,27,254,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,160,207,6,0,0,0,0,0,0,0,27,254,65,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,127,254,21,0,0,0,0,0,0,0,20,239,65,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,77,254,21,0,0,0,0,0,0,0,0,195,65,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,70,254,21,0,0,0,0,0,0,0,0,195,142,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,56,251,21,0,0,0,0,0,0,0,0,195,227,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,222,153,5,0,0,0,0,0,0,0,120,240,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,251,40,0,0,0,0,0,0,0,94,255,69,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,234,184,0,0,0,0,0,0,0,19,245,69,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,234,169,0,0,0,0,0,0,0,3,199,182,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,154,205,4,0,0,26,72,128,203,208,254,254,131,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,61,254,129,113,186,245,251,189,75,56,136,254,73,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,15,216,233,233,159,104,52,0,0,0,38,254,73,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18,254,73,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18,254,73,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,206,106,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,186,159,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,209,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,25,130,155,254,254,254,157,30,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,103,253,253,253,253,253,253,253,253,114,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,208,253,253,253,253,253,253,253,253,253,253,107,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,253,253,253,253,253,253,253,253,253,253,253,215,101,3,0,0,0,0,0,0,0,0,0,0,0,0,23,210,253,253,253,248,161,222,222,246,253,253,253,253,253,39,0,0,0,0,0,0,0,0,0,0,0,0,136,253,253,253,229,77,0,0,0,70,218,253,253,253,253,215,91,0,0,0,0,0,0,0,0,0,0,5,214,253,253,253,195,0,0,0,0,0,104,224,253,253,253,253,215,29,0,0,0,0,0,0,0,0,0,116,253,253,253,247,75,0,0,0,0,0,0,26,200,253,253,253,253,216,4,0,0,0,0,0,0,0,0,254,253,253,253,195,0,0,0,0,0,0,0,0,26,200,253,253,253,253,5,0,0,0,0,0,0,0,0,254,253,253,253,99,0,0,0,0,0,0,0,0,0,25,231,253,253,253,36,0,0,0,0,0,0,0,0,254,253,253,253,99,0,0,0,0,0,0,0,0,0,0,223,253,253,253,129,0,0,0,0,0,0,0,0,254,253,253,253,99,0,0,0,0,0,0,0,0,0,0,127,253,253,253,129,0,0,0,0,0,0,0,0,254,253,253,253,99,0,0,0,0,0,0,0,0,0,0,139,253,253,253,90,0,0,0,0,0,0,0,0,254,253,253,253,99,0,0,0,0,0,0,0,0,0,78,248,253,253,253,5,0,0,0,0,0,0,0,0,254,253,253,253,216,34,0,0,0,0,0,0,0,33,152,253,253,253,107,1,0,0,0,0,0,0,0,0,206,253,253,253,253,140,0,0,0,0,0,30,139,234,253,253,253,154,2,0,0,0,0,0,0,0,0,0,16,205,253,253,253,250,208,106,106,106,200,237,253,253,253,253,209,22,0,0,0,0,0,0,0,0,0,0,0,82,253,253,253,253,253,253,253,253,253,253,253,253,253,209,22,0,0,0,0,0,0,0,0,0,0,0,0,1,91,253,253,253,253,253,253,253,253,253,253,213,90,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,18,129,208,253,253,253,253,159,129,90,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,141,202,254,193,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,165,254,179,163,249,244,72,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,135,254,150,0,0,189,254,243,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,82,248,209,5,0,0,164,236,254,115,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,211,254,58,0,0,0,0,33,230,212,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,119,254,156,3,0,0,0,0,18,230,254,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,212,254,35,0,0,0,0,0,33,254,254,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,116,254,154,3,0,0,0,0,0,33,254,254,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,124,254,115,0,0,0,0,0,0,160,254,239,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,203,254,35,0,0,0,0,0,0,197,254,178,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,23,239,221,11,0,0,0,0,0,0,198,255,123,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,23,238,178,0,0,0,0,0,0,10,219,254,96,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,30,249,204,0,0,0,0,0,0,25,235,254,62,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,26,243,204,0,0,0,0,0,0,91,254,248,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,254,204,0,0,0,0,0,67,241,254,133,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,254,214,7,0,0,0,50,242,254,194,24,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,193,254,78,0,0,19,128,254,195,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,103,254,222,74,143,235,254,228,83,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,30,242,254,254,254,254,252,84,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,23,64,158,200,174,61,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,82,152,71,51,51,21,41,51,51,51,51,113,193,152,30,0,0,0,0,0,0,0,0,0,0,0,0,0,122,253,252,253,252,223,243,253,252,253,252,253,252,233,30,0,0,0,0,0,0,0,0,0,0,0,0,0,123,102,41,102,102,102,102,102,102,102,162,254,253,142,0,0,0,0,0,0,0,0,0,0,0,0,0,0,203,102,0,0,0,0,0,0,0,0,183,253,212,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,203,142,0,0,0,0,0,0,0,11,213,254,91,0,0,0,0,0,0,0,0,0,0,0,0,0,0,41,243,102,0,0,0,0,0,0,0,51,252,172,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,41,223,102,0,0,0,0,0,0,0,214,253,102,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,20,0,0,0,0,0,0,0,253,252,102,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,254,253,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,102,253,171,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,163,254,91,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,203,253,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,51,253,254,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,51,252,253,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,51,253,254,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,51,252,213,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,152,253,82,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,233,252,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,21,255,253,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,253,212,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,21,130,190,254,254,250,175,135,96,96,16,4,0,0,0,0,0,0,0,0,0,0,0,0,0,26,102,186,254,254,248,222,222,225,254,254,254,254,254,206,112,4,0,0,0,0,0,0,0,0,0,0,0,207,254,254,177,117,39,0,0,56,248,102,48,48,103,192,254,135,0,0,0,0,0,0,0,0,0,0,0,91,111,36,0,0,0,0,0,72,92,0,0,0,0,12,224,210,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,139,240,254,66,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,121,220,254,244,194,15,0,0,0,0,0,0,0,0,0,0,0,0,0,8,107,112,112,112,87,112,141,218,248,177,68,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,77,221,254,254,254,254,254,225,104,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,32,32,32,32,130,215,195,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,111,231,174,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,47,18,0,0,0,0,0,0,0,0,0,40,228,205,35,0,0,0,0,0,0,0,0,0,0,0,0,22,234,42,0,0,0,0,0,0,0,0,0,0,56,212,226,38,0,0,0,0,0,0,0,0,0,0,0,96,157,0,0,0,0,0,0,0,0,0,0,0,0,30,215,188,9,0,0,0,0,0,0,0,0,0,0,96,142,0,0,0,0,0,0,0,0,0,0,0,0,0,86,254,68,0,0,0,0,0,0,0,0,0,0,71,202,15,0,0,0,0,0,0,0,0,0,0,0,0,6,214,151,0,0,0,0,0,0,0,0,0,0,10,231,86,2,0,0,0,0,0,0,0,0,0,0,0,0,191,207,0,0,0,0,0,0,0,0,0,0,0,93,248,129,7,0,0,0,0,0,0,0,0,0,0,117,238,112,0,0,0,0,0,0,0,0,0,0,0,0,94,248,209,73,12,0,0,0,0,0,0,42,147,252,136,9,0,0,0,0,0,0,0,0,0,0,0,0,0,48,160,215,230,158,74,64,94,153,223,250,214,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,129,189,234,224,255,194,134,75,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,41,149,156,179,254,254,201,119,46,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,147,241,253,253,254,253,253,253,253,245,160,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,224,253,253,180,174,175,174,174,174,174,223,247,145,6,0,0,0,0,0,0,0,0,0,0,0,0,7,197,254,253,165,2,0,0,0,0,0,0,12,102,184,16,0,0,0,0,0,0,0,0,0,0,0,0,152,253,254,162,18,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,235,254,158,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,74,250,253,15,0,0,0,16,20,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,199,253,253,0,0,25,130,235,254,247,145,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,253,253,177,100,219,240,253,253,254,253,253,125,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,193,253,253,254,253,253,200,155,155,238,253,229,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,61,249,254,241,150,30,0,0,0,215,254,254,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,36,39,30,0,0,0,0,0,214,253,234,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,41,241,253,183,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,201,253,253,102,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,114,254,253,154,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,254,255,241,30,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,118,235,253,249,103,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,81,0,102,211,253,253,253,135,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,79,243,234,254,253,253,216,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,245,253,254,207,126,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,60,136,136,147,254,255,199,111,18,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,152,253,253,253,253,253,253,253,253,253,124,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,135,225,244,253,202,200,181,164,216,253,253,211,151,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,30,149,78,3,0,0,0,20,134,253,253,224,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,206,253,253,224,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,78,253,253,253,224,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,99,234,253,253,224,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,142,220,219,236,253,253,240,121,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24,253,253,253,253,235,233,253,253,185,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,150,194,194,194,53,40,97,253,253,170,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,122,253,253,170,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,237,253,253,170,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,130,253,253,253,170,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,12,120,193,253,253,214,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,153,253,253,253,253,212,30,0,0,0,0,0,0,0,0,0,0,0,0,0,33,136,70,6,0,27,67,186,253,253,253,253,234,31,0,0,0,0,0,0,0,0,0,0,0,0,0,26,231,253,253,191,183,223,253,253,253,253,172,216,112,0,0,0,0,0,0,0,0,0,0,0,0,0,0,36,215,253,253,253,253,253,253,253,253,253,47,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,87,223,253,253,253,244,152,223,223,109,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,50,176,148,78,16,0,12,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,24,24,97,253,253,253,253,255,180,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,30,186,252,252,253,252,252,252,252,253,252,227,29,0,0,0,0,0,0,0,0,0,0,0,0,0,0,38,155,252,252,252,253,252,252,227,79,222,252,252,129,0,0,0,0,0,0,0,0,0,0,0,0,0,85,233,252,252,252,252,253,252,252,202,11,180,252,252,119,0,0,0,0,0,0,0,0,0,0,0,0,43,240,253,252,252,252,252,253,252,252,244,126,201,252,252,150,0,0,0,0,0,0,0,0,0,0,0,7,212,253,255,253,253,253,232,221,42,0,104,253,255,253,205,21,0,0,0,0,0,0,0,0,0,0,0,25,223,252,253,252,252,214,18,0,0,34,215,252,253,223,56,0,0,0,0,0,0,0,0,0,0,0,0,0,99,246,253,252,252,77,0,7,70,203,252,252,173,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,42,253,252,252,236,103,160,252,252,218,108,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,148,252,252,252,252,253,231,106,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24,253,253,253,253,255,159,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,43,118,252,240,244,252,253,231,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,19,164,246,253,187,50,99,246,253,252,69,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,80,232,252,203,58,0,0,135,253,252,121,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,43,246,252,200,11,0,0,0,116,253,252,69,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,162,253,192,11,0,0,0,0,179,255,253,69,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,178,252,119,0,5,47,47,140,244,253,252,69,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,186,252,227,184,191,252,252,252,252,253,240,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,144,227,252,252,253,252,252,252,252,98,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,137,242,253,231,137,137,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,15,48,143,186,244,143,31,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,83,209,253,252,252,252,252,192,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,166,241,252,253,252,170,162,252,252,113,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,61,234,252,252,243,121,44,2,21,245,252,122,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,80,252,252,243,163,50,0,0,0,5,101,88,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,105,234,252,210,88,0,0,0,0,74,199,240,43,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,252,210,21,0,4,12,41,231,249,252,252,55,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,242,252,218,154,154,184,252,253,252,252,248,184,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,209,252,252,252,252,252,252,253,252,252,196,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,57,142,95,142,61,81,253,252,209,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,177,255,230,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,124,252,245,57,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,135,252,252,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,79,248,252,233,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,231,252,202,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,175,248,252,136,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,109,252,252,159,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,218,252,252,192,141,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,132,252,252,252,205,74,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,132,252,252,146,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,169,207,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,168,254,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,84,249,254,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,89,254,254,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,89,254,193,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,89,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,204,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,89,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,209,254,178,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,37,209,254,254,69,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,107,254,254,254,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,187,254,254,134,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,254,155,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,254,238,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,254,254,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,254,231,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,255,87,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,173,254,87,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,87,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,254,87,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,128,255,191,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,191,255,255,255,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,191,255,255,255,255,255,255,255,191,191,191,128,128,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,128,0,0,0,0,0,0,0,0,0,0,0,0,0,128,128,191,128,128,191,255,255,255,255,255,255,255,255,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,191,255,255,255,255,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,255,255,255,255,255,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,255,255,255,255,255,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,255,255,255,255,255,191,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,191,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,191,255,255,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,128,128,0,0,0,0,0,0,64,255,255,255,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,191,255,128,0,0,0,0,0,0,64,255,255,255,64,0,0,0,0,0,0,0,0,0,0,0,0,0,64,255,255,64,0,0,0,0,0,0,191,255,255,255,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,191,64,0,0,0,64,191,255,255,255,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,255,255,255,255,255,255,255,255,255,255,191,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,255,255,191,128,128,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,222,253,253,192,113,88,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,85,226,249,252,252,252,253,252,246,209,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,156,252,253,233,195,195,195,196,214,252,252,221,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,252,252,162,56,0,0,0,0,28,121,252,252,216,18,0,0,0,0,0,0,0,0,0,0,0,0,0,57,252,173,0,0,0,0,0,0,0,25,205,252,253,27,0,0,0,0,0,0,0,0,0,0,0,0,0,57,253,253,0,0,0,0,0,0,0,0,92,253,255,27,0,0,0,0,0,0,0,0,0,0,0,0,0,38,224,252,126,0,0,0,0,0,0,51,243,252,253,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,53,195,110,0,0,0,0,0,51,101,252,252,190,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,29,29,92,243,252,252,252,253,177,53,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,126,165,252,252,253,252,252,252,252,253,252,195,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,226,253,253,253,255,215,140,140,140,192,253,253,146,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,178,252,242,167,106,18,0,0,0,12,228,252,223,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,19,55,49,0,0,0,0,0,0,0,225,252,223,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16,92,243,252,129,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16,203,253,252,220,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,26,207,253,255,215,31,0,0,0,0,0,0,0,0,0,0,101,225,175,0,0,0,0,0,10,85,147,225,231,252,252,168,33,0,0,0,0,0,0,0,0,0,0,0,113,252,208,57,57,57,57,166,203,252,253,252,239,195,118,0,0,0,0,0,0,0,0,0,0,0,0,0,38,234,252,252,252,253,252,252,252,252,225,176,65,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,221,252,252,253,127,112,112,112,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,38,236,73,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,96,253,92,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,207,214,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,92,253,97,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,144,253,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,231,253,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,54,254,192,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,130,249,107,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,221,221,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,221,221,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,78,255,208,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,107,253,111,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,188,253,68,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,188,253,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,202,181,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,45,253,143,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,146,253,90,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,215,187,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,45,253,66,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,165,124,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,39,91,91,91,91,91,191,255,142,91,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,23,211,253,253,253,253,253,253,253,253,253,181,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,180,253,253,253,253,253,253,253,253,253,253,228,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,180,253,253,253,253,253,253,253,253,253,253,228,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,101,253,253,253,253,253,253,253,253,253,253,228,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,224,253,253,211,114,96,253,253,253,253,228,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,204,114,18,0,156,253,253,253,253,228,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,15,0,0,0,230,253,253,253,253,185,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,156,244,253,253,253,192,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,209,253,253,253,253,245,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,215,253,253,253,245,130,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,124,253,253,253,253,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,41,220,253,253,253,238,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,222,253,253,253,253,98,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,107,253,253,253,253,210,35,0,0,0,5,42,42,42,42,18,0,0,0,0,0,0,0,0,0,0,0,0,107,253,253,253,253,253,178,131,131,131,143,253,253,253,253,182,8,0,0,0,0,0,0,0,0,0,0,0,83,240,253,253,253,253,253,253,253,253,253,253,253,253,240,195,12,0,0,0,0,0,0,0,0,0,0,0,0,197,253,253,253,253,253,253,253,253,253,253,229,106,82,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,179,241,253,253,253,253,253,253,246,179,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,75,89,195,89,89,89,89,81,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,144,254,130,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,46,157,251,253,222,91,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,95,232,253,253,253,253,249,133,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,43,254,253,253,228,137,85,253,235,34,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,47,229,254,253,228,57,0,54,253,253,107,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,226,253,254,202,55,0,0,54,253,253,107,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,133,253,253,214,11,0,0,0,54,253,253,107,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,182,253,253,168,0,0,0,0,0,54,253,253,107,0,0,0,0,0,0,0,0,0,0,0,0,0,4,65,253,252,233,50,0,0,0,0,0,131,253,252,101,0,0,0,0,0,0,0,0,0,0,0,0,0,68,253,253,226,0,0,0,0,0,0,0,187,253,226,0,0,0,0,0,0,0,0,0,0,0,0,0,0,202,254,235,49,0,0,0,0,0,0,131,254,255,94,0,0,0,0,0,0,0,0,0,0,0,0,0,204,251,230,53,0,0,0,0,0,0,0,201,253,253,93,0,0,0,0,0,0,0,0,0,0,0,0,85,249,228,126,0,0,0,0,0,0,0,123,248,253,253,93,0,0,0,0,0,0,0,0,0,0,0,35,215,253,186,0,0,0,0,0,0,0,123,247,253,236,151,15,0,0,0,0,0,0,0,0,0,0,0,108,253,253,81,0,0,0,0,0,0,191,248,253,253,164,0,0,0,0,0,0,0,0,0,0,0,0,0,108,253,194,14,0,0,0,3,54,188,250,253,253,175,18,0,0,0,0,0,0,0,0,0,0,0,0,0,108,253,228,68,41,132,174,178,253,255,253,253,176,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,40,237,253,253,253,253,253,253,253,255,242,177,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,221,253,253,253,253,253,252,240,241,68,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,45,233,253,253,232,120,114,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,69,128,199,199,220,255,177,242,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,59,180,252,254,254,254,254,254,254,254,254,120,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,179,254,230,186,136,73,56,56,56,63,254,140,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,110,50,16,0,0,0,0,0,0,76,237,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,205,214,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,47,252,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,208,219,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,96,254,78,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,209,187,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,155,254,58,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,222,240,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,87,254,71,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,213,238,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,31,254,203,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,172,254,152,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,254,151,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,191,226,27,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,23,254,102,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,158,249,47,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,165,160,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/tests/test_bigquery.py b/tests/test_bigquery.py new file mode 100644 index 00000000..cbee03ae --- /dev/null +++ b/tests/test_bigquery.py @@ -0,0 +1,38 @@ +import unittest +import os +import threading + +from http.server import BaseHTTPRequestHandler, HTTPServer + +from google.cloud import bigquery + +HOSTNAME = "127.0.0.1" +PORT = 8000 +URL = "http://%s:%s" % (HOSTNAME, PORT) + +class TestBigQuery(unittest.TestCase): + def test_proxy(self): + httpd = HTTPServer((HOSTNAME, PORT), HTTPHandler) + threading.Thread(target=httpd.serve_forever).start() + client = bigquery.Client() + + try: + for ds in client.list_datasets(): pass + except: + pass + + httpd.shutdown() + self.assertTrue(HTTPHandler.called, msg="Fake server did not recieve a request from the BQ client.") + self.assertTrue(HTTPHandler.header_found, msg="X-KAGGLE-PROXY-DATA header was missing from the BQ request.") + +class HTTPHandler(BaseHTTPRequestHandler): + called = False + header_found = False + + def do_HEAD(s): + s.send_response(200) + + def do_GET(s): + HTTPHandler.called = True + HTTPHandler.header_found = any(k for k in s.headers if k == "X-KAGGLE-PROXY-DATA" and s.headers[k] == "test-key") + s.send_response(200) diff --git a/tests/test_bokeh.py b/tests/test_bokeh.py new file mode 100644 index 00000000..1b186814 --- /dev/null +++ b/tests/test_bokeh.py @@ -0,0 +1,7 @@ +import unittest + +from bokeh.plotting import figure + +class TestBokeh(unittest.TestCase): + def test_figure(self): + figure(title="Hello World") diff --git a/tests/test_essentia.py b/tests/test_essentia.py new file mode 100644 index 00000000..749b9466 --- /dev/null +++ b/tests/test_essentia.py @@ -0,0 +1,7 @@ +import unittest + +from essentia.standard import Windowing + +class TestEssentia(unittest.TestCase): + def test_windowing(self): + Windowing(type = 'hann') diff --git a/tests/test_fastai.py b/tests/test_fastai.py new file mode 100644 index 00000000..275d2776 --- /dev/null +++ b/tests/test_fastai.py @@ -0,0 +1,9 @@ +import unittest + +from fastai.core import partition + +class TestFastAI(unittest.TestCase): + def test_partition(self): + result = partition([1,2,3,4,5], 2) + + self.assertEqual(3, len(result)) diff --git a/tests/test_fasttext.py b/tests/test_fasttext.py new file mode 100644 index 00000000..e7112ae7 --- /dev/null +++ b/tests/test_fasttext.py @@ -0,0 +1,9 @@ +import unittest + +import fastText + +class TestFastText(unittest.TestCase): + def test_tokenize(self): + tokens = fastText.FastText.tokenize("Hello World") + + self.assertEqual(["Hello", "World"], tokens) diff --git a/tests/test_keras.py b/tests/test_keras.py new file mode 100644 index 00000000..ca107563 --- /dev/null +++ b/tests/test_keras.py @@ -0,0 +1,25 @@ +import unittest + +import pandas as pd + +from keras.models import Sequential +from keras.layers import Dense +from keras.optimizers import RMSprop +from keras.utils.np_utils import to_categorical + +class TestKeras(unittest.TestCase): + def test_train(self): + train = pd.read_csv("/input/tests/data/train.csv") + + x_train = train.iloc[:,1:].values.astype('float32') + y_train = to_categorical(train.iloc[:,0].astype('int32')) + + model = Sequential() + model.add(Dense(units=10, input_dim=784, activation='softmax')) + + model.compile( + loss='categorical_crossentropy', + optimizer=RMSprop(lr=0.001), + metrics=['accuracy']) + + model.fit(x_train, y_train, epochs=1, batch_size=32) diff --git a/tests/test_kmapper.py b/tests/test_kmapper.py new file mode 100644 index 00000000..c75deea3 --- /dev/null +++ b/tests/test_kmapper.py @@ -0,0 +1,7 @@ +import unittest + +import kmapper as km + +class TestKMapper(unittest.TestCase): + def test_init(self): + km.KeplerMapper() diff --git a/tests/test_learntools.py b/tests/test_learntools.py new file mode 100644 index 00000000..c4c4efb0 --- /dev/null +++ b/tests/test_learntools.py @@ -0,0 +1,9 @@ +import unittest + +from learntools.core import binder; binder.bind(globals()) +from learntools.python.ex1 import * + +class TestLearnTools(unittest.TestCase): + def test_check(self): + color="blue" + q0.check() diff --git a/tests/test_matplotlib.py b/tests/test_matplotlib.py new file mode 100644 index 00000000..4b2605ec --- /dev/null +++ b/tests/test_matplotlib.py @@ -0,0 +1,18 @@ +import unittest +import os.path + +import matplotlib.pyplot as plt +import numpy as np + +from mpl_toolkits.basemap import Basemap + +class TestMatplotlib(unittest.TestCase): + def test_plot(self): + plt.plot(np.linspace(0,1,50), np.random.rand(50)) + plt.savefig("plot1.png") + + self.assertTrue(os.path.isfile("plot1.png")) + + def test_basemap(self): + Basemap(width=100,height=100,projection='aeqd', + lat_0=40,lon_0=-105) diff --git a/tests/test_nltk.py b/tests/test_nltk.py new file mode 100644 index 00000000..45bee936 --- /dev/null +++ b/tests/test_nltk.py @@ -0,0 +1,9 @@ +import unittest + +import nltk + +class TestNLTK(unittest.TestCase): + def test_tokenize(self): + tokens = nltk.word_tokenize("At eight o'clock") + + self.assertEqual(["At", "eight", "o'clock"], tokens) diff --git a/tests/test_numpy.py b/tests/test_numpy.py new file mode 100644 index 00000000..2d809f7b --- /dev/null +++ b/tests/test_numpy.py @@ -0,0 +1,16 @@ +import unittest + +import numpy as np +from numpy.distutils.system_info import get_info + +class TestNumpy(unittest.TestCase): + def test_array(self): + array = np.array([1, 3]) + + self.assertEqual((2,), array.shape) + + # Numpy must be linked to the MKL. (Occasionally, a third-party package will muck up the installation + # and numpy will be reinstalled with an OpenBLAS backing.) + def test_mkl(self): + # This will throw an exception if the MKL is not linked correctly. + get_info("blas_mkl") diff --git a/tests/test_opencv.py b/tests/test_opencv.py new file mode 100644 index 00000000..be0e92f8 --- /dev/null +++ b/tests/test_opencv.py @@ -0,0 +1,9 @@ +import unittest + +import cv2 + +class TestOpenCV(unittest.TestCase): + def test_imread(self): + img = cv2.imread('/input/tests/data/dot.png') + + self.assertEqual(1, img.shape[0]) diff --git a/tests/test_pandas.py b/tests/test_pandas.py new file mode 100644 index 00000000..f9163c57 --- /dev/null +++ b/tests/test_pandas.py @@ -0,0 +1,9 @@ +import unittest + +import pandas as pd + +class TestPandas(unittest.TestCase): + def test_read_csv(self): + data = pd.read_csv("/input/tests/data/train.csv") + + self.assertEqual(2, len(data.shape)) diff --git a/tests/test_plotly.py b/tests/test_plotly.py new file mode 100644 index 00000000..0c24a05e --- /dev/null +++ b/tests/test_plotly.py @@ -0,0 +1,9 @@ +import unittest + +import plotly.graph_objs as go + +class TestPlotly(unittest.TestCase): + def test_figure(self): + trace = {'x': [1, 2], 'y': [1, 3]} + data = [ trace ] + go.Figure(data=data) diff --git a/tests/test_pyfasttext.py b/tests/test_pyfasttext.py new file mode 100644 index 00000000..1ebf5211 --- /dev/null +++ b/tests/test_pyfasttext.py @@ -0,0 +1,9 @@ +import unittest + +from pyfasttext import FastText + +class TestPyFasttext(unittest.TestCase): + def test_vector(self): + model = FastText() + + model.supervised(input='/input/tests/data/text.txt', output='model', epoch=1, lr=0.7) diff --git a/tests/test_pytorch.py b/tests/test_pytorch.py new file mode 100644 index 00000000..4f250572 --- /dev/null +++ b/tests/test_pytorch.py @@ -0,0 +1,13 @@ +import unittest + +import torch +import torch.nn as tnn +import torch.autograd as autograd + +class TestPyTorch(unittest.TestCase): + # PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html + def test_nn(self): + torch.manual_seed(31337) + linear_torch = tnn.Linear(5,3) + data_torch = autograd.Variable(torch.randn(2, 5)) + linear_torch(data_torch) diff --git a/tests/test_seaborn.py b/tests/test_seaborn.py new file mode 100644 index 00000000..3986bc8c --- /dev/null +++ b/tests/test_seaborn.py @@ -0,0 +1,7 @@ +import unittest + +import seaborn as sns + +class TestSeaborn(unittest.TestCase): + def test_option(self): + sns.set(style="darkgrid") diff --git a/tests/test_shap.py b/tests/test_shap.py new file mode 100644 index 00000000..992d5c5e --- /dev/null +++ b/tests/test_shap.py @@ -0,0 +1,7 @@ +import unittest + +import shap + +class TestShap(unittest.TestCase): + def test_init(self): + shap.initjs() diff --git a/tests/test_skimage.py b/tests/test_skimage.py new file mode 100644 index 00000000..4a53b12e --- /dev/null +++ b/tests/test_skimage.py @@ -0,0 +1,8 @@ +import unittest + +from skimage import data, filters + +class TestSkImage(unittest.TestCase): + def test_filter(self): + image = data.coins() + filters.sobel(image) diff --git a/tests/test_sklearn.py b/tests/test_sklearn.py new file mode 100644 index 00000000..d2d4b027 --- /dev/null +++ b/tests/test_sklearn.py @@ -0,0 +1,18 @@ +import unittest + +from sklearn import datasets +from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import LinearRegression + +class TestSklearn(unittest.TestCase): + def test_random_forest_classifier(self): + iris = datasets.load_iris() + X, y = iris.data, iris.target + rf1 = RandomForestClassifier() + rf1.fit(X,y) + + def test_linearn_classifier(self): + boston = datasets.load_boston() + X, y = boston.data, boston.target + lr1 = LinearRegression() + lr1.fit(X,y) diff --git a/tests/test_tensorflow.py b/tests/test_tensorflow.py new file mode 100644 index 00000000..120b773d --- /dev/null +++ b/tests/test_tensorflow.py @@ -0,0 +1,12 @@ +import unittest + +import tensorflow as tf + +class TestTensorflow(unittest.TestCase): + def test_addition(self): + op = tf.add(2, 3) + sess = tf.Session() + + result = sess.run(op) + + self.assertEqual(5, result) diff --git a/tests/test_theano.py b/tests/test_theano.py new file mode 100644 index 00000000..e4714157 --- /dev/null +++ b/tests/test_theano.py @@ -0,0 +1,20 @@ +import unittest + +import theano +from theano import tensor + +class TestTheano(unittest.TestCase): + def test_addition(self): + # declare two symbolic floating-point scalars + a = tensor.dscalar() + b = tensor.dscalar() + + # create a simple expression + c = a + b + + # convert the expression into a callable object that takes (a,b) + # values as input and computes a value for c + f = theano.function([a,b], c) + + # bind 1.5 to 'a', 2.5 to 'b', and evaluate 'c' + self.assertEqual(4.0, f(1.5, 2.5)) diff --git a/tests/test_vowpalwabbit.py b/tests/test_vowpalwabbit.py new file mode 100644 index 00000000..f6f62161 --- /dev/null +++ b/tests/test_vowpalwabbit.py @@ -0,0 +1,10 @@ +import unittest + +from vowpalwabbit import pyvw + +class TestVowpalwabbit(unittest.TestCase): + def test_basic(self): + vw = pyvw.vw(quiet=True) + ex = vw.example('1 | a b c') + vw.learn(ex) + self.assertEqual(0.632030725479126, vw.predict(ex)) diff --git a/tests/test_wordbatch.py b/tests/test_wordbatch.py new file mode 100644 index 00000000..4d64c8f7 --- /dev/null +++ b/tests/test_wordbatch.py @@ -0,0 +1,15 @@ +import unittest + +import wordbatch + +from wordbatch.extractors import WordBag + +class TestWordBatch(unittest.TestCase): + def test_wordbatch(self): + wordbatch.WordBatch(extractor=(WordBag, { + "hash_ngrams":2, + "hash_ngrams_weights":[0.5, -1.0], + "hash_size":2**23, + "norm":'l2', + "tf":'log', + "idf":50.0})) diff --git a/tests/test_xgboost.py b/tests/test_xgboost.py new file mode 100644 index 00000000..e1bc12f8 --- /dev/null +++ b/tests/test_xgboost.py @@ -0,0 +1,12 @@ +import unittest + +from sklearn import datasets +from xgboost import XGBClassifier + +class TestXGBoost(unittest.TestCase): + def test_classifier(self): + boston = datasets.load_boston() + X, y = boston.data, boston.target + + xgb1 = XGBClassifier(n_estimators=3) + xgb1.fit(X[0:70],y[0:70]) diff --git a/tests/text_mxnet.py b/tests/text_mxnet.py new file mode 100644 index 00000000..8c9d0712 --- /dev/null +++ b/tests/text_mxnet.py @@ -0,0 +1,9 @@ +import unittest + +import mxnet as mx + +class TestMxNet(unittest.TestCase): + def test_array(self): + x = mx.nd.array([[1, 2, 3], [4, 5, 6]]) + + self.assertEqual((2, 3), x.shape) From 8fbdaff9eb07479b959d745be5f949db920488d5 Mon Sep 17 00:00:00 2001 From: Ben Hamner Date: Fri, 10 Aug 2018 20:58:06 -0700 Subject: [PATCH 0359/1660] Added tensorflow_hub --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1e272503..85dc779e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -494,6 +494,7 @@ RUN pip install flashtext && \ pip install conx && \ pip install pandasql && \ pip install trackml && \ + pip install tensorflow_hub && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From 3646d64d48ee240e233e02331651c7b6837f6f7c Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 13 Aug 2018 16:46:24 +0000 Subject: [PATCH 0360/1660] Capitalize and add punctuation to comment --- tests/test_theano.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_theano.py b/tests/test_theano.py index e4714157..156beb7b 100644 --- a/tests/test_theano.py +++ b/tests/test_theano.py @@ -5,16 +5,16 @@ class TestTheano(unittest.TestCase): def test_addition(self): - # declare two symbolic floating-point scalars + # Declare two symbolic floating-point scalars. a = tensor.dscalar() b = tensor.dscalar() - # create a simple expression + # Create a simple expression. c = a + b - # convert the expression into a callable object that takes (a,b) - # values as input and computes a value for c + # Convert the expression into a callable object that takes (a,b) + # values as input and computes a value for 'c'. f = theano.function([a,b], c) - # bind 1.5 to 'a', 2.5 to 'b', and evaluate 'c' + # Bind 1.5 to 'a', 2.5 to 'b', and evaluate 'c'. self.assertEqual(4.0, f(1.5, 2.5)) From cab99a4285ebf42503232756ffe5485a590d7648 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 13 Aug 2018 17:07:49 +0000 Subject: [PATCH 0361/1660] add tracking issue for tensorflow python 3.7 compatibility --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 85dc779e..f0495a89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,11 +48,11 @@ RUN apt-get update && apt-get install -y python-software-properties zip && \ apt-get update && apt-get install -y bazel && \ apt-get upgrade -y bazel -# Tensorflow +# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517 # Fix to install tf 1.10:: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2 RUN conda install -y python=3.6.6 && \ pip install pandas==0.23.2 && \ -# Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518 + # Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518 pip install keras_applications==1.0.4 --no-deps && \ pip install keras_preprocessing==1.0.2 --no-deps && \ cd /usr/local/src && \ From 8dda3a7683c0206d44f2cdd53e731da963343e8d Mon Sep 17 00:00:00 2001 From: Chris Crawford Date: Tue, 14 Aug 2018 00:10:41 +0000 Subject: [PATCH 0362/1660] Ugly and inelegant fix for geopandas --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f0495a89..5d97c5af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -227,7 +227,6 @@ RUN pip install scipy && \ cd /usr/local/src && git clone git://github.com/nicolashennetier/pyeconometrics.git && \ cd pyeconometrics && python setup.py install && \ apt-get install -y graphviz && pip install graphviz && \ - apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ # Pandoc is a dependency of deap apt-get install -y pandoc && \ cd /usr/local/src && git clone git://github.com/scikit-learn-contrib/py-earth.git && \ @@ -324,8 +323,10 @@ RUN pip install fancyimpute && \ pip install stemming && \ conda install -y -c conda-forge fbprophet && \ conda install -y -c conda-forge -c ioam holoviews geoviews && \ -# Temp fix : Fiona is already installed by pip and conda installs another version as a dependency for holoviews - conda uninstall -y fiona && \ + #Temp fix: After installing holoviews and geoviews, deps for fiona and geopandas get really messed up. This is a very unelegant fix. + conda uninstall -y fiona geopandas && \ + pip uninstall -y fiona geopandas && \ + apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ pip install hypertools && \ # Nxviz has been causing an installation issue by trying unsuccessfully to remove setuptools. #pip install nxviz && \ From cc404542a0fdb365bd0dca19c9fc421a066b9cb7 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 22 Aug 2018 09:40:53 -0700 Subject: [PATCH 0363/1660] pin notebook package to 5.5.0 to fix styling issue --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e4f353c5..d5cc3ce6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -433,7 +433,7 @@ RUN pip install bcolz && \ pip install mistune && \ pip install nbconvert && \ pip install nbformat && \ - pip install notebook && \ + pip install notebook==0.5.5 && \ pip install numpy && \ pip install olefile && \ pip install opencv-python && \ From 94c03a970609f4f33ae3e1f4a93bb9929e69071f Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 22 Aug 2018 23:00:58 +0000 Subject: [PATCH 0364/1660] install tensorpack using pip --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5cc3ce6..6f689c0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -360,8 +360,7 @@ RUN pip install kmeans-smote --no-dependencies && \ cd /opt/ && git clone https://github.com/PAIR-code/facets && cd facets/ && jupyter nbextension install facets-dist/ --user && \ export PYTHONPATH=$PYTHONPATH:/opt/facets/facets_overview/python/ && \ pip install --no-dependencies ethnicolr && \ - # Update setuptools and add tensorpack - pip install --upgrade --ignore-installed setuptools && pip install --no-cache-dir git+git://github.com/ppwwyyxx/tensorpack && \ + pip install tensorpack && \ pip install pycountry && pip install iso3166 && \ pip install pydash && \ pip install kmodes --no-dependencies && \ @@ -524,4 +523,4 @@ ENV PYTHONUSERBASE "/root/.local" ADD patches/sitecustomize.py /root/.local/lib/python3.6/site-packages/sitecustomize.py # Set backend for matplotlib -ENV MPLBACKEND "agg" \ No newline at end of file +ENV MPLBACKEND "agg" From 16307753cd6e84fd4e55618abca2e474a7a7bfa8 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 23 Aug 2018 15:06:11 -0700 Subject: [PATCH 0365/1660] Typo in notebook version. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6f689c0f..d7a63932 100644 --- a/Dockerfile +++ b/Dockerfile @@ -432,7 +432,7 @@ RUN pip install bcolz && \ pip install mistune && \ pip install nbconvert && \ pip install nbformat && \ - pip install notebook==0.5.5 && \ + pip install notebook==5.5.0 && \ pip install numpy && \ pip install olefile && \ pip install opencv-python && \ From c2b8335832c3a01f0c61276639471ed80c7f9f88 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 24 Aug 2018 16:40:01 +0000 Subject: [PATCH 0366/1660] Install pytorch for cpu on master --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d7a63932..cb447b43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -235,7 +235,7 @@ RUN pip install scipy && \ # PyTorch export CXXFLAGS="-std=c++11" && \ export CFLAGS="-std=c99" && \ - conda install -y pytorch torchvision -c pytorch && \ + conda install -y pytorch-cpu torchvision-cpu -c pytorch && \ # PyTorch Audio apt-get install -y sox libsox-dev libsox-fmt-all && \ pip install cffi && \ From 4432a8f3c3b0e5d1f3476062dc93e4e6c2a41b22 Mon Sep 17 00:00:00 2001 From: Gyczero <2997347185@qq.com> Date: Sat, 1 Sep 2018 18:50:30 +0800 Subject: [PATCH 0367/1660] ADD support for gensim and jieba --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cb447b43..61a11394 100644 --- a/Dockerfile +++ b/Dockerfile @@ -156,7 +156,7 @@ RUN apt-get update && \ conda install basemap && \ # Pillow (PIL) apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev && \ - pip install Pillow + pip install Pillow RUN cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep.git && \ cd opendeep && python setup.py develop && \ @@ -495,6 +495,8 @@ RUN pip install flashtext && \ pip install pandasql && \ pip install trackml && \ pip install tensorflow_hub && \ + pip install gensim && \ + pip install jieba && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From 3fe25c22f6ca2cd1ee6da68913bfded9fc6e2912 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 11 Sep 2018 18:41:19 +0000 Subject: [PATCH 0368/1660] Install basemap from source --- Dockerfile | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb447b43..49c709c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -142,21 +142,23 @@ RUN apt-get install -y libfreetype6-dev && \ ENV LD_LIBRARY_PATH=/opt/conda/lib # Install Basemap via conda temporarily -RUN apt-get update && \ - #apt-get -y install libgeos-dev && \ - #pip install matplotlib && \ - #pip install pyshp && \ - #pip install pyproj && \ - #cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \ - #cd basemap/geos-3.3.3 && \ - #export GEOS_DIR=/usr/local && \ - #./configure --prefix=$GEOS_DIR && \ - #make && make install && \ - #cd .. && python setup.py install && \ - conda install basemap && \ - # Pillow (PIL) - apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev && \ - pip install Pillow +RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \ + # Pillow is an optional dependency of basemap + pip install Pillow && \ + pip install matplotlib && \ + pip install pyshp && \ + pip install pyproj && \ + cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \ + cd basemap && \ + git checkout v1.1.0 && \ + # Install geos + cd geos-3.3.3 && \ + export GEOS_DIR=/usr/local && \ + ./configure --prefix=$GEOS_DIR && \ + make && make install && \ + # Install basemap + cd .. && python setup.py install && \ + pip install basemap --no-binary basemap RUN cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep.git && \ cd opendeep && python setup.py develop && \ From 5095d04c40c4cd8df2860bb484637d2de1b2485c Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 13 Sep 2018 10:55:18 -0700 Subject: [PATCH 0369/1660] Move gensim after downgrading to python 3.6 Otherwise, the package gets silently removed. --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 49c709c1..100464f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & apt-get -y install cmake RUN pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ - husl geopy ml_metrics mne pyshp gensim && \ + husl geopy ml_metrics mne pyshp && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ python -m spacy download en_core_web_lg && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source @@ -65,6 +65,7 @@ RUN conda install -y python=3.6.6 && \ RUN apt-get install -y libfreetype6-dev && \ apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ + pip install gensim && \ # textblob pip install textblob && \ #word cloud From 3af3acbb1968266e646970336abcb4d4cd355aef Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 13 Sep 2018 15:08:56 -0700 Subject: [PATCH 0370/1660] Add PDPbox --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 100464f5..6317d1bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -498,6 +498,7 @@ RUN pip install flashtext && \ pip install pandasql && \ pip install trackml && \ pip install tensorflow_hub && \ + pip install PDPbox && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From adffdb69ba1e03dff47b7e89d955c927e1fc85a3 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 14 Sep 2018 10:01:37 -0700 Subject: [PATCH 0371/1660] add tests for gensim --- tests/test_gensim.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/test_gensim.py diff --git a/tests/test_gensim.py b/tests/test_gensim.py new file mode 100644 index 00000000..bf239017 --- /dev/null +++ b/tests/test_gensim.py @@ -0,0 +1,9 @@ +import unittest + +from gensim import corpora + +class TestGensim(unittest.TestCase): + def test_dictionary(self): + dic = corpora.Dictionary([['lorem', 'ipsum']]) + + self.assertEqual(2, len(dic.token2id)) From d4d81c2ebc4cad9d4be76c8704a2da65df7fcc65 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 14 Sep 2018 10:59:47 -0700 Subject: [PATCH 0372/1660] Remove duplicate install for opencv --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6317d1bc..c3f2f09c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,9 +32,6 @@ RUN pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick* -# OpenCV install (from pip or source) -RUN pip install opencv-python - RUN apt-get update && apt-get install -y python-software-properties zip && \ echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ From f087d24fe550f869f9dadb26ef31df760459852f Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 14 Sep 2018 11:03:34 -0700 Subject: [PATCH 0373/1660] Remove duplicate install for Pillow --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c3f2f09c..fe62e9eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -141,8 +141,6 @@ ENV LD_LIBRARY_PATH=/opt/conda/lib # Install Basemap via conda temporarily RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \ - # Pillow is an optional dependency of basemap - pip install Pillow && \ pip install matplotlib && \ pip install pyshp && \ pip install pyproj && \ From a175e913e08fca81245c94e86ad8f9e183c9b9a3 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 14 Sep 2018 11:14:23 -0700 Subject: [PATCH 0374/1660] Add ggplot --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index fe62e9eb..9f41bc9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -494,6 +494,7 @@ RUN pip install flashtext && \ pip install trackml && \ pip install tensorflow_hub && \ pip install PDPbox && \ + pip install ggplot && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From e8339b3f119e03429e85d6626871a38613b44aeb Mon Sep 17 00:00:00 2001 From: Michael Apers <32524673+perfectclear@users.noreply.github.com> Date: Fri, 14 Sep 2018 11:50:54 -0700 Subject: [PATCH 0375/1660] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 9f41bc9e..036d005b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -495,6 +495,7 @@ RUN pip install flashtext && \ pip install tensorflow_hub && \ pip install PDPbox && \ pip install ggplot && \ + pip install cesium && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From eb6742f9b5760d8a5357e5241541b3f02095a9bc Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 17 Sep 2018 16:03:21 +0000 Subject: [PATCH 0376/1660] Merge and improve gpu tests. Add annotations --- tests/__init__.py | 0 tests/common.py | 6 ++++++ tests/test_nvidia.py | 26 ++++++++++++++++++++++++++ tests/test_pytorch.py | 12 ++++++++++++ tests/test_tensorflow.py | 16 ++++++++++++++++ 5 files changed, 60 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/common.py create mode 100644 tests/test_nvidia.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/common.py b/tests/common.py new file mode 100644 index 00000000..823212a2 --- /dev/null +++ b/tests/common.py @@ -0,0 +1,6 @@ +"""Common testing setup""" + +import os +import unittest + +gpu_test = unittest.skipIf(len(os.environ.get('CUDA_VERSION', '')) == 0, 'Not running GPU tests') diff --git a/tests/test_nvidia.py b/tests/test_nvidia.py new file mode 100644 index 00000000..a6f3c428 --- /dev/null +++ b/tests/test_nvidia.py @@ -0,0 +1,26 @@ +"""Tests for general GPU support""" + +import os +import subprocess +import sys +import unittest + +import common +import pycuda.driver + +from common import gpu_test + + +class TestNvidia(unittest.TestCase): + @gpu_test + def test_system_management_interface(self): + smi = subprocess.Popen(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + smi.communicate() + self.assertEqual(0, smi.returncode) + + @gpu_test + def test_pycuda(self): + pycuda.driver.init() + gpu_name = pycuda.driver.Device(0).name() + self.assertNotEqual(0, len(gpu_name)) + diff --git a/tests/test_pytorch.py b/tests/test_pytorch.py index 4f250572..16b17364 100644 --- a/tests/test_pytorch.py +++ b/tests/test_pytorch.py @@ -4,6 +4,9 @@ import torch.nn as tnn import torch.autograd as autograd +from common import gpu_test + + class TestPyTorch(unittest.TestCase): # PyTorch smoke test based on http://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html def test_nn(self): @@ -11,3 +14,12 @@ def test_nn(self): linear_torch = tnn.Linear(5,3) data_torch = autograd.Variable(torch.randn(2, 5)) linear_torch(data_torch) + + @gpu_test + def test_gpu(self): + cuda = torch.device('cuda') + a = torch.tensor([1., 2.], device=cuda) + + result = a.sum() + + self.assertEqual(torch.tensor([3.], device=cuda), result) diff --git a/tests/test_tensorflow.py b/tests/test_tensorflow.py index 120b773d..10a66837 100644 --- a/tests/test_tensorflow.py +++ b/tests/test_tensorflow.py @@ -1,7 +1,11 @@ import unittest +import numpy as np import tensorflow as tf +from common import gpu_test + + class TestTensorflow(unittest.TestCase): def test_addition(self): op = tf.add(2, 3) @@ -10,3 +14,15 @@ def test_addition(self): result = sess.run(op) self.assertEqual(5, result) + + @gpu_test + def test_gpu(self): + with tf.device('/gpu:0'): + m1 = tf.constant([2.0, 3.0], shape=[1, 2], name='a') + m2 = tf.constant([3.0, 4.0], shape=[2, 1], name='b') + op = tf.matmul(m1, m2) + + sess = tf.Session() + result = sess.run(op) + + self.assertEqual(np.array(18, dtype=np.float32, ndmin=2), result) From f9f5ee8002c66e20ff04d35434156fedeb90dbfa Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 17 Sep 2018 20:52:42 +0000 Subject: [PATCH 0377/1660] Fix nvidia tests for CPU --- tests/test_nvidia.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_nvidia.py b/tests/test_nvidia.py index a6f3c428..b5ad61ce 100644 --- a/tests/test_nvidia.py +++ b/tests/test_nvidia.py @@ -5,9 +5,6 @@ import sys import unittest -import common -import pycuda.driver - from common import gpu_test @@ -19,7 +16,8 @@ def test_system_management_interface(self): self.assertEqual(0, smi.returncode) @gpu_test - def test_pycuda(self): + def test_pycuda(self): + import pycuda.driver pycuda.driver.init() gpu_name = pycuda.driver.Device(0).name() self.assertNotEqual(0, len(gpu_name)) From 228c99e27736236658c01a54d4e9128a2d4f1173 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 17 Sep 2018 20:53:24 +0000 Subject: [PATCH 0378/1660] Update build, test, push script to support gpu --- build | 29 ++++++++++++++++++++++++----- push | 25 ++++++++++++++++++++++--- test | 24 +++++++++++++++++++++--- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/build b/build index d3f34959..8d3fc81f 100755 --- a/build +++ b/build @@ -1,10 +1,29 @@ #!/bin/bash +# +# Build a new Python Docker image. +# +# Usage: +# ./build [--gpu] [--use-cache] +# +# Options: +# --gpu: Build an image with GPU support. +# --use-cache: Use layer cache when building a new image. +# set -e +set -x + +CACHE_FLAG="--no-cache" +IMAGE_TAG="kaggle/python-build" +DOCKERFILE="Dockerfile" + +if [[ "$1" == "--gpu" ]]; then + IMAGE_TAG="kaggle/python-gpu-build" + DOCKERFILE="gpu.Dockerfile" + shift +fi -# Default behavior is to do everything from scratch. -# The --use-cache option is useful if you're iterating on a broken build. if [[ "$1" == "--use-cache" ]]; then - docker build --rm -t kaggle/python-build . -else - docker build --pull --rm --no-cache -t kaggle/python-build . + CACHE_FLAG="" fi + +docker build --rm -t $CACHE_FLAG $IMAGE_TAG -f $DOCKERFILE . diff --git a/push b/push index ac3d2136..85f11611 100755 --- a/push +++ b/push @@ -1,13 +1,32 @@ #!/bin/bash +# +# Push a newly-built image with the given label to gcr.io and DockerHub. +# +# Usage: +# ./push [--gpu] [label] +# +# Description: +# label: Docker image label. Defaults to "testing". +# +# Options: +# --gpu: Psuh the image with GPU support. +# set -e -set +x +set -x + +IMAGE_TAG="kaggle/python-build:latest" + +if [[ "$1" == "--gpu" ]]; then + IMAGE_TAG="kaggle/python-gpu-build:latest" + shift +fi label=${1:-testing} -docker tag kaggle/python-build:latest gcr.io/kaggle-images/python:${label} +docker tag $IMAGE_TAG gcr.io/kaggle-images/python:${label} gcloud docker -- push gcr.io/kaggle-images/python:${label} if [[ "$label" == "latest" ]]; then - docker tag kaggle/python-build:latest kaggle/python:${label} + docker tag $IMAGE_TAG kaggle/python:${label} docker push kaggle/python:${label} fi diff --git a/test b/test index 2528b6af..080fc5ff 100755 --- a/test +++ b/test @@ -1,6 +1,24 @@ #!/bin/bash +# +# Run tests for a newly-built Python Docker image. +# By default, it runs the tests for the CPU image. +# +# Usage: +# ./test [--gpu] +# +# Options: +# --gpu: Run tests for the GPU image +# + set -e -set +x +set -x + +IMAGE_TAG="kaggle/python-build" + +if [[ "$1" == "--gpu" ]]; then + IMAGE_TAG="kaggle/python-gpu-build" + shift +fi rm -rf /tmp/python-build docker rm jupyter_test || true @@ -8,7 +26,7 @@ mkdir -p /tmp/python-build/tmp mkdir -p /tmp/python-build/devshm mkdir -p /tmp/python-build/working # Check that Jupyter server can run; if it dies on startup, the `docker kill` command will throw an error -docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm kaggle/python-build jupyter notebook --allow-root --ip="*" +docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm $IMAGE_TAG jupyter notebook --allow-root --ip="*" sleep 3 docker kill jupyter_test && docker rm jupyter_test docker run --rm -t --read-only --net=none \ @@ -18,5 +36,5 @@ docker run --rm -t --read-only --net=none \ -v $PWD:/input:ro -v /tmp/python-build/working:/working \ -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm \ -w=/working \ - kaggle/python-build \ + $IMAGE_TAG \ /bin/bash -c 'python -m unittest discover -s /input/tests' From 18b07e269a53272ba189819bcf5a8809f8dbb062 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 17 Sep 2018 23:20:29 +0000 Subject: [PATCH 0379/1660] fix push and test script --- push | 23 +++++++++++++---------- test | 3 +++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/push b/push index 85f11611..a938abb5 100755 --- a/push +++ b/push @@ -3,10 +3,10 @@ # Push a newly-built image with the given label to gcr.io and DockerHub. # # Usage: -# ./push [--gpu] [label] +# ./push [--gpu] [LABEL] # # Description: -# label: Docker image label. Defaults to "testing". +# LABEL: Docker image label. Defaults to "testing". # # Options: # --gpu: Psuh the image with GPU support. @@ -14,19 +14,22 @@ set -e set -x -IMAGE_TAG="kaggle/python-build:latest" +SOURCE_IMAGE="kaggle/python-build" +TARGET_IMAGE="gcr.io/kaggle-images/python" if [[ "$1" == "--gpu" ]]; then - IMAGE_TAG="kaggle/python-gpu-build:latest" + SOURCE_IMAGE="kaggle/python-gpu-build" + TARGET_IMAGE="gcr.io/kaggle-private-byod/python" shift fi -label=${1:-testing} +LABEL=${1:-testing} -docker tag $IMAGE_TAG gcr.io/kaggle-images/python:${label} -gcloud docker -- push gcr.io/kaggle-images/python:${label} +docker tag $SOURCE_IMAGE:latest $TARGET_IMAGE:$LABEL +gcloud docker -- push $TARGET_IMAGE:$LABEL -if [[ "$label" == "latest" ]]; then - docker tag $IMAGE_TAG kaggle/python:${label} - docker push kaggle/python:${label} +# Only CPU images are made public at this time. +if [[ "$LABEL" == "latest" && SOURCE_IMAGE = "kaggle/python-build" ]]; then + docker tag $SOURCE_IMAGE:latest kaggle/python:$LABEL + docker push kaggle/python:$LABEL fi diff --git a/test b/test index 080fc5ff..b2086cc8 100755 --- a/test +++ b/test @@ -14,9 +14,11 @@ set -e set -x IMAGE_TAG="kaggle/python-build" +ADDITONAL_OPTS="" if [[ "$1" == "--gpu" ]]; then IMAGE_TAG="kaggle/python-gpu-build" + ADDITONAL_OPTS="-v /tmp/empty_dir:/usr/local/cuda/lib64/stubs:ro" shift fi @@ -36,5 +38,6 @@ docker run --rm -t --read-only --net=none \ -v $PWD:/input:ro -v /tmp/python-build/working:/working \ -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm \ -w=/working \ + $ADDITONAL_OPTS \ $IMAGE_TAG \ /bin/bash -c 'python -m unittest discover -s /input/tests' From c57c0896a88f45fce43ed9f8aee0bb8cacb66a7e Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 18 Sep 2018 21:41:50 +0000 Subject: [PATCH 0380/1660] improve pytorch gpu tests --- tests/test_pytorch.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_pytorch.py b/tests/test_pytorch.py index 16b17364..61dd71ff 100644 --- a/tests/test_pytorch.py +++ b/tests/test_pytorch.py @@ -16,10 +16,20 @@ def test_nn(self): linear_torch(data_torch) @gpu_test - def test_gpu(self): + def test_gpu_computation(self): cuda = torch.device('cuda') a = torch.tensor([1., 2.], device=cuda) result = a.sum() self.assertEqual(torch.tensor([3.], device=cuda), result) + + @gpu_test + def test_cuda_nn(self): + # These throw if cuda is misconfigured + tnn.GRUCell(10,10).cuda() + tnn.RNNCell(10,10).cuda() + tnn.LSTMCell(10,10).cuda() + tnn.GRU(10,10).cuda() + tnn.LSTM(10,10).cuda() + tnn.RNN(10,10).cuda() From fb787224199d5ecd09f8724f13ec5aaaede4cfdd Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 18 Sep 2018 21:46:55 +0000 Subject: [PATCH 0381/1660] fix build command --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 8d3fc81f..c2b44598 100755 --- a/build +++ b/build @@ -26,4 +26,4 @@ if [[ "$1" == "--use-cache" ]]; then CACHE_FLAG="" fi -docker build --rm -t $CACHE_FLAG $IMAGE_TAG -f $DOCKERFILE . +docker build --rm $CACHE_FLAG -t $IMAGE_TAG -f $DOCKERFILE . From ba4a5237f2009c7c5a785f0dde864a9e1954920b Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 18 Sep 2018 21:48:46 +0000 Subject: [PATCH 0382/1660] Install tensorflow from pre-built wheel --- Dockerfile | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 45290385..1e2dce6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,17 @@ +FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl FROM continuumio/anaconda3:5.0.1 +# This is necessary to for apt to access HTTPS sources +RUN apt-get update && \ + apt-get install apt-transport-https + ADD patches/ /tmp/patches/ ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl # Use a fixed apt-get repo to stop intermittent failures due to flaky httpredir connections, # as described by Lionel Chan at http://stackoverflow.com/a/37426929/5881346 RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list && \ - apt-get update && apt-get install -y build-essential && \ + apt-get update && apt-get install -y build-essential unzip && \ # https://stackoverflow.com/a/46498173 conda update -y conda && conda update -y python && \ pip install --upgrade pip && \ @@ -32,33 +37,18 @@ RUN pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick* -RUN apt-get update && apt-get install -y python-software-properties zip && \ - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ - echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ - apt-get update && \ - echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ - echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ - apt-get install -y oracle-java8-installer && \ - echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ - curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ - apt-get update && apt-get install -y bazel && \ - apt-get upgrade -y bazel - # Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517 # Fix to install tf 1.10:: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2 RUN conda install -y python=3.6.6 && \ pip install pandas==0.23.2 && \ # Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518 pip install keras_applications==1.0.4 --no-deps && \ - pip install keras_preprocessing==1.0.2 --no-deps && \ - cd /usr/local/src && \ - git clone https://github.com/tensorflow/tensorflow && \ - cd tensorflow && \ - cat /dev/null | ./configure && \ - bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ - bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg && \ - pip install /tmp/tensorflow_pkg/tensorflow*.whl + pip install keras_preprocessing==1.0.2 --no-deps + +# Install tensorflow from a pre-built wheel +COPY --from=tensorflow_whl /tmp/tensorflow_cpu/*.whl /tmp/tensorflow_cpu/ +RUN pip install /tmp/tensorflow_cpu/tensorflow*.whl && \ + rm -rf /tmp/tensorflow_cpu RUN apt-get install -y libfreetype6-dev && \ apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ From 5ee32f807974de2d452e8d251a54d071eccdff72 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 18 Sep 2018 21:49:10 +0000 Subject: [PATCH 0383/1660] GPU Dockerfile --- gpu.Dockerfile | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 gpu.Dockerfile diff --git a/gpu.Dockerfile b/gpu.Dockerfile new file mode 100644 index 00000000..89840ce0 --- /dev/null +++ b/gpu.Dockerfile @@ -0,0 +1,48 @@ +FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 AS nvidia +FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl +FROM kaggle/python-build + +# Cuda support +COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ +COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ +COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg + +ENV CUDA_VERSION=9.1.85 +ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 +LABEL com.nvidia.volumes.needed="nvidia_driver" +LABEL com.nvidia.cuda.version="${CUDA_VERSION}" +ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +# The stub is useful to us both for built-time linking and run-time linking, on CPU-only systems. +# When intended to be used with actual GPUs, make sure to (besides providing access to the host +# CUDA user libraries, either manually or through the use of nvidia-docker) exclude them. One +# convenient way to do so is to obscure its contents by a bind mount: +# docker run .... -v /non-existing-directory:/usr/local/cuda/lib64/stubs:ro ... +ENV LD_LIBRARY_PATH="/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs" +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility +ENV NVIDIA_REQUIRE_CUDA="cuda>=9.0" +RUN apt-get update && apt-get install -y --no-install-recommends \ + cuda-cudart-$CUDA_PKG_VERSION \ + cuda-libraries-$CUDA_PKG_VERSION \ + cuda-libraries-dev-$CUDA_PKG_VERSION \ + cuda-nvml-dev-$CUDA_PKG_VERSION \ + cuda-minimal-build-$CUDA_PKG_VERSION \ + cuda-command-line-tools-$CUDA_PKG_VERSION \ + libcudnn7=7.0.5.15-1+cuda9.1 \ + libcudnn7-dev=7.0.5.15-1+cuda9.1 \ + libnccl2=2.2.12-1+cuda9.1 \ + libnccl-dev=2.2.12-1+cuda9.1 && \ + ln -s /usr/local/cuda-9.1 /usr/local/cuda && \ + ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 + +# Reinstall packages with a separate version for GPU support +# Tensorflow +COPY --from=tensorflow_whl /tmp/tensorflow_gpu/*.whl /tmp/tensorflow_gpu/ +RUN pip uninstall -y tensorflow && \ + pip install /tmp/tensorflow_gpu/tensorflow*.whl && \ + rm -rf /tmp/tensorflow_gpu && \ + conda uninstall -y pytorch-cpu torchvision-cpu && \ + conda install -y pytorch torchvision -c pytorch + +# Install GPU-only packages +RUN pip install pycuda From c8dd5650433de9db922457d537fa34d29b706e23 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 18 Sep 2018 21:57:56 +0000 Subject: [PATCH 0384/1660] Update Jenkins config to support GPU build --- Jenkinsfile | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cef5a0b4..00996d1a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,7 @@ pipeline { } stages { - stage('Docker Build') { + stage('Docker CPU Build') { steps { slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash @@ -28,7 +28,7 @@ pipeline { } } - stage('Test Image') { + stage('Test CPU Image') { steps { slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} test image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash @@ -39,8 +39,32 @@ pipeline { ''' } } + + stage('Docker GPU Build') { + steps { + slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + sh '''#!/bin/bash + set -exo pipefail + + ./build --gpu | ts + ''' + } + } + + stage('Test GPU Image') { + agent { label 'ephemeral-linux-gpu' } + steps { + slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} test image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + sh '''#!/bin/bash + set -exo pipefail + + date + ./test --gpu + ''' + } + } - stage('Push Image') { + stage('Push Images') { steps { slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} pushing image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash @@ -48,6 +72,7 @@ pipeline { date ./push staging + ./push --gpu staging ''' } } From 6cfb569a70eecb4264f9a398a76d63a58c51e79c Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 19 Sep 2018 17:00:59 +0000 Subject: [PATCH 0385/1660] Fix typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1e2dce6e..695aa187 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl FROM continuumio/anaconda3:5.0.1 -# This is necessary to for apt to access HTTPS sources +# This is necessary for apt to access HTTPS sources RUN apt-get update && \ apt-get install apt-transport-https From 408cc24ef2ed6e705b75ad472958304398f415d1 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 19 Sep 2018 17:01:21 +0000 Subject: [PATCH 0386/1660] Make our helper scripts more robus --- build | 60 +++++++++++++++++++++++++++++++++++++++-------------------- push | 60 ++++++++++++++++++++++++++++++++++++++--------------------- test | 47 +++++++++++++++++++++++++++++++++++++--------- 3 files changed, 117 insertions(+), 50 deletions(-) diff --git a/build b/build index c2b44598..8bcd1a0e 100755 --- a/build +++ b/build @@ -1,29 +1,49 @@ #!/bin/bash -# -# Build a new Python Docker image. -# -# Usage: -# ./build [--gpu] [--use-cache] -# -# Options: -# --gpu: Build an image with GPU support. -# --use-cache: Use layer cache when building a new image. -# set -e -set -x + +usage() { +cat << EOF +Usage: $0 [OPTIONS] +Build a new Python Docker image. + +Options: + -g, --gpu Build an image with GPU support. + -c, --use-cache Use layer cache when building a new image. +EOF +} CACHE_FLAG="--no-cache" -IMAGE_TAG="kaggle/python-build" DOCKERFILE="Dockerfile" +IMAGE_TAG="kaggle/python-build" + +while :; do + case "$1" in + -h|--help) + usage + exit + ;; + -g|--gpu) + IMAGE_TAG="kaggle/python-gpu-build" + DOCKERFILE="gpu.Dockerfile" + ;; + -c|--use-cache) + CACHE_FLAG="" + ;; + -?*) + usage + printf 'ERROR: Unknown option: %s\n' "$1" >&2 + exit + ;; + *) + break + esac -if [[ "$1" == "--gpu" ]]; then - IMAGE_TAG="kaggle/python-gpu-build" - DOCKERFILE="gpu.Dockerfile" shift -fi +done -if [[ "$1" == "--use-cache" ]]; then - CACHE_FLAG="" -fi +readonly CACHE_FLAG +readonly DOCKERFILE +readonly IMAGE_TAG -docker build --rm $CACHE_FLAG -t $IMAGE_TAG -f $DOCKERFILE . +set -x +docker build --rm $CACHE_FLAG -t "$IMAGE_TAG" -f "$DOCKERFILE" . diff --git a/push b/push index a938abb5..79ad8f13 100755 --- a/push +++ b/push @@ -1,35 +1,53 @@ #!/bin/bash -# -# Push a newly-built image with the given label to gcr.io and DockerHub. -# -# Usage: -# ./push [--gpu] [LABEL] -# -# Description: -# LABEL: Docker image label. Defaults to "testing". -# -# Options: -# --gpu: Psuh the image with GPU support. -# set -e -set -x + +usage() { +cat << EOF +Usage: $0 [OPTIONS] [LABEL] +Push a newly-built image with the given LABEL to gcr.io and DockerHub. + +Options: + -g, --gpu Push the image with GPU support. +EOF +} SOURCE_IMAGE="kaggle/python-build" TARGET_IMAGE="gcr.io/kaggle-images/python" -if [[ "$1" == "--gpu" ]]; then - SOURCE_IMAGE="kaggle/python-gpu-build" - TARGET_IMAGE="gcr.io/kaggle-private-byod/python" +while :; do + case "$1" in + -h|--help) + usage + exit + ;; + -g|--gpu) + SOURCE_IMAGE="kaggle/python-gpu-build" + TARGET_IMAGE="gcr.io/kaggle-private-byod/python" + ;; + -?*) + usage + printf 'ERROR: Unknown option: %s\n' "$1" >&2 + exit + ;; + *) + break + esac + shift -fi +done LABEL=${1:-testing} -docker tag $SOURCE_IMAGE:latest $TARGET_IMAGE:$LABEL -gcloud docker -- push $TARGET_IMAGE:$LABEL +readonly SOURCE_IMAGE +readonly TARGET_IMAGE +readonly LABEL + +set -x +docker tag "${SOURCE_IMAGE}:latest" "${TARGET_IMAGE}:${LABEL}" +gcloud docker -- push "${TARGET_IMAGE}:${LABEL}" # Only CPU images are made public at this time. if [[ "$LABEL" == "latest" && SOURCE_IMAGE = "kaggle/python-build" ]]; then - docker tag $SOURCE_IMAGE:latest kaggle/python:$LABEL - docker push kaggle/python:$LABEL + docker tag "${SOURCE_IMAGE}:latest" "kaggle/python:${LABEL}" + docker push "kaggle/python:${LABEL}" fi diff --git a/test b/test index b2086cc8..c336bb5e 100755 --- a/test +++ b/test @@ -1,7 +1,7 @@ #!/bin/bash # -# Run tests for a newly-built Python Docker image. -# By default, it runs the tests for the CPU image. +# +# # # Usage: # ./test [--gpu] @@ -9,19 +9,48 @@ # Options: # --gpu: Run tests for the GPU image # - set -e -set -x + +usage() { +cat << EOF +Usage: $0 [OPTIONS] +Run tests for a newly-built Python Docker image. +By default, it runs the tests for the CPU image. + +Options: + -g, --gpu Run tests for the GPU image. +EOF +} IMAGE_TAG="kaggle/python-build" ADDITONAL_OPTS="" -if [[ "$1" == "--gpu" ]]; then - IMAGE_TAG="kaggle/python-gpu-build" - ADDITONAL_OPTS="-v /tmp/empty_dir:/usr/local/cuda/lib64/stubs:ro" +while :; do + case "$1" in + -h|--help) + usage + exit + ;; + -g|--gpu) + IMAGE_TAG="kaggle/python-gpu-build" + ADDITONAL_OPTS="-v /tmp/empty_dir:/usr/local/cuda/lib64/stubs:ro" + ;; + -?*) + usage + printf 'ERROR: Unknown option: %s\n' "$1" >&2 + exit + ;; + *) + break + esac + shift -fi +done +readonly IMAGE_TAG +readonly ADDITONAL_OPTS + +set -x rm -rf /tmp/python-build docker rm jupyter_test || true mkdir -p /tmp/python-build/tmp @@ -39,5 +68,5 @@ docker run --rm -t --read-only --net=none \ -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm \ -w=/working \ $ADDITONAL_OPTS \ - $IMAGE_TAG \ + "$IMAGE_TAG" \ /bin/bash -c 'python -m unittest discover -s /input/tests' From 1c2c907ce9f9d7e3f7a6ec1e3be01002b0caf068 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 19 Sep 2018 17:06:42 +0000 Subject: [PATCH 0387/1660] Remove extra comments --- test | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test b/test index c336bb5e..2545ce39 100755 --- a/test +++ b/test @@ -1,14 +1,4 @@ #!/bin/bash -# -# -# -# -# Usage: -# ./test [--gpu] -# -# Options: -# --gpu: Run tests for the GPU image -# set -e usage() { From 77739d38954caa0fdc1cb34907b3e563878f7d81 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 19 Sep 2018 17:09:06 +0000 Subject: [PATCH 0388/1660] Minor fixes --- test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test b/test index 2545ce39..42bc2c11 100755 --- a/test +++ b/test @@ -47,7 +47,7 @@ mkdir -p /tmp/python-build/tmp mkdir -p /tmp/python-build/devshm mkdir -p /tmp/python-build/working # Check that Jupyter server can run; if it dies on startup, the `docker kill` command will throw an error -docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm $IMAGE_TAG jupyter notebook --allow-root --ip="*" +docker run -d --name=jupyter_test --read-only --net=none -e HOME=/tmp -v $PWD:/input:ro -v /tmp/python-build/working:/working -w=/working -v /tmp/python-build/tmp:/tmp -v /tmp/python-build/devshm:/dev/shm "$IMAGE_TAG" jupyter notebook --allow-root --ip="*" sleep 3 docker kill jupyter_test && docker rm jupyter_test docker run --rm -t --read-only --net=none \ From 9cb9f600687c14e2a3921c0505fc7cc0afddbafb Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 19 Sep 2018 18:36:35 +0000 Subject: [PATCH 0389/1660] Fix Jenkins config --- Jenkinsfile | 24 ++++++++++++++++++++++-- gpu.Dockerfile | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 00996d1a..07a455da 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,6 +3,7 @@ String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1-5" : "" pipeline { agent { label 'ephemeral-linux' } options { + // The Build GPU stage depends on the image from the Push CPU stage disableConcurrentBuilds() } triggers { @@ -39,8 +40,27 @@ pipeline { ''' } } + + stage('Push CPU Image') { + steps { + slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} pushing image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL + sh '''#!/bin/bash + set -exo pipefail + + date + ./push staging + ''' + } + } stage('Docker GPU Build') { + // A GPU is not required to build this image. However, in our current setup, + // the default runtime is set to nvidia (as opposed to runc) and there + // is no option to specify a runtime for the `docker build` command. + // + // TODO(rosbo) don't set `nvidia` as the default runtime and use the + // `--runtime=nvidia` flag for the `docker run` command when GPU support is needed. + agent { label 'ephemeral-linux-gpu' } steps { slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash @@ -64,14 +84,14 @@ pipeline { } } - stage('Push Images') { + stage('Push GPU Image') { + agent { label 'ephemeral-linux-gpu' } steps { slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} pushing image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash set -exo pipefail date - ./push staging ./push --gpu staging ''' } diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 89840ce0..6124280f 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -1,6 +1,6 @@ FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 AS nvidia FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl -FROM kaggle/python-build +FROM gcr.io/kaggle-images/python:staging # Cuda support COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ From c17404600129b767bc04b8bece3441fdb3c5ddaf Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 20 Sep 2018 10:59:51 -0700 Subject: [PATCH 0390/1660] Remove duplicate install of seaborn already being install at line 20. --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 695aa187..481b656a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -439,7 +439,6 @@ RUN pip install bcolz && \ pip install PyYAML && \ pip install pyzmq && \ pip install qtconsole && \ - pip install seaborn && \ pip install simplegeneric && \ pip install six && \ pip install terminado && \ From 8ff8232d975e2b0a4b8cc11c4083624d2cb9b774 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 21 Sep 2018 11:22:44 -0700 Subject: [PATCH 0391/1660] Update Dockerfile --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 481b656a..4560b620 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,7 +63,10 @@ RUN apt-get install -y libfreetype6-dev && \ cd /usr/local/src && mkdir xgboost && cd xgboost && \ git clone --depth 1 --recursive https://github.com/dmlc/xgboost.git && cd xgboost && \ make && cd python-package && python setup.py install && \ - pip install lightgbm && \ + # Pinning to an older version. The latest release (2.2.0) depends on an old GLIBC version. + # This cause the package to fail at import time with "version `GLIBC_2.23' not found" + # TODO: Unpin once the latest release is fixed. + pip install lightgbm==2.1.2 && \ #lasagne cd /usr/local/src && mkdir Lasagne && cd Lasagne && \ git clone --depth 1 https://github.com/Lasagne/Lasagne.git && cd Lasagne && \ From 8085c063ffcdfa60e24f17a8cd961d25f116a5d5 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 21 Sep 2018 18:53:17 +0000 Subject: [PATCH 0392/1660] Add test for the lightgbm package --- tests/test_lightgbm.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/test_lightgbm.py diff --git a/tests/test_lightgbm.py b/tests/test_lightgbm.py new file mode 100644 index 00000000..5727cacd --- /dev/null +++ b/tests/test_lightgbm.py @@ -0,0 +1,33 @@ +import unittest + +import lightgbm as lgb + +from sklearn.datasets import load_iris + +class TestLightgbm(unittest.TestCase): + # Based on the "simple_example" from their documentation: + # https://github.com/Microsoft/LightGBM/blob/master/examples/python-guide/simple_example.py + def test_simple(self): + # Load a dataset aleady on disk + iris = load_iris() + + lgb_train = lgb.Dataset(iris.data[:100], iris.target[:100]) + lgb_eval = lgb.Dataset(iris.data[100:], iris.target[100:], reference=lgb_train) + + params = { + 'task': 'train', + 'boosting_type': 'gbdt', + 'objective': 'regression', + 'metric': {'l2', 'auc'}, + 'num_leaves': 31, + 'learning_rate': 0.05, + 'feature_fraction': 0.9, + 'bagging_fraction': 0.8, + 'bagging_freq': 5, + 'verbose': 0 + } + + # Run only one round for faster test + gbm = lgb.train(params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, early_stopping_rounds=1) + + self.assertEqual(1, gbm.best_iteration) From 4fb5b680af7954806ab6d372033834c42d505918 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 21 Sep 2018 18:56:34 +0000 Subject: [PATCH 0393/1660] formatting --- tests/test_lightgbm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_lightgbm.py b/tests/test_lightgbm.py index 5727cacd..213eb0ca 100644 --- a/tests/test_lightgbm.py +++ b/tests/test_lightgbm.py @@ -28,6 +28,10 @@ def test_simple(self): } # Run only one round for faster test - gbm = lgb.train(params, lgb_train, num_boost_round=1, valid_sets=lgb_eval, early_stopping_rounds=1) + gbm = lgb.train(params, + lgb_train, + num_boost_round=1, + valid_sets=lgb_eval, + early_stopping_rounds=1) self.assertEqual(1, gbm.best_iteration) From a15031bac4beba79ec44bf52e261e9b0616be0fd Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 21 Sep 2018 14:09:29 -0700 Subject: [PATCH 0394/1660] Always pull the latest parent images at build --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 8bcd1a0e..0b8c662b 100755 --- a/build +++ b/build @@ -46,4 +46,4 @@ readonly DOCKERFILE readonly IMAGE_TAG set -x -docker build --rm $CACHE_FLAG -t "$IMAGE_TAG" -f "$DOCKERFILE" . +docker build --rm --pull $CACHE_FLAG -t "$IMAGE_TAG" -f "$DOCKERFILE" . From 561f3abad225f95869eefcd85e6528a48d4e7f5a Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 20 Sep 2018 21:22:42 +0000 Subject: [PATCH 0395/1660] Expand keras tests to cover conv net --- tests/test_keras.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/test_keras.py b/tests/test_keras.py index ca107563..cdaa7cb9 100644 --- a/tests/test_keras.py +++ b/tests/test_keras.py @@ -1,10 +1,11 @@ import unittest +import numpy as np import pandas as pd from keras.models import Sequential -from keras.layers import Dense -from keras.optimizers import RMSprop +from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D +from keras.optimizers import RMSprop, SGD from keras.utils.np_utils import to_categorical class TestKeras(unittest.TestCase): @@ -23,3 +24,34 @@ def test_train(self): metrics=['accuracy']) model.fit(x_train, y_train, epochs=1, batch_size=32) + + def test_conv2d(self): + # Generate dummy data + x_train = np.random.random((100, 100, 100, 3)) + y_train = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10) + x_test = np.random.random((20, 100, 100, 3)) + y_test = keras.utils.to_categorical(np.random.randint(10, size=(20, 1)), num_classes=10) + + model = Sequential() + # input: 100x100 images with 3 channels -> (100, 100, 3) tensors. + # this applies 32 convolution filters of size 3x3 each. + model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(100, 100, 3))) + model.add(Conv2D(32, (3, 3), activation='relu')) + model.add(MaxPooling2D(pool_size=(2, 2))) + model.add(Dropout(0.25)) + + model.add(Conv2D(64, (3, 3), activation='relu')) + model.add(Conv2D(64, (3, 3), activation='relu')) + model.add(MaxPooling2D(pool_size=(2, 2))) + model.add(Dropout(0.25)) + + model.add(Flatten()) + model.add(Dense(256, activation='relu')) + model.add(Dropout(0.5)) + model.add(Dense(10, activation='softmax')) + + sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) + model.compile(loss='categorical_crossentropy', optimizer=sgd) + + model.fit(x_train, y_train, batch_size=32, epochs=1) + score = model.evaluate(x_test, y_test, batch_size=32) From f9386a24daf660bf6c583016cf63a8d7ea668385 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 20 Sep 2018 22:40:20 +0000 Subject: [PATCH 0396/1660] use libcudnn7.2 and cuda 9.2 --- gpu.Dockerfile | 30 ++++++++++++++++-------------- tests/test_keras.py | 1 + 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 6124280f..45515596 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 AS nvidia +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 AS nvidia FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl FROM gcr.io/kaggle-images/python:staging @@ -7,8 +7,8 @@ COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg -ENV CUDA_VERSION=9.1.85 -ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 +ENV CUDA_VERSION=9.2.88 +ENV CUDA_PKG_VERSION=9-2=$CUDA_VERSION-1 LABEL com.nvidia.volumes.needed="nvidia_driver" LABEL com.nvidia.cuda.version="${CUDA_VERSION}" ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} @@ -22,17 +22,19 @@ ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility ENV NVIDIA_REQUIRE_CUDA="cuda>=9.0" RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-cudart-$CUDA_PKG_VERSION \ - cuda-libraries-$CUDA_PKG_VERSION \ - cuda-libraries-dev-$CUDA_PKG_VERSION \ - cuda-nvml-dev-$CUDA_PKG_VERSION \ - cuda-minimal-build-$CUDA_PKG_VERSION \ - cuda-command-line-tools-$CUDA_PKG_VERSION \ - libcudnn7=7.0.5.15-1+cuda9.1 \ - libcudnn7-dev=7.0.5.15-1+cuda9.1 \ - libnccl2=2.2.12-1+cuda9.1 \ - libnccl-dev=2.2.12-1+cuda9.1 && \ - ln -s /usr/local/cuda-9.1 /usr/local/cuda && \ + cuda-cupti-$CUDA_PKG_VERSION \ + cuda-cudart-$CUDA_PKG_VERSION \ + cuda-cudart-dev-$CUDA_PKG_VERSION \ + cuda-libraries-$CUDA_PKG_VERSION \ + cuda-libraries-dev-$CUDA_PKG_VERSION \ + cuda-nvml-dev-$CUDA_PKG_VERSION \ + cuda-minimal-build-$CUDA_PKG_VERSION \ + cuda-command-line-tools-$CUDA_PKG_VERSION \ + libcudnn7=7.2.1.38-1+cuda9.2 \ + libcudnn7-dev=7.2.1.38-1+cuda9.2 \ + libnccl2=2.2.13-1+cuda9.2 \ + libnccl-dev=2.2.13-1+cuda9.2 && \ + ln -s /usr/local/cuda-9.2 /usr/local/cuda && \ ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 # Reinstall packages with a separate version for GPU support diff --git a/tests/test_keras.py b/tests/test_keras.py index cdaa7cb9..fc6448e3 100644 --- a/tests/test_keras.py +++ b/tests/test_keras.py @@ -1,5 +1,6 @@ import unittest +import keras import numpy as np import pandas as pd From 626ba4cbd0bf3043446ab26b8fb8c778cafa0d32 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 21 Sep 2018 17:12:20 +0000 Subject: [PATCH 0397/1660] Use libcudnn 7.2 --- gpu.Dockerfile | 31 +++++++++++++++---------------- tests/test_keras.py | 8 +++++--- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 45515596..247c7dd4 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 AS nvidia +FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 AS nvidia FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl FROM gcr.io/kaggle-images/python:staging @@ -7,8 +7,8 @@ COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg -ENV CUDA_VERSION=9.2.88 -ENV CUDA_PKG_VERSION=9-2=$CUDA_VERSION-1 +ENV CUDA_VERSION=9.1.85 +ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 LABEL com.nvidia.volumes.needed="nvidia_driver" LABEL com.nvidia.cuda.version="${CUDA_VERSION}" ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} @@ -22,19 +22,18 @@ ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility ENV NVIDIA_REQUIRE_CUDA="cuda>=9.0" RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-cupti-$CUDA_PKG_VERSION \ - cuda-cudart-$CUDA_PKG_VERSION \ - cuda-cudart-dev-$CUDA_PKG_VERSION \ - cuda-libraries-$CUDA_PKG_VERSION \ - cuda-libraries-dev-$CUDA_PKG_VERSION \ - cuda-nvml-dev-$CUDA_PKG_VERSION \ - cuda-minimal-build-$CUDA_PKG_VERSION \ - cuda-command-line-tools-$CUDA_PKG_VERSION \ - libcudnn7=7.2.1.38-1+cuda9.2 \ - libcudnn7-dev=7.2.1.38-1+cuda9.2 \ - libnccl2=2.2.13-1+cuda9.2 \ - libnccl-dev=2.2.13-1+cuda9.2 && \ - ln -s /usr/local/cuda-9.2 /usr/local/cuda && \ + cuda-cupti-$CUDA_PKG_VERSION \ + cuda-cudart-$CUDA_PKG_VERSION \ + cuda-libraries-$CUDA_PKG_VERSION \ + cuda-libraries-dev-$CUDA_PKG_VERSION \ + cuda-nvml-dev-$CUDA_PKG_VERSION \ + cuda-minimal-build-$CUDA_PKG_VERSION \ + cuda-command-line-tools-$CUDA_PKG_VERSION \ + libcudnn7=7.2.1.38-1+cuda9.0 \ + libcudnn7-dev=7.2.1.38-1+cuda9.0 \ + libnccl2=2.2.12-1+cuda9.1 \ + libnccl-dev=2.2.12-1+cuda9.1 && \ + ln -s /usr/local/cuda-9.1 /usr/local/cuda && \ ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 # Reinstall packages with a separate version for GPU support diff --git a/tests/test_keras.py b/tests/test_keras.py index fc6448e3..9f8a1e4c 100644 --- a/tests/test_keras.py +++ b/tests/test_keras.py @@ -25,7 +25,8 @@ def test_train(self): metrics=['accuracy']) model.fit(x_train, y_train, epochs=1, batch_size=32) - + + # Uses convnet which depends on libcudnn when running on GPU def test_conv2d(self): # Generate dummy data x_train = np.random.random((100, 100, 100, 3)) @@ -52,7 +53,8 @@ def test_conv2d(self): model.add(Dense(10, activation='softmax')) sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) - model.compile(loss='categorical_crossentropy', optimizer=sgd) + # This throws if libcudnn is not properly installed with on a GPU + model.compile(loss='categorical_crossentropy', optimizer=sgd) model.fit(x_train, y_train, batch_size=32, epochs=1) - score = model.evaluate(x_test, y_test, batch_size=32) + model.evaluate(x_test, y_test, batch_size=32) From ec25fd3099e5c921c9818cbafe48a520e991d87a Mon Sep 17 00:00:00 2001 From: Gyczero <2997347185@qq.com> Date: Sat, 22 Sep 2018 18:53:54 +0800 Subject: [PATCH 0398/1660] ADD unit test for jieba --- tests/test_jieba.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/test_jieba.py diff --git a/tests/test_jieba.py b/tests/test_jieba.py new file mode 100644 index 00000000..745545f0 --- /dev/null +++ b/tests/test_jieba.py @@ -0,0 +1,10 @@ +# encoding=utf-8 +import unittest + +import jieba + + +class TestJieba(unittest.TestCase): + def test_text_split(self): + sentence = "我爱北京天安门" + seg_list = jieba.cut(sentence) \ No newline at end of file From 3d3aeb69ed8ae29e3ecc87d9bcbf240619f0e698 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 1 Oct 2018 11:00:40 -0700 Subject: [PATCH 0399/1660] Prevent disk from filling up on GPU build The GPU build is still using a permanent worker (not ephemeral like the CPU build). Docker images are building up and end up taking all the space and the build start failing. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 07a455da..5e32b76c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,7 +65,7 @@ pipeline { slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash set -exo pipefail - + docker image prune # remove previously built image to prevent disk from filling up ./build --gpu | ts ''' } From 321eeb52d623138d3191d01b146bd17c0a2327cb Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 1 Oct 2018 16:23:07 -0700 Subject: [PATCH 0400/1660] Remove all untagged images, not just dangling Context: gpu build still uses a permanent worker. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5e32b76c..702898fb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,7 +65,7 @@ pipeline { slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash set -exo pipefail - docker image prune # remove previously built image to prevent disk from filling up + docker image prune -a # remove previously built image to prevent disk from filling up ./build --gpu | ts ''' } From 560af468065cfd51c3b96d6526a19fe5d2817715 Mon Sep 17 00:00:00 2001 From: Gyczero <2997347185@qq.com> Date: Tue, 2 Oct 2018 12:18:13 +0800 Subject: [PATCH 0401/1660] Add an assertion for jieba --- .../inspectionProfiles/Project_Default.xml | 14 + .../inspectionProfiles/profiles_settings.xml | 7 + tests/.idea/misc.xml | 14 + tests/.idea/modules.xml | 8 + tests/.idea/tests.iml | 11 + tests/.idea/workspace.xml | 698 ++++++++++++++++++ tests/test_jieba.py | 5 +- 7 files changed, 756 insertions(+), 1 deletion(-) create mode 100644 tests/.idea/inspectionProfiles/Project_Default.xml create mode 100644 tests/.idea/inspectionProfiles/profiles_settings.xml create mode 100644 tests/.idea/misc.xml create mode 100644 tests/.idea/modules.xml create mode 100644 tests/.idea/tests.iml create mode 100644 tests/.idea/workspace.xml diff --git a/tests/.idea/inspectionProfiles/Project_Default.xml b/tests/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000..7700bea2 --- /dev/null +++ b/tests/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,14 @@ + + + + \ No newline at end of file diff --git a/tests/.idea/inspectionProfiles/profiles_settings.xml b/tests/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 00000000..c60c33bb --- /dev/null +++ b/tests/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/tests/.idea/misc.xml b/tests/.idea/misc.xml new file mode 100644 index 00000000..559ad2b6 --- /dev/null +++ b/tests/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/.idea/modules.xml b/tests/.idea/modules.xml new file mode 100644 index 00000000..dac5cbb1 --- /dev/null +++ b/tests/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/tests/.idea/tests.iml b/tests/.idea/tests.iml new file mode 100644 index 00000000..08bd4824 --- /dev/null +++ b/tests/.idea/tests.iml @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/tests/.idea/workspace.xml b/tests/.idea/workspace.xml new file mode 100644 index 00000000..f9541e62 --- /dev/null +++ b/tests/.idea/workspace.xml @@ -0,0 +1,698 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Python + + + + + PyPep8Inspection + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1537613612117 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_jieba.py b/tests/test_jieba.py index 745545f0..44e49b0e 100644 --- a/tests/test_jieba.py +++ b/tests/test_jieba.py @@ -7,4 +7,7 @@ class TestJieba(unittest.TestCase): def test_text_split(self): sentence = "我爱北京天安门" - seg_list = jieba.cut(sentence) \ No newline at end of file + seg_list = jieba.cut(sentence) + seg_list = list(seg_list) + + self.assertEqual(4, len(seg_list)) \ No newline at end of file From 2a22bbe2993eab9c0b5ff0f070503b3468dd9e6e Mon Sep 17 00:00:00 2001 From: Gyczerinvis <2997347185@qq.com> Date: Tue, 2 Oct 2018 12:21:08 +0800 Subject: [PATCH 0402/1660] Delete Project_Default.xml --- tests/.idea/inspectionProfiles/Project_Default.xml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 tests/.idea/inspectionProfiles/Project_Default.xml diff --git a/tests/.idea/inspectionProfiles/Project_Default.xml b/tests/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 7700bea2..00000000 --- a/tests/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - \ No newline at end of file From e36a92c0a8c31313b881bfcaa7c95051087acd69 Mon Sep 17 00:00:00 2001 From: Gyczerinvis <2997347185@qq.com> Date: Tue, 2 Oct 2018 12:21:17 +0800 Subject: [PATCH 0403/1660] Delete profiles_settings.xml --- tests/.idea/inspectionProfiles/profiles_settings.xml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 tests/.idea/inspectionProfiles/profiles_settings.xml diff --git a/tests/.idea/inspectionProfiles/profiles_settings.xml b/tests/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index c60c33bb..00000000 --- a/tests/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - \ No newline at end of file From a14b211232c8ad5bb4a7cb921ea8ce520f626653 Mon Sep 17 00:00:00 2001 From: Gyczerinvis <2997347185@qq.com> Date: Tue, 2 Oct 2018 12:21:26 +0800 Subject: [PATCH 0404/1660] Delete misc.xml --- tests/.idea/misc.xml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 tests/.idea/misc.xml diff --git a/tests/.idea/misc.xml b/tests/.idea/misc.xml deleted file mode 100644 index 559ad2b6..00000000 --- a/tests/.idea/misc.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file From 639e2232e1d1dea7daf7716bd3407ad1626a1b7d Mon Sep 17 00:00:00 2001 From: Gyczerinvis <2997347185@qq.com> Date: Tue, 2 Oct 2018 12:21:46 +0800 Subject: [PATCH 0405/1660] Delete workspace.xml --- tests/.idea/workspace.xml | 698 -------------------------------------- 1 file changed, 698 deletions(-) delete mode 100644 tests/.idea/workspace.xml diff --git a/tests/.idea/workspace.xml b/tests/.idea/workspace.xml deleted file mode 100644 index f9541e62..00000000 --- a/tests/.idea/workspace.xml +++ /dev/null @@ -1,698 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Python - - - - - PyPep8Inspection - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1537613612117 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 45d9be2b9d70fb2101c6dd90184ac128fcc41770 Mon Sep 17 00:00:00 2001 From: Gyczerinvis <2997347185@qq.com> Date: Tue, 2 Oct 2018 12:21:56 +0800 Subject: [PATCH 0406/1660] Delete modules.xml --- tests/.idea/modules.xml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 tests/.idea/modules.xml diff --git a/tests/.idea/modules.xml b/tests/.idea/modules.xml deleted file mode 100644 index dac5cbb1..00000000 --- a/tests/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file From b54644ad067fc1d964477f46afa4a2f51a36138d Mon Sep 17 00:00:00 2001 From: Gyczerinvis <2997347185@qq.com> Date: Tue, 2 Oct 2018 12:22:04 +0800 Subject: [PATCH 0407/1660] Delete tests.iml --- tests/.idea/tests.iml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 tests/.idea/tests.iml diff --git a/tests/.idea/tests.iml b/tests/.idea/tests.iml deleted file mode 100644 index 08bd4824..00000000 --- a/tests/.idea/tests.iml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - \ No newline at end of file From 94872226ebff5dad9285b4b507615267a54291e0 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 2 Oct 2018 18:00:25 +0000 Subject: [PATCH 0408/1660] Install hep_ml, xgboost and lasagne using pip --- Dockerfile | 17 +++-------------- tests/test_hep_ml.py | 13 +++++++++++++ tests/test_lasagne.py | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 tests/test_hep_ml.py create mode 100644 tests/test_lasagne.py diff --git a/Dockerfile b/Dockerfile index 4a1a74f5..f29ad383 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,24 +53,15 @@ RUN pip install /tmp/tensorflow_cpu/tensorflow*.whl && \ RUN apt-get install -y libfreetype6-dev && \ apt-get install -y libglib2.0-0 libxext6 libsm6 libxrender1 libfontconfig1 --fix-missing && \ pip install gensim && \ - # textblob pip install textblob && \ - #word cloud pip install wordcloud && \ - #igraph conda install -y -c conda-forge python-igraph && \ - #xgboost - cd /usr/local/src && mkdir xgboost && cd xgboost && \ - git clone --depth 1 --recursive https://github.com/dmlc/xgboost.git && cd xgboost && \ - make && cd python-package && python setup.py install && \ + pip install xgboost && \ # Pinning to an older version. The latest release (2.2.0) depends on an old GLIBC version. # This cause the package to fail at import time with "version `GLIBC_2.23' not found" # TODO: Unpin once the latest release is fixed. pip install lightgbm==2.1.2 && \ - #lasagne - cd /usr/local/src && mkdir Lasagne && cd Lasagne && \ - git clone --depth 1 https://github.com/Lasagne/Lasagne.git && cd Lasagne && \ - pip install -r requirements.txt && python setup.py install && \ + pip install git+git://github.com/Lasagne/Lasagne.git && \ #keras cd /usr/local/src && mkdir keras && cd keras && \ git clone --depth 1 https://github.com/fchollet/keras.git && \ @@ -100,8 +91,7 @@ RUN apt-get install -y libfreetype6-dev && \ apt-get install -y libatlas-base-dev && \ cd /usr/local/src && git clone --depth 1 https://github.com/ztane/python-Levenshtein && \ cd python-Levenshtein && python setup.py install && \ - cd /usr/local/src && git clone --depth 1 https://github.com/arogozhnikov/hep_ml.git && \ - cd hep_ml && pip install . && \ + pip install hep_ml && \ # chainer pip install chainer && \ # NLTK Project datasets @@ -132,7 +122,6 @@ RUN apt-get install -y libfreetype6-dev && \ # Make sure the dynamic linker finds the right libstdc++ ENV LD_LIBRARY_PATH=/opt/conda/lib -# Install Basemap via conda temporarily RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \ pip install matplotlib && \ pip install pyshp && \ diff --git a/tests/test_hep_ml.py b/tests/test_hep_ml.py new file mode 100644 index 00000000..37afc476 --- /dev/null +++ b/tests/test_hep_ml.py @@ -0,0 +1,13 @@ +import unittest + +import numpy as np + +from hep_ml.preprocessing import BinTransformer + +class TestHepML(unittest.TestCase): + def test_preprocessing(self): + X = np.array([[1.1, 1.2, 1.3],[5.1, 6.4, 10.5]]) + transformer = BinTransformer().fit(X) + new_X = transformer.transform(X) + + self.assertEqual((2, 3), new_X.shape) diff --git a/tests/test_lasagne.py b/tests/test_lasagne.py new file mode 100644 index 00000000..08e9dbf8 --- /dev/null +++ b/tests/test_lasagne.py @@ -0,0 +1,15 @@ +import unittest + +import lasagne +import theano.tensor as T + +class TestLasagne(unittest.TestCase): + def test_network_definition(self): + input_var = T.tensor4('X') + + network = lasagne.layers.InputLayer((None, 3, 32, 32), input_var) + network = lasagne.layers.Conv2DLayer(network, 64, (3, 3)) + + params = lasagne.layers.get_all_params(network, trainable=True) + + self.assertEqual(2, len(params)) From 566014a1b0e143f11591a47447945bc8195edcc0 Mon Sep 17 00:00:00 2001 From: Nikita Titov Date: Tue, 9 Oct 2018 02:45:09 +0300 Subject: [PATCH 0409/1660] update LightGBM --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f29ad383..30f04615 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,10 +57,7 @@ RUN apt-get install -y libfreetype6-dev && \ pip install wordcloud && \ conda install -y -c conda-forge python-igraph && \ pip install xgboost && \ - # Pinning to an older version. The latest release (2.2.0) depends on an old GLIBC version. - # This cause the package to fail at import time with "version `GLIBC_2.23' not found" - # TODO: Unpin once the latest release is fixed. - pip install lightgbm==2.1.2 && \ + pip install lightgbm && \ pip install git+git://github.com/Lasagne/Lasagne.git && \ #keras cd /usr/local/src && mkdir keras && cd keras && \ From 687df0e926ff70cbe2058c9dd7c7c54c58c5a205 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 9 Oct 2018 09:15:03 -0700 Subject: [PATCH 0410/1660] Let prompt-toolkit used older version jupyter-console installs an older version of that package. --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 30f04615..9c7391a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -419,7 +419,6 @@ RUN pip install bcolz && \ pip install pexpect && \ pip install pickleshare && \ pip install Pillow && \ - pip install prompt-toolkit && \ pip install ptyprocess && \ pip install Pygments && \ pip install pyparsing && \ From d9aacd14c702a378ad13d26ce4490349f0b4905c Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 10 Oct 2018 00:04:18 +0000 Subject: [PATCH 0411/1660] Fix compatibility issue with matplotlib and basemap --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9c7391a2..d996cfa0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -125,7 +125,7 @@ RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \ pip install pyproj && \ cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \ cd basemap && \ - git checkout v1.1.0 && \ + git checkout v1.2.0rel && \ # Install geos cd geos-3.3.3 && \ export GEOS_DIR=/usr/local && \ From d0a8bc4b7b6ecc769149f05c3f9e5b62d6a98cb2 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 9 Oct 2018 19:56:29 -0700 Subject: [PATCH 0412/1660] Pin matplotlib to prevent compatibility issue with basemap Remove the duplicate pip install for matplotlib --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d996cfa0..744c80c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,12 +120,12 @@ RUN apt-get install -y libfreetype6-dev && \ ENV LD_LIBRARY_PATH=/opt/conda/lib RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \ - pip install matplotlib && \ + pip install matplotlib==2.2.3 && \ pip install pyshp && \ pip install pyproj && \ cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \ cd basemap && \ - git checkout v1.2.0rel && \ + git checkout v1.1.0 && \ # Install geos cd geos-3.3.3 && \ export GEOS_DIR=/usr/local && \ @@ -405,7 +405,6 @@ RUN pip install bcolz && \ pip install jupyter-console && \ pip install jupyter-core && \ pip install MarkupSafe && \ - pip install matplotlib && \ pip install mistune && \ pip install nbconvert && \ pip install nbformat && \ From 655a5084791a571d6aa267fa3f4db3cee5f4247a Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 9 Oct 2018 20:18:08 -0700 Subject: [PATCH 0413/1660] Removing unmaintained and unused opendeep package The package was broken for quite a while and no user noticed/complained. The last update on github for this package was 3 years ago and the package is still marked as an alpha and in active development, which is clearly not the case. --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 744c80c9..f7e7c1bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,11 +134,8 @@ RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \ # Install basemap cd .. && python setup.py install && \ pip install basemap --no-binary basemap - -RUN cd /usr/local/src && git clone https://github.com/vitruvianscience/opendeep.git && \ - cd opendeep && python setup.py develop && \ - # sasl is apparently an ibis dependency - apt-get -y install libsasl2-dev && \ +# sasl is apparently an ibis dependency +RUN apt-get -y install libsasl2-dev && \ # ...as is psycopg2 apt-get install -y libpq-dev && \ pip install ibis-framework && \ From 68a3b18f87f8481657f3cd379cbf7a79d1664bd2 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 10 Oct 2018 03:26:27 +0000 Subject: [PATCH 0414/1660] Install shapely from pip and add tests --- Dockerfile | 3 +-- tests/test_shapely.py | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 tests/test_shapely.py diff --git a/Dockerfile b/Dockerfile index f7e7c1bc..7af93feb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -142,8 +142,7 @@ RUN apt-get -y install libsasl2-dev && \ # Cartopy plus dependencies yes | conda install proj4 && \ pip install packaging && \ - cd /usr/local/src && git clone https://github.com/Toblerity/Shapely.git && \ - cd Shapely && python setup.py install && \ + pip install shapely && \ cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ cd cartopy && python setup.py install && \ # MXNet diff --git a/tests/test_shapely.py b/tests/test_shapely.py new file mode 100644 index 00000000..14c4843c --- /dev/null +++ b/tests/test_shapely.py @@ -0,0 +1,9 @@ +import unittest + +from shapely.geometry import Point + +class TestShapely(unittest.TestCase): + def test_geometry(self): + p = Point(0.0, 0.0) + + self.assertEqual("Point", p.geom_type) From d7138c2024c4b45d0261f7d2a24a765288102883 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 10 Oct 2018 16:57:21 +0000 Subject: [PATCH 0415/1660] Install cartopy from pip and add tests --- Dockerfile | 4 ++-- tests/test_cartopy.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 tests/test_cartopy.py diff --git a/Dockerfile b/Dockerfile index 7af93feb..147571e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,6 +134,7 @@ RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \ # Install basemap cd .. && python setup.py install && \ pip install basemap --no-binary basemap + # sasl is apparently an ibis dependency RUN apt-get -y install libsasl2-dev && \ # ...as is psycopg2 @@ -143,8 +144,7 @@ RUN apt-get -y install libsasl2-dev && \ yes | conda install proj4 && \ pip install packaging && \ pip install shapely && \ - cd /usr/local/src && git clone https://github.com/SciTools/cartopy.git && \ - cd cartopy && python setup.py install && \ + pip install cartopy && \ # MXNet pip install mxnet && \ # h2o diff --git a/tests/test_cartopy.py b/tests/test_cartopy.py new file mode 100644 index 00000000..e372b280 --- /dev/null +++ b/tests/test_cartopy.py @@ -0,0 +1,8 @@ +import unittest + +import cartopy.crs as ccrs + +class TestCartopy(unittest.TestCase): + def test_projection(self): + ccrs.PlateCarree() + ccrs.Mollweide() From 599680f3a710b9b1eb9a284b1d36621a7ccdc284 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 10 Oct 2018 17:43:48 +0000 Subject: [PATCH 0416/1660] install scikit-learn from pip --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 147571e1..bc1ba901 100644 --- a/Dockerfile +++ b/Dockerfile @@ -172,12 +172,9 @@ RUN apt-get -y install libsasl2-dev && \ # scikit-learn dependencies RUN pip install scipy && \ - # Scikit-Learn pinned to 0.19.X until 0.20.0 (many packages break with scikitlearn 0.20.0dev) - cd /usr/local/src && git clone https://github.com/scikit-learn/scikit-learn.git && \ - cd scikit-learn && python setup.py build && python setup.py install && \ + pip install scikit-learn && \ # HDF5 support conda install h5py && \ - # https://github.com/biopython/biopython pip install biopython && \ # PUDB, for local debugging convenience pip install pudb && \ From 7d554a4f832031e964cdeb47b64cc8ed3632e9e7 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 01:28:14 +0000 Subject: [PATCH 0417/1660] install datashader from pip and add tests --- Dockerfile | 2 +- tests/test_datashader.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/test_datashader.py diff --git a/Dockerfile b/Dockerfile index bc1ba901..1fa64546 100644 --- a/Dockerfile +++ b/Dockerfile @@ -190,7 +190,7 @@ RUN pip install scipy && \ pip install orderedmultidict && \ pip install smhasher && \ conda install -y -c bokeh bokeh && \ - conda install -y -c bokeh datashader && \ + pip install datashader && \ # Boruta (python implementation) cd /usr/local/src && git clone https://github.com/danielhomola/boruta_py.git && \ cd boruta_py && python setup.py install && \ diff --git a/tests/test_datashader.py b/tests/test_datashader.py new file mode 100644 index 00000000..c8ff9295 --- /dev/null +++ b/tests/test_datashader.py @@ -0,0 +1,38 @@ +import unittest + +import numpy as np +import pandas as pd +import datashader as ds +import datashader.transfer_functions as tf + +class TestDatashader(unittest.TestCase): + # based on https://github.com/pyviz/datashader/blob/master/datashader/tests/test_pipeline.py + def test_pipeline(self): + df = pd.DataFrame({ + 'x': np.array(([0.] * 10 + [1] * 10)), + 'y': np.array(([0.] * 5 + [1] * 5 + [0] * 5 + [1] * 5)), + 'f64': np.arange(20, dtype='f8') + }) + df.f64.iloc[2] = np.nan + + cvs = ds.Canvas(plot_width=2, plot_height=2, x_range=(0, 1), y_range=(0, 1)) + + pipeline = ds.Pipeline(df, ds.Point('x', 'y')) + img = pipeline((0, 1), (0, 1), 2, 2) + agg = cvs.points(df, 'x', 'y', ds.count()) + self.assertTrue(img.equals(tf.shade(agg))) + + color_fn = lambda agg: tf.shade(agg, 'pink', 'red') + pipeline.color_fn = color_fn + img = pipeline((0, 1), (0, 1), 2, 2) + self.assertTrue(img.equals(color_fn(agg))) + + transform_fn = lambda agg: agg + 1 + pipeline.transform_fn = transform_fn + img = pipeline((0, 1), (0, 1), 2, 2) + self.assertTrue(img.equals(color_fn(transform_fn(agg)))) + + pipeline = ds.Pipeline(df, ds.Point('x', 'y'), ds.sum('f64')) + img = pipeline((0, 1), (0, 1), 2, 2) + agg = cvs.points(df, 'x', 'y', ds.sum('f64')) + self.assertTrue(img.equals(tf.shade(agg))) From 50ef11973c55eea558e1eba9b67d792a63d18c30 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 01:22:53 +0000 Subject: [PATCH 0418/1660] only install packages after downgrading to python 3.6 --- Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1fa64546..d1367e4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,14 @@ RUN sed -i "s/httpredir.debian.org/debian.uchicago.edu/" /etc/apt/sources.list & pip install --upgrade pip && \ apt-get -y install cmake +# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517 +# Fix to install tf 1.10:: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2 +RUN conda install -y python=3.6.6 && \ + pip install pandas==0.23.2 && \ + # Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518 + pip install keras_applications==1.0.4 --no-deps && \ + pip install keras_preprocessing==1.0.2 --no-deps + RUN pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ husl geopy ml_metrics mne pyshp && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ @@ -37,14 +45,6 @@ RUN pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ # clean up ImageMagick source files cd ../ && rm -rf ImageMagick* -# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517 -# Fix to install tf 1.10:: Downgrade python 3.7->3.6.6 and downgrade Pandas 0.23.3->0.23.2 -RUN conda install -y python=3.6.6 && \ - pip install pandas==0.23.2 && \ - # Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518 - pip install keras_applications==1.0.4 --no-deps && \ - pip install keras_preprocessing==1.0.2 --no-deps - # Install tensorflow from a pre-built wheel COPY --from=tensorflow_whl /tmp/tensorflow_cpu/*.whl /tmp/tensorflow_cpu/ RUN pip install /tmp/tensorflow_cpu/tensorflow*.whl && \ From 428217dbd390acf2b0c61907be1126fea35737c2 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 02:53:05 +0000 Subject: [PATCH 0419/1660] install fbprophet from pip and add tests --- Dockerfile | 2 +- tests/test_fbprophet.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/test_fbprophet.py diff --git a/Dockerfile b/Dockerfile index d1367e4c..2d010414 100644 --- a/Dockerfile +++ b/Dockerfile @@ -291,7 +291,7 @@ RUN pip install fancyimpute && \ pip install pyexcel-ods && \ pip install sklearn-pandas && \ pip install stemming && \ - conda install -y -c conda-forge fbprophet && \ + pip install fbprophet && \ conda install -y -c conda-forge -c ioam holoviews geoviews && \ #Temp fix: After installing holoviews and geoviews, deps for fiona and geopandas get really messed up. This is a very unelegant fix. conda uninstall -y fiona geopandas && \ diff --git a/tests/test_fbprophet.py b/tests/test_fbprophet.py new file mode 100644 index 00000000..f8badbd6 --- /dev/null +++ b/tests/test_fbprophet.py @@ -0,0 +1,16 @@ +import unittest + +import numpy as np +import pandas as pd + +from fbprophet import Prophet + +class TestFbProphet(unittest.TestCase): + def test_fit(self): + train = pd.DataFrame({ + 'ds': np.array(['2012-05-18', '2012-05-20']), + 'y': np.array([38.23, 21.25]) + }) + + forecaster = Prophet() + forecaster.fit(train) From 03e180b04f2bba446f1550067491f0328c338b2e Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 04:56:36 +0000 Subject: [PATCH 0420/1660] add tests for fiona --- tests/data/coutwildrnp.shp | Bin 0 -> 109548 bytes tests/data/coutwildrnp.shx | Bin 0 -> 636 bytes tests/test_fiona.py | 10 ++++++++++ 3 files changed, 10 insertions(+) create mode 100644 tests/data/coutwildrnp.shp create mode 100644 tests/data/coutwildrnp.shx create mode 100644 tests/test_fiona.py diff --git a/tests/data/coutwildrnp.shp b/tests/data/coutwildrnp.shp new file mode 100644 index 0000000000000000000000000000000000000000..f1f62b5d80e9ff767be06eeca2c001b3fe7d293f GIT binary patch literal 109548 zcmagn3D}Rt|Ns3VLK{kp7ENg1q(w?KiWHR++N2>#L@J?uN=ju*ds0IwMM_8|Oxhzw zrIIp26p=~`;=Y{E*XRHJ=f02Qb{yYdzsK>OnRCwB&-40RpUap7w=@5r|GxZl#Wu!N zBo_DX)Gn3%zy6B>r@FHLrrIZ8oXY-d?%v$QZTav2&%biUlpmwAOt-a{q_Xt>tVKoM zbn%M9MOkGDO{@PEmUdcG#N1_E$JPJ&e`&Xragb~jKNR~u>x;7UYV6+sgKO=+lJb`S zj@KytY1rM5Q|sS~xPSI1(pUYpu)9vaX$|??%RN>Xj)%$L{8HF_$i0Q0@<8maUqyc1 zdwAz(^7m8vr9YAXckIu_K2W|t8r$a7*LJ{h9X1{1({;b=S8%<`tBU){3zinPop4M` z3j5{qLpbgg!25)x?UZw?}lLeCHCd8KZE)qe_gqH(=W@Tc{*Tmu4fX?; z7xrf={V%j%en#b8LHp*)TrZW}y>xxzej|1tQNFpF_K2OhzMClTL;cbvv`4JO{h0IA zK98&Yx5GxZwQzox{QeW#E0#P9=jPNWU8wRJVcQXhS2X_CP~TjS@#&YT{x@TDoci;9 z96FHSo~pR!cKV$0o)1)g7Gv8%?K4dCp1xFk z6He!-f95G(NB8pt`OV+Dp9NgcAEoit0EdQZue(?e@}u0ZlkFJY|B?-ikD**IT_s;Y zed4>UKVeUtf5l#Q%}JlV(!Y(vZfcJkHWuZ_4b(Rr|8L=~*TeE<>*^j_SMB>1iZwdj z`z~!WawBv7+wYsWE$gcL-@0l}F-2N4EA8~UT&Z>`+MY350fj)f%NWtjgKM!b!hr!k>8KoRE#~}S{{Y-MB3l|^)F-d zS@{?2E6n5i1ODsO3x|{D6!G*qW82GNERNNP2aSFAKI6&!M%=Z>ZmOFd{o?zJp`xf|8;E4e6&9) zUSIwayGHL7`JG&v`{r2s=l^)C`CfLit299CMtW4 z2|sg75f2^46;4;*T-YrfTR6UUV`2X>_U-x>w!ghdd?F6Du!-2Oe7=b1cl(h44Qw0t zru?sP=z9a+cTADqecg-l7h-c6_8;Kf1e>REm~(xRJ_a0bz~LO64#jC_><`1{Wv(Bq z%2(m^#dSq_X2x?xeQaHvYhic&wOnt`XyU7T7WTbxY=qD}qroQm@a@`f&yw=>Rb zt|a|lI2?gfW2OJ|iXxty<9G~qdtlqPQxQ+w;Pm$8l)sjr}$yc;5 zuICph{_LfN-F-M!!@i#4FSH}RGY*&GSP9#=E}^_?I1jtHaIA^lri;kmK>26lxJc~_1hhw|&Jjy#j?J*wb5jcH}{jOS1j#B=MaXtmR7qP1+uhIH4pZuxW3#?}~RNf;v zUvn<^E2rJCeV6>^opXwK9*u2R9Ntj*C&@cgp8f4?(jSS_V>s-O<0&})r1F-ZRiqE? zb-kBx?9Bc9={PS_`+bMabew14^a3_lV)qmdzc;5oQ*oMtQw_?GS7Xx%+g-4~2!{{P zB>#P?&u|>eQQz1d+h#c3hs`zedpN0o(pshemALx>`~7Gi_nhLL)qeZpto||W&!9dF zh}&bD6?R8!zq{`A!l9FV9FFDLFWtB7{ce!_l>DRiSDg*gq1t~35znt-zYLqU?ANh{ zuJ;ZOugX=}-@`$wUwv#UQU7p__WyR=pBaq(ue#sK@}cxcsLA;u^iuy_Ctt7rtNse( z`#AMqS8SSK`>s3|hoziXY@-&m=OvtPY;Wv8$8jNcH76G3n*$hMu{q9v6HnJ-`?Sg% zg?&?b0}cn^(0Ec&o<9xy;n=>R^e0R%;<1sg_Z0Te;Jg5bnwn3`uo;idM>u)rr)w@R zZCTu}uPC3*e0PcKyPGxtuf=H!j+d&wzbkzg)wiqGlksxK_6_B~g7wHeiOqL%Jsies zJzYurqz+^kTzhyX|i%lc;i~NtytKVtAS%~v->_=v-`nQettMhPZhQn&c zgYS&pG>wm>{H615Yxcu*xbm0Re%cnNd$gZ=IV;`}$3xkV?R;JDBJI!B)uhw|dC*k+s##c>?xt8^I7-8qltii#h31>>eEN#>3g0h;ve!}dS0om`84LbBE5e|^JyJ+w`2bl`O`f4;v0(e@oeUAya%V& z*jD9v%1pwpyPmK5YkfK&r}{jL<^j0$Y?|~u=T9bXcH{ZaF4B5c|CZwV=_;IiVE+@> z^It2y)~E22(r@)VTfvxZO3%gdnb(VF`gl8gRqS^s?=Jrs__zIkpJQDj0?k?i)WO*14H)FE|$GSN69>D!y$Gptf+*{aO(6Q*3 z@HqCLuuhr%a6XK6Fx^9bcO&bl9fR}hth4F2`-}4YF|6yM$DqRb*0z*iWpLs2sP>5! z*dM|^;wL>oede>zxN{yV90%h(YL-r|m$0LR9;o8^AW8a5;(j2dNjdP0Y`@gY! zhke*hd6fJ*2bjl)7f!QCZ@+r7aIVeueHc;Lujl$S@7rAoj3Nb2)};s;~<2DaPaP#>G`vHL~ml&_vC z@|&M={BbnCU2gt-;k+7$=3@%S$-4gDFA%SU^0V^J z%pX5P`Cnyzg@Y*HOk@7}@ou&E6|5KGWc5dT`KVWD-@CP*41JmFAIo~@FUFw}`*1iG$M&qZ=4$%e zcJ5L1PiVn-2lhqoFdp-2rGNdoB7K@jdHF7l|HkaA zzSq-~cRS8ancv|J?Zf#o^505+f99j4|Aq3yPOLwsGW&QOG^~ibBgr36VEu}%aF{)W z_#>oGyRzPe_pqDDdTAz*-tBl_k=}3FyUMn#_!sM|>5k3Pdy2U2L;9`vv$9vk6-xVJ z+H^P1Jnx-KTaW2oI6Pl=?JfJ_VD`i<|JUx`r-XT{rMHwuS|%(Ju> zn<+d$+BSVD@5!dcwbM`>9%w>cUdLwjX?QlacVqXn@}G$9ChV&I*GBhdZojd(o|RLT zn+p5Vy&s6pZ?uJ%V=c-{>&TxL$t$tzM}4E5D|0_y>BBf|_EGuga{sX&cGYR0cmNJs zm;4Dh9;)`e7pDz4FU58g{o!idT-;Cm8{1kqpUU%aI!f`XT4$SL|1ZwHu^o1H(cbQ) zTj-x-&ncWcVKW5#M{ua$vWVLk3wT2oG+IZ_m#A6%kDOvqwY}pAF&&NZ3R6~Rtd%R z%o1!r!})lEG6O z7T2=}v5&-;aC!@eS=f2@6~9R7Phnqi|6=nVHVyj~<+-ZbM|#PNaC{Eiwp|!+<}T8| zjPq{&sn7DOin#BIjciBZw2rv>Sm}?_^Wj?T?!&RV*2*X3!*PBd+Xrxb4X2-RsMW2w zerc_>EwEXq^q*W!{TE}m9{V@2*+cuv@c-KGiPJ*T$I|^Mf7pil=WEGt@1%Y53hd9t zVORFCSOceRv3XDJy+G-^{nu`0sXs}dkHh&7?B8cR7JJE-4WupOH$KL`=J#X%1UaqJ z`f;!Jy}z)nL3_G-az*Urv_1W87Hd8Gf$?LHkOwl}!ueW1J2D>4d92^D3HGvU!T8SQ zw7xgecwepkW|W?LK9CR9`x=9N4bA5Z+3);ktQTP&d#kC(`eMG;Ub`#nk6DF%Yt|>z zNc-nWtaqsoc6C@EO<$aD*K^0ky51+Oujw1gbA7ZPzoqNH#(EvJ*XL#D6yqi1w*7dGt}2lfR%C*A)|D5{Tj+E!m!M@}l8JBJ| z>El2g&XS+dxUH$rEjq>3c77fEaNIy!`a#;a-ywZEm~rl#P@cJmah_|bK3#P#n5O$b zQeKA58+!gfMeSFUc@S>KLG~Nv(}~A6^hatVmp+%c7dB1Qe~Xw$ZUlB2$3JmcgVW{o zznh268u>^Zj$yo5jjc2YyKNMo!q{^aaA_?18`XaIlipvYbIn%cuTt6PK`zudcD-+4 zPu#yjF~?k|OAE)Z%f`l*alArcE+_gn4e18gQXXDh}6;3~4U%fH$O4k+X^E){2 ziOoKz7V&rlcD-@BOzx!f`K{O=)1*iro=|)jotI}~GX|&Ss-NNfoIl6ra2(dEetob# z_4=ZIX(Z?CaI5?Zwg>bgeix1x$PZ#O0{c(pKjk)@zx{!3@);;wLh%?Tt8GDKAz3Prz}r`f~$z z!*DvEaqbrNEc(mTXI)8`;!sWN)VbFbaW{^2FV@1vST7sj*~lLe&%L@A>2pK!yR+p( zaQydb$~yyx_Si1e^=5P{;m^X~G&IDSHVxTCS}r*-&m$_uM$KU-dLeLf;yfD`&?p9eL5M&W#@*7+~6o6Pt#vlQ=& zV}ERq)IM-6PD{0pUx(xU^4-`s#_ks7U%~i~?QtHg`OpHV12kXSDE>0@$=-_NzdAn+ zmmA^ufczfwH7mWnPxIFkPx|~suBP<+DgJwpV!W7jtOt1(cIWAQH6Ev3ak>TPzqH;o z$F{EWU!(q+NBMpS9M7P>`F-_oFRdp}tAAfnJjg-o%ZWHG$N3YDAE)+v4oCg`*`K9& zBenk#IDe-8sVg6^^=4b8Z;SmYj92rs;s-(QO7x7e|xVZ}FF-pJo3gYj{J>}2kZE$=P=Q}$U>ElWA!PtDs^}-HvOKktS zyh!iIYCL|6v&SytbSh52bS%>68aOJsIL^g!Gd3^LA9l0y*U@@)80Gt`)StC*{2AxRNbgS2 z`c)hIdN{4q^{OlXEYADp53UzV=YKhM)bl`PUH?FwZ&3bUwg2~4d=mB@6~9q_lk)Rx zoIm1|I6bfPNC(~Td)Uoqyrs=Jw7~XPoNFq+2B+85p0XWFd%Nwqe!2qZ4NCt3<0+hh zV_$g(>@@!3!Ak!P{Tr@V{3-RfY+K^ol(?Co@$?GkGus5aw%C8wp%{;5Po3Y+!gd4a zJ^zF+;^`lpyU5=v-U8dw`{3$0&*MB9HeE*k-*p~cjB|HvpTn+(+zs2ooM%Hl9Hw#J zjVs$1<=NA9{;h{o+mnm)sd<;PvwQob!g&}r&E;}6`t-v78u?YlC#rr`7~g)G-mg2HYHB>R zlP6+3R_pbFjIaEY=1&EArQ-LKK5BlO-zh(|VE)Di#9cY9ACD2wgXIaT&m0^t(DfUz zAGw1xzWd@(jrkGh;B-6AU$Z}kFK{|c`)zCN-o<`j#m`{B4$ZLdfuqXzwK*S{a*8+P zJe>Pbzp#k7-BtCsiqBX755Q?1{gsz1K2qbMvd-h%W7}MQm;ScrU^fW+-in{8@o^2d z+sXqJ|B?B%_48)svd?RDDg7S9Pa4~$&$89Ikhy-Vo?E;A@43}Xy@YwIb9<^E|*j)&>`hw7YOYRl1_liaCF z{{rWd;FZ2HcDG@Bw9YM~<@2zA2j_t}R;Ir8O>8dI{mJ=u-QPIH@53g@PhsCt<$bFB za;`;x`8$-pulld_bEIYRbmFe3#>Xg}CS&uE;-8hqi@X*G#p53B7+;Hs+veEpMf-)1 zae771DK%O5^D69H;P|Q1U#5NGA)K#Q`puLdZo{D+&LuD7Tv*z7vOf3i=8)d~NZkC1 z(^oo2)?!~VJ+W(q?d~|pX|~Rp=W1ViK%$cUzPu>_SI(Gzn6Vf z`>fafyo}S15+dsgL5PU~{hGmnz;8 zyXO^u82c|2e+wts|Bb^G;<*|1b1z`uPxteI^1rC~Hd-I1E8Y&Zg#<;eN zc&}}?lvP$nXs-9*^E59%)_e2E$!}|8H;Cu|a5C@RZRz(j=2Ew;&%>AM>iY4#Wh#)~ zWuCXf3Y<^U^PRq5!1UDfRX^U7m+n10!TrVK6(6ei;0KW3m)dF%mG?4jm&)@zXlAP| z8*o2cw^!MBVU&w%_tIxW(pt{qrL#xzS(YvBev}%$>_1z7&L!NbpM%}0_r3??a4?P+ z={>HV<4k=Vp5t5=^`6;S`5oSax}}^OeJ`9I$7ZYd-dn#Hw_Ck8-}=4!*6-c7elPE~ zdT;;#dau9Ld;d~fm3}T@mvD~Db2!JPUv;h|2}<;$$4I!s%UH8hfSBA(dSEoYK~xV+vb+&M9p5IjF5a zC-wjO9Mx8zv)cM|SaGY*X{D_`$F=qN-@a@f^(_57=@;f)=zUzV#+W|*EHC{~Hnz9S zqeK5Uk6aVY*Gi20r0)+&EpgbL^}yH0b`tAZ(EAb7Oh1>~Qg&IfVGiq67>(oEI8DTE zXZd05H|ghyeX-HcEsf?`TBr5O^Soq6$d~JRsRIrh=tsXdcB^oDfc4R@$F8e>jyYZF zC#Y@CU|g2k%DsPLaX;=3;_)bKdMbazhDAJ%)%D+MK>8Z2=kXnEWcTy&McjTxJpY8# zd|Bz;aN5&OBflMk-GlUHd{{rnEcLByza@RxS3lSOS=T=r+reBfAEf=MHubmlv3rE~ z8@Zm+52QUp$?cezPTz-;yJ?&P^Xy(6&sK>kkiv@|wf)${x=8h;aT-W$gT ztg~B>&mGFfdA-s&|MSq|J1g=pN71%#*DdU8l#Puos+PIi{gahRzIPEHr zWL$&}2Nz|Tvo&rO9auQ_(70(KS7034jR*Yi+FLGrtbCBlv)K06bNl;>-%LB@Z*crk z?fMW79pqiGI}ZCd_AjpICeTlIIL>>^Qxt!KacdXibTIv%Ud8@u#);oc@w<=tZ(FuM z8vjPyn&tHir}irULu}qw{`!j7p}w|;JRYZev2RHGnaPTGJE*vy&>QD_Y5!8+`WvyI zLjBT=LyGivG|mrT_XhXlG&bD3y1&)*uicaTH?mQCY&|}9EPKyU8fRg|?Zw`ax*tee zJjvb?s+65mwv4mO|2NM3hc^{z^D5oPGVF)aj_Hh>c~0G*ddKzHt;4SNEk)eR>3nS7 zByNY`aGShB9)xr2TZ{bWBY6t8M;%+#Jsfdc5f4kSX^Ydt+ACzU28X&zKTPY%@091N zXg&K$<*&f;Sn8Ah)cSUX?ym;z?mwVC(&HM3^|3ig>rX9gwH^h<<138gydCkp633}G zaz9)8qO2R-KG?6R`z>pT*>ilweQ2E z&zI5u{sru3(f{rPl|M)0V=1;1u=^c{OR@h(Uc(xfcjo^6qgwOS9`VL{^zSLe-I>^Q zQTjDU(mq|)exq>wobvsLIGl^kZP;!v2b{j*++-%mN8tPh&j0H3TYn!>T+cs^eJ_=# zx#LzHTExxA#N#hG&cUhiK}9^z!?rJTFCSPs$Klva=eUos=}Ub}bJg64U4P=KGq!7$ zzrXwx*E0#nat9aJ56f|`f$b%3$eTAI(Za43ELx{a=P zDf214sOw$I+7kvILVdIr`L2iKdx-nHu*sUw`_cYkG>)5ydym~DoV6Ypm1pK_KHqW} z?LQPphwZKZb!_ur+qyX4M|$_R)`PiRFFvmO-<$eHhhsyHmx|bSR{htiJxBi6c_p@0 zmA)zCE2zAn@oKv0dT$Xg*>9`)(O&U^*fqy~K8{+CQ+4K>SAY5=<-4d~7@+xhit0NL zyLlQf-)lZ^wLk1swr2fZI?sHiv+E4ag>8Q*0|8U#@{v0suNE&aBPe7$Fz;_h3zc0)rH)jE4AJ8 zILslQFI4-?raZG5htj(E2IqP^i2B>J>5F_d_ZQyLIsZ%DulmYVr+w_H#O(!Y?-i71 zM#@|DSJkrbkxT0L_`pQ=up4U>bIyMIT;aT?tnOR-y-lgt%YKijl(v=Dhtlt>S7SZ% zzr0f9_0tY0$}*?QGqHc}BQ{;cVjOMb;~SI4HuK5f2Tq>xbjq{b_4C&BX2+;{FjFO5f*kq~iL%k2Jq-k>7WJf%_ec^Qtk0 z(+K%NZ0^9R3AUY-fAjOyZ%-UP!DeEeqWm}-`xe*@$MK8W)PJb*zqw!G)L(9eO>1ly z+9K`_lSksxShEXialNICJ$LjzlxIlqDq~mrzLwd07xB=PxF3&W!1*QYm*6y2uCMlx z!(Z49CvN`4zV%*3`Q{b*$~_CaVK|?O{b@Kg$99vhcQ{V7aj1s#>#E;xHH-4HzW>O) zfJ@(pD6mN{<+vK+um42hHw{8#0Ye0S6kJw(O`oD;M7aZ#<{rNatw>$TH7)~{@ ztAO*TyA^T!E7vn!aQqzm^}81Fv@7E={UO)V_}s+WVj5~bXst<4Vl#tyI2)&`y54B! zpWA@bDCWDJiv4QV3tx}+@ZV^?_#3;AS#R7?^rv5pQyq|W}I70DCUFh#dx%{ zwMS`W5bKdTwDZR!1Ux|VmKX4Idjxh-p( zK06aTmi5(^b$D*s|MV{{=}JA1`>RN64(e9eSHWolE|rykz^+vG2tH?E4ko>SSD!oR zr2Lz39Q$`se*6QwqyFJ~v*ibHcp3XIWUu^P))(p1sY<^H$2K_h+(>y>$;;&@u&MuV z5f63v+=IJJZja4CT=x4*#r?Zw*j7(MSacHLedsZmoX0Gn%0r>-*|CaxjZ>vasx|82_-xjaJxpQTFP7nJ3 z^eXsG>}F%P3!m$Va(EYqPn3QgwpF$(@_RYoj`JtP^VitDf?bX6i}a?0;!AMYAN#pG zP@hUT^xLtpRou?NQJ-r~Rdy=kL7%hD<*VV1H$Km=MdLylS4#!)y4*jC~+=g8T?5-hy zs4ZWlb#Z&u_jvgk?l0`7^xNxx55{?{+M_Er*JIn4>lwATe@XS5%6JIBs(ue@e5}Q} zi97+@gK+MtcsU%;P`n}z3STAKa_*{WyGzV<#NdK4u*2TZ)vQH?ls)7P{WY*uRF&EsDRZ^uO~tuiOXQOK`jy zySj>>D~B$uH|no^mb`%a<$hNd@%)79dk#($uscM48HcreZY^|?U&QfbZ1%vp3AU^F zT%5f{@po~0O7VU4xw%Jie3Z}K*(0dGABod&Y=+^mT=#b`HdoQ#u03|I$ea0`Us{Qi zK4%>Ia&E}S>T{r<;`AGz8#E0$cciJf*xSnXw9;PZH6Lz8*w;>_BD&J>8$pD6_0Q+5Z*j89KO6f|YVQxK7WMZ@@wzyzP<$2ZssBmd8@tzW)OwsR!?_RZ zv;CR-^=q}Be}dgj*vY0IPL;`@hO-}pZrGoO)85=qK0y1&1=#N)chmjOVgE^2D}E3A zS-4L=0^7f^nO}+i`$hBP8k{d@KAN|+Kb|f($M$IKcalF~e#S}cuU57LaImD0=dvG% z8&%#=?e8P8ZGhcmxwp;-GjKWq$D6R9$N9o0>Tkcpeun0A4a!S5(VoW2x2S!G>AbQC z$2FX1>}c92&ELfNrV{1(YjA8UAEWp^ILY}_UH^LGc~90O_c3-?X??2C_2M)fZ^f=L z?U`Scr{FMF-iZC}*jA%GOcNZoReSxz_3TedAF&^R?VhTSY`Q4kkhtlC<9>2eY=%-_ z`>@W3cj*2`W8W70JFz`Y_tzDh>Nx+)dDQNR!xz|Bz-AiG>(rhT6<>`_#O`VOP=o#rCu7@4UZ&@b?%2J7&0E+!iS0q!ugBtW9}Ye7 z*3UoH|L^ZNZ~8;A_ojx6X{%S>VXuC*?5tO=Y_s{lzTZ55GM9aN66rx)jt{#({z*BpmgTF2{Pzd!4|`|^__y+0r4YO@QQ&lUe^7Jdzfao8@WywDkk zYqc-zh*JY>@BEnhJVJfq>DYI}_F$ZD(!Q|+Hs#3gYAW7L^{FOLSAAdqi1IJjx%o64 zhT%N_!y@jswC$E>(_Yx$gWc;N6mj2D@l)mVaNPJl`FEE;#%6}vV=NA9sIPfg>F-zj zcU1aW>aWT;H&B0#f3GOtjHZ9nP;9%(k1JkY{XbT&$Ud1qz^)bhp#4no2iVu*1RVF2 zyD9xW+UJg!UtnL$m*Q|c`&fEWzJq-#?MwaBKkQ4UKJArG#r`z4Z+-Tu^ft~tv`OK0q1tKpLq|5K{yS-CMv#+cSvRowtuZF)8GqM;-)(eM`7C(=jFc^@w8a=X@mVz)#s^IMcnpK`5)r^ zl=44;-Fw*A!RAQDpR2Tz`W=q#=|2~a^R!+y#LlsvYok^i|fV5)ZY7iS2#7$c&xFga5#wZ9xlYLirfq5J4v5j z$Fa5M!&#Z~2jX}XXCb@IFGYQ_&PL{Bd0*mrG7j?=Q=ffT)BfL6p8ZJPr0ZY1hW7Xr zhuxUp_FL?~!e%G(`vI&EsiMl;hxNoSR{GZNn` zk*c5Gd&fE77wzdkChp5&*FgC{p#R-;90uWBh3mO9uvtXhEMffE*C{VQi2Wdy-%{gi z9Jb#w{!%NI-%jIoHnw%JyO!(u2>$qMV&w6XF zRDBw2J^qdQ#|m^-P@eU@^rKCTsxRdT@wbIYP@e0Z_9k_p|`n&$LS6Za{o~H3p zm-e(?z7^;DuwAG2-bwxa4gD9s!TA^L>N8&QzKjQdA2uIpziq4cT<>c9Ji+rrSWA9W zmFJQCjK;@3*fdo9aGqbbexBXA?B|Q6?|F(Fd$PBT)!wFe8s?I+vtzlkXSWZxJizg9 z>*wqEU03#%y*X$4q1gSPbLK$B=VAY#;&W)p)x#4k@_kf-=CM$k~{DtD>#v9r#33;*jyqC(WuktG^{W_gnE6JZ=zY7j;;M`F8pT+hn`lZeCnce zaBmzQ$9WLWW3iut?Q=>$7rUAABJAH#{xvw=qWCf#F30X2`68vCt@Jx%UxlAndOdfV zL$KSQcsd4$ReJ8KhVuwLC)}>`uEV(t_B$xv8t3!$T-H$O*YO-;&%~}b_J88|FHW*M zf&8wouHQk=Ekjh_#)|8Eox%>-UQXQqNqO;H<^LR;{p3fnd!6gsn{@qqvAGoIjySc& zerKHbz@{FKm9T4z!$LiWYCPD1^7X`RU7UuiygPX=jrx2{7=mr(?(82+v9mb+rSvDs z8?kAN?HZ-;iQN($F2H#@w#Vb3=VH4??N>wTf57Hfo-<>{;eKr9;n)#}kL11eTsjlG z&3bMfhy5z~QJh}EX%fz7%BvM`i_O8**YwA^i|YThe4RW2TYYa>`cdvj-0w{N{5W|x z99QD>Jn7wUINyfNze?X3hs%1fzdS5IgY&1#|1@?(<$gFEfU~|Q#gFB^gXygFPs*)S zUR%AFxKR0<$#*M#J#0r}vz_vPi&HtAce^I|l5A_MBa~6yr;dBwUH{v)9n|;-O6P5mK>?h-V1rBdw z_n_iSvDNr1ojr3Gjn8$&(*TW!?^WKD*nXk-yNc_3`NCbyk6cstHwUK`>YqKB-*y)E zLG!&P&gEFI!uJ|q6S4mSo6fAK@im+lYJGhiyDF^LM&sYerO&5Xr8i${{(MV*Hx%1{ zRGvQnW>2F$cQ|o#wf48F%Krln$Klw8{nekZ^!?bcQ$K7!)_GwN4&8O0xl`%8a^8t0 z&%^FKrN4mlSKb%rl{%lT)_Pe{=gs%E-rj=!{fd{<`SlLP@6mgj%jGk49^U%%id%i2 zahI~+vn+i+-Me+g?+WB4i;8zdp$osWnoch(Ys+)Xv3h7J`%hY+?+X~OdN1SmN&PHr zKEp-cE!hsnW=mORv;+D5Rx^rvr|JCus~Mwn@N2sEu9R1*iy1{bhcAiy*LCh{tlv-m zi+;;96@P_(_3y};aS(sTK{gNQ_i+>H-AvV|x5mW)`7g$CJV)1C&A9j5>GyRH(7bw@ zxLd?LH+^w<75j-Q?`qbKbgtUtcHWnUSLKSt^MSOVyC26Na0q(8dN2KvH_>V zmuUmGn{v_qrM`AMlHZp4em#zBN$>a2^?${&E9*|W6q_5dlkIlech6V+RJD)$y{M0C zjNK_Xv@Xqi)&FapKi2rUl>Q3MvE4!QvZ~^D;#5cN|FrtwVSgLrBixMRk2uay`;EhP z12z|s-#Y5!@1g(8o{x(9`=!(;pG^JjdDzOazv_3A?sqKfggqS_J-2N=f6BgUq`WKr ze$aJy6nj{>gSIjoZ!Vl3FPj@%=3&#a|CRnJ=36pv@`1OIcQNxO>i0^+FxDS?q~dEe zk4nGy^rh-~H}SMTF8v(Lhejv($|(KvKUueP@OyCwck z;&Fw_e*=e$_`N!R4Cmo+3AP<@T&nVS;d*YFte@l9W~2`XXg} z;yfFhHjG_=Jn8*~*!08pC(?&X)Zebe?ilPe_T35`8x>8N?%dy@~0}Vvc}WZ zIR2>da|JH_Tr-YS`p!!KlHxbuR9^Yt(RiLhdzi|s6KN=RJ7A~3BNJ-Lhp0VUX+C|S z{=5mtdvK08_r-C0=3^X)!_V?iY?_qL572GYW)%@PF zmTXx=wpu@SE&F}=pmiu*c~jB%ZXee8-1Az-*5I29yj;KEI|s*3*gVU)az|nNpvKkGK1E&fDVhfhuv;eU?_JsMj02N< zQ}w9iof9$v8`cdm{o^-?i+PbWX?c?$ulpmVPW5{og$KeK*-w)4+szfAvymhxTL-Hzj_ijTx$ z4*l=j;q;c#V%D2~G^APq|;;j3LXX11^_wNqGv60$$ zH|1}J?Ud__>$|3;&&SK#F}}=xIOx4u-W#XVbIsc8iv0d%+RIJCaXL0V<+UwLP_8xG@{yZ&FwE6rthC$`rUw_1mz{{BRsL;IMfy585=oh=Vjd>@?7#Q8v+ zPr<1xHuaSLK^(R3xF4}Ul(?;<_R>CT>Z`qeRDT|e^9$&I`DXHU}lvJU6U ziq|B)dr0qrT43w3>4C$cINq=H|L8e+xZ>Ynw+Q=_a5$3j;TE!X#vA2HIzK-x>%ER& zihb!lPe;bLdtc}4S=jun@=nltwG!u%TF+in{k~Fsx$5_W)}spCkMF{InzzO28SH;j z{%7T-IKPDBP#nf%_dVCQ`ul(3be#0}nY`X>#M0ky(q7_bupi{Ur1$f*f4r#ti?OZD z^-}ptV=7whgMR(f#oP>! zq+-vs-TN0#f0xajE&Jd|ZGvV0NwuF~9nkw(^Tp$Z;|Rv79fH&OI964>wZ`?V$BOj% za;3it2c56NzmIb5R?L%l8@A^%kKNRViJzu<{>?*_H&yfe?*|L}Ep;!eZ+rn4_p+t_ zov?|@zc04UO6iA@zmewgB{(jjzF`NPWi$POBE2b3dVex@uaiFSIJAg|pXdiOZgAnW zR{ee^j+bITZcq`A9W-vP#bI}BR}3uTex;tX8^{-6w=K4NE57mmBE5O%2il_|@#J_8 zP8$^eSkKL0;5ZnYyKy)lr}j7>E4P+E;kn#(!S)QjW4RfpM&B3Lcd~o@yTYb9@$k{Y z!eQ-w+<*L*cmr(uE+}kg-OK%#!~QaCmVQ&jUBdz7ANMu+kHlf-d_3%)BA#o1S=g_< zyRhr`dEwj*r-$bi4(G^?aPH0AFcs$(@zPw0AALst5t=)#uxmp8@bjmn|3uy%=YO%i zXigFLm8hS8adu%-9h*wn9f9Lpvx<1ECcpDhVb?;QgyRtGx+=a0?cwhKko+&Iy?TC7 zIQ@<7C+`;yr_;XiqW22hJJjB_aGHv(!FdACliw}U`-gDa2Al3U{5_L+z`g-aZ(`f~ z?ILbA%bQc-e5=OCXdD;e(%R+LGM>yCma36uS}F&QttP94+(Pj-h?s71*_){ljXl2Yb>#xzUhfJlcrE#W)=E zG4(qS8%O{9J8`PY_y`k~|5=2!sbMr_R;$KAddIqv>kwq__5Mg)_&Sk9*@&_ z*3Wo2`>#6)r`FiW?m8Tr63?=)OL=y@;;+k#a9WDZo;nZwj?*}7#wh(-Y_Gi@NT61LktMSG4`yd!oExnBGlhX>?cbk>=z`W%Q; zCHA9eu<42Orz-zzY!_hLlKsrC#_0>3rYZlo?2mpK_MZK=bk>TWvmd8lmA)4HwS5M= z9oX+(8@crJ)m~iB{Jg9fkKq9MfFBv3D`_7uhc~fXPkMi^+E3@oa3PL+kl*h{`@4#q zU(#Iq$JLj&I{)on_Fi|4&XMLf?PX^&*Ua9$i*a?z?xkGW;j2~I|4RQ3Am(3TUYXl* zJPX@%Z!P*Y9E-#I*jLaz-1~OM$)(KO&>Q;&%xkksz7D6CdGD2*Vt z>x^T6Y;`Z;H5_kHd^z?5xqd#SUyu_Zn+bO7C8Kd^94Ar!{#pR1M|=i#_l1U z?v?++@fPKu!#wokvC%v;cVOQ}^Qct5$9}BRe=Cnu`rmM>!q`gnnP;XNHqB(C_#hlU zBEQ{_b7)$L<25)>lWQ_h?Q;1NY!+i*mw6i(;?NDd&#_sJ?NY_7YF?{8aS%53s9!97 zX7VH)4%NB6zwW22)&c!}X5S5$=2jZ0_&cQkKhK|EYCLS2Yh`oH)y84H)}ikhkMTF! z$Bo0OmD+bOHVd^bUW-G-=`h86VLL_h<5{i4cWXWzj&pm(Z`SzO7AK8Y_qyiaG>vzU za|4a{KbWuPGv<%$ruW#-XgzqI`JMX9O*P*YH&5SI%rCz^<>g&*IvvMHw4QXr;bfeL zVmpKND2&Hm-}mR`cq?(!hq$>~>Dwv&WjLRq^7dE!H|kqj8^f3KR_oy&##DGr>uTE2 zy||ZD_pPFDeT7R4M_;zqY*|-_mKN2rzpMJpl|?KZ$+~2o#kQ{2#l5hrgQFapu&$MU zpV=OWO)t`$?>JAH+myevo`<^PIG6L7uZPpaIKA7M`gD;;;Mg3;`>=gp=fmzw-%UOp z`*zsws`QhwTX02DzWWvD(%*SHLFJb`3g?GNpAMw{@p9#VO83(ehi|aI2*<{>hg00Y zuJ$@g=`T@xuIfa6CSms{&Kc*;*dD5VV;zncV5{$+h{v#x1pQsKqOZ%k*{|+clyB-1 zkC$LuL+;$6h^LcrIu?7`Z}3IjexUJjz5J)@H~+FC9#7T%e1*duy5A#Ip7udI3)>Ub zKKi>~exUmM3+m@@mup^LlphwVe;Z@dS^X#HXZ8GW8F71(JXGn+%d-@JUC+;p6z_rU zY@BM~G+gOB@;sAzVSg-k^>98$@tv{hspp@sxu0;G((CUUhFul^fc7v!&sWFbG>zxA zR8j5yF%H{ezmn&>R1N30*t|^pTR9&ow;`Svw=de$)Wp6%w!L_MG_6=G!YZCO(~j8B zz@@dsUx3YV#M3=`UfoaSRlue*j$?Vgb(3%yjQtMOFX{Q%ou~1$O792s_hiy2>=zLC z58=>I}~RYiS0IO4}Jd9b;G5-EbK%37=3=uZB%>d^KtPf-WP@D zlxIGqe_~Z#?`--nZ=inmUAgrAPbV@S;+dom585{fhd2ELWZN zFTbhs253DjeP7l*`3B{m!TM@W!08G0H>dtI&uM*sNBup4_1_M|t||LPI!zARKYr<2 zj34(TcK6HI;BYRs=i#`o;&l~2`6}|)P<%4uIseG@!=>2EuA;m%>9;=b)GYh{j%qqL z<*!~S%FZ*NEyiZn&k|C@vc4);wwW&a-?_=$z;mv>i+;~L%bOV|W+m@2Qh98jl(%Kv zrCvC{q;>5!?C;RJHXEDka9)I6Q=HzG4`tlv?{Uh^i(H%fgsX5oMfDrQyviqIGnjd3 z^q$4+j&on-pUu2=cjDMaRy=>Nd493V>x0AJ)W}j?-H2ncorV2ic>zv$ z>AlgmT31HNA7Jy0(s$!}?r!W4(>l{e>HA|}9h+wre_8Lr?#A{6xt!9!sh=ZltMYHh z;W(~e8XIPm?q@jhxJK(}9m)$|Ol2(XtNOKm124vQ=9I!|1MTJQYlY1u+A}{nsjyp( zT`L?9da@X+_81&4!r?r{@5AmzoF?G-fzl5fQRGi8CKu($pRt*U^YKp+--y#toVTNX zZkFP!6yJc|gV@!6y2$T##HQrYTtELx`goG+GYXqOCQ_ekasB`Y*$v0(F5>Aj`6i|R z>s8wCI2`W4aV_^3+TfgU?1cSQiceR(y1X8{uV}w4hYRHm#M4{quTIp*tk?XUgMDkw z&+T;o*K0iPC4a?uF?-{9ug2pV@`sK%O~Up>Y)@AC2Vys$`h+IRe?N{5xLzJKp=cl9 z9=ldJ+>BEtY~?)k71~pdr{mC$^!BmwMcm2e6daZi4`EypPphz>F}860LDy@D!&DqM zy-51!<MaJ@HiZi!*v(| z=Fx@y_oUCK;Cw9oX)0iM0pr0`#kM`;r?i*YTFeJOhrK226^r?i|Hl3*Y%A${Q*hW; z*V}1ik>0K#y*(ehib{VIHtprXI4n{A$K+?ZKbLSmSnV?f$Cu?Nuw5_TgI!13*FK>9 z%dvkQn>zG|{YmK^PKS>w>K~t%pTQ~7zxG-AAjX61it{v$mxhY}EbolnLXDqyb-mK~ z8iD;B;z`a`7>{-gar+YX4=a6-mx}&&FQ~lv*e;N(Grn_2t{)p|{QGB$`|~GYQ|H;j z?qBlz4{;c!@(&(Of9{C$McAyNKIRVWcOahnVAoIXB(KM@FU~!=zWo-*b6#frzVKXe zJ(HNfsq}s33$eTJdFo$U52oVyA?rbw%`U7b_7tVxr1hu)4ntV)!(y)I9%FwCE3y9p z+jXS3Q`ry0O6C6(n}={}{AN-AtnXJh!*NpF{2{+bdRLwOHtGA)O)YFzj-fw#W4kwJ zJv#x%l54Vm+h0f@Pi231Yq9@X=Yx-BotNA#*c{0Dsr3El`F!lpQu=Kn&!8l(00poQy#pmL12zF<1 z-nN5eJ!jcRmHxDkiu|b`jveG<6(5Jq_iFD2*gvE8m2-dWrxDK;6<^KubJBS2i__y8 zf0tl$nZ~c2>MH$YrLQg@M)`3Q^CLZk!<#s)#wOsnSJ|^j*<*-nsq|Z(O}4m}((jGK zH_U(gF8RII6Z^cb*NC{kTG!iE>02uOYOP=Gae734UFi?idbSPMv&&ik^3FI9#&$XB z^I?h)#%3t%r)_}!Cic5jYOj0QFZ1s7zgf+GYVOl`dLKJEOjLXd@iT|R{J z;-%QOraxSB9EYg?wtBwXv+VP4uarLjwqK88Ey;HcqU{f9Q#hVjwicBu>x_wI|115g z-hp+eJ?m&X$dmRE>_+lFD-YH>+oydIH~Zmm#HEEpbDg`I;#8M)J^a~@^sBTF-Hz>{ z@&PzMuYF?hB}IDo3=Y>~-y6FwitnWKLvVhSeaU@+Lp_}J_sHW{?6dwirT5yW$1DAU z7mKzs9poW6eSL9JpWI6BBmeyZ>35WeVE^Mqw8w$ieuv{LV~X@?98SIDF7mgH_%!IDV-3`jJK4Tk2z0$<1(@ zgYCC+x2W{oU%u_>!toH@|36O^PP@x@Vz-w3xhD1-h?AV&98CJtiKh-Y-61a=RK)H3*glB8(fw6&jRVt*3zJ1(KTI2D`uIIqL@57lRH){l5L`R!HM9?t!m$F;uvi~VGrr_+9U zuJWHxd%JIOnu<+%*2}m8+eTVHw;firk8g?nsW@GT(?A@W;Lwi#O{d|wkH*tbrQehB z7oJf(;y47m-5I}Wl+t&=;YS>%;=D%rYdl)iFUqbhcG@r8{+b`xu)nzJIMiZ4veV?6 z>{n?&>>77u{W{?BqI|Re<%R8YIE=^nvnPspzN1qS&(ntE7qDBx{LihgDB|f6)|1=} z$APRjLEqC{|e`mb-X&d zKMld|C&k~zX};?Jh3x{In$iEfEA{huY-ixmhVk*ubw6E>-5b~}Wqi#)+@FI!4@+Ng zf49>$9}d@Zc@tg#bLAH_KgQy4?D=KBxWlynsQ$k4VK}VPvBYE5C*aVPczOt@9poPs zUlZHwI+gn2AC8x|=*;{)O65E8`Geo_i87yk7o1PVRyMEVJeas?_GBp^*3#!`pJBI` zK2OvBE!E@mk^D1p|D`;0Y^k5F;PV%|_tP9-d(Sv*9A2vS8&9CTejkEahRr5w&+^>u z^!r-U9TQ9WIG4Cz&U1H~MSEp0TiL_f9y;F}^;|xR@i8A#->=X3nN8K+vs|yZ8ctv6 zdXJ&MaewBA{e|}kre=S@*~tKY}sdOppySx)ZHmki@t?_QZpbt;{^th&jr{NEnL z(s1@A?iOsW!sc1k-9nx%@2b4?0nQWIXTdQDYm_|4_W`A(w=L9{S~tI zHRdNA2WX$t>|w^XmCE0MgZ43YrQs#+uU7jNlE{$kz-xG%BM`x842#~&Uq?WK2Bzx|kE^SC_!30$pz@sq`Q z4bDHe8cX~)Z0>)mIPJ%H#uHf^>2b!px>oY8%m=%x;*V-yaVPb|MC|Hf*O>X_KIC}u zKAgVBp&j!v@1*)W3@&-Y9fQpj=Cjk^+e~LN|I_=_Pd#zC6T2buUD#g8J|Q(%dBb`# z!x@h_N#0C0TAzn9esMDK@F3%Bx5KtM_8a5;(C9LLuB+lBxIX@A)qh&=i#lVoCw8q= z{zqN!!PtK-cT#^Es{CKrx5eggJ@?e*cy1Ntr@L`}7Tc)!Gujtv{)8^tr`%2dLu>XK zZa$7%vaiUEIA6CZ`-m_Mr{Clzj91p@8TLi&KGHs8p5i;P4+t+|ujkr-|K8yu_5sNe zH!Zl&rzds%OXdA>T#>6Q={dHhzjh7c@i)#le!#xLw!yxI_8kKmzx)^di+Vqp4#bsq zSTBt^zofsHS=~3LZd@N@wQs9~;~6;3XFb~njAuB5xP6}X%+@MDK*!hL!?Xi&?neIb ze^B0iMZPgN;J7OJ(`=*bIfQ(hJ8-?@lghsjGe4~Iv+pbKruc9irzyYRq5D_O{ciLB z^ZlRUp=o7o{LfdFu@3{s727(MYrEpU`G3Fv^PSgAEY8uqUGi#ieh=rx*u2R)4T}{2 zjCCI#!Tx0Bfn8O8ZQbYcE2TX%fxHnq%958k{sQH(SFt$+ryJzG6(9N%{cEc{ zcQ_8W;{4|e#HT6m{W!7MAE*21kt@=^^NXnZweQ^gI-@m>%oPu5F>!|;P+JE+1 zyhP>K#o=7e$KMo6JYFe}?o*uh$M*W(#qI^QH@_$4U)S+|>rrg#sr*nJ_F(+nP1p~> zu@$zPFy3MJ?xlVnC~tF3aafGwuUFH4Z?1n>iMXAJV_lVBh|N!wkH>Sp%`dDqqxn!> zd-ee4qwPoe@POv$7dqa)FLJ#b(!N`Q?bg_Q!2FK=)t~oS4|aRb$7((K$8~;BXnpOX z^ZyB_zi~W<^%e$7mw4=<_#)vF#lkeA!tQo^>Dd`w1Ky3}JkF z$yZ~)fqb;~d1t*>&NmI`erW$;eB4glm)(mRzt?b@heLht(|#oNV=wIVcUseQ>89KYP_SM0m~=6c6(h^IT45Ajj$ zQ${ks%oW;4KE?d=)qSSsV_x_W{ylfuAFHvroOKrU{b0U7>(CA+p6+9vhCT16KgZ#u?~iqpS=VObkP^=` zWJmw)oAQ}BcE@%Hjo&pmy@CBW?Dy074OjhzIIV^A>BGwL>@K`-FdM{TH-Nm8Mqqm* z&g*FZ{W^BH;_w=_4IeJ`O<%<=cFWn<*+w`_B+tcDasE~BBPL-#1;=`N-|+`?&peE) zbJ6rud_Ci^L*`nptk>wD$nWFpEt0V-&S%+e44#| zJmC7`yjLY>t{8_t{~ZUn`JfUDACe!!gqw@gSnaF2V7E;9WEbquQT|z1<;UQB$3WV9 zRpocb<|wr{>ZTI+UvfNu9(MQY-nbRcbro-j%@Fp1<{;I7Q2WZk1L$uz?NgiLygAOV z-&o>#f9-3J!|7i3z3!|2)L+29JUoNLXWF-4jpM_54!BCbM$Zjxuv>6L*~f&9`s2I@^{t+R&6+sP4CVNFP(RMMCyq<|7Q0KZ zxe>?7*p0ycUfPdCaNddjgtKM!Fa3H$X)h-A_dFct;Jg;jildzb;TIqZmQ{Fs}?Rz*M z%=m>RIG!x8to9?ef6<<=dW?=YU*#KO^CLFDb}#*jy54p}#Si8BnRRe}QrGh(%G=Lz zcn-UNu>HK+p62tB)UW>Dsc;px?G*2Y!_7FJrt(@J_6OSY^;kb)U7Rn#=2dcun~STt z#kOUAr;RBe50dIG?8ZbQ4bPG~WiR{JqS_Gzq(f*z9r(>!&&EGu6fJP}XnyoAR+K>p#l&Qykt< zd+o>_&84M%E#)teMgPoIoNi_ zxt-!KsQfKBZh31N|Fk;!-|UC;A-W%IhuwG_=TYB`sNS!L`zGv{b4S(xf&H|59vvS)^Z>#;S6>qKfXRE$d{7ZQ!o&TpeuP#r-c0=|5 zBplbkes}p|U7tSsJo6kJ_K`KeY){TF{G<7B4&zaKf47ytpSGm>{ia*E;kvxUb#iOqd@?qZhnKkjhx>B+3fp^dJcRQ0BIZflPhMa5 z<*Mq@&X?n;=cqUZ`?WP*k1IY6`xnV0@z@b%yiC+SpbJhN$wRp# zb{~h+8EB!Ttu$#yEMman!w^e=pT#I{!_VY!= z?HxM)bUoKTjqMFM$>vOKPE))A4o2-g$8&RPio@;lIPCR(+V)nB&*8*l8^+ICT)Cf@ zxoZzn{iC>^_Im6)C{GT;Zcojpzj%)K58$Ndp!})aPVWU?*8DjhhhaDzsOP4Rv~RxG zdxhWWpZQg8qWi{bs(&iyQ~kL>H;M5!4e5`)iSu_>{aab%^*i7wuYvN%T_dVAQ z$D_C}2EC_n`n!Y^et_eftF-UfUFS1{eL)(< z`I`IXBXJ(6XONAt`&sdK_1yT8_8oU%H<#mu!{pEDpWQ)zk$pj`i){};aI&_tlq7}Q)A*`Tvy7sRs9Co zMb&?_ONqN~950-K<0kY!)s;_>2VSoBaQ^O6tlw{&re9KQMydRai;MjHjHyY0!SdXDE&Tr_wPJPBdw`BeLbFiI( zLt9;+tH~F3RqQ@h{@76U7t5F6^cIem>+QByydU!;yr=fpWd8U`IKP73W|U7aTvpB} zAA`gC%)fZ6+WSTE&#`|VC#!t;F*Y|VUyq=^`2f3raM+&wn~z|=huv`+i>>l|I-B+5 zM-mS|VDlX|XI{bh&*1)M?_@p3i@D$C#n|nk`?qKPnwIPbLUnzAr2R}s)_>fP{e(G- zd>uMy|1v?|SNoT1l+VXt+Zp?tN0Kk=lW)?1N15*jli#g=ZZVC}{YF3c=r$u?`Bg?y zKjL^k&hP4e@G|zz6hHKF+W(gJ%p@FIs{9k9OWd|2Kf24t6#EX!pKCoq{5j>@$FSR( z{Or0tS>j==++u8T{1nH&*lw-;%zZfPer}$_X$En7{!`T7pZkAkfWtG|58R>l_tSns zw%u@kO6}jm{=&_|ZWZ=JVGG)`OSL~5fOGXd)>Cq`{$>8SHJ&d0$^EsTJ`e{z=cT)_ zkL=guo7jAd?fTD@`u<()_Q&BjoG0Pf^k(MkRGdG+W;Rax+`>PET?^hThG$j(EqP_0 zUHl`rmG-=zWz6Td7u(Y)U!G+uk1_F6o!@zPaK3HS{|hOfkJj}WNBeme*Wd4fV@Kvo z-dM-4ulbY|pP>1=1g8fc8S zj&HM$;yrRJ){XnlgdkHa?D z?ZG;6PjP(T7KhI`A9EzmN6N=wyNY}P_N_iB{k0RZn=-YyT$>e_WzqY^rn|~N{%*0; zbFJSS+l%R+T?@w-IiFBRalLP{TVTH#=Wpb=Cia`D{(wMe)_#>-yVo7zwfKGZ)fVhl5svx z`|yc4+`{_EJyiZq-S>{hW@p{E9QM!ZzIiE*$71(^;tdpEtoR$;ckNx+Y=CoXoIcg} zh5g9+gxNTsuk(9ap7BOGpVSVgW3YLEeSv$I+>!RvK0>)9HfNvV-zFZPV_#yoq`v!_ zeM{OE``_8;$$>i!TE;~#M5ya zpCjo{4A}jlebuiTuYuZEUBP@yAK|F^mu|%A3*uoHm0#oSGJa+j$20rm_?i0K58I&g z`vzBYv%81+o(B^TTWbCMh*LMMm%p(6Sn=w3HPia;O5BXrdLF9sm%c~;CTYCKV|y~^ zpPP_x{H@rvR{l9h?uPAZDu27;vo*eR6|YA;?Lt0F6IA~KY))gm{6xhc()it?`^g5{ zx4uEX49_v&yzDLOrS^W+xbpM#)%%{AK8Lm0wq?oXzA^XZa(+0 zQr66uhaXZL_9svK8?YH7-;P}e`Wc6DEHerF6L9_uTlK}Y;Jy(i6AxXn8$o;d3Hb+X z+j8IW4G%5-v7>Y!+7nmrTV@D$e-O9*IiGO6_5e)|XDrUZ_DLK!)qU}@BZzN-!(&Gl zM~Cy-#}vE66`$S;ucrDf+fcp>#}8+;ElyivGvWl|J7B-xiN*Or`CS}0SNmV%Fb4Y` z*e{S*!fy4M*3;X+U+}oGq z#Q`|ne|B*-7r4LAp??!p|NRd1uRXS#o?9GNQ~iI4+dJuB?16nFY#+d}Bj@KMHj^|y z{c!4|@gA@G1F^e-S`5uuKlVawW@x?5#Q9e29#{FDapm`PQvVUw zb39G1kMkQkp7Nu8oBHlF<;NAdxpMq?9r-nEj{S$^W4{OW{R7-L@}}5r%zY=0*83)X zzj3}w=i7$+U^rIRzBu1}0sVbh_tDF+TbcWEZlU;k+~@5{*ldYoOWHF#vv0_UW4{mk zko*bPBQ(~&r6IQ3XU5F+4)+sJ>)`y6{OEa%M?2cLr(<)n_9aK)uoL|?Lsfn^_7QF{ zPOY*3koAyu#&N3F;}JMqt@XBp>et0_0qy(suxW|ICD>i8eNYFSCu<)yo%~=2X?!lg zW*82m$S3|Pc^i2Txhwf6e#&_G{^Y0d9X4-~&)g?Cww4DdzKDEjn{n^=?QnH3;2Oz$ zE54L`YbH=XuZh!a9PU1e>+u%0kCCrq8|jBphbp_}OtK?!RVy+$i}7&4>SBGf>xK3C{0hJ3{66(R?`?hZ}I(58Icq zeWX=6UND*ueX%(er_UANj`d^5)8BME&iiOTpyxz?IX2B%kG3A;VUEG3vBu{<`B`kg z!Rc?Dj%VNMn&|%i5)PMPUtjyux3RfU`_i{m{(ku)oa>NJ{MI`D+wyk09-k@SbY*)%QKmKT+>eg=dn$gF%I}EdAvnJ=lKNX=zYR81xlf1xj39n3c5mX?s(l%^{0mN9$y0Wj z;vXyT43l@j>0X=%k!M0%#S?kNZjHkmN5Uzk=zoylPPabm)D?ue=QD;uoS}+L$2p%hvDpEeZ&dy; z>?1a};5blu^gES58K>Pxm-hS(%DYF5q5a>L*C*j>E{IF9eNK7)4|%q7#SGOyTXLzmJ`M{yc81_1Zx7+u{^y-}J@yCLHg< z;X3SJ!fp`Gi?F{%@gG$G8`Xc9eBq4xvkmzs{7O9Z!s&CB|3Ufa9kq8c`77V8@~dj! zbON^Hu-ONj!R)h~oL1IzTwBHO#!>ZiFY;?Vm$+|0K6dR?{u1pY9d^%QzZuSd;P4;j zYuuasZl1+vAh!45@VVl>a6VA?i)~fDBeplN9&B@*kJ5TMUGbXx+cuT=LzVwmzVfKw zSSwR`$Kj?fDNe6c>aMs?4yD|R|HdgxODqo6Jl^uBV*8Zl`7_wvuX!!|^|bC6{!H6z zVfP)b?srUsUugd!_7ia<9Bw_j+y}!&l#c_jxe(`uiZA5&X*R}pW*6$awbh@NYOgW&gA_jqn>%s5SiV5cVztC7T8b0 zN#*@7IJY347b(x~f#X)%FJ6hmb=bY3_HI&qU*?{z$NULLV|R(%1$*VyaIfNniRWRe z-$C=Owc;PhXUe0QZ{Eo*$eZCDY{uYpAxt*T;6L{Eo)&VDehlc%@gA2Zw4ryOQVP6r7&Lp$YBfaX1as@#iZq zYOR|V`s$+ zRrvupf1>$Yy&tTq@}DZM_pNC$HuEW;Hs|6-n+w*)K9;0FZbtDJ~sYO$@i|Q>UYNG zc=Dg?g3}c3D-XiCE#>V|^7Cr%Lisp3sQxkX%Q(!VJu@2{JqP)@*sZDfAM#x757y&+ zou1S3E;vnB{4i`D!|5!QzZ%D@a5xLw`>{P9=Vw&^1o=<-CF~8?Cr*`D)Ad@8({m*dfj-!qjd#Zj^`O8)Q4xG2e?i%cyD82&@%4haB?tft! z=i`@fzpUNAZClv~?pu9dqPVW-S>~rj; ztc!fF+!crIu%G_}=e`%tvKy!OTE8wX@iYNPIqoiB`hAJp-EbI${g3S1{7v$6+Q+rU z?oOPW$-{Bl35Wlv{yK`!RD1($2V?gG{qq+qz68euasF6wIlYR_Ma08!Y-M|=;=2$x zFDSk;wsQJ}eXM^-@fYQuIL^{O_YCa+z@{zEOK~~@y9p|PiON5N?QPihRr!7@f4<@> zA6sF!8}Zy-@m*z=k4LM$W$gR&{@A>Y<5oCM!(j`>$K$j;cKsCJ9H*|Tzp~1A#CF|0`UzUnXGJ*jVj?bq1I>1lZv)sHx>i-R2A{g(Osy2^ip z;{&{BHHRxc1m|OL-a_wXUm;h-^%Q>%+g^v2Tw-6xp&tG>x5z=cCU4B=_SJaUiCkpY zSj2cKH&vZR5|6(UHxcJ0D!(@Eg+FoF5W7w2zp10+ZG*$AIBqBJgY!y^SJL+x_)gf| zq4v7s{GQ4`jNLpOo>lqHIe+&yw*53d6LESBrx$TtsPUSv_%AsA2j}^!zYK>qjDPw< z@t)YIewdBjEVWmc>+Qdn*U@2eEQ0GkKpt5p9l?C-^H zW99Q_a6T8O8MvCi(?a=d;(iu3Tg$&IUJtv5-2c)lYHxQO7jS%ci2Ni@XJa=K$8&M$ zg-ut*FI4%DaB8CRKVrW`~uy3 z@iBQ8${obJT-LbW$VJ{;#Y_T_xG!{#__SI2Q2PEV-+Z((yj&VR_QusdeI z(w_aiPZ`g&QDy8a!*1WhrU~{9uw8L&D)I0`Z^r*voTgwukM{B;#m~UzdK}s+elyOm z(Vx&7r$N~M*sHW32V*}IyJqrgDt{&8laC-C?&tca_2lh!J!bSQ?b)+&{#O10yD#Pb zv=<-2v4h6{5bRD;e0~qwy9uX9N>a=$~2Go%55^2iRPw_&f5^n$O>0 zx0259Th*VnSGgW>hT`vFH&Ff=+g8|)m5;>cMb=XueGTU`4#$!5-db-1aQ=CZa=g41 z4s&pbS93leVRHsHS7N&>&Tpyy-&d9LW}M1@hvVfczY)$`V_#oh1G{x`+I4s4qrv8= z-58HKSJI!acP+MqaQGJIs}w&GhYmQsxl1XZ24HtDHdo+$Yg5`EitUd(7n|XVUy4%; z?9W#G=WeAxemfky$!G7x@y3(;LZ=;R?|B?g$MFg5THt?k5|hpV|Bc5UK$h_27i zsy|0=spI_z=Yz5Tl=*croFsQ*{^pL@-HDy@ zrRyobOTP2(sJ$Sy!ex>%v5N6i$QW6@97nAzy&w zGS%N!?vHI99L~n>wXWq{TqA7G!)bk-26f?F2jI{L+ny?a9ri~lJ{sq_I-kLczlZbQ zI2=Ryye5vlaD4Xia=f%q?cam_PuQM`?PQhT3!Bfd|MRj^Kd#4k_&+W!w!aY1H(|3A zb{FCFr~11d_T90WcS))5&X7mq)C}9Zv9E{YwKz;w9yu4AA8lwC!1)ZEZ^mYC9Hwqv=7af-x$5r1=|^l njRPH0@pyQXR{ZB-m^ z!09xcR;9mb7o3k%|MyY(-55XD%+lUTIDWk){drL1dm+v*%QtSp@lN9URo7a$4Ci{p z(>>U?#BP1fk591q2!|fD?+?`cT8cw!>>4#H#|!5u{yI)W6rW1}^J_R=&iR>{D!)+Y zzcTB^?8*4Wr*Rm9{aLIx|24LwaoUaX57V*jA#b4d-<0cV&&GB-4*eBhQ~5->shV$^ z7*ppl<*oGU2W2j$wN@?X>ifN3Y{pc^qE01Q?4hK!;{U8Qt;E8hH7WZHb^UGl-?koI zuf%P&?O7WZr_0`@ZP~4X^C!gp)l+DHuDl11S8r76o2J;^fbEtz_i9+;p)NMZZ9@5p zZw32fHFak_pkPj>IiW`okceW&_!ee2V| z-q`GiV=wFm>iYD={)BZY-=5>8HrUGk>g#Vq*n4Uxy#!0ryu3RHP~OT z`a_v7p}*$ZeVR|2d-p-u2$mtNyXQ~w%eQKdQ|6f>Y(vmo4K9z z{pwcE<8(dl*8FXt`J?_={oHE4m$noV2&f?HX^LFBe9IHJ}n`3hs z_HSHJ;z3UBu)SR6KkZ!N`3jC7XUM;>j?BB*)W_~t?0YiL^U>I~WnH>kI+gb9`C8YT z;dHC!?F$`=&&07C4xcOD3Y$A{I8o)-#py`wZ)Bd^ov_;wyRR7gd^S!GV%r&4uZ_Kh zb>VstcW-K4yobYE^2+MZ)^Zme|47z>U4X+^4Oj=maa@Q)ADq5b{01EU#<4#AiS?K- z{&(zTdnxNY&Q%8% z9A~av;y&Uqoci9(EphV{4oh-zFgP##lJ?%GemZboabB$BwZd*E?2p2+XPt7q(Cq6{ zJ_elk#OZcJ|94kh*E%o9<}u3q*S;zB&AD~y|E>#ZuL1puO%@fq8#G?m;Pi>ccl&RN z_htP31K97S>;C}`_pVXecXJi*qCB@Nb1}_O9?{&)dy*HzF&xjFt30s-{d1QpuZ*Gp z{#qOd;P3$DNc4<6M>BAGH`9`%AU|ZLIZ? zC~x|)e*6&fUY`Fs^T#Xi?T7R4YJV*2H@=0#AFRjpj^Y>MJVwWRQu$y-E~}(6_dWT- z9-!xg7nDCfp}xIE?x(yymwaPy(S4&S`N$68z7jW6J{-dM)aJ*XRvc6N!D_6UE2Dl# zj%!kx>vbvxRj%^?Gu5)yJe#inka1a2wi2(_wXK*(z9tXX*5A2OXI8cS>NDJq$}!p# zn-zQB75&(_@~`?;Evx6f+Wl|s{y5d_pZ{lnU9y-MacHId+y3nmw@b*6 zaf5e?gZl1H#l9=&k`H>f#Pb5>tFbsfM}7A)Hq+JK7VpvCCLAx{huvy8z9V0X&HM6T z@}Fz*eyQ*GBmcRFussQ<<~%RB8On!e$ghzP?XNg}i))`JnpQq%?4;}Fc47||Z}_#P zTXB6(J&SYguj_WxivN+Xcx;Cfi-Z4Y=_>z|zY?b|IF7~UI-LHL8!jv5^AYD#zYn(S zoL8LM|5@U3XY7aIus3$y|0;1K+oS(3PVI>2IdX&ZOM7;8_8hS>4kut+_14_C@?I)G zQTMTfWXpXk?I>S^?aDZ3?3Qu-cnbHmbRcolg!^b3qVjuc9=(R+rCKAlW9hHEk9iXg z$F{%bMH_7Hkvrh{I?f~HA90+C{Z7oIJO|sSv0q#J`3}sZaD==F+naFct$ofU#edN} zTZEm{zNij!Cta_3wh6Xx;p*J--(WkR`gud{(|)Adn?pP~Y<43an<@S%Hg|G9;acUV zQ*d4}S4|}+e#`i}EwG)X>(g1~Z^r3Av}g5q;lp4YPE!1I9dCd6bNZKmW&Y-`JF)GkczYZ?cH7{15{|DizIJV#uES|M>nF6rahTRy9hL8n^V_<9{jnP?zpV1@ zarzG1U2$lp{;ZGVW$Mr0%ojTco9A%YP~&kpj;|5-R^^Aw?U;|1-(OMYx4)hFoleH7 zGxOio#r9CvPiS}!>vu!eSKjsP;(W8#-{+qAMC>Qxa3(en6r} zc_a?c>HOEgt~ujlJ|{n>jKgKvUCZ_L6UeXbIk_VaR@vs4&Ax`n1=E-zjOU&w=eT2^}%K+4u4^LB{mJH?{>jX*4?6Tq>P~F$myc0+b*i)Bx2sQiS+V>09vY{b) zw)Xu(?fZw?`ny*Cj)GM`(p}mcZu|{%@P*N34%&K^oV%hQuT}o(%ic1s#G+-s=L2y1 zhWQ-_K2zfUGwgqSy4W?t=>=??YdzeF{S~Tz{ZplUn1XY2Y`4?;`fO~8o5!>sFT>$k z#pgUx;&B3wZ{g5iUgJsnvndW=6HlW!UKp$Ly>WU7d-XSLO8qpE>l+$ja|m|7QQq99 z`O^=_RdL=1+v%F`)5mbWi!~oTwoNG?64xR(QvF%FHvLq;daYj5{QsM{X^i7Cm0$8p zOINvuVJ41!aQp)MW99WYzPae{4^d}6$`2fzx|AoVkI=>5_W&XaX@wf!X{y&uZxepGrae7$s#W+sG`6-p3 zClAJ9jS2MkPHdZCtMaC|e21=IAM6iWQjTx#!)c`a33lUgT!Q0m*suItIiCGZ*I$2^ z+AY`hcf{jmluwsoKZyC{?pFNhrRDf>n0yxw*W-pZu2hc)qls&ckV<+yR@ZFA#5|e0ai(^k-!pr()lR_QTKe z20C8Tmr8l_4gK{UaoCD@KH%jN&s}xAk@9aiEyZqY?JtjfrIfcTc3vf zr*3j*oOi%!0rrQf{yCHA|7dJJ!S))R&pK~#{JXJp*k6V1HQ4T~^ZgFHaXQ{LZak|D<;BmgBj1WbI}A`aCbi z<&<{~cz%nY(I5AvK1Vo7{lAgtzx+PuoA==PFxC6_*2CrtJ>Pz!^|@T*dlPmIi06ZG+)?kFer0_} zyA36?Dof5?^WGz z%9m&AdL4z`ZK{7Tj+@|ciQ;!^{5HaNFyj-ulFwZWjaOgYAKt-XTkfy^aoRK2X}|C* z$Ir{yUxX%d!uA8o=iZd}uXB9!4t9^yzI#?4#CUuCodWwT*Qfe-1pF_0U(lNNQ*Rs| z!Er;J8_}LQf%TUA;h4zBwfq0wD&J34{k?5wr@zWtG_O$BoU^pp?p3*VE1n}N4;|I7 zy!oHR^d2z&{l|aHRtou*mKCS^8kg@CKTYE}8Rw?hKaSHb*xn~k!frS=dY|rY!(ln` z&=q^b@$AVszezmqirqwP_WZr{FW%S=aro-{}8PU8l>ii#YCs!xuO$|Fx7i zC+j-ji&J0CyT5)ZaqDn60EdTlyp?_~aW_}*mCpRBIL~4pn2}tIcoYuRYtu+^IV@!! z`1>dyTFKwzH29yEu5x`+7ut87uzeWYqj3BV2fYW+&D6il+z8Wfb*`koI==;)JJ)jl zLGPu~HjG!SL;GfXjsG8dkM#r&de3Ek!gh)J*PZKU4%0kemHCxlAf7*yv-;ng`I+~k zyjzL69V6pgo$K}ou16Y0`E;`8Yt8)Ly|PZK&)e0{s@*@F>%X66ulVeG#X8xl@=ss? z#;Tul>vM3^j(*su=e0|fviTXVWBLjErn;{6zal<^^RR2l{cv2W@<-!vE4CZp++Xot zdOtWx-T}vrX|H-N=2&dkA|ATS?d5*h%vAlh^GpBio7kNz4^zB94lm17uog3BfW2n9Ud;Fj8Uva~}Jv~*8Ecf{{ww9 z0~Z$COL(qyyU=&P4$pPr9vsit@pNwSH=g6{&ML3>4DR_w9KR0FZRrl|ug397Y`)NQ z+eXw+U$PIie-IBxu@4Q`tAE>RAKnzZZ;7W(71!s6@p5eRp2j|`^6P6KESuGMPRutl z9(e-iZ<=HOwVvZ=e9d^>jQu0nJdQ(uoYvKI`fWIWDSwFVri^EPMV^h#v)CNM_}U+^ zAFb=xivFa=dTzc|*RK)J$@Y82H`H_Qr;6XJeg4(*V#e1Wgx!XEZaG43rRTnlaNb<` z{yp{2<1}3TUon>}Q#o#r^AXtozMza(evS70k2pTc@zO@B{|Eg^YpMNxahyTCG_2--1%IB4POjz&JG8b+8(~HYH z{}p+*Q{|t&YQO&=)qEaM`*}g_=Lxl+H`IO}QTusC?dKV_pLfI$+SAvOjI%!mhdT0O zXO(!mS?gS%_gIa)xr}%|h;3L1`Fw1~$VVE zN8r2$HWO6;?FAg~UF^HbALG~%`&rn0IKPxPpQ!#=?BB%B<2+vVH&OgyobUXq)X%bC zJ`evv+#)`LiPe{WR_sD8d2);^rMc@}FqH9iRUs`P!uee1Bw zKYdlp2G4cM?;cjl`l;F<9)ZIVTnE<$n=f&0CV!!E|KZS5-)+day6v#le%+jXNQv8< zRKAb=I_<^1aoAhFtR;QFRL9#w{vNw^akxj=w{)%(2oO2j`F#T(eV^4Xm z=0VU{dByu+8;ILIvFV9pYnA_x&gVAl`lvq%hoi8YqWVkdzd0WJ2|AzcaM~QFH?iG` z>tP$y-?TQ46NrbMwazZbt`V+Y`#6$y>UJO=|6v{bUN}Fhb>30+k5~L#arsW+!4o3m6vrq_o?o?PZ3Xt z;;>Nd4Q4Hd=9-UMoB8_|%+C*q+a=g_rM^)-97lgD-`-iun?tc#uK8bw@iE`ZvvAsx z^7&xK-+qnV%{X3-eSNNHYQ=g@1GxVAMy>x_aOL-(mE-4j%&*$~v3F&SEnKDYpURil z9y4gqmX@}o@8c_LXGI>`v=l4W*t@L%pb=|Y_Pha^S%G!N2->P*dYM*|M+UmjlbDynR+Rl1k&XeD6BMo8WX1Hm#Ze>7(W4 zI@@EFA5X(+rq;_K9QRfIJFpo_|GZQE(OlQKXV#C;DOA5eb&RP)8*cs=u}I=Az2nr}A~ul$@`nICyG+Rsz5 zIaue{n)&K(7UYb92R9 zTd~fDulOJN3a=POI?t1F+JJeH4#vI_^CsQVhjHwJ!-C$$cBbNOae9w=7p~}4;-Me&(tXsE z`pvZNe!+Ht=H+#YKaF!UoUX%uRqS+p|6q^OUaIb6F2?>jmABX}!EU$irMziF|Lj9o z)4!L9hi$IHZ);vmy|Oq?!1faC*JQlhj#rd;_MA^z(6!hpuY^P8zQp~~F0|KIdGl4A zzfzuk7~6Aoyj7`h*3wa?XC0O zQ}Nw(en-pi(7rzb=iwYb^k!Y>CO930?Y%mmi{`a-l_?$vW3vkO-6`+R;(defB(E=T zuJ-ESe7x%4io-fM?1`&uEB2XFju+;!*32*cNFRyaAXzc~iG^SIvmR~@f4$4~cY{Ho`7t=wQ%Ilenv<9W%)#c>+8 zOJ)?CA+#4($MH&zSIxCyq54;kc-Zio(~~jAzM%56h=)ltOMBrImEYr&;EeXdB# zS9>$wac8*`>pM2b_HV8KiP&}4eQs6d!`{U6yEv|^>)TxUwLK0OD!)#_{(Kw`q&;&6 z`P}@b`_*RT`}_fR!{yyoekeAF>iAb<_ayD7#k7~+;r!jHI6k29`IzII-i%+GrtvzK z`5VrWXEOeN5$B&9Xui$FaUI1Q;?Pj>wYYw9W7ea+6o>w-pV%LpdRkwNt7iR$?y7&D z=5H&FZ#TIOP9vCK;dRAF$=7mz{to$moNlFlx*NL%953I5^Re{Dk5K)2*nO??7tsH3 z54PuWe)$a5@1y*48&2zyKmAiGzn${uI2^Xcc5AMWTT}V%D(p|g?rm(pr@iU(Kd``)^%-g~!ZT?hS~lugPLuM&4J{{If&%xmdZ^g-pl9511K-eX=%R|)67*tgMj zcmtcKlt(|t;U%tX{0FBWx$bdw-rKvAG!OJ1-<-`nGVO_nG0c0j&N=N;<+x!(t&_*G zeM;+k{SGBw9ow|gnf3@5QDLHs|R0cVT-jwzuK9RP(wwcFUMo z{vfWc`xU3=I4{sV)3wiwbl>=Zc-V^T6CXIg^fxIF+cwxb;;t1Ao_(y@MCW&|yb&&Q zsWMKr^Jl-x|MvSTmH$+}s-IcePt^Abyf2D7RQ9kd=Hce0SozQYd|p-adDg$5FI7M9 zs`)(3)_h)OYd%l=pU>NBK98&Yyw25po>%*MU+w3CwVxOM&*zCXpEuTi9_ea6ue2BI zea?35_tW~gj9q2K;xfk4Z${i}Z1c}N52WXb+mXsQFJjY-d6Czky{vg<^m%LL_kXl> zmG-SZPpw|Nu)`V5n}x({uh;&@tXBOw|9soxQr7K0zO)rCdZsvRP{~~@u6z9z|KZCH zT1dNhv%XDBdAaiN@^5IX(Y%(fQY8<=W@Vh3$-h%RfAV!HZ?-2MFT?&7oafFj@%%it z^JV4fd-UCBIebmHO^U z@>94Pha<5c!tv4&@?ZLp_T70n_Qt6`#ur7;u)Le^e2zSZh+ih@hKcH?S_4|f752rUeh_{eA8AqU8;P4*heLv=Q3Vy z^oRIh-H(<}r~V(hUN__X1)RZz*@@dc-Af5g*O*<1`%i#%5E>r+w&ex|sIO zc^aQp=udh@;=H>`K^=f3uu#xSQu<_sko318fhLpX9kVA0h99(;S@7)pK!w+Vg*ZSjIn$ z#_6N!_)HvUeNdd&q`p~iTCv%6QaQfu__8S=bDvzwrs|I&psX9~>9( zjyBe(Ju{r~G85F^5sYu>jKk(QEm!{stNb8r+u+dbWya%njsM3l7Q407-lH!R`?Y!Q z^JhL!`5l=L>5YlSVQp;g!sblPhtse>ThEOXo-5_;H1%)(1lk`jx5vRyKHv9jiN`Bx zFSo(@1?I0GJHEutw#N#oc`w*b^SNO_G#uv*hlBH9rG!lqyE1A7V|+r*Bj1#6CXo7 zjhs|$p67hR4~h75@|&;XJt&_~$L=)s?*<$$qCf61mH$TLJ^K~fKb-56Z^fxD*U!ED za*5l7;~{cy=0kSae8GH4Z@g5>r{T<}bQ+F_yi~47KIX*|&-*g}(t$5f{!E+)JYVcI zKjY|$#o-do*S&CFs`=jOIpW8&KEh5o-ho}`2_wk{q*8sjod9vd|iNi@bKsa;Z$E^bF|*}Bk>GD&F$ZMSmkHh7w-y#KR5-!rCj zp6Ar_Icx2;*Is+A_1pX3|NJvuna@mx{Rr574hQ*@kD_0tD(SxnyMFS2k^0?g)*Cm7 z2Pi(v#arNTv-;s+;_KQ-ejxF;A7MX?)&4X=Za;y;FgWbae&!?eywd)M<*<22`EnA^ z1GhW*ls3U(9rAoA&olSE^aqQNSN^@K=UuJxxiiljKl_!kKe0W}@AQXx@ON;&L_C7$ zeclfaYxR4ZyTSPY&Ie%z><$*Mpq^q|*e_U6<4ooJcMfcq!M-Qw55ItVOuae3q}!-h zJBss<8v~m~oR@r~*NQ#!FV0hbDe_?6Dftqb!116Jh0Sl7FRp>pfO%zn{AOj5$8E&R z9nbkOo}&7@tXu_@igZ%J{&>*g#??!;rv`C9t*{{e^@xZ44YQmm%dR_!lu*5#h%~5dfgka&5B3cPl`MaBHr#zwh&iQuc?0g=tbDi_@5$dKG1sZ;(V5t!+zWHqHnGc z_tbf6EAfcCuPE{`1hzWg_}@vsb!CyqJBep%^lssXuSZl@URPaI|9aNk{v~$yT6~X3 zUPvtK-h;f8F{V@HTVD;|ubk#ccM0>vS>lV~5MVnGzwBT*?E$;r!?6$RvbVr_4eO2* zB>#fABynC0+qaOrX0SV`FKfAmbC%m6d#j0ao-ZDubMXw=-wOLNu(=m@H;6aFW&j*( zHU20#&4TM==h|?t^WBh#Wt_{smwtEH*1+aZoa5an*dE6@(B2KZPjwD_6!w1-zoPM1 zz<#9G|EA*d6*dDeD*A4<@_KB) z!tr|8pGyEkRm3!`X>i>^aC8Ohx6ye)BYZ| z3)zo!IUKf-AHM$cvc`~qu8r1nhw7}s`{z@G_t!Uk|9!*v=Qn)+e#7_oH>{^)Ds^d8 zm#%4b?^LO~Je{?<4xI|8t1GotSC{*%C@X*cTlSjj;2m_$BGrx9jSvs|O|fNflfF2g z0_SIt=ab?19PCypPhW!LDCOzV|ATL@UOWJvT@Gogwb3et7_#Yhl|4PTSbm{0;t@Vd_ga$o~`Ht#Orj+Wo|J zv954@f%S%yVe^FeKJ~FHV80Uk<~!o=kB8GB*lxuiQ?3134>#PuV=M2IKSX(u&r%Nv zyjGBAo>yutud1xQE)Qxd-}-V*dQ*?BiIZ;zoB6~s><-)e;q+5Dt6!$w;Pf-(uCeqR z!$Plr=u z@ffjk$*$0TKBL@v9?lzKI~evSF+R_rPV;Q>I-X~_-v4%b-Zdw_d7=98ad3J-edGq% zjTP^u{apk5--_45@o4fRj)zSn^3ybfLpAK5qR)hD#LucPEr#7^dL5->pR(Wnzw|tO z0d}f2yBD<=ry}?5;H2kqeQlasm@oevxxE5^Y_)2)De*}9eK5ZO4$WnMC-uFb!sbKb z9shwd^rT6qq-E1s-zl^=!c&xL=0 z{VwMf4(G%1NH|~mMA7$8qn}@zRXDyY{(ffRFb;k5=NZWFIKS9)Z$Aci?OE8L5BqQ7 zxEZ#G_d@<5oZ9y(oLW7K{p&6)YumggR`F^7G<+2tzo{*3@916pv-5TDo~ZcNaxYI; zT~OrdO75*;2K!<9ac}h-+0SIS2dD1rcgZbNIkzk$pW;*K$Cu!=C+sG`{y6OWP2{HE zjz9JY*o~f3{H@R3bn&xrSLx4VJ?0+R4u|7m%I5*%%VGEOGi7`_4GvE$KDWSm4f)-$ z9-J||)}M#-FXzxxmX9d5!hH`EHmxeY*45Sfm2Z9d{T}~OR{2?Ya(ZDmn|1i{j}?yX z;Mh_8j{N@TqeUM2ipRia0PBb+!0r;(;Vy%HmDbe)dwKFxHLfy$dPerAuuj`e{=7;Z zhf84VU^@(sqsS|(&s7cmm3L11SHt#6*}q2e>oxx*IBb`{+h8*UwjaXrWcfcrJW%@0 z;qkc!|vLtMc-D#=1|xyq)vT%*ms5VdD8DeUAyDq z{G$5OJ{rH9`ce}(by45=atizVAbrJ8fL%8@T?E_iuLX=*+O8fx5H<^7 zzY82T!LGyPq95jK{5KC5&dcC57!GUH2e(c_{yZF}!tn*!UpcYJV@t;SPU4Pmo*I#V zrTMF1(-TfBCKS0j2DSs?cmbUEfW!ZZ=Zr7<_UwmCzS{P%>piisuNL1J3%fzEUp1j{ z>?VGHeBrR}p<*x1e+XU$+Z)F*{%qJyc#!ea#ufc&#v*?RPIui0p8PddyhSMRFRv< zq(AzO!fCsDQ)@V0jQ#wNk;s?8u_bc99X8`&dkX&M6&gQH>v;`MU%+mx^g9tBezv{jAF7n5{fIKaOy<>cSPrMWM z&2r`2-(bH{@_BH0nEdrs(%+zb-W|@J$^URMY=1(%xJ$JkC&G3z9AAOMQrPvR{@gFv zU-O3ePBHkaiu2%nlL-q0tK8HHpE`A2KH;C_?Nxgjh0RFuS z`x{{U5gb?Gzd36b_2I-{!~Q|)(d;oB`5CIuX0SPo`iz}nf9XBskJ#-AXMHYu{$BNY z3ieG;I2{7Vlc_IT4cm?6bDj&Q>)^_7eX_nwl>duh`<>*T&&_sT^}0d-K0i_UqR)xX z7peYl{rfuUN#puWxlvK>~6%KxeN{)cpgWwzj!kK1nj5xcz*j{usxge zfxSikY=+H8a9pnQPao{tD$YyhRoJwJ?PlKpVFz*VGDCU)hrJH=f7km@4ufM`-v5!C z!C~~X7bH`8-C(8UFT+{)HlN_2duSQ~yUEkb`usn!XYYBuaF{Rq>)^Noc5OIMI^7LS z)vTgle^;=>;j}k*hrARvKVv@I<_YX4**_P~Ggx05$ay`zPyPAJVSkM3aWHIa74JVX zJ`ROL2k}*KuH`%*YT)!M*smg={0#ZqLHV_9Ch?d?ewzDW`*Y>*-Hdnph=*uB>YwQk z_?N|%eJ=J=fAqsi+*jhwu)j$6G}V(|D?Uj2zve!aS5mL8M%)Wd|3Kemo)zPeaeVrWN+Pur2>&aja@chEgzGydDd;jje`HTdKHN<7j@>SJQte^QyR z;eORre5n6_yYu($h}|Of#8cZA_WM?RsoVEmE8qHxlI0fo=}tp8X>8goTd!lQ-gZ0` zzkTcdiFHmppFz6eDB@u@ u|A%fx9w+R`;UU=Hir@9N^OKWmTt(lHK46Er>wSrn@WpCAb13>d%>VzCG>ZNJ literal 0 HcmV?d00001 diff --git a/tests/data/coutwildrnp.shx b/tests/data/coutwildrnp.shx new file mode 100644 index 0000000000000000000000000000000000000000..7eae7002fa8fbb935b0f012c4aec5b9a918ce079 GIT binary patch literal 636 zcmaiwPe@c@6vcngZu!aCkF z>;Djm?q?h3oL}KPE$k-!#Wwe5p=jQ~JxhN?>sOh?|GMmc=f7*Tj^KSfZwHpU)H#2p z#yqX?DQ7MJ*}U<)u0Dcl_pJT0zX4y&6G&}*bWRhLe_9mb>r?yUH~m$%Hro^bmNj|U HlBaM88B%7g literal 0 HcmV?d00001 diff --git a/tests/test_fiona.py b/tests/test_fiona.py new file mode 100644 index 00000000..51934a72 --- /dev/null +++ b/tests/test_fiona.py @@ -0,0 +1,10 @@ +import unittest + +import fiona +import pandas as pd + +class TestFiona(unittest.TestCase): + def test_read(self): + with fiona.open("/input/tests/data/coutwildrnp.shp") as source: + self.assertEqual(67, len(source)) + From 9b813778b74d6b55d6565a6d6be47a9d044115b9 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 04:56:51 +0000 Subject: [PATCH 0421/1660] add tests for geopandas --- tests/test_geopandas.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/test_geopandas.py diff --git a/tests/test_geopandas.py b/tests/test_geopandas.py new file mode 100644 index 00000000..10603fa0 --- /dev/null +++ b/tests/test_geopandas.py @@ -0,0 +1,8 @@ +import unittest + +import geopandas + +class TestGeopandas(unittest.TestCase): + def test_read(self): + df = geopandas.read_file(geopandas.datasets.get_path('nybb')) + self.assertTrue(df.size > 1) From 1c5c6542b39ff3636c28645313c79497229bfff4 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 04:57:07 +0000 Subject: [PATCH 0422/1660] add tests for geoviews --- tests/test_geoviews.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/test_geoviews.py diff --git a/tests/test_geoviews.py b/tests/test_geoviews.py new file mode 100644 index 00000000..36ce8882 --- /dev/null +++ b/tests/test_geoviews.py @@ -0,0 +1,11 @@ +import unittest + +import geoviews.feature as gf + +from cartopy import crs + +class TestGeoviews(unittest.TestCase): + def test_viz(self): + (gf.ocean + gf.land + gf.ocean * gf.land * gf.coastline * gf.borders).options( + 'Feature', projection=crs.Geostationary(), global_extent=True, height=325 + ).cols(3) From 7ec3690bb337e46fd48e8c32fcbd7d2a01f6d682 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 04:57:35 +0000 Subject: [PATCH 0423/1660] add tests for holoviews --- tests/test_holoviews.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/test_holoviews.py diff --git a/tests/test_holoviews.py b/tests/test_holoviews.py new file mode 100644 index 00000000..88671590 --- /dev/null +++ b/tests/test_holoviews.py @@ -0,0 +1,10 @@ +import unittest + +import holoviews as hv + +class TestHoloviews(unittest.TestCase): + def test_curve(self): + xs = range(-10,11) + ys = [100-x**2 for x in xs] + + hv.Curve((xs, ys)) From 71a80e877eddcffac8b3402360be96d556ea2c37 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 04:57:57 +0000 Subject: [PATCH 0424/1660] reduce log spamming for fbprophet tests --- tests/test_fbprophet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_fbprophet.py b/tests/test_fbprophet.py index f8badbd6..69caa984 100644 --- a/tests/test_fbprophet.py +++ b/tests/test_fbprophet.py @@ -12,5 +12,5 @@ def test_fit(self): 'y': np.array([38.23, 21.25]) }) - forecaster = Prophet() + forecaster = Prophet(mcmc_samples=1) forecaster.fit(train) From b8ef08e46b69bf7d9e375b865abbfefab71f63d2 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 04:58:18 +0000 Subject: [PATCH 0425/1660] install holoviews and geoviews with pip --- Dockerfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2d010414..7f4b40e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -274,7 +274,6 @@ RUN pip install fancyimpute && \ pip install descartes && \ pip install geojson && \ pip install pysal && \ - #conda install -y gdal && \ pip install pyflux && \ pip install terminalplot && \ pip install raccoon && \ @@ -292,11 +291,8 @@ RUN pip install fancyimpute && \ pip install sklearn-pandas && \ pip install stemming && \ pip install fbprophet && \ - conda install -y -c conda-forge -c ioam holoviews geoviews && \ - #Temp fix: After installing holoviews and geoviews, deps for fiona and geopandas get really messed up. This is a very unelegant fix. - conda uninstall -y fiona geopandas && \ - pip uninstall -y fiona geopandas && \ - apt-get install -y libgdal1-dev && GDAL_CONFIG=/usr/bin/gdal-config pip install fiona && pip install geopandas && \ + pip install holoviews && \ + pip install geoviews && \ pip install hypertools && \ # Nxviz has been causing an installation issue by trying unsuccessfully to remove setuptools. #pip install nxviz && \ From 77bc7cf171fe7deec1f10f2d8ecf9d87a2f18403 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Thu, 11 Oct 2018 19:55:32 +0000 Subject: [PATCH 0426/1660] install feather-format and add tests --- Dockerfile | 4 ++++ tests/data/feather-0_3_1.feather | Bin 0 -> 672 bytes tests/test_pandas.py | 7 ++++++- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/data/feather-0_3_1.feather diff --git a/Dockerfile b/Dockerfile index 7f4b40e7..b093a493 100644 --- a/Dockerfile +++ b/Dockerfile @@ -425,6 +425,10 @@ RUN pip install bcolz && \ pip install wcwidth && \ pip install webencodings && \ pip install widgetsnbextension && \ + # Latest version of pyarrow conflicts with pandas + # https://github.com/pandas-dev/pandas/issues/23053 + pip install pyarrow==0.10.0 && \ + pip install feather-format && \ cd /usr/local/src && git clone --depth=1 https://github.com/fastai/fastai && \ cd fastai && python setup.py install && \ # clean up pip cache diff --git a/tests/data/feather-0_3_1.feather b/tests/data/feather-0_3_1.feather new file mode 100644 index 0000000000000000000000000000000000000000..5a2c7b3dcc684b3676a94287702ea01aefa057af GIT binary patch literal 672 zcmZvaF;2r!42GRH6%j;Hh9YHPs1O4KL)q8}5_IMO2&n~CAXRkf$jr*b0gw=fKs^*E zAbkIOF+@;D_Otzd|8|^AF3ye(Nn|8HPIP-QYtvcl*z0)x_%bv*zkYri?8fZpX?t6H zxz_sISSiy5(TliUEtb!#> zF9YhAU;SJ@TFyNZ>fxdp=8NTY#aTB^!?9gsBtEC}yvO@q#oZA)Lc3hT-$In}#Q8{= z_3vU`Iu7A~UhS%)Ca0=oYemE*>a##cXt$5{aKpsvqrCR3eU%_L>}5wh5`Q0$OWWUs wPyBryFa9V6J>vJ&m(`r=t6g;GlRsN7H9B~0T=zM4NF3)??l1odi3$JaHx?%;)c^nh literal 0 HcmV?d00001 diff --git a/tests/test_pandas.py b/tests/test_pandas.py index f9163c57..abf70398 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -6,4 +6,9 @@ class TestPandas(unittest.TestCase): def test_read_csv(self): data = pd.read_csv("/input/tests/data/train.csv") - self.assertEqual(2, len(data.shape)) + self.assertEqual(14915, data.size) + + def test_read_feather(self): + data = pd.read_feather("/input/tests/data/feather-0_3_1.feather") + + self.assertEqual(10, data.size) From 7b7dfd6ee8e58e8371953e9c43dc7ccd1aa8d5fd Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Sat, 13 Oct 2018 01:02:20 +0000 Subject: [PATCH 0427/1660] Expand fastai tests --- tests/test_fastai.py | 30 ++++++++++++++++++++++++++++++ tests/test_pandas.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/test_fastai.py b/tests/test_fastai.py index 275d2776..7d28b347 100644 --- a/tests/test_fastai.py +++ b/tests/test_fastai.py @@ -1,9 +1,39 @@ import unittest +import fastai +import pandas as pd +import torch + +from fastai.docs import * +from fastai.tabular import * from fastai.core import partition +from fastai.torch_core import tensor class TestFastAI(unittest.TestCase): def test_partition(self): result = partition([1,2,3,4,5], 2) self.assertEqual(3, len(result)) + + def test_has_version(self): + self.assertGreater(len(fastai.__version__), 1) + + # based on https://github.com/fastai/fastai/blob/master/tests/test_torch_core.py#L17 + def test_torch_tensor(self): + a = tensor([1, 2, 3]) + b = torch.tensor([1, 2, 3]) + + self.assertTrue(torch.all(a == b)) + + def test_tabular(self): + df = pd.read_csv("/input/tests/data/train.csv") + + train_df, valid_df = df[:-5].copy(),df[-5:].copy() + dep_var = "label" + cont_names = [] + for i in range(784): + cont_names.append("pixel" + str(i)) + + data = tabular_data_from_df("", train_df, valid_df, dep_var, cont_names=cont_names, cat_names=[]) + learn = get_tabular_learner(data, layers=[200, 100]) + learn.fit(epochs=1) diff --git a/tests/test_pandas.py b/tests/test_pandas.py index abf70398..f533a043 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -6,7 +6,7 @@ class TestPandas(unittest.TestCase): def test_read_csv(self): data = pd.read_csv("/input/tests/data/train.csv") - self.assertEqual(14915, data.size) + self.assertEqual(19, len(data.index)) def test_read_feather(self): data = pd.read_feather("/input/tests/data/feather-0_3_1.feather") From d74125053ba6218aed13cf1cbcb8b9efc61e42b6 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Sat, 13 Oct 2018 19:17:51 +0000 Subject: [PATCH 0428/1660] Pin fastai to 0.7.0 --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b093a493..a485c1d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -429,8 +429,7 @@ RUN pip install bcolz && \ # https://github.com/pandas-dev/pandas/issues/23053 pip install pyarrow==0.10.0 && \ pip install feather-format && \ - cd /usr/local/src && git clone --depth=1 https://github.com/fastai/fastai && \ - cd fastai && python setup.py install && \ + pip install fastai=0.7.0 && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ cd && rm -rf /usr/local/src/* From c83eaeeabf742e520569c1faa94fb5f84129d26d Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Sun, 14 Oct 2018 12:00:05 -0700 Subject: [PATCH 0429/1660] fix pip command typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a485c1d9..641cc92d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -429,7 +429,7 @@ RUN pip install bcolz && \ # https://github.com/pandas-dev/pandas/issues/23053 pip install pyarrow==0.10.0 && \ pip install feather-format && \ - pip install fastai=0.7.0 && \ + pip install fastai==0.7.0 && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ cd && rm -rf /usr/local/src/* From ecd64a15e9866229dfba2b222dc05b41351c7683 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 15 Oct 2018 03:49:46 +0000 Subject: [PATCH 0430/1660] simplify fastai tests imports --- tests/test_fastai.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_fastai.py b/tests/test_fastai.py index 7d28b347..31236e98 100644 --- a/tests/test_fastai.py +++ b/tests/test_fastai.py @@ -4,8 +4,7 @@ import pandas as pd import torch -from fastai.docs import * -from fastai.tabular import * +from fastai.tabular import tabular_data_from_df, get_tabular_learner from fastai.core import partition from fastai.torch_core import tensor From 79e96f06edab69109b40d1a400d5efc2a6058698 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 15 Oct 2018 17:14:06 +0000 Subject: [PATCH 0431/1660] Install fastai from source and pinned version --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 641cc92d..f7b0e8ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -429,7 +429,9 @@ RUN pip install bcolz && \ # https://github.com/pandas-dev/pandas/issues/23053 pip install pyarrow==0.10.0 && \ pip install feather-format && \ - pip install fastai==0.7.0 && \ + # This version is not available in pypi + # Also, installing from source to prevent static TLS (thread local storage) issue + pip install git+git://github.com/fastai/fastai.git@v0.7.2 && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ cd && rm -rf /usr/local/src/* From bcda2d4f154a1498b0c53e3257e6e35c7cc969b1 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 15 Oct 2018 18:48:47 +0000 Subject: [PATCH 0432/1660] Prevent fastai install from downgrading pytorch --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f7b0e8ef..c549ced6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -429,9 +429,9 @@ RUN pip install bcolz && \ # https://github.com/pandas-dev/pandas/issues/23053 pip install pyarrow==0.10.0 && \ pip install feather-format && \ - # This version is not available in pypi - # Also, installing from source to prevent static TLS (thread local storage) issue - pip install git+git://github.com/fastai/fastai.git@v0.7.2 && \ + # Don't install dependencies for fastai because it requires pytorch<0.4. + # which downgrades pytorch. fastai does work with pytorch 0.4. + pip install fastai==0.7.0 --no-deps && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ cd && rm -rf /usr/local/src/* From 8a430bb21e2e4928ec16ef370f7e4b4019b24ff4 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 15 Oct 2018 20:39:32 +0000 Subject: [PATCH 0433/1660] Remove fastai tests using 1.x api --- Dockerfile | 1 + tests/test_fastai.py | 45 ++++++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index c549ced6..500441ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -432,6 +432,7 @@ RUN pip install bcolz && \ # Don't install dependencies for fastai because it requires pytorch<0.4. # which downgrades pytorch. fastai does work with pytorch 0.4. pip install fastai==0.7.0 --no-deps && \ + pip install torchtext && \ # clean up pip cache rm -rf /root/.cache/pip/* && \ cd && rm -rf /usr/local/src/* diff --git a/tests/test_fastai.py b/tests/test_fastai.py index 31236e98..94223fe3 100644 --- a/tests/test_fastai.py +++ b/tests/test_fastai.py @@ -4,9 +4,8 @@ import pandas as pd import torch -from fastai.tabular import tabular_data_from_df, get_tabular_learner from fastai.core import partition -from fastai.torch_core import tensor +from fastai.layer_optimizer import LayerOptimizer class TestFastAI(unittest.TestCase): def test_partition(self): @@ -14,25 +13,31 @@ def test_partition(self): self.assertEqual(3, len(result)) - def test_has_version(self): - self.assertGreater(len(fastai.__version__), 1) - - # based on https://github.com/fastai/fastai/blob/master/tests/test_torch_core.py#L17 - def test_torch_tensor(self): - a = tensor([1, 2, 3]) - b = torch.tensor([1, 2, 3]) + # based on https://github.com/fastai/fastai/blob/0.7.0/tests/test_layer_optimizer.py + def test_layer_optimizer(self): + lo = LayerOptimizer(FakeOpt, fastai_params_('A', 'B', 'C'), 1e-2, 1e-4) + fast_check_optimizer_(lo.opt, [(nm, 1e-2, 1e-4) for nm in 'ABC']) - self.assertTrue(torch.all(a == b)) - def test_tabular(self): - df = pd.read_csv("/input/tests/data/train.csv") +class Par(object): + def __init__(self, x, grad=True): + self.x = x + self.requires_grad = grad + def parameters(self): return [self] - train_df, valid_df = df[:-5].copy(),df[-5:].copy() - dep_var = "label" - cont_names = [] - for i in range(784): - cont_names.append("pixel" + str(i)) - data = tabular_data_from_df("", train_df, valid_df, dep_var, cont_names=cont_names, cat_names=[]) - learn = get_tabular_learner(data, layers=[200, 100]) - learn.fit(epochs=1) +class FakeOpt(object): + def __init__(self, params): self.param_groups = params + + +def fastai_params_(*names): return [Par(nm) for nm in names] + +def fast_check_optimizer_(opt, expected): + actual = opt.param_groups + assert len(actual) == len(expected) + for (a, e) in zip(actual, expected): fastai_check_param_(a, *e) + +def fastai_check_param_(par, nm, lr, wd): + assert par['params'][0].x == nm + assert par['lr'] == lr + assert par['weight_decay'] == wd From 2aa9b9c1b85630b64cb787e51d1bca7b52188628 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 15 Oct 2018 15:25:23 -0700 Subject: [PATCH 0434/1660] Prune images previously built to prevent permanent gpu worker from filing up disk --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 702898fb..c8ab54da 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,7 +65,7 @@ pipeline { slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} docker build>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL sh '''#!/bin/bash set -exo pipefail - docker image prune -a # remove previously built image to prevent disk from filling up + docker image prune -a -f # remove previously built image to prevent disk from filling up ./build --gpu | ts ''' } From 6eb31c1581767691aa766dc3633d118ee3e9e9b2 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 15 Oct 2018 15:58:09 -0700 Subject: [PATCH 0435/1660] Update README to mention that GCR is the source of truth for our docker images. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 90faac2e..655c3977 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![example script](http://i.imgur.com/yrWycNA.png) -This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://registry.hub.docker.com/u/kaggle/python/) the Docker image on Dockerhub. +This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://https://gcr.io/kaggle-images/python) the Docker image on Google Container Registry. ## Getting started @@ -17,7 +17,7 @@ We can merge your request quickly if you check that it builds correctly. Here's Start by running this image on your system: ``` -me@my-computer:/home$ docker run --rm -it kaggle/python +me@my-computer:/home$ docker run --rm -it gcr.io/kaggle-images/python root@d72b81a003e1:/# ``` From 549f89ac7f3a15070de428199b5619eb53eac485 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 19 Oct 2018 17:20:35 -0400 Subject: [PATCH 0436/1660] Prevent upgrade of matplotlib plotnine 0.5 is depending on matplotlib >= 3.0 which is not compatible with basemap. --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 500441ab..07ed960a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -311,7 +311,9 @@ RUN pip install fancyimpute && \ pip install folium && \ pip install scikit-plot && \ pip install dipy && \ - pip install plotnine && \ + # plotnine 0.5 is depending on matplotlib >= 3.0 which is not compatible with basemap. + # once basemap support matplotlib, we can unpin this package. + pip install plotnine==0.4.0 && \ pip install git+https://github.com/dvaida/hallucinate.git && \ pip install scikit-surprise && \ pip install pymongo && \ From 8048a37411e7bd7e032a132358c1bca7552e2f2f Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 19 Oct 2018 19:33:41 +0000 Subject: [PATCH 0437/1660] Upgrade anaconda base to 5.2.0 --- Dockerfile | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07ed960a..2351c92d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM gcr.io/kaggle-images/python-tensorflow-whl:1.11.0-py36 as tensorflow_whl -FROM continuumio/anaconda3:5.0.1 +FROM continuumio/anaconda3:5.2.0 # This is necessary for apt to access HTTPS sources RUN apt-get update && \ @@ -30,13 +30,13 @@ RUN pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ conda install -y -c conda-forge spacy && python -m spacy download en && \ python -m spacy download en_core_web_lg && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source - apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid ghostscript gsfonts hicolor-icon-theme \ + apt-get -y install dbus fontconfig fontconfig-config fonts-dejavu-core fonts-droid-fallback ghostscript gsfonts hicolor-icon-theme \ libavahi-client3 libavahi-common-data libavahi-common3 libcairo2 libcap-ng0 libcroco3 \ libcups2 libcupsfilters1 libcupsimage2 libdatrie1 libdbus-1-3 libdjvulibre-text libdjvulibre21 libfftw3-double3 libfontconfig1 \ libfreetype6 libgdk-pixbuf2.0-0 libgdk-pixbuf2.0-common libgomp1 libgraphite2-3 libgs9 libgs9-common libharfbuzz0b libijs-0.35 \ - libilmbase6 libjasper1 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1-0 libltdl7 libmagickcore-6.q16-2 \ - libmagickcore-6.q16-2-extra libmagickwand-6.q16-2 libnetpbm10 libopenexr6 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ - libpaper-utils libpaper1 libpixman-1-0 libpng12-0 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ + libilmbase12 libjbig0 libjbig2dec0 libjpeg62-turbo liblcms2-2 liblqr-1-0 libltdl7 libmagickcore-6.q16-3 \ + libmagickcore-6.q16-3-extra libmagickwand-6.q16-3 libnetpbm10 libopenexr22 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 \ + libpaper-utils libpaper1 libpixman-1-0 libpng16-16 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ cd /usr/local/src && \ wget http://transloadit.imagemagick.org/download/ImageMagick.tar.gz && \ @@ -126,13 +126,7 @@ RUN apt-get -y install zlib1g-dev liblcms2-dev libwebp-dev libgeos-dev && \ cd /usr/local/src && git clone https://github.com/matplotlib/basemap.git && \ cd basemap && \ git checkout v1.1.0 && \ - # Install geos - cd geos-3.3.3 && \ - export GEOS_DIR=/usr/local && \ - ./configure --prefix=$GEOS_DIR && \ - make && make install && \ - # Install basemap - cd .. && python setup.py install && \ + python setup.py install && \ pip install basemap --no-binary basemap # sasl is apparently an ibis dependency From eb7b055c5ba92087074d948e34ac5e5f65de1bde Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 26 Oct 2018 18:33:39 +0000 Subject: [PATCH 0438/1660] install fasttesxt using pip git feature --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2351c92d..a5a54e68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -341,7 +341,7 @@ RUN pip install --upgrade cython && \ pip install --upgrade cysignals && \ pip install pyfasttext && \ pip install ktext && \ - cd /usr/local/src && git clone --depth=1 https://github.com/facebookresearch/fastText.git && cd fastText && pip install . && \ + pip install git+git://github.com/facebookresearch/fastText.git && \ apt-get install -y libhunspell-dev && pip install hunspell && \ pip install annoy && \ pip install category_encoders && \ From 736bad3134ee889e5cadfcb96f546e425a28c06f Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Fri, 26 Oct 2018 18:19:09 +0000 Subject: [PATCH 0439/1660] Don't preload tensorflow and torch --- patches/sitecustomize.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index f0dfab45..9221ba59 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -1,13 +1,6 @@ -# Monkey patches BigQuery client creation to use proxy. - -# Import tensorflow and torch before anything else. This is a hacky workaround to an error on dlopen -# reporting a limit on static TLS, tracked in: -# tensorflow: https://github.com/tensorflow/tensorflow/issues/19010 -# torch: https://github.com/pytorch/pytorch/issues/2575 -import tensorflow -import torch import os +# Monkey patches BigQuery client creation to use proxy. kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN") if kaggle_proxy_token: from google.auth import credentials From f9c4830c3bbe42cfcce579081568e4c45b0463fa Mon Sep 17 00:00:00 2001 From: Chris Gorgolewski Date: Sun, 28 Oct 2018 11:22:37 -0700 Subject: [PATCH 0440/1660] Add libGl1 to satisfy vtk depenencies --- Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2351c92d..17367a23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -210,11 +210,19 @@ RUN pip install scipy && \ conda clean -i -l -t -y && \ rm -rf /usr/local/src/* + +# vtk with dependencies +RUN apt-get install -y libgl1-mesa-glx && \ + pip install vtk && \ + # ~~~~ CLEAN UP ~~~~ + rm -rf /root/.cache/pip/* && \ + apt-get autoremove -y && apt-get clean + + RUN pip install --upgrade mpld3 && \ pip install mplleaflet && \ pip install gpxpy && \ pip install arrow && \ - pip install vtk && \ pip install nilearn && \ pip install nibabel && \ pip install pronouncing && \ From 5389746d750a50f3f31e12dad402f6121343a931 Mon Sep 17 00:00:00 2001 From: Chris Gorgolewski Date: Sun, 28 Oct 2018 14:33:26 -0700 Subject: [PATCH 0441/1660] adding xvfb --- Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index 2351c92d..89ae6c49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -210,6 +210,15 @@ RUN pip install scipy && \ conda clean -i -l -t -y && \ rm -rf /usr/local/src/* + +# xvfbwrapper with dependencies +RUN apt-get install -y xvfb && \ + pip install xvfbwrapper && \ + # ~~~~ CLEAN UP ~~~~ + rm -rf /root/.cache/pip/* && \ + apt-get autoremove -y && apt-get clean + + RUN pip install --upgrade mpld3 && \ pip install mplleaflet && \ pip install gpxpy && \ From 035f3d13c1cd3079e4adfab6932fd47e901147ca Mon Sep 17 00:00:00 2001 From: Chris Gorgolewski Date: Mon, 29 Oct 2018 09:49:45 -0700 Subject: [PATCH 0442/1660] adding xvfbwrapper test --- tests/test_xvfbwrapper.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/test_xvfbwrapper.py diff --git a/tests/test_xvfbwrapper.py b/tests/test_xvfbwrapper.py new file mode 100644 index 00000000..0b8dac45 --- /dev/null +++ b/tests/test_xvfbwrapper.py @@ -0,0 +1,11 @@ +import unittest +import os.path +from xvfbwrapper import Xvfb + +class TestXvfbwrapper(unittest.TestCase): + def test_xvfb(self): + vdisplay = Xvfb() + vdisplay.start() + display_var = ':{}'.format(vdisplay.new_display) + self.assertEqual(display_var, os.environ['DISPLAY']) + vdisplay.stop() From 8b04aaee44e3e297ddc08c9354862fed69a58f31 Mon Sep 17 00:00:00 2001 From: Chris Gorgolewski Date: Mon, 29 Oct 2018 09:58:18 -0700 Subject: [PATCH 0443/1660] adding test for dipy renderer --- tests/test_dipy.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/test_dipy.py diff --git a/tests/test_dipy.py b/tests/test_dipy.py new file mode 100644 index 00000000..de12ef74 --- /dev/null +++ b/tests/test_dipy.py @@ -0,0 +1,19 @@ +import unittest +import os.path +from dipy.viz import window +from xvfbwrapper import Xvfb + +class TestDipy(unittest.TestCase): + out_file = 'test.png' + def test_renderer(self): + vdisplay = Xvfb() + vdisplay.start() + + ren = window.Renderer() + window.record(ren, n_frames=1, out_path=self.out_file, size=(600, 600)) + self.assertTrue(os.path.exists(self.out_file)) + + vdisplay.stop() + + def tearDown(self): + os.remove(self.out_file) From 1e73362cc332ea7103de410df5551f64cb6f2411 Mon Sep 17 00:00:00 2001 From: Chris Gorgolewski Date: Mon, 29 Oct 2018 10:01:23 -0700 Subject: [PATCH 0444/1660] adding vtk import test --- tests/test_vtk.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/test_vtk.py diff --git a/tests/test_vtk.py b/tests/test_vtk.py new file mode 100644 index 00000000..7100223a --- /dev/null +++ b/tests/test_vtk.py @@ -0,0 +1,6 @@ +import unittest + + +class TestVTK(unittest.TestCase): + def test_import(self): + import vtk From 67b8d337101972ee9ace1d4d187c0160dd611b81 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 30 Oct 2018 11:01:22 -0400 Subject: [PATCH 0445/1660] Preload tensorflow If fastai is used prior to tensorflow, an older version of libcudnn is loaded and tensorflow fails with the following error message: ``` tensorflow/stream_executor/cuda/cuda_dnn.cc:343] Loaded runtime CuDNN library: 7.1.2 but source was compiled with: 7.2.1. CuDNN library major and minor version needs to match or have higher minor version in case of CuDNN 7.0 or later version. If using a binary install, upgrade your CuDNN library. If building from sources, make sure the library loaded at runtime is compatible with the version specified during compile configuration. ``` Preloading for now (like before) until we find how to fix the issue with fastai --- patches/sitecustomize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 9221ba59..7447f9cc 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -1,3 +1,5 @@ +# TODO(rosbo): Remove this once we fix the issue with fastai importing older libcudnn if imported prior to tensorflow +import tensorflow import os # Monkey patches BigQuery client creation to use proxy. From ff8079a18b7988efb03bf0694933fb771ceb607d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Ruales?= Date: Tue, 30 Oct 2018 23:36:39 -0700 Subject: [PATCH 0446/1660] Fix broken link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 655c3977..453a4ca8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![example script](http://i.imgur.com/yrWycNA.png) -This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://https://gcr.io/kaggle-images/python) the Docker image on Google Container Registry. +This is the Dockerfile (etc.) used for building the image that runs python scripts on Kaggle. [Here's](https://gcr.io/kaggle-images/python) the Docker image on Google Container Registry. ## Getting started From 4c6ed0fec1b8385169ec30eaef0a55b055d9972d Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 12 Nov 2018 19:30:01 +0000 Subject: [PATCH 0447/1660] Install latest version of seaborn --- Dockerfile | 5 +++-- tests/test_seaborn.py | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8e0430aa..ea4f47f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,8 +25,9 @@ RUN conda install -y python=3.6.6 && \ pip install keras_applications==1.0.4 --no-deps && \ pip install keras_preprocessing==1.0.2 --no-deps -RUN pip install seaborn python-dateutil dask pytagcloud pyyaml joblib \ - husl geopy ml_metrics mne pyshp && \ +# The anaconda base images includes outdated versions of these packages. Update them to include the latest version. +RUN pip install --upgrade seaborn python-dateutil dask && \ + pip install pyyaml joblib pytagcloud husl geopy ml_metrics mne pyshp && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ python -m spacy download en_core_web_lg && \ # The apt-get version of imagemagick is out of date and has compatibility issues, so we build from source diff --git a/tests/test_seaborn.py b/tests/test_seaborn.py index 3986bc8c..b20cbe27 100644 --- a/tests/test_seaborn.py +++ b/tests/test_seaborn.py @@ -1,7 +1,14 @@ import unittest +from distutils.version import StrictVersion + import seaborn as sns class TestSeaborn(unittest.TestCase): + # Fails if seaborn gets downgraded by other package installations. + def test_version(self): + self.assertGreaterEqual(StrictVersion(sns.__version__), StrictVersion("0.9.0")) + + def test_option(self): sns.set(style="darkgrid") From 3a2c83983c866f82bb8b563a90d6a290698f3b41 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Mon, 12 Nov 2018 20:20:16 +0000 Subject: [PATCH 0448/1660] fix-typo-comment --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ea4f47f4..3525c526 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN conda install -y python=3.6.6 && \ pip install keras_applications==1.0.4 --no-deps && \ pip install keras_preprocessing==1.0.2 --no-deps -# The anaconda base images includes outdated versions of these packages. Update them to include the latest version. +# The anaconda base image includes outdated versions of these packages. Update them to include the latest version. RUN pip install --upgrade seaborn python-dateutil dask && \ pip install pyyaml joblib pytagcloud husl geopy ml_metrics mne pyshp && \ conda install -y -c conda-forge spacy && python -m spacy download en && \ From 429dec5b0e53c0e43b1664c14a7061083e5a9d25 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 13 Nov 2018 02:36:45 +0000 Subject: [PATCH 0449/1660] Install h2o dependencies --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3525c526..71f236c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -142,12 +142,11 @@ RUN apt-get -y install libsasl2-dev && \ pip install cartopy && \ # MXNet pip install mxnet && \ - # h2o - # Temporary sync of conda's numpy with pip's, needed to avoid an install error - #conda upgrade -y numpy && \ + # h2o (requires java) # Upgrade numpy with pip to avoid install errors pip install --upgrade numpy && \ - # This requires python-software-properties and Java, which were installed above. + # requires java + apt-get install -y default-jdk && \ cd /usr/local/src && mkdir h2o && cd h2o && \ wget http://h2o-release.s3.amazonaws.com/h2o/latest_stable -O latest && \ wget --no-check-certificate -i latest -O h2o.zip && rm latest && \ From 5b9224e5cd500abd961db834e67b688db3f9e2e0 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 13 Nov 2018 02:37:22 +0000 Subject: [PATCH 0450/1660] Add tests for h2o --- tests/test_h2o.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/test_h2o.py diff --git a/tests/test_h2o.py b/tests/test_h2o.py new file mode 100644 index 00000000..7b6b212d --- /dev/null +++ b/tests/test_h2o.py @@ -0,0 +1,9 @@ +import unittest + +import h2o + +class TestH2o(unittest.TestCase): + def test_init_read(self): + h2o.init() + train = h2o.import_file("/input/tests/data/train.csv", destination_frame="train") + self.assertEqual(19, train.nrow) From 0020fba526ec69378d5ae59e50d31796fb289269 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 13 Nov 2018 19:41:22 +0000 Subject: [PATCH 0451/1660] close h2o session in test --- tests/test_h2o.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_h2o.py b/tests/test_h2o.py index 7b6b212d..94fc57b8 100644 --- a/tests/test_h2o.py +++ b/tests/test_h2o.py @@ -3,6 +3,9 @@ import h2o class TestH2o(unittest.TestCase): + def tearDown(self): + h2o.cluster().shutdown(False) + def test_init_read(self): h2o.init() train = h2o.import_file("/input/tests/data/train.csv", destination_frame="train") From 8fcae6112175c2a1eef8f1c7246cb7b43fffadeb Mon Sep 17 00:00:00 2001 From: Wendy Kan Date: Mon, 19 Nov 2018 14:10:35 -0800 Subject: [PATCH 0452/1660] adding kaggle api --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 71f236c3..a8212c59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -475,6 +475,7 @@ RUN pip install flashtext && \ pip install PDPbox && \ pip install ggplot && \ pip install cesium && \ + pip install kaggle && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From 40499323b7c83503ccfe849120e8cf1e1cbf78bd Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 20 Nov 2018 09:38:46 -0800 Subject: [PATCH 0453/1660] Update ImageMagick binary url --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a8212c59..abda466c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN pip install --upgrade seaborn python-dateutil dask && \ libpaper-utils libpaper1 libpixman-1-0 libpng16-16 librsvg2-2 librsvg2-common libthai-data libthai0 libtiff5 libwmf0.2-7 \ libxcb-render0 libxcb-shm0 netpbm poppler-data p7zip-full && \ cd /usr/local/src && \ - wget http://transloadit.imagemagick.org/download/ImageMagick.tar.gz && \ + wget https://imagemagick.org/download/ImageMagick.tar.gz && \ tar xzf ImageMagick.tar.gz && cd `ls -d ImageMagick-*` && pwd && ls -al && ./configure && \ make -j $(nproc) && make install && \ # clean up ImageMagick source files From 922c5a9030748ebab4da23d7dbdd4cb89a7b86a1 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Wed, 21 Nov 2018 09:06:11 -0800 Subject: [PATCH 0454/1660] Remove the kaggle api package This breaks the twosigma competition because the code for both lives in a python package called `kaggle`. The code for the twosigma competition under `kaggle.competitions` should probably be move. Additionally, the kaggle api client package requires an API secret key. We don't offer a way to manage secrets for now and we don't want to encourage people adding their secret in the code (easy to leak if ever made public). --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index abda466c..0aff17f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -475,7 +475,6 @@ RUN pip install flashtext && \ pip install PDPbox && \ pip install ggplot && \ pip install cesium && \ - pip install kaggle && \ ##### ^^^^ Add new contributions above here ^^^^ ##### # clean up pip cache rm -rf /root/.cache/pip/* From 5976f3c19b36060790863af361b769786643a72c Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 18 Sep 2018 22:22:43 +0000 Subject: [PATCH 0455/1660] Create process to prebuild tensorflow wheels --- tensorflow-whl/CHANGELOG.md | 1 + tensorflow-whl/Dockerfile | 100 ++++++++++++++++++++++++++++++++++++ tensorflow-whl/README.md | 22 ++++++++ tensorflow-whl/build | 11 ++++ tensorflow-whl/push | 25 +++++++++ 5 files changed, 159 insertions(+) create mode 100644 tensorflow-whl/CHANGELOG.md create mode 100644 tensorflow-whl/Dockerfile create mode 100644 tensorflow-whl/README.md create mode 100755 tensorflow-whl/build create mode 100755 tensorflow-whl/push diff --git a/tensorflow-whl/CHANGELOG.md b/tensorflow-whl/CHANGELOG.md new file mode 100644 index 00000000..52ab3b13 --- /dev/null +++ b/tensorflow-whl/CHANGELOG.md @@ -0,0 +1 @@ +1.11.0-py36: Tensorflow 1.11.0 wheels built with python 3.6 diff --git a/tensorflow-whl/Dockerfile b/tensorflow-whl/Dockerfile new file mode 100644 index 00000000..db5bc8df --- /dev/null +++ b/tensorflow-whl/Dockerfile @@ -0,0 +1,100 @@ +FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 AS nvidia +FROM continuumio/anaconda3:5.0.1 + +# Avoid interactive configuration prompts/dialogs during apt-get. +ENV DEBIAN_FRONTEND=noninteractive + +# This is necessary to for apt to access HTTPS sources +RUN apt-get update && \ + apt-get install apt-transport-https + +# Cuda support +COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ +COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ +COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg + +ENV CUDA_VERSION=9.1.85 +ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 +LABEL com.nvidia.volumes.needed="nvidia_driver" +LABEL com.nvidia.cuda.version="${CUDA_VERSION}" +ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +# The stub is useful to us both for built-time linking and run-time linking, on CPU-only systems. +# When intended to be used with actual GPUs, make sure to (besides providing access to the host +# CUDA user libraries, either manually or through the use of nvidia-docker) exclude them. One +# convenient way to do so is to obscure its contents by a bind mount: +# docker run .... -v /non-existing-directory:/usr/local/cuda/lib64/stubs:ro ... +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs" +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility +ENV NVIDIA_REQUIRE_CUDA="cuda>=9.0" +RUN apt-get update && apt-get install -y --no-install-recommends \ + cuda-cudart-$CUDA_PKG_VERSION \ + cuda-libraries-$CUDA_PKG_VERSION \ + cuda-libraries-dev-$CUDA_PKG_VERSION \ + cuda-nvml-dev-$CUDA_PKG_VERSION \ + cuda-minimal-build-$CUDA_PKG_VERSION \ + cuda-command-line-tools-$CUDA_PKG_VERSION \ + libcudnn7=7.0.5.15-1+cuda9.1 \ + libcudnn7-dev=7.0.5.15-1+cuda9.1 \ + libnccl2=2.2.12-1+cuda9.1 \ + libnccl-dev=2.2.12-1+cuda9.1 && \ + ln -s /usr/local/cuda-9.1 /usr/local/cuda && \ + ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ + rm -rf /var/lib/apt/lists/* + +# Install bazel +RUN apt-get update && apt-get install -y python-software-properties zip && \ + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list && \ + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 C857C906 2B90D010 && \ + apt-get update && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get install -y oracle-java8-installer && \ + echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \ + curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \ + apt-get update && apt-get install -y bazel && \ + apt-get upgrade -y bazel + +# Tensorflow doesn't support python 3.7 yet. See https://github.com/tensorflow/tensorflow/issues/20517 +RUN conda install -y python=3.6.6 && \ + # Another fix for TF 1.10 https://github.com/tensorflow/tensorflow/issues/21518 + pip install keras_applications==1.0.4 --no-deps && \ + pip install keras_preprocessing==1.0.2 --no-deps + +# Fetch tensorflow +RUN cd /usr/local/src && \ + git clone https://github.com/tensorflow/tensorflow && \ + cd tensorflow && \ + git checkout r1.11 + +# Create a tensorflow wheel for CPU +RUN cd /usr/local/src/tensorflow && \ + cat /dev/null | ./configure && \ + bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_cpu && \ + bazel clean + +# Create a tensorflow wheel for GPU/cuda +ENV TF_NEED_CUDA=1 +ENV TF_CUDA_VERSION=9.1 +ENV TF_CUDA_COMPUTE_CAPABILITIES=3.7,6.0 +ENV TF_CUDNN_VERSION=7 +ENV TF_NCCL_VERSION=2 +ENV NCCL_INSTALL_PATH=/usr/ + +RUN cd /usr/local/src/tensorflow && \ + # TF_NCCL_INSTALL_PATH is used for both libnccl.so.2 and libnccl.h. Make sure they are both accessible from the same directory. + ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/lib/ && \ + cat /dev/null | ./configure && \ + echo "/usr/local/cuda-${TF_CUDA_VERSION}/targets/x86_64-linux/lib/stubs" > /etc/ld.so.conf.d/cuda-stubs.conf && ldconfig && \ + bazel build --config=opt \ + --config=cuda \ + --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ + //tensorflow/tools/pip_package:build_pip_package && \ + rm /etc/ld.so.conf.d/cuda-stubs.conf && ldconfig && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_gpu && \ + bazel clean + +# Print out the built .whl files +RUN ls -R /tmp/tensorflow* diff --git a/tensorflow-whl/README.md b/tensorflow-whl/README.md new file mode 100644 index 00000000..ba0e0003 --- /dev/null +++ b/tensorflow-whl/README.md @@ -0,0 +1,22 @@ +# Build new Tensorflow wheels + +``` +./build +``` + +# Push the new wheels + +1. Add an entry in the [CHANGELOG](CHANGELOG.md) with an appropriate `LABEL`. +2. Push the new image using the `LABEL` you picked above. + + ``` + ./push LABEL + ``` + +# Use the new wheels + +Update the line below in the [CPU Dockerfile](../Dockerfile) and the [GPU Dockerfile](../gpu.Dockerfile) to use the new `LABEL`. + +``` +FROM gcr.io/kaggle-images/python-tensorflow-whl: as tensorflow_whl +``` diff --git a/tensorflow-whl/build b/tensorflow-whl/build new file mode 100755 index 00000000..b5678526 --- /dev/null +++ b/tensorflow-whl/build @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +set -x + +# Default behavior is to do everything from scratch. +# The --use-cache option is useful if you're iterating on a broken build. +if [[ "$1" == "--use-cache" ]]; then + docker build --rm -t kaggle/python-tensorflow-whl . +else + docker build --pull --rm --no-cache -t kaggle/python-tensorflow-whl . +fi diff --git a/tensorflow-whl/push b/tensorflow-whl/push new file mode 100755 index 00000000..a27de8a1 --- /dev/null +++ b/tensorflow-whl/push @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Push a newly-built image with the given label to gcr.io and DockerHub. +# +# Usage: +# ./push LABEL +# +# Description: +# LABEL: Image label. See CHANGELOG.md +# +set -e +set -x + +SOURCE_IMAGE="kaggle/python-tensorflow-whl" +TARGET_IMAGE="gcr.io/kaggle-images/python-tensorflow-whl" + +LABEL=$1 + +if [[ -z "$LABEL" ]]; then + echo "You must provide a label for the image" + exit 1 +fi + +docker tag $SOURCE_IMAGE:latest $TARGET_IMAGE:$LABEL +gcloud docker -- push $TARGET_IMAGE:$LABEL From 377387914cd323bda8163c1800e81c627d1474e0 Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 27 Nov 2018 02:39:59 +0000 Subject: [PATCH 0456/1660] Add TODOs and improve build/push script --- gpu.Dockerfile | 2 ++ tensorflow-whl/Dockerfile | 8 +++++-- tensorflow-whl/build | 47 ++++++++++++++++++++++++++++++++------- tensorflow-whl/push | 45 ++++++++++++++++++++++++++++--------- 4 files changed, 81 insertions(+), 21 deletions(-) diff --git a/gpu.Dockerfile b/gpu.Dockerfile index 247c7dd4..164b0d25 100644 --- a/gpu.Dockerfile +++ b/gpu.Dockerfile @@ -7,6 +7,8 @@ COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg +# Ensure the cuda libraries are compatible with the custom Tensorflow wheels. +# TODO(b/120050292): Use templating to keep in sync. ENV CUDA_VERSION=9.1.85 ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 LABEL com.nvidia.volumes.needed="nvidia_driver" diff --git a/tensorflow-whl/Dockerfile b/tensorflow-whl/Dockerfile index db5bc8df..5b7b421b 100644 --- a/tensorflow-whl/Dockerfile +++ b/tensorflow-whl/Dockerfile @@ -13,6 +13,8 @@ COPY --from=nvidia /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/ COPY --from=nvidia /etc/apt/trusted.gpg /etc/apt/trusted.gpg.d/cuda.gpg +# Ensure the cuda libraries are compatible with the GPU image. +# TODO(b/120050292): Use templating to keep in sync. ENV CUDA_VERSION=9.1.85 ENV CUDA_PKG_VERSION=9-1=$CUDA_VERSION-1 LABEL com.nvidia.volumes.needed="nvidia_driver" @@ -28,14 +30,16 @@ ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility ENV NVIDIA_REQUIRE_CUDA="cuda>=9.0" RUN apt-get update && apt-get install -y --no-install-recommends \ + cuda-cupti-$CUDA_PKG_VERSION \ cuda-cudart-$CUDA_PKG_VERSION \ + cuda-cudart-dev-$CUDA_PKG_VERSION \ cuda-libraries-$CUDA_PKG_VERSION \ cuda-libraries-dev-$CUDA_PKG_VERSION \ cuda-nvml-dev-$CUDA_PKG_VERSION \ cuda-minimal-build-$CUDA_PKG_VERSION \ cuda-command-line-tools-$CUDA_PKG_VERSION \ - libcudnn7=7.0.5.15-1+cuda9.1 \ - libcudnn7-dev=7.0.5.15-1+cuda9.1 \ + libcudnn7=7.2.1.38-1+cuda9.0 \ + libcudnn7-dev=7.2.1.38-1+cuda9.0 \ libnccl2=2.2.12-1+cuda9.1 \ libnccl-dev=2.2.12-1+cuda9.1 && \ ln -s /usr/local/cuda-9.1 /usr/local/cuda && \ diff --git a/tensorflow-whl/build b/tensorflow-whl/build index b5678526..dc705f17 100755 --- a/tensorflow-whl/build +++ b/tensorflow-whl/build @@ -1,11 +1,42 @@ #!/bin/bash set -e -set -x -# Default behavior is to do everything from scratch. -# The --use-cache option is useful if you're iterating on a broken build. -if [[ "$1" == "--use-cache" ]]; then - docker build --rm -t kaggle/python-tensorflow-whl . -else - docker build --pull --rm --no-cache -t kaggle/python-tensorflow-whl . -fi +usage() { +cat << EOF +Usage: $0 [OPTIONS] +Build new Tensorflow wheels for use in the Kaggle Docker Python base images. + +Options: + -c, --use-cache Use layer cache when building a new image. +EOF +} + +CACHE_FLAG="--no-cache" +IMAGE_TAG="kaggle/python-build" + +while :; do + case "$1" in + -h|--help) + usage + exit + ;; + -c|--use-cache) + CACHE_FLAG="" + ;; + -?*) + usage + printf 'ERROR: Unknown option: %s\n' "$1" >&2 + exit + ;; + *) + break + esac + + shift +done + +readonly CACHE_FLAG +readonly IMAGE_TAG + +set -x +docker build --rm --pull $CACHE_FLAG -t "$IMAGE_TAG" . diff --git a/tensorflow-whl/push b/tensorflow-whl/push index a27de8a1..dfb64ec8 100755 --- a/tensorflow-whl/push +++ b/tensorflow-whl/push @@ -1,19 +1,37 @@ #!/bin/bash -# -# Push a newly-built image with the given label to gcr.io and DockerHub. -# -# Usage: -# ./push LABEL -# -# Description: -# LABEL: Image label. See CHANGELOG.md -# set -e + +usage() { +cat << EOF +Usage: $0 [LABEL] +Push a newly-built image with the given LABEL to gcr.io and DockerHub. +See CHANGELOG.md file for LABEL naming convention. +EOF +} + set -x SOURCE_IMAGE="kaggle/python-tensorflow-whl" TARGET_IMAGE="gcr.io/kaggle-images/python-tensorflow-whl" +while :; do + case "$1" in + -h|--help) + usage + exit + ;; + -?*) + usage + printf 'ERROR: Unknown option: %s\n' "$1" >&2 + exit + ;; + *) + break + esac + + shift +done + LABEL=$1 if [[ -z "$LABEL" ]]; then @@ -21,5 +39,10 @@ if [[ -z "$LABEL" ]]; then exit 1 fi -docker tag $SOURCE_IMAGE:latest $TARGET_IMAGE:$LABEL -gcloud docker -- push $TARGET_IMAGE:$LABEL +readonly SOURCE_IMAGE +readonly TARGET_IMAGE +readonly LABEL + +set -x +docker tag "$SOURCE_IMAGE:latest" "$TARGET_IMAGE:$LABEL" +gcloud docker -- push "$TARGET_IMAGE:$LABEL" From 52b051e44b53a816f62080ed17228d699787b1fb Mon Sep 17 00:00:00 2001 From: Vincent Roseberry Date: Tue, 27 Nov 2018 02:45:17 +0000 Subject: [PATCH 0457/1660] fix typo --- tensorflow-whl/README.md | 2 +- tensorflow-whl/build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow-whl/README.md b/tensorflow-whl/README.md index ba0e0003..509591a2 100644 --- a/tensorflow-whl/README.md +++ b/tensorflow-whl/README.md @@ -18,5 +18,5 @@ Update the line below in the [CPU Dockerfile](../Dockerfile) and the [GPU Dockerfile](../gpu.Dockerfile) to use the new `LABEL`. ``` -FROM gcr.io/kaggle-images/python-tensorflow-whl: as tensorflow_whl +FROM gcr.io/kaggle-images/python-tensorflow-whl: