From 0e739e2a6cf6aa0b3d83a74329a611a38ff9da2e Mon Sep 17 00:00:00 2001
From: Jean-Luc Parouty <Jean-Luc.Parouty@simap.grenoble-inp.fr>
Date: Wed, 24 Jan 2024 09:59:39 +0100
Subject: [PATCH] Update docker config (v3.0.4)
---
Misc/99-Scratchbook.ipynb | 327 ------------------
README.ipynb | 16 +-
README.md | 4 +-
docker/Dockerfile | 10 +-
docker/jupyter_lab_config.py | 2 +-
..._packages_cpu.txt => requirements-cpu.txt} | 2 +
..._packages-old.txt => requirements-gpu.txt} | 43 +--
fidle/about.yml | 2 +-
fidle/ci/default.yml | 4 +-
9 files changed, 40 insertions(+), 370 deletions(-)
delete mode 100644 Misc/99-Scratchbook.ipynb
rename docker/{requirements_packages_cpu.txt => requirements-cpu.txt} (97%)
rename docker/{requirements_packages-old.txt => requirements-gpu.txt} (65%)
diff --git a/Misc/99-Scratchbook.ipynb b/Misc/99-Scratchbook.ipynb
deleted file mode 100644
index 9e61e8e..0000000
--- a/Misc/99-Scratchbook.ipynb
+++ /dev/null
@@ -1,327 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "id": "alpha-bahrain",
- "metadata": {},
- "source": [
- "<img width=\"800px\" src=\"../fidle/img/header.svg\"></img>\n",
- "\n",
- "# <!-- TITLE --> [SCRATCH1] - Scratchbook\n",
- "<!-- DESC --> A scratchbook for small examples\n",
- "<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->\n",
- "\n",
- "## Objectives :\n",
- " - Take a quick look at thousands of little things\n",
- "\n",
- "## Inside this scratchbook :\n",
- "\n",
- "[1 - LSTM Keras layer](#1---LSTM-Keras-layer) \n",
- "[2 - TimeseriesGenerator](#2---TimeseriesGenerator) \n",
- "[3 - Upsampling](#3---Upsampling) \n",
- "[4 - Reduce mean](#4---Reduce-mean) \n",
- "[5 - Gradient tape](#5---Gradient-tape)\n"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "accessory-church",
- "metadata": {},
- "source": [
- "# One init to rule them all"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "floppy-organic",
- "metadata": {},
- "outputs": [],
- "source": [
- "import tensorflow as tf\n",
- "from tensorflow import keras\n",
- "from tensorflow.keras.callbacks import TensorBoard\n",
- "from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator\n",
- "\n",
- "import numpy as np\n",
- "import math, random"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "danish-rebound",
- "metadata": {},
- "source": [
- "## 1 - LSTM Keras layer\n",
- "[Back to home](#)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "opposite-plasma",
- "metadata": {},
- "outputs": [],
- "source": [
- "inputs = tf.random.normal([32, 20, 8])\n",
- "\n",
- "lstm = tf.keras.layers.LSTM(16)\n",
- "output = lstm(inputs)\n",
- "\n",
- "print('Inputs shape is : ', inputs.shape)\n",
- "print('Output shape is : ', output.shape)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "forbidden-murray",
- "metadata": {},
- "outputs": [],
- "source": [
- "lstm = tf.keras.layers.LSTM(18, return_sequences=True, return_state=True)\n",
- "\n",
- "output, memory_state, carry_state = lstm(inputs)\n",
- "\n",
- "print('Output shape : ', output.shape)\n",
- "print('Memory state : ', memory_state.shape)\n",
- "print('Carry state : ', carry_state.shape)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "verified-fruit",
- "metadata": {},
- "outputs": [],
- "source": [
- "# --- See the last vector of the output\n",
- "output[-1,-1]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "homeless-library",
- "metadata": {},
- "outputs": [],
- "source": [
- "# ---- Memory state is the last output\n",
- "memory_state[-1]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "preliminary-psychiatry",
- "metadata": {},
- "outputs": [],
- "source": [
- "carry_state[-1]"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "41d326b2-376e-49d6-9429-07016d98dc09",
- "metadata": {},
- "source": [
- "## 2 - TimeseriesGenerator\n",
- "[Back to home](#)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "42276389-4ea6-42d1-93bc-6650062ef86a",
- "metadata": {},
- "outputs": [],
- "source": [
- "from keras.preprocessing.sequence import TimeseriesGenerator\n",
- "\n",
- "# ---- Define a dataset\n",
- "\n",
- "series = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])\n",
- "\n",
- "# ---- Generator\n",
- "\n",
- "generator = TimeseriesGenerator(series, series, length=5, batch_size=1)\n",
- "\n",
- "# ---- Samples\n",
- "\n",
- "nb_batch = len(generator)\n",
- "\n",
- "print(f'Number of batch : {nb_batch}\\n')\n",
- "for i in range(nb_batch):\n",
- " x, y = generator[i]\n",
- " print(f'#{i} : {x} => {y}')"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "67e3c888-aaa4-4166-90a1-cdb63920fd7d",
- "metadata": {},
- "source": [
- "## 3 - Upsampling\n",
- "[Back to home](#)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "20f12cf0-1fdb-4b53-92c6-d03b140e42d1",
- "metadata": {},
- "outputs": [],
- "source": [
- "x = np.array([1,2,3,4])\n",
- "x = x.reshape(2,2)\n",
- "print('\\nInitial : ', x.shape)\n",
- "print(x)\n",
- "\n",
- "x = x.reshape((1,2,2,1))\n",
- "print('\\nReshape as a batch of (2,2) vectors : ', x.shape)\n",
- "print(x)\n",
- "\n",
- "y = tf.keras.layers.UpSampling2D( size=(2, 2), interpolation=\"nearest\" )(x)\n",
- "\n",
- "y = np.array(y)\n",
- "print('\\ny shape : ',y.shape)\n",
- "\n",
- "y = y.reshape(4,4)\n",
- "print('\\n After a (4,4) reshape :')\n",
- "print(y)\n",
- "\n"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "e8fb1472",
- "metadata": {},
- "source": [
- "## 4 - Reduce mean\n",
- "[Back to home](#)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "09ac4e52-8953-41d9-b712-e6a83a9ae860",
- "metadata": {},
- "outputs": [],
- "source": [
- "import numpy as np\n",
- "import tensorflow as tf\n",
- "\n",
- "c = np.array([[3.,4], [5.,6], [7.,8]])\n",
- "print(np.mean(c,1))\n",
- "\n",
- "print(tf.reduce_mean(c,1))\n"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "72be9368",
- "metadata": {},
- "source": [
- "## 5 - Gradient tape\n",
- "[Back to home](#)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "13fcc722",
- "metadata": {},
- "outputs": [],
- "source": [
- "import numpy as np\n",
- "import tensorflow as tf\n",
- "\n",
- "# ---- My function f\n",
- "#\n",
- "def f(x):\n",
- " y = x**2 + 4*x - 5\n",
- " return y\n",
- "\n",
- "# ---- Examples :\n",
- "#\n",
- "print('f(1) is : ', f(1))\n",
- "print('f(2) is : ', f(2))\n",
- "\n",
- "# ---- With a tensor :\n",
- "#\n",
- "x = tf.Variable(2.0)\n",
- "\n",
- "print('f(x) is : ', f(x))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "6fab93ce",
- "metadata": {},
- "outputs": [],
- "source": [
- "# ---- Derivative function of f(x)\n",
- "#\n",
- "def g(x):\n",
- " y = 2*x + 4\n",
- " return y\n",
- "\n",
- "print('Derivative of f(x) for x=3 : ', g(3))\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "3f3f0c4c",
- "metadata": {},
- "outputs": [],
- "source": [
- "# ---- Derivative using tf\n",
- "#\n",
- "with tf.GradientTape() as tape:\n",
- " x = tf.Variable(3.0)\n",
- " y = f(x)\n",
- "\n",
- "dy = tape.gradient(y, x) # dy/dx\n",
- "\n",
- "print(dy)\n"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "43f9a625",
- "metadata": {},
- "source": [
- "---\n",
- "<img width=\"80px\" src=\"../fidle/img/logo-paysage.svg\"></img>"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3.9.2 ('fidle-env')",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.9.2"
- },
- "vscode": {
- "interpreter": {
- "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/README.ipynb b/README.ipynb
index 9ba07a4..71714c4 100644
--- a/README.ipynb
+++ b/README.ipynb
@@ -3,13 +3,13 @@
{
"cell_type": "code",
"execution_count": 1,
- "id": "b1e32492",
+ "id": "e6fc2701",
"metadata": {
"execution": {
- "iopub.execute_input": "2024-01-23T11:51:57.867131Z",
- "iopub.status.busy": "2024-01-23T11:51:57.866328Z",
- "iopub.status.idle": "2024-01-23T11:51:57.876478Z",
- "shell.execute_reply": "2024-01-23T11:51:57.875658Z"
+ "iopub.execute_input": "2024-01-24T08:27:28.475895Z",
+ "iopub.status.busy": "2024-01-24T08:27:28.475122Z",
+ "iopub.status.idle": "2024-01-24T08:27:28.485786Z",
+ "shell.execute_reply": "2024-01-24T08:27:28.484875Z"
},
"jupyter": {
"source_hidden": true
@@ -68,7 +68,7 @@
"## Jupyter notebooks\n",
"\n",
"<!-- TOC_BEGIN -->\n",
- "<!-- Automatically generated on : 23/01/24 12:51:56 -->\n",
+ "<!-- Automatically generated on : 24/01/24 09:27:27 -->\n",
"\n",
"### Linear and logistic regression\n",
"- **[LINR1](LinearReg/01-Linear-Regression.ipynb)** - [Linear regression with direct resolution](LinearReg/01-Linear-Regression.ipynb) \n",
@@ -181,8 +181,6 @@
"PyTorch est l'un des principaux framework utilisé dans le Deep Learning\n",
"- **[TSB1](Misc/04-Using-Tensorboard.ipynb)** - [Tensorboard with/from Jupyter ](Misc/04-Using-Tensorboard.ipynb) \n",
"4 ways to use Tensorboard from the Jupyter environment\n",
- "- **[SCRATCH1](Misc/99-Scratchbook.ipynb)** - [Scratchbook](Misc/99-Scratchbook.ipynb) \n",
- "A scratchbook for small examples\n",
"<!-- TOC_END -->\n",
"\n",
"## Installation\n",
@@ -213,7 +211,7 @@
"from IPython.display import display,Markdown\n",
"display(Markdown(open('README.md', 'r').read()))\n",
"#\n",
- "# This README is visible under Jupiter Lab ;-)# Automatically generated on : 23/01/24 12:51:57"
+ "# This README is visible under Jupiter Lab ;-)# Automatically generated on : 24/01/24 09:27:27"
]
}
],
diff --git a/README.md b/README.md
index 615d42d..f7f6529 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ Have a look about **[How to get and install](https://fidle.cnrs.fr/installation)
## Jupyter notebooks
<!-- TOC_BEGIN -->
-<!-- Automatically generated on : 23/01/24 12:51:56 -->
+<!-- Automatically generated on : 24/01/24 09:27:27 -->
### Linear and logistic regression
- **[LINR1](LinearReg/01-Linear-Regression.ipynb)** - [Linear regression with direct resolution](LinearReg/01-Linear-Regression.ipynb)
@@ -160,8 +160,6 @@ pandas is another essential tool for the Scientific Python.
PyTorch est l'un des principaux framework utilisé dans le Deep Learning
- **[TSB1](Misc/04-Using-Tensorboard.ipynb)** - [Tensorboard with/from Jupyter ](Misc/04-Using-Tensorboard.ipynb)
4 ways to use Tensorboard from the Jupyter environment
-- **[SCRATCH1](Misc/99-Scratchbook.ipynb)** - [Scratchbook](Misc/99-Scratchbook.ipynb)
-A scratchbook for small examples
<!-- TOC_END -->
## Installation
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 86588d7..67005e7 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -7,7 +7,7 @@ FROM ${docker_image_base}
LABEL maintainer1=soraya.arias@inria.fr maintainer2=jean-luc.parouty@simap.grenoble-inp.fr
ARG ARCH_VERSION=cpu
-ARG FIDLE_VERSION=3.0.3
+ARG BRANCH=pre-master
# Ensure a sane environment
ENV TZ=Europe/Paris LANG=C.UTF-8 LC_ALL=C.UTF-8 DEBIAN_FRONTEND=noninteractive
@@ -22,11 +22,11 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone &
rm -fr /var/lib/apt/lists/*
# copy Python requirement packages list in docker image
-COPY requirements_packages_${ARCH_VERSION}.txt /root/requirements_packages_${ARCH_VERSION}.txt
+COPY requirements-${ARCH_VERSION}.txt /root/requirements-${ARCH_VERSION}.txt
# Update Python tools and install requirements packages for Fidle
RUN python3 -m pip install --upgrade pip && \
- pip3 install --no-cache-dir --upgrade -r /root/requirements_packages_${ARCH_VERSION}.txt
+ pip3 install --no-cache-dir --upgrade -r /root/requirements-${ARCH_VERSION}.txt
# Install tensorboard & update jupyter
RUN pip3 install --no-cache-dir --upgrade tensorboard tensorboardX jupyter ipywidgets
@@ -44,8 +44,8 @@ RUN mkdir /data && \
# Get Fidle notebooks and create link
RUN mkdir /notebooks/ && \
- fid install_notebooks --notebooks fidle-pre-master --quiet --install_dir /notebooks && \
- ln -s /notebooks/fidle-pre-master-${FIDLE_VERSION} /notebooks/fidle-master
+ fid install_notebooks --notebooks fidle-${BRANCH} --quiet --install_dir /notebooks && \
+ ln -s $(ls -1td /notebooks/* | head -1) /notebooks/last
# Add Jupyter configuration (no browser, listen all interfaces, ...)
COPY jupyter_lab_config.py /root/.jupyter/jupyter_lab_config.py
diff --git a/docker/jupyter_lab_config.py b/docker/jupyter_lab_config.py
index dae4a64..8c89001 100644
--- a/docker/jupyter_lab_config.py
+++ b/docker/jupyter_lab_config.py
@@ -907,7 +907,7 @@ import os
#fidle_master_version = os.environ.get('FIDLE_MASTER_VERSION')
-c.ServerApp.root_dir = f'/notebooks/fidle-master'
+c.ServerApp.root_dir = f'/notebooks/last'
## The session manager class to use.
# Default: 'jupyter_server.services.sessions.sessionmanager.SessionManager'
diff --git a/docker/requirements_packages_cpu.txt b/docker/requirements-cpu.txt
similarity index 97%
rename from docker/requirements_packages_cpu.txt
rename to docker/requirements-cpu.txt
index e5da40f..86fad23 100644
--- a/docker/requirements_packages_cpu.txt
+++ b/docker/requirements-cpu.txt
@@ -20,11 +20,13 @@
torchdata
lightning
keras
+ transformers
numpy
Scikit-image
Scikit-learn
Matplotlib
plotly
+ seaborn
barviz
pyarrow
Pandas
diff --git a/docker/requirements_packages-old.txt b/docker/requirements-gpu.txt
similarity index 65%
rename from docker/requirements_packages-old.txt
rename to docker/requirements-gpu.txt
index 98399d3..1580817 100644
--- a/docker/requirements_packages-old.txt
+++ b/docker/requirements-gpu.txt
@@ -10,25 +10,26 @@
#
# To install your Fidle env, see https://fidle.cnrs.fr/installation
#
-# OLD version (Keras 2) - To be considered obsolete
+# Keras 3 / PyTorch version (Python 3.9.2)
-torch
-torchvision
-torch-geometric
-torchtext
-torchdata
-portalocker
-lightning
--f https://download.pytorch.org/whl/cpu
-tensorflow_cpu>=2.7
-keras
-numpy
-pandas
-jupyterlab
-scikit-image
-scikit-learn
-matplotlib
-plotly
-barviz
-opencv-python
-fidle
\ No newline at end of file
+ torch
+ torchvision
+ torch-geometric
+ torchtext
+ torchdata
+ lightning
+ keras
+ transformers
+ numpy
+ Scikit-image
+ Scikit-learn
+ Matplotlib
+ plotly
+ seaborn
+ barviz
+ pyarrow
+ Pandas
+ Pandoc
+ pyyaml
+ Jupyterlab
+ fidle
diff --git a/fidle/about.yml b/fidle/about.yml
index 8c8ff99..158ee98 100644
--- a/fidle/about.yml
+++ b/fidle/about.yml
@@ -13,7 +13,7 @@
#
# This file describes the notebooks used by the Fidle training.
-version: 3.0.3
+version: 3.0.4
content: notebooks
name: Notebooks Fidle
description: All notebooks used by the Fidle training
diff --git a/fidle/ci/default.yml b/fidle/ci/default.yml
index faf3cd6..2aff81f 100644
--- a/fidle/ci/default.yml
+++ b/fidle/ci/default.yml
@@ -1,6 +1,6 @@
campain:
version: '1.0'
- description: Automatically generated ci profile (23/01/24 12:51:56)
+ description: Automatically generated ci profile (24/01/24 09:27:27)
directory: ./campains/default
existing_notebook: 'remove # remove|skip'
report_template: 'fidle # fidle|default'
@@ -220,5 +220,3 @@ PYTORCH1:
TSB1:
notebook: Misc/04-Using-Tensorboard.ipynb
overrides: ??
-SCRATCH1:
- notebook: Misc/99-Scratchbook.ipynb
--
GitLab