Skip to content
Snippets Groups Projects
Commit 32315257 authored by Jean-Luc Parouty's avatar Jean-Luc Parouty
Browse files

Activation-Functions correction

parent 997dbfdd
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
GTSRB/02-First-convolutions.ipynb
+ 23
− 265
View file @ 32315257
source diff could not be displayed: it is too large. Options to address this: view the blob.
Misc/Activation-Functions.ipynb
+ 13
− 11
View file @ 32315257
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img> <img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
# <!-- TITLE --> [ACTF1] - Activation functions # <!-- TITLE --> [ACTF1] - Activation functions
<!-- DESC --> Some activation functions, with their derivatives. <!-- DESC --> Some activation functions, with their derivatives.
<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) --> <!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
## Objectives : ## Objectives :
- View the main activation functions - View the main activation functions
Les fonctions d'activation dans Keras : Les fonctions d'activation dans Keras :
https://www.tensorflow.org/api_docs/python/tf/keras/activations https://www.tensorflow.org/api_docs/python/tf/keras/activations
## What we're going to do : ## What we're going to do :
- Juste visualiser les principales fonctions d'activation - Juste visualiser les principales fonctions d'activation
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
import numpy as np import numpy as np
import matplotlib import matplotlib
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import math import math
from math import erfc, sqrt, exp from math import erfc, sqrt, exp
from math import pi as PI from math import pi as PI
from math import e as E from math import e as E
import sys import sys
sys.path.append('..') sys.path.append('..')
import fidle.pwk as pwk import fidle.pwk as pwk
datasets_dir = pwk.init('ACTF1') datasets_dir = pwk.init('ACTF1')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
SELU_A = -sqrt(2/PI)/(erfc(1/sqrt(2))*exp(1/2)-1) SELU_A = -sqrt(2/PI)/(erfc(1/sqrt(2))*exp(1/2)-1)
SELU_L = (1-erfc(1/sqrt(2))*sqrt(E))*sqrt(2*PI) / (2*erfc(sqrt(2))*E*E+PI*erfc(1/sqrt(2))**2*E-2*(2+PI)*erfc(1/sqrt(2))*sqrt(E)+PI+2)**0.5 SELU_L = (1-erfc(1/sqrt(2))*sqrt(E))*sqrt(2*PI) / (2*erfc(sqrt(2))*E*E+PI*erfc(1/sqrt(2))**2*E-2*(2+PI)*erfc(1/sqrt(2))*sqrt(E)+PI+2)**0.5
def heaviside(z): def heaviside(z):
return np.where(z<0,0,1) return np.where(z<0,0,1)
def sign(z): def sign(z):
return np.where(z<0,-1,1) return np.where(z<0,-1,1)
# return np.sign(z) # return np.sign(z)
def sigmoid(z): def sigmoid(z):
return 1 / (1 + np.exp(-z)) return 1 / (1 + np.exp(-z))
def tanh(z): def tanh(z):
return np.tanh(z) return np.tanh(z)
def relu(z): def relu(z):
return np.maximum(0, z) return np.maximum(0, z)
def leaky_relu(z,a=0.05): def leaky_relu(z,a=0.05):
return np.maximum(a*z, z) return np.maximum(a*z, z)
def elu(z,a=1): def elu(z,a=1):
#y=z.copy() #y=z.copy()
y=a*(np.exp(z)-1) y=a*(np.exp(z)-1)
y[z>0]=z[z>0] y[z>0]=z[z>0]
return y return y
def selu(z): def selu(z):
return SELU_L*elu(z,a=SELU_A) return SELU_L*elu(z,a=SELU_A)
def derivative(f, z, eps=0.000001): def derivative(f, z, eps=0.000001):
return (f(z + eps) - f(z - eps))/(2 * eps) return (f(z + eps) - f(z - eps))/(2 * eps)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
pw=5 pw=5
ph=5 ph=5
z = np.linspace(-5, 5, 200) z = np.linspace(-5, 5, 200)
# ------ Heaviside # ------ Heaviside
# #
fig, ax = plt.subplots(1, 1) fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph) fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5) ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2) ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray') ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray') ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(0, 0, "rx", markersize=10) ax.plot(0, 0, "rx", markersize=10)
ax.plot(z, sign(z), linewidth=2, linestyle='-', color='steelblue', label="Heaviside") ax.plot(z, heaviside(z), linestyle='-', label="Heaviside")
ax.plot(z, derivative(np.sign, z), label="dHeaviside/dx") ax.plot(z, derivative(heaviside, z), linewidth=3, alpha=0.6, label="dHeaviside/dx")
ax.plot(z, sign(z), label="Heaviside") # ax.plot(z, sign(z), label="Heaviside")
ax.set_title("Heaviside") ax.set_title("Heaviside")
pwk.save_fig('Heaviside') pwk.save_fig('Heaviside')
plt.show() plt.show()
# ----- Logit/Sigmoid # ----- Logit/Sigmoid
# #
fig, ax = plt.subplots(1, 1) fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph) fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5) ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2) ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray') ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray') ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, sigmoid(z), label="Sigmoid") ax.plot(z, sigmoid(z), label="Sigmoid")
ax.plot(z, derivative(sigmoid, z), label="dSigmoid/dx") ax.plot(z, derivative(sigmoid, z), linewidth=3, alpha=0.6, label="dSigmoid/dx")
ax.set_title("Logit") ax.set_title("Logit")
pwk.save_fig('Logit') pwk.save_fig('Logit')
plt.show() plt.show()
# ----- Tanh # ----- Tanh
# #
fig, ax = plt.subplots(1, 1) fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph) fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5) ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2) ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray') ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray') ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, tanh(z), label="Tanh") ax.plot(z, tanh(z), label="Tanh")
ax.plot(z, derivative(tanh, z), label="dTanh/dx") ax.plot(z, derivative(tanh, z), linewidth=3, alpha=0.6, label="dTanh/dx")
ax.set_title("Tanh") ax.set_title("Tanh")
pwk.save_fig('Tanh') pwk.save_fig('Tanh')
plt.show() plt.show()
# ----- Relu # ----- Relu
# #
fig, ax = plt.subplots(1, 1) fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph) fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5) ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2) ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray') ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray') ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, relu(z), label="ReLU") ax.plot(z, relu(z), label="ReLU")
ax.plot(z, derivative(relu, z), label="dReLU/dx") ax.plot(z, derivative(relu, z), linewidth=3, alpha=0.6, label="dReLU/dx")
ax.set_title("ReLU") ax.set_title("ReLU")
pwk.save_fig('ReLU') pwk.save_fig('ReLU')
plt.show() plt.show()
# ----- Leaky Relu # ----- Leaky Relu
# #
fig, ax = plt.subplots(1, 1) fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph) fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5) ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2) ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray') ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray') ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, leaky_relu(z), label="Leaky ReLU") ax.plot(z, leaky_relu(z), label="Leaky ReLU")
ax.plot(z, derivative( leaky_relu, z), label="dLeakyReLU/dx") ax.plot(z, derivative( leaky_relu, z), linewidth=3, alpha=0.6, label="dLeakyReLU/dx")
ax.set_title("Leaky ReLU (α=0.05)") ax.set_title("Leaky ReLU (α=0.05)")
pwk.save_fig('LeakyReLU') pwk.save_fig('LeakyReLU')
plt.show() plt.show()
# ----- Elu # ----- Elu
# #
fig, ax = plt.subplots(1, 1) fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph) fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5) ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2) ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray') ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray') ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, elu(z), label="ReLU") ax.plot(z, elu(z), label="ReLU")
ax.plot(z, derivative( elu, z), label="dExpReLU/dx") ax.plot(z, derivative( elu, z), linewidth=3, alpha=0.6, label="dExpReLU/dx")
ax.set_title("ELU (α=1)") ax.set_title("ELU (α=1)")
pwk.save_fig('ELU') pwk.save_fig('ELU')
plt.show() plt.show()
# ----- Selu # ----- Selu
# #
fig, ax = plt.subplots(1, 1) fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph) fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5) ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2) ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray') ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray') ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, selu(z), label="SeLU") ax.plot(z, selu(z), label="SeLU")
ax.plot(z, derivative( selu, z), label="dSeLU/dx") ax.plot(z, derivative( selu, z), linewidth=3, alpha=0.6, label="dSeLU/dx")
ax.set_title("ELU (SELU)") ax.set_title("ELU (SELU)")
pwk.save_fig('SeLU') pwk.save_fig('SeLU')
plt.show() plt.show()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
--- ---
<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img> <img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
......
fidle/batch-tests-slurm.sh
+ 1
− 1
View file @ 32315257
...@@ -32,7 +32,7 @@ NOTEBOOK_DIR="$WORK/fidle/fidle" ...@@ -32,7 +32,7 @@ NOTEBOOK_DIR="$WORK/fidle/fidle"
export FIDLE_OVERRIDE_RUNCI_profile_name='./ci/smart_gpu.yml' export FIDLE_OVERRIDE_RUNCI_profile_name='./ci/smart_gpu.yml'
export FIDLE_OVERRIDE_RUNCI_reset='False' export FIDLE_OVERRIDE_RUNCI_reset='False'
export FIDLE_OVERRIDE_RUNCI_filter='Nb_GTSRB.*|Nb_AE.*' export FIDLE_OVERRIDE_RUNCI_filter='Nb_GTSRB.*'
NOTEBOOK_SRC1="02-running-ci-tests.ipynb" NOTEBOOK_SRC1="02-running-ci-tests.ipynb"
......
fidle/ci/smart_gpu.yml
+ 1
− 1
View file @ 32315257
...@@ -132,7 +132,7 @@ Nb_GTSRB5_r1: ...@@ -132,7 +132,7 @@ Nb_GTSRB5_r1:
notebook_src: 05-Full-convolutions.ipynb notebook_src: 05-Full-convolutions.ipynb
notebook_tag: =1==done== notebook_tag: =1==done==
overrides: overrides:
run_dir: ./run/GTSRB5_done run_dir: default
enhanced_dir: '{datasets_dir}/GTSRB/enhanced' enhanced_dir: '{datasets_dir}/GTSRB/enhanced'
datasets: "['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']" datasets: "['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']"
models: "{'v1':'get_model_v1', 'v2':'get_model_v2', 'v3':'get_model_v3'}" models: "{'v1':'get_model_v1', 'v2':'get_model_v2', 'v3':'get_model_v3'}"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment