Skip to content
Snippets Groups Projects
Commit 32315257 authored by Jean-Luc Parouty's avatar Jean-Luc Parouty
Browse files

Activation-Functions correction

parent 997dbfdd
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
GTSRB/02-First-convolutions.ipynb
+ 23
− 265
View file @ 32315257
source diff could not be displayed: it is too large. Options to address this: view the blob.
Misc/Activation-Functions.ipynb
+ 13
− 11
View file @ 32315257
%% Cell type:markdown id: tags:
<img width="800px" src="../fidle/img/00-Fidle-header-01.svg"></img>
# <!-- TITLE --> [ACTF1] - Activation functions
<!-- DESC --> Some activation functions, with their derivatives.
<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->
## Objectives :
- View the main activation functions
Les fonctions d'activation dans Keras :
https://www.tensorflow.org/api_docs/python/tf/keras/activations
## What we're going to do :
- Juste visualiser les principales fonctions d'activation
%% Cell type:code id: tags:
```
``` python
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import math
from math import erfc, sqrt, exp
from math import pi as PI
from math import e as E
import sys
sys.path.append('..')
import fidle.pwk as pwk
datasets_dir = pwk.init('ACTF1')
```
%% Cell type:code id: tags:
```
``` python
SELU_A = -sqrt(2/PI)/(erfc(1/sqrt(2))*exp(1/2)-1)
SELU_L = (1-erfc(1/sqrt(2))*sqrt(E))*sqrt(2*PI) / (2*erfc(sqrt(2))*E*E+PI*erfc(1/sqrt(2))**2*E-2*(2+PI)*erfc(1/sqrt(2))*sqrt(E)+PI+2)**0.5
def heaviside(z):
return np.where(z<0,0,1)
def sign(z):
return np.where(z<0,-1,1)
# return np.sign(z)
def sigmoid(z):
return 1 / (1 + np.exp(-z))
def tanh(z):
return np.tanh(z)
def relu(z):
return np.maximum(0, z)
def leaky_relu(z,a=0.05):
return np.maximum(a*z, z)
def elu(z,a=1):
#y=z.copy()
y=a*(np.exp(z)-1)
y[z>0]=z[z>0]
return y
def selu(z):
return SELU_L*elu(z,a=SELU_A)
def derivative(f, z, eps=0.000001):
return (f(z + eps) - f(z - eps))/(2 * eps)
```
%% Cell type:code id: tags:
```
``` python
pw=5
ph=5
z = np.linspace(-5, 5, 200)
# ------ Heaviside
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(0, 0, "rx", markersize=10)
ax.plot(z, sign(z), linewidth=2, linestyle='-', color='steelblue', label="Heaviside")
ax.plot(z, derivative(np.sign, z), label="dHeaviside/dx")
ax.plot(z, sign(z), label="Heaviside")
ax.plot(z, heaviside(z), linestyle='-', label="Heaviside")
ax.plot(z, derivative(heaviside, z), linewidth=3, alpha=0.6, label="dHeaviside/dx")
# ax.plot(z, sign(z), label="Heaviside")
ax.set_title("Heaviside")
pwk.save_fig('Heaviside')
plt.show()
# ----- Logit/Sigmoid
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, sigmoid(z), label="Sigmoid")
ax.plot(z, derivative(sigmoid, z), label="dSigmoid/dx")
ax.plot(z, derivative(sigmoid, z), linewidth=3, alpha=0.6, label="dSigmoid/dx")
ax.set_title("Logit")
pwk.save_fig('Logit')
plt.show()
# ----- Tanh
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, tanh(z), label="Tanh")
ax.plot(z, derivative(tanh, z), label="dTanh/dx")
ax.plot(z, derivative(tanh, z), linewidth=3, alpha=0.6, label="dTanh/dx")
ax.set_title("Tanh")
pwk.save_fig('Tanh')
plt.show()
# ----- Relu
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, relu(z), label="ReLU")
ax.plot(z, derivative(relu, z), label="dReLU/dx")
ax.plot(z, derivative(relu, z), linewidth=3, alpha=0.6, label="dReLU/dx")
ax.set_title("ReLU")
pwk.save_fig('ReLU')
plt.show()
# ----- Leaky Relu
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, leaky_relu(z), label="Leaky ReLU")
ax.plot(z, derivative( leaky_relu, z), label="dLeakyReLU/dx")
ax.plot(z, derivative( leaky_relu, z), linewidth=3, alpha=0.6, label="dLeakyReLU/dx")
ax.set_title("Leaky ReLU (α=0.05)")
pwk.save_fig('LeakyReLU')
plt.show()
# ----- Elu
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, elu(z), label="ReLU")
ax.plot(z, derivative( elu, z), label="dExpReLU/dx")
ax.plot(z, derivative( elu, z), linewidth=3, alpha=0.6, label="dExpReLU/dx")
ax.set_title("ELU (α=1)")
pwk.save_fig('ELU')
plt.show()
# ----- Selu
#
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(pw,ph)
ax.set_xlim(-5, 5)
ax.set_ylim(-2, 2)
ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')
ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')
ax.plot(z, selu(z), label="SeLU")
ax.plot(z, derivative( selu, z), label="dSeLU/dx")
ax.plot(z, derivative( selu, z), linewidth=3, alpha=0.6, label="dSeLU/dx")
ax.set_title("ELU (SELU)")
pwk.save_fig('SeLU')
plt.show()
```
%% Cell type:markdown id: tags:
---
<img width="80px" src="../fidle/img/00-Fidle-logo-01.svg"></img>
......
fidle/batch-tests-slurm.sh
+ 1
− 1
View file @ 32315257
......@@ -32,7 +32,7 @@ NOTEBOOK_DIR="$WORK/fidle/fidle"
export FIDLE_OVERRIDE_RUNCI_profile_name='./ci/smart_gpu.yml'
export FIDLE_OVERRIDE_RUNCI_reset='False'
export FIDLE_OVERRIDE_RUNCI_filter='Nb_GTSRB.*|Nb_AE.*'
export FIDLE_OVERRIDE_RUNCI_filter='Nb_GTSRB.*'
NOTEBOOK_SRC1="02-running-ci-tests.ipynb"
......
fidle/ci/smart_gpu.yml
+ 1
− 1
View file @ 32315257
......@@ -132,7 +132,7 @@ Nb_GTSRB5_r1:
notebook_src: 05-Full-convolutions.ipynb
notebook_tag: =1==done==
overrides:
run_dir: ./run/GTSRB5_done
run_dir: default
enhanced_dir: '{datasets_dir}/GTSRB/enhanced'
datasets: "['set-24x24-L', 'set-24x24-RGB', 'set-48x48-L', 'set-48x48-RGB', 'set-24x24-L-LHE', 'set-24x24-RGB-HE', 'set-48x48-L-LHE', 'set-48x48-RGB-HE']"
models: "{'v1':'get_model_v1', 'v2':'get_model_v2', 'v3':'get_model_v3'}"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment