Newer
Older
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<img width=\"800px\" src=\"../fidle/img/header.svg\"></img>\n",
"\n",
"# <!-- TITLE --> [ACTF1] - Activation functions\n",
"<!-- DESC --> Some activation functions, with their derivatives.\n",
"<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->\n",
"\n",
"## Objectives :\n",
" - View the main activation functions \n",
"\n",
"Les fonctions d'activation dans Keras : \n",
"https://www.tensorflow.org/api_docs/python/tf/keras/activations\n",
"\n",
"## What we're going to do :\n",
"\n",
" - Juste visualiser les principales fonctions d'activation\n"
]
},
{
"cell_type": "code",
"source": [
"import numpy as np\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"\n",
"import math\n",
"from math import erfc, sqrt, exp\n",
"from math import pi as PI\n",
"from math import e as E\n",
"import sys\n",
"\n",
"run_id, run_dir, datasets_dir = fidle.init('ACTF1')"
]
},
{
"cell_type": "code",
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
"metadata": {},
"outputs": [],
"source": [
"SELU_A = -sqrt(2/PI)/(erfc(1/sqrt(2))*exp(1/2)-1)\n",
"SELU_L = (1-erfc(1/sqrt(2))*sqrt(E))*sqrt(2*PI) / (2*erfc(sqrt(2))*E*E+PI*erfc(1/sqrt(2))**2*E-2*(2+PI)*erfc(1/sqrt(2))*sqrt(E)+PI+2)**0.5\n",
"\n",
"\n",
"def heaviside(z):\n",
" return np.where(z<0,0,1)\n",
"\n",
"def sign(z):\n",
" return np.where(z<0,-1,1)\n",
"# return np.sign(z)\n",
"\n",
"def sigmoid(z):\n",
" return 1 / (1 + np.exp(-z))\n",
"\n",
"def tanh(z):\n",
" return np.tanh(z)\n",
"\n",
"def relu(z):\n",
" return np.maximum(0, z)\n",
"\n",
"def leaky_relu(z,a=0.05):\n",
" return np.maximum(a*z, z)\n",
"\n",
"def elu(z,a=1):\n",
" #y=z.copy()\n",
" y=a*(np.exp(z)-1)\n",
" y[z>0]=z[z>0]\n",
" return y\n",
"\n",
"def selu(z):\n",
" return SELU_L*elu(z,a=SELU_A)\n",
"\n",
"def derivative(f, z, eps=0.000001):\n",
" return (f(z + eps) - f(z - eps))/(2 * eps)"
]
},
{
"cell_type": "code",
"metadata": {
"jupyter": {
"source_hidden": true
"source": [
"pw=5\n",
"ph=5\n",
"\n",
"z = np.linspace(-5, 5, 200)\n",
"\n",
"\n",
"# ------ Heaviside\n",
"#\n",
"fig, ax = plt.subplots(1, 1)\n",
"fig.set_size_inches(pw,ph)\n",
"ax.set_xlim(-5, 5)\n",
"ax.set_ylim(-2, 2)\n",
"ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.plot(0, 0, \"rx\", markersize=10)\n",
"ax.plot(z, heaviside(z), linestyle='-', label=\"Heaviside\")\n",
"ax.plot(z, derivative(heaviside, z), linewidth=3, alpha=0.6, label=\"dHeaviside/dx\")\n",
"# ax.plot(z, sign(z), label=\"Heaviside\")\n",
"ax.set_title(\"Heaviside\")\n",
"fidle.scrawler.save_fig('Heaviside')\n",
"plt.show()\n",
"\n",
"\n",
"# ----- Logit/Sigmoid\n",
"#\n",
"fig, ax = plt.subplots(1, 1)\n",
"fig.set_size_inches(pw,ph)\n",
"ax.set_xlim(-5, 5)\n",
"ax.set_ylim(-2, 2)\n",
"ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.plot(z, sigmoid(z), label=\"Sigmoid\")\n",
"ax.plot(z, derivative(sigmoid, z), linewidth=3, alpha=0.6, label=\"dSigmoid/dx\")\n",
"plt.show()\n",
"\n",
"# ----- Tanh\n",
"#\n",
"fig, ax = plt.subplots(1, 1)\n",
"fig.set_size_inches(pw,ph)\n",
"ax.set_xlim(-5, 5)\n",
"ax.set_ylim(-2, 2)\n",
"ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.plot(z, tanh(z), label=\"Tanh\")\n",
"ax.plot(z, derivative(tanh, z), linewidth=3, alpha=0.6, label=\"dTanh/dx\")\n",
"plt.show()\n",
"\n",
"# ----- Relu\n",
"#\n",
"fig, ax = plt.subplots(1, 1)\n",
"fig.set_size_inches(pw,ph)\n",
"ax.set_xlim(-5, 5)\n",
"ax.set_ylim(-2, 2)\n",
"ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.plot(z, relu(z), label=\"ReLU\")\n",
"ax.plot(z, derivative(relu, z), linewidth=3, alpha=0.6, label=\"dReLU/dx\")\n",
"plt.show()\n",
"\n",
"# ----- Leaky Relu\n",
"#\n",
"fig, ax = plt.subplots(1, 1)\n",
"fig.set_size_inches(pw,ph)\n",
"ax.set_xlim(-5, 5)\n",
"ax.set_ylim(-2, 2)\n",
"ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.plot(z, leaky_relu(z), label=\"Leaky ReLU\")\n",
"ax.plot(z, derivative( leaky_relu, z), linewidth=3, alpha=0.6, label=\"dLeakyReLU/dx\")\n",
"ax.set_title(\"Leaky ReLU (α=0.05)\")\n",
"fidle.scrawler.save_fig('LeakyReLU')\n",
"plt.show()\n",
"\n",
"# ----- Elu\n",
"#\n",
"fig, ax = plt.subplots(1, 1)\n",
"fig.set_size_inches(pw,ph)\n",
"ax.set_xlim(-5, 5)\n",
"ax.set_ylim(-2, 2)\n",
"ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.plot(z, elu(z), label=\"ReLU\")\n",
"ax.plot(z, derivative( elu, z), linewidth=3, alpha=0.6, label=\"dExpReLU/dx\")\n",
"ax.set_title(\"ELU (α=1)\")\n",
"plt.show()\n",
"\n",
"# ----- Selu\n",
"#\n",
"fig, ax = plt.subplots(1, 1)\n",
"fig.set_size_inches(pw,ph)\n",
"ax.set_xlim(-5, 5)\n",
"ax.set_ylim(-2, 2)\n",
"ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
"ax.plot(z, selu(z), label=\"SeLU\")\n",
"ax.plot(z, derivative( selu, z), linewidth=3, alpha=0.6, label=\"dSeLU/dx\")\n",
"ax.set_title(\"ELU (SELU)\")\n",
"plt.show()\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"---\n",
"<img width=\"80px\" src=\"../fidle/img/logo-paysage.svg\"></img>"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.2 ('fidle-env')",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.2"
},
"vscode": {
"interpreter": {
"hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
}
}
},
"nbformat": 4,
"nbformat_minor": 4
}