{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "<img width=\"800px\" src=\"../fidle/img/header.svg\"></img>\n", "\n", "# <!-- TITLE --> [ACTF1] - Activation functions\n", "<!-- DESC --> Some activation functions, with their derivatives.\n", "<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->\n", "\n", "## Objectives :\n", " - View the main activation functions \n", "\n", "Les fonctions d'activation dans Keras : \n", "https://www.tensorflow.org/api_docs/python/tf/keras/activations\n", "\n", "## What we're going to do :\n", "\n", " - Juste visualiser les principales fonctions d'activation\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", "\n", "import math\n", "from math import erfc, sqrt, exp\n", "from math import pi as PI\n", "from math import e as E\n", "import sys\n", "\n", "import fidle\n", "\n", "# Init Fidle environment\n", "run_id, run_dir, datasets_dir = fidle.init('ACTF1')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "SELU_A = -sqrt(2/PI)/(erfc(1/sqrt(2))*exp(1/2)-1)\n", "SELU_L = (1-erfc(1/sqrt(2))*sqrt(E))*sqrt(2*PI) / (2*erfc(sqrt(2))*E*E+PI*erfc(1/sqrt(2))**2*E-2*(2+PI)*erfc(1/sqrt(2))*sqrt(E)+PI+2)**0.5\n", "\n", "\n", "def heaviside(z):\n", " return np.where(z<0,0,1)\n", "\n", "def sign(z):\n", " return np.where(z<0,-1,1)\n", "# return np.sign(z)\n", "\n", "def sigmoid(z):\n", " return 1 / (1 + np.exp(-z))\n", "\n", "def tanh(z):\n", " return np.tanh(z)\n", "\n", "def relu(z):\n", " return np.maximum(0, z)\n", "\n", "def leaky_relu(z,a=0.05):\n", " return np.maximum(a*z, z)\n", "\n", "def elu(z,a=1):\n", " #y=z.copy()\n", " y=a*(np.exp(z)-1)\n", " y[z>0]=z[z>0]\n", " return y\n", "\n", "def selu(z):\n", " return SELU_L*elu(z,a=SELU_A)\n", "\n", "def derivative(f, z, eps=0.000001):\n", " return (f(z + eps) - f(z - eps))/(2 * eps)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "jupyter": { "source_hidden": true }, "tags": [] }, "outputs": [], "source": [ "pw=5\n", "ph=5\n", "\n", "z = np.linspace(-5, 5, 200)\n", "\n", "\n", "# ------ Heaviside\n", "#\n", "fig, ax = plt.subplots(1, 1)\n", "fig.set_size_inches(pw,ph)\n", "ax.set_xlim(-5, 5)\n", "ax.set_ylim(-2, 2)\n", "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.plot(0, 0, \"rx\", markersize=10)\n", "ax.plot(z, heaviside(z), linestyle='-', label=\"Heaviside\")\n", "ax.plot(z, derivative(heaviside, z), linewidth=3, alpha=0.6, label=\"dHeaviside/dx\")\n", "# ax.plot(z, sign(z), label=\"Heaviside\")\n", "ax.set_title(\"Heaviside\")\n", "fidle.scrawler.save_fig('Heaviside')\n", "plt.show()\n", "\n", "\n", "# ----- Logit/Sigmoid\n", "#\n", "fig, ax = plt.subplots(1, 1)\n", "fig.set_size_inches(pw,ph)\n", "ax.set_xlim(-5, 5)\n", "ax.set_ylim(-2, 2)\n", "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.plot(z, sigmoid(z), label=\"Sigmoid\")\n", "ax.plot(z, derivative(sigmoid, z), linewidth=3, alpha=0.6, label=\"dSigmoid/dx\")\n", "ax.set_title(\"Logit\")\n", "fidle.scrawler.save_fig('Logit')\n", "plt.show()\n", "\n", "# ----- Tanh\n", "#\n", "fig, ax = plt.subplots(1, 1)\n", "fig.set_size_inches(pw,ph)\n", "ax.set_xlim(-5, 5)\n", "ax.set_ylim(-2, 2)\n", "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.plot(z, tanh(z), label=\"Tanh\")\n", "ax.plot(z, derivative(tanh, z), linewidth=3, alpha=0.6, label=\"dTanh/dx\")\n", "ax.set_title(\"Tanh\")\n", "fidle.scrawler.save_fig('Tanh')\n", "plt.show()\n", "\n", "# ----- Relu\n", "#\n", "fig, ax = plt.subplots(1, 1)\n", "fig.set_size_inches(pw,ph)\n", "ax.set_xlim(-5, 5)\n", "ax.set_ylim(-2, 2)\n", "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.plot(z, relu(z), label=\"ReLU\")\n", "ax.plot(z, derivative(relu, z), linewidth=3, alpha=0.6, label=\"dReLU/dx\")\n", "ax.set_title(\"ReLU\")\n", "fidle.scrawler.save_fig('ReLU')\n", "plt.show()\n", "\n", "# ----- Leaky Relu\n", "#\n", "fig, ax = plt.subplots(1, 1)\n", "fig.set_size_inches(pw,ph)\n", "ax.set_xlim(-5, 5)\n", "ax.set_ylim(-2, 2)\n", "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.plot(z, leaky_relu(z), label=\"Leaky ReLU\")\n", "ax.plot(z, derivative( leaky_relu, z), linewidth=3, alpha=0.6, label=\"dLeakyReLU/dx\")\n", "ax.set_title(\"Leaky ReLU (α=0.05)\")\n", "fidle.scrawler.save_fig('LeakyReLU')\n", "plt.show()\n", "\n", "# ----- Elu\n", "#\n", "fig, ax = plt.subplots(1, 1)\n", "fig.set_size_inches(pw,ph)\n", "ax.set_xlim(-5, 5)\n", "ax.set_ylim(-2, 2)\n", "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.plot(z, elu(z), label=\"ReLU\")\n", "ax.plot(z, derivative( elu, z), linewidth=3, alpha=0.6, label=\"dExpReLU/dx\")\n", "ax.set_title(\"ELU (α=1)\")\n", "fidle.scrawler.save_fig('ELU')\n", "plt.show()\n", "\n", "# ----- Selu\n", "#\n", "fig, ax = plt.subplots(1, 1)\n", "fig.set_size_inches(pw,ph)\n", "ax.set_xlim(-5, 5)\n", "ax.set_ylim(-2, 2)\n", "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n", "ax.plot(z, selu(z), label=\"SeLU\")\n", "ax.plot(z, derivative( selu, z), linewidth=3, alpha=0.6, label=\"dSeLU/dx\")\n", "ax.set_title(\"ELU (SELU)\")\n", "fidle.scrawler.save_fig('SeLU')\n", "plt.show()\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "---\n", "<img width=\"80px\" src=\"../fidle/img/logo-paysage.svg\"></img>" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3.9.2 ('fidle-env')", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.2" }, "vscode": { "interpreter": { "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d" } } }, "nbformat": 4, "nbformat_minor": 4 }