{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img width=\"800px\" src=\"../fidle/img/header.svg\"></img>\n",
    "\n",
    "# <!-- TITLE --> [ACTF1] - Activation functions\n",
    "<!-- DESC --> Some activation functions, with their derivatives.\n",
    "<!-- AUTHOR : Jean-Luc Parouty (CNRS/SIMaP) -->\n",
    "\n",
    "## Objectives :\n",
    " - View the main activation functions \n",
    "\n",
    "Les fonctions d'activation dans Keras :  \n",
    "https://www.tensorflow.org/api_docs/python/tf/keras/activations\n",
    "\n",
    "## What we're going to do :\n",
    "\n",
    " - Juste visualiser les principales fonctions d'activation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import math\n",
    "from math import erfc, sqrt, exp\n",
    "from math import pi as PI\n",
    "from math import e as E\n",
    "import sys\n",
    "\n",
    "import fidle\n",
    "\n",
    "# Init Fidle environment\n",
    "run_id, run_dir, datasets_dir = fidle.init('ACTF1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SELU_A = -sqrt(2/PI)/(erfc(1/sqrt(2))*exp(1/2)-1)\n",
    "SELU_L = (1-erfc(1/sqrt(2))*sqrt(E))*sqrt(2*PI) / (2*erfc(sqrt(2))*E*E+PI*erfc(1/sqrt(2))**2*E-2*(2+PI)*erfc(1/sqrt(2))*sqrt(E)+PI+2)**0.5\n",
    "\n",
    "\n",
    "def heaviside(z):\n",
    "    return np.where(z<0,0,1)\n",
    "\n",
    "def sign(z):\n",
    "    return np.where(z<0,-1,1)\n",
    "#    return np.sign(z)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def tanh(z):\n",
    "    return np.tanh(z)\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def leaky_relu(z,a=0.05):\n",
    "    return np.maximum(a*z, z)\n",
    "\n",
    "def elu(z,a=1):\n",
    "    #y=z.copy()\n",
    "    y=a*(np.exp(z)-1)\n",
    "    y[z>0]=z[z>0]\n",
    "    return y\n",
    "\n",
    "def selu(z):\n",
    "    return SELU_L*elu(z,a=SELU_A)\n",
    "\n",
    "def derivative(f, z, eps=0.000001):\n",
    "    return (f(z + eps) - f(z - eps))/(2 * eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pw=5\n",
    "ph=5\n",
    "\n",
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "\n",
    "# ------ Heaviside\n",
    "#\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "fig.set_size_inches(pw,ph)\n",
    "ax.set_xlim(-5, 5)\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.plot(0, 0, \"rx\", markersize=10)\n",
    "ax.plot(z, heaviside(z),              linestyle='-',  label=\"Heaviside\")\n",
    "ax.plot(z, derivative(heaviside, z),  linewidth=3, alpha=0.6, label=\"dHeaviside/dx\")\n",
    "# ax.plot(z, sign(z),                  label=\"Heaviside\")\n",
    "ax.set_title(\"Heaviside\")\n",
    "fidle.scrawler.save_fig('Heaviside')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "# ----- Logit/Sigmoid\n",
    "#\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "fig.set_size_inches(pw,ph)\n",
    "ax.set_xlim(-5, 5)\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.plot(z, sigmoid(z),             label=\"Sigmoid\")\n",
    "ax.plot(z, derivative(sigmoid, z), linewidth=3, alpha=0.6, label=\"dSigmoid/dx\")\n",
    "ax.set_title(\"Logit\")\n",
    "fidle.scrawler.save_fig('Logit')\n",
    "plt.show()\n",
    "\n",
    "# ----- Tanh\n",
    "#\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "fig.set_size_inches(pw,ph)\n",
    "ax.set_xlim(-5, 5)\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.plot(z, tanh(z),             label=\"Tanh\")\n",
    "ax.plot(z, derivative(tanh, z), linewidth=3, alpha=0.6, label=\"dTanh/dx\")\n",
    "ax.set_title(\"Tanh\")\n",
    "fidle.scrawler.save_fig('Tanh')\n",
    "plt.show()\n",
    "\n",
    "# ----- Relu\n",
    "#\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "fig.set_size_inches(pw,ph)\n",
    "ax.set_xlim(-5, 5)\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.plot(z, relu(z),             label=\"ReLU\")\n",
    "ax.plot(z, derivative(relu, z), linewidth=3, alpha=0.6, label=\"dReLU/dx\")\n",
    "ax.set_title(\"ReLU\")\n",
    "fidle.scrawler.save_fig('ReLU')\n",
    "plt.show()\n",
    "\n",
    "# ----- Leaky Relu\n",
    "#\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "fig.set_size_inches(pw,ph)\n",
    "ax.set_xlim(-5, 5)\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.plot(z, leaky_relu(z),              label=\"Leaky ReLU\")\n",
    "ax.plot(z, derivative( leaky_relu, z), linewidth=3, alpha=0.6, label=\"dLeakyReLU/dx\")\n",
    "ax.set_title(\"Leaky ReLU (α=0.05)\")\n",
    "fidle.scrawler.save_fig('LeakyReLU')\n",
    "plt.show()\n",
    "\n",
    "# ----- Elu\n",
    "#\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "fig.set_size_inches(pw,ph)\n",
    "ax.set_xlim(-5, 5)\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.plot(z, elu(z),              label=\"ReLU\")\n",
    "ax.plot(z, derivative( elu, z), linewidth=3, alpha=0.6, label=\"dExpReLU/dx\")\n",
    "ax.set_title(\"ELU (α=1)\")\n",
    "fidle.scrawler.save_fig('ELU')\n",
    "plt.show()\n",
    "\n",
    "# ----- Selu\n",
    "#\n",
    "fig, ax = plt.subplots(1, 1)\n",
    "fig.set_size_inches(pw,ph)\n",
    "ax.set_xlim(-5, 5)\n",
    "ax.set_ylim(-2, 2)\n",
    "ax.axhline(y=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.axvline(x=0, linewidth=1, linestyle='--', color='lightgray')\n",
    "ax.plot(z, selu(z),              label=\"SeLU\")\n",
    "ax.plot(z, derivative( selu, z), linewidth=3, alpha=0.6, label=\"dSeLU/dx\")\n",
    "ax.set_title(\"ELU (SELU)\")\n",
    "fidle.scrawler.save_fig('SeLU')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "<img width=\"80px\" src=\"../fidle/img/logo-paysage.svg\"></img>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.2 ('fidle-env')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "vscode": {
   "interpreter": {
    "hash": "b3929042cc22c1274d74e3e946c52b845b57cb6d84f2d591ffe0519b38e4896d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}