diff options
Diffstat (limited to 'logistic_regression_notebook.ipynb')
| -rw-r--r-- | logistic_regression_notebook.ipynb | 525 |
1 files changed, 525 insertions, 0 deletions
diff --git a/logistic_regression_notebook.ipynb b/logistic_regression_notebook.ipynb new file mode 100644 index 0000000..3b4f165 --- /dev/null +++ b/logistic_regression_notebook.ipynb @@ -0,0 +1,525 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Logistic Regression\n", + "\n", + "Logistic regression is a *classification algorithm*." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 1.0, 'Data representation')" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEWCAYAAABliCz2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFMBJREFUeJzt3Xu0ZGV95vHvQx8aPMgtgTBpbh1jBB1jgO5wMXgZjG2GaJOVoEbFIY5rBJNJjInjKMwyZlw4ZGaSSRw0YqLCjIh4DS3EoVkhwcsYpBsU5TawtOUOrYRrA03Db/7Y+4TqQ5/uajhFVff7/ay119l71/vu+tXbfZ7a9VadXakqJEnbvx3GXYAk6Zlh4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl7YzSV6S5Ppx16HJY+DrKUmyJslDSe5Pck+S/5vk5CRD/Z9KsjhJJZkada3bkiQvT3LLVvapJM+d2a6qr1XVQfNfnbZ1Br6ejtdU1a7AgcDpwH8EPj6OQub7icMnIm2PDHw9bVV1b1WtAF4PnJjkhQBJfjXJlUnuS3JzkvcPdPtq//OeJA8kOSrJzya5JMmPk/woyTlJ9pjrfvsz299JcgNwQ7/v4CQXJ7k7yfVJXjfQ/qwkH+1vvz/JpUkOfBrHOzbJNf2xbk3yroHbXp3k2wOvfl40cNuaJO9KclWSe5Ocl2TnJLsAXwEW9WPyQJJFSQ5P8s3+WLcnOSPJwv5YM+P4nb7962e/Skjy/CT/0Pe/OsnyWWPy4SQX9o/jsiQ/u6V/c22jqsrFZasXYA3wy5vYfxPw9n795cDP051YvAi4E/i1/rbFQAFTA32fC7wS2AnYm+5J4c83U0MBFwM/ATwL2AW4GXgLMAUcCvwIeEHf/izgfuCl/X38BfD1p3G824GX9Ot7Aof164cCdwFHAAuAE/vx2mlg7L4FLOrv61rg5IExu2XW41wCHNnXsLhv//uz6n7uwPY/HwPYEbgROAVYCBzTj8FBA2PyY+Dw/vjnAJ8Z9/8vl9EsnuFrvt1GF2JU1T9U1Xer6vGqugo4F3jZXB2r6saquriqHqmqtcCfba59779U1d1V9RDwamBNVX2yqjZU1ZXAF4DXDrS/sKq+WlWPAKcCRyXZ/yke71HgBUl2q6p/qqor+v1vA86sqsuq6rGqOht4hC60Z3yoqm6rqruBLwOHbGZcVlfVP/Y1rAHOHGJcZhwJPBs4varWV9UlwAXAGwbafKmqvlVVG+gCf85atG0z8DXf9gXuBkhyRJK/T7I2yb3AycBec3VMsk+Sz/TTI/cBn9pc+97NA+sHAkf0Uxf3JLkHeBPwLzbVvqoe6Gtd9BSP9xvAscAP++mhowb6/eGsfvvPup87BtbX0YXyJiV5XpILktzRj8sH2fK4zFgE3FxVjw/s+yHdv9NW16Jtm4GveZPkF+mC5Ov9rk8DK4D9q2p34KNA+ts2dZnWD/b7f76qdgNOGGg/l8Hj3AxcWlV7DCzPrqq3D7T557P5JM+mezVy21M5XlVdXlXHAT8F/A3w2YF+p83qN11V527hscy+/xl/CVwH/Fw/Lqew5XGZcRuw/6xPTx0A3Dpkf21HDHw9bUl2S/Jq4DPAp6rqu/1NuwJ3V9XDSQ4H3jjQbS3wOPCcgX27Ag8A9ybZF/gPW1nKBcDzkrw5yY798otJnj/Q5tgkR/dven4A+MequnnTh5v7eEkWJnlTkt2r6lHgvv7xAPwVcHL/CidJdkn3BvauQzyGO4GfTLL7wL5d++M/kORg4O2b6PMcNu0yurP2d/f1vxx4Dd2/lRpj4Ovp+HKS++nOaE+lm3N/y8Dtvw38577N+3jiDJiqWgecBnyjn/Y4Evhj4DDgXuBC4ItbU0xV3Q8sA36T7sz2DuBP6N6gnfFp4I/opnKW0L2KeKrHezOwpp9mOZluuoeqWgX8O+AM4J/o3jT9rSEfw3V073V8vx+XRcC76J4s76d7MjlvVrf3A2f37V8363jr6QL+X9O94fwR4N/096PGpMovQFEbkpxF9+mV/zTuWqRx8Axfkhph4EtSI5zSkaRGeIYvSY2YqAtE7bXXXrV48eJxlyFJ24zVq1f/qKr2HqbtRAX+4sWLWbVq1bjLkKRtRpIfDtvWKR1JaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhoxssBP8okkdyX53qjuQ5I0vFH+4dVZdNcD/18jvA8AVqyAlSth993h3nth2TJYvnzTbV/7WrjkEjjmGPjc57q+f/AHcOed8KIXwaGHPtF/xQo488yu3/Q0XHopPPAA7Lgj7LADPPggJLBgQbdvehoee6zbvvvubv1Zz4KHHura7bDDE7fvvPMTx5ia6mq/+2549FHYZx84+mi48MKuL3Rtpqe7ZcMGOPBAeNWr4Lzz4NZbYY89YN99u77T03D55V2fgw6Ck07qjrFy5ZMf2513dn1OOmnuMZO0nRjlN6QDi4HvDdt+yZIltbXOP79qeroKnlimp7v9sx1//MbtXvziqgULNt430/+UU6p22unJt22Ly9RU1cKFGz+2me2ZZeHCTY+ZpMkGrKohM3bsc/hJ3pZkVZJVa9eu3er+K1fCunUb71u3rts/2yWXbLy9enV3xj3bunXdGfAjj2x1ORNpwwZYv75bn3lsM9sz1q/f9JhJ2n6MPfCr6mNVtbSqlu6991DX/9nIsmXdFMag6elu/2zHHLPx9pIl3fTKbNPT3fTGTjs9+bZt0dQULFzYrc88tpntGQsXbnrMJG0/JuriaU/F8uVw7rnDzeF/7nNbN4d/xBHb7xz+zGNzDl9qx0i/ACXJYuCCqnrhMO2XLl1aXi1TkoaXZHVVLR2m7Sg/lnku8E3goCS3JHnrqO5LkrRlI5vSqao3jOrYkqStN/Y3bSVJzwwDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0YKvCTvCPJbul8PMkVSZaNujhJ0vwZ9gz/31bVfcAyYE/gzcDpI6tKkjTvhg389D+PBf53VV09sE+StA0YNvBXJ1lJF/gXJdkVeHx0ZUmS5tvUkO3eChwCfL+q1iX5SeAtoytLkjTfhgr8qno8yWLghCQFfL2qvjTKwiRJ82vYT+l8BDgZ+C7wPeCkJB8eZWGSpPk17JTOMcDzq6oAkpwNXDOyqiRJ827YN21vBA4Y2N4fuGH+y5EkjcqwZ/i7Atcm+RZQwOHAqiQrAKpq+YjqkyTNk2ED/30jrUKSNHLDfkrn0lEXIkkaraECP8n9dFM5AAuBHYEHq2q3URUmSZpfw57h7zqzniTAccCRoypKkjT/tvryyNX5G+BVI6hHkjQiw07p/PrA5g7AUuDhkVQkSRqJYT+l85qB9Q3AGrppHUnSNmLYOXwvlCZJ27hhr6WzX5IvJbmrX76QZL9RFydJmj/Dvmn7SWAFsKhfvtzvkyRtI4YN/L2r6pNVtaFfzgL2HmFdkqR5Nmzg/zjJCUkW9MsJwI9HWZgkaX4N/SXmwOuAO4DbgePxG68kaZuyxU/pJFkA/LpXxJSkbdsWz/Cr6jHgDc9ALZKkERr2D6++keQM4DzgwZmdVXXFSKqSJM27YQP/kP7nH/c/Q3f1zGPmvSJJ0kgMG/gX0AV8+u0C7ktySFV9eySVSZLm1bCf0lkCnAz8NN0fXp1Ed7XMv0ry7hHVJkmaR8Oe4e8HHFZVDwAk+SPgQuClwGrgv46mPEnSfBn2DP+ngEcGth8F9qmqh2btlyRNqGHP8M8BLktyfr/9GuDTSXYBrhlJZZKkeTXs5ZE/kOQrwC/1u06uqlX9+ptGUpkkaV4Ne4ZPH/CrtthQkjSRtvo7bSVJ2yYDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaMdLAT/IrSa5PcmOS94zyviRJmzc1qgMnWQB8GHglcAtweZIVVXXNqO5T0tZZsQJWroTdd4d774Vly2D58k23PfXUrv3y5XDaaRv3X7as255Zv+wyOOccmOoT5r77YLfd4KCD4KSTun1nntn9/P734cYbYa+94Oij4brrurY33ACPPAI77AAPPggJ7LwzTE93tT78MFTBwoXdvgcf7O5v0SK4445uG2DffWHPPeHgg+HKK2HNmq7f1BQsWACPPQaPPtq1XbQI1q/v1l/2MvjBD+Cmm+CAA+B979t4bOYaj5nHddJJT7QfHKfZ47u52+ZdVY1kAY4CLhrYfi/w3s31WbJkSUl6Zpx/ftX0dFUXf90yPd3tn+2UUzZud8opG/dfuLBqp5269ampjdvOXqamttxmEpepqSfGZq7xmBmDmTE5//yNx2n2+G7utmEBq2rIXB7llM6+wM0D27f0+zaS5G1JViVZtXbt2hGWI2nQypWwbt3G+9at6/bPtmLFk7cH+69f352NA2zYsPn73bBhy20m0YYNT4zNXOMxMwbQjcnKlRuP0+zx3dxtozD2N22r6mNVtbSqlu69997jLkdqxrJl3VTIoOnpJ6ZnBs2eali+fOP+CxfCTjt161NbmCiemtpym0k0NfXE2Mw1HjNjAN2YLFu28TjNHt/N3TYKoxz2W4H9B7b36/dJmgDLl8O55w43hz84Rz04Zz3Tv7U5/LnG47Of3fQc/uA4DY7v4L/BMzGHn24KaAQHTqaA/we8gi7oLwfeWFVXz9Vn6dKltWrVqpHUI0nboySrq2rpMG1HdoZfVRuS/HvgImAB8InNhb0kabRGOpNWVX8L/O0o70OSNJyxv2krSXpmGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0Y2R9ePRVJ1gI/HNi1F/CjMZUz6RybuTk2m+f4zG1bHJsDq2qo69JMVODPlmTVsH9B1hrHZm6OzeY5PnPb3sfGKR1JaoSBL0mNmPTA/9i4C5hgjs3cHJvNc3zmtl2PzUTP4UuS5s+kn+FLkuaJgS9JjZi4wE+yf5K/T3JNkquTvGPcNU2aJAuSXJnkgnHXMmmS7JHk80muS3JtkqPGXdOkSPLO/nfqe0nOTbLzuGsapySfSHJXku8N7PuJJBcnuaH/uec4a5xvExf4wAbgD6vqBcCRwO8kecGYa5o07wCuHXcRE+ovgP9TVQcDv4DjBECSfYHfA5ZW1QvpvpToN8db1didBfzKrH3vAf6uqn4O+Lt+e7sxcYFfVbdX1RX9+v10v7D7jreqyZFkP+BXgb8edy2TJsnuwEuBjwNU1fqqume8VU2UKeBZ/dePTgO3jbmesaqqrwJ3z9p9HHB2v3428GvPaFEjNnGBPyjJYuBQ4LLxVjJR/hx4N/D4uAuZQD8DrAU+2U95/XWSXcZd1CSoqluB/w7cBNwO3FtVK8db1UTap6pu79fvAPYZZzHzbWIDP8mzgS8Av19V9427nkmQ5NXAXVW1ety1TKgp4DDgL6vqUOBBtrOX5E9VPxd9HN2T4iJglyQnjLeqyVbdZ9a3q8+tT2TgJ9mRLuzPqaovjrueCfJLwPIka4DPAMck+dR4S5ootwC3VNXMK8LP0z0BCH4Z+EFVra2qR4EvAi8ec02T6M4kPw3Q/7xrzPXMq4kL/CShm4O9tqr+bNz1TJKqem9V7VdVi+necLukqjxL61XVHcDNSQ7qd70CuGaMJU2Sm4Ajk0z3v2OvwDe0N2UFcGK/fiJw/hhrmXcTF/h0Z7Fvpjt7/Xa/HDvuorTN+F3gnCRXAYcAHxxzPROhf9XzeeAK4Lt0v/vb9WUEtiTJucA3gYOS3JLkrcDpwCuT3ED3quj0cdY437y0giQ1YhLP8CVJI2DgS1IjDHxJaoSBL0mNMPAlqREGvrZ7SX6vv3LmOVvZb3GSN46qLumZZuCrBb8NvLKq3rSV/RYDWx34SRZsbR/pmWDga7uW5KPAc4CvJDm1vwb6t/qLqx3Xt1mc5GtJruiXmUsOnA68pP/jv3cm+a0kZwwc+4IkL+/XH0jyp0m+AxyVZEmSS5OsTnLRzJ/rS+Nk4Gu7VlUn010G+F8Bu9BdjuLwfvu/9VfTvIvuFcBhwOuBD/Xd3wN8raoOqar/sYW72gW4rKp+ge7qrv8TOL6qlgCfAE6b54cmbbWpcRcgPYOW0V187l399s7AAXRPCGckOQR4DHjeUzj2Y3QX/AM4CHghcHF32RoW0F2SWBorA18tCfAbVXX9RjuT9wN30n1D1g7Aw3P038DGr4oHvyLw4ap6bOB+rq4qv15RE8UpHbXkIuB3+6tFkuTQfv/uwO1V9Tjdhftm3nS9H9h1oP8a4JAkOyTZHzh8jvu5Hth75vt0k+yY5F/O6yORngIDXy35ALAjcFWSq/ttgI8AJ/ZvuB5M98UpAFcBjyX5TpJ3At8AfkB3yeUP0V158kmqaj1wPPAn/TG/jdee1wTwapmS1AjP8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJasT/B6/+dhyoEz3zAAAAAElFTkSuQmCC\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "from sklearn.datasets import make_blobs\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "\n", + "# create linearly separable data\n", + "sep = False\n", + "while not sep:\n", + " X, Y = make_blobs(n_samples=200, n_features\n", + " =1, centers=2, cluster_std=0.5, center_box=(0, 10))\n", + " sep = True\n", + " for x_1 in X[Y == 1]:\n", + " for x_0 in X[Y == 0]:\n", + " if Y[X.argmin()] == 0 and x_0 > x_1:\n", + " sep = False\n", + " elif x_0 < x_1:\n", + " sep = False\n", + " \n", + "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33)\n", + "\n", + "plt.scatter(X, Y, s=10, c='b')\n", + "plt.xlabel('feature')\n", + "plt.ylabel('groups')\n", + "plt.yticks([0, 1])\n", + "plt.title('Data representation')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model\n", + "\n", + "We have to fit a **model** trough our data. \n", + "A *sigmoid* (also called *logistic*) function is used:\n", + "$$\\sigma(x) = \\frac{1}{1 + e^{-x}}$$\n", + "\n", + "It produce an 'S' shape like function that stays between 0 and 1" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 1.0, 'Sigmoid function')" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VPXd9/H3NzuQkECAsBP2VVEWcaUgqOBSeLzrdqNWrXVp1dr2aSt1abWXt21vt7q0PtyK2+1aSilVSxUxdUGRXfZ9C4YlCYSErDPze/6YwQYIkISZnJnJ53Vdc2WWM+d8JiSf/PjNmXPMOYeIiMSXBK8DiIhI+KncRUTikMpdRCQOqdxFROKQyl1EJA6p3EVE4pDKXZqMmU0xs/ejbbtmlmdmNx/jMTOzF81sn5l9GbmUdW77H2b23abcpsQP037uEk5mdi7we2Aw4AfWAHc75xZ6Guw4zCwP+F/n3PN1PHYe8AbQ3zl3MIIZfg30cc5dG6ltSPOS5HUAiR9m1hp4B7gdeBtIAc4DqrzMdZJ6AFsjWewikaBpGQmnfgDOuTecc37nXIVz7n3n3FcAZnaDmX16aGEzu9DM1plZiZn90cz+dWh6JLTsZ2b2hJntN7PNZnZ26P4dZran9pSFmWWa2StmttfMtpnZfWaWcIztXmBma0PbfQawul6MmX0PeB44y8zKzOzBI9cVWs6ZWZ/Q9ZfM7Fkze9fMSs1sgZn1rrXsYDP7wMyKzWy3mf3SzCYAvwSuCm1neWjZb6aLzCwh9Jq2hV77K2aWGXosN5Thu2a23cwKzezeRv8rSlxQuUs4rQf8ZvaymU00szbHWtDM2gEzgKlANrAOOPuIxUYBX4Uefx14ExgJ9AGuBZ4xs/TQsk8DmUAv4FvA9cCNx9juTOA+oB2wCTinrozOuReA24DPnXPpzrlfnegbEHI18CDQBtgIPBzadgYwF5gDdA69jg+dc3OA/wLeCm1naB3rvCF0GRt6jenAM0cscy7QHxgHPGBmA+uZV+KQyl3Cxjl3gGDBOOB/gL1mNtvMcupY/GJglXNupnPOBzwF7DpimS3OuRedc37gLaAb8JBzrso59z5QDfQxs0SChTrVOVfqnNsKPAZcd5ztznDO1QBP1rHdk/VX59yXodf1GnBa6P5LgV3Oucecc5WhrAvquc4pwOPOuc3OuTKCfxSvNrPaU6sPhv63tBxYDtT1R0KaCZW7hJVzbo1z7gbnXFdgCMER6pN1LNoZ2FHreQ7IP2KZ3bWuV4SWO/K+dIIj8GRgW63HtgFd6rndHXUsdzJq/7EoD2WE4B+nTY1cZ2eOfn1JQO0/nMfarjRDKneJGOfcWuAlgiV/pAKg66EbZma1bzdQIVBD8M3PQ7oDO4+x3W5HbLdbHcsdy0GgZa3nd2zAc3cQnFKpy4l2W/uao1+fj8P/AIp8Q+UuYWNmA8zsp2bWNXS7G3AN8EUdi78LnGJmk0NTCz8EGlKU3whN27wNPGxmGWbWA/gJ8L/H2O5gM7s8tN27Grjd5aHnn2ZmacCvG/Dcd4BOZna3maWGso4KPbYbyD30JnAd3gB+bGY9Q+8zHJqj9zVg+9KMqNwlnEoJvgm6wMwOEiz1lcBPj1zQOVcIXEFwn/giYBCwiMbvNnknwVH1ZuBTgm/ATj/Odn8b2m5f4LP6bsQ5tx54iOAboxtC26rvc0uBC4DLCE6hbCD4BinAn0Nfi8xsSR1Pnw68CnwMbAEqCb5mkTrpQ0wSFUIj1nxginPuI6/ziMQ6jdzFM2Z2kZllmVkqwf28jbqncESkgVTu4qWzCO49UkhwqmKyc67C20gi8UHTMiIicUgjdxGROOTZgcPatWvncnNzI7b+gwcP0qpVq4itP9KU3zuxnB2U32uRzr948eJC51z7Ey3nWbnn5uayaNGiiK0/Ly+PMWPGRGz9kab83onl7KD8Xot0fjPbduKlNC0jIhKXVO4iInFI5S4iEoei6kxMNTU15OfnU1lZedLryszMZM2aNWFI5Y2mzJ+WlkbXrl1JTk5uku2JSORFVbnn5+eTkZFBbm4uwYP1NV5paSkZGRlhStb0miq/c46ioiLy8/Pp2bNnxLcnIk3jhNMyZjY9dFqvlcd43MzsKTPbaGZfmdmwxoaprKwkOzv7pItd6s/MyM7ODsv/lkQketRnzv0lYMJxHp9I8Mh6fYFbgD+dTCAVe9PT91wk/pxwWsY597GZ5R5nkUnAK6Ez2nwROhBUJ+dcQZgyiohEBeccvoCj2heg2hegxh+gKvT10P2b9/tJ31pMjd/hDzh8gQA+f/B5h24HnMPndwScwx8Av3MEQo8H73MEHARC9x+63pDDxYRjzr0Lh5+mLD9031Hlbma3EBzdk5OTQ15e3mGPZ2ZmUlpaGoZI4Pf7w7YuLzR1/srKyqP+PU5GWVlZWNfXlGI5Oyj/Ib6A42ANHKxxlPscFTWOCj9U1Dgq/VDpc1T4oMrvqPL/+2u131Hjh6pA8GtNAGpqXa9XvX7x+UnnP1lN+oaqc24aMA1gxIgR7shPca1ZsyZsbyLqDdWGSUtL4/TTTw/b+mL5U4axnB3iN79zjgOVPvYcqGT3gSp2H6hkb1kVRWVVFJZVU1hWxb7yavYdrGFfeTXl1f4Tbis1KYFWqUm0TEmkZUoiLdISaZuSSIvkRNJCl9SkBNKSE0lJSiA1KYGUxARSkv59SU4IfU1MICnRWLNqJcNOG0pSopGUkBD6aiQmBG8HvwZvJyQYiWYkJECiHX5fYoJhBglmoUtwCtV+V7/vYzjKfSeHn4OyK3Wfu1LCaPPmzTz88MOUlJQwY8YMr+OIhEVFtZ9tB/z8Y0UB24vL2V5cTv6+Cr7eH7wcrKOw05ITaJeeSnarFNqnp9KvQwZZLVNo0zKZ1i2SyWyRTOsWSbROSyYjLZmMtCRapSbRKiWRpMTwf9Qnde9aRvc74aFfIi4c5T4buMPM3iR4irUSzbdHXq9evXjhhRf4zne+43UUkQarqPazbncp63eVsnZXKRv2lLJpTxlfl4T22pofPNNgVstkurZpQa/2rTi3bzs6Z7YgJzONnIxUclqn0S4jlVYpidopoA4nLHczewMYA7Qzs3zgV0AygHPuOeA94GJgI1AO3BipsF6rqKhgwoQJzJs3j8TExDqXqa6uZvz48cybN4+kpKj6GIGIJ3z+AGt3lbJ42z6+yi9h5c4SNuwpJRCavE5LTqBvhwxG9cqmV7tWVOzdxiWjR9KtbUtap+mDdY1Vn71lrjnB447gmevj3vTp07n88suPWewAKSkpjBs3jrfeeospU6Y0YTqR6FDjD7B8x37mbyrii81FLNux/5v573bpqZzSpTUXDc5hUOdMBnTMoFvbliQm/HvknZe3k8GdM72KHzc0tKzD8uXLufPOOyksLGTt2rU457j//vuZO3cur7/++jfLjR07ll/+8pdccMEF3HfffZSUlPD0008zefJkpk6dGtFyLyoq4t5772Xp0qU88sgjTJ06NWLbEjmRgpIK5q3dw7w1e/hicxEHq/2YwcCOrblieFeG57ZleI82dM5M0xRKE4nacn/w76tY/fWBRj/f7/cfNcIe1Lk1v7ps8HGfV1lZyVVXXcUrr7zCGWecwf33309lZSX33nsv06ZNo/YJRh588EEeeOAB9uzZw9KlS5k9ezYAQ4YMYeHChUet+7zzzqtz98ZHH32U8ePHN+j1ZWdn89xzzzXoOSLhtLXwIO+uKOC9FQWsCv2udmvbgv8zrAvn9G7Hmb2yadMqxeOUzVfUlrtX5s6dy7BhwzjjjDMAOPXUU5kzZw5FRUVkZWUdtuzo0aNxzvH444+Tl5f3zR+TxMREUlJSjtqd8ZNPPmm6FyISAUVlVfx16U5mLdvJyp3BQh/WPYt7Jg5g3IAO9OmQrpF5lIjacj/RCPtEGruf+MqVKznllFO+ub1kyRKGDRtGixYtjjr+yooVKygoKCA7O/uobVVVVZGWlnbYffUZuTflL4ZOji71EQg4Pt6wlze/3MHcNbvxBRxDu2Zy3yUDufiUTnTOauF1RKlD1Ja7V7Kzs5k3bx4A69evZ+bMmcyfP582bdrg9/uprKwkLS2NgoICpkyZwt/+9jfuuusu5syZw4QJwUPwFBUV0a5du6MOoVufkfuhwg33h5jmzp1LcXExRUVF3H777WFbr8Svg1U+Zi7J58XPtrK58CDZrVK48ZxcrhjRjX45sfsBweZC5X6Ea665htmzZzNkyBDatWvHG2+8QXZ2NgAXXnghn376KWeffTaXX345jz32GAMHDuT+++/nF7/4xTfl/tFHH3HJJZc0avuHPpxUVFTErFmzjnp8xYoVZGZm0r179+OuZ9asWd88f+LEiVx88cVcdNFFfPbZZ43KJc3H/vJqpn+2lZc+28KBSh9Du2byh6tPY+KQTqQk6fw+sULlfoT09HT+/ve/1/nYD3/4Q5544gnGjx/P55//+9gRo0ePPuz266+/zm9/+9tGbf/Qh5MmT55c5+OLFy8mNzf3sHKvqanhV7/6FeXl5QQCAZ566ikmT578zToCgQAPP/wwP/vZz1i0aBEjR45sVDaJb/sOVvP8p5t5ef42yqp8XDQ4h1tG92ZY9yzNo8cglXsDDBs2jLFjx9a5J84h1dXVTJ48mX79+oV126tXr+app55i3bp1ZGRk8Oabb/Lggw+Sk5PDtGnTqKioICsriy1bthz13ISEBO6///6w5pH4UeXz8/L8rTw9byNlVT4uHtKJO8f1YUDH1l5Hk5Ogcm+gm2666biPp6SkcP3114d9u4MGDeK5557jpZdeIjc397ADKy1dupRnn32W1NTUsG9X4pdzjndXFPDbf6wlf18FY/q3Z+rEgfTvqPn0eKAJtChTVFTEbbfdxvLly3nkkUfq9ZxJkyZxww038POf/5w5c+ZEOKHEg+1F5Vw//UvueH0p6alJvPq9M3jpxjNU7HFEI/coc+jDScfaW+aGG2446r7LLruMyy67rAnSSayr8Qd44dMtPDl3PUkJCTz47cFce2aPwz7+L/FB5S7STGwpPMjdby1j+Y79XDQ4h19/ezCdMrWPerxSuYvEOeccby3cwUPvrCY5MYFn/3MYl5zayetYEmEqd5E4VlpZw89nfMU/Vu7i7N7ZPHblUI3Wm4moK3fnnPapbWI6DEF82rinlFteXcy2onKmThzA98/rRYLm1puNqCr3tLQ0ioqKyM7OVsE3EeccRUVFRx0HR2LbnJW7+Onby2iRkshrN4/izF7ZXkeSJhZV5d61a1fy8/PZu3fvSa/r0DFgYlVT5k9LS6Nr165Nsi2JLOcc72yqZsacxQztlsVz1w7TNEwzFVXlnpycTM+ePcOyrry8PE4//fSwrMsLsZ5fmp7PH+BXs1cxY0MNk07rzO+/cyqpScc+a5jEt6gqdxFpnPJqH3e9sZS5a/ZwSc9knrjyNM2vN3Mqd5EYV1pZw40vLmTJ9n38ZtJgulVtVbGLDj8gEssOVNZw/fQvWbZjP09fM4zrzsr1OpJECY3cRWJUSXkN109fwOqCAzw7ZRgXDe7odSSJIip3kRh0oLKGa19YwLpdpTx37XDGDczxOpJEGZW7SIyprPFz80uLWLvrANOuG8HYAR28jiRRSOUuEkNq/AHueH0JC7cV89TVp6vY5Zj0hqpIjAgEHL+Y8RVz1+zhoUlDuGxoZ68jSRRTuYvEiP9+fx0zl+7kpxf047oze3gdR6Kcyl0kBsxYnM+f8jZxzRndueP8Pl7HkRigcheJcgu3FjN15lec3TubhyYN1kH1pF5U7iJRbEdxObe+upiubVryxynDSE7Ur6zUj35SRKJURbWf77+yCJ8/wAvfHUFWyxSvI0kM0a6QIlHIOce9s1awbncpL94wkl7t072OJDFGI3eRKPTWwh3MXLKTO8/vy5j+2pddGk7lLhJlVu4s4YHZqzivbzt+NK6v13EkRqncRaLIgcoafvDaErJbpfDkVaeRqEP3SiNpzl0kijwwayU791fw9q1nkp2e6nUciWEauYtEib8t28msZV9z5/l9GN6jrddxJMap3EWiwM79Fdw3ayXDumdxx1h9AlVOnspdxGP+gOMnby0jEHA8edXpJOmDShIGmnMX8djzn2xmwZZiHr1iKN2zW3odR+KEhggiHtq0t4zHPljPRYNz+I9hXbyOI3FE5S7ikUPHZ2+RnMhvJg/RAcEkrFTuIh555fOtLNq2jwcuHUSHjDSv40icUbmLeGBHcTm/m7OOb/Vrz+WajpEIULmLNDHnHPfM/IrEBOO/Lj9F0zESESp3kSY2e/nXfLaxiF9M6E+XrBZex5E4pXIXaUIlFTX85p01DO2ayX+O0nlQJXK0n7tIE3r8/XUUH6zixRtG6qBgElEauYs0kRX5Jbz6xTauO7MHp3TN9DqOxDmVu0gT8Acc981aQdtWqfzkwv5ex5FmQOUu0gTeXrSD5fkl3H/pQDJbJHsdR5oBlbtIhB2orOHRf67jjNy2fHtoZ6/jSDOhN1RFIuzpDzdQXF7Ny5cN0j7t0mQ0cheJoC2FB3lp/lauHN6NIV30Jqo0HZW7SAQ9/O5qUpMS+b8X6U1UaVoqd5EI+Xj9Xuau2cMd5/ehfYbOhypNS+UuEgH+gOPhd9fQI7slN56T63UcaYZU7iIR8Jcl+azbXcovJgwgNSnR6zjSDKncRcKsotrP4++v57RuWUwc0tHrONJMqdxFwmz6Z1vYdaCSqRMHaNdH8YzKXSSMisqq+FPeJsYPzGFUr2yv40gzpnIXCaOn522kvNrHPRO166N4S+UuEiY7ist5bcE2rhrZjT4dMryOI82cyl0kTJ6Yu54EM340rp/XUURU7iLhsGF3KX9dupPvnp1Lx8w0r+OIqNxFwuHxD9bTKiWJ277V2+soIoDKXeSkfZW/n3+s3MXN5/WkbasUr+OIACp3kZP26PvradMyme+d29PrKCLfULmLnIQFm4v4eP1efjCmDxlpOsOSRA+Vu0gjOed47IP1dMhI5bqzengdR+QwKneRRvp8UxFfbinmB2N6k5asg4NJdFG5izSCc44n5q6nY+s0rj6ju9dxRI6ichdphE83FrJw6z5+OFajdolOKneRBnLO8cQH6+mcmcaVI7t5HUekTip3kQb6eEMhS7bv5wdj++hEHBK1VO4iDXBo1N4lqwVXjtCoXaKXyl2kAT7eUMiyHfv5wdjepCTp10eil346RerJOccf5gbn2r8zvKvXcUSOS+UuUk/zNxWxZPt+bh/TW3PtEvVU7iL19IcPN5DTOpUrNNcuMUDlLlIPX2wOfhr19m9pv3aJDSp3kXp46sMNtM9I1adRJWao3EVOYNHWYuZvKuLW0b00apeYoXIXOYGn520ku1UKU0bpyI8SO1TuIsexfMd+/rV+Lzef14sWKRq1S+xQuYscxzMfbSSzRbKO1y4xR+UucgxrCg7wwerd3HROT9JTk7yOI9IgKneRY3jmo42kpyZxw9m5XkcRaTCVu0gdNu4p470VBVx/Vg8yW+rcqBJ7VO4idfhj3kbSkhL53rk9vY4i0ij1Knczm2Bm68xso5ndU8fjN5jZXjNbFrrcHP6oIk1jT3mAvy37mimjupOdnup1HJFGOeG7RGaWCDwLXADkAwvNbLZzbvURi77lnLsjAhlFmtR7m2tINOP7o3t5HUWk0eozcj8D2Oic2+ycqwbeBCZFNpaINwpKKvh0p48rR3Ylp3Wa13FEGq0++3d1AXbUup0PjKpjuf8ws9HAeuDHzrkdRy5gZrcAtwDk5OSQl5fX4MD1VVZWFtH1R5rye+O1NVUEnGNoyt6YzA+x+70/RPnDI1w77/4deMM5V2VmtwIvA+cfuZBzbhowDWDEiBFuzJgxYdr80fLy8ojk+iNN+ZteYVkVn3w4j7O7JHPFxUf9+MaMWPze16b84VGfaZmdQO0DWHcN3fcN51yRc64qdPN5YHh44ok0nRc+3UKVL8ClvbTro8S++pT7QqCvmfU0sxTgamB27QXMrFOtm98G1oQvokjk7S+v5tXPt3HJKZ3o2Ep7CEvsO+FPsXPOB9wB/JNgab/tnFtlZg+Z2bdDi91lZqvMbDlwF3BDpAKLRMJL87dSVuXjjvP7eB1FJCzqNefunHsPeO+I+x6odX0qMDW80USaRmllDdM/3cKFg3IY0LE1u9Z6nUjk5On/n9LsvfrFNg5U+rjz/L5eRxEJG5W7NGvl1T6e/2QLY/q355SumV7HEQkblbs0a68v2E7xwWru1Fy7xBmVuzRblTV+pn28mbN7ZzO8R1uv44iElcpdmq0/L9rBntIq7SEjcUnlLs1Slc/PH/M2MTK3DWf1yvY6jkjYqdylWZqxOJ+CkkruGtcXM/M6jkjYqdyl2an2BfjjR5s4vXsW5/Zp53UckYhQuUuz89el+ezcX6FRu8Q1lbs0KzX+AM98tJFTu2Yypl97r+OIRIzKXZqVWUt3sqO4grvO16hd4pvKXZoNX2jUPqhTa8YN7OB1HJGIUrlLszFz6U62FZXz4wv6adQucU/lLs1CjT/A0/M2cEqXTMZr1C7NgMpdmoW/LM5nR3EFP75Ac+3SPKjcJe5V+wI8PW8jQ7tlMba/Ru3SPKjcJe7NWBzcr/3u8Rq1S/Ohcpe4Vu0L8OxHGzm9e5b2a5dmReUuce3NhdvZub+Cn2gPGWlmVO4St8qrfTz14UbO7NVWx5CRZkflLnHr5fnbKCyr4mcX9deoXZodlbvEpZKKGp771ybOH9BBZ1mSZknlLnHp+U82U1JRw08v7Od1FBFPqNwl7hSWVfHCp1u49NRODO6c6XUcEU+o3CXuPDNvI5U1fn58gUbt0nyp3CWubCs6yGsLtnHVyO70bp/udRwRz6jcJa789z/XkZSQwI/H9/U6ioinVO4SN5bt2M87XxXw/dG96NA6zes4Ip5SuUtccM7xyHtraJeewi2je3kdR8RzKneJCx+t28OCLcX8aHw/0lOTvI4j4jmVu8Q8nz/AI++tpVe7Vlw9spvXcUSigspdYt7rX25nw54y7pk4gORE/UiLgMpdYtz+8moe/2A95/TJ5oJBOV7HEYkaKneJaU/O3cCBihruv3SQDg4mUovKXWLW+t2lvPrFNqaM6sGAjq29jiMSVVTuEpOcc/zmndW0SknUYQZE6qByl5j0werdfLKhkLvH96NtqxSv44hEHZW7xJzyah8P/n01/XMyuO6sHl7HEYlK+rSHxJynPtzIzv0V/Pm2s7Tro8gx6DdDYsr63aU8/8lmrhjelZG5OsOSyLGo3CVmOOe4b9ZKWqUmcc/EAV7HEYlqKneJGTOX7OTLLcXcM3EA2empXscRiWoqd4kJe0ur+M27qzm9exZXjdDxY0ROROUuMeHXs1dRXuXn9/9xKgkJ+iSqyImo3CXqzVlZwLsrCrhrXB/65mR4HUckJqjcJartL6/mvlmrGNSpNbd+q7fXcURihvZzl6j20Dur2V9ezcs3jdQ+7SINoN8WiVpzVu5i5pKd3Pat3gzunOl1HJGYonKXqLT7QCVTZ37FkC6tuWtcX6/jiMQclbtEnUDA8X//vJyKGj9PXnU6KUn6MRVpKP3WSNR5cf5WPtlQyH2XDKJPh3Sv44jEJJW7RJU1BQf43Zy1jBvQgSmjunsdRyRmqdwlapRW1vCD15bQOi2Z333nVJ02T+QkaFdIiQrOOe75ywq2F5fz2s2jaKdjx4icFI3cJSq8NH8r764o4GcX9efMXtlexxGJeSp38dzibft4+N01jB+Yw62je3kdRyQuqNzFU7tKKvnBa4vplJXGY1cO1Ty7SJio3MUz5dU+bn5lIWWVPv7n+hFktkj2OpJI3NAbquKJQMDxk7eWs+rrAzx//QgGdGztdSSRuKKRu3jisQ/WMWfVLu69eCDjBuZ4HUck7qjcpcm9vmA7z360iatHduN75/b0Oo5IXFK5S5N6b0UB985awZj+7Xlo0hC9gSoSISp3aTKfbijk7jeXMbx7G/40ZbgOCCYSQfrtkiaxdPs+bnl1Eb3at+KF746kRUqi15FE4prKXSJu8bZ9XP/Cl7RLT+WVm84gs6V2eRSJNJW7RNTCrcVc/8ICstNTeOvWM+nQOs3rSCLNgvZzl4j5YnMRN720kI6Zabzx/TPJUbGLNBmVu0TE4t0+ps39ku5tW/La90fRIUPFLtKUNC0jYffy/K08s7SKgZ1a8+YtZ6rYRTygkbuETSDg+N2ctfy/jzdzeodEXv/+mdorRsQjKncJiwOVNfzkreXMXbOba8/szvmZhSp2EQ9pWkZO2obdpUx+5jM+WreHBy4dxG8mDSFBnzwV8ZRG7nJS3vnqa34+4ytapiTy+s2jGKWzKIlEBZW7NEppZQ2/nr2avyzJ5/TuWfxpynA6ZuqNU5FooXKXBlu8rZi731rGzn0V3Hl+H+4a15fkRM3wiUQTlbvUW1mVj8feX8fL87fSOasFb996FiNy23odS0TqoHKXepm7ejcP/G0lBQcqmTKqO7+YMICMNB0jRiRaqdzluDbuKeWR99by4do99M/J4On/HMbwHm28jiUiJ6BylzoVlVXx5NwNvP7ldlomJzJ14gBuOren5tZFYoTKXQ5TVFbFtE828+rn26jyBZgyqjs/GteX7PRUr6OJSAOo3AWAgpIKXvxsK69+vo1Kn5/LTu3MXeP60qdDutfRRKQRVO7N3NLt+5j+2VbeW1GAc45Jp3Xhh2P7qNRFYpzKvRkqqahh9rKdvL0onxU7S8hITeKmc3K5/qxcurVt6XU8EQkDlXszUeXz8/H6Qv6+/Gv+uWoXVb4AAzu15qFJg7l8WFfSU/WjIBJP9Bsdx8qqfHy6YS/vr97NB6t2U1rlI6tlMleO6MZVI7sxpEum1xFFJEJU7nEkEHCs213KZxsL+df6vXyxuYgav6N1WhIThnTk0qGdObt3tnZnFGkGVO4xzOcPsKaglEXbilm0dR9fbC6i6GA1AL3at+LGc3py/oAODO/RRoUu0syo3GNEjT/AlsKDrNxZwsqdB4Jfvy6hvNoPQOfMNL7Vrz1n92nH2b2z6ZzVwuPEIuIllXuUKa/2sbWwnC8LfHyfS+NHAAAG/ElEQVT14QY27ilj/e5SNu0to8bvAEhLTmBQp9ZcOaIbw3u0YXiPNipzETmMyr0JBQKOooPV7D5Qye4DlXy9v4KvS4JfdxSXs724gsKyqlrPWE+XrBb0y0lnTP8O9O+YzqBOmfRu34okTbOIyHHUq9zNbALwByAReN4599sjHk8FXgGGA0XAVc65reGNGl2qfQHKqnyUVtZQUlHDgQofJRU17K+oZt/BavaV17DvYDWFB6spLK2i6GAVhWXV+APusPUkJxodM9Po1qYl4wd2oFvblvTIbknx1jVcMWGMzkMqIo1ywnI3s0TgWeACIB9YaGaznXOray32PWCfc66PmV0N/A64KhKBAZxz+AMOX+DfX33+AP6AoyZ0/euyAGsKDuDzO6r9Aap9AWpCX6v9Aap8fqp9Aap8AapqgrcrawJU1PipqPFTWe2nvNpPeY2fimofB6v8HKz2UVbpo6zKR5UvcNyMLVMSadMyhXbpKXTKTGNIl9a0z0glp3UaHTLSyGmdSpesFrRLTyUh4ejzjeYVr1exi0ij1Wfkfgaw0Tm3GcDM3gQmAbXLfRLw69D1GcAzZmbOucOHqbWs313K+Y/mEXCOgCP4NRC87g9d94dK/ND1QIBv7quXTz+p33IhCQYtkhNpkZJIalIiLVMOXZLonJVMq9QkWqUmkR66ZKQlkZGWTOu0JDJbJJPZMpmsFilktUwmLVnFLCLeqU+5dwF21LqdD4w61jLOOZ+ZlQDZQGHthczsFuAWgPScXNonVWIGBiSY1bp+xAUjISGBhCMeS0yARLN/3w7dl2RQU11FqxZpJCUE709OMJISCF2M5ARCFyM5MXg90cCs9ijaAb7QpQ41oUspVBC87KrHN7Q+ysrKyMvLC9Paml4s54/l7KD8XouW/E36hqpzbhowDWDEiBHurbsnRGxbeXl5jBkzJmLrjzTl904sZwfl91q05K/PLhc7gW61bncN3VfnMmaWBGQSfGNVREQ8UJ9yXwj0NbOeZpYCXA3MPmKZ2cB3Q9e/A8w73ny7iIhE1gmnZUJz6HcA/yS4K+R059wqM3sIWOScmw28ALxqZhuBYoJ/AERExCP1mnN3zr0HvHfEfQ/Uul4JXBHeaCIi0lj6mKOISBxSuYuIxCGVu4hIHFK5i4jEIfNqj0Uz2wtsi+Am2nHEJ2RjjPJ7J5azg/J7LdL5ezjn2p9oIc/KPdLMbJFzboTXORpL+b0Ty9lB+b0WLfk1LSMiEodU7iIicSiey32a1wFOkvJ7J5azg/J7LSryx+2cu4hIcxbPI3cRkWZL5S4iEofivtzN7E4zW2tmq8zs917naQwz+6mZOTNr53WW+jKz/w59378ys7+aWZbXmerDzCaY2Toz22hm93idpyHMrJuZfWRmq0M/7z/yOlNDmVmimS01s3e8ztIYZpZlZjNCP/trzOwsr7LEdbmb2ViC53cd6pwbDDzqcaQGM7NuwIXAdq+zNNAHwBDn3KnAemCqx3lOqNbJ4CcCg4BrzGyQt6kaxAf81Dk3CDgT+GGM5Qf4EbDG6xAn4Q/AHOfcAGAoHr6WuC534Hbgt865KgDn3B6P8zTGE8DPCZ7QNWY45953zh06+ewXBM/gFe2+ORm8c64aOHQy+JjgnCtwzi0JXS8lWCxdvE1Vf2bWFbgEeN7rLI1hZpnAaILnt8A5V+2c2+9Vnngv937AeWa2wMz+ZWYjvQ7UEGY2CdjpnFvudZaTdBPwD69D1ENdJ4OPmXKszcxygdOBBd4maZAnCQ5kAl4HaaSewF7gxdDU0vNm1sqrME16guxIMLO5QMc6HrqX4OtrS/C/qCOBt82sVzSdAvAE+X9JcEomKh0vu3Pub6Fl7iU4XfBaU2ZrzswsHfgLcLdz7oDXeerDzC4F9jjnFpvZGK/zNFISMAy40zm3wMz+ANwD3O9VmJjmnBt/rMfM7HZgZqjMvzSzAMGD+uxtqnwncqz8ZnYKwZHAcjOD4LTGEjM7wzm3qwkjHtPxvvcAZnYDcCkwLpr+oB5HfU4GH9XMLJlgsb/mnJvpdZ4GOAf4tpldDKQBrc3sf51z13qcqyHygXzn3KH/Lc0gWO6eiPdpmVnAWAAz6wekECNHm3POrXDOdXDO5Trncgn+4AyLlmI/ETObQPC/2N92zpV7naee6nMy+KhlwVHAC8Aa59zjXudpCOfcVOdc19DP+tXAvBgrdkK/mzvMrH/ornHAaq/yxPzI/QSmA9PNbCVQDXw3RkaQ8eAZIBX4IPQ/jy+cc7d5G+n4jnUyeI9jNcQ5wHXACjNbFrrvl6FzIEvTuBN4LTQ42Azc6FUQHX5ARCQOxfu0jIhIs6RyFxGJQyp3EZE4pHIXEYlDKncRkTikchcRiUMqdxGROPT/AfZTssY3+RLaAAAAAElFTkSuQmCC\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "def sigmoid(x):\n", + " return 1 / (1 + np.exp(-x))\n", + "\n", + "xs = np.linspace(-7, 7, 100)\n", + "ys = sigmoid(xs)\n", + "plt.plot(xs, ys, label=r'$\\sigma(x) = {\\frac{1}{1 + e^{-x}}}$')\n", + "plt.yticks([0, 0.5, 1])\n", + "plt.xlim([-7, 7])\n", + "plt.legend()\n", + "plt.grid()\n", + "plt.title('Sigmoid function')" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 1.0, 'Approximation')" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XeY1NW9x/H3d7axyy4gJSgILMhKICCoxF7BgtGr0VjAQjHWGEtuTK6xG2tiNEpiooiIFay5GqOxQYLXSglB0eCqwNI7LG3rnPvHGWBAll1gds+Uz+t55tkpv5357PDwmd+cXznmnENERNJLJHQAERFJPJW7iEgaUrmLiKQhlbuISBpSuYuIpCGVu4hIGlK5i8Qxs3Vm1q2JXuthM7upKV5LMo9pP3dpSmb2D6AvsKdzrjJwnCZjZsOBi5xzR4TOIplBa+7SZMysGDgScMCpjfQa2Y3xvCKpRuUuTWko8BEwFhi26U4zGxsbonjbzNaa2T/NrEvc487MrjKzb8xsuZnda2aR2GPDzex9M/u9ma0AbjWziJndaGZzzWypmT1pZi1jy59jZrPNrEXs9klmttjM2sW9Vve4XH8yszdiwzXvm9meZvaAma0ys/+Y2f5xOa8zs69jf8PnZnZ67P6ewMPAobHnWR33/HfE/f7FZvaVma00s1fNrMM278FlZlZqZqvN7CEzswT/+0gaUblLUxoKPBO7nGhm7eMeOw+4HWgLTI8tE+90oD9wAHAacGHcYwcD3wDtgTuB4bHLsUA3oBD4I4Bz7jngA2CkmbUBHsMPlyyrI/PZwI2xXJXAh8C02O0Xgfvjlv0a/82kJXAb8LSZ7eWc+wK4DPjQOVfonGu17YuY2QDg7tjr7QXMBcZvs9gpwPeB/WLLnVhHZhGVuzQNMzsC6AI875ybii/Cc+MW+ZtzblJsHP4G/Fpup7jHf+OcW+mcKwMeAIbEPbbQOfcH51yNc24j/oPifufcN865dcCvgMFxQzZXAAOAfwB/dc69toPof3HOTXXOVQB/ASqcc08652qB54DNa+7OuReccwudc9HYh0gpcFAD36LzgDHOuWmx9+BXsfegOG6Ze5xzq2PvwUSgXwOfWzKQyl2ayjDgLefc8tjtZ4kbmgHmbboSK+SVQIftPY5fq63rMWKPzd1m+Wz8mj3OudXAC0Bv4L56ci+Ju75xO7cLN90ws6FmNj02bLI69vxt63n+7WaOvQcrgI5xyyyOu74h/rVFtqWNT9LozCwfP4yQZWabCioPaGVmfWO3O8UtXwi0BhbGPU0nYGbseudtHtt2l6+F+G8Jm3QGaogVs5n1ww/rjANGAoN26Q+LE9tG8CgwED/8Umtm04FN4+L17Za2VWYzaw60ARbsbjbJTFpzl6bwQ6AW6IUfSugH9ATew4/DA/zAzI4ws1z82PtHzrn4NfJfmNkesaGaq/FDInUZB/zMzLrGPijuAp5zztWYWTPgaeB6YATQ0cx+koC/sTm+wJcBmNkI/Jr7JkuAvWN/X12ZR5hZPzPLi2X+2Dk3JwHZJAOp3KUpDAMed86VOecWb7rgN3Keh/8G+SxwC3445kDg/G2e4xVgKn5j69/wG0LrMgZ4CpgEzAYqgCtjj90NzHPO/Tk2tn0+cIeZlezOH+ic+xw/xPMhvsj7AO/HLTIB/81jsZkt387vvwPcBLwELAL2AQbvTibJbDqISYIzs7HAfOfcjXU87oAS59xXTRpMJIVpzV1EJA2p3EVE0pCGZURE0pDW3EVE0lCw/dzbtm3riouLQ728iEhKmjp16nLnXLv6lgtW7sXFxUyZMiXUy4uIpCQzm1v/UhqWERFJSyp3EZE0pHIXEUlDKncRkTSkchcRSUMqdxGRNKRyFxFJQ/WWu5mNiU0y/Fkdj5uZjYxN7DvDzA5IfEwREdkZDTmIaSz+vNtP1vH4SUBJ7HIw8OfYz0Yxa2YN33xewT7d/QQ3X31tdC8x9t0XMPvW5dHHIkyYaAwYaBx1FIwbB19+CfvuCwcfDLW1UFICPXrArFkwcSIsXgzLl8Nnn4Fz0K8ffP01zI0dOtC6tb+vpgbKyqC8HLKyYOlSqKqCli39yzsHkQisXw/t2vnb69ZBhw4Qjfrl8/PhuON8jmefhYULoUULOPBAv3xREQwY4B/flGvlSv/8rVr5HAsW+Ow9esCQ2MyipaXf/rsAunTZ+m8WkfTUoBOHxSbpfc0513s7jz0C/MM5Ny52exZwjHNu0Y6es3///m5nj1CdNQte+8nr/HzCyTv1e/FqieAwokSIEsGyIjiLkJ0TobI6QnU0Qk3UP1ZLFrVkESVCDdnUkkUN2ZsvtWRRTQ41ZFNNDlXkbv5ZRS6V5FFFLhU0o5I8KmjGRvI3X9bTnA0UsIHmrKWQtRRRTgvKacEaWlJLNpEIZGf7D4y1a/2Hx6Z/spycLR8i0ai/3bcvtG/vP4Bqa2HwYHjkEf9BVVXlf+/734fCQrj+ehW8SKoxs6nOuf71LZeI0w90ZOsJiufH7vtWuZvZJcAlAJ07d97pFyothaWtv8tbx9/LnNm+4boWO1avcvT+nqPnd51vutjl+eccpV86mudH2bDekZ3lW9BFHYYjL7uWbl0dOVlRWhTWsniho3xNlIr1UVytr/YI0VjF126u9U3Xc6jefF8O1RSwIVbr/pJH5VaXZlQQqXcqzS3WU8Cq6B6srGpN+crWLHNtWBL9Dstox2LaszTagYXRvVhoHVmWvReWlc2SJdC8uV+jLyuDDz/03xaKimDNGqiogLw8/02jtFTlLpKumvTcMs65UcAo8GvuO/v7JSXwQkE3xhddy8oCv9a6xx5Q2wH2ux7YpqjWtId7/hssCrW50KmTL7zqar823KoVHNbLr8UOHgxPjYKpU2FZFVTU1p8nK8uvHTecI5cq8tlIweZ19vUUsp7mrKOItRSxlhaU05I1tGQNrVlFG1vJdyIr6FHzH45gEm1Y4T8karY8c01tFgurOrLCurCyZh/WrOpOu8IS9uvfk8/z92XO2jxqavzfXVnp1/JLdmtiORFJZoko9wXEzVwP7E0jzdjeo4cfStg0ngxbjy1v6+KL/c8JE/y4dX1j7t26NfaYu7FuXR4dOuQRjbbaqTH3RXFj7quW19Kiajmdsxeyf/uFVM9ZQM03ZXTLLqNHzmxyy94ke9pYH3YSHJGVxeq23VneaX9q++zPqn0OpM2gg9i3R1Fj/DOJSBJIxJj7ycBPgR/gN6SOdM4dVN9z7sqYu+yE9ev9J98XX/jLjBnwr3/5TyPwnzp9+sDhh8PAgXDssf5rkIgktYaOuddb7mY2DjgGaIuf1f0WIAfAOfewmRl+b5pBwAZghHOu3tZWuQeyYgVMmeIH4z/4YMugfCQC/fvDqafC6adDz57+64eIJJWElXtjUbkniepq+PhjeOcdeOMN+OQTf3+PHnDuuTB0KGhSFZGkoXKXXbNgAbz6KrzwAvzjH37g/6ij4PLL4Uc/8ltiRSSYhpa7Tj8gW+vY0Rf5hAkwZw7ceaffyjtkCHTtCnfdBatWhU4pIvVQuUvdOnf2uyfNmgWvvQa9esENN/hhmttu8zvOi0hSUrlL/SIROPlkeOstmD7d711z662+5O+7z4/bi0hSUbnLzunbF15+GaZNg0MPhWuvhf32g7ffDp1MROKo3GXX7L8/vP46/PWvfs39hBP83jUrV4ZOJiKo3GV3nXKKP5T31lv9HjZ9+vjhGxEJSuUuu69ZM7jlFvjoI3/CnhNPhGuu0Vi8SEAqd0mcAw/0Z1676ip48EF/0pwlS0KnEslIKndJrGbNfLE//TRMnuwLf/Lk0KlEMo7KXRrHeef5c9fk5MAxx8Cbb4ZOJJJRVO7SePr18ycmKymB//ovGD8+dCKRjKFyl8a1557wz3/6feLPPRcefjh0IpGMoHKXxteyJfz97363ycsvhzFjQicSSXsqd2ka+fl+P/gTT/RTZD3/fOhEImlN5S5NJy/Pn7rg8MP9BtfXXgudSCRtqdylaRUU+FLv1w/OPtufo0ZEEk7lLk2vRQtf8O3a+Wn9Fi0KnUgk7ajcJYz27f2MT6tWwQ9/CBs3hk4kklZU7hJO377+SNZPPoGLLvJT+olIQqjcJazTT4fbb4dnn4XRo0OnEUkbKncJ7/rr/UnGrroKZs4MnUYkLajcJbxIBJ56ym9oPecc2LAhdCKRlKdyl+Sw556+4GfOhJ/9LHQakZSncpfkccIJ8MtfwqhR/nQFIrLLVO6SXH79a+jZEy69FNauDZ1GJGWp3CW55OXBY4/BvHlw3XWh04ikLJW7JJ9DD4Wrr4Y//QkmTQqdRiQlqdwlOd1xB3Tr5g9uqqgInUYk5ajcJTk1bw6PPAKlpXD//aHTiKQclbskr+OOgzPOgDvvhPnzQ6cRSSkqd0lu990H0ajfRVJEGkzlLsmtuBh+8QsYNw7+7/9CpxFJGSp3SX7XXQedOsGVV0Jtbeg0IilB5S7Jr6AAfvtbmD4dnnkmdBqRlKByl9Rw9tlwwAFw881QWRk6jUjSU7lLaohE4O67Ye5cf+4ZEdkhlbukjuOPh2OO8Qc4rVsXOo1IUlO5S+ow82vvS5fCAw+ETiOS1FTukloOOQROOw3uvRdWrAidRiRpqdwl9dxxB5SXw8iRoZOIJC2Vu6Se3r39xNojR8KaNaHTiCQllbukphtugNWr/WmBReRbVO6Smg48EAYN8meMXL8+dBqRpKNyl9R1442wfDk8+mjoJCJJR+Uuqevww/1+7/feqwk9RLahcpfUdsMNsHAhPP106CQiSUXlLqlt4EDo29ePvTsXOo1I0lC5S2ozg5//HL74At58M3QakaShcpfUd8450KGDn7VJRACVu6SD3Fw/kcc778CMGaHTiCQFlbukh0su8ZN63H9/6CQiSUHlLumhdWsYMQKefRYWLQqdRiQ4lbukj2uugZoaePjh0ElEglO5S/ro3t2fkmDUKKiqCp1GJCiVu6SXK66AxYvhL38JnUQkKJW7pJdBg6BrV3joodBJRIJSuUt6ycqCyy+H996DTz8NnUYkGJW7pJ8LL4RmzbT2LhlN5S7pp00bGDzYn0xMMzVJhlK5S3q64go/iceTT4ZOIhKEyl3SU//+framRx/V2SIlI6ncJX1dconfqPrxx6GTiDQ5lbukryFDoHlzTcMnGUnlLumrqMgX/PjxUF4eOo1Ik1K5S3q7+GLYsMGfUEwkg6jcJb19//uw334ampGMo3KX9GbmN6xOmwZTp4ZOI9JkVO6S/s47D/LzYfTo0ElEmozKXdJfq1Zwxhkwbhxs3Bg6jUiTULlLZrjwQn8qgv/939BJRJqEyl0ywzHHQHExjBkTOolIk1C5S2aIRGDYMHj3XSgrC51GpNGp3CVzDB/uzzPzxBOhk4g0OpW7ZI7iYhgwAB5/HKLR0GlEGpXKXTLLiBEwezZMmhQ6iUijUrlLZjnjDGjRwq+9i6QxlbtkloICOPtseOklWLcudBqRRqNyl8wzbJifpemll0InEWk0KnfJPIcfDvvso71mJK2p3CXzmMHQoTBxIsydGzqNSKNQuUtmGjrU/3zqqbA5RBqJyl0yU3ExHH00PPmkJtCWtKRyl8w1bBiUlsJHH4VOIpJwKnfJXGee6XeN1IZVSUMqd8lcRUX+oKbnnoOKitBpRBJK5S6ZbehQWL0aXnstdBKRhFK5S2YbMAA6dPAbVkXSSIPK3cwGmdksM/vKzK7bzuPDzWyZmU2PXS5KfFSRRpCV5edYfeMNWLo0dBqRhKm33M0sC3gIOAnoBQwxs17bWfQ551y/2EUzEUvqGDoUampg/PjQSUQSpiFr7gcBXznnvnHOVQHjgdMaN5ZIE+rdG/bfX0MzklYaUu4dgXlxt+fH7tvWj8xshpm9aGadtvdEZnaJmU0xsynLli3bhbgijWToUJg6FWbODJ1EJCEStUH1r0Cxc24/4G1guzsOO+dGOef6O+f6t2vXLkEvLZIAQ4b48XedjkDSREPKfQEQvya+d+y+zZxzK5xzlbGbo4EDExNPpIm0bw+DBsHTT0Ntbeg0IrutIeU+GSgxs65mlgsMBl6NX8DM9oq7eSrwReIiijSRoUNhwQJ/tkiRFFdvuTvnaoCfAm/iS/t559xMM/u1mZ0aW+wqM5tpZv8GrgKGN1ZgkUZz6qnQsqU2rEpaMBfojHj9+/d3U6ZMCfLaInW69FI/NLN4sT89gUiSMbOpzrn+9S2nI1RF4g0dChs2wMsvh04isltU7iLxDjvMT8GnoRlJcSp3kXjxU/CVlYVOI7LLVO4i2zr/fD8709NPh04isstU7iLb6tYNjjzST+KhKfgkRancRbZn+HD48kv4+OPQSUR2icpdZHvOOstPwTd2bOgkIrtE5S6yPUVF8KMf+dMAb9wYOo3ITlO5i9Rl2DBYswZeeSV0EpGdpnIXqcuxx0KnThqakZSkchepSyTi197fftufUEwkhajcRXZk2DCIRrXPu6QclbvIjnTvDkccAY8/rn3eJaWo3EXqc+GFMGsWfPBB6CQiDaZyF6nPWWdBYSGMHh06iUiDqdxF6lNYCIMHw/PPQ3l56DQiDaJyF2mIH//Yn+f9uedCJxFpEJW7SEMcfDD06gWPPRY6iUiDqNxFGsIMLrrIn0jss89CpxGpl8pdpKEuuABycrT2LilB5S7SUG3bwmmnwVNPQUVF6DQiO6RyF9kZl14KK1bASy+FTiKyQyp3kZ0xYIA/avWRR0InEdkhlbvIzohE4JJL4L33YObM0GlE6qRyF9lZw4dDbq7W3iWpqdxFdla7dnDmmfDkk/7AJpEkpHIX2RWXXeZnadIRq5KkVO4iu+KII/wRq3/+c+gkItulchfZFWZ+7X3yZPjkk9BpRL5F5S6yq4YNg6IiGDkydBKRb1G5i+yqFi38RB7PPw+LFoVOI7IVlbvI7vjpT6GmBh5+OHQSka2o3EV2R/fu8IMf+HKvrAydRmQzlbvI7rrqKli61A/PiCQJlbvI7jr+eOjZEx58EJwLnUYEULmL7D4zv/Y+dSpMmhQ6jQigchdJjGHD/GkJfvvb0ElEAJW7SGLk5/u199dfh08/DZ1GROUukjA/+Qk0bw733hs6iYjKXSRhWreGiy+GceOgrCx0GslwKneRRPrv//Y/f//7sDkk46ncRRKpUyc491x49FE/16pIICp3kUT7n//xk3jcd1/oJJLBVO4iidarF5xzDvzhD7B8eeg0kqFU7iKN4eabYf16+N3vQieRDKVyF2kMPXvCkCHwxz/CsmWh00gGUrmLNJabb4aNG7XfuwShchdpLD16+D1nHnoIFi8OnUYyjMpdpDHdcgtUVcGtt4ZOIhlG5S7SmLp396clePRR+Pzz0Gkkg6jcRRrbTTf5ibR/+cvQSSSDqNxFGlvbtnDDDfC3v8G774ZOIxlC5S7SFK68Erp0gWuvhWg0dBrJACp3kabQrBncdRdMnw5jxoROIxlA5S7SVIYMgaOO8uee0YFN0shU7iJNxQz+9CcoL9fGVWl0KneRpvS97/lx97FjNZm2NCqVu0hTu+kmv3H18sv9AU4ijUDlLtLUCgr8CcU+/9xvZBVpBCp3kRBOOQXOOw/uuAMmTw6dRtKQyl0klD/+EfbcEy64wJ89UiSBVO4iobRq5TeszpoF110XOo2kGZW7SEjHHeePXh05Et56K3QaSSMqd5HQ7rnH7yJ57rlQVhY6jaQJlbtIaAUF8PLLfrfIM8+EysrQiSQNqNxFksG++8ITT/g9Z665JnQaSQMqd5Fkcfrp/rQEDz8Mo0eHTiMpTuUukkzuvBNOPBEuuwzeeCN0GklhKneRZJKdDS+8APvtB2edBVOnhk4kKUrlLpJsior8rE1t28LJJ8M334ROJClI5S6SjPbayw/LVFXBscfC7NmhE0mKUbmLJKuePeGdd2DdOjj6aPj669CJJIWo3EWS2QEHwIQJsGGDL/jS0tCJJEWo3EWSXd++MHGiH6I57DB4//3QiSQFqNxFUkGfPr7U99gDBg6E8eNDJ5Ikp3IXSRUlJfDhh3DQQX6y7Vtvhdra0KkkSancRVJJmzbw9tswbBjcdhuccAIsWhQ6lSQhlbtIqsnLg8cfhzFj/Jp8v37w97+HTiVJRuUukorMYMQImDIF2rWDk07ya/MrVoROJklC5S6Synr18gV/443w7LN+3/innoJoNHQyCUzlLpLqmjWD22+HadOgWzcYOtRvdJ04MXQyCUjlLpIu+vSBDz7wa+7LlsGAATBoELz3XuhkEoDKXSSdRCJw/vl+0u3f/MafVfKoo+CII+DVV7XrZAZRuYuko2bN/MQfc+b4ybfnzYPTTvPDNrffDgsXhk4ojUzlLpLOmjeHK6+Er77y54nfd1+4+Wbo1Mkf6TpqlPawSVMqd5FMkJPjJ99++21/8rEbboD58+HSS6F9ez90c889MGOG9rRJE+acC/LC/fv3d1OmTAny2iICOAf//je89JKfHORf//L3t23ry/7oo+GQQ/yJy/LywmaVzcxsqnOuf73LqdxFBPDj8G++Cf/8p7/MmePvz8nx0/716+d/9unj96dv394fTCVNKqHlbmaDgAeBLGC0c+6ebR7PA54EDgRWAOc45+bs6DlV7iJJbt48mDwZPvnE/5wxA5Yv3/J4ixZ+DL9bN+jaFYqLoXNn2Htv6NgRWrdW+TeChpZ7dgOeKAt4CDgemA9MNrNXnXOfxy32Y2CVc667mQ0GfgOcs2vRRaSxzJrlh9yzsvxekSUl0KPH9pebOLET0IljR5xBj3sA5/jq/SUsnzCDrlWzyJ3zJTUzZ9Hy42lkv/wXIjXVWz1HdSSXipbtqW3bnvLcdmwsbEtVUVu+WNKaFbWtKOi4B9ltW1G4VxE1zVvwxntFzFtVSFbLQpavz4dIhPz8LZ8PK1fCmjX+M6VLF2jZcsvtJUv8MVwFBXDMMXDkkX5X/3fe8fOcmMHGjX4noooKKC/3n0X9+vn3o1s3P/o0fbpftndvP7vhpvfmjTf8aXwOPdSf6WHLe+Svxy8b/z5v+/7WdX9jqHfN3cwOBW51zp0Yu/0rAOfc3XHLvBlb5kMzywYWA+3cDp5ca+4iTWvWLLjrLj9r3/TpvtgKC+H6679dQNddB59+6m/37u13mQf/+1lZvmjN/Onl586Fz/5dS/6qhXR08+nIAjpH5tM+uohOuUv4jltCG7ecPaLLaV27jELWNyjvBvLZQAEbKKCCZmwknwqaUUkeleRRRe7mn9XkbL5UkYvlZLOxOodqsqkli5rYz+1dnEWIkoWLRMAi1EQjFBRG6NY9wtnnGEuXGWOfMIhEiDpj+Aij/Z7GM88ac8sMh9Gps3HBUKPj3saChcYLLxoWMWqdcfZZ/svM/AXGc88bkSyjtPXBXH1b610q+IStuQMdgXlxt+cDB9e1jHOuxszWAG2A5fELmdklwCUAnTt3bsBLi0iibFpjz8vzxzrl5fnbpaVbl3tpqf8AKCryt9et2zK7X1aWH3kpK/Pl3revP05qQ2UW5bmdmFfViWgU8rL99tr8Zv61WrXya9DLl0N2tIqWbjUtWU373NXskbOWnA3lFLpymrOeQtZRyDry2Rir9g00o2LzJY9KCthAa1aSQzW5VJFL1eZ6z6WK7Ooasqkhh2qyqCXCDlZiNz0Uf3xXOTDNX/YCfh+/fOyD7hfx9y0BJvurHYFr4h+b4H/sDfw8dtfdJ02itPTIRl17b0i5J4xzbhQwCvyae1O+tkimKynxQzGVlX5vx8pKv620pOTbyxUWwuzZ/naXLluWqa31xV5Y6Mu9rMzvXLNwIaxa5ZeJxHawjkYhN9ffrqnxy2dnQ3V1Lsv4Dsv4DvMiUJgPa6qgeutRnQYx8x8i2yos9B8mm/bqNKLbXW+PECXHYj+zouRm1VJb42izR5T++9fyi2sd88ocv77NkR2J4qKOm2507N3R8cDvHaVfOgxHSXfHNVc7irs45sx2PD7GkRVx1Nb6k3cWd3HMneMY+7gjKwvmFfbkjJJv504kDcuIZJCdG3P31+PHk+PHjGHL9W++8ae0cc6fqHLx4i1j1126+KEb8K87caIfK+/Wzc89suee/v5XXvFj50VFsHat/32NuX9bwvaWiZX1l8BAYAH+y8e5zrmZcctcAfRxzl0W26B6hnPu7B09r8pdRGTnJWzMPTaG/lPgTfyukGOcczPN7NfAFOfcq8BjwFNm9hWwEhi8e/FFRGR3NGjM3Tn3OvD6NvfdHHe9AjgrsdFERGRX6dwyIiJpSOUuIpKGVO4iImlI5S4ikoZU7iIiaUjlLiKShoKdz93MlgFz4+5qyzbnopGt6P2pm96buum92bFUfH+6OOfa1bdQsHLflplNachRV5lK70/d9N7UTe/NjqXz+6NhGRGRNKRyFxFJQ8lU7qNCB0hyen/qpvembnpvdixt35+kGXMXEZHESaY1dxERSRCVu4hIGgpe7mbWycwmmtnnZjbTzK4OnSnZmFmWmf3LzF4LnSWZmFkrM3vRzP5jZl/EZg2TGDP7Wez/1GdmNs7MmoXOFIqZjTGzpWb2Wdx9rc3sbTMrjf3cI2TGRAte7kAN8HPnXC/gEOAKM+sVOFOyuRr4InSIJPQg8Hfn3HeBvug92szMOgJXAf2dc73xE+1k8iQ6Y4FB29x3HfCuc64EeDd2O20EL3fn3CLn3LTY9bX4/6Adw6ZKHma2N3AyMDp0lmRiZi2Bo/CzgOGcq3LOrQ6bKulkA/mxqTILgIWB8wTjnJuEnyUu3mnAE7HrTwA/bNJQjSx4ucczs2Jgf+DjsEmSygPAL4Fo6CBJpiuwDHg8NmQ12syahw6VLJxzC4DfAWXAImCNc+6tsKmSTnvn3KLY9cVA+5BhEi1pyt3MCoGXgGucc+Wh8yQDMzsFWOqcmxo6SxLKBg4A/uyc2x9YT5p9rd4dsfHj0/Afgh2A5mZ2fthUycv5fcLTar/wpCh3M8vBF/szzrmXQ+dJIocDp5rZHGA8MMDMng4bKWnMB+Y75zZ9y3sRX/biHQfMds4tc85VAy8DhwXOlGyWmNleALGfSwPnSajg5W5mhh83/cI5d3/oPMnEOfcr59wecQlnAAAAt0lEQVTezrli/MawCc45rX0BzrnFwDwz6xG7ayDwecBIyaYMOMTMCmL/xwaiDc7behUYFrs+DHglYJaEC17u+LXTC/BrpdNjlx+EDiUp4UrgGTObAfQD7gqcJ2nEvtG8CEwDPsX/X0/bQ+3rY2bjgA+BHmY238x+DNwDHG9mpfhvOveEzJhoOv2AiEgaSoY1dxERSTCVu4hIGlK5i4ikIZW7iEgaUrmLiKQhlbuISBpSuYuIpKH/BwvMdXWTThkTAAAAAElFTkSuQmCC\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# helper function for approximation\n", + "def sigmoid_mu_s(x, mu, s):\n", + " return 1 / (1 + np.exp(-(x + mu) / s))\n", + "\n", + "plt.scatter(X, Y, s=10, c='b', alpha=0.5)\n", + "sig_x = np.linspace(X.min(), X.max(), 100)\n", + "y_flip = 1 if Y[X.argmin()] == 0 else -1\n", + "sig_y = sigmoid_mu_s(sig_x, -X.mean(), y_flip * 0.5)\n", + "plt.plot(sig_x, sig_y, c='r')\n", + "plt.yticks([0, 0.5, 1])\n", + "plt.title('Approximation')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Hypothesis and Prediction\n", + "\n", + "We can tweak the sigmoid function by passing the dot product of $x$ (vector of feature) and $\\theta$ (the vector of weights).\n", + "\n", + "$$h(x) = \\sigma(x \\cdot \\theta) = \\sigma(\\theta_0 x_0 + \\theta_1 x_1 + \\cdots + \\theta_n x_n)$$\n", + "$x_0$ being equal to 1. \n", + "$\\theta$ is not a parameter of $h$ because it's considered as a constant.\n", + "\n", + "$h$ is our hypothesis function. It return a probability of an $x$ to belong to one of the group. \n", + "To predict some new $x$ we input it to $h$ with the weights and see if the returned value if above or below 0.5.\n", + "\n", + "$$\n", + "\\text{prediction} =\n", + " \\begin{cases} \n", + " 1, & h(x) \\ge 0.5 \\\\\n", + " 0, & h(x) \\lt 0.5\n", + " \\end{cases}\n", + "$$\n", + "\n", + "This is equivalent to:\n", + "\n", + "$$\n", + "\\text{prediction} =\n", + " \\begin{cases} \n", + " 1, & x \\cdot \\theta \\ge 0 \\\\\n", + " 0, & x \\cdot \\theta \\lt 0\n", + " \\end{cases}\n", + "$$\n", + "\n", + "Indeed, in the sigmoid funcion: \n", + "* $x$ is *above* 0 then $e^{-x}$ tend to be small, $\\frac{1}{1 + \\text{small number}} \\approx 1$. \n", + "* $x$ is *below* 0 then $e^{-x}$ tend to be big, $\\frac{1}{1 + \\text{big number}} \\approx 0$. \n", + "\n", + "Therefore, a prediction can be visualized by projecting $x$ onto $\\theta$. \n", + "It will be more relevant the more the two vectors go in the same direction\n", + "(or opposite direction for negative projection),\n", + "for exemple if $x \\cdot \\theta = 0$ then $x$ is perpendicular to $\\theta$,\n", + "$h(x) = 0.5$ right in the middle of the sigmoid function,\n", + "the prediction become more irelevant as $x \\cdot \\theta$ tend to 0." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "def hypothesis(x, theta):\n", + " # suppose that x_0 = 1 and isn't mentionned\n", + " return sigmoid(theta[0] + x.dot(theta[1:]))\n", + "\n", + "\n", + "def predict(x, theta):\n", + " return int(hypothesis(x, theta) >= 0.5) # a bit more clear\n", + " # or\n", + " return int(theta[0] + x.dot(theta[1:]) >= 0)\n", + " # there is a nice vector interpretation for this one\n", + " # and it probably compute faster" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cost function\n", + "\n", + "We have to evaluate our model by introducing a cost function, this one is called the *logloss*\n", + "\n", + "For a single $x$:\n", + "$$\n", + "j(h(x), y) = \n", + " \\begin{cases} \n", + " -\\ln(h(x)), & y = 1 \\\\\n", + " -\\ln(1 - h(x)), & y = 0 \n", + " \\end{cases}\n", + "$$\n", + "It can be rewritten as a single continuous function:\n", + "$$ j(h(x), y) = -y \\ln(h(x)) - (1 - y) \\ln(1 - h(x))$$\n", + "\n", + "Even if it may seem more complicated at first sight, it's just a switch with the values of $y$. \n", + "- if $y = 1$ then $- (1 - y) \\ln(1 - h(x)) = 0$ and the $-\\ln(h(x))$ stays.\n", + "- if $y = 0$ then $-y \\ln(h(x)) = 0$ and the $-\\ln(1 - h(x))$ stays." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 1.0, 'cost function')" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEICAYAAABCnX+uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd4XNW57/Hv0qj33qsly0XusokLGIwNpncIhE4CJ+dCThLSTs5NwiE3yeEEQhJCEkIPhAAJzUAwphlsDO62bLnIlmVZvfcuzaz7xx7JTbaKZ7SnvJ/nmWc0nj17v8oz/LK09ipKa40QQgj35WN2AUIIIc6MBLkQQrg5CXIhhHBzEuRCCOHmJMiFEMLNSZALIYSbkyAXHkMpdbVSqlwp1aGUmjuB171ZKfXBRF1PiBMpGUcu3IFS6lPgb1rrp09zzCHgfq31KifWkQkcBvy01gPOuo4QYyEtcuFJMoA9ZhchxESTIBdOoZRKU0q9oZSqV0o1KqUet/+7j1LqJ0qpI0qpOqXUC0qpCPt7gUqpv9mPb1FKbVFKJSilfgmcAzxu7zZ5/IRrBSilOgALUGBvmaOU0kqpnGOOe14p9Qv7z+cppSqUUt+z11GtlLrzmGODlFK/sdfZqpT6XCkVBKyzH9Jir2WRUuoOpdTnx3x2sb32Vvvz4mPe+1Qp9f+UUhuUUu1KqQ+UUrGO/V9feBsJcuFwSikL8C5wBMgEUoBX7G/fYX8sAyYBocBgMN8ORABpQAzwTaBba/1/gfXAfVrrUK31fcdeT2vdq7UOtb+crbXOHmWpifbrpQBfB/6olIqyv/cIkA8sBqKBHwI2YKn9/Uh7LV+e8LtHA/8CHrP/Do8C/1JKxRxz2NeAO4F4wB/4/ijrFWJYEuTCGc4CkoEfaK07tdY9WuvBFuvNwKNa6xKtdQfwY+BGpZQv0I8Rfjlaa6vWepvWus2JdfYDP9da92ut3wM6gClKKR/gLuDbWutKey1faK17R3HOS4GDWusXtdYDWuuXgf3A5ccc85zW+oDWuhv4BzDHsb+W8DYS5MIZ0oAjp7gZmIzRUh90BPAFEoAXgTXAK0qpKqXUr5VSfk6ss/GEGrsw/kKIBQKBQ+M454m/H/bXKce8rhnmmkKMmwS5cIZyIN3eyj5RFcZNyUHpwABQa28ZP6i1no7RpXEZcJv9uPEMr+oCgo95nTjKzzUAPcBwXTQj1XHi7wfG71g5ymsLMWYS5MIZNgPVwENKqRD7Tcwl9vdeBr6rlMpSSoUCvwJe1VoPKKWWKaVm2vvY2zC6Pmz2z9Vi9KmPxU7ga0opi1LqIuDc0XxIa20DngUeVUol2z+/SCkVANTbazpVLe8BuUqprymlfJVSXwWmY9wzEMIpJMiFw2mtrRh9wjlAGVABfNX+9rMYXSjrMMZj9wDfsr+XCLyGEeL7gM/sxwL8HrhOKdWslHpslKV8215HC0bf/Ftj+DW+D+wGtgBNwP8CPlrrLuCXwAb7yJqFx35Ia92I8ZfE94BGjJukl2mtG8ZwbSHGRCYECSGEm5MWuRBCuLnhbkadRClVCrQDVmBAaz3fmUUJIYQYvVEFud0y6ecTQgjXI10rQgjh5kZ1s1MpdRhoxhhD+xet9ZPDHHMPcA9ASEhIflZOLgdq20mPDiYiyJlzOoQQwg11NUJLGSTMAIsf27Zta9Bax43nVKMN8hStdaVSKh74EPiW1nrdqY6fP3++fvODdZzz67U8cv1srstPHU9tQgjhuTb9BVb/EH54GIKjUUptG+/9x1F1rWitK+3PdcCbGGtpnFaAr3Hq3gHreOoSQgjPNtBjPPsGnPGpRgxy+8y8sMGfgQuBwpE+F+BrAaCn3zbCkUII4YX6B4M88IxPNZpRKwnAm0qpweP/rrV+f6QPBfhJi1wIIU5poAd8/MDHcsanGjHItdYlwOyxnnioa0Va5Kbq7++noqKCnp4es0sRLiQwMJDU1FT8/GQggmkGeh3SGoexjSMfE6UU/r4+9EiL3FQVFRWEhYWRmZmJ/a8q4eW01jQ2NlJRUUFWVpbZ5XivgR6H9I+Dk8eRB/j6SIvcZD09PcTExEiIiyFKKWJiYuSvNLMN9DisRe7UIA/0s9A7IEFuNglxcSL5TriAgR7wc4MgD/D1kZudQggxHAf2kUvXihBCmMF9+sgt0iIXQojh9LtNH7mP9JELh7rrrruIj49nxowZTjl/aOjI+yB3d3dz7rnnYrUajZSPP/6YW2+9dcTjhtPX18fSpUsZGBgY9rXwYO5yszPA10JPv7TIhePccccdvP/+iPPRnOrZZ5/lmmuuwWIxJnIUFBQwd+7cEY8bjr+/P8uXL+fVV18d9rXwYG7TRy4tcgEUFhayePHiodfbt29n+fLl4zrX0qVLiY6OdlRpwyotLWXatGncfffd5OXlceGFF9Ld3T30/ksvvcSVV1459LqgoICamhqWLl1Keno6H3300bDHLVu2jA8//BCAn/zkJ3zrW8ZWpVdddRUvvfTS0HEnvhYeyoF95E6bEARys9PVPPjOHvZWtTn0nNOTw3ng8rzTHzN9OiUlJVitViwWC/fffz+PPvroccecc845tLe3n/TZRx55hBUrVji05tE4ePAgL7/8Mk899RQ33HADr7/+Orfccgt9fX2UlJSQmZk5dGxBQQE333wz69at48033+Sll15i6dKlJx334IMP8rOf/Yy6ujp27NjB22+/DcCMGTPYsmXL0HEnvhYeyoFdK04N8kA/i8zsFPj4+JCXl8eePXs4ePAgGRkZzJs377hj1q9f77Trr1ixgpqampP+/Ze//OVxLeZjZWVlMWfOHADy8/MpLS0FoKGhgcjIyKHj+vv7aWxs5Hvf+97Q68jIyJOOA+OvCa01jz76KJ9++ulQl4vFYsHf35/29nbCwsJOei08lAPHkUuL3IuM1HJ2poULF7Jhwwb+9Kc/DdvH7cwW+WBXx1gEBBz9k9disQx1rQQFBR03I3Lfvn3Mnj0bHx+jl3LXrl3MmDHjpOMAdu/eTXV1NTExMScFdG9vL4GBgad8LTyQ2/SRy/BDYbdw4UJ+8pOfcPXVV5OSknLS++vXr2fnzp0nPUYb4suXL6eystLRZZ8kKioKq9U6FNIFBQXMnn10Tbldu3Yxa9ask46rrq7m5ptvZtWqVYSGhh73f2aNjY3ExsYOLWB14mvhodxnHLnc7BSGqVOnEhAQwI9+9KMzOs9NN93EokWLKCoqIjU1lWeeeQabzUZxcbHTb4IOuvDCC/n8888BI8hnzZo19F5hYeHQ0MjB47q6urjmmmv4zW9+w7Rp0/jpT3/Kgw8+OPSZtWvXcumll57ytfBANitY+xzWIkdr7fBHfn6+1lrrh9/fr7P+811ts9m0MMfevXvNLkFrrfW9996rn3/+eaece/fu3fq73/2uU849nG3btulbbrnFYcddffXVuqio6JSvncVVvhteqbdT6wfCtV7/26F/ArbqcWau01vkNg391pH3BRWe6dChQ0ydOpXu7m5uv/12p1xjxowZJ42CcaZ58+axbNmy0070Ge1xfX19XHXVVeTm5g77WnioAcftDgROvtkZGWz08TV39ZEQLjduvFF2djb79+83uwyHu+uuuxxynL+/P7fddtspXwsP1dVoPAdFOeR0Tm2Rp0QFAVDR3D3CkUII4UVayoznyDSHnM6pQZ4caQR5ZYsEuRBCDGktN54jUh1yOue2yO1BXiVBLoQQR7VWgLJAWLJDTufUIA8L9CM80JdK6VoRQoijWsohPBksjrlN6dQgB0iJCpauFSGEOFZrOUQ4pn8cJiLIIwOla0UIIY7VWu6w/nGYkCAPkq4VIYQYZLNCW5XDRqzAhHStBNHeO0Brd7+zLyWEEK6vvRpsA+7VtZIsI1eEA73//vtMmTKFnJwcHnroIYeffzxbvTlq+7lTbRk33DWHc+w2cbJlnAtrrTCe3SnIB4cgSveKOFNWq5V7772X1atXs3fvXl5++WX27t074XWcuIWbo7afO9WWccNdczjHbhMnW8a5sBb7GHJ361oBmRTkzRy11dvmzZvJyclh0qRJ+Pv7c+ONN7Jq1SpHlgqMfas3R20/d6ot44a75mi2jZMt41xUq31WpwNvdjp1rRWA2JAA/H19JMhdwer/hJrdjj1n4ky4+PRdHI7a6q2yspK0tKOtmNTUVDZt2uSAX+JkY9nqzVGG2zJuxYoVw15zNNvGyZZxLqqlHIJjwD/EYad0epD7+CiSIwIlyL2YJ2/15qjrn2rLuFNdc7TbxsmWcS6otcKhrXGYgCAHo3tF+shdwAgtZ2dyxFZvKSkplJeXD71XUVEx7G5DJ3LmVm+jMZrrn2rLuFNdc7TbxsmWcS6otRxichx6Sqf3kYN9LLm0yL2aI7Z6W7BgAQcPHuTw4cP09fXxyiuvcMUVVwDmbfXmKKfaMm64a4522zjZMs4FaW10rUSmO/S0ExLkyZFB1Lf3yv6dXswRW735+vry+OOPs3LlSqZNm8YNN9xAXl6eqVu9wfDbz43V6baMO/aaY9k2TraMc0HdzdDf6fCuFadu9TboH1vKdMaP3tWH6zscuVmSGAVX2c7LG7d6M+Oax24TN9KWca7y3fAqVTuNLd72rDrpLVx1q7dBMgTRe3nzVm8Tfc1jt4mTLeNclBPGkMMYbnYqpSzAVqBSa33ZWC6SGhkMyKQgb+TtW71N5DWP3SZOtoxzUUMbSpjXR/5tYN94LpIYEYhS0iIXQni51grwDYJgx97PGVWQK6VSgUuBp8dzEX9fH+LDAiTIhRDeraXM6FZRyqGnHW2L/HfADwHbqQ5QSt2jlNqqlNpaX19/0vuynK15jPsoQhwl3wmTOHhDiUEjBrlS6jKgTmu97XTHaa2f1FrP11rPj4uLO+n95MggqlolyCdaYGAgjY2N8h+uGKK1prGxUSYKmcEJszphdDc7lwBXKKUuAQKBcKXU37TWt4zlQilRQXywpxabTePj49g/K8SppaamUlFRwXB/JQnvFRgYSGqq4wNFnEZ/N3TWO3zECowiyLXWPwZ+DKCUOg/4/lhDHCA1Mog+q436jl4SwqUlMFH8/PzIysoyuwwhxNA65I4dsQITNLMTZCy5EMLLtTpnDDmMMci11p+OdQz5oGTZYEII4c0GJwM5oY98wlrk6dHBKAXFdR0TdUkhhHAdDQfANxDCkh1+6gkL8mB/X7LjQimsbJ2oSwohhOuo2gkJM8Di+NXDJyzIAWamRLBbglwI4W1sNqgugOQ5Tjn9hAb5jJQI6tp7qWtz7FrOQgjh0ppKoK8dkjwgyGemRABIq1wI4V2qdxrPntAiz0sORykJciGEl6naAZYAiJvqlNNPaJCHBPgyKTZEbngKIbxLdQEkzgCLc7bdm9AgB7nhKYTwMoM3Op3UPw4mBPmMlAhq23qpa5cbnkIIL9B8GHrbnNY/Dia1yAHpXhFCeIeqHcazJ7XI81IijBueFW0TfWkhhJh41TuNG53x05x2iQkP8tAAX7JiQ6SfXAjhHap2QkKe0250gglBDkb3inStCCE8ns0G1buc2j8OJgZ5TVsP9e29ZlxeCCEmRvNh6G11av84mBTkM+SGpxDCGwze6PTEFnlecjggMzyFEB6ueidY/CHOeTc6waQgDwv0Y5Lc8BRCeLrBG52+/k69jClBDjA3PYotpU1YbbK7uxDCA/X3QMVWSF3g9EuZFuRLc2Np6eqXVrkQwjOVfQkD3ZC93OmXMi3Iz86JRSlYd6DerBKEEMJ5Dn0MPn6QebbTL2VakMeEBjAjOUKCXAjhmYo/gfSFEBDq9EuZFuRgdK/sKG+hraffzDKEEMKx2qqhbg/kOL9bBcwO8slxWG2aL4obzCxDCCEc69AnxvME9I+DyUE+LyOK0ABfPjsgQS6E8CCHPoaQeEiYMSGXMzXI/Sw+LM6OYd2BerSWYYhCCA9gs8KhtUa3is/ERKypQQ6wNDeOypZuSho6zS5FCCHOXPVO6G6asG4VcIEgPzc3DpBhiEIID1H8CaAge9mEXdL0IE+LDiYrNkSCXAjhGQ59DEmzISR2wi5pepADLJ0cy8aSJnoHrGaXIoQQ49fTCuWbJ2zY4SCXCPLzpsbT3W9lvYxeEUK4swNrQFth8oUTelmXCPKzc2KJDvHnrZ2VZpcihBDjt+sfEJEGqWdN6GVdIsj9LD5cOjOJj/bV0tE7YHY5Qggxdh31xkSgmddN2LDDQS4R5ABXzU2mp9/GmsIas0sRQoix2/Om0a0y84YJv7TLBPm89CjSooOke0UI4Z52/8OYyZkwfcIvPWKQK6UClVKblVIFSqk9SqkHnVGIUoorZ6ewobiBuvYeZ1xCCCGco6kEKrbAzOtNufxoWuS9wPla69nAHOAipdRCZxRz1dxkbBreLah2xumFEMI5dr8GKKN/3AQjBrk2dNhf+tkfTlkYJSc+jLzkcFZJ94oQwl1obYxWyVgCEammlDCqPnKllEUptROoAz7UWm8a5ph7lFJblVJb6+vHP0vzqjkpFFS0UlLfMfLBQghhtuqd0HgQZpnTrQKjDHKttVVrPQdIBc5SSp20NqPW+kmt9Xyt9fy4uLhxF3T57GSUgje2S6tcCOEGdr5sbOk2/UrTShjTqBWtdQuwFrjIOeVAYkQg50+J5+XNZTJlXwjh2nrbYeffjRAPijKtjNGMWolTSkXafw4CLgD2O7OoO5Zk0tjZxzty01MI4cp2/h362mHhv5taxmha5EnAWqXULmALRh/5u84s6uycWCbHh/LchsOy4YQQwjXZbLDpL5AyH1Lnm1rKaEat7NJaz9Vaz9Jaz9Ba/9zZRSmluGNJJnuq2thS2uzsywkhxNgVfwhNh0xvjYMLzew80TVzU4kI8uO5DYfNLkUIIU628c8QlmTqTc5BLhvkQf4WbjwrjTV7aqho7jK7HCGEOKpuP5SshQVfB4uf2dW4bpAD3LYoE6UUL355xOxShBDiqE1PgCUA8u80uxLAxYM8JTKIi/IS+fvmMlq7+80uRwghoL0WCl4xJgBN4HZup+PSQQ7wf5Zl094zwDPrS8wuRQgh4PPfgrUPzr7f7EqGuHyQ5yVHcPGMRJ7dUEpzZ5/Z5QghvFlbFWx9FmbfBDHZZlczxOWDHOC7F+TS2TfAk9IqF0KYaf1vjM0jzv2B2ZUcxy2CPDchjMtnJfP8hlIaOnrNLkcI4Y1aymDbX2HurRCVaXY1x3GLIAf49orJ9A5YeeLTQ2aXIoTwRuseBqVg6ffNruQkbhPk2XGhXD03lRc3HqG2TXYQEkJMoKYS2PES5N9h2prjp+M2QQ7wnRWT0Rr+932nrtklhBDH++Cn4BvgUiNVjuVWQZ4WHcw3zsnije2VbDvSZHY5QghvUPwx7H/X6FIJTzK7mmG5VZAD3Lssh8TwQB54ew9Wm6yMKIRwooE+WP0jiJ4Ei+4zu5pTcrsgDwnw5b8unUZhZRuvbik3uxwhhCfb9ISxjdtFDxldKy7K7YIc4PJZSZyVFc3Da/bT0iWThIQQTtBeA5/9L0xeCbkrza7mtNwyyJVS/PflebR29/PwmiKzyxFCeKIPfmpMxb/of8yuZERuGeQA05PDuWNxFi9tKuOLQw1mlyOE8CRFq2H3P+Ds77rUVPxTcdsgB/jByilkxgTzw9d20dk7YHY5QghP0NUE73wbEmbAOa43+Wc4bh3kQf4WHr5+NpUt3Ty0WsaWCyEc4P0fQ2cDXPUn8PU3u5pRcesgB1iQGc2di7N4ceMR6WIRQpyZotWw6xU453uQNNvsakbN7YMcju9iaeuRDSiEEOPQ2Xi0S2Wpa61uOBKPCPIgfwu/uWE21a09/PiN3WgtE4WEEGNgs8Gb/wbdzXDVn92mS2WQRwQ5QH5GNN+7MJd/7arm75vLzC5HCOFOvvwDFH8IK38FSbPMrmbMPCbIAb65NJuluXE8+M5e9la1mV2OEMIdlG+Gjx6E6VfCgm+YXc24eFSQ+/goHr1hNpFBftz39+10yJBEIcTpdDXBa3cZS9Ne8QdjvXE35FFBDhAbGsBjN82ltLGTH/yzAJssrCWEGI51AF7/hjEV//rnIDDC7IrGzeOCHGDhpBj+65JprC6s4bFPDppdjhDCFX30ABz6GC79DaTkm13NGfE1uwBn+frZWeyvaed3Hx1kSkIYF890zXWEhRAm2PESfPk4nPVvkH+72dWcMY9skYOxsNYvr57BvPRI7v9HAXuqWs0uSQjhCso3w7vfgUnnGaNUPIDHBjlAgK+FJ27NJzLYj68/v5Wqlm6zSxJCmKnxELzyNQhPgeueA4tndEp4dJADxIcF8sztC+jsHeD2ZzfL+uVCeKuOOvjbNWCzws3/hOBosytyGI8PcjCWvP3LbfkcaeziG3/dSk+/1eyShBATqbcdXrrOCPOb/wmxk82uyKG8IsgBFmfH8uhXZ7OtrJlvvbyDAavN7JKEEBNhoBdevQVqCuH6v0LqfLMrcjivCXKAy2Yl88Bl0/lwby3f/2eBbN4shKez9sM/74SST40JP7kXml2RU3hGT/8Y3LEki84+Kw+vKcLX4sOvr52Fj497zuYSQpyGdQBe/zoU/QsueQTm3mx2RU4zYpArpdKAF4AEQANPaq1/7+zCnOneZTkMWDW//egAvj6KX109U8JcCE9is8Kb98DeVcYQw7PuNrsipxpNi3wA+J7WertSKgzYppT6UGu918m1OdV/LM+h32rj8bXFKAW/uGomFglzIdyftR/e+ncofB1WPAiL7jW7IqcbMci11tVAtf3ndqXUPiAFcOsgV0rxvQtz0Wj+uPYQXX1WHrl+Nn4Wr7ptIIRn6e8xFsEq+hcsfwDO/o7ZFU2IMfWRK6UygbnApmHeuwe4ByA9Pd0BpTmfUoofrJxKsL8vD68poqvPyh9umkugn8Xs0oQQY9XXaUz2KfkULn4YvnKP2RVNmFE3P5VSocDrwHe01ict9q21flJrPV9rPT8uLs6RNTrdvctyePCKPD7cW8vX/7qFdtkuTgj30tUEL14Nh9fBlX/yqhCHUQa5UsoPI8Rf0lq/4dySzHH74kx+c/1sNpY0ccNfNlLb1mN2SUKI0WguhWcuhKodxrR7Dx6dciojBrlSSgHPAPu01o86vyTzXJufyrN3LKCssZOr/7iBA7XtZpckhDidqh3w9AXQWQe3vgV5V5ldkSlG0yJfAtwKnK+U2ml/XOLkukxzbm4cr/7bIvptmmv//AUbihvMLkkIMZyi9+G5S8E3AO76ADKXmF2RaUYMcq3151prpbWepbWeY3+8NxHFmWVGSgRv/PtikiICue3ZzbzwZanZJQkhBmkNn/8OXr4RYnPg6x9C/FSzqzKVjLU7hbToYF7/98WclxvHz1bt4f++uZt+WZ9FCHP198Cb3zR298m7Cu58H8Jl0xgJ8tMIC/Tjydvm881zs3lpUxk3P7WJuna5CSqEKVrK4flLYNcrsOwnxo1N/2Czq3IJEuQjsPgo/vPiqfz+xjnsqmzh0sc+Z/PhJrPLEsK7FH8Mf1kK9Qfgq3+Dc3/gtjveO4ME+ShdOSeFt+5dQmiALzc9tZGn15egtayeKIRT2Wzw2cPwt2shLBHu+RSmXW52VS5HgnwMpiaGs+q+JayYFs8v/rWPu1/YSlOn7DgkhFO018Lfroa1v4CZ18E3PjJuboqTSJCPUXigH0/cks8Dl09n3YEGLvn9ejaWNJpdlhCepfgjeGIJlG2Cyx+Da54C/xCzq3JZEuTjoJTiziVZvPF/FhPkb+FrT23k4TX76RuQUS1CnJH+Hnj/v4yulJA4oysl/3bpDx+BBPkZmJESwbvfOptr56Xyx7WHuPpPMhtUiHGrLoAnz4WNf4QFd8Pdn3j9+PDRkiA/QyEBvjx8/Wz+cms+1a09XPaHz3l6fYlsIyfEaFn7Yd3D8NRy6G6BW16HSx8BvyCzK3MbXrfVm7OszEtkXnoUP35jF7/41z7e213Nr6+bTU58qNmlCeG6anbDqnuN1nje1XDpoxAcbXZVbkda5A4UFxbAU7fN53dfnUNJQyeXPLaeP31aLDNChTjRQC+s/R948jxoq4IbXoDrn5cQHydpkTuYUoqr5qawJCeWn60q5NfvF/H2zir+55qZzE2PMrs8IcxXugHe/Q40HIBZX4WLHpIAP0PSIneSuLAA/nxLPn+5NZ+Wrn6u+fMX/PStQtpk0wrhrbqajG6U5y+BgR64+TW45kkJcQeQFrmTrcxLZElOLI+sKeKvX5ayurCGH188lWvmpaBkSJXwBjYrbP8rfPxz6GmDJd+Bc38k66Q4kHLGNPP58+frrVu3Ovy87m53RSs/XVXIzvIW5mdE8eCVeeQlR5hdlhDOU7EV3vu+sQFExhK45GFIyDO7KpeklNqmtZ4/rs9KkE8sm03z2rYKHnp/P81dfdy4II37L5hCXFiA2aUJ4ThtVfDRg8ZKhaGJcKF9mr38FXpKZxLk0rUywXx8FDcsSGNlXiKPfXKQv35RyjsF1dx3fg53LM4k0M9idolCjF9fJ3zxOGz4HdgG4Ozvwtn3Q2C42ZV5NGmRm6ykvoNfvbePj/bVkRIZxPdX5nLl7BR8fKTlItyIdQB2vgRrfwUdNTD9Srjg5xCVaXZlbkO6VjzAF8UN/Gr1Pgor25ieFM6PLp7K0smxckNUuDatoWg1fPwg1O+H1AVwwf+DjEVmV+Z2JMg9hM2meWdXFQ+vKaKiuZuvZEXzw4umkJ8hw7OECzq8zhiJUrEFoifBiv+GaVdIP/g4SZB7mN4BK69sLucPnxTT0NHLsilx3H/BFGamyggX4QLKN8PaX0LJpxCWDOf9CObcDBY/sytzaxLkHqqrb4DnNpTy5LoSWrv7WTEtge+smMyMFAl0YYKKrUYf+KGPITjGuIm54BvgF2h2ZR5BgtzDtfX08/yGUp5eX0JbzwArpsVz3/mTmZMWaXZpwhsc+QLWPXI0wBf/B5x1t2z04GAS5F5iMNCf3XCYlq5+zpkcy73LcvhKVrTcFBWOpTWUrDUC/MgGY5OHRfca64QHyIqeziBB7mU6egd4aeMRnlpfQkNHH3PTI/nmudlcMC1Bhi2KM2Ozwt5V8PnWLjfhAAATy0lEQVRvoWaX0Qe+5Nsw7zaZUu9kEuReqqffyj+3lvPk+hLKm7rJjgvh7nMmcdXcFJlYJMamr8sYB/7lH6H5MMTkGAE+66vgK7OOJ4IEuZcbsNp4r7CGJz49xN7qNmJD/bltUSa3LMwgOsTf7PKEK2uvhS1PwZZnoLsJUvKNRa2mXgo+0hiYSBLkAgCtNV8eauSp9SWsLaonwNeHq+akcOfZmUxNlCnS4hhVO2DjE1D4ujGVfuqlsOg+SF8o48BNImutCMDY1GJxTiyLc2I5WNvOc1+U8sb2Cl7dWs7i7BhuW5TJimnx+FpkGXqvZO2HfW/D5qeg7EvwD4X5d8JZ/waxOWZXJ86AtMg9XEtXHy9vLufFL0upau0hOSKQmxdmcMP8NFlx0Vu0VcH2F2Drc8Y6KFGZxuiTebdCoMxJcBXStSJGNGC18fH+Ol74spQNxY34WRQr8xK5ZWGGDF/0RDYbHP7U6PsuWg3aCjkXwFn3QM4K8JG/ylyNBLkYk+K6Dv6+qYzXtpXT1jPApLgQblqQzjXzUogJlVa6W2uvhZ1/g+0vGqNPgqJh7i2QfwfEZJtdnTgNCXIxLt19Vt7dVcXLm8vYXtaCn0VxYV4iX52fxpKcWCwyJt09WAeMWZfbX4AD7xs3LzPOhvzbjeVkZfigW5AgF2esqKadV7aU8eaOSlq6+kmJDOLa/FSum5dKeoxMBHFJDcVG63vny0bfd3AszLkJ5t0OsZPNrk6MkVODXCn1LHAZUKe1njGak0qQu6/eASsf7q3lH1srWH+wHq3hrMxors1P4ZKZSYQFygp3pupugT1vwM6/G8vHKgtMvtDoPsldKSsQujFnB/lSoAN4QYLcu1S1dPPmjkpe31ZBSUMngX4+XDA9kavnJnPO5Dj8ZBjjxBjog+IPoeAVo+vE2gdx02DO12DWDRCWaHaFwgGc3rWilMoE3pUg905aa7aXtfDmjgre3VVNS1c/MSH+XDYriSvmpDAvPVJGvTiazWaM9d79T9j7FnQ3G10nM68zps0nz5WJOx7GJYJcKXUPcA9Aenp6/pEjR8ZTj3BxfQM2Pi2q462dlXy0r46+ARtp0UFcMTuZy2YlMzUxTEJ9vLSG6gJjtmXhG9BWAX7BMOUSI7yzl0nXiQdziSA/lrTIvUN7Tz9r9tSyamclXxxqxGrT5MSHctmsJC6blUROfJjZJbo+raF2j9HqLnwDmg6Bjy9knw8zb4ApF8uysV5CglyYrrGjl9WFNbxTUMXm0ia0htyEUC6ZmcQlM5OYHB8qLfVBWkNtobFc7J63oPEgKB/IPAdmXAvTLodg2afV20iQC5dS19bD6sIa/rW7mi32UJ8UF8IlM5K4aEYiecnh3hfqWkPldmOtk72rjMk6ygcyz4a8q2Hq5RAaZ3aVwkTOHrXyMnAeEAvUAg9orZ853WckyMWgurYe1uypYXVhDRtLGrFpSIkMYmVeIivzEpifGe25E4+s/cY2afvfhX3vQnuV0W2StdTYbX7qZRLeYohMCBJuobGjl4/31bFmTw3rixvoG7ARHeLP+VPjuWB6AudMjiXY380X5Oxth+KPoeg9OLAGelrANwhylhvBnbtSuk3EsCTIhdvp6B3gs6J6Ptxbwyf762jrGcDf14cl2TGsmJ7A8qkJJEa4ye7sLeXG+O6i1VC63hjnHRQNuRfB1Esge7lskyZGJEEu3Fq/1cbmw018tK+Wj/bVUt7UDUBecjjLp8azbGo8s1MjXWc/UpvVmFV5YI3xqNtj/Ht0tjHKZMrFkLYQLG7+14WYUBLkwmNorTlY18HH++r4ZH8t2440Y9MQE+LPublxnDc1nqWTY4kMnuAt7DobjC6Tgx8YC1R1NxvT4zMWG90luRfJ+ibijEiQC4/V3NnHuoP1rN1fx2cH6mnu6sdHwey0SM7NjePc3DhmpUY6/oapdQAqt0LxR8ajaiegISTOWNd78gqjyyQo0rHXFV5Lglx4BatNU1DRwmdF9Xx2oJ6Ciha0hshgP5bkxLJ0ciznTI4jOTJofBdoPgKHPjFa3CXroLfVGCKYepaxGUPOckiaI5syCKeQIBdeqamzj8+LG1h3oJ71B+upbesFIDsuhHMmx3F2TixfmRR96hUbu1ug9HMoWWsEeFOJ8e/hqZBzvjG7ctJ5EBQ1Ib+P8G4S5MLraa05UNvB+oP1rD/YwKbDjfT027D4KOakRbIkJ5YlmaHMpQj/svVQ8hlUbQdtA78QyDoHJi0z1jOJzZUFqcSEkyAX4gQ9/VZ2lNZxuOBzdMlnZHVsJ18dIED1Y8WHuvCZMOlc4mavxDftLPCd4JunQpzgTIJcxkcJz2Hth6odUPo5gaWfs6hsI4v6O423EmdQEXkLn1un84/6NArqbFAHoTs6WZC5k4WTYlg4KYa85HB8ZZ114WYkyIX7Gug11i858jmUboDyzWAP7qGNF7KWQubZWIKjyQAygJuBho5eNpY08uWhRjaWNLK2qB6A0ABf8jOi+MqkaL6SFcPMlAj8fSXYhWuTrhXhPnrbjbAu+9JYw6RiK1iNG5zE5xljujPPhowlY17DpK69h00lTWwsaWTz4SYO1nUAEOjnw7z0KBZkRnNWVjRz0yPdfxkB4ZKkj1x4prZqKN8IZZug7Auo2W3cnFQWSJplBHb6IiPAHbx+SUNHL1sON7HpcBNbSpvYW92G1mDxUcxIDmdBZjTzM6PIz4gmLkx2qRdnToJcuD+bFer2Qvkme6t7I7TYd5nyDYLU+UZopy+EtLMgYGI3rWjr6WfbkWa2ljaxpbSZneUt9A3YAMiMCSY/I5r8jCjyM6KYHB/qOssJCLchQS7cT3czVGwzgrtis/FzX7vxXmgCpH3FHtoLjda3i21x1jtgpbCyjW1Hmtha2sy2I800dvYBEBboy9z0KOalR5KfEcXstEjCTzWWXQg7CXLh2qwDRmu7YgtUbjNa3I0HjfeUDyTkGbMnB1vbkRluN45ba82Rxi6j1X6kmR1lzRTVtqO18avkxIUyNz2SuelRzE2PZHJ8mOeuwy7GRYJcuA6tobXCCOzKrUZLu3on9HcZ7wfHGKGdtgBSF0DyPI/dk7K9p5+C8lZ2lDWzvayZHeUttHT1AxDib2FmagRz0qKYkxbJnLRI91m2VziFBLkwT2ejMXa7arsxFLByG3TWGe9Z/CFxphHYKfONfu6oTLdrbTuK1prSxi52lBl97DvKWthf00a/1fhvMCE8gFmpRqjPSo1gVkokEcHSJeMtZEKQmBg9rVBdYAR35XbjefCGJMqY2p6zAlLmQUo+JMyQGZPHUEqRFRtCVmwI18xLBYwZqPuq29hZ3sKuilYKylv4cG/t0GcyYoKZlRrJ7NQIZqQYj9AA+c9WHE++EWJ43S1Qs8tYvrV6p/HcdOjo+5HpRrfI/LuM4E6aA4Hh5tXrpgL9LPZ+86MLc7V297O7opVdlS3sKm9lW2kT7xRUAcYfM1mxIcxMiWCmPdjzksNPvTCY8ArStSKMTROqC45/NB8++n5EGiTNhuQ5RngnzYGQGPPq9UINHb3srmxlV3kruytbKaxspaatZ+j9rNgQo8WeHE5eshHuUSHy15A7kT5yMTpaG10h1buMyTU1u4yf26uOHhOZbgR18hxItId3SKx5NYtTqm/vpdAe6oVVrRRWtlHZ0j30fkpkENOSwslLNh7Tk8NJiQxCeek9ClcnfeTiZP09UL8PagqhttAe3IXGZglgDPuLzTWmtCfNMlrciTNl7W03EhcWwDL7nqaDmjv72FPVxp6q1qHnj/fXMtheiwjyY3pSONOSwpmWFMa0pHAmJ4QS4Gsx6bcQjiAtcnenNbTXGGFdW3g0uBsOgrYax/iFQMJ0SJxlhHXiTIifLju7e4muvgH2Vbezr7qNvdVt7Klqo6imjZ5+Y2aqr48iOy50KNinJYUzNSmMuNAAab1PIGmRe4veDqjfD7V7jAk2tXuMR3fT0WMi0ozRIlMvOxraUVmyPZkXC/b3HVo+YJDVpilt7GRvVRv7qo3HxpIm3tp5tJstJsSfqUlhTEkwgn1qYhiT48MI8pfWu6uRFrkrsvZDY7E9rPdC3T6o2wPNpUeP8QuB+GlGSzthpjE7MmG6dI2IM9Lc2cf+mnb21xjhXlTTTlFt+1DrXSnIiA5mSmIYUxLDmZIQxpTEUDJjQmQd9zMkLXJ3ZbNC02GjL7vumEdjMdiMGYAoC8TkQPJcmHOz0SWSMB0iM6WVLRwuKsSfRdkxLMo+OirJatOUNXVRVNPG/pp2I9xr2vlwby02ezvQ3+LDpLgQchPCyE0ItT+HkRYdLEsRTABpkU8E64AxnK9+v/1RBHX7oeHA0fW0wVhjJH6a8Yizt7Zjc8FXlkkVrqen30pxXQdFNe0cqGvnQE07B2o7jhs5E+jnQ3acEew58aFMjg+VgD8FaZG7iv4eozXdUAT1B+zPRca/WfuOHheRDvFTIfs8I7Djp0LsFI9dc0R4pkA/y9Bs02N19A5wsLadg7UdHKht50BdB5tKGnlzR+XQMf6+PkyKDWFyQhg5caHkxBuPzNhgGUEzDhLk49HdbIwKqS8yWtUNB4yfW44YGx8AoCAqwwjoyRdAnD2s4ySwhWcLDfA9abYqGIuIHazroNj+OFDbzo6y5qFZq2Bs3JEeHUy2Pdyz40KM5/hQWQr4NCTIT8VmNYK5odgI6saDRng3HIDO+qPHWfyNPuyk2TDzeiOoY3MhdjL4BZlXvxAuJizQj3npUcw7IeC7+gYoqe8cCvhD9cbzZwfqhhYUA2PcfHZcCNlxoUyKCx36OTkyyOu7abw7yLWGriYjpBuLjaBuLDYeTSXHd4cERRvhnHuRPahzIS7X6Nf2kT8FhRivYH/fYbtoBqw2ypu7h8L9kP353V3VtHb3Dx3n7+tDVkwIk+KMBckmxYWSFRtCdlwIkcHesUyBdwR5b4cRzI3F0HjIWPxpMLR7Wo4e5+MH0VkQMxlyVxrPg61rB+8JKYQ4PV+Lz9BqkReQMPTvWmuaOvsoaejkUF0HJQ2dlNR3DI2kGbAdbcVHBfsNBfuxj8yYEI8aD+85o1b6uoyRIYNB3XjIHt6HoKPm+GPDkiE2x+gSic42gjomx2hdW7zj/9uE8ET9VhvlTV0cbuikpL6TkoZODjd0UFLfSV1773HHJoYHkhkbPBTsmfaQT48OJtBv4kPee0at9HYYYd1UcjSkm+yvj134CSAkzgjpnOUQPQlisu3BPQn8Q8ypXwjhVH4WHybZ+9CXTzv+vY7eAUobOjnc0Gk8NxrPa/bU0tR5tBtVKUgKDyQjJoTM2GDj2f5zenQwwf6uF5uuVZHW0NVohHPz4eOfm0qO7jwzKCTOmH4+6VwjtKOzjMCOzpa1sYUQxwkNGL4vHow14EsbOilt7KS0oYsjjUbQnxjyAPFhAWTEGAGfER1M+jE/Rwb7mbI+zaiCXCl1EfB7wAI8rbV+aNxXtPZDa7kx3by51B7WpUZgNx+B3rbjjw9LNgI6d6XRmo7OMp6jsiSshRAOERHkx+y0SGanRZ70Xmt3P2WNXZQ2dlLW1DUU+OsO1J/UXRMW6GuEfHQIadHBZMQYrfj06GCSIgKdtozBiEGulLIAfwQuACqALUqpt7XWe0/5IduAselu82FjCN9gaDcfMTbmHVyVD4zhe5EZxl6O6YuMgI7KtId1hgzhE0KYKiLIj5mpEcxMPbkl391npazJaMGXNXXZf+5ib3UbH+ytOW74pMVHkRIZRHp0MGn2cE+LDhoK+jMxmhb5WUCx1roEQCn1CnAlcOogr9kNT59/9HVInBHOqQtg1g3Gz4OPsGRZM0QI4ZaC/C32BcTCTnrPatNUt3ZT1tRF+TEhX97czZo9NSd12ZyJ0QR5ClB+zOsK4CsnHqSUuge4x/6yVz3YVnj03Tbg0IkfEcIMsUCD2UUIMYwp4/2gw252aq2fBJ4EUEptHe8wGiGcSb6bwlUppcY9Zns0fRqVQNoxr1Pt/yaEEMIFjCbItwCTlVJZSil/4EbgbeeWJYQQYrRG7FrRWg8ope4D1mAMP3xWa71nhI896YjihHAC+W4KVzXu76ZTpugLIYSYODLuTwgh3JwEuRBCuLlxB7lS6iKlVJFSqlgp9Z/DvB+glHrV/v4mpVTmmRQqxFiM4vt5h1KqXim10/74hhl1Cu+jlHpWKVWnlCo8xftKKfWY/bu7Syk1b6RzjivIj5m2fzEwHbhJKTX9hMO+DjRrrXOA3wL/O55rCTFWo/x+AryqtZ5jfzw9oUUKb/Y8cNFp3r8YmGx/3AP8eaQTjrdFPjRtX2vdBwxO2z/WlcBf7T+/BixXZiwLJrzRaL6fQphCa70OaDrNIVcCL2jDRiBSKZV0unOON8iHm7afcqpjtNYDQCsQM87rCTEWo/l+Alxr/9P1NaVU2jDvC2GG0X5/h8jNTuGt3gEytdazgA85+tejEG5nvEE+mmn7Q8copXyBCKBxnNcTYixG/H5qrRu11oOLST8N5E9QbUKMZMzLoow3yEczbf9t4Hb7z9cBn2iZfSQmxojfzxP6HK8A9k1gfUKcztvAbfbRKwuBVq119ek+MK7VD081bV8p9XNgq9b6beAZ4EWlVDFGx/6N47mWEGM1yu/nfyilrgAGML6fd5hWsPAqSqmXgfOAWKVUBfAA4AegtX4CeA+4BCgGuoA7RzynNJKFEMK9yc1OIYRwcxLkQgjh5iTIhRDCzUmQCyGEm5MgF0IINydBLoQQbk6CXAgh3Nz/B+pXIWE2XuOkAAAAAElFTkSuQmCC\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "xs = np.linspace(0.001, 0.999, 100)\n", + "ys_0 = [-np.log(1 - x) for x in xs]\n", + "ys_1 = [-np.log(x) for x in xs]\n", + "plt.plot(xs, ys_1, label=r'$y = 1, -\\ln(h(x))$')\n", + "plt.plot(xs, ys_0, label=r'$y = 0, -\\ln(1 - h(x))$')\n", + "plt.ylim([0, 5])\n", + "plt.xlim([0, 1])\n", + "plt.xticks([0, 0.5, 1])\n", + "plt.legend()\n", + "plt.title('cost function')" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "def logloss(x, y, theta):\n", + " if y == 1:\n", + " return -np.log(hypothesis(x, theta))\n", + " elif y == 0:\n", + " return -np.log(1 - hypothesis(x, theta))\n", + " # or (less obvious)\n", + " return -y * np.log(hypothesis(x, theta)) - (1 - y) * np.log(1 - hypothesis(x, theta))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that if $y$ was suppose to be 0 but is close to 1, the cost will be high. \n", + "Same logic if $y$ was suppose to be 1 but is close to 0.\n", + "\n", + "Then we generalize the cost function for all $x$:\n", + "$$J(\\theta) = \\frac{1}{n} \\sum_{i = 1}^{n} j(h(x_i), y_i)$$\n", + "$n$ is the number of elements in $X$ and $Y$." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "def cost(xs, ys, theta):\n", + " return sum([logloss(x, y, theta) for x, y in zip(xs, ys)]) / len(xs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optimization\n", + "\n", + "### Gradient descent\n", + "\n", + "Finally, use gradient descent to minimize the cost function:\n", + "\n", + "$$\\theta_i := \\theta_i - \\alpha \\frac{\\partial}{\\partial \\theta_i} J(\\theta)\\ \\text{ for } i = 1, ..., n$$\n", + "\n", + "Or with gradient notation:\n", + "\n", + "$$ \\theta := \\theta - \\alpha \\nabla J(\\theta)$$\n", + "\n", + "\n", + "$\\alpha$ is the learning rate (how fast we want the step to be). \n", + "Since $\\theta_i$ is only used in the dot product, all of them will have similar partial derivative.\n", + "\n", + "let $z = x \\cdot \\theta_j$\n", + "\n", + "$\\newcommand{\\pt}{\\frac{\\partial}{\\partial \\theta_j}}$\n", + "$$\n", + "\\begin{align}\n", + " & \\pt J(\\theta) \\\\\n", + " = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n} j(h(x_i, \\theta), y_i) \\right] \\\\\n", + " = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n", + " (-y_i \\ln(h(x_i, \\theta)) - (1 - y_i) \\ln(1 - h(x_i, \\theta))) \\right] \\\\\n", + " = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n", + " (-y_i \\ln(\\sigma(z)) - (1 - y_i) \\ln(1 - \\sigma(z))) \\right] \\\\\n", + " = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n", + " (-y_i \\ln(\\frac{1}{1 + e^{-z}}) -\n", + " (1 - y_i) \\ln(1 - \\frac{1}{1 + e^{-z}})) \\right] \\\\\n", + " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\pt \\left[ \n", + " -y_i \\ln(\\frac{1}{1 + e^{-z}}) -\n", + " (1 - y_i) \\ln(1 - \\frac{1}{1 + e^{-z}}) \\right] \\\\\n", + " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", + " -y_i \\pt \\left[ \\ln(\\frac{1}{1 + e^{-z}}) \\right] -\n", + " (1 - y_i) \\pt \\left[ \\ln(1 - \\frac{1}{1 + e^{-z}}) \\right] \\right) \\\\\n", + " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", + " -y_i(1 + e^{-z}) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n", + " - (1 - y_i) \\frac{1}{1 - \\frac{1}{1 + e^{-z}}} \\pt \\left[ 1 - \\frac{1}{1 + e^{-z}} \\right]\n", + " \\right) \\\\\n", + " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", + " -y_i(1 + e^{-z}) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n", + " - (1 - y_i) \\frac{1 + e^{-z}}{e^{-z}} (-1) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n", + " \\right) \\\\\n", + " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", + " -y_i(1 + e^{-z}) \\frac{e^{-z}}{(1 + e^{-z})^2} \\pt z\n", + " - (1 - y_i) \\frac{1 + e^{-z}}{e^{-z}} (-1) \\frac{e^{-z}}{(1 + e^{-z})^2} \\pt z\n", + " \\right) \\\\\n", + " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", + " -y_i \\frac{e^{-z}}{1 + e^{-z}} x_{ij}\n", + " + (1 - y_i) \\frac{1}{1 + e^{-z}} x_{ij}\n", + " \\right) \\\\\n", + " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", + " \\frac{-y_i e^{-z}}{1 + e^{-z}}\n", + " + \\frac{(1 - y_i)}{1 + e^{-z}} \\right) x_{ij} \\\\\n", + " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", + " \\frac{-y_i e^{-z} + 1 - y_i}{1 + e^{-z}} \\right) x_{ij} \\\\\n", + "\\end{align}\n", + "$$\n", + "\n", + "$$\n", + "\\boxed{\n", + " = \\frac{1}{n} \\sum_{i = 1}^{n} \\left( (h(x_i) - y_i) x_{ij} \\right)\n", + "}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "def cost_partial_theta(xs, ys, theta, theta_j):\n", + " total = 0\n", + " for x_i, y_i in zip(xs, ys):\n", + " temp = hypothesis(x_i, theta) - y_i\n", + " if theta_j != 0:\n", + " temp *= x_i[theta_j - 1]\n", + " total += temp\n", + " return total / len(xs)\n", + "\n", + "\n", + "def cost_gradient(xs, ys, theta):\n", + " return np.array([cost_partial_theta(xs, ys, theta, i)\n", + " for i in range(len(theta))])" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [], + "source": [ + "def gradient_descent(xs, ys, theta, alpha=0.1, iterations=100):\n", + " for _ in range(iterations):\n", + " theta = theta - alpha * cost_gradient(xs, ys, theta) \n", + " return theta" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "weights: 10.159570236692419 -1.6360674177180405\n", + "train cost: 0.011264210236902399\n", + "test cost: 0.011623228480782\n" + ] + } + ], + "source": [ + "theta = np.random.randn(len(X_train[0]) + 1)\n", + "theta = gradient_descent(X_train, Y_train, theta, 1, 500)\n", + "print('weights: ', *theta)\n", + "print('train cost:', cost(X_train, Y_train, theta))\n", + "print('test cost: ', cost(X_test, Y_test, theta))" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 1.0, 'Optimized model')" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt8VNW99/HPbyYJYUJIYgIIgoCggOEaCHIRDaJIRaPHW0vVeh6t2lbbc3p6bLW2te3r9HKe06enetS2PraKl4LXR6NSxQtgxQsCoqKAgoCEawgQLiH39fyxBgyQQEgm2ZPJ9/16zSuZ2XvP/LKVb1bWXnstc84hIiKJJRR0ASIiEnsKdxGRBKRwFxFJQAp3EZEEpHAXEUlACncRkQSkcJe4ZmYnm9leMws38/i9ZnZKjGuab2bfjOV7NvFzC8ysuIn7/tzMHm3tmiR+Kdwlpszsn83sIzMrN7MtZvZHM8s8juPXmdm5B547575wznVxztU2p57osZ8351iR9kzhLjFjZj8A/hO4FcgAxgF9gVfMLCXI2kQ6GoW7xISZdQV+AXzXOfeSc67aObcOuBLoB1wd3e/nZvaUmT1uZnvMbKmZjYhuewQ4GXg+2p3yQzPrZ2bOzJKi+8w3s/8ws7ei+zxvZtlm9piZ7Taz98ysX726nJkNNLNe0f0PPMrNzNXb7zozW2FmO83sZTPrW2/beWa20szKzOwewI5yHn5uZk+a2aPRn+8jMzvNzG43s21mtsHMptbbv5eZFZnZDjNbbWY31NvW2cweitb0CZB/2Gf1MrOnzazEzNaa2feO+z+cJCyFu8TKBCAVeKb+i865vcAc4Lx6L18MPAmcAPwNeNbMkp1z1wBfABdFu1P+dyOf9TXgGuAkYADwNvBg9P1WAHcefoBzblP0Pbs457oA/w+YDWBmFwM/Bi4FugH/AGZFt+VEf6afADnAGmDiMc7FRcAjQBbwPvAy/t/aScAvgT/X23c2UAz0Ai4Hfm1m50S33Rn9+QYA5wPXHjjIzELA88AH0fedAvyrmZ1/jNqkg1C4S6zkANudczUNbNsc3X7AEufcU865auD3+F8K447jsx50zq1xzpUBfwfWOOdejX72k8Coox1sZj8CBgPXRV/6FvAb59yK6Hv8GhgZbb1fAHxcr94/AFuOUd8/nHMv16unG/Db6PGzgX5mlmlmffC/KH7knKtwzi0DHgC+EX2fK4FfOed2OOc2AHfX+4x8oJtz7pfOuarodYX/i//FJ0JS0AVIwtgO5JhZUgMB3zO6/YANB75xztVFR4D0Oo7P2lrv+/0NPO/S2IFm9hXgX4AznHP7oy/3Be4ys/9Tf1d8i7jXYfU6M9vA0R1ez/Z6F4QPfGaX6HvvcM7tqbf/emBM9PtDPju67YC+QC8z21XvtTD+rw4RtdwlZt4GKvFdGweZWRfgK8Br9V7uU297COgNbIq+1GrTlJrZIGAmcGW0JXzABuAm51xmvUdn59xb+L866tdr9Z+30CbgBDNLr/faycDG6PeHfHZ0W/2a1x5Wc7pz7oIY1SbtnMJdYiLaRfIL4H/MbJqZJUcvbD6B71N+pN7uo83s0uhF0n/F/1J4J7ptKxDTcelw8ILvc8Adzrk3D9v8J+B2M8uN7pthZldEt70I5Nar93vAibGoKfoL5i3gN2aWambDgeuBA+PTn4jWlWVmvYHv1jt8EbDHzH4UvfAaNrOhZnbIRVfpuBTuEjPRC6A/Bn4H7AbexbcwpzjnKuvt+hzwVWAn/sLopdH+aIDfAD8xs11m9u8xLC8PGAT8d/1RM9G6/x9+COdsM9sNLMf/tYFzbjtwBfBboBQ4FVgYw7pm4EcTbcJf5L3TOfdqdNsv8F0xa4G51PsFGe3muRAYGd2+Hd9fnxHD2qQdMy3WIW3JzH4ODHTOXR10LSKJTC13EZEEpHAXEUlA6pYREUlAarmLiCSgwG5iysnJcf369Qvq40VE2qUlS5Zsd851O9Z+gYV7v379WLx4cVAfLyLSLpnZ+mPvpW4ZEZGEpHAXEUlACncRkQSkWSFFJCaqq6spLi6moqIi6FISQmpqKr179yY5OblZxyvcRSQmiouLSU9Pp1+/fvjJM6W5nHOUlpZSXFxM//79m/Ue6pYRkZioqKggOztbwR4DZkZ2dnaL/gpSuItIzCjYY6el5/KY4W5mf40u7Lu8ke1mZndHF/f90MzyWlSRiIi0WFP63B8C7gEebmT7V/BzXJ8KnAH8Mfq1VRWtKmLumrlMPHEqoyKF9OwJGY3MZL18OSxcCBMnwtChMGtpEY8vnsugpKlcM7aQpCQOHl9WBp9+6o/r3Bleegk2boSdo+7grdIi0isH0zurB+GaDKooY0TXqZxWV8hnoSL+WHwTZVXb6Zs0lqsqF7KzexFvlP8ZVwfZ1SNJO6GM6k5bWbF9JX1SB1PpytlWvpWs5B6cdepIKq2MTZ9n8HbJy9SlljCw81jCqeVUJm+le6QHU7NvIi+tkIWlRSzZOZdIOIPqUBknd89g+54yeuybyrT+heTlwYIt/vxMHTCVwkGFAHzxhT8Xm7sW8f7uQ7eJyKEO3GiZk5PTon2C0qSJw6Ir6rzgnBvawLY/A/OdcwdWi18FFDjnNh/tPceMGeOae4dq0aoi/u9/XomrqCRknRhbdxunpE6k8J9CdOkahlAIwmEIh1mzLsy/3RqmvCpMUkqYy257g9+t+gEVoQpq61LJWPQnpg+9kJyeyXztmmQemJnMgjfDVFQa69fDtm1QV3AHTPq1X1XTcejX6ggpS/6VqvzfQDh6Lh1QMhiyPofkqi9fa+j4Axp6/bB9rDaFLh/9O3tz/4BLLj/iGKuO0O2NWUyYAC+nz2B/TTmR5AizLpvFyM6FXH89FKcV8enwGdSFv9ymgJdYWLFiBUOGDAm6jJiJh3Bv6Jya2RLn3JhGDvlyvxiE+wv4ld3fjD5/Db+a+xHJbWY3AjcC9OjRY/Ts2bOP+dkN2bB7Ayc+9ywp+ysx50h1ETrVdiWSWkdy+NCfp7wc9uzxWV9bC+FIGVXs+7Km6jQi4QxCIUhJgYoKqKk1qi2JKpdETTiZ2swSajs5apOTqElOojYpiZqUJGqSk6lJSaYmnE5NpI7qlGSqO6VQm5yET9tm/XhHZTWdcUn7G90eruhOUhJUJm07+Fr3tO5k0IfNm6EmbQMV4UO39ekaqyVBpSPLyMhg4MCBgdawfv16Lr30UvLz83n33XfJy8vj6quv5te//jUlJSU88MADnHLKKdx8882sW7eOzp07c/fddzN06FBKS0u57rrr2Lx5M2PHjmXevHm88cYbZGdnM3v2bP70pz9RXV3NmDFj+P3vf084HGbo0KEsWLCA7OzsVvl5Vq9eTVlZ2SGvTZ48uUnh3qZDIZ1z9wP3g2+5FxQUNOt9ilYVccmAlyivLieZCBdX+5bpLbdARlcHdXU+yWtr+eSjWm76ai01lbVEUmq49s45/O7Db4OrILkqle6LfscFAyfRI7uG6VOreeOZaj5ZVoWrrGLn1ir2l1WRNORROg18jk61kFqD/1odze66JMKbxlF70psHw7zWYF9Vd/ZklrKncy17OsHuFChLhd2dYGdn2JMCrv4Vj5i33L91WMu9gP/5nwMt928d0nIvGNS8/w4i9a1YsYL09Oha3y+9BFu2xPYDTjwRpk076i5dunTh888/5+mnnyY3N5f8/HyeffZZ3n77bYqKirjrrrvo06cP+fn5vPDCC7z++ut8+9vfZtmyZdxxxx0UFBTws5/9jBdffJGHH36YLl26UFxcTFFREe+88w7Jycl85zvfoaioiG984xuYGV26dPny546x1NRURo0a1axjYxHuGzl0hfbefLl6e6soHFTIrMtmNdLnbge7ZABOHwuPPl+/z/2fOWHpCV/2uX/n0D73Kwsa6nO/nJ2j7uD1+n3u1V1xNTvISz2LwRVns6HmBZ794g7qyndxSu0gpu37FTWpC/hizyuk7aghq6IbKZ0rqU3ay/by7XRJ6cauVNgQLqe6azf6Dx7KjowaPt15Igt2ve773CMN9LlPLmRh6RmN97lPOdDnPuuIPve//AWWLy9kc9dZ6nOXhNW/f3+GDRsGQG5uLlOmTMHMGDZsGOvWrWP9+vU8/fTTAJxzzjmUlpaye/du3njjDZ555hkApk+fTlZWFgCvvfYaS5YsIT/frz2+f/9+unfvHsBPdnxiEe5FwC1mNht/IbXsWP3tsVA4qLDJwTR0qH8cMCOvkBl5DR+bkQH59daP//K4X0UfjbmJf+Omw1675Mtvq6v91dqyMti1C3bu9I/SUv/YV83B3qKcGb6V0rMn9OoFJ50E9VoGkykEjv6zF2YceX5OPtk/aMLxIi1yjBZ2a+rUqdPB70Oh0MHnoVCImpqa477j0znHtddey29+85uY1tnajhnuZjYLKAByzKwYuBNIBnDO/QmYA1wArAbKgf/VWsW2a8nJkJPjH4dzDnbvhpISfwV361b/J+3q1X4bQGYm9Onj07l/f8jOBo0pFjlukyZN4rHHHuOnP/0p8+fPJycnh65du3LWWWfxt7/9jZ/85Cf8/e9/Z+fOnQBMmTKFiy++mO9///t0796dHTt2sGfPHvr27RvwT3J0xwx359yMY2x3wM0xq6gjMvN/MmRkQP0LUtXVPuQ3boQNG2DdOvjoI7+ta1c45RQ47TQYMADqtVZEpHE///nPue666xg+fDiRSISZM2cCcOeddzJjxgxyc3OZMGECJ/s/czn99NP5j//4D6ZOnUpdXR3Jycnce++9cR/uga2h2pKhkB2Wc74r5/PPYe1a/3X/fn99oV8/yM2FwYMhEgm6UumAEm0oZDxoyVBITRzWnpjBCSf4x5gxflTQhg3+CvCKFVBUBC+84FvyI0fCoEGQpP/EIh2R/uW3Z6EQ9O3rH+ee67twPv4YPvwQnnzSt+CHD4exY/0vBBHpMBTuicLMj67p2RPOOcd32bz/PixaBO++6/vmx4/3vwh0IVYk4SncE1Eo5C/MDhzob8997z1YvBhWrfKjbc4+21+MVciLJCyFe6JLT/ct+bPO8i35N9+ERx6B3r3hvPN8S15EEo7CvaNISvJ3Z+XlwbJlsGABPPggDBni++tbaW4MEQmGFuvoaMJhGD0avvtd36Jfswbuuw/mzYOamqCrE2m2Xbt2cd999zXr2AsuuIBdu3Y1+7O7dOly1O0tqa25FO4dVXKy76r53vf8+PgFC+CPf/Tj50XaoaMFaM0xGi5z5swhMzOzNcoCFO4ShC5d4NJL4Zpr/Lj5mTPh73/3d8eKtCO33XYba9asYeTIkdx6663Mnz+fSZMmUVhYyOmnnw7AJZdcwujRo8nNzeX+++8/eGy/fv3Yvn0769atY8iQIdxwww3k5uYydepU9u8/corttWvXMn78eIYNG8ZPfvKTg6/v3buXKVOmkJeXx7Bhw3juuecarK2x/WLKORfIY/To0U7iTFWVc3PmOHfnnc7de69zW7YEXZG0I5988slxH7Nrl3MrVvivLbV27VqXm5t78Pm8efNcJBJxn3/++cHXSktLnXPOlZeXu9zcXLd9+3bnnHN9+/Z1JSUlbu3atS4cDrv333/fOefcFVdc4R555JEjPuuiiy5yM2fOdM45d88997i0tDTnnHPV1dWurKzMOedcSUmJGzBggKurqzuitsb2O1xD5xRY7JqQsWq5y5eSk+ErX4Grr/arnNx/PyxZEnRVkqDKyuCee/zgrXvu8c9jbezYsfTv3//g87vvvpsRI0Ywbtw4NmzYwGeffXbEMf3792fkyJEAjB49mnXr1h2xz8KFC5kxw0+7dc011xx83TnHj3/8Y4YPH865557Lxo0b2bp16xHHN3W/ltBoGTnSwIHw7W/DM8/A88/Dpk0+9DWVgcTQ5s2+DdG3L6xf7583tg5yc6WlpR38fv78+bz66qu8/fbbRCIRCgoKqKioOOKY+lMGh8PhBrtlAKyB+0Qee+wxSkpKWLJkCcnJyfTr16/Bz2jqfi2hlrs0LC0NrroKJk3yrfeHHvI3RInESM+efoaM9ev91549W/Z+6enp7DnK/6NlZWVkZWURiURYuXIl77zzTrM/a+LEiRxYJvSxxx475DO6d+9OcnIy8+bNY/369Q3W1th+saRwl8aFQjBlClx5pZ9n/i9/ge3bg65KEkRGBtxyi7+Wf8stLW+1Z2dnM3HiRIYOHcqtt956xPZp06ZRU1PDkCFDuO222xg3blyzP+uuu+7i3nvvZdiwYWzc+OXCc1dddRWLFy9m2LBhPPzwwwwePLjB2hrbL5Y05a80zaZN8NhjfkTN17/uFw4RqUdT/sZeS6b8VctdmqZXL/jmN/3fzzNnfrnQrIjEJYW7NF1WFlx3HXTvDo8/7iciE5G4pHCX45OWBt/4hl/A+4knYOXKoCsSkQYo3OX4pab6q2A9e/qAVwteJO4o3KV5UlP9zU49e/pVn1phKJeINJ/CXZovNdWPhc/MhFmzIMZ32IlI8yncpWUiEd+CT0mBRx+FFkybKtISLZ158Q9/+APl5eXH3G/+/PlceOGFR91n2bJlzJkzp9m1xILCXVouM9MHfE0N/O1vUFkZdEXSAbVVuDeFwl0SR/fucMUV/g7Wp5/2NzuJtKHDp9UF+K//+i/y8/MZPnw4d955JwD79u1j+vTpjBgxgqFDh/L4449z9913s2nTJiZPnszkyZOPeO+XXnqJwYMHk5eXxzPPPHPw9UWLFjF+/HhGjRrFhAkTWLVqFVVVVfzsZz/j8ccfZ+TIkTz++OMN7tfqmjJ1ZGs8NOVvgnr3XT9l8CuvBF2JtLHmTPn73Mrn3M0v3uyeW/lciz//8Gl1X375ZXfDDTe4uro6V1tb66ZPn+4WLFjgnnrqKffNb37z4H67ovMNH5j293D79+93vXv3dp9++qmrq6tzV1xxhZs+fbpzzrmysjJXXV3tnHPulVdecZdeeqlzzrkHH3zQ3XzzzQffo7H9jqUlU/5qmj+Jrfx8Pw/Nm2/61vzw4UFXJHGqaFURM56eQXl1OQ8ue5BZl82icFBhzN5/7ty5zJ07l1GjRgF+IY3PPvuMSZMm8YMf/IAf/ehHXHjhhUyaNOmo77Ny5Ur69+/PqaeeCsDVV199cKGPsrIyrr32Wj777DPMjOpGFrlp6n6xpG4ZiS0zPz1w375+uuCSkqArkjg1d81cyqt9H3d5dTlz18yN6fs757j99ttZtmwZy5YtY/Xq1Vx//fWcdtppLF269OAqSr/85S+b/Rk//elPmTx5MsuXL+f5559vdNrepu4XSwp3ib1wGC6/3I+geeIJqKoKuiKJQ1MHTCWSHAEgkhxh6oCpLXq/w6fVPf/88/nrX//K3r17Adi4cSPbtm1j06ZNRCIRrr76am699VaWLl3a4PEHDB48mHXr1rFmzRoAZs2adXBbWVkZJ510EgAPPfRQo7U0tl9rUrhL60hPh8su8xdYAx41IPGpcFAhsy6bxc35N8ekS+bwaXWnTp3K17/+9YNrnV5++eXs2bOHjz76iLFjxzJy5Eh+8YtfHFwD9cYbb2TatGlHXFBNTU3l/vvvZ/r06eTl5dG9e/eD2374wx9y++23M2rUqEMW4Z48eTKffPLJwQuqje3XmjTlr7SuefNgwQK45BKILl0miUlT/saepvyV+HX22dCvn2+979wZdDUiHYbCXVpXKAT/9E/+Quuzz2r8u0gbUbhL68vIgAsu8JOLvf120NVIKwqqmzcRtfRcKtylbQwfDkOGwOuva4KxBJWamkppaakCPgacc5SWlpKamtrs99BNTNI2zOCii+C++3z3zA03+C4bSRi9e/emuLiYEt3bEBOpqan07t272ccr3KXtRCK+e+aJJ+Cdd2DChKArkhhKTk6mf//+QZchUWo6SdsaMgQGDfJDJDV6RqTVKNylbZnB9Om+S+b550H9syKtQuEuba9rVzj3XPj8c/jww6CrEUlICncJxpgx0KcPzJ0LbTCJkkhHo3CXYBzonikv9/3vIhJTCncJzokn+hb8okUa+y4SYwp3CdY550Dnzn7uGV1cFYkZhbsEq3NnmDLFT02wfHnQ1YgkDIW7BG/UKOjVC155Bdpg+TGRjkDhLsELheD882H3bn/nqoi0mMJd4kPfvjB4MPzjHxBdFk1Emk/hLvHjvPOgpsav3CQiLaJwl/iRne2HRi5ZAppZUKRFFO4SX84+G5KT4dVXg65EpF1TuEt8SUuDM8+EVatgw4agqxFptxTuEn/OOMOH/Guv6cYmkWZSuEv8SUmBs86Cdev8zJEictwU7hKfRo+GzEy13kWaSeEu8SkpCQoKYNMmWLky6GpE2h2Fu8Sv4cMhJwdefx3q6oKuRqRdUbhL/AqFfOu9pAQ++SToakTaFYW7xLfTT4du3fxdq+p7F2kyhbvEt1DI39ik1rvIcVG4S/xT613kuCncJf6FQn7c+7Ztar2LNJHCXdqH3Fw/ckatd5EmUbhL+1C/9b5qVdDViMQ9hbu0H0OHQlaWX9BDrXeRo1K4S/sRCvkZIzduhLVrg65GJK4p3KV9GTEC0tN9611EGqVwl/YlKQkmTPAtd833LtIohbu0P6NHQySi1rvIUSjcpf1JSfELenz6qR89IyJHULhL+zR2rA/5hQuDrkQkLincpX3q3Bny8uCjj6CsLOhqROKOwl3ar/Hj/de33w62DpE4pHCX9isjA4YNg6VLYf/+oKsRiSsKd2nfJk6EqipYtCjoSkTiisJd2rfu3eG00+Ddd6G6OuhqROKGwl3av4kTobwcPvgg6EpE4obCXdq/k0+Gk07yF1a1kLYIoHCXRGDmpyQoLfU3NomIwl0SxJAhkJkJb70VdCUicUHhLokhFPLj3r/4QhOKiaBwl0QyahSkpuqmJhEU7pJIUlIgPx9WrIAdO4KuRiRQCndJLGPH+i6ad94JuhKRQCncJbGkp/spCd5/X1MSSIemcJfEM368v1t1yZKgKxEJjMJdEk+PHnDKKX5KgtraoKsRCYTCXRLThAmwZw8sXx50JSKBaFK4m9k0M1tlZqvN7LYGtv+zmZWY2bLo45uxL1XkOAwYAN26+WGRzgVdjUibO2a4m1kYuBf4CnA6MMPMTm9g18edcyOjjwdiXKfI8THzfe9btsDatUFXI9LmmtJyHwusds597pyrAmYDF7duWSIxMHw4pKXppibpkJKasM9JQP37uYuBMxrY7zIzOwv4FPi+c+6Ie8DN7EbgRoAePXowf/784y5Y5HhkJCeT9corbExLozozM+hyRNpMU8K9KZ4HZjnnKs3sJmAmcM7hOznn7gfuBxgzZowrKCiI0ceLNCI/H/77v+kXDoP+f5MOpCndMhuBPvWe946+dpBzrtQ5Vxl9+gAwOjblibRQWhqMGOEX8ti3L+hqRNpMU8L9PeBUM+tvZinA14Ci+juYWc96TwuBFbErUaSFxo+HmhpYvDjoSkTazDHD3TlXA9wCvIwP7Seccx+b2S/NrDC62/fM7GMz+wD4HvDPrVWwyHHLyfHrrC5a5ENepAMwF9AY4DFjxrjFaklJW1m7FmbOhMJCyMsLuhqRZjOzJc65McfaT3eoSsfQrx+ceKJuapIOQ+EuHcOBdVZLSmD16qCrEWl1CnfpOHJzoWtXrbMqHYLCXTqOcBjOOMP3v2/eHHQ1Iq1K4S4dy+jRfjk+TUkgCU7hLh1LaqoP+OXLoaws6GpEWo3CXTqeM6JTI2mdVUlgCnfpeDIz/cXVJUugoiLoakRahcJdOqYJE6CqSlMSSMJSuEvH1LOnX2f1nXc0JYEkJIW7dFwTJ8LevfDRR0FXIhJzCnfpuE45xU9J8NZbmpJAEo7CXTqu+lMSfPpp0NWIxJTCXTq23Fw/eubNN9V6l4SicJeOLRz2rfcNG+CLL4KuRiRmFO4io0b55fjefDPoSkRiRuEukpzs71r97DPYsiXoakRiQuEuApCf7ycUW7gw6EpEYkLhLgLQubMP+OXLYceOoKsRaTGFu8gB48b5C6xqvUsCULiLHJCe7i+uLlum6YCl3VO4i9R35pn+q1rv0s4p3EXqy8iAESNg6VLYsyfoakSaTeEucrhJk6CuTgtpS7umcBc5XFYWDBvm53rfty/oakSaReEu0pBJk/w872q9SzulcBdpSE6Ob70vWuTnfBdpZxTuIo05+2yordWcM9IuKdxFGpOd7UfOLF4Mu3cHXY3IcVG4ixzNWWf5kTP/+EfQlYgcF4W7yNFkZfm7VpcuhV27gq5GpMkU7iLHctZZ/uuCBcHWIXIcFO4ix5KRAWPH+jlnSkqCrkakSRTuIk0xaZKf7/3VV4OuRKRJFO4iTRGJ+EnFVq3SWqvSLijcRZrqjDP8tMCvvALOBV2NyFEp3EWaKiUFCgpgwwbfgheJYwp3keMxapSfmuCVV/zdqyJxSuEucjxCITj/fCgt9fPOiMQphbvI8Tr1VBg40I9715TAEqcU7iLNcf75UFUF8+YFXYlIgxTuIs3RrRvk58OSJbB1a9DViBxB4S7SXAUFkJoKc+ZoaKTEHYW7SHN17gznngvr18MHHwRdjcghFO4iLZGXB336wNy5UF4edDUiByncRVrCDC68ECoq4LXXgq5G5CCFu0hL9egB48b5i6sbNgRdjQigcBeJjYICPzVwURHU1ARdjYjCXSQmUlJ890xJiRb1kLigcBeJlVNPhZEjYeFC2LQp6Gqkg1O4i8TStGmQlgbPPqvuGQmUwl0kllJT4aKLYNs2dc9IoBTuIrF22ml+auA33/Q3OIkEQOEu0hqmTYOsLHj6adi/P+hqpANSuIu0hk6d4PLL/ZTARUWae0banMJdpLX06gVTpsCKFf4GJ5E2pHAXaU3jx8OAAfDSSxoeKW1K4S7Smszgssv88MjHH9fKTdJmFO4irS0Sga9+1Qf7U09BXV3QFUkHoHAXaQu9evnpCdauhVdfDboa6QCSgi5ApMMYOdL3u7/1FmRnw+jRQVckCUzhLtKWpk2DnTvhxReha1c/H41IK1C3jEhbCoXgiiv8HPBPPglbtgRdkSQohbtIW0tJga9/3a/B+uijUFoadEV62jYPAAALZklEQVSSgBTuIkFIT4err/YjZx5+GHbtCroiSTAKd5GgdOsG3/gGVFbCzJmwe3fQFUkCUbiLBOnEE+Gaa6C8XAEvMaVwFwnaSSfBVVfB3r3w17+qD15iQuEuEg9OPhmuvRaqquDBB2Hr1qArknZO4S4SL3r1guuu88MlH3wQPv886IqkHVO4i8STnBwf8F27+mGS770XdEXSTincReJNZiZcfz0MHOjvZJ0zB2prg65K2hmFu0g86tQJvvY1Px/8okW+m0Zj4eU4KNxF4lUoBOef76crKCmBP/8ZVq4MuippJxTuIvEuNxduusl318ye7ddkragIuiqJcwp3kfbghBN8P/yZZ8L778N998Hq1UFXJXFM4S7SXiQlwbnn+pDv1MmPpnnySSgrC7oyiUMKd5H2pndv301TUACrVsE998Abb0B1ddCVSRzRYh0i7VFSkg/3kSPh5Zfh9df9mPizz4ZRoyAcDrpCCZjCXaQ9y8z0i2+vWwevvQYvvAALF/q++REj/C8B6ZDMORfIB48ZM8YtXrw4kM8WSUjO+Yus8+b5tVq7dIFx4yAvDyKRoKuTGDGzJc65McfaT7/WRRKFmV+TdeBAWLvWt+BffRXmz/fDKceM8f31ZkFXKm1A4S6SaMzglFP8Y+tWWLwYPvjAP3JyYPhwGDYMsrKCrlRakbplRDqCykpYvhw+/BDWr/ev9eoFQ4bA4ME+9NWibxea2i2jcBfpaHbt8kG/ciUUF/vXsrJgwAD/6N8fUlODrVEapT53EWlYZqYfTXPmmX5Zv1Wr/IXYDz/0XThm0KMH9O0Lffr4laIyM9Wyb2cU7iIdWdeukJ/vH7W1viW/bp3vulm6FN591+8XifhunB49/LqvPXr4KRE01DJu6b+MiHjhsG+t9+3rn9fWwrZtsHGjf2ze7EfhHJhbPhTy3Tk5OT7oDzwyMvwjOTm4n0WaFu5mNg24CwgDDzjnfnvY9k7Aw8BooBT4qnNuXWxLFZE2FQ5Dz57+MSbaxVtbC9u3+9AvKfGP0lJYswZqag49Pi3N/2XQtSukp/tx9wcekYjfHon4/n11+cTcMcPdzMLAvcB5QDHwnpkVOec+qbfb9cBO59xAM/sa8J/AV1ujYBFpvrIy3wCPRKC83Od2RkbD+85aWsQLK+Zy4ZCpzMgr9MfvDbN5Zw96ntwDTvbv1bMnzFn9HM8vfJFhFWO54MR8dqwtI213GdkVu9mx/g02bvmA9NowlTWVbNy9me37tnNSpD9Du49gbdkakiJdWFO+lXILkd21P2v2F7Ov027CSScwNKOQPdXJzK98iB3hrXTL6MnArmexomwltUkhJmVfy+Lap/lH6csQSuGfBnyHy7r9iqzsEJ9+ZszedgcfVRWRw2A6dSln/b5VlNeW0S2lL5N6ns/HO5aR3gWuGnwTXTcXkprqe55OO+3Lc9PguTjKuSxaVcTcNXOZOmAqhYMKG/xvcLRzHwvHHC1jZuOBnzvnzo8+vx3AOfebevu8HN3nbTNLArYA3dxR3lyjZUTaVlmZn2Ns504/5H34cN+LcsstR4bMrKVFXPv8DKopJ5kIMy+axQUDCrnnHh9koeiUg3V1sDpcxDOhGdRYOVRHiMyZRdfNhVRUQPdJRazNm0G1lROqg7Sq6KMaItWQVmmk1jk6V0PnGkitgdRqSK2FTjXQqRZSqkMYddBY495xxLbI9jOpXX0uFX1ep+rkN6gNQa3hv4agzvzzunoPRxKdP7mZ2k1j6NEzxNBhxoWFIZZuXcI9S++i2qoIWQrfzf8Bo3uewYtzjN37QqxdZ/Tvb3RJNy4qND7c+Q6/Xfhb9tdU0im5E7dP+jET+kwAM/bug2efNdZmjiKUndXguT+WmA2FNLPLgWnOuW9Gn18DnOGcu6XePsuj+xRHn6+J7rP9sPe6EbgRoEePHqNnz559fD+ViDRbRQXs2OF7QHbv9r0lzvmAP3zk49odG9hRue3g8xM6dadnpA87dkBKig948K3WbZUb2B/6cl/2dSe5vA91dWCZG6jptI0WcY5QbS3hGv8I1dYSin4N19ZhB7/WEaqrJVRbR6gmmVBpHyx9LRaqwOrqCNU5rK4Oc9Gvdc4/nMOcfx6uSsfKM0kO19EpBdK71LG7cif7q3f7k+UgLZxGRnIG5fvAcFTuryM11W9OS61jX20Zeyv3Yvj9u6SkkZHiT3ZNDewvh+JzprIrq2eD5/5YJk+eHH9DIZ1z9wP3g2+5FxQUtOXHi3RojbXcp08/svW4eWkRtz//rUNa7uMHFByl5f6tw1ruBfVa7t+i2sq/fPMDLW0HuBCE6g5tfR/eEq9JAquDcN2R7wFQF/Lb6x2Ts/rHVL/0b5SPu4Pqcfd8+XmHt/7rvWa1nch69QnqVhQyaBCcdx78+7/DnDWN/xWzY4cfQTriFH9t+ZZbYMGWImY8PYPy6nIiyRFmXTbrYNfMgf8G5Zsgsqvhcx8r6pYR6UBa3Oder78Y6vW5ryniiSVzOS08lUuGFFJczMG+69XhIhZumUtGagZlFWW8V/w+K0tWkp9zDjNyr+H19XM5IZLBexuWUVUFgzNH8lHJMiqTt9KFHpyTcRP79sHzu39JGV9wcsbJjEw/n+XblxEymJp9E+9VPsI/tr1AEil8tf8tXJnzK7Ky4NNPSbg+91h2yyQBnwJTgI3Ae8DXnXMf19vnZmCYc+5b0Quqlzrnrjza+yrcRUSOX8zuUHXO1ZjZLcDL+KGQf3XOfWxmvwQWO+eKgL8Aj5jZamAH8LWWlS8iIi3RpD5359wcYM5hr/2s3vcVwBWxLU1ERJpLa6iKiCQghbuISAJSuIuIJCCFu4hIAlK4i4gkIIW7iEgCCmyZPTMrAdY3sCkH2N7A66JzczQ6N43TuWlcezw3fZ1z3Y61U2Dh3hgzW9yUu686Ip2bxuncNE7npnGJfG7ULSMikoAU7iIiCSgew/3+oAuIYzo3jdO5aZzOTeMS9tzEXZ+7iIi0XDy23EVEpIUU7iIiCSguwt3M+pjZPDP7xMw+NrN/CbqmeGNmYTN738xeCLqWeGJmmWb2lJmtNLMV0ZXDBDCz70f/PS03s1lmdpyrdSYWM/urmW2Lrvl84LUTzOwVM/ss+jUryBpjKS7CHagBfuCcOx0YB9xsZqcHXFO8+RdgRdBFxKG7gJecc4OBEegcAWBmJwHfA8Y454biF9rp6IvoPARMO+y124DXnHOnAq9FnyeEuAh359xm59zS6Pd78P9ATwq2qvhhZr2B6cADQdcST8wsAzgLvxIYzrkq59yuYKuKK0lA5+hSmRFgU8D1BMo59wZ+pbj6LgZmRr+fCVzSpkW1orgI9/rMrB8wCng32Eriyh+AHwJ1x9qxg+kPlAAPRrusHjCztKCLigfOuY3A74AvgM1AmXNubrBVxaUezrnN0e+3AD2CLCaW4irczawL8DTwr8653UHXEw/M7EJgm3NuSdC1xKEkIA/4o3NuFLCPBPqzuiWifccX438B9gLSzOzqYKuKb86PC0+YseFxE+5mlowP9secc88EXU8cmQgUmtk6YDZwjpk9GmxJcaMYKHbOHfgr7yl82AucC6x1zpU456qBZ4AJAdcUj7aaWU+A6NdtAdcTM3ER7mZm+H7TFc653wddTzxxzt3unOvtnOuHvyD2unNOLTDAObcF2GBmg6IvTQE+CbCkePIFMM7MItF/X1PQxeaGFAHXRr+/FnguwFpiKi7CHd86vQbfKl0WfVwQdFHSLnwXeMzMPgRGAr8OuJ64EP1r5ilgKfAR/t96wt5q3xRmNgt4GxhkZsVmdj3wW+A8M/sM/9fOb4OsMZY0/YCISAKKl5a7iIjEkMJdRCQBKdxFRBKQwl1EJAEp3EVEEpDCXUQkASncRUQS0P8HWh7JwlgvdS8AAAAASUVORK5CYII=\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.scatter(X_train, Y_train, s=8, c='b', alpha=0.4, label='train data')\n", + "plt.scatter(X_test, Y_test, s=10, c='g', label='test data')\n", + "sig_x = np.linspace(X.min(), X.max(), 100)\n", + "sig_y = [hypothesis(np.array([x]), theta) for x in sig_x]\n", + "plt.plot(sig_x, sig_y, c='r', alpha=0.5, label='model')\n", + "plt.yticks([0, 0.5, 1])\n", + "plt.legend()\n", + "plt.grid(axis='y')\n", + "plt.title('Optimized model')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} |