From a1c7fe1b6d95e2560e62c12453da287d36d4a714 Mon Sep 17 00:00:00 2001 From: Charles Cabergs Date: Sat, 29 Aug 2020 17:13:57 +0200 Subject: Removed bloat, Added logreg_train --- logistic_regression_notebook.ipynb | 706 ------------------------------------- 1 file changed, 706 deletions(-) delete mode 100644 logistic_regression_notebook.ipynb (limited to 'logistic_regression_notebook.ipynb') diff --git a/logistic_regression_notebook.ipynb b/logistic_regression_notebook.ipynb deleted file mode 100644 index b56ea55..0000000 --- a/logistic_regression_notebook.ipynb +++ /dev/null @@ -1,706 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Logistic Regression\n", - "\n", - "Logistic regression is a *binary classification algorithm*." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Text(0.5, 1.0, 'Data')" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEWCAYAAABsY4yMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAASeElEQVR4nO3de5Cdd13H8fcn2SaQmjYiMQNNIOJAC9YxbWJDBUGLrJVLqlgvYBSVGVoviHhBaGcQ61RRB6+AtHJTGooIYldQGphiocq0JDVCLxQ6WCEFmwD2RqElydc/npPJZtnL2eyePbu/vF8zz5zzPOf3e57veXbzybO/c87vpKqQJLVn2bALkCQNhgEvSY0y4CWpUQa8JDXKgJekRhnwktQoA16SGmXA67iT5I4kX0tyX5K7k/xHkguTzPjvIcnGJJVkZCFqlebCgNfx6rlVtRp4LPAa4HeANw+3JGl+GfA6rlXVPVU1BvwU8MIkpyd5dpL/THJvks8nefW4Lh/p3d6d5P4kZyf5ziTXJPlyki8l2ZFkzYI/GWkCA14CquoGYC/w/cBXgZ8D1gDPBn4pyY/2mj6td7umqr6lqj4GBPhD4NHAE4ENwKsXrnppcga8dMQXgEdU1b9V1Ser6lBVfQK4Enj6VJ2q6vaq+mBVPVhV+4E/na69tFB8oUg64hTgK0m20o3Lnw6sAFYC/zBVpyTrgL+gu/pfTXfh9H8Dr1aagVfwEpDke+kC/jrgHcAYsKGqTgbeSDcMAzDZ9Kt/0Nv+3VV1ErB9XHtpaAx4HdeSnJTkOcA7gSuq6pN0V+FfqaqvJzkLeMG4LvuBQ8Djxm1bDdwP3JPkFOC3F6Z6aXpxPngdb5LcAawDDtCF9S3AFcAbq+pgkvOB1wKPAK4F7qB7UXV7r/8lwC8BJwDnAvcBfwecCtwOvB14WVWtX7hnJX0zA16SGuUQjSQ1yoCXpEYZ8JLUKANekhq1qD7o9MhHPrI2btw47DIkacnYvXv3l6pq7WSPLaqA37hxI7t27Rp2GZK0ZCT5n6kec4hGkhplwEtSowx4SWqUAS9JjTLgJalRBrwkNcqAl6RGDSzgk7wlyb4kNw3qGJKkqQ3yg05vA15HN0/2QF18MYyNwbZtcOmls+s7NgY7d8LoaNd/bAwuuwzuuutIm3XrYNMm2LOn275uHVxwwZH2v/EbcOedsGZN1/7++2H9ehgZgdNOgw99CO6+G1auhGXL4KGH4MQTYfXqbn8rV8KhQ1AFZ57Zbbvjjq7teed1+/zAB7rjnnEGXHst3HMPHDjQPXa4/4MPwooVXV3r1sHJJ3ftRkfh+uvh8sshgac/HR54AG67Db7xDdi+vdvPjh3dPg4ehMc8Bl71qm77ZZd1txdc0O3n8LneuvXoc9fvOT5W87Uf6bhRVQNbgI3ATf2237x5c83WRRdVddHYLRdd1H/fq66qWrWq67dqVdd35cqj9zfVsmJF137Zsv7aD3MZGTm2fsuXH913+fLJ97tqVXcu+znHU7Wb7c/qWPcjtQbYVVNk6tDH4JO8OMmuJLv2798/6/5jY9OvT2fnzu5KFrrbsbHuCrYfDz3UtT90qP/jDcvhK/3ZOnjw6L4HD06+3wce6M7lZCae46nazWS+9iMdT4Ye8FV1eVVtqaota9dOOl/OtCb+qT6bP91HR2HVqu7+qlVd35Ur++t7eChk2dDP4MxGjnEgbvnyo/suXz75flet6s7lZCae46nazWS+9iMdTxbVZGPH4vCY+7GMwW/bBldeefS47tatsxuD37rVMfjpxsQnO8fHYr72Ix1PBvqdrEk2Au+rqtP7ab9ly5ZyNklJ6l+S3VW1ZbLHBvk2ySuBjwGnJtmb5EWDOpYk6ZsNbIimqp4/qH1Lkma2BF4ilCQdCwNekhplwEtSowx4SWqUAS9JjTLgJalRBrwkNcqAl6RGGfCS1CgDXpIaZcBLUqMMeElqlAEvSY0y4CWpUQa8JDXKgJekRhnwktQoA16SGmXAS1KjDHhJapQBL0mNMuAlqVEGvCQ1yoCXpEYZ8JLUKANekhplwEtSowx4SWqUAS9JjTLgJalRBrwkNcqAl6RGGfCS1CgDXpIaZcBLUqMMeElqlAEvSY0y4CWpUQa8JDXKgJekRhnwktQoA16SGmXAS1KjDHhJapQBL0mNMuAlqVEGvCQ1yoCXpEYZ8JLUKANekhplwEtSowx4SWqUAS9JjTLgJalRBrwkNcqAl6RGGfCS1CgDXpIaZcBLUqMMeElqlAEvSY0y4CWpUQa8JDXKgJekRhnwktQoA16SGmXAS1KjDHhJapQBL0mNMuAlqVEGvCQ1yoCXpEYZ8JLUKANekhplwEtSowx4SWqUAS9Jjeor4JO8NMlJ6bw5yY1JRgddnCTp2PV7Bf+LVXUvMAp8K/CzwGsGVpUkac76Dfj0bp8FvL2qbh63TZK0CPUb8LuT7KQL+KuTrAYODa4sSdJcjfTZ7kXAJuCzVfVAkm8DfmFwZUmS5qqvgK+qQ0k2AtuTFHBdVb13kIVJkuam33fRvAG4EPgkcBNwQZLXD7IwSdLc9DtEcw7wxKoqgCR/C9wysKokSXPW74ustwOPGbe+AfjM/JcjSZov/V7BrwZuTXIDUMBZwK4kYwBVtW1A9UmSjlG/Af+qgVYhSZp3/b6L5tpBFyJJml99BXyS++iGZgBWACcAX62qkwZVmCRpbvq9gl99+H6SAOcBTx5UUZKkuZv1dMHV+SfghwdQjyRpnvQ7RPO8cavLgC3A1wdSkSRpXvT7Lprnjrt/ALiDbphGkrRI9TsG78RikrTE9DsXzfok702yr7e8J8n6QRcnSTp2/b7I+lZgDHh0b/nn3jZJ0iLVb8Cvraq3VtWB3vI2YO0A65IkzVG/Af/lJNuTLO8t24EvD7IwSdLc9P2l28BPAv8LfBE4H7/RSZIWtRnfRZNkOfA8Z4yUpKVlxiv4qjoIPH8BapEkzaN+P+j070leB/w98NXDG6vqxoFUJUmas34DflPv9vd6t6GbXfKcea9IkjQv+g3499EFenrrBdybZFNV7RlIZZKkOen3XTSbgQuBR9F90OkCutkk/ybJywdUmyRpDvq9gl8PnFlV9wMk+V3g/cDTgN3AHw+mPEnSser3Cv7bgQfHrX8DWFdVX5uwXZK0SPR7Bb8DuD7JVb315wLvSHIicMtAKpMkzUm/0wX/fpJ/BZ7S23RhVe3q3f+ZgVQmSZqTfq/g6QX6rhkbSpIWhVl/J6skaWkw4CWpUQa8JDXKgJekRhnwktQoA16SGmXAS1KjDHhJapQBL0mNMuAlqVEGvCQ1yoCXpEYZ8JLUKANekhplwEtSowx4SWqUAS9JjTLgJalRBrwkNcqAl6RGGfCS1CgDXpIaZcBLUqMMeElqlAEvSY0y4CWpUQa8JDXKgJekRhnwktQoA16SGmXAS1KjDHhJapQBL0mNMuAlqVEGvCQ1yoCXpEYZ8JLUKANekhplwEtSowx4SWqUAS9JjTLgJalRBrwkNcqAl6RGGfCS1CgDXpIaZcBLUqMMeElqlAEvSY0y4CWpUQa8JDXKgJekRhnwktQoA16SGmXAS1KjDHhJapQBL0mNMuAlqVEGvCQ1yoCXpEYZ8JLUKANekhplwEtSowx4SWqUAS9JjTLgJalRBrwkNcqAl6RGGfCS1CgDXpIaZcBLUqMMeElqlAEvSY0y4CWpUQMN+CTnJrktye1JXjHIY0mSjjYyqB0nWQ68HngmsBf4eJKxqrplUMeUNLWxMbjssu7+BRfAtm1w8cWwYweMjMCpp8KmTbBnz5E2ADt3wuho1x66PmNjR9avuALuvbdbDh2CFStgwwZYswbWrev2ec89cNddcO21XZ/HPx4+8xl48MHu/sqVcOONkMCjH931Bbjppq7NKafAG94Al1wCn/scnHRS1+bOO2Hfvu64D384PPax8OlPd+sAy5bBE54Az3hG9xyuv76r/bTT4LrrYP9+WLsWnvpU+NSnuue0des3n6fD5++SS7o+27fDpZd22yaen4nnfKbHJzvWvKmqgSzA2cDV49ZfCbxyuj6bN28uSfPvqquqVq6sgm5ZsaLq/POPrE+2jIwc6bNqVbePiy6avs9iXkZG+mu3bNnR5+mqq7plYv/zz+/Oy/jzM/Gcz/T4xJ/JxDb9AHbVFJk6yCGaU4DPj1vf29t2lCQvTrIrya79+/cPsBzp+LVzZ3clfNhDD8E110zf58CBI30eeKDbx9jY4GoctAMH+mt3+OofuvO0c2e3TOx/zTXdeYEj52e8nTtnfnziz2Rim7ka+ousVXV5VW2pqi1r164ddjlSk0ZHu2GQw1asgHPOmb7PyMiRPqtWTT3MsFSM9DkgvWxcKq5Y0T3v0dFv7n/OOd15gSPnZ7zR0Zkfn/gzmdhmrgY2Bg/cCWwYt76+t03SAtu2Dd71rrmPwY8fjz7exuDf857ZjcFv2wZXXjn945P9TOZTuiGc+ZdkBPg08Ay6YP848IKqunmqPlu2bKldu3YNpB5JalGS3VW1ZbLHBnYFX1UHkvwqcDWwHHjLdOEuSZpfgxyioar+BfiXQR5DkjS5ob/IKkkaDANekhplwEtSowx4SWqUAS9JjTLgJalRA/ug07FIsh/4KvClYdcyC49kadULS6/mpVYvWPNCWGr1wmBqfmxVTTrPy6IKeIAku6b6VNZitNTqhaVX81KrF6x5ISy1emHha3aIRpIaZcBLUqMWY8BfPuwCZmmp1QtLr+alVi9Y80JYavXCAte86MbgJUnzYzFewUuS5oEBL0mNWnQBn+Qnktyc5FCSRf0WqCTnJrktye1JXjHsemaS5C1J9iW5adi19CPJhiQfTnJL73fipcOuaSZJHpbkhiT/1av594ZdUz+SLE/yn0neN+xa+pHkjiSfTLInyaL/lqAka5K8O8mnktya5OyFOO6iC3jgJuB5wEeGXch0kiwHXg/8CPAk4PlJnjTcqmb0NuDcYRcxCweA36yqJwFPBn5lCZzjB4Fzqup7gE3AuUmePOSa+vFS4NZhFzFLP1hVm5bIe+H/AvhAVZ0GfA8LdK4XXcBX1a1Vdduw6+jDWcDtVfXZqnoIeCdw3pBrmlZVfQT4yrDr6FdVfbGqbuzdv4/uH8Upw61qetW5v7d6Qm9Z1O9kSLIeeDbwpmHX0qIkJwNPA94MUFUPVdXdC3HsRRfwS8gpwOfHre9lkYfPUpZkI3AGcP1wK5lZb7hjD7AP+GBVLfaa/xx4OXBo2IXMQgE7k+xO8uJhFzOD7wD2A2/tDYO9KcmJC3HgoQR8kg8luWmSZVFfAWs4knwL8B7g16vq3mHXM5OqOlhVm4D1wFlJTh92TVNJ8hxgX1XtHnYts/TUqjqTboj0V5I8bdgFTWMEOBP466o6g26+rQV5zW6g38k6lar6oWEcd57dCWwYt76+t03zKMkJdOG+o6r+cdj1zEZV3Z3kw3SveyzWF7afAmxL8izgYcBJSa6oqu1DrmtaVXVn73ZfkvfSDZku1tft9gJ7x/0l924WKOAdojl2Hwcen+Q7kqwAfhoYG3JNTUkSunHLW6vqT4ddTz+SrE2ypnf/4cAzgU8Nt6qpVdUrq2p9VW2k+x2+ZrGHe5ITk6w+fB8YZfH+B0pV/S/w+SSn9jY9A7hlIY696AI+yY8l2QucDbw/ydXDrmkyVXUA+FXgaroX/95VVTcPt6rpJbkS+BhwapK9SV407Jpm8BTgZ4Fzem+H29O70lzMHgV8OMkn6C4CPlhVS+Kth0vIOuC6JP8F3AC8v6o+MOSaZvISYEfv92IT8AcLcVCnKpCkRi26K3hJ0vww4CWpUQa8JDXKgJekRhnwktQoA17NS/JrvRn8dsyy38YkLxhUXdKgGfA6Hvwy8Myq+plZ9tsIzDrgezONSkNnwKtpSd4IPA741yQX9+bEv6E36dN5vTYbk3w0yY295ft63V8DfH/vA1YvS/LzSV43bt/vS/IDvfv3J3lt78M3ZyfZnOTa3mRYVyd51MI+c8mAV+Oq6kLgC8APAifSfRT/rN76n/Q+6r6P7gr/TOCngL/sdX8F8NHenON/NsOhTgSu780Dfz3wV8D5VbUZeAtw6Tw/NWlGQ5lsTBqSUbqJtX6rt/4w4DF0/wG8Lskm4CDwhGPY90G6SdEATgVOBz7YTafDcuCLc6hbOiYGvI4nAX584hfKJHk1cBfdN+0sA74+Rf8DHP1X78PG3f96VR0cd5ybq2pBvpZNmopDNDqeXA28pDdLJUnO6G0/GfhiVR2im9zs8Iuk9wGrx/W/A9iUZFmSDXRT1E7mNmDt4e/dTHJCku+a12ci9cGA1/Hk9+m+Qu8TSW7urQO8AXhh7wXS0+i+kAHgE8DB3hdovwz4d+C/6aZ6/UvgxskO0vsKx/OBP+rtcw/wfZO1lQbJ2SQlqVFewUtSowx4SWqUAS9JjTLgJalRBrwkNcqAl6RGGfCS1Kj/B6ua70hcje7mAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "from sklearn.datasets import make_blobs\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "\n", - "# create linearly separable data\n", - "sep = False\n", - "while not sep:\n", - " X, Y = make_blobs(n_samples=200, n_features\n", - " =1, centers=2, cluster_std=0.5, center_box=(0, 10))\n", - " sep = True\n", - " for x_1 in X[Y == 1]:\n", - " for x_0 in X[Y == 0]:\n", - " if Y[X.argmin()] == 0 and x_0 > x_1:\n", - " sep = False\n", - " elif x_0 < x_1:\n", - " sep = False\n", - " \n", - "X = np.hstack([X, np.ones((X.shape[0], 1))])\n", - "\n", - " \n", - "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33)\n", - "\n", - "plt.scatter(X[:, 0], Y, s=10, c='b')\n", - "plt.xlabel('feature')\n", - "plt.ylabel('groups')\n", - "plt.yticks([0, 1])\n", - "plt.title('Data')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Binary target\n", - "\n", - "### Sigmoid function\n", - "\n", - "We have to fit a **model** trough our data. \n", - "The *sigmoid* (also called *logistic*) function is used:\n", - "$$\\boxed{\n", - "\\sigma(x) = \\frac{1}{1 + e^{-x}}\n", - "}$$\n", - "\n", - "It produce an 'S' shape like function that stays between 0 and 1" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXhU9d338fc3O5CQQICwE/ZVURZxpSCo4FJ4vOt2o1atdWnV2vZpK3VptZe3bW+3urQ+3Irb7VpKKVVLFTF1QZFd9n0LhiUJhISsM/N7/pjBBgiQhMmcmcnndV1zzXZmzmdg8skvvzlzjjnnEBGR2JfgdQAREQkPFbqISJxQoYuIxAkVuohInFChi4jECRW6iEicUKFLxJjZFDN7P9rWa2Z5ZnbzMe4zM3vRzPaZ2ZdNl7LOdf/DzL4byXVKbDNthy7hZGbnAr8HBgN+YA1wt3NuoafBjsPM8oD/dc49X8d95wFvAP2dcwebMMOvgT7OuWubah0S/5K8DiDxw8xaA+8AtwNvAynAeUCVl7lOUg9ga1OWuUi4aMpFwqkfgHPuDeec3zlX4Zx73zn3FYCZ3WBmnx5a2MwuNLN1ZlZiZn80s38dmvoILfuZmT1hZvvNbLOZnR26fYeZ7ak9HWFmmWb2ipntNbNtZnafmSUcY70XmNna0HqfAayuF2Nm3wOeB84yszIze/DI5wot58ysT+jyS2b2rJm9a2alZrbAzHrXWnawmX1gZsVmttvMfmlmE4BfAleF1rM8tOw3U0FmlhB6TdtCr/0VM8sM3ZcbyvBdM9tuZoVmdm+j/xclZqnQJZzWA34ze9nMJppZm2MtaGbtgBnAVCAbWAecfcRio4CvQve/DrwJjAT6ANcCz5hZemjZp4FMoBfwLeB64MZjrHcmcB/QDtgEnFNXRufcC8BtwOfOuXTn3K9O9A8QcjXwINAG2Ag8HFp3BjAXmAN0Dr2OD51zc4D/At4KrWdoHc95Q+g0NvQa04FnjljmXKA/MA54wMwG1jOvxAkVuoSNc+4AwVJxwP8Ae81stpnl1LH4xcAq59xM55wPeArYdcQyW5xzLzrn/MBbQDfgIedclXPufaAa6GNmiQRLdKpzrtQ5txV4DLjuOOud4ZyrAZ6sY70n66/OuS9Dr+s14LTQ7ZcCu5xzjznnKkNZF9TzOacAjzvnNjvnygj+IrzazGpPmz4Y+qtoObAcqOsXg8QxFbqElXNujXPuBudcV2AIwZHok3Us2hnYUetxDsg/YpndtS5XhJY78rZ0giPtZGBbrfu2AV3qud4ddSx3Mmr/gigPZYTgL6RNjXzOzhz9+pKA2r8sj7VeaSZU6NJknHNrgZcIFvuRCoCuh66YmdW+3kCFQA3BDzAP6Q7sPMZ6ux2x3m51LHcsB4GWtR7fsQGP3UFwuqQuJ9rc7GuOfn0+Dv+lJ82cCl3CxswGmNlPzaxr6Ho34BrgizoWfxc4xcwmh6YNfgg0pBy/EZqSeRt42MwyzKwH8BPgf4+x3sFmdnlovXc1cL3LQ48/zczSgF834LHvAJ3M7G4zSw1lHRW6bzeQe+iD3Dq8AfzYzHqGPjc4NOfua8D6Jc6p0CWcSgl+kLnAzA4SLPKVwE+PXNA5VwhcQXCb9SJgELCIxm/ieCfB0fNm4FOCH6JOP856fxtab1/gs/quxDm3HniI4IebG0Lrqu9jS4ELgMsITo9sIPghJ8CfQ+dFZrakjodPB14FPga2AJUEX7PIN/TFIokKoZFpPjDFOfeR13lEYpFG6OIZM7vIzLLMLJXgdthG3dMzIlIPKnTx0lkEt/ooJDgNMdk5V+FtJJHYpSkXEZE4oRG6iEic8GznXO3atXO5ublN9vwHDx6kVatWTfb8TU35vRPL2UH5vdbU+RcvXlzonGtf132eFXpubi6LFi1qsufPy8tjzJgxTfb8TU35vRPL2UH5vdbU+c1s27Hu05SLiEicUKGLiMQJFbqISJyIqiMW1dTUkJ+fT2Vl5Uk/V2ZmJmvWrAlDKm9EMn9aWhpdu3YlOTk5IusTkaYRVYWen59PRkYGubm5BHeC13ilpaVkZGSEKVnkRSq/c46ioiLy8/Pp2bNnk69PRJrOCadczGx66JBXK49xv5nZU2a20cy+MrNhjQ1TWVlJdnb2SZe51J+ZkZ2dHZa/ikTEW/WZQ38JmHCc+ycS3GNdX+AW4E8nE0hlHnn6NxeJDyeccnHOfWxmucdZZBLwSujIL1+EdrbUyTlXEKaMIiJRwTmHL+Co9gWo9gWo8QeoCp0fun3zfj/pW4up8Tv8AYcvEMDnDz7u0PWAc/j8joBz+APgd45A6P7gbY6Ag0Do9kOXT7SrlnDMoXfh8EN45YduO6rQzewWgqN4cnJyyMvLO+z+zMxMSktLwxAJ/H5/2J7LC5HOX1lZedT/x8koKysL6/NFUixnB+U/xBdwHKyBgzWOcp+josZR4YeKGkelHyp9jgofVPkdVf5/n1f7HTV+qAoEz2sCUFPrcr32fvXF5yedvzEi+qGoc24aMA1gxIgR7shvU61ZsyZsHwTqQ9GGSUtL4/TTTw/b88Xyt/1iOTvEb37nHAcqfew5UMnuA1XsPlDJ3rIqisqqKCyrprCsin3l1ew7WMO+8mrKq/0nXFdqUgKtUpNomZJIy5REWqQl0jYlkRbJiaSFTqlJCaQlJ5KSlEBqUgIpiQmkJP37lJwQOk9MICnRWLNqJcNOG0pSopGUkBA6NxITgteD58HrCQlGohkJCZBoh9+WmGCYQYJZ6BScHrXfHfv1hKPQd3L4MRm7UvexHCWMNm/ezMMPP0xJSQkzZszwOo5IWFRU+9l2wM8/VhSwvbic7cXl5O+r4Ov9wdPBOko6LTmBdumpZLdKoX16Kv06ZJDVMoU2LZNp3SKZzBbJtG6RROu0ZDLSkslIS6JVahKtUhJJSgz/V3FS965ldL86d7XS5MJR6LOBO8zsTYKHHyvR/HnT69WrFy+88ALf+c53vI4i0mAV1X7W7S5l/a5S1u4qZcOeUjbtKePrktDWVvODR+HLaplM1zYt6NW+Fef2bUfnzBbkZKaRk5FKTus02mWk0iolUR/sh5yw0M3sDWAM0M7M8oFfAckAzrnngPeAi4GNQDlwY1OF9VpFRQUTJkxg3rx5JCYm1rlMdXU148ePZ968eSQlRdVm/iKe8PkDrN1VyuJt+/gqv4SVO0vYsKeUQGgyOi05gb4dMhjVK5te7VpRsXcbl4weSbe2LWmdpi+7NUR9tnK55gT3O4JHbI9706dP5/LLLz9mmQOkpKQwbtw43nrrLaZMmRLBdCLRocYfYPmO/czfVMQXm4tYtmP/N/PZ7dJTOaVLay4anMOgzpkM6JhBt7YtSUz49wg7L28ngztnehU/pmkIWYfly5dz5513UlhYyNq1a3HOcf/99zN37lxef/31b5YbO3Ysv/zlL7ngggu47777KCkp4emnn2by5MlMnTq1SQu9qKiIe++9l6VLl/LII48wderUJluXyIkUlFQwb+0e5q3ZwxebizhY7ccMBnZszRXDuzI8ty3De7Shc2aapkeaUNQW+oN/X8Xqrw80+vF+v/+okfSgzq351WWDj/u4yspKrrrqKl555RXOOOMM7r//fiorK7n33nuZNm0atQ/K8eCDD/LAAw+wZ88eli5dyuzZswEYMmQICxcuPOq5zzvvvDo3RXz00UcZP358g15fdnY2zz33XIMeIxJOWwsP8u6KAt5bUcCq0M9qt7Yt+D/DunBO73ac2SubNq1SPE7ZvERtoXtl7ty5DBs2jDPOOAOAU089lTlz5lBUVERWVtZhy44ePRrnHI8//jh5eXnf/AJJTEwkJSXlqE0PP/nkk8i9EJEmUFRWxV+X7mTWsp2s3Bks8WHds7hn4gDGDehAnw7pGoF7KGoL/UQj6RNp7HbcK1eu5JRTTvnm+pIlSxg2bBgtWrQ4an8nK1asoKCggOzs7KPWVVVVRVpa2mG31WeEHskfBh0gXOojEHB8vGEvb365g7lrduMLOIZ2zeS+SwZy8Smd6JzVwuuIEhK1he6V7Oxs5s2bB8D69euZOXMm8+fPp02bNvj9fiorK0lLS6OgoIApU6bwt7/9jbvuuos5c+YwYUJwlzdFRUW0a9fuqN3R1meEfqhkw/3Forlz51JcXExRURG333572J5X4tfBKh8zl+Tz4mdb2Vx4kOxWKdx4Ti5XjOhGv5zY/dJePFOhH+Gaa65h9uzZDBkyhHbt2vHGG2+QnZ0NwIUXXsinn37K2WefzeWXX85jjz3GwIEDuf/++/nFL37xTaF/9NFHXHLJJY1a/6EvDBUVFTFr1qyj7l+xYgWZmZl07979uM8za9asbx4/ceJELr74Yi666CI+++yzRuWS5mN/eTXTP9vKS59t4UClj6FdM/nD1acxcUgnUpJ0TJxopkI/Qnp6On//+9/rvO+HP/whTzzxBOPHj+fzz/+9r4bRo0cfdv3111/nt7/9baPWf+gLQ5MnT67z/sWLF5Obm3tYodfU1PCrX/2K8vJyAoEATz31FJMnT/7mOQKBAA8//DA/+9nPWLRoESNHjmxUNolv+w5W8/ynm3l5/jbKqnxcNDiHW0b3Zlj3LM2LxwgVegMMGzaMsWPH1rkFzSHV1dVMnjyZfv36hXXdq1ev5qmnnmLdunVkZGTw5ptv8uCDD5KTk8O0adOoqKggKyuLLVu2HPXYhIQE7r///rDmkfhR5fPz8vytPD1vI2VVPi4e0ok7x/VhQMfWXkeTBlKhN9BNN9103PtTUlK4/vrrw77eQYMG8dxzz/HSSy+Rm5t72M6Lli5dyrPPPktqamrY1yvxyznHuysK+O0/1pK/r4Ix/dszdeJA+nfU/His0oRYlCkqKuK2225j+fLlPPLII/V6zKRJk7jhhhv4+c9/zpw5c5o4ocSD7UXlXD/9S+54fSnpqUm8+r0zeOnGM1TmMU4j9Chz6AtDx9rK5YYbbjjqtssuu4zLLrssAukk1tX4A7zw6RaenLuepIQEHvz2YK49s8dhX72X2KVCF2kmthQe5O63lrF8x34uGpzDr789mE6Z2oY8nqjQReKcc463Fu7goXdWk5yYwLP/OYxLTu3kdSxpAip0kThWWlnDz2d8xT9W7uLs3tk8duVQjcrjWNQVunNO27xGmHYBEJ827inlllcXs62onKkTB/D983qRoLnyuBZVhZ6WlkZRURHZ2dkq9QhxzlFUVHTUfmckts1ZuYufvr2MFimJvHbzKM7sle11JImAqCr0rl27kp+fz969e0/6uQ7tcyVWRTJ/WloaXbt2jci6pGk553hnUzUz5ixmaLcsnrt2mKZYmpGoKvTk5GR69uwZlufKy8sL61HsIy3W80vk+fwBfjV7FTM21DDptM78/junkpp07KNrSfyJqkIXkcYpr/Zx1xtLmbtmD5f0TOaJK0/TfHkzpEIXiXGllTXc+OJClmzfx28mDaZb1VaVeTOlr/6LxLADlTVcP/1Llu3Yz9PXDOO6s3K9jiQe0ghdJEaVlNdw/fQFrC44wLNThnHR4I5eRxKPqdBFYtCByhqufWEB63aV8ty1wxk3MMfrSBIFVOgiMaayxs/NLy1i7a4DTLtuBGMHdPA6kkQJFbpIDKnxB7jj9SUs3FbMU1efrjKXw+hDUZEYEQg4fjHjK+au2cNDk4Zw2dDOXkeSKKNCF4kR//3+OmYu3clPL+jHdWf28DqORCEVukgMmLE4nz/lbeKaM7pzx/l9vI4jUUqFLhLlFm4tZurMrzi7dzYPTRqsHdfJManQRaLYjuJybn11MV3btOSPU4aRnKgfWTk2vTtEolRFtZ/vv7IInz/AC98dQVbLFK8jSZTTZosiUcg5x72zVrBudykv3jCSXu3TvY4kMUAjdJEo9NbCHcxcspM7z+/LmP7a1lzqR4UuEmVW7izhgdmrOK9vO340rq/XcSSGqNBFosiByhp+8NoSslul8ORVp5Go3eBKA2gOXSSKPDBrJTv3V/D2rWeSnZ7qdRyJMRqhi0SJvy3byaxlX3Pn+X0Y3qOt13EkBqnQRaLAzv0V3DdrJcO6Z3HHWH0TVBpHhS7iMX/A8ZO3lhEIOJ686nSS9OUhaSTNoYt47PlPNrNgSzGPXjGU7tktvY4jMUxDAREPbdpbxmMfrOeiwTn8x7AuXseRGKdCF/HIof2bt0hO5DeTh2inW3LSVOgiHnnl860s2raPBy4dRIeMNK/jSBxQoYt4YEdxOb+bs45v9WvP5ZpqkTBRoYtEmHOOe2Z+RWKC8V+Xn6KpFgkbFbpIhM1e/jWfbSziFxP60yWrhddxJI6o0EUiqKSiht+8s4ahXTP5z1E6LqiEl7ZDF4mgx99fR/HBKl68YaR2vCVhpxG6SISsyC/h1S+2cd2ZPTila6bXcSQOqdBFIsAfcNw3awVtW6Xykwv7ex1H4pQKXSQC3l60g+X5Jdx/6UAyWyR7HUfilApdpIkdqKzh0X+u44zctnx7aGev40gc04eiIk3s6Q83UFxezcuXDdI259KkNEIXaUJbCg/y0vytXDm8G0O66INQaVoqdJEm9PC7q0lNSuT/XqQPQqXpqdBFmsjH6/cyd80e7ji/D+0zdHxQaXoqdJEm4A84Hn53DT2yW3LjOblex5FmQoUu0gT+siSfdbtL+cWEAaQmJXodR5oJFbpImFVU+3n8/fWc1i2LiUM6eh1HmhEVukiYTf9sC7sOVDJ14gBtpigRpUIXCaOisir+lLeJ8QNzGNUr2+s40syo0EXC6Ol5Gymv9nHPRG2mKJGnQhcJkx3F5by2YBtXjexGnw4ZXseRZkiFLhImT8xdT4IZPxrXz+so0kyp0EXCYMPuUv66dCffPTuXjplpXseRZkqFLhIGj3+wnlYpSdz2rd5eR5FmTIUucpK+yt/PP1bu4ubzetK2VYrXcaQZU6GLnKRH319Pm5bJfO/cnl5HkWZOhS5yEhZsLuLj9Xv5wZg+ZKTpSETiLRW6SCM553jsg/V0yEjlurN6eB1HRIUu0lifbyriyy3F/GBMb9KStQMu8Z4KXaQRnHM8MXc9HVuncfUZ3b2OIwKo0EUa5dONhSzcuo8fjtXoXKKHCl2kgZxzPPHBejpnpnHlyG5exxH5hgpdpIE+3lDIku37+cHYPjp4hUQVFbpIAxwanXfJasGVIzQ6l+iiQhdpgI83FLJsx35+MLY3KUn68ZHoonekSD055/jD3ODc+XeGd/U6jshRVOgi9TR/UxFLtu/n9jG9NXcuUUmFLlJPf/hwAzmtU7lCc+cSpVToIvXwxebgt0Jv/5a2O5fopUIXqYenPtxA+4xUfStUopoKXeQEFm0tZv6mIm4d3Uujc4lqKnSRE3h63kayW6UwZZT2qCjRTYUuchzLd+znX+v3cvN5vWiRotG5RDcVushxPPPRRjJbJGt/5xITVOgix7Cm4AAfrN7NTef0JD01yes4IiekQhc5hmc+2kh6ahI3nJ3rdRSRelGhi9Rh454y3ltRwPVn9SCzpY4VKrFBhS5Shz/mbSQtKZHvndvT6ygi9VavQjezCWa2zsw2mtk9ddx/g5ntNbNlodPN4Y8qEhl7ygP8bdnXTBnVnez0VK/jiNTbCT/pMbNE4FngAiAfWGhms51zq49Y9C3n3B1NkFEkot7bXEOiGd8f3cvrKCINUp8R+hnARufcZudcNfAmMKlpY4l4o6Ckgk93+rhyZFdyWqd5HUekQeqzLVYXYEet6/nAqDqW+w8zGw2sB37snNtx5AJmdgtwC0BOTg55eXkNDlxfZWVlTfr8TU35vfHamioCzjE0ZW9M5ofY/bc/RPkbL1wb1/4deMM5V2VmtwIvA+cfuZBzbhowDWDEiBFuzJgxYVr90fLy8mjK529qyh95hWVVfPLhPM7ukswVFx/19o0ZsfhvX5vyN159plx2ArV3AN01dNs3nHNFzrmq0NXngeHhiScSOS98uoUqX4BLe2kzRYlN9Sn0hUBfM+tpZinA1cDs2guYWadaV78NrAlfRJGmt7+8mlc/38Ylp3SiYyttzSux6YTvXOecD7gD+CfBon7bObfKzB4ys2+HFrvLzFaZ2XLgLuCGpgos0hRemr+Vsiofd5zfx+soIo1Wrzl059x7wHtH3PZArctTganhjSYSGaWVNUz/dAsXDsphQMfW7FrrdSKRxtHfltLsvfrFNg5U+rjz/L5eRxE5KSp0adbKq308/8kWxvRvzyldM72OI3JSVOjSrL2+YDvFB6u5U3PnEgdU6NJsVdb4mfbxZs7unc3wHm29jiNy0lTo0mz9edEO9pRWacsWiRsqdGmWqnx+/pi3iZG5bTirV7bXcUTCQoUuzdKMxfkUlFRy17i+mJnXcUTCQoUuzU61L8AfP9rE6d2zOLdPO6/jiISNCl2anb8uzWfn/gqNziXuqNClWanxB3jmo42c2jWTMf3aex1HJKxU6NKszFq6kx3FFdx1vkbnEn9U6NJs+EKj80GdWjNuYAev44iEnQpdmo2ZS3eyraicH1/QT6NziUsqdGkWavwBnp63gVO6ZDJeo3OJUyp0aRb+sjifHcUV/PgCzZ1L/FKhS9yr9gV4et5GhnbLYmx/jc4lfqnQJe7NWBzc7vzu8RqdS3xToUtcq/YFePajjZzePUvbnUvcU6FLXHtz4XZ27q/gJ9qyRZoBFbrErfJqH099uJEze7XVPlukWVChS9x6ef42Csuq+NlF/TU6l2ZBhS5xqaSihuf+tYnzB3TQ0Yik2VChS1x6/pPNlFTU8NML+3kdRSRiVOgSdwrLqnjh0y1cemonBnfO9DqOSMSo0CXuPDNvI5U1fn58gUbn0ryo0CWubCs6yGsLtnHVyO70bp/udRyRiFKhS1z573+uIykhgR+P7+t1FJGIU6FL3Fi2Yz/vfFXA90f3okPrNK/jiEScCl3ignOOR95bQ7v0FG4Z3cvrOCKeUKFLXPho3R4WbCnmR+P7kZ6a5HUcEU+o0CXm+fwBHnlvLb3ateLqkd28jiPiGRW6xLzXv9zOhj1l3DNxAMmJektL86V3v8S0/eXVPP7Bes7pk80Fg3K8jiPiKRW6xLQn527gQEUN9186SDvgkmZPhS4xa/3uUl79YhtTRvVgQMfWXscR8ZwKXWKSc47fvLOaVimJ+oq/SIgKXWLSB6t388mGQu4e34+2rVK8jiMSFVToEnPKq308+PfV9M/J4LqzengdRyRq6BsYEnOe+nAjO/dX8OfbztJmiiK16KdBYsr63aU8/8lmrhjelZG5OhKRSG0qdIkZzjnum7WSVqlJ3DNxgNdxRKKOCl1ixswlO/lySzH3TBxAdnqq13FEoo4KXWLC3tIqfvPuak7vnsVVI7S/FpG6qNAlJvx69irKq/z8/j9OJSFB3wgVqYsKXaLenJUFvLuigLvG9aFvTobXcUSilgpdotr+8mrum7WKQZ1ac+u3ensdRySqaTt0iWoPvbOa/eXVvHzTSG1zLnIC+gmRqDVn5S5mLtnJbd/qzeDOmV7HEYl6KnSJSrsPVDJ15lcM6dKau8b19TqOSExQoUvUCQQc//fPy6mo8fPkVaeTkqS3qUh96CdFos6L87fyyYZC7rtkEH06pHsdRyRmqNAlqqwpOMDv5qxl3IAOTBnV3es4IjFFhS5Ro7Syhh+8toTWacn87jun6pByIg2kzRYlKjjnuOcvK9heXM5rN4+infbVItJgGqFLVHhp/lbeXVHAzy7qz5m9sr2OIxKTVOjiucXb9vHwu2sYPzCHW0f38jqOSMxSoYundpVU8oPXFtMpK43HrhyqeXORk6BCF8+UV/u4+ZWFlFX6+J/rR5DZItnrSCIxTR+KiicCAcdP3lrOqq8P8Pz1IxjQsbXXkURinkbo4onHPljHnFW7uPfigYwbmON1HJG4oEKXiHt9wXae/WgTV4/sxvfO7el1HJG4oUKXiHpvRQH3zlrBmP7teWjSEH0IKhJGKnSJmE83FHL3m8sY3r0Nf5oyXDvdEgkz/URJRCzdvo9bXl1Er/ateOG7I2mRkuh1JJG4o0KXJrd42z6uf+FL2qWn8spNZ5DZUpsnijQFFbo0qYVbi7n+hQVkp6fw1q1n0qF1mteRROKWtkOXJvPF5iJuemkhHTPTeOP7Z5KjMhdpUip0aRKLd/uYNvdLurdtyWvfH0WHDJW5SFPTlIuE3cvzt/LM0ioGdmrNm7ecqTIXiRCN0CVsAgHH7+as5f99vJnTOyTy+vfP1NYsIhGkQpewOFBZw0/eWs7cNbu59szunJ9ZqDIXiTBNuchJ27C7lMnPfMZH6/bwwKWD+M2kISToG6AiEacRupyUd776mp/P+IqWKYm8fvMoRuloQyKeUaFLo5RW1vDr2av5y5J8Tu+exZ+mDKdjpj78FPGSCl0abPG2Yu5+axk791Vw5/l9uGtcX5ITNXsn4jUVutRbWZWPx95fx8vzt9I5qwVv33oWI3Lbeh1LREJU6FIvc1fv5oG/raTgQCVTRnXnFxMGkJGmfbKIRBMVuhzXxj2lPPLeWj5cu4f+ORk8/Z/DGN6jjdexRKQOKnSpU1FZFU/O3cDrX26nZXIiUycO4KZze2quXCSKqdDlMEVlVUz7ZDOvfr6NKl+AKaO686NxfclOT/U6moicgApdACgoqeDFz7by6ufbqPT5uezUztw1ri99OqR7HU1E6kmF3swt3b6P6Z9t5b0VBTjnmHRaF344to+KXCQGqdCboZKKGmYv28nbi/JZsbOEjNQkbjonl+vPyqVb25ZexxORRlKhNxNVPj8fry/k78u/5p+rdlHlCzCwU2semjSYy4d1JT1VbwWRWKef4jhWVuXj0w17eX/1bj5YtZvSKh9ZLZO5ckQ3rhrZjSFdMr2OKCJhpEKPI4GAY93uUj7bWMi/1u/li81F1PgdrdOSmDCkI5cO7czZvbO16aFInFKhxzCfP8CaglIWbStm0dZ9fLG5iKKD1QD0at+KG8/pyfkDOjC8RxuVuEgzoEKPETX+AFsKD7JyZwkrdx4Inn9dQnm1H4DOmWl8q197zu7TjrN7Z9M5q4XHiUUk0lToUaa82sfWwnK+LPDx1Ycb2JdqRDgAAAb4SURBVLinjPW7S9m0t4wavwMgLTmBQZ1ac+WIbgzv0YbhPdqowEVEhR5JgYCj6GA1uw9UsvtAJV/vr+DrkuD5juJythdXUFhWVesR6+mS1YJ+OemM6d+B/h3TGdQpk97tW5GkKRQROUK9Ct3MJgB/ABKB551zvz3i/lTgFWA4UARc5ZzbGt6o0aXaF6CsykdpZQ0lFTUcqPBRUlHD/opq9h2sZl95DfsOVlN4sJrC0iqKDlZRWFaNP+AOe57kRKNjZhrd2rRk/MAOdGvbkh7ZLSneuoYrJozRcTlFpN5OWOhmlgg8C1wA5AMLzWy2c251rcW+B+xzzvUxs6uB3wFXNUVgAOcc/oDDF/j3uc8fwB9w1IQuf10WYE3BAXx+R7U/QLUvQE3ovNofoMrnp9oXoMoXoKomeL2yJkBFjZ+KGj+V1X7Kq/2U1/ipqPZxsMrPwWofZZU+yqp8VPkCx83YMiWRNi1TaJeeQqfMNIZ0aU37jFRyWqfRISONnNapdMlqQbv0VBISjj7+Zl7xepW5iDRIfUboZwAbnXObAczsTWASULvQJwG/Dl2eATxjZuacO3w4Wsv63aWc/2geAecIOILngeBlf+iyP1Tchy4HAnxzW718+kn9lgtJMGiRnEiLlERSkxJpmXLolETnrGRapSbRKjWJ9NApIy2JjLRkWqclkdkimcyWyWS1SCGrZTJpySpjEYms+hR6F2BHrev5wKhjLeOc85lZCZANFNZeyMxuAW4BSM/JpX1SJWZgQIJZrctHnDASEhJIOOK+xARINPv39dBtSQY11VW0apFGUkLw9uQEIymB0MlITiB0MpITg5cTDeywo9U7wBc61aEmdCqFCoKnXfX4B62PsrIy8vLywvRskRfL+WM5Oyi/17zMH9EPRZ1z04BpACNGjHBv3T2hydaVl5fHmDFjmuz5m5ryeyeWs4Pye83L/PXZVGIn0K3W9a6h2+pcxsySgEyCH46KiEiE1KfQFwJ9zaynmaUAVwOzj1hmNvDd0OXvAPOON38uIiLhd8Ipl9Cc+B3APwlutjjdObfKzB4CFjnnZgMvAK+a2UagmGDpi4hIBNVrDt059x7w3hG3PVDrciVwRXijiYhIQ+jrhiIicUKFLiISJ1ToIiJxQoUuIhInzKutC81sL7CtCVfRjiO+qRpjlN87sZwdlN9rTZ2/h3OufV13eFboTc3MFjnnRnido7GU3zuxnB2U32te5teUi4hInFChi4jEiXgu9GleBzhJyu+dWM4Oyu81z/LH7Ry6iEhzE88jdBGRZkWFLiISJ+K+0M3sTjNba2arzOz3XudpDDP7qZk5M2vndZb6MrP/Dv27f2VmfzWzLK8z1YeZTTCzdWa20czu8TpPQ5hZNzP7yMxWh97vP/I6U0OZWaKZLTWzd7zO0hhmlmVmM0Lv/TVmdlYk1x/XhW5mYwke73Soc24w8KjHkRrMzLoBFwLbvc7SQB8AQ5xzpwLrgake5zmhWgdEnwgMAq4xs0HepmoQH/BT59wg4EzghzGWH+BHwBqvQ5yEPwBznHMDgKFE+LXEdaEDtwO/dc5VATjn9nicpzGeAH5O8ACnMcM5975z7tDBWL8geKSraPfNAdGdc9XAoQOixwTnXIFzbknocinBMunibar6M7OuwCXA815naQwzywRGEzw+BM65aufc/khmiPdC7wecZ2YLzOxfZjbS60ANYWaTgJ3OueVeZzlJNwH/8DpEPdR1QPSYKcTazCwXOB1Y4G2SBnmS4OAl4HWQRuoJ7AVeDE0bPW9mrSIZIKIHiW4KZjYX6FjHXfcSfH1tCf75ORJ428x6RdPh8U6Q/5cEp1ui0vGyO+f+FlrmXoJTAa9FMltzZmbpwF+Au51zB7zOUx9mdimwxzm32MzGeJ2nkZKAYcCdzrkFZvYH4B7g/kgGiGnOufHHus/Mbgdmhgr8SzMLENxxzt5I5TuRY+U3s1MI/sZfbmYQnLJYYmZnOOd2RTDiMR3v3x7AzG4ALgXGRdMv0eOozwHRo5qZJRMs89ecczO9ztMA5wDfNrOLgTSgtZn9r3PuWo9zNUQ+kO+cO/RX0QyChR4x8T7lMgsYC2Bm/YAUYmQvbs65Fc65Ds65XOdcLsE3y7BoKfMTMbMJBP98/rZzrtzrPPVUnwOiRy0L/uZ/AVjjnHvc6zwN4Zyb6pzrGnqvX03wQPOxVOaEfjZ3mFn/0E3jgNWRzBDzI/QTmA5MN7OVQDXw3RgZKcaDZ4BU4IPQXxhfOOdu8zbS8R3rgOgex2qIc4DrgBVmtix02y9DxwSWyLgTeC00INgM3BjJleur/yIicSLep1xERJoNFbqISJxQoYuIxAkVuohInFChi4jECRW6iEicUKGLiMSJ/w+a+bLGSkCgbAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "def sigmoid(x):\n", - " return 1 / (1 + np.exp(-x))\n", - "\n", - "xs_sig = np.linspace(-7, 7, 100)\n", - "ys_sig = sigmoid(xs_sig)\n", - "plt.plot(xs_sig, ys_sig, label=r'$\\sigma(x) = {\\frac{1}{1 + e^{-x}}}$')\n", - "plt.yticks([0, 0.5, 1])\n", - "plt.xlim([-7, 7])\n", - "plt.legend()\n", - "plt.grid()\n", - "plt.title('Sigmoid function')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5xU1f3/8ddnZwu7LL0LyEpEFAttEYyKoCaC2BNr7IXYicbeomgQlagRNQTFrwGjGKLmZ6yJUSxYIigWBJQSZJGO9O17fn+cWVlwYXdhd86U9/PxuI/dmXu59zMDvOfMufeeY845REQk8aWFLkBEROqHAl1EJEko0EVEkoQCXUQkSSjQRUSShAJdRCRJKNBFqjCzjWbWNUbHGmdmt8biWJIaTNehSyyZ2VSgJ9DeOVccuJyYMbNzgQudc4eErkWSl1roEjNmlgccCjjguAY6RnpD7FckESjQJZbOBj4EngTOqXzSzJ6Mdj/828w2mNnbZtalynpnZlea2QIzW2Vm95lZWnTduWY2zcweMLPVwO1mlmZmt5jZIjNbYWYTzaxZdPtTzWyhmTWNPh5qZsvMrE2VY+1Zpa5HzezVaFfMNDNrb2YPmtn3ZjbHzHpXqfMGM5sffQ1fmdmJ0ef3AcYBB0X3s7bK/u+q8ucvMrN5ZrbGzF40s922eQ8uNrNvzGytmT1iZlbPfz+S4BToEktnA3+NLkeZWbsq634F3Am0BmZGt6nqRCAf6AMcD5xfZV1/YAHQDvg9cG50GQx0BXKBhwGcc88C7wMPmVkrYAK+K2Tldmo+BbglWlcx8AHwSfTx34H7q2w7H/8NpBlwB/CUmXVwzs0GLgY+cM7lOueab3sQMzscuDt6vA7AImDyNpsdA/QDDohud9R2apYUpUCXmDCzQ4AuwN+cczPw4XdGlU1eds69E+1Xvxnfmu1cZf09zrk1zrlvgQeB06us+845N9Y5V+acK8R/ONzvnFvgnNsI3AicVqU75jLgcGAq8E/n3Es7KP0F59wM51wR8AJQ5Jyb6JwrB54FfmihO+emOOe+c85VRD84vgEOrOVb9CvgCefcJ9H34Mboe5BXZZvRzrm10ffgLaBXLfctKUKBLrFyDvAv59yq6OOnqdLtAiyu/CUawmuA3apbj2+9bm8d0XWLttk+Hd+Cxzm3FpgC7Af8oYa6l1f5vbCax7mVD8zsbDObGe0SWRvdf+sa9l9tzdH3YDXQsco2y6r8vrnqsUXA/yMXaVBmlo3vIoiYWWUoZQHNzaxn9HHnKtvnAi2B76rspjMwK/r77tus2/ZSre/w3wYq7Q6UEQ1jM+uF77J5BngIGLJTL6yKaJ//Y8AR+K6VcjObCVT2c9d0OdlWNZtZY6AVsGRXa5PUoRa6xMIJQDnQA99N0AvYB3gX368OcLSZHWJmmfi+9A+dc1Vb3teaWYtoN8wIfHfH9jwDXGVme0Q/HEYBzzrnysysEfAUcBNwHtDRzC6th9fYGB/aKwHM7Dx8C73ScqBT9PVtr+bzzKyXmWVFa/7IOfe/eqhNUoQCXWLhHOD/nHPfOueWVS74E5W/wn9TfBr4Hb6rpS9w5jb7+H/ADPwJ05fxJzO35wlgEvAOsBAoAq6IrrsbWOyc+1O0r/pM4C4z67YrL9A59xW+++YDfHjvD0yrssmb+G8Yy8xsVTV//g3gVuA5YCnwE+C0XalJUo9uLJLgzOxJoMA5d8t21jugm3NuXkwLE0kwaqGLiCQJBbqISJJQl4uISJJQC11EJEkEuw69devWLi8vL9ThRUQS0owZM1Y559pUty5YoOfl5TF9+vRQhxcRSUhmtmh769TlIiKSJBToIiJJQoEuIpIkFOgiIklCgS4ikiQU6CIiSUKBLiKSJGoMdDN7IjrR7pfbWW9m9lB0ctvPzaxP/ZcpIiI1qc2NRU/ix62euJ31Q4Fu0aU/8Kfozwbz6qvwwQdw0EEwdGjt/9zcufDNN9CtG3TvvvW+OneGJUtg5Uro1Qs6dIClS2Hx4i3Hqfzzn34Kb7wBubmw557+uZYt4ZBDYPZsv66sDDIyoHFjaNMGNm3y++7ZE1q3hmnTYO+94bDD4K23/D5ycvyxV64E5+Dww31NU6f6n2aw227QpAnMmQPFxXDooXDmmVBeDpHIlp8vvwwLF8IJJ8DAgTB27NaPn3nGv57WraFVK2jfHgYPhgULtry3Xbv62sCvgx+/f7V9n3dGfe1HJFXUanCu6ES1Lznn9qtm3Z+Bqc65Z6KP5wKDnHNLd7TP/Px8tzN3ir7/wEd0uu50KixCORHadojQpFnEp1gkAunp1S4bitKZPS+DikgGZWSwX58M1hVm8ua7mRS7TL7fnEWxZVFqWRSTRfP2jfh2RSMiTbIpizTihDNy+GR2NvOX5vDB543Z4BqzkVw20RjSIjgH2dmweXPdXk96ug/hqn8NZpCW5heA0tId76N5c+jXD+bPh5/8BD75BNau9ftJT/cfTosXbzlely7w7bdQUgIVFZCV5T90OnaEggJo1AiKiqBTJ//hApCX5z/AWrb09d50U/UhO3cujBq15cNle9vVpL72I5JszGyGcy6/unX1cet/R7aepLcg+tyPAt3MhgPDAdq1a8fUqVPrfLCNZYto2XtP0inHlVXwfXoFRVkVWEUFVl7ufxYWYmVlW54rL6eipIy9SstJKy7HyspIf7OMdqWlnFFaRqS8lEhFKVY1VStfUXH05wMwbDs1lUayKMnKobhRY0pa+p/F2bkUZzehODuXwpwmFOU0pTCnGZtzm1OY25zNuc0pymnqU7caZluH/I6Y+W8CpaX+W8HQoT6o09L8Pir3Vfmz8vnKxQwyM7c8rgz0yg8E8M+np/tvByUl/hvC0mo+stet898sMjN3vF1N6ms/IqkkpmO5OOfGA+PBt9AHDRpU5328WgjDxp1DWpoPrYcfrl23S3UtvgUL4PLLfffIiuWOdMrITismi2J6dC1i6cIi2jUtpHFaIRecUcicTwv5vmATBV9vJtttIpeN5LKRpqUbaFK6gRaF68kpX09z1tKMNbRgAS1ZQw6F1dZUQgbLaM8y2lNApx+Wb+nCorQ9+DYtj5W0obSs+tCvpBa6iEACdrlA4vWh79ayiPR1q6lYsYq+XVbRMWMFK2etYK+my9i72VKKFi6l8doltCtdTG75+q1qLs5swqLMPfmGbizM3JvV7Xqwsk0P3lqyFxtKstSHLpJidtTlUh+BPgy4HDgafzL0IefcgTXtc1cCPamtX++bzwsX+mX+fJ9q33zj07aiwm+Xng49evhPnz59oH9/6N3bN7dFJGntUh+6mT0DDAJam1kBfmb2DADn3DjgFXyYzwM2A+fVT9kpqmlT2G8/v2yrqAi+/hpmzYIvvoCZM+Ff/4KJ0QuQMjJ8uA8c6C+fOeQQaNYstvWLSDDBpqBTC70eLVkCH33kl2nT4L//9WdIIxHfcj/qKN9n1LfvlktnRCQh7XKXS0NQoDegwkL48EP4z3/g9ddhxgx/mUrHjnD88XDSSTBokA98EUkoCvRUt2qVP/v7wgvw2ms+8Dt0gNNPh7PO8v3wIpIQdhTo+v6dClq39sH9/PM+3KdM8V0xY8f6E6n9+sGECf52VhFJWAr0VJOTA7/8pW+tL1vmQ72wEC680HfJXH89fPdd6CpFZCco0FNZy5b+zqovvoD33vMnT8eM8XcRnX8+zJsXukIRqQMFuvhbSg8+GJ591l/vPnw4TJ7sRw8bPnzLbaYiEtcU6LK1rl39eArz58Oll8Jf/uJv1bzxRti4MXR1IrIDCnSpXocO8NBD/kamU0+F0aNhr71g0qTajxomIjGlQJcd69LFt9I/+MCfND37bDjiCD8MgYjEFQW61M6AAf5O1HHjYPp02H9/34KvHFtGRIJToEvtpaXBr3/tx5I57DAYMQJ+9jMNVC4SJxToUnedO/vxeSdM8EMM9Ozp70AVkaAU6LJzzPy16tOn+8HUhw6Fm29WF4xIQAp02TX77OP71i+80E8xdMIJsGFD6KpEUpICXXZddjaMH++vX3/lFT/dka6CEYk5BbrUDzO47DI/4cbSpfDTn/q5+kQkZhToUr8OP9xPspGV5cdcnzo1dEUiKUOBLvVv7719qHfqBEOGwD/+EboikZSgQJeG0akTvPuuH2/95JMV6iIxoECXhtOypZ8Cr29fOOUU+Oc/Q1ckktQU6NKwmjb1od6rF/ziF/6GJBFpEAp0aXjNmvmrXw44wM+W9P77oSsSSUoKdImN5s39RNWdO8Oxx8KcOaErEkk6CnSJnTZt/JgvGRl+ujvNXSpSrxToEltdu/q7SdesgWHDYNOm0BWJJA0FusRenz4wZQp8/jmcd55mQBKpJwp0CWPIELjnHh/so0aFrkYkKSjQJZzf/hbOPBNuuQVefDF0NSIJT4Eu4Zj5URrz832wz5sXuiKRhKZAl7Cys+G55yA9HU49FYqLQ1ckkrAU6BLe7rvDk0/CJ5/AtdeGrkYkYSnQJT4cdxxcdRWMHQvPPx+6GpGEpECX+DF6NPTr5+cqXbw4dDUiCUeBLvEjMxOeeQbKynyoa8JpkTpRoEt8+clP4A9/gDfegD/9KXQ1IglFgS7xZ/hwf+PRtdfC11+HrkYkYSjQJf6YwYQJ0KgRnHMOlJeHrkgkISjQJT7tths88gh8+KG/8kVEaqRAl/h12mlw9NF+aIBFi0JXIxL3FOgSv8y2nBi95BKNyihSAwW6xLfdd4ff/97PdjR5cuhqROKaAl3i3+WX+xuORozwE2OISLUU6BL/IhE/KuPq1XDbbaGrEYlbCnRJDL16waWX+j71mTNDVyMSlxTokjhGjoSWLeGKK3SCVKQaCnRJHC1a+AG83nsPnn46dDUicUeBLonlvPP8CdJrroENG0JXIxJXFOiSWNLS4OGHYdky31oXkR8o0CXxHHggnHEG3H+/xk0XqUKBLolp1Ch/YvTmm0NXIhI3FOiSmLp08VPWTZoEM2aErkYkLijQJXHdcAO0bg2//a0uYxRBgS6JrFkzuOMOePtteOml0NWIBKdAl8R20UWw556+L11zkEqKU6BLYsvIgDvvhC++8BNMi6QwBbokvlNO8WO93HYblJSErkYkGAW6JL60NH8Z44IF8PjjoasRCUaBLslhyBA49FDf/bJpU+hqRIJQoEtyMIO77/ZDAjzySOhqRIJQoEvyOPhgOOoouO8+2LgxdDUiMadAl+Ryxx2wapUfwEskxSjQJbn07w9Dh/pWuobXlRSjQJfkc8cdfjLpsWNDVyISUwp0ST79+sExx8CYMbBuXehqRGJGgS7J6fbb4fvv1UqXlKJAl+TUty8MGwYPPqgrXiRlKNAled1yC6xeDePGha5EJCYU6JK8BgyAI4/0femFhaGrEWlwCnRJbrfeCsuXa4wXSQkKdEluAwf6MV7uvReKi0NXI9KgFOiS/G69FQoK4C9/CV2JSINSoEvyO/JIf236vfdCWVnoakQajAJdkp8Z3HgjzJ8Pf/976GpEGowCXVLD8cfD3nvD6NHgXOhqRBqEAl1SQ1oaXH89fPYZvPpq6GpEGoQCXVLHGWdA585+IgyRJKRAl9SRmQnXXAPvvecXkSSjQJfUcuGF0Lq170sXSTIKdEktOTlwxRXw8svw5ZehqxGpVwp0ST2XXeaDfcyY0JWI1CsFuqSeVq1818tf/wqLF4euRqTeKNAlNV11lb8e/cEHQ1ciUm8U6JKa8vLgtNNg/Hg/s5FIElCgS+q69lo/m9Gjj4auRKReKNAldfXsCUcd5ecdLSoKXY3ILlOgS2q77jo/AcakSaErEdllCnRJbYMHQ58+/hLGiorQ1YjsEgW6pDYz30r/+mt48cXQ1YjskloFupkNMbO5ZjbPzG6oZv25ZrbSzGZGlwvrv1SRBvKLX/irXu67L3QlIrukxkA3swjwCDAU6AGcbmY9qtn0Wedcr+iiGXklcaSnw9VXw/vvw7RpoasR2Wm1aaEfCMxzzi1wzpUAk4HjG7YskRg7/3xo2VKtdElo6bXYpiNQ9f7oAqB/Ndv9wswGAl8DVznnfnRPtZkNB4YDtGvXjqlTp9a5YJGGkjdsGF2eeoqPJ05k8+67hy5HpM5qE+i18U/gGedcsZn9GvgLcPi2GznnxgPjAfLz892gQYPq6fAi9aBHD5gyhQPffRceeyx0NSJ1VpsulyVA5yqPO0Wf+4FzbrVzrjj68HGgb/2UJxJDbdvCuefCxImwbFnoakTqrDaB/jHQzcz2MLNM4DRgq+u7zKxDlYfHAbPrr0SRGLr6aigt9XePiiSYGgPdOVcGXA68jg/qvznnZpnZSDM7LrrZlWY2y8w+A64Ezm2ogkUaVLducOKJfnyXDRtCVyNSJ+acC3Lg/Px8N3369CDHFtmhjz6CAQPggQfgN78JXY3IVsxshnMuv7p1ulNUZFv9+8Ohh8L99/vuF5EEoUAXqc511/nZjJ59NnQlIrWmQBepztFHw777wr33+pmNRBKAAl2kOmlpfgKML76A114LXY1IrSjQRbbn9NOhUye4557QlYjUigJdZHsyM/1k0m+/7a98EYlzCnSRHbnoImje3Peli8Q5BbrIjjRpApddBi+8AHPmhK5GZIcU6CI1ufJKyMrS0LoS9xToIjVp2xYuuMBPJF1QELoake1SoIvUxjXX+Emk778/dCUi26VAF6mNvDx/GeP48bB6dehqRKqlQBepreuvh02b4OGHQ1ciUi0Fukht7bcfHHssPPQQbNwYuhqRH1Ggi9TFTTfBmjUwblzoSkR+RIEuUhcDBsCRR8KYMVBYGLoaka0o0EXq6pZbYPlymDAhdCUiW1Ggi9TVwIFwyCF+0K6SktDViPxAgS5SV2a+lV5QABMnhq5G5AcKdJGd8fOfQ9++cPfdUFYWuhoRQIEusnPM4LbbYMECeOqp0NWIAAp0kZ137LHQpw/ceacmk5a4oEAX2VlmcPvtaqVL3FCgi+yKY47xfel33aVWugSnQBfZFVVb6ZMmha5GUpwCXWRXDRsG+flqpUtwCnSRXWUGI0fCwoW6e1SCUqCL1IchQ/zdoyNHaowXCUaBLlIfzGDUKFi6VOOlSzAKdJH6cuihvqU+ejSsWxe6GklBCnSR+vT73/vx0jX3qASgQBepT336wMkn+0Bfvjx0NZJiFOgi9e3OO/2J0TvuCF2JpBgFukh9694dLr4Yxo+HOXNCVyMpRIEu0hB+9zvIyYHrrw9diaQQBbpIQ2jTxk8o/eKLMHVq6GokRSjQRRrKiBHQuTNccw1UVISuRlKAAl2koWRn+5uNZszQVHUSEwp0kYZ0xhkwYADccINuNpIGp0AXaUhpaTB2LKxY4cd5EWlACnSRhpafDxdcAA89BLNnh65GkpgCXSQWRo2Cxo39iVLnQlcjSUqBLhILbdr4Lpd//xueey50NZKkFOgisXLppdC7N1x5JaxdG7oaSUIKdJFYSU+Hxx7zg3bdeGPoaiQJKdBFYqlvX9+PPm4cTJsWuhpJMgp0kVgbORJ23x2GD4eSktDVSBJRoIvEWm4uPPoofPUV3HVX6GokiSjQRUIYNgzOOstfzjh9euhqJEko0EVC+eMfoX17OPtsKCoKXY0kAQW6SCgtWsCECf7u0dtuC12NJAEFukhIRx0Fv/41jBkD774buhpJcAp0kdDuuw+6dvUjM65eHboaSWAKdJHQmjSByZP9DUfnn6+xXmSnKdBF4kF+Ptx7r5+ybuzY0NVIglKgi8SLESPguOPg2mv9LEcidaRAF4kXZvDEE9CuHZx4op8UQ6QOFOgi8aRVK3jhBVi5Ek45BUpLQ1ckCUSBLhJv+vb1ozK+/TZcfXXoaiSBpIcuQESqceaZ8OmncP/9cMABcNFFoSuSBKAWuki8uucef+PRJZfAq6+GrkYSgAJdJF6lp8OUKbD//nDyybryRWqkQBeJZ02awCuv+JOlw4bBwoWhK5I4pkAXiXcdOsBrr/nJMI48EgoKQlckcUqBLpII9tnHh/rKlXD44bB0aeiKJA4p0EUSxYEH+lD/7js44gjdeCQ/okAXSSQ//anvU//f/2DQIHW/yFYU6CKJZuBAfxljQQEcfDB8/XXoiiROKNBFEtFhh8HUqVBYCIccAp98EroiiQMKdJFE1acPvPceZGf7Vvs//hG6IglMgS6SyPbaCz74AHr0gJNOgrvv1gQZKUyBLpLodtvND+R16qlw001+HJiNG0NXJQEo0EWSQXY2PP003HUXPPOMnwHp889DVyUxpkAXSRZmcPPN8J//wLp10L8/jBunLpgUokAXSTaDB8Nnn/kTpZdcAkOGwLffhq5KYkCBLpKM2rb116o/+ihMmwb77Qd//jNUVISuTBqQAl0kWaWl+Rb6F19Av35w8cVw0EHw8cehK5MGokAXSXZ77AFvvAGTJvmul/794cILYcmS0JVJPVOgi6QCM38549y5fp7SiRNhzz3hmmtg1arQ1Uk9UaCLpJKmTWHMGD/+y6mnwgMP+Bb8tdf6URwloSnQRVJRXh48+aTvXz/2WD8ZdV4eXHABzJwZuDjZWQp0kVTWo4e/IWnePBg+3N+U1Lu3H6Z30iTYvDl0hVIHCnQR8d0uDz/sT5Q+8ACsXg1nnw3t28N558Gbb0J5eegqpQYKdBHZokUL+M1vYM4ceOstOPlkeO45P0NShw5w0UX++vaiotCVSjXMBbotOD8/302fPj3IsUWkDjZvhpdfhuefh5de8gN/ZWf7O1KHDPETV++9t7+SRhqcmc1wzuVXu06BLiK1VlTku19ee8231OfN88+3besn3Tj4YBgwAHr1gqyssLUmqR0FenqsixGRBNaoERx9tF8AFizwMydVLlOm+OczM2H//f0J1t694YADYN99fZeONBi10EWk/ixZAh99BB9+6KfF+/RTWLNmy/oOHWCffaBbNz85x557+hOyeXnQpEmwshPJLne5mNkQ4I9ABHjcOTd6m/VZwESgL7AaONU5978d7VOBLpICnIPFi+HLL2HWLL/MnetvbKoa9AAtW0KnTtC5s//ZocOWpW1bv7RpA40bp3R//S51uZhZBHgE+BlQAHxsZi86576qstkFwPfOuT3N7DTgHuDUXS9dRHbG3Ln+IhXw5y67d/dd3i+95DOxf39/FWIkAosW+e26dPHPdevmt6/06qt+lrvOnf36d9/1c2esXg0ZGX6O6kGDfG537uzzt1s3eOcdeOEFo2vX3enZc3fe+vRo1q+HPfJhTVdYt2A1vZstIM8tJGf5QpqtXUTOgsV0+KaATvYRjTdXPyRBEVmsS29FcW4r1rgWrCprzqqyFqxPa0ajtk3p1rcZexzQhGWbmjBrUS7NO+XSbLfGvDy1MZ/MyWGvXjn0+mkO5RmN6Nw1g/IKIxLZ+rXPnesvyV+xwt93NXTolvf1m29+/B5Vfd93tL7yvTzooC37rE81ttDN7CDgdufcUdHHNwI45+6uss3r0W0+MLN0YBnQxu1g52qhizSMuXPhhhv8TaDgR84dNgxGjvTzXpSX+xEA9t8fZs/2gzJW/k/t1w9yc/1MdpUfApdfDmVlfsiXSAQ2bPjxMTMy/AfF+vU+rDZs8DeclpX5EXvT0vxP534830bluqrMoFWTEhpvXE57ltE+bQWtylfQ2q2gFat/WJqzlhZ8T3PW0ox1NGU9adS+G7mcNErTGlFIIyoyG1GenkVOiyyWf5/J2s2ZlFgWLj2Dbvtmkt0kg7kLMqiIZFBqGex7QDrNWmf4NyU9ne83pPPR9AguEqHcRTjo0HRatY34FxiJMHdehH/8Mw1nEZ5vfBZ3jGu3U6G+S10uZvZLYIhz7sLo47OA/s65y6ts82V0m4Lo4/nRbVZts6/hwHCAdu3a9Z08eXLdX42I7NC6db5lWVbmH0ciPnArg7i83AdmdjYUF/u8AR+qTZr4x82bQ7NmfniXNWv8c8XF1QdypcxMf8ycHCgpgdJSf5ydHYJ9216VWp3uc47MkkJy3WYimzeTXVaI21hIRkkR6cVFpJcUkVFaTEZpMZmuhEhJMVkVxVhxKY0oIVJWSnpFCeWFZaSVlJJWUUakvIwsSkl3ZVSUlBOpKMPKykh35URcOVbuF1de4X+vqMAqKkir8L9XZ/J146no2Y3ddqv7+zJ48OD4uMrFOTceGA++hT5o0KBYHl4kJcydu2WYFtjSQr/vvp1rod96a5gWetOmW+a6jkS27KsmTZv6EQ0KCvy+16zxH1AFBVtv07y5398++8D8Jf5Ky9xcOO00+MMf/LldM//BNn48dO0Ko0bxQ/dM5XtU9X2vdn1FBZSX8/qrFVw9opz0tAo2T8nmoUER6jsCaxPoS4DOVR53ij5X3TYF0S6XZviToyISY927w+jRP+5D79Sp7n3oQ4f6EQF2vg/dB2HPnr6e9ev9RS1r1mzZvkUL/3jjRn8VZHY2HHecr3PWLH+cvfaCjh3hscdg4UI/IkHPnn4fS5fC2rW+3n339YNIDh7s91VZd4cO8Le/wYwZ0LevH6qm6mvetg+9a9fq+9Bvumn7feTdu29nfVoapKVx1HEwJiN8H3o68DVwBD64PwbOcM7NqrLNZcD+zrmLoydFT3LOnbKj/aoPXUSk7nbpKhfnXJmZXQ68jr9s8Qnn3CwzGwlMd869CEwAJpnZPGANcFr9lS8iIrVRqz5059wrwCvbPHdbld+LgJPrtzQREakLjbYoIpIkFOgiIklCgS4ikiQU6CIiSUKBLiKSJBToIiJJIth46Ga2EtgEVD+kWvxqTWLVnGj1QuLVnGj1gmqOhYaqt4tzrk11K4IFOoCZTd/eHU/xKtFqTrR6IfFqTrR6QTXHQoh61eUiIpIkFOgiIkkidKCPD3z8nZFoNSdavZB4NSdavaCaYyHm9QbtQxcRkfoTuoUuIiL1RIEuIpIkgge6mZ1sZrPMrMLM4vaSJDMbYmZzzWyemd0Qup6amNkTZrYiOt9r3DOzzmb2lpl9Ff33MCJ0TTUxs0Zm9l8z+yxa8x2ha6oNM4uY2adm9lLoWmrDzP5nZl+Y2UwzS4hZccysuZn93czmmNlsMzsoFscNHujAl8BJwDuhC9keM4sAj0fanBkAAAKtSURBVABDgR7A6WbWI2xVNXoSGBK6iDooA37rnOsBDAAuS4D3uBg43DnXE+gFDDGzAYFrqo0RwOzQRdTRYOdcrwS6Dv2PwGvOub2BnsTo/Q4e6M652c65uaHrqMGBwDzn3ALnXAkwGTg+cE075Jx7Bz97VEJwzi11zn0S/X0D/j9Ax7BV7ZjzotMYkxFd4voqAzPrBAwDHg9dS7Iys2bAQPxMbjjnSpxza2Nx7OCBniA6AourPC4gzsMmkZlZHtAb+ChsJTWLdl/MBFYA/3bOxXvNDwLXARWhC6kDB/zLzGaY2fDQxdTCHsBK4P+iXVuPm1njWBw4JoFuZm+Y2ZfVLHHdypXYM7Nc4DngN8659aHrqYlzrtw51wvoBBxoZvuFrml7zOwYYIVzbkboWuroEOdcH3yX52VmNjB0QTVIB/oAf3LO9caPWRWT8261mlN0VznnjozFcRrQEqBzlcedos9JPTKzDHyY/9U593zoeurCObfWzN7Cn7eI1xPRBwPHmdnRQCOgqZk95Zw7M3BdO+ScWxL9ucLMXsB3gcbtOTf8N/iCKt/W/k6MAl1dLrXzMdDNzPYws0zgNODFwDUlFTMzfJ/jbOfc/aHrqQ0za2NmzaO/ZwM/A+aErWr7nHM3Ouc6Oefy8P+G34z3MDezxmbWpPJ34OfE7wcmAM65ZcBiM+sefeoI4KtYHDt4oJvZiWZWABwEvGxmr4euaVvOuTLgcuB1/Mm6vznnZoWtasfM7BngA6C7mRWY2QWha6rBwcBZwOHRy9NmRluS8awD8JaZfY7/0P+3cy4hLgVMIO2A98zsM+C/wMvOudcC11QbVwB/jf7b6AWMisVBdeu/iEiSCN5CFxGR+qFAFxFJEgp0EZEkoUAXEUkSCnQRkSShQBcRSRIKdBGRJPH/AZg50RKotGwHAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "# helper function for approximation\n", - "def sigmoid_mu_s(x, mu, s):\n", - " return 1 / (1 + np.exp(-(x + mu) / s))\n", - "\n", - "plt.scatter(X[:, 0], Y, s=10, c='b', alpha=0.5)\n", - "sig_x = np.linspace(X[:, 0].min(), X[:, 0].max(), 100)\n", - "y_flip = 1 if Y[X[:, 0].argmin()] == 0 else -1\n", - "sig_y = sigmoid_mu_s(sig_x, -X[:, 0].mean(), y_flip * 0.5)\n", - "plt.plot(sig_x, sig_y, c='r')\n", - "plt.yticks([0, 0.5, 1])\n", - "plt.grid(axis='y')\n", - "plt.title('Approximation')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Hypothesis and Prediction\n", - "\n", - "We can tweak the sigmoid function by passing the dot product of $x$ (vector of feature) and $\\theta$ (the vector of weights).\n", - "\n", - "$$h(x) = \\sigma(x \\cdot \\theta) = \\sigma(\\theta_0 x_0 + \\theta_1 x_1 + \\cdots + \\theta_n x_n)$$\n", - "$x_0$ being equal to 1. \n", - "$\\theta$ is not a parameter of $h$ because it's considered as a constant.\n", - "\n", - "$h$ is our hypothesis function. It return a probability of an $x$ to belong to one of the group. \n", - "To predict some new $x$ we input it to $h$ with the weights and see if the returned value if above or below 0.5.\n", - "\n", - "$$\n", - "\\text{prediction} =\n", - " \\begin{cases} \n", - " 1, & h(x) \\ge 0.5 \\\\\n", - " 0, & h(x) \\lt 0.5\n", - " \\end{cases}\n", - "$$\n", - "\n", - "This is equivalent to:\n", - "\n", - "$$\n", - "\\text{prediction} =\n", - " \\begin{cases} \n", - " 1, & x \\cdot \\theta \\ge 0 \\\\\n", - " 0, & x \\cdot \\theta \\lt 0\n", - " \\end{cases}\n", - "$$\n", - "\n", - "Indeed, in the sigmoid funcion: \n", - "* $x$ is *above* 0 then $e^{-x}$ tend to be small, $\\frac{1}{1 + \\text{small number}} \\approx 1$. \n", - "* $x$ is *below* 0 then $e^{-x}$ tend to be big, $\\frac{1}{1 + \\text{big number}} \\approx 0$. \n", - "\n", - "Therefore, a prediction can be visualized by projecting $x$ onto $\\theta$. \n", - "It will be more relevant the more the two vectors go in the same direction\n", - "(or opposite direction for negative projection),\n", - "for exemple if $x \\cdot \\theta = 0$ then $x$ is perpendicular to $\\theta$,\n", - "$h(x) = 0.5$ right in the middle of the sigmoid function,\n", - "the prediction become more irelevant as $x \\cdot \\theta$ tend to 0." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def hypothesis(x, theta):\n", - " # suppose that x_0 = 1 and isn't mentionned\n", - " return sigmoid(x.dot(theta))\n", - "\n", - "\n", - "def predict(x, theta):\n", - " return int(hypothesis(x, theta) >= 0.5) # a bit more clear\n", - " # or\n", - " return int(x.dot(theta) >= 0)\n", - " # there is a nice vector interpretation for this one\n", - " # and it probably compute faster" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Cost function\n", - "\n", - "We have to evaluate our model by introducing a cost function, this one is called the *logloss*\n", - "\n", - "For a single $x$:\n", - "$$\n", - "j(h(x), y) = \n", - " \\begin{cases} \n", - " -\\ln(h(x)), & y = 1 \\\\\n", - " -\\ln(1 - h(x)), & y = 0 \n", - " \\end{cases}\n", - "$$\n", - "It can be rewritten as a single continuous function:\n", - "$$ j(h(x), y) = -y \\ln(h(x)) - (1 - y) \\ln(1 - h(x))$$\n", - "\n", - "Even if it may seem more complicated at first sight, it's just a switch with the values of $y$. \n", - "- if $y = 1$ then $- (1 - y) \\ln(1 - h(x)) = 0$ and the $-\\ln(h(x))$ stays.\n", - "- if $y = 0$ then $-y \\ln(h(x)) = 0$ and the $-\\ln(1 - h(x))$ stays." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEICAYAAABCnX+uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3Rc1bn38e/WqPcuq1qyZMm25Cob3HCJDaaYDg6ETgJJLmmElJt7k8slb5JLAiEJgYTQQiDUUGJ6MxYu4G7LXbYsy1bv3aoz+/3jjOQmW8UzOjOa57PWrNFojs55RMa/bO2zi9JaI4QQwn15mV2AEEKIcyNBLoQQbk6CXAgh3JwEuRBCuDkJciGEcHMS5EII4eYkyMWooZS6WilVopRqVUpNH8Hr3qSU+nikrifEqZSMIxfuQCmVB/xTa/30WY45BPxQa73SiXWkAocBH611j7OuI8RQSItcjCZjgT1mFyHESJMgF06hlEpWSr2plKpRStUppR6zf99LKfVzpdQRpVS1Uup5pVSY/T1/pdQ/7cc3KqU2K6XilFK/Bi4AHrN3mzx2yrX8lFKtgAXIt7fMUUpppVTGCcc9p5T6lf3rRUqpUqXUffY6KpRSd5xwbIBS6vf2OpuUUuuUUgHAGvshjfZa5iilbldKrTvhZ+faa2+yP8894b08pdT/U0qtV0q1KKU+VkpFO/a/vvA0EuTC4ZRSFuBd4AiQCiQCr9jfvt3+WAyMA4KB3mC+DQgDkoEo4FtAu9b6v4G1wHe01sFa6++ceD2tdafWOtj+cqrWOn2QpY6xXy8R+DrwuFIqwv7ew0AuMBeIBH4C2IAF9vfD7bV8ecrvHgm8Bzxq/x0eAd5TSkWdcNjXgDuAWMAX+NEg6xWiXxLkwhnOAxKAH2ut27TWHVrr3hbrTcAjWusirXUr8DPgBqWUN9CNEX4ZWmur1nqr1rrZiXV2A7/UWndrrd8HWoEspZQXcCfwfa11mb2WL7TWnYM452XAQa31C1rrHq31y8B+4PITjvm71vqA1rodeA2Y5thfS3gaCXLhDMnAkTPcDEzAaKn3OgJ4A3HAC8BHwCtKqXKl1O+UUj5OrLPulBqPYfyFEA34A4eGcc5Tfz/srxNPeF3ZzzWFGDYJcuEMJUCKvZV9qnKMm5K9UoAeoMreMn5Aaz0Jo0tjOXCr/bjhDK86BgSe8HrMIH+uFugA+uuiGaiOU38/MH7HskFeW4ghkyAXzrAJqAAeVEoF2W9izrO/9zJwr1IqTSkVDPwGeFVr3aOUWqyUmmzvY2/G6Pqw2X+uCqNPfSh2AF9TSlmUUhcDCwfzQ1prG/As8IhSKsH+83OUUn5Ajb2mM9XyPpCplPqaUspbKfVVYBLGPQMhnEKCXDic1tqK0SecARwFSoGv2t9+FqMLZQ3GeOwO4Lv298YAr2OE+D7gc/uxAH8CrlNKNSilHh1kKd+319GI0Tf/7yH8Gj8CdgGbgXrgt4CX1voY8GtgvX1kzewTf0hrXYfxl8R9QB3GTdLlWuvaIVxbiCGRCUFCCOHmpEUuhBBurr+bUadRShUDLYAV6NFaz3RmUUIIIQZvUEFut1j6+YQQwvVI14oQQri5Qd3sVEodBhowxtD+TWv9ZD/H3A3cDeDv758bn5RCWauNmABFkI9ycNlCDI/NZsPLS9ovwnw+3c34d1TTFpyKTXlz4MCBWq11zHDONdggT9RalymlYoFPgO9qrdec6fisrCz96ZfbueB3q3n4+qlcl5s0nNqEcLi8vDwWLVpkdhlCwMa/wQc/gZ8chsBIlFJbh3v/cVBNE611mf25GngLYy2Ns/LzNk7d2WMdTl1CCDG69XQYz95+53yqAYPcPjMvpPdr4CJg90A/5+dtAaCj2zbAkUII4YG6e4Pc/5xPNZhRK3HAW0qp3uNf0lp/ONAP+flIi1wIIc6opwO8fMDLcs6nGjDItdZFwNShnriva0Va5Kbq7u6mtLSUjo4Os0txCWFhYezbt8/sMkzn7+9PUlISPj7OXFxSnFVPp0Na4zC0ceRDopTC19uLDmmRm6q0tJSQkBBSU1Ox/1Xl0VpaWggJCTG7DFNpramrq6O0tJS0tDSzy/FcPR0O6R8HJ48j9/P2kha5yTo6OoiKipIQF32UUkRFRclfaWbr6XBYi9ypQe7vY6GzR4LcbBLi4lTymXABPR3g4wZB7uftJTc7hRCiPw7sI5euFSGEMIP79JFbpEUuhBD96XabPnIv6SMXDnXnnXcSGxtLTk6OU84fHDzwPsjt7e0sXLgQq9VopKxatYpbbrllwOP609XVxYIFC+jp6en3tRjF3OVmp5+3hY5uaZELx7n99tv58MMB56M51bPPPss111yDxWJM5MjPz2f69OkDHtcfX19flixZwquvvtrvazGKuU0fubTIBbB7927mzp3b93rbtm0sWbJkWOdasGABkZGRjiqtX8XFxUycOJG77rqL7OxsLrroItrb2/vef/HFF7nyyiv7Xufn51NZWcmCBQtISUnh008/7fe4xYsX88knnwDw85//nO9+19iq9KqrruLFF1/sO+7U12KUcmAfudMmBIHc7HQ1D7yzh73lzQ4956SEUO6/PPvsx0yaRFFREVarFYvFwg9/+EMeeeSRk4654IILaGlpOe1nH374YZYuXerQmgfj4MGDvPzyyzz11FOsWLGCN954g5tvvpmuri6KiopITU3tOzY/P5+bbrqJNWvW8NZbb/Hiiy+yYMGC04574IEH+J//+R+qq6vZvn07b7/9NgA5OTls3ry577hTX4tRyoFdK04Ncn8fi8zsFHh5eZGdnc2ePXs4ePAgY8eOZcaMGScds3btWqddf+nSpVRWVgInr0f+61//+qQW84nS0tKYNm0aALm5uRQXFwNQW1tLeHh433Hd3d3U1dVx33339b0ODw8/7Tgw/prQWvPII4+Ql5fX1+VisVjw9fXtm3V66msxSjlwHLm0yD3IQC1nZ5o9ezbr16/nL3/5S7993M5skfd2dcDgp+j7+R3/k9disfR1rQQEBJw0I3Lfvn1MnTq17/8cdu7cSU5OzmnHAezatYuKigqioqJOq6GzsxN/f/8zvhajkNv0kcvwQ2E3e/Zsfv7zn3P11VeTmJh42vtr165lx44dpz0GG+JLliyhrKzM0WWfJiIiAqvV2hfS+fn5TJ16fE25nTt3MmXKlNOOq6io4KabbmLlypUEBwef9H9mdXV1REdH9y1gdeprMUq5zzhyudkpDBMmTMDPz4+f/vSn53SeG2+8kTlz5lBQUEBSUhLPPPMMNpuNwsJCp98E7XXRRRexbt06wAjyKVOm9L23e/fuvqGRvccdO3aMa665ht///vdMnDiRX/ziFzzwwAN9P7N69Wouu+yyM74Wo5DNCtYuh7XI0Vo7/JGZmam11vqhD/frtP98V9tsNi3MsXfvXrNL0Fprfc899+jnnnvOKefetWuXvvfeewd1bHNz8zlfb+vWrfrmm2922HFXX321LigoOONrZ3GVz4ZH6mzT+v5Qrdf+oe9bwBY9zMx1eovcpqHbOvC+oGJ0OnToEBMmTKC9vZ3bbrvNKdfIyck5bRSMM82YMYPFixefdaLPYI/r6uriqquuIjMzs9/XYpTqcdzuQODkm53hgUYfX8OxLuJC5caNJ0pPT2f//v1ml+Fwd955p0OO8/X15dZbbz3jazFKHaszngMiHHI6p7bIEyMCAChtaB/gSCGE8CCNR43n8GSHnM6pQZ4QbgR5WaMEuRBC9GkqMZ7DkhxyOue2yO1BXi5BLoQQxzWVgrJASIJDTufUIA/x9yHU35sy6VoRQojjGksgNAEsjrlN6dQgB0iMCJSuFSGEOFFTCYQ5pn8cRiLIw/2la0UIIU7UVOKw/nEYkSAPkK4VIYToZbNCc7nDRqzAiHStBNDS2UNTe7ezLyWEEK6vpQJsPe7VtZIgI1eEA3344YdkZWWRkZHBgw8+6PDzD2erN0dtP3emLeP6u2Z/TtwmTraMc2FNpcazOwV57xBE6V4R58pqtXLPPffwwQcfsHfvXl5++WX27t074nWcuoWbo7afO9OWcf1dsz8nbhMnW8a5sEb7GHJ361oBmRTkyRy11dumTZvIyMhg3Lhx+Pr6csMNN7By5UpHlgoMfas3R20/d6Yt4/q75mC2jZMt41xUk31WpwNvdjp1rRWA6CA/fL29JMhdwQf/CZW7HHvOMZPhkrN3cThqq7eysjKSk4+3YpKSkti4caMDfonTDWWrN0fpb8u4pUuX9nvNwWwbJ1vGuajGEgiMAt8gh53S6UHu5aVICPOXIPdgo3mrt6Fe/0QnXv9MW8ad6ZqD3TZOtoxzQU2lDm2NwwgEORjdK9JH7gIGaDk7kyO2ektMTKSkpKTvvdLS0n53GzqVM7d6G4wTr38mZ9oy7kzXHOy2cbJlnAtqKoGoDIee0ul95GAfSy4tco/miK3eZs2axcGDBzl8+DBdXV288sorXHHFFYB5W705ypm2jOvvmoPdNk62jHNBWhtdK+EpDj3tiAR5QngANS2dsn+nB3PEVm/e3t489thjLFu2jIkTJ7JixQqys7NN3eoN+t9+bqjOtmXcidccyrZxsmWcC2pvgO42h3etOHWrt16vbT6qx/70XX24ptWRmyWJQXCV7bw8cas3RxrOtnEDbRnnKp8Nj1K+w9jibc/K097CVbd66yVDED2XJ2/1NtLXPHGbONkyzkU5YQw5DOFmp1LKAmwByrTWy4dykaTwQEAmBXkiT9/qbSSveeI2cbJlnIvq21DCvD7y7wP7hnORMWH+KCUtciGEh2sqBe8ACHTs/ZxBBblSKgm4DHh6OBfx9fYiNsRPglwI4dkajxrdKko59LSD7Vr5I/AT4IwDcJVSdwN3A8TExJCXl3fS+yFe3ew5XE5eXsPwKhXDEhYWRnNzM8rBHxx3ZbVa+x2v7mm01nR0dJz271Q414zSvfR4h7DTwf/dBwxypdRyoFprvVUptehMx2mtnwSeBMjKytKLFp186Ovl29hV1sSp3xfO1TvmOioqSsKcwU8IGs201tTV1REeHn7GRbqEk2xugrR5Ds/BwbTI5wFXKKUuBfyBUKXUP7XWNw/lQokRAXy8pwqbTePlJYEyUpKSkigtLaWmpsbsUlxCR0eHzHQE/P39SUpy8FhmcXbd7dBW4/ARKzCIINda/wz4GYC9Rf6joYY4QFJ4AF1WGzWtncSFyj+kkeLj40NaWprZZbiMvLw8aYUKc/StQ+7YESswQjM7QcaSCyE8XJNzxpDDEINca5031DHkvRJkgwkhhCfrnQzk6On5jGCLPCUyEKWgsLp1pC4phBCuo/YAePtDSILDTz1iQR7o6016TDC7y5pG6pJCCOE6yndAXA5YHL96+IgFOcDkxDB2SZALITyNzQYV+ZAwzSmnH9Egz0kMo7qlk+pmx67lLIQQLq2+CLpaIH4UBPnkxDAAaZULITxLxQ7jeTS0yLMTQlFKglwI4WHKt4PFD2ImOOX0IxrkQX7ejIsOkhueQgjPUpEPY3LA4pxt90Y0yEFueAohPEzvjU4n9Y+DCUGekxhGVXMn1S1yw1MI4QEaDkNns9P6x8GkFjkg3StCCM9Qvt14Hk0t8uzEMOOGZ2nzSF9aCCFGXsUO40Zn7ESnXWLEgzzYz5u06CDpJxdCeIbyHRCX7bQbnWBCkIPRvSJdK0KIUc9mg4qdTu0fBxODvLK5g5qWTjMuL4QQI6PhMHQ2ObV/HEwK8hy54SmE8AS9NzpHY4s8OyEUkBmeQohRrmIHWHwhxnk3OsGkIA/x92Gc3PAUQox2vTc6vX2dehlTghxgekoEm4vrsdq0WSUIIYTzdHdA6RZImuX0S5kW5Asyo2k81i2tciHE6HT0S+hph/QlTr+UaUE+PyMapWDNgRqzShBCCOc5tAq8fCB1vtMvZVqQRwX7kZMQJkEuhBidCj+DlNngF+z0S5kW5GB0r2wvaaS5o9vMMoQQwrGaK6B6D2Q4v1sFzA7y8TFYbZovCmvNLEMIIRzr0GfG8wj0j4PJQT5jbATBft58fkCCXAgxihxaBUGxEJczIpczNch9LF7MTY9izYEatJZhiEKIUcBmhUOrjW4Vr5GJWFODHGBBZgxlje0U1baZXYoQQpy7ih3QXj9i3SrgAkG+MDMGkGGIQohRovAzQEH64hG7pOlBnhwZSFp0kAS5EGJ0OLQK4qdCUPSIXdL0IAdYMD6aDUX1dPZYzS5FCCGGr6MJSjaN2LDDXi4R5IsmxNLebWWtjF4RQrizAx+BtsL4i0b0si4R5PMzookM8uXfO8rMLkUIIYZv52sQlgxJ543oZV0iyH0sXlw2OZ5P91XR2tljdjlCCDF0rTXGRKDJ143YsMNeLhHkAFdNT6Cj28ZHuyvNLkUIIYZuz1tGt8rkFSN+aZcJ8hkpESRHBkj3ihDCPe16zZjJGTdpxC89YJArpfyVUpuUUvlKqT1KqQecUYhSiiunJrK+sJbqlg5nXEIIIZyjvghKN8Pk6025/GBa5J3AV7TWU4FpwMVKqdnOKOaq6QnYNLybX+GM0wshhHPseh1QRv+4CQYMcm1otb/0sT+csjBKRmwI2QmhrJTuFSGEu9DaGK0ydh6EJZlSgvdgDlJKWYCtQAbwuNZ6Yz/H3A3cDRATE0NeXt6wCsoJ6ebVgi5eee8zxgS5TBe+GCVaW1uH/dkUoj/BLYXMrDtIQdRFVJj02RpUkGutrcA0pVQ48JZSKkdrvfuUY54EngTIysrSixYtGlZBE6Z38NqDqyj1TuSGRVnDOocQZ5KXl8dwP5tC9Ov998HLh6yrf0xWQIQpJQypyau1bgRWAxc7pxwYE+bPV7JieXnTUZmyL4RwbZ0tsOMlmHQlmBTiMLhRKzH2ljhKqQDgQmC/M4u6fV4qdW1dvCM3PYUQrmzHS9DVArO/bWoZg2mRxwOrlVI7gc3AJ1rrd51Z1PyMaMbHBvP39YdlwwkhhGuy2WDj3yBxJiTNNLWUwYxa2am1nq61nqK1ztFa/9LZRSmluH1eKnvKm9lc3ODsywkhxNAVfgL1h0xvjYMLzew81TXTkwgL8OHv6w+bXYoQQpxuw18hJN7oHzeZywZ5gK+FG85L5qM9lZQ2HDO7HCGEOK56PxSthllfB4uP2dW4bpAD3DonFaUUL3x5xOxShBDiuI1PgMUPcu8wuxLAxYM8MTyAi7PH8NKmozS1d5tdjhBCQEsV5L8CU64f0e3czsalgxzgPxan09LRwzNri8wuRQghYN0fwNoF839odiV9XD7IsxPCuCRnDM+uL6ahrcvscoQQnqy5HLY8C1NvhKh0s6vp4/JBDnDvhZm0dfXwpLTKhRBmWvt7Y/OIhT82u5KTuEWQZ8aFcPmUBJ5bX0xta6fZ5QghPFHjUdj6D5h+C0Skml3NSdwiyAG+v3Q8nT1Wnsg7ZHYpQghPtOYhUAoW/MjsSk7jNkGeHhPM1dOTeGHDEaqaZQchIcQIqi+C7S9C7u2mrTl+Nm4T5AA/WDoereG3Hzp1zS4hhDjZx78Abz+XGqlyIrcK8uTIQL5xQRpvbitj65F6s8sRQniCwlWw/12jSyU03uxq+uVWQQ5wz+IMxoT6c//be7DaZGVEIYQT9XTBBz+FyHEw5ztmV3NGbhfkQX7e/NdlE9ld1syrm0vMLkcIMZptfALqDsLFDxpdKy7K7YIc4PIp8ZyXFslDH+2n8ZhMEhJCOEFLJXz+Wxi/DDKXmV3NWbllkCul+N/Ls2lq7+ahjwrMLkcIMRp9/AtjKv7F/2d2JQNyyyAHmJQQyu1z03hx41G+OFRrdjlCiNGk4APY9RrMv9elpuKfidsGOcCPl2WRGhXIT17fSVtnj9nlCCFGg2P18M73IS4HLnC9yT/9cesgD/C18ND1UylrbOfBD2RsuRDCAT78GbTVwlV/AW9fs6sZFLcOcoBZqZHcMTeNFzYckS4WIcS5KfgAdr4CF9wH8VPNrmbQ3D7I4eQuluYO2YBCCDEMbXXHu1QWuNbqhgMZFUEe4Gvh9yumUtHUwc/e3IXWMlFICDEENhu89U1ob4Cr/uo2XSq9RkWQA+SOjeS+izJ5b2cFL206anY5Qgh38uWfofATWPYbiJ9idjVDNmqCHOBbC9JZkBnDA+/sZW95s9nlCCHcQckm+PQBmHQlzPqG2dUMy6gKci8vxSMrphIe4MN3XtpGqwxJFEKczbF6eP1OY2naK/5srDfuhkZVkANEB/vx6I3TKa5r48f/yscmC2sJIfpj7YE3vmFMxb/+7+AfZnZFwzbqghxg9rgo/uvSiXywu5JHPztodjlCCFf06f1waBVc9ntIzDW7mnPibXYBzvL1+Wnsr2zhj58eJCsuhEsmu+Y6wkIIE2x/Eb58DM77JuTeZnY152xUtsjBWFjr11fnMCMlnB++ls+e8iazSxJCuIKSTfDuD2DcImOUyigwaoMcwM/bwhO35BIe6MPXn9tCeWO72SUJIcxUdwhe+RqEJsJ1fwfL6OiUGNVBDhAb4s8zt82irbOH257dJOuXC+GpWqvhn9eAzQo3/QsCI82uyGFGfZCDseTt327N5UjdMb7xjy10dFvNLkkIMZI6W+DF64wwv+lfED3e7IocyiOCHGBuejSPfHUqW4828N2Xt9NjtZldkhBiJPR0wqs3Q+VuuP4fkDTT7IoczmOCHGD5lATuXz6JT/ZW8aN/5cvmzUKMdtZu+NcdUJRnTPjJvMjsipxidPT0D8Ht89Jo67Ly0EcFeFu8+N21U/Dycs/ZXEKIs7D2wBtfh4L34NKHYfpNZlfkNAMGuVIqGXgeiAM08KTW+k/OLsyZ7lmcQY9V84dPD+DtpfjN1ZMlzIUYTWxWeOtu2LvSGGJ43l1mV+RUg2mR9wD3aa23KaVCgK1KqU+01nudXJtTfW9JBt1WG4+tLkQp+NVVk7FImAvh/qzd8O9vw+43YOkDMOcesytyugGDXGtdAVTYv25RSu0DEgG3DnKlFPddlIlG8/jqQxzrsvLw9VPxsXjUbQMhRpfuDmMRrIL3YMn9MP8HZlc0IobUR66USgWmAxv7ee9u4G6AmJgY8vLyzr26ETDLD64b78PrO8o5Wl7Ft6f64WuRlvlo1dra6jafTTE0XtYOcnb/hsiGfA5m3E2ZdQZ4yP/WarC76SilgoHPgV9rrd8827FZWVm6oKDAAeWNnH98Ucz9b+9hXkYUT9ycS4i/j9klCSfIy8tj0aJFZpchHO1YPbx8A5Ruhisec8sbm0qprVrrYY2NHFQ/glLKB3gDeHGgEHdXt81N5ffXT2VDUT0r/raBquYOs0sSQgxGQzE8cxGUbzem3bthiJ+rAYNcKaWAZ4B9WutHnF+Sea7NTeLZ22dxtK6Nqx9fz4GqFrNLEkKcTfl2ePpCaKuGW/4N2VeZXZEpBtMinwfcAnxFKbXD/rjUyXWZZmFmDK9+cw7dNs21f/2C9YW1ZpckhOhPwYfw98vA2w/u/BhS55ldkWkGDHKt9TqttdJaT9FaT7M/3h+J4sySkxjGm9+eS3yYP7c+u4nnvyw2uyQhRC+tYd0fjT7x6Az4+icQO8HsqkwlY+3OIDkykDe+PZdFmTH8z8o9/Pdbu+iW9VmEMFd3B7z1LWN3n+yr4I4PIVQ2jZEgP4sQfx+evHUm31qYzosbj3LTUxupbpGboEKYorEEnrsUdr4Ci39u3Nj0DTS7KpcgQT4Ai5fiPy+ZwJ9umMbOskYue3Qdmw7Xm12WEJ6lcBX8bQHUHICv/hMW/thtd7x3BgnyQbpyWiL/vmcewX7e3PjUBp5eW8Rgx+ALIYbJZoPPH4J/XgshY+DuPJh4udlVuRwJ8iGYMCaUld+Zx9KJsfzqvX3c9fwW6ttkxyEhnKKlCv55Naz+FUy+Dr7xqXFzU5xGgnyIQv19eOLmXO6/fBJrDtRy6Z/WsqGozuyyhBhdCj+FJ+bB0Y1w+aNwzVPgG2R2VS5LgnwYlFLcMS+NN/9jLgG+Fr721AYe+mg/XT0yqkWIc9LdAR/+l9GVEhRjdKXk3ib94QOQID8HOYlhvPvd+Vw7I4nHVx/i6r/IbFAhhq0iH55cCBseh1l3wV2fefz48MGSID9HQX7ePHT9VP52Sy4VTR0s//M6nl5bJNvICTFY1m5Y8xA8tQTaG+HmN+Cyh8EnwOzK3IbHbfXmLMuyxzAjJYKfvbmTX723j/d3VfC766aSERtsdmlCuK7KXbDyHqM1nn01XPYIBEaaXZXbkRa5A8WE+PHUrTP541enUVTbxqWPruUveYUyI1SIU/V0wur/gycXQXM5rHgern9OQnyYpEXuYEoprpqeyLyMaP5n5W5+92EBb+8o5/+umcz0lAizyxPCfMXr4d0fQO0BmPJVuPhBCfBzJC1yJ4kJ8eOvN+fyt1tyaTzWzTV//YJf/Hs3zR3dZpcmhDmO1RvdKM9dCj0dcNPrcM2TEuIOIC1yJ1uWPYZ5GdE8/FEB//iymA92V/KzSyZwzYxElAypEp7AZoVt/4BVv4SOZpj3A1j4U1knxYGkRT4Cgv28+d8rsnn7nvkkRQRw37/yuf6JL9lT3mR2aUI4V+kWeHoJvHsvxE6Cb62FCx+QEHcwCfIRNDnJWOf8d9dOoai2jeV/XsfP3txJTUun2aUJ4VjN5fDmN40Qb66Aa56G29+DuGyzKxuVpGtlhHl5KVbMSmZZ9hge/ewg//iimHfyK/jOVzK4fW4q/j4Ws0sUYvi62uCLx2D9H8HWA/Pvhfk/BP9Qsysb1STITRIW6MMvlk/ipvNT+M37+3jwg/288OURfrQskyunJuLlJf3nwo1Ye2DHi7D6N9BaCZOuhAt/CRGpZlfmEaRrxWTjYoJ5+rZZvPSN84kI8uHeV/NZ/ud1fH6gRpbJFa5Pa9j/vrHA1Tvfg/BkY9eeFc9LiI8gCXIXMTcjmrfvmc+fbphGc0c3tz27iRue3MDWI7KJhXBRh9fAMxfCKzeCtcsI769/AmPnmF2Zx5GuFRfi5aW4cloiF+eM4ZVNJfz5s0Ku/euXLM6K4YcXZjE5KczsEoWAkk2w+tdQlAchCXD5n2DaTWDxMbsyjyVB7oL8vC66VWsAABYnSURBVC3cNjeV62cm8ff1xTy5pojLH1vH0olx/GDpeHISJdCFCUq3GH3gh1ZBYBRc9GuY9Q3w8Te7Mo8nQe7CAn29uWdxBrfMGctz64t5em0Ry/9cxdKJsXznK+OZlhxudonCExz5AtY8fDzAlz4A590lGz24EAlyNxDq78P3lozn9nmpPLe+mGfXH+aqx9dzwfho7lmcwflpkTJLVDiW1lC02gjwI+uNTR6W/q+xTrifrOjpaiTI3UhvoN85P40XNxzhqbVF3PDkBqanhPOthelcODFOhi2Kc2Ozwt6VsO4PULnT6AO/+Lcw41aZjenCJMjdULCfN99cmM5tc1P515YSnlxbxDdf2Ep6TBB3XTCOq6YnysQiMTRdx4xx4F8+Dg2HISoDrvizsTqht5/Z1YkBKGeMVc7KytIFBQUOP6/oX4/Vxvu7K3ki7xB7K5qJDvbl1jmp3Dx7LJFBvmaX51Ly8vJYtGiR2WW4jpYq2PwUbH4G2ushMddY1GrCZeAljYGRpJTaqrWeOZyflRb5KOBt8eKKqQlcPiWeLw/V8dTaIh755ACPry7kqmmJ3DE/lQljZIq0OEH5dtjwBOx+w5hKP+EymPMdSJktGx27IQnyUUQpxdyMaOZmRHOwqoW/f1HMm9tKeXVLCXPTo7h1TipLJ8bibZF5YB7J2g373oZNT8HRL8E3GGbeAed9E6IzzK5OnAPpWhnlGo918fKmEl74spjypg4Swvy5afZYVsxMJibE8/o+PbJrpbkctj0PW/5urIMSkWqMPplxC/jLnARXcS5dKxLkHqLHamPV/mqe/7KY9YV1+FgUy7LHcPPssR41fNFjgtxmg8N5Rt93wQegrZBxIZx3N2QsBS/5q8zVSB+5GJC3xYtl2WNYlj2GwupWXtp4lNe3lvDuzgrGxQRx46wUrpmRSFSw57XSR5WWKtjxT9j2gjH6JCAS5twDubdDVLrZ1QknkRa5B2vvsvLuznJe3nSUbUcb8bEoLsoew1dnJjMvIxrLKByTPipb5NYeY9bltufhwIfGzcux8yH3NmM5WRk+6BakRS6GJcDXwvUzk7l+ZjIFlS28svkob20v472dFSSGB3BtbhLXzUgiJUomgrik2kKj9b3jZaPvOzAaZn8bZtwG0ePNrk6MoAFb5EqpZ4HlQLXWOmcwJ5UWufvq7LHyyd4qXttSytqDNWgN56VGcm1uIpdOjifE371XuHP7Fnl7I+x5E3a8BKWbQVlg/EUw/WbIXCYrELoxp97sVEotAFqB5yXIPUt5YztvbS/jja2lFNW24e/jxYWTxnD19AQuGB+DjxsOY3TLIO/pgsJPIP8Vo+vE2gUxE2Ha12DKCggZY3aFwgGc2rWitV6jlEodzsmFe0sID+CexRn8x6J0th1t5K3tpby7s4J38suJCvJl+ZR4rpiWyIyUcI8Z9TJibDZjrPeuf8Hef0N7g9F1MvNOY9p8wnSZuCP6DOpmpz3I3z1bi1wpdTdwN0BMTEzua6+95qAShSvpsWl21lj5oryHHTVWemwQE6A4P96b8+O9SQpWLh3qra2tBAe76Op9WhPcWkRs9Vpiq9fi31mL1cuP2ujzqYpbSEPENLSX3NYarRYvXuzcceSDCfITSdeKZ2jp6OajPVWs3FHGF4fqsNo0GbHBLJ8Sz/Ip8WTEhphd4mlcrmtFa6jaY7S6d78J9YfAyxvSvwKTV0DWJbJsrIeQUSvCFCH+PlyXm8R1uUnUtXbywe5K3skv50+rDvLHTw+SGRfMpZPjuXRyPONjg126pT6itIaq3cZysXv+DXUHQXlB6gUw7/sw8XIIjDS7SuFGJMiFQ0QF+3Hz7LHcPHss1c0dfLC7kvd2VfSF+riYIC7NiefinDFkJ4R6XqhrDWXbjLVO9q40JusoL0idD3P+AyZcDsExZlcp3NRgRq28DCwCooEq4H6t9TNn+xnpWhG9qps7+GhPJR/srmRDUR02DYnhAfZZpnHMTI0c0YlHI9q1Yu02tknb/y7sexdayo1uk7QFMPEKmLBcwlv0kbVWhFuoa+1k1b5qPtpTydrCWrp6bEQG+fKVCbFcOCmOC8ZHE+jr3D8SnR7knS1QuAoK3ocDH0FHI3gHQMYSI7gzl0m3ieiX9JELtxAV7MeKWcmsmJVMa2cPnxfU8MneSj7eU8nrW0vx9fZiXnoUSyfFsWRCHGPC3GR39sYSY3x3wQdQvNYY5x0QCVmXwoRLIX2JbJMmnEqCXJgi2M+by6bEc9mUeLqtNjYdrufTfVV8uq+K1W/V8N/sJjshlCUTYlk8IZapSeGusx+pzWrMqjzwkfGo3mN8PzLdWF0w6xJIng0W+eclRoZ0rQiXorXmYHUrq/ZV89n+KrYeacCmISrIl4WZMSyaEMuC8dGEBw5vC7thd6201RpdJgc/Nhaoam8wpsePnWt0l2ReLOubiHMiXSti1FBKkRkXQmZcCN9elE5DWxdrDtawen81qwuqeXN7GV4KpiaHszAzhoWZMUxJCnf8DVNrD5RtgcJPjUf5DkBDUAxkXgLjlxpdJgHhjr2uEMMgLXLhNqw2TX5pI58X1PD5gRrySxvRGsIDfZiXEc2C8dFcMD6GhPCAM57jrC3yhiNw6DOjxV20BjqbjCGCSecZmzFkLIH4abIpg3AKaZELj2DxUsxIiWBGSgT3XphJfVsX6wprWXOghrUHa3hvZwUA6TFBXDA+hvkZ0Zw/LvLMKza2N0LxOihabQR4fZHx/dAkyL7SmF05bhEERIzI7yfEcEmLXIwKWmsOVLWy9mANaw/WsvFwHR3dNixeimnJ4czLiGZeajCWnf9iZngTFH0O5dtA28AnCNIugHGLIX0xRGfKglRixMk4ciFO0dFtZXtxNYfz16GLPietdRu56gB+qhsrXlSHToZxC4mZugzv5PPAe3g3T4VwFOlaEQKMmZTl26F4Hf7F65hzdANzutuMt8bkUBp+Myur41hlm0Z+tQ2qIXh7G7NSdzB7XBSzx0WRnRCKtxuusy48mwS5cF89ncb6JUfWQfF6KNkE9uDu23ghbQGkzscSGMlYYEpeHt9btIja1k42FNXx5aE6NhTVsbqgBjDGt+eOjeD8cZGcnxbF5MQwfL0l2IVrkyAX7qOzxQjro18aa5iUbgFrp/FebLYR3KnzYey8AdcwiQ72Y/mUBJZPSQCguqWDjUX1bCiqY9Phen73odE16O/jxYyUCGalRnJeWiTTU8KdvoyAEEMln0jhuporoGQDHN0IR7+Ayl3GzUllgfgpcN5dkDLHmJRzjuuXxIb4c/nUBC6fagR7bWsnmw/Xs/FwPZuL63n0s4NobYycyUkIZVZqJDNTI8gdG0lMiOxSL8wlQS5cg80K1XuhZKO91b0BGo8Y73kHQNJMuOBHkDIbks8DP+duWhEd7Mclk+O5ZHI8AM0d3Ww90sCW4no2Fzfw/IYjPL3uMACpUYHkjo0kd2wEuWMjGB8b7DrLCQiPIEEuzNHeAKVbjeAu3WR83dVivBccB8nnw/nfNNYsiZ9i+u7wof4+LM6KZXFWLACdPVZ2lzWz9Ug9W4obyCuo5o1tpQCE+HszPSWCGSnh5I6NYGpyOKFnGssuhANIkAvns/YYre3SzVC21Whx1x003lNeEJdt7Abf29oOH+vy47j9vC19LfC7Fxjj2I/UHTNa7Uca2H60gT+tMrpjlIKMmGCmp4QzPSWC6SnhjI8NGdF12MXoJkEuHEtraCo1Artsi9HSrtgB3ceM9wOjjCnv026EpFmQMGNU7EmplCI1OojU6CCuzU0CjD1N80ua2H60gW1HG/h4bxWvbTFa7UG+FiYnhTEtOYJpyeFMSw53n2V7hcuRIBfnpq3OGLtdvs0YCli2FdqqjfcsvjBmMsy4FRJnGv3cEaku39p2lBB/H+aPj2b++GjAaLUX1x1j+9EGdpQ0sv1oI8+sK6LbakzKiwv1Y0qSEepTksKYkhhOWKB0yYiBSZCLwetogop8I7jLthnPvTckUcbU9oylkDgDEnMhLkdmTJ5AKUVadBBp0UFcM8NotXd0W9lX0cyOkkZ2ljaRX9LIJ3ur+n5mbFQgU5LCmZoURk6i8Qj2k3+24mTyiRD9a2+Eyp3G8q0VO4zn+kPH3w9PMbpFZt5pBHf8NPAPNa9eN+XvY7H3mx9fmKupvZtdpU3sLGtkZ0kTW4vreSe/HDD+mEmLDmJyYhiT7cGenRB65oXBhEeQIBfGpgkV+Sc/Gg4ffz8sGeKnGv3aCfbQDooyr95RLizg5C4ZMMa17yprYmdJE7vKmthYVM/KHeV976dFBxkt9oRQshOMcI8Ikr+GPIUEuSfR2ugKqdhpTK6p3Gl83XI8EAhPMYJ6xi0wZiokTIOg6DOfU4yI6GC/k4Y/AtS0dLK7rMl4lDex7UhDX8sdIDE8gInxoWQnGI9JCaEkhgegPOQehSeRIB+tujugZh9U7oaq3fbg3m1slgDGsL/oTGNKe/wUo8U9ZrKsve1GYkL8WGzf07RXQ1sXe8qb2VPe1Pe8an8VvYuchgX4MCk+lInxoUyMD2FifCjj44Lx87aY9FsIR5Agd3daQ0ulEdZVu48Hd+1B0FbjGJ8giJsEk68zwnrMZIidJDu7j0IRQb6ndcsc6+phX0UL+yqa2VvRzJ7yZl7adISObhsA3l6K9JjgvmCfGB/KhPgQYoL9pPXuJiTI3UlnK9Tsh6o9xgSbqj3Go73++DFhycZokQnLj4d2RJpsT+bBAn29+yYv9bLaNMV1bewtb2ZfhfHYUFTPv0/od48K8mVCfAhZcUawTxgTwvjYEAJ8pfXuaiTIXZG1G+oK7WG9F6r3QfUeaCg+foxPEMROhInLIW6yMTsybpJ0jYhBsdhb4ekxwX0LhYHRNbO/soX9lUa4F1S2nNR6VwrGRgaSNSaErDGhZMWFkDUmmNSoIFnH3UQS5GayWaH+sNGXXX3Co64QbN3GMcoCURmQMB2m3WR0icRNgvBUaWULh4sI8mVOehRz0o+PSrLaNEfrj1FQ2cz+yhYK7I9P9lZhs/e9+1q8GBcTRGZcCJlxwfbnEJIjA2UpghEgQT4SrD3GcL6a/fZHAVTvh9oDx9fTBmONkdiJkHWxsTFC3CTjhqS3LJMqzGPxOj6R6eKc+L7vd3RbKaxupaCyhQPVLRyobGHrkQbePmHkjL+PF+kxRrBnxAYzPjZYAt4JJMgdqbvDaE3XFkDNAftzgfE9a9fx48JSIHYCpC8yAjt2AkRnjYo1R4Tn8Pex9M02PVFrZw8Hq1o4WNXKgaoWDlS3srGojre2l/Ud4+vtxbjoIMbHhZARE0xGrPFIjQ6UETTDIEE+HO0NxqiQmgKjVV17wPi68Yix8QEACiLGGgE9/kKIsYd1jAS2GN2C/bxPm60KxiJiB6tbKbQ/DlS1sP3oyWPfLV6KlMhA0u3hnh4TZDzHBstSwGchQX4mNqsRzLWFRlDXHTTCu/YAtNUcP87ia/Rhx0+FydcbQR2dCdHjwSfAvPqFcDEh/j7MSIlgxikBf6yrh6Katr6AP1RjPH9+oLpvQTEwxs2nxwSRHhPMuJjgvq8TwgM8vpvGs4NcazhWb4R0XaER1HWFxqO+6OTukIBII5wzL7YHdSbEZBr92l7yp6AQwxXo691vF02P1UZJQ3tfuB+yP7+7s4Km9u6+43y9vUiLCmJcjNGPPy4mmLToINJjgggP9IxlCjwjyDtbjWCuK4S6Q8biT72h3dF4/DgvH4hMg6jxkLnMeO5tXZ/jnpBCiKHxtnj13WS9kLi+72utqW/roqi2jUPVrRTVtlFU09o3kqbHdrwVHxHo0xfsJz5So4JG1Xj40RPkXceMkSG9QV13yB7eh6C18uRjQxIgOgNyroHIdCOoozKM1rVl9PwnEWI0UkoRFexHVLAfs1JPbmB1W22U1B/jcG0bRTVtFNW2cbi2lTUHanh9a+lJx44J9Sc1OrAv2FPtIZ8SGYi/j3uFvHulVmerEdb1RcdDut7++sSFnwCCYoyQzlgCkeMgKt0I68hx4BtkTv1CCKfysXgxzt6HvmTiye+1dvZQXNvG4do247nOeP5oTxX1bce7UZWC+FB/xkYFkRodaDzbv06JDCTQ1/Vi07Uq0hqO1Rnh3HD45Of6ouM7z/QKijGmn49baIR2ZJoR2JHpsja2EOIkwX7998WDsQZ8cW0bxXVtFNce40idEfSnhjxAbIgfY6OMgB8bGUjKCV+HB/qYsj7NoIJcKXUx8CfAAjyttX5w2Fe0dkNTiTHdvKHYHtbFRmA3HIHO5pOPD0kwAjpzmdGajkwzniPSJKyFEA4RFuDD1ORwpiaHn/ZeU3s3R+uOUVzXxtH6Y32Bv+ZADdUtnScdG+LvbYR8ZBDJkYGMjTJa8SmRgcSH+TttGYMBg1wpZQEeBy4ESoHNSqm3tdZ7z/gz2mpsuttw2BjC1xvaDUeMjXl7V+UDY/he+FhjL8eUOUZAR6Taw3qsDOETQpgqLMCHyUlhTE46vSXf3mXlaL3Rgj9af8z+9TH2VjTz8d7Kk4ZPWrwUieEBpEQGkmwP9+TIgL6gPxeDaZGfBxRqrYsAlFKvAFcCZwzy4NbD8PRXjn8jKMYI56RZMGWF8XXvIyRB1gwRQrilAF+LfQGxkNPes9o0FU3tHK0/RskJIV/S0M5HeypP67I5F4MJ8kSg5ITXpcD5px6klLobuNv+slM90Lz7+LvNwKFTf0QIM0QDtWYXIUQ/sob7gw672am1fhJ4EkAptUVrPdNR5xbCUeSzKVyVUmrLcH92MH0aZUDyCa+T7N8TQgjhAgYT5JuB8UqpNKWUL3AD8LZzyxJCCDFYA3ataK17lFLfAT7CGH74rNZ6zwA/9qQjihPCCeSzKVzVsD+bSms98FFCCCFcloz7E0IINydBLoQQbm7YQa6UulgpVaCUKlRK/Wc/7/sppV61v79RKZV6LoUKMRSD+HzerpSqUUrtsD++YUadwvMopZ5VSlUrpXaf4X2llHrU/tndqZSaMdA5hxXkJ0zbvwSYBNyolJp0ymFfBxq01hnAH4DfDudaQgzVID+fAK9qrafZH0+PaJHCkz0HXHyW9y8BxtsfdwN/HeiEw22R903b11p3Ab3T9k90JfAP+9evA0uUGcuCCU80mM+nEKbQWq8B6s9yyJXA89qwAQhXSsWf7ZzDDfL+pu0nnukYrXUP0AREDfN6QgzFYD6fANfa/3R9XSmV3M/7QphhsJ/fPnKzU3iqd4BUrfUU4BOO//UohNsZbpAPZtp+3zFKKW8gDKgb5vWEGIoBP59a6zqtde9i0k8DuSNUmxADGfKyKMMN8sFM238buM3+9XXAZ1pmH4mRMeDn85Q+xyuAfSNYnxBn8zZwq330ymygSWtdcbYfGNbqh2eatq+U+iWwRWv9NvAM8IJSqhCjY/+G4VxLiKEa5Ofze0qpK4AejM/n7aYVLDyKUuplYBEQrZQqBe4HfAC01k8A7wOXAoXAMeCOAc8pjWQhhHBvcrNTCCHcnAS5EEK4OQlyIYRwcxLkQgjh5iTIhRDCzUmQCyGEm5MgF0IIN/f/Af/YIBo4wyssAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "xs_log = np.linspace(0.001, 0.999, 100)\n", - "ys_0 = [-np.log(1 - x) for x in xs_log]\n", - "ys_1 = [-np.log(x) for x in xs_log]\n", - "plt.plot(xs_log, ys_1, label=r'$y = 1, -\\ln(h(x))$')\n", - "plt.plot(xs_log, ys_0, label=r'$y = 0, -\\ln(1 - h(x))$')\n", - "plt.ylim([0, 5])\n", - "plt.xlim([0, 1])\n", - "plt.xticks([0, 0.5, 1])\n", - "plt.legend()\n", - "plt.grid()\n", - "plt.title('cost function')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "def logloss(x, y, theta):\n", - " if y == 1:\n", - " return -np.log(hypothesis(x, theta))\n", - " elif y == 0:\n", - " return -np.log(1 - hypothesis(x, theta))\n", - " # or (less obvious)\n", - " return -y * np.log(hypothesis(x, theta)) - (1 - y) * np.log(1 - hypothesis(x, theta))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see that if $y$ was suppose to be 0 but is close to 1, the cost will be high. \n", - "Same logic if $y$ was suppose to be 1 but is close to 0.\n", - "\n", - "Then we generalize the cost function for all $x$:\n", - "$$J(\\theta) = \\frac{1}{n} \\sum_{i = 1}^{n} j(h(x_i), y_i)$$\n", - "$n$ is the number of elements in $X$ and $Y$." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "def cost(xs, ys, theta):\n", - " return sum([logloss(x, y, theta) for x, y in zip(xs, ys)]) / len(xs)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Gradient descent\n", - "\n", - "Finally, use gradient descent to minimize the cost function:\n", - "\n", - "$$\\theta_i := \\theta_i - \\alpha \\frac{\\partial}{\\partial \\theta_i} J(\\theta)\\ \\text{ for } i = 1, ..., n$$\n", - "\n", - "Or with gradient notation:\n", - "\n", - "$$ \\theta := \\theta - \\alpha \\nabla J(\\theta)$$\n", - "\n", - "\n", - "$\\alpha$ is the learning rate (how fast we want the step to be). \n", - "Since $\\theta_i$ is only used in the dot product, all of them will have similar partial derivative.\n", - "\n", - "let $z = x \\cdot \\theta_j$\n", - "\n", - "$\\newcommand{\\pt}{\\frac{\\partial}{\\partial \\theta_j}}$\n", - "$$\n", - "\\begin{align}\n", - " & \\pt J(\\theta) \\\\\n", - " = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n} j(h(x_i, \\theta), y_i) \\right] \\\\\n", - " = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n", - " (-y_i \\ln(h(x_i, \\theta)) - (1 - y_i) \\ln(1 - h(x_i, \\theta))) \\right] \\\\\n", - " = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n", - " (-y_i \\ln(\\sigma(z)) - (1 - y_i) \\ln(1 - \\sigma(z))) \\right] \\\\\n", - " = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n", - " (-y_i \\ln(\\frac{1}{1 + e^{-z}}) -\n", - " (1 - y_i) \\ln(1 - \\frac{1}{1 + e^{-z}})) \\right] \\\\\n", - " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\pt \\left[ \n", - " -y_i \\ln(\\frac{1}{1 + e^{-z}}) -\n", - " (1 - y_i) \\ln(1 - \\frac{1}{1 + e^{-z}}) \\right] \\\\\n", - " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", - " -y_i \\pt \\left[ \\ln(\\frac{1}{1 + e^{-z}}) \\right] -\n", - " (1 - y_i) \\pt \\left[ \\ln(1 - \\frac{1}{1 + e^{-z}}) \\right] \\right) \\\\\n", - " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", - " -y_i(1 + e^{-z}) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n", - " - (1 - y_i) \\frac{1}{1 - \\frac{1}{1 + e^{-z}}} \\pt \\left[ 1 - \\frac{1}{1 + e^{-z}} \\right]\n", - " \\right) \\\\\n", - " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", - " -y_i(1 + e^{-z}) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n", - " - (1 - y_i) \\frac{1 + e^{-z}}{e^{-z}} (-1) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n", - " \\right) \\\\\n", - " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", - " -y_i(1 + e^{-z}) \\frac{e^{-z}}{(1 + e^{-z})^2} \\pt z\n", - " - (1 - y_i) \\frac{1 + e^{-z}}{e^{-z}} (-1) \\frac{e^{-z}}{(1 + e^{-z})^2} \\pt z\n", - " \\right) \\\\\n", - " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", - " -y_i \\frac{e^{-z}}{1 + e^{-z}} x_{ij}\n", - " + (1 - y_i) \\frac{1}{1 + e^{-z}} x_{ij}\n", - " \\right) \\\\\n", - " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", - " \\frac{-y_i e^{-z}}{1 + e^{-z}}\n", - " + \\frac{(1 - y_i)}{1 + e^{-z}} \\right) x_{ij} \\\\\n", - " = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n", - " \\frac{-y_i e^{-z} + 1 - y_i}{1 + e^{-z}} \\right) x_{ij} \\\\\n", - "\\end{align}\n", - "$$\n", - "\n", - "$$\n", - "\\boxed{\n", - " = \\frac{1}{n} \\sum_{i = 1}^{n} (h(x_i) - y_i) x_{ij}\n", - "}$$" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def gradient(xs, ys, theta):\n", - " g = np.array([])\n", - " for j in range(len(theta)):\n", - " g = np.append(g, [sum([(hypothesis(x, theta) - y) * x[j]\n", - " for x, y in zip(xs, ys)]) / len(xs)],\n", - " axis=0)\n", - " return g" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "def gradient_descent(xs, ys, theta, alpha=1, epoch=100):\n", - " for _ in range(epoch):\n", - " theta = theta - alpha * gradient(xs, ys, theta) \n", - " return theta" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata