path: root/notebooks/logistic-regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression\n",
    "\n",
    "Logistic regression is a *binary classification algorithm*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Data')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEWCAYAAABsY4yMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAATSElEQVR4nO3de5RdZXnH8e+TTCYwISZAQhYCZkAs4MLFJQMWUqmFNmjR4FJSxWCplwK21larrYQuVFy0WqtdWrxglYrl0hZrm1lWMS4QtF6ABCJyKS0qKqAQwHCLJCR5+sfes2Yymcycuew5M2++n7XOOme/Z797P2cf+GXPe/Z5T2QmkqTyzGh3AZKkZhjwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvHY7EXFfRPwqIp6MiI0R8Z2IOC8iRvz/ISK6IyIjomMyapXGw4DX7uqVmTkXWAx8EPhL4HPtLUmaWAa8dmuZ+Xhm9gKvBc6OiCMj4rSIuC0inoiIn0XE+wZ0+WZ9vzEinoqIEyLi+RFxfUQ8GhGPRMSVETF/0l+MNIgBLwGZeTNwP/AS4Gng94H5wGnAWyPiVfWqJ9X38zNzr8z8LhDA3wDPBY4ADgLeN3nVS0Mz4KV+DwL7ZOYNmfmDzNyembcDVwO/uatOmXlvZn49Mzdn5gbgo8OtL00WPyiS+h0APBYRL6Yalz8S6ARmA9fsqlNELAI+RnX2P5fqxOmXjVcrjcAzeAmIiOOoAv6/gauAXuCgzJwHfJpqGAZgqOlX/7puf1FmPgc4a8D6UtsY8NqtRcRzIuIVwL8AV2TmD6jOwh/LzGci4njg9QO6bAC2A4cMaJsLPAU8HhEHAO+enOql4YXzwWt3ExH3AYuArVRhfRdwBfDpzNwWEWcAHwH2AW4E7qP6UPWsuv9FwFuBWcDLgCeBLwCHAfcC/wy8IzMPnLxXJe3MgJekQjlEI0mFMuAlqVAGvCQVyoCXpEJNqS86LViwILu7u9tdhiRNG+vWrXskMxcO9dyUCvju7m7Wrl3b7jIkadqIiJ/s6jmHaCSpUAa8JBXKgJekQhnwklQoA16SCtVYwEfEZRHxcETc0dQ+JEm71uRlkp8HLqGaZa9Rvb1w6aXV43PPheXLR9d3zRqYNw/Wr995GxdcAFdeCQsWwIUXwk03VX36nu/thcMPh0WLYNmyqu2ii+Cuu2DrVpgzB7Zvh2eegRkz4Nhj4aGH4IEHYI89qv2uXAkXX9xf04oV8KUvVf06OmDxYpg/HzZurLa5ciXccAOsWwcHH1ztY8MG2LYNHnkEZs2q9jdzJkRUfefMgQcfhM2bq7bubjjmGLjlluq1HXxw/+NTT4XHH69eT9/rXLoUbr4Z9toLTjwRurrgxhshE845p7/+we8FVMd34LZaeT9aXV/SMDKzsRvQDdzR6vpLlizJ0Vq9OnP27MwqajI7O6u2Vvt2dfX3HbyNVat2bI/Yed3B/WbOHH6dXd1WrapqOuOMsfVv4tbVVR2HE09srf7Vq6tj0NfW0dG/3LetVt+PVtaXlAmszV1katvH4CPinIhYGxFrN2zYMOr+a9ZUZ6V9tmyp2lrtu2nTzu192+jt3bF9pJmVt2ypzqLHom9f118/tv5N2LSpOg7r1o28bt+Z95Yt/W1bt/Yv921rOAPfj1bWlzS8tgd8Zn4mM3sys2fhwiG/bTusZctg9uz+5c7O/qGSVvp2de3c3reNwUMEMcKPsHV2VsMiY9G3r5NPHlv/JnR1VcdhyZKR112+vFq3s7O/raOjf7lvW8MZ+H60sr6k4TX6gx8R0Q18OTOPbGX9np6eHMtUBY7BOwYv7a4iYl1m9gz5XAkBL0m7q+ECvsnLJK8GvgscFhH3R8Sbm9qXJGlnjV0mmZlnNrVtSdLI2v4hqySpGQa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKlRLAR8RSyNiTv34rIj4aEQsbrY0SdJ4tHoG/ylgU0QcBfw58EPgC41VJUkat1YDfmtmJnA6cElmfgKY21xZkqTx6mhxvScj4nzgDcBLImIGMKu5siRJ49XqGfxrgc3AmzLzF8CBwIcbq0qSNG4tBXwd6lcBe0fEK4EtmekYvCRNYa1eRfMW4Gbg1cAZwPci4k1NFiZJGp9Wx+DfDRyTmY8CRMS+wHeAy5oqTJI0Pq2OwT8KPDlg+cm6TZI0RbV6Bn8vcFNErAb6Lpe8PSLeCZCZH22oPknSGLUa8D+sb31W1/deCy9JU1RLAZ+Z72+6EEnSxGop4CPiG1RDMzvIzJMnvCJJ0oRodYjmXQMe7wG8Btg68eVIkiZKq0M06wY1fTsibm6gHknSBGl1iGafAYszgCXAvEYqkiRNiFaHaNZRjcEH1dDMj4E3N1WUJGn8Wh2iObjpQiRJE6vVIZpZwFuBk+qmG4BLM/PZhuqSJI1Tq0M0n6Ka//2T9fIb6ra3NFGUJGn8Wg344zLzqAHL10fE95soSJI0MVqdbGxbRDy/byEiDgG2NVOSJGkijOaLTt+IiB9RXUmzGHhjY1VJksZtxICPiJnAUcALgMPq5nsyc3OThUmSxmfEIZrM3AacmZmbM/P2+ma4S9IU1+oQzbcj4hLgX4Gn+xoz89ZGqpIkjVurAX90fd83bXBQfbPV2SQlaYpqNeC/TP9UBdSPn4iIozNzfSOVSZLGpdXLJJcA5wH7A88FzgVOBf4xIv6iodokSePQ6hn8gcCxmfkUQES8F/gvqqkL1gF/20x5kqSxavUMfj9g4JUzzwKLMvNXg9olSVNEq2fwVwI3RUTfj22/ErgqIuYAdzVSmSRpXFqdLvgDEfFVYGnddF5mrq0fr2ykMknSuLR6Bk8d6GtHXFGSNCW0OgYvSZpmDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVAGvCQVyoCXpEIZ8JJUKANekgplwEtSoQx4SSqUAS9JhTLgJalQBrwkFcqAl6RCGfCSVCgDXpIKZcBLUqEMeEkqlAEvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1KhDHhJKpQBL0mFMuAlqVCNBnxEvCwi7omIeyPiPU3uS5K0o8YCPiJmAp8AXg68EDgzIl7Y1P6kqaq3F972tuq+yT6D+592WnUbuI3B2x243ooV0N0NPT1wxBEwaxbsvTcceijstx/svz/Mng0R1X1PT9W/t7daL6K6H9wWAfPmwQUX9NexYgXMnVut07fNGTOq+5kzYa+9oKOjauvshKVLq7qXLq2ej6hq7O2t6ujurrbf93oOPbS/rc8FF8DixTvW3Xcs+vr19Ox8zCb6vWl6ezvIzEZuwAnA1wYsnw+cP1yfJUuWpFSS1aszu7oyobpfvbqZPoP7d3ZW/SFz9uyqbfB2V62qnutbbyy3mTNHt/6qVZlnnDG+fQ53mzFj6H2uWrXzen2vffbszI6OHZ/v7Bz6uI/3vWlie8Da3EWmNjlEcwDwswHL99dtO4iIcyJibUSs3bBhQ4PlSJNvzRrYtKl6vGlTtdxEn8H9t2zpX968uWobvN3e3uq58di2bXTr9/bC9dePb5/D2b596H0OPjvevr3/tW/eDFu37vj8li1DH/fxvjdNb2+wtn/ImpmfycyezOxZuHBhu8uRJtSyZdDVVT3u6qqWm+gzuH9nZ//y7NlV2+DtLl9ePTceM2eObv3ly+Hkk8e3z+HMGCLRli+vboPX63vts2dXw0EDdXYOfdzH+940vb3BOkZeZcweAA4asHxg3SbtNpYvh6uvrs7Mli3bOWgmqs/g/tdcA5deWi2fe27/NgZv98Uv7l+vqwtuuQUWLICnn4Z7763GwvfdF554ogrzxx6rzm47O+FFL4ILL6z6nn02bNwI8+fD5Zfv2AbwnOdU48wXX1wtr1gB115bBeumTdU2I6oBkhkzYM894ZlnqjPtjg447jg45hi47Tb43veq9sMPhw99CC66CB55BFau7H8999xTnZWvXNm/T4ArroCFC/vr7jsWUPV76CFYtGjHYzaR703T2xssqiGciRcRHcD/AqdQBfstwOsz885d9enp6cm1a9c2Uo8klSgi1mVmz1DPNXYGn5lbI+JtwNeAmcBlw4W7JGliNTlEQ2Z+BfhKk/uQJA2t7R+ySpKaYcBLUqEMeEkqlAEvSYVq7DLJsYiIDcBPJmBTC4BHJmA7k2261g3Tt3brnnzTtfapWvfizBzyW6JTKuAnSkSs3dV1oVPZdK0bpm/t1j35pmvt07Fuh2gkqVAGvCQVqtSA/0y7Cxij6Vo3TN/arXvyTdfap13dRY7BS5LKPYOXpN2eAS9JhSou4CPivoj4QUSsj4hpM/dwRMyPiC9GxP9ExN0RcUK7axpJRBxWH+e+2xMR8WftrqtVEfGOiLgzIu6IiKsjYo9219SKiPjTuuY7p/rxjojLIuLhiLhjQNs+EfH1iPi/+n7vdtY4lF3UvaI+5tsjYlpcLllcwNd+KzOPnmbXrH4MuDYzDweOAu5ucz0jysx76uN8NLAE2AT8R5vLaklEHAC8HejJzCOpprR+XXurGllEHAn8IXA81X8nr4iIQ9tb1bA+D7xsUNt7gOsy8wXAdfXyVPN5dq77DuDVwDcnvZoxKjXgp5WImAecBHwOIDO3ZObG9lY1aqcAP8zMifgm8mTpAPasf5ymC3iwzfW04gjgpszclJlbgRupQmdKysxvAo8Naj4dqH/3icuBV01qUS0Yqu7MvDsz72lTSWNSYsAnsCYi1kXEOe0upkUHAxuAf4qI2yLisxExp91FjdLrgKvbXUSrMvMB4O+AnwI/Bx7PzAn+yeNG3AG8JCL2jYgu4HfZ8acxp4NFmfnz+vEvgEXtLKZkJQb8b2TmscDLgT+OiJPaXVALOoBjgU9l5jHA00zNP1uHFBGdwHLgmnbX0qp63Pd0qn9cnwvMiYiz2lvVyDLzbuBDwBrgWmA9sK2tRY1DVtdpe612Q4oL+PrMjMx8mGo8+Pj2VtSS+4H7M/OmevmLVIE/XbwcuDUzH2p3IaPw28CPM3NDZj4LfAk4sc01tSQzP5eZSzLzJOCXVL99PJ08FBH7A9T3D7e5nmIVFfARMSci5vY9BpZR/Uk7pWXmL4CfRcRhddMpwF1tLGm0zmQaDc/Ufgr8ekR0RURQHfMp/8E2QETsV98/j2r8/ar2VjRqvcDZ9eOzgdVtrKVoRX2TNSIOof8qjg7gqsy8uI0ltSwijgY+C3QCPwLemJm/bG9VI6v/If0pcEhmPt7uekYjIt4PvBbYCtwGvCUzN7e3qpFFxLeAfYFngXdm5nVtLmmXIuJq4KVUU+0+BLwX+E/g34DnUU0P/nuZOfiD2LbaRd2PAf8ALAQ2Ausz89R21diKogJektSvqCEaSVI/A16SCmXAS1KhDHhJKpQBL0mFMuBVvIh4ez1D55Wj7NcdEa9vqi6paQa8dgd/BPxOZq4cZb9uYNQBHxEzR9tHaoIBr6JFxKeBQ4CvRsQF9TzfN9eTup1er9MdEd+KiFvrW9+UBR+kmthrfT13/B9ExCUDtv3liHhp/fipiPhIRHwfOCEizqr3sz4iLjX01Q4GvIqWmedRTQP8W8Ac4PrMPL5e/nD9TdyHqc7wj6X6ZuvH6+7vAb5Vz3n/9yPsag7VNL5HAY/W21laz5W/DRjtXw/SuHW0uwBpEi0DlkfEu+rlPai+Lv8gcEk9XcQ24NfGsO1twL/Xj0+h+gGUW6ppbtgTJ9RSGxjw2p0E8JrBP9oQEe+jmm/kKKq/ap/ZRf+t7PhX78Cf+HsmM/um7Q3g8sw8fyKKlsbKIRrtTr4G/Ek9eyQRcUzdPg/4eWZuB95A9fN9AE8Ccwf0vw84OiJmRMRB7Hoq6uuAMwbM+rhPRCye0FcitcCA1+7kA8As4PaIuLNeBvgkcHb9AenhVD+4AnA7sC0ivh8R7wC+DfyYairnjwO3DrWTzLwL+CuqXxa7Hfg6sH8zL0naNWeTlKRCeQYvSYUy4CWpUAa8JBXKgJekQhnwklQoA16SCmXAS1Kh/h+VWvvMg0wsLwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "# create linearly separable data\n",
    "sep = False\n",
    "while not sep:\n",
    "    X, Y = make_blobs(n_samples=200, n_features\n",
    "                      =1, centers=2, cluster_std=0.5, center_box=(0, 10))\n",
    "    sep = True\n",
    "    for x_1 in X[Y == 1]:\n",
    "        for x_0 in X[Y == 0]:\n",
    "            if Y[X.argmin()] == 0 and x_0 > x_1:\n",
    "                sep = False\n",
    "            elif x_0 < x_1:\n",
    "                sep = False\n",
    "         \n",
    "X = np.hstack([X, np.ones((X.shape[0], 1))])\n",
    "\n",
    "                \n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33)\n",
    "\n",
    "plt.scatter(X[:, 0], Y, s=10, c='b')\n",
    "plt.xlabel('feature')\n",
    "plt.ylabel('groups')\n",
    "plt.yticks([0, 1])\n",
    "plt.title('Data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binary target\n",
    "\n",
    "### Sigmoid function\n",
    "\n",
    "We have to fit a **model** trough our data.  \n",
    "The *sigmoid* (also called *logistic*) function is used:\n",
    "$$\\boxed{\n",
    "\\sigma(x) = \\frac{1}{1 + e^{-x}}\n",
    "}$$\n",
    "\n",
    "It produce an 'S' shape like function that stays between 0 and 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXhU9d338fc3O5CQQICwE/ZVURZxpSCo4FJ4vOt2o1atdWnV2vZpK3VptZe3bW+3urQ+3Irb7VpKKVVLFTF1QZFd9n0LhiUJhISsM/N7/pjBBgiQhMmcmcnndV1zzXZmzmdg8skvvzlzjjnnEBGR2JfgdQAREQkPFbqISJxQoYuIxAkVuohInFChi4jECRW6iEicUKFLxJjZFDN7P9rWa2Z5ZnbzMe4zM3vRzPaZ2ZdNl7LOdf/DzL4byXVKbDNthy7hZGbnAr8HBgN+YA1wt3NuoafBjsPM8oD/dc49X8d95wFvAP2dcwebMMOvgT7OuWubah0S/5K8DiDxw8xaA+8AtwNvAynAeUCVl7lOUg9ga1OWuUi4aMpFwqkfgHPuDeec3zlX4Zx73zn3FYCZ3WBmnx5a2MwuNLN1ZlZiZn80s38dmvoILfuZmT1hZvvNbLOZnR26fYeZ7ak9HWFmmWb2ipntNbNtZnafmSUcY70XmNna0HqfAayuF2Nm3wOeB84yszIze/DI5wot58ysT+jyS2b2rJm9a2alZrbAzHrXWnawmX1gZsVmttvMfmlmE4BfAleF1rM8tOw3U0FmlhB6TdtCr/0VM8sM3ZcbyvBdM9tuZoVmdm+j/xclZqnQJZzWA34ze9nMJppZm2MtaGbtgBnAVCAbWAecfcRio4CvQve/DrwJjAT6ANcCz5hZemjZp4FMoBfwLeB64MZjrHcmcB/QDtgEnFNXRufcC8BtwOfOuXTn3K9O9A8QcjXwINAG2Ag8HFp3BjAXmAN0Dr2OD51zc4D/At4KrWdoHc95Q+g0NvQa04FnjljmXKA/MA54wMwG1jOvxAkVuoSNc+4AwVJxwP8Ae81stpnl1LH4xcAq59xM55wPeArYdcQyW5xzLzrn/MBbQDfgIedclXPufaAa6GNmiQRLdKpzrtQ5txV4DLjuOOud4ZyrAZ6sY70n66/OuS9Dr+s14LTQ7ZcCu5xzjznnKkNZF9TzOacAjzvnNjvnygj+IrzazGpPmz4Y+qtoObAcqOsXg8QxFbqElXNujXPuBudcV2AIwZHok3Us2hnYUetxDsg/YpndtS5XhJY78rZ0giPtZGBbrfu2AV3qud4ddSx3Mmr/gigPZYTgL6RNjXzOzhz9+pKA2r8sj7VeaSZU6NJknHNrgZcIFvuRCoCuh66YmdW+3kCFQA3BDzAP6Q7sPMZ6ux2x3m51LHcsB4GWtR7fsQGP3UFwuqQuJ9rc7GuOfn0+Dv+lJ82cCl3CxswGmNlPzaxr6Ho34BrgizoWfxc4xcwmh6YNfgg0pBy/EZqSeRt42MwyzKwH8BPgf4+x3sFmdnlovXc1cL3LQ48/zczSgF834LHvAJ3M7G4zSw1lHRW6bzeQe+iD3Dq8AfzYzHqGPjc4NOfua8D6Jc6p0CWcSgl+kLnAzA4SLPKVwE+PXNA5VwhcQXCb9SJgELCIxm/ieCfB0fNm4FOCH6JOP856fxtab1/gs/quxDm3HniI4IebG0Lrqu9jS4ELgMsITo9sIPghJ8CfQ+dFZrakjodPB14FPga2AJUEX7PIN/TFIokKoZFpPjDFOfeR13lEYpFG6OIZM7vIzLLMLJXgdthG3dMzIlIPKnTx0lkEt/ooJDgNMdk5V+FtJJHYpSkXEZE4oRG6iEic8GznXO3atXO5ublN9vwHDx6kVatWTfb8TU35vRPL2UH5vdbU+RcvXlzonGtf132eFXpubi6LFi1qsufPy8tjzJgxTfb8TU35vRPL2UH5vdbU+c1s27Hu05SLiEicUKGLiMQJFbqISJyIqiMW1dTUkJ+fT2Vl5Uk/V2ZmJmvWrAlDKm9EMn9aWhpdu3YlOTk5IusTkaYRVYWen59PRkYGubm5BHeC13ilpaVkZGSEKVnkRSq/c46ioiLy8/Pp2bNnk69PRJrOCadczGx66JBXK49xv5nZU2a20cy+MrNhjQ1TWVlJdnb2SZe51J+ZkZ2dHZa/ikTEW/WZQ38JmHCc+ycS3GNdX+AW4E8nE0hlHnn6NxeJDyeccnHOfWxmucdZZBLwSujIL1+EdrbUyTlXEKaMIiJRwTmHL+Co9gWo9gWo8QeoCp0fun3zfj/pW4up8Tv8AYcvEMDnDz7u0PWAc/j8joBz+APgd45A6P7gbY6Ag0Do9kOXT7SrlnDMoXfh8EN45YduO6rQzewWgqN4cnJyyMvLO+z+zMxMSktLwxAJ/H5/2J7LC5HOX1lZedT/x8koKysL6/NFUixnB+U/xBdwHKyBgzWOcp+josZR4YeKGkelHyp9jgofVPkdVf5/n1f7HTV+qAoEz2sCUFPrcr32fvXF5yedvzEi+qGoc24aMA1gxIgR7shvU61ZsyZsHwTqQ9GGSUtL4/TTTw/b88Xyt/1iOTvEb37nHAcqfew5UMnuA1XsPlDJ3rIqisqqKCyrprCsin3l1ew7WMO+8mrKq/0nXFdqUgKtUpNomZJIy5REWqQl0jYlkRbJiaSFTqlJCaQlJ5KSlEBqUgIpiQmkJP37lJwQOk9MICnRWLNqJcNOG0pSopGUkBA6NxITgteD58HrCQlGohkJCZBoh9+WmGCYQYJZ6BScHrXfHfv1hKPQd3L4MRm7UvexHCWMNm/ezMMPP0xJSQkzZszwOo5IWFRU+9l2wM8/VhSwvbic7cXl5O+r4Ov9wdPBOko6LTmBdumpZLdKoX16Kv06ZJDVMoU2LZNp3SKZzBbJtG6RROu0ZDLSkslIS6JVahKtUhJJSgz/V3FS965ldL86d7XS5MJR6LOBO8zsTYKHHyvR/HnT69WrFy+88ALf+c53vI4i0mAV1X7W7S5l/a5S1u4qZcOeUjbtKePrktDWVvODR+HLaplM1zYt6NW+Fef2bUfnzBbkZKaRk5FKTus02mWk0iolUR/sh5yw0M3sDWAM0M7M8oFfAckAzrnngPeAi4GNQDlwY1OF9VpFRQUTJkxg3rx5JCYm1rlMdXU148ePZ968eSQlRdVm/iKe8PkDrN1VyuJt+/gqv4SVO0vYsKeUQGgyOi05gb4dMhjVK5te7VpRsXcbl4weSbe2LWmdpi+7NUR9tnK55gT3O4JHbI9706dP5/LLLz9mmQOkpKQwbtw43nrrLaZMmRLBdCLRocYfYPmO/czfVMQXm4tYtmP/N/PZ7dJTOaVLay4anMOgzpkM6JhBt7YtSUz49wg7L28ngztnehU/pmkIWYfly5dz5513UlhYyNq1a3HOcf/99zN37lxef/31b5YbO3Ysv/zlL7ngggu47777KCkp4emnn2by5MlMnTq1SQu9qKiIe++9l6VLl/LII48wderUJluXyIkUlFQwb+0e5q3ZwxebizhY7ccMBnZszRXDuzI8ty3De7Shc2aapkeaUNQW+oN/X8Xqrw80+vF+v/+okfSgzq351WWDj/u4yspKrrrqKl555RXOOOMM7r//fiorK7n33nuZNm0atQ/K8eCDD/LAAw+wZ88eli5dyuzZswEYMmQICxcuPOq5zzvvvDo3RXz00UcZP358g15fdnY2zz33XIMeIxJOWwsP8u6KAt5bUcCq0M9qt7Yt+D/DunBO73ac2SubNq1SPE7ZvERtoXtl7ty5DBs2jDPOOAOAU089lTlz5lBUVERWVtZhy44ePRrnHI8//jh5eXnf/AJJTEwkJSXlqE0PP/nkk8i9EJEmUFRWxV+X7mTWsp2s3Bks8WHds7hn4gDGDehAnw7pGoF7KGoL/UQj6RNp7HbcK1eu5JRTTvnm+pIlSxg2bBgtWrQ4an8nK1asoKCggOzs7KPWVVVVRVpa2mG31WeEHskfBh0gXOojEHB8vGEvb365g7lrduMLOIZ2zeS+SwZy8Smd6JzVwuuIEhK1he6V7Oxs5s2bB8D69euZOXMm8+fPp02bNvj9fiorK0lLS6OgoIApU6bwt7/9jbvuuos5c+YwYUJwlzdFRUW0a9fuqN3R1meEfqhkw/3Forlz51JcXExRURG333572J5X4tfBKh8zl+Tz4mdb2Vx4kOxWKdx4Ti5XjOhGv5zY/dJePFOhH+Gaa65h9uzZDBkyhHbt2vHGG2+QnZ0NwIUXXsinn37K2WefzeWXX85jjz3GwIEDuf/++/nFL37xTaF/9NFHXHLJJY1a/6EvDBUVFTFr1qyj7l+xYgWZmZl07979uM8za9asbx4/ceJELr74Yi666CI+++yzRuWS5mN/eTXTP9vKS59t4UClj6FdM/nD1acxcUgnUpJ0TJxopkI/Qnp6On//+9/rvO+HP/whTzzxBOPHj+fzz/+9r4bRo0cfdv3111/nt7/9baPWf+gLQ5MnT67z/sWLF5Obm3tYodfU1PCrX/2K8vJyAoEATz31FJMnT/7mOQKBAA8//DA/+9nPWLRoESNHjmxUNolv+w5W8/ynm3l5/jbKqnxcNDiHW0b3Zlj3LM2LxwgVegMMGzaMsWPH1rkFzSHV1dVMnjyZfv36hXXdq1ev5qmnnmLdunVkZGTw5ptv8uCDD5KTk8O0adOoqKggKyuLLVu2HPXYhIQE7r///rDmkfhR5fPz8vytPD1vI2VVPi4e0ok7x/VhQMfWXkeTBlKhN9BNN9103PtTUlK4/vrrw77eQYMG8dxzz/HSSy+Rm5t72M6Lli5dyrPPPktqamrY1yvxyznHuysK+O0/1pK/r4Ix/dszdeJA+nfU/His0oRYlCkqKuK2225j+fLlPPLII/V6zKRJk7jhhhv4+c9/zpw5c5o4ocSD7UXlXD/9S+54fSnpqUm8+r0zeOnGM1TmMU4j9Chz6AtDx9rK5YYbbjjqtssuu4zLLrssAukk1tX4A7zw6RaenLuepIQEHvz2YK49s8dhX72X2KVCF2kmthQe5O63lrF8x34uGpzDr789mE6Z2oY8nqjQReKcc463Fu7goXdWk5yYwLP/OYxLTu3kdSxpAip0kThWWlnDz2d8xT9W7uLs3tk8duVQjcrjWNQVunNO27xGmHYBEJ827inlllcXs62onKkTB/D983qRoLnyuBZVhZ6WlkZRURHZ2dkq9QhxzlFUVHTUfmckts1ZuYufvr2MFimJvHbzKM7sle11JImAqCr0rl27kp+fz969e0/6uQ7tcyVWRTJ/WloaXbt2jci6pGk553hnUzUz5ixmaLcsnrt2mKZYmpGoKvTk5GR69uwZlufKy8sL61HsIy3W80vk+fwBfjV7FTM21DDptM78/junkpp07KNrSfyJqkIXkcYpr/Zx1xtLmbtmD5f0TOaJK0/TfHkzpEIXiXGllTXc+OJClmzfx28mDaZb1VaVeTOlr/6LxLADlTVcP/1Llu3Yz9PXDOO6s3K9jiQe0ghdJEaVlNdw/fQFrC44wLNThnHR4I5eRxKPqdBFYtCByhqufWEB63aV8ty1wxk3MMfrSBIFVOgiMaayxs/NLy1i7a4DTLtuBGMHdPA6kkQJFbpIDKnxB7jj9SUs3FbMU1efrjKXw+hDUZEYEQg4fjHjK+au2cNDk4Zw2dDOXkeSKKNCF4kR//3+OmYu3clPL+jHdWf28DqORCEVukgMmLE4nz/lbeKaM7pzx/l9vI4jUUqFLhLlFm4tZurMrzi7dzYPTRqsHdfJManQRaLYjuJybn11MV3btOSPU4aRnKgfWTk2vTtEolRFtZ/vv7IInz/AC98dQVbLFK8jSZTTZosiUcg5x72zVrBudykv3jCSXu3TvY4kMUAjdJEo9NbCHcxcspM7z+/LmP7a1lzqR4UuEmVW7izhgdmrOK9vO340rq/XcSSGqNBFosiByhp+8NoSslul8ORVp5Go3eBKA2gOXSSKPDBrJTv3V/D2rWeSnZ7qdRyJMRqhi0SJvy3byaxlX3Pn+X0Y3qOt13EkBqnQRaLAzv0V3DdrJcO6Z3HHWH0TVBpHhS7iMX/A8ZO3lhEIOJ686nSS9OUhaSTNoYt47PlPNrNgSzGPXjGU7tktvY4jMUxDAREPbdpbxmMfrOeiwTn8x7AuXseRGKdCF/HIof2bt0hO5DeTh2inW3LSVOgiHnnl860s2raPBy4dRIeMNK/jSBxQoYt4YEdxOb+bs45v9WvP5ZpqkTBRoYtEmHOOe2Z+RWKC8V+Xn6KpFgkbFbpIhM1e/jWfbSziFxP60yWrhddxJI6o0EUiqKSiht+8s4ahXTP5z1E6LqiEl7ZDF4mgx99fR/HBKl68YaR2vCVhpxG6SISsyC/h1S+2cd2ZPTila6bXcSQOqdBFIsAfcNw3awVtW6Xykwv7ex1H4pQKXSQC3l60g+X5Jdx/6UAyWyR7HUfilApdpIkdqKzh0X+u44zctnx7aGev40gc04eiIk3s6Q83UFxezcuXDdI259KkNEIXaUJbCg/y0vytXDm8G0O66INQaVoqdJEm9PC7q0lNSuT/XqQPQqXpqdBFmsjH6/cyd80e7ji/D+0zdHxQaXoqdJEm4A84Hn53DT2yW3LjOblex5FmQoUu0gT+siSfdbtL+cWEAaQmJXodR5oJFbpImFVU+3n8/fWc1i2LiUM6eh1HmhEVukiYTf9sC7sOVDJ14gBtpigRpUIXCaOisir+lLeJ8QNzGNUr2+s40syo0EXC6Ol5Gymv9nHPRG2mKJGnQhcJkx3F5by2YBtXjexGnw4ZXseRZkiFLhImT8xdT4IZPxrXz+so0kyp0EXCYMPuUv66dCffPTuXjplpXseRZkqFLhIGj3+wnlYpSdz2rd5eR5FmTIUucpK+yt/PP1bu4ubzetK2VYrXcaQZU6GLnKRH319Pm5bJfO/cnl5HkWZOhS5yEhZsLuLj9Xv5wZg+ZKTpSETiLRW6SCM553jsg/V0yEjlurN6eB1HRIUu0lifbyriyy3F/GBMb9KStQMu8Z4KXaQRnHM8MXc9HVuncfUZ3b2OIwKo0EUa5dONhSzcuo8fjtXoXKKHCl2kgZxzPPHBejpnpnHlyG5exxH5hgpdpIE+3lDIku37+cHYPjp4hUQVFbpIAxwanXfJasGVIzQ6l+iiQhdpgI83FLJsx35+MLY3KUn68ZHoonekSD055/jD3ODc+XeGd/U6jshRVOgi9TR/UxFLtu/n9jG9NXcuUUmFLlJPf/hwAzmtU7lCc+cSpVToIvXwxebgt0Jv/5a2O5fopUIXqYenPtxA+4xUfStUopoKXeQEFm0tZv6mIm4d3Uujc4lqKnSRE3h63kayW6UwZZT2qCjRTYUuchzLd+znX+v3cvN5vWiRotG5RDcVushxPPPRRjJbJGt/5xITVOgix7Cm4AAfrN7NTef0JD01yes4IiekQhc5hmc+2kh6ahI3nJ3rdRSRelGhi9Rh454y3ltRwPVn9SCzpY4VKrFBhS5Shz/mbSQtKZHvndvT6ygi9VavQjezCWa2zsw2mtk9ddx/g5ntNbNlodPN4Y8qEhl7ygP8bdnXTBnVnez0VK/jiNTbCT/pMbNE4FngAiAfWGhms51zq49Y9C3n3B1NkFEkot7bXEOiGd8f3cvrKCINUp8R+hnARufcZudcNfAmMKlpY4l4o6Ckgk93+rhyZFdyWqd5HUekQeqzLVYXYEet6/nAqDqW+w8zGw2sB37snNtx5AJmdgtwC0BOTg55eXkNDlxfZWVlTfr8TU35vfHamioCzjE0ZW9M5ofY/bc/RPkbL1wb1/4deMM5V2VmtwIvA+cfuZBzbhowDWDEiBFuzJgxYVr90fLy8mjK529qyh95hWVVfPLhPM7ukswVFx/19o0ZsfhvX5vyN159plx2ArV3AN01dNs3nHNFzrmq0NXngeHhiScSOS98uoUqX4BLe2kzRYlN9Sn0hUBfM+tpZinA1cDs2guYWadaV78NrAlfRJGmt7+8mlc/38Ylp3SiYyttzSux6YTvXOecD7gD+CfBon7bObfKzB4ys2+HFrvLzFaZ2XLgLuCGpgos0hRemr+Vsiofd5zfx+soIo1Wrzl059x7wHtH3PZArctTganhjSYSGaWVNUz/dAsXDsphQMfW7FrrdSKRxtHfltLsvfrFNg5U+rjz/L5eRxE5KSp0adbKq308/8kWxvRvzyldM72OI3JSVOjSrL2+YDvFB6u5U3PnEgdU6NJsVdb4mfbxZs7unc3wHm29jiNy0lTo0mz9edEO9pRWacsWiRsqdGmWqnx+/pi3iZG5bTirV7bXcUTCQoUuzdKMxfkUlFRy17i+mJnXcUTCQoUuzU61L8AfP9rE6d2zOLdPO6/jiISNCl2anb8uzWfn/gqNziXuqNClWanxB3jmo42c2jWTMf3aex1HJKxU6NKszFq6kx3FFdx1vkbnEn9U6NJs+EKj80GdWjNuYAev44iEnQpdmo2ZS3eyraicH1/QT6NziUsqdGkWavwBnp63gVO6ZDJeo3OJUyp0aRb+sjifHcUV/PgCzZ1L/FKhS9yr9gV4et5GhnbLYmx/jc4lfqnQJe7NWBzc7vzu8RqdS3xToUtcq/YFePajjZzePUvbnUvcU6FLXHtz4XZ27q/gJ9qyRZoBFbrErfJqH099uJEze7XVPlukWVChS9x6ef42Csuq+NlF/TU6l2ZBhS5xqaSihuf+tYnzB3TQ0Yik2VChS1x6/pPNlFTU8NML+3kdRSRiVOgSdwrLqnjh0y1cemonBnfO9DqOSMSo0CXuPDNvI5U1fn58gUbn0ryo0CWubCs6yGsLtnHVyO70bp/udRyRiFKhS1z573+uIykhgR+P7+t1FJGIU6FL3Fi2Yz/vfFXA90f3okPrNK/jiEScCl3ignOOR95bQ7v0FG4Z3cvrOCKeUKFLXPho3R4WbCnmR+P7kZ6a5HUcEU+o0CXm+fwBHnlvLb3ateLqkd28jiPiGRW6xLzXv9zOhj1l3DNxAMmJektL86V3v8S0/eXVPP7Bes7pk80Fg3K8jiPiKRW6xLQn527gQEUN9186SDvgkmZPhS4xa/3uUl79YhtTRvVgQMfWXscR8ZwKXWKSc47fvLOaVimJ+oq/SIgKXWLSB6t388mGQu4e34+2rVK8jiMSFVToEnPKq308+PfV9M/J4LqzengdRyRq6BsYEnOe+nAjO/dX8OfbztJmiiK16KdBYsr63aU8/8lmrhjelZG5OhKRSG0qdIkZzjnum7WSVqlJ3DNxgNdxRKKOCl1ixswlO/lySzH3TBxAdnqq13FEoo4KXWLC3tIqfvPuak7vnsVVI7S/FpG6qNAlJvx69irKq/z8/j9OJSFB3wgVqYsKXaLenJUFvLuigLvG9aFvTobXcUSilgpdotr+8mrum7WKQZ1ac+u3ensdRySqaTt0iWoPvbOa/eXVvHzTSG1zLnIC+gmRqDVn5S5mLtnJbd/qzeDOmV7HEYl6KnSJSrsPVDJ15lcM6dKau8b19TqOSExQoUvUCQQc//fPy6mo8fPkVaeTkqS3qUh96CdFos6L87fyyYZC7rtkEH06pHsdRyRmqNAlqqwpOMDv5qxl3IAOTBnV3es4IjFFhS5Ro7Syhh+8toTWacn87jun6pByIg2kzRYlKjjnuOcvK9heXM5rN4+infbVItJgGqFLVHhp/lbeXVHAzy7qz5m9sr2OIxKTVOjiucXb9vHwu2sYPzCHW0f38jqOSMxSoYundpVU8oPXFtMpK43HrhyqeXORk6BCF8+UV/u4+ZWFlFX6+J/rR5DZItnrSCIxTR+KiicCAcdP3lrOqq8P8Pz1IxjQsbXXkURinkbo4onHPljHnFW7uPfigYwbmON1HJG4oEKXiHt9wXae/WgTV4/sxvfO7el1HJG4oUKXiHpvRQH3zlrBmP7teWjSEH0IKhJGKnSJmE83FHL3m8sY3r0Nf5oyXDvdEgkz/URJRCzdvo9bXl1Er/ateOG7I2mRkuh1JJG4o0KXJrd42z6uf+FL2qWn8spNZ5DZUpsnijQFFbo0qYVbi7n+hQVkp6fw1q1n0qF1mteRROKWtkOXJvPF5iJuemkhHTPTeOP7Z5KjMhdpUip0aRKLd/uYNvdLurdtyWvfH0WHDJW5SFPTlIuE3cvzt/LM0ioGdmrNm7ecqTIXiRCN0CVsAgHH7+as5f99vJnTOyTy+vfP1NYsIhGkQpewOFBZw0/eWs7cNbu59szunJ9ZqDIXiTBNuchJ27C7lMnPfMZH6/bwwKWD+M2kISToG6AiEacRupyUd776mp/P+IqWKYm8fvMoRuloQyKeUaFLo5RW1vDr2av5y5J8Tu+exZ+mDKdjpj78FPGSCl0abPG2Yu5+axk791Vw5/l9uGtcX5ITNXsn4jUVutRbWZWPx95fx8vzt9I5qwVv33oWI3Lbeh1LREJU6FIvc1fv5oG/raTgQCVTRnXnFxMGkJGmfbKIRBMVuhzXxj2lPPLeWj5cu4f+ORk8/Z/DGN6jjdexRKQOKnSpU1FZFU/O3cDrX26nZXIiUycO4KZze2quXCSKqdDlMEVlVUz7ZDOvfr6NKl+AKaO686NxfclOT/U6moicgApdACgoqeDFz7by6ufbqPT5uezUztw1ri99OqR7HU1E6kmF3swt3b6P6Z9t5b0VBTjnmHRaF344to+KXCQGqdCboZKKGmYv28nbi/JZsbOEjNQkbjonl+vPyqVb25ZexxORRlKhNxNVPj8fry/k78u/5p+rdlHlCzCwU2semjSYy4d1JT1VbwWRWKef4jhWVuXj0w17eX/1bj5YtZvSKh9ZLZO5ckQ3rhrZjSFdMr2OKCJhpEKPI4GAY93uUj7bWMi/1u/li81F1PgdrdOSmDCkI5cO7czZvbO16aFInFKhxzCfP8CaglIWbStm0dZ9fLG5iKKD1QD0at+KG8/pyfkDOjC8RxuVuEgzoEKPETX+AFsKD7JyZwkrdx4Inn9dQnm1H4DOmWl8q197zu7TjrN7Z9M5q4XHiUUk0lToUaa82sfWwnK+LPDx1Ycb2JdqRDgAAAb4SURBVLinjPW7S9m0t4wavwMgLTmBQZ1ac+WIbgzv0YbhPdqowEVEhR5JgYCj6GA1uw9UsvtAJV/vr+DrkuD5juJythdXUFhWVesR6+mS1YJ+OemM6d+B/h3TGdQpk97tW5GkKRQROUK9Ct3MJgB/ABKB551zvz3i/lTgFWA4UARc5ZzbGt6o0aXaF6CsykdpZQ0lFTUcqPBRUlHD/opq9h2sZl95DfsOVlN4sJrC0iqKDlZRWFaNP+AOe57kRKNjZhrd2rRk/MAOdGvbkh7ZLSneuoYrJozRcTlFpN5OWOhmlgg8C1wA5AMLzWy2c251rcW+B+xzzvUxs6uB3wFXNUVgAOcc/oDDF/j3uc8fwB9w1IQuf10WYE3BAXx+R7U/QLUvQE3ovNofoMrnp9oXoMoXoKomeL2yJkBFjZ+KGj+V1X7Kq/2U1/ipqPZxsMrPwWofZZU+yqp8VPkCx83YMiWRNi1TaJeeQqfMNIZ0aU37jFRyWqfRISONnNapdMlqQbv0VBISjj7+Zl7xepW5iDRIfUboZwAbnXObAczsTWASULvQJwG/Dl2eATxjZuacO3w4Wsv63aWc/2geAecIOILngeBlf+iyP1Tchy4HAnxzW718+kn9lgtJMGiRnEiLlERSkxJpmXLolETnrGRapSbRKjWJ9NApIy2JjLRkWqclkdkimcyWyWS1SCGrZTJpySpjEYms+hR6F2BHrev5wKhjLeOc85lZCZANFNZeyMxuAW4BSM/JpX1SJWZgQIJZrctHnDASEhJIOOK+xARINPv39dBtSQY11VW0apFGUkLw9uQEIymB0MlITiB0MpITg5cTDeywo9U7wBc61aEmdCqFCoKnXfX4B62PsrIy8vLywvRskRfL+WM5Oyi/17zMH9EPRZ1z04BpACNGjHBv3T2hydaVl5fHmDFjmuz5m5ryeyeWs4Pye83L/PXZVGIn0K3W9a6h2+pcxsySgEyCH46KiEiE1KfQFwJ9zaynmaUAVwOzj1hmNvDd0OXvAPOON38uIiLhd8Ipl9Cc+B3APwlutjjdObfKzB4CFjnnZgMvAK+a2UagmGDpi4hIBNVrDt059x7w3hG3PVDrciVwRXijiYhIQ+jrhiIicUKFLiISJ1ToIiJxQoUuIhInzKutC81sL7CtCVfRjiO+qRpjlN87sZwdlN9rTZ2/h3OufV13eFboTc3MFjnnRnido7GU3zuxnB2U32te5teUi4hInFChi4jEiXgu9GleBzhJyu+dWM4Oyu81z/LH7Ry6iEhzE88jdBGRZkWFLiISJ+K+0M3sTjNba2arzOz3XudpDDP7qZk5M2vndZb6MrP/Dv27f2VmfzWzLK8z1YeZTTCzdWa20czu8TpPQ5hZNzP7yMxWh97vP/I6U0OZWaKZLTWzd7zO0hhmlmVmM0Lv/TVmdlYk1x/XhW5mYwke73Soc24w8KjHkRrMzLoBFwLbvc7SQB8AQ5xzpwLrgake5zmhWgdEnwgMAq4xs0HepmoQH/BT59wg4EzghzGWH+BHwBqvQ5yEPwBznHMDgKFE+LXEdaEDtwO/dc5VATjn9nicpzGeAH5O8ACnMcM5975z7tDBWL8geKSraPfNAdGdc9XAoQOixwTnXIFzbknocinBMunibar6M7OuwCXA815naQwzywRGEzw+BM65aufc/khmiPdC7wecZ2YLzOxfZjbS60ANYWaTgJ3OueVeZzlJNwH/8DpEPdR1QPSYKcTazCwXOB1Y4G2SBnmS4OAl4HWQRuoJ7AVeDE0bPW9mrSIZIKIHiW4KZjYX6FjHXfcSfH1tCf75ORJ428x6RdPh8U6Q/5cEp1ui0vGyO+f+FlrmXoJTAa9FMltzZmbpwF+Au51zB7zOUx9mdimwxzm32MzGeJ2nkZKAYcCdzrkFZvYH4B7g/kgGiGnOufHHus/Mbgdmhgr8SzMLENxxzt5I5TuRY+U3s1MI/sZfbmYQnLJYYmZnOOd2RTDiMR3v3x7AzG4ALgXGRdMv0eOozwHRo5qZJRMs89ecczO9ztMA5wDfNrOLgTSgtZn9r3PuWo9zNUQ+kO+cO/RX0QyChR4x8T7lMgsYC2Bm/YAUYmQvbs65Fc65Ds65XOdcLsE3y7BoKfMTMbMJBP98/rZzrtzrPPVUnwOiRy0L/uZ/AVjjnHvc6zwN4Zyb6pzrGnqvX03wQPOxVOaEfjZ3mFn/0E3jgNWRzBDzI/QTmA5MN7OVQDXw3RgZKcaDZ4BU4IPQXxhfOOdu8zbS8R3rgOgex2qIc4DrgBVmtix02y9DxwSWyLgTeC00INgM3BjJleur/yIicSLep1xERJoNFbqISJxQoYuIxAkVuohInFChi4jECRW6iEicUKGLiMSJ/w+a+bLGSkCgbAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "xs_sig = np.linspace(-7, 7, 100)\n",
    "ys_sig = sigmoid(xs_sig)\n",
    "plt.plot(xs_sig, ys_sig, label=r'$\\sigma(x) = {\\frac{1}{1 + e^{-x}}}$')\n",
    "plt.yticks([0, 0.5, 1])\n",
    "plt.xlim([-7, 7])\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.title('Sigmoid function')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU5dn/8c81WQkEEJCAgiCKC0LZUtBWBGu1QN21givuT1vRYp+6YVVURKxa9XFrrY/i0oLVPtZqS9Vardoqgj+wiuwqgrIIYQ1LSHL//rgmZqQJJGaSMzP5vl+v88rMnDNnrkzgO2fuc5/7thACIiKS/mJRFyAiIsmhQBcRyRAKdBGRDKFAFxHJEAp0EZEMoUAXEckQCnSRBGb2KzO7rolea4iZLWiK15LmwdQPXZqamb0G9AU6hRC2R1xOkzGzAPQMISyOuhbJTDpClyZlZt2BIUAAjm+k18hujP2KpDoFujS1c4C3gSnAmKoHzWxKvLnjZTPbZGb/MLNuCeuDmV1mZh+Z2Rozu93MYvF155rZP83sLjNbC0wwszZm9riZfWFmS83s52YWM7N2ZrbczI6LP7eVmS02s3MS6pgYvz0svu2VZrbazFaY2YlmNtLMFppZiZmNT6hxkJm9ZWbr49veZ2a58XWvxzd7z8w2m9moqv0nPP9gM3st/vy5ZnZ8wropZna/mf05/v7MMLP9kv3HkfSmQJemdg7w2/jyPTMrSlh3JnAz0AGYE98m0UlAMTAAOAE4P2HdYOAjoAi4BbgXaAP0AIbGX/e8EEJJ/Hm/MbOOwF3AnBDC47XU2wnIB/YGrgd+A5wFDMS/aVxnZvvGt60ALo/XfxhwFPBjgBDCEfFt+oYQWoUQnkp8ETPLAZ4HXgI6ApcCvzWzAxM2Gw3cCOwBLI7/niLVQghatDTJAhwO7AA6xO/PBy6P354CTEvYthUekF3j9wMwPGH9j4FX4rfPBT5NWJcFlAG9Eh77L+C1hPv3Au8DnwHtEx6fAkyM3x4GbAWy4vcL43UMTtj+XeDEWn7fccCzCfcDsH/C/WHA8vjtIcBKIJawfiowIaGuhxPWjQTmR/031ZJai47QpSmNAV4KIayJ3/8dCc0uwLKqGyGEzUAJsFdN64Glu1jXAciJb5O4/d4J9x8CegNTQghrd1Hz2hBCRfz21vjPVQnrt+IfPpjZAWb2gpmtNLONwKR4LXWxF7AshFC5i5pXJtzeUvW6IlUU6NIkzKwFcBowNB54K/Hmib5m1je+WdeE7VsB7YDPE3bTNeH2PjutS+yutQb/JtBtp+0/i+87Cw/0x4Efm9n+DfjVEj2If+voGUJoDYwHrI7P/RzoWnVeYOeaRepCgS5N5US8CaUX0C++HAy8gbdvA4w0s8PjJxJvBt4OISQeeV9hZnuYWVfgJ8BX2qGrxI+ofw/cYmaF8ZOrPwWejG8yHv8AOB+4HXg8HvINVQhsBDab2UHAj3Zavwpv06/JDPyo+0ozyzGzYcBxwLQk1CXNhAJdmsoY4NEQwqchhJVVC3AffjI0G2+CuQFvahmIn3xM9BzeZj0H+DPwv7t4vUuBUvxE6ZvxfT9iZgPxcD8nHvy34eF+dRJ+x58BZwCb8JOnO3/gTAAei/diOS1xRQihDA/wEfg3jAfiNc5PQl3STOjCIkkJZjYFP0H481rW66Ickd3QEbqISIZQoIuIZAg1uYiIZAgdoYuIZIjIBjHq0KFD6N69e1QvLyKSlt599901IYQ9a1oXWaB3796dWbNmRfXyIiJpycyW1rZOTS4iIhlCgS4ikiEU6CIiGUKBLiKSIRToIiIZYreBbmaPxKff+qCW9WZm/xOfxuvfZjYg+WWKiMju1KXb4hR8RLzapugaAfSML4PxMaEHJ6O4mixYAK++6rePPBIOPHDX2+/83EWLICsLKiqqf/bs6ftZsACmToXVq+G44/w5b70Fhx321ds9elTvZ+lS+OADX9auhfbtYe+9YcMGCMG3XbcOli2DLl1gyJCv1j19Otx2G3z0EbRpA926wb77+nNLSuCAA3x/r74KGzdC9+7QoYO//vLlEIvB5s1gBh07+j5CgHbtvLZ58/z24MH+eAjQr5//3ivj0yV06vTVmhYsgHvvhbff9nUnnAD//rfXeOKJcNFF1e/p9OnV78uIEdXvcdV7Wpe/R122FZHdq9Ol//GZ2l8IIfSuYd2v8am9psbvLwCGhRBW7GqfxcXFob790BcsgEmXf8HqD1ZTTjb7HZjNFdf4T3JyIDsbcnP9dk6Op13icyd5+M2ZA/vtB0uWeLi1agWjR8Odd8KMGR6OeXm+u1at/Dngt7dt8/Bp0wZmzoStWz3I6zKCgpkH/uGHw+TJHpBnnumBv/N2VUtWlu+7ogIqK/1XisWgvLz214nFqsN758fz8vzxVq289hD8A6J/f/9gATj3XHjnHX898Peh6rVzc+Huuz3Up0+HsWP98cpKuPpqePPN6g/K8eNrD+qqv0ddthWRamb2bgihuMZ1SQj0F4DJIYQ34/dfAa4KIfxHWpvZxcDFAEVFRQOnTavf2P0bNkCnJ6Yx+A+/rtP2IRajMieHkJ1NRXYO5Vk5VGTnsCMrl8qcXMqzcgj5eVTk5hLy8thieZRl5VOem0dZTj7leS2gVT6l1oKy/AJirQvYHCugolUrYnu0pKSiJWVZeVRU1nVSGg+wli39aLq01I+Sa/sTWHy3yRxupyrsq36Cf/bl5kJRfLrmTz+FsrLan7vHHv4t4vPP/VtEbq5v37JldeiXlUHbtv7BV5MNG2D9+rptKyLVjjzyyFoDvUmvFA0hPIRP/UVxcXEYNmxYvZ6/YAH8z/rOPFl0FFmhnO5dyhl1yg46dyiHHTt8KS/3hNixA9uxg6yyMti+nc2ry/j3zO1Qtp3Sku3s0WIblZu30q5gG/lspUPL9eR+sZWs7VsoCKW0ZEudatpOLuvYgxLasZb2rKU9a+jAajqyiiJW05EVdI4ve5HXofArR+hjx6beEfrkybs/Qh82zI/Qr7tOR+giqSKtmlygCdvQv19JVtlWZr9ZyuBDNpO9dRMLZm6k774b2avlBtZ+tIHcrRvY8vk6tixfR/nqElps9UhvV/EFrbd/QdaXcwtX257fGrp2Ja9HF+jenQVl+zLt7e68U7I/a9v1pEOP1mpDF5FaNXaTy/eBscBI/GTo/4QQBu1un1830NNGZaUfeq9aBStW+PL5557Cy5d7u8Ynn3gDfKKOHeHgg6FXL1++8Q3o21ftESIC7DrQd9vkYmZTgWFABzNbjs/5mAMQQvgV8Bc8zBfjk9yel5yy01ws5mdA27f3YK7Npk1+6LtkiR+uLlzoh9W/+503NFfp3h0GDoRBg3z55je90VpEJC6yCS4y/gi9oULwo/r33vNl9myYNcvDH7xhe8AAOOIIb9AeOtQbxkUkozW4yaUxKNC/pjVr/IzlP/8Jr7/ut8vKvKvKt74Fw4d7o/dBB1V3kxGRjKFAz2Tbtnm4v/SSL3Pm+OP77w8nn+wd7Pv1U7iLZAgFenOyfDk8/zw89xy88or3bzzwQDjjDL9iaJ99oq5QRBpgV4GuwbkyTZcu8KMfwV//6v0SH3oI9toLJkzwE6sjR8Kzz3q/RRHJKAr0TNa+vXca//vf/WTqz3/uHcpPPtmbZO66yzu3i0hGUKA3F927w003+dVGf/iDH8n/9KfQtatf7llSEnWFItJACvTmJivLj9DfeMNHFzvmGJg40Yd5vPZaH2BFRNKSAr05Ky6Gp5+G99/3tvVJk7wp5r77fFwcEUkrCnSB3r3hqafg3Xd9qIFLL/XHXn456spEpB4U6FJtwADv6vj8836l6jHH+IDtq1ZFXZmI1IECXb7KDI491nvDXH89PPOMX3X6298md2B2EUk6BbrULD8fbrzRx5Hp1QvOOsuvOlVvGJGUpUCXXTvoIB8zZtIk+L//gz594LXXoq5KRGqgQJfdy8qCa67xgcAKC+G734Xbb1cTjEiKUaBL3fXv733XTzoJrrwSTjlFV5qKpBAFutRPYSH8/vc+bMDzz8O3v+2zL4lI5BToUn9mMG6cDwC2bJlPWKqRM0Uip0CXr++oo+Bf//IeMUccAS+8EHVFIs2aAl0aplcvePttOOQQb1t/6qmoKxJpthTo0nBFRX6F6be+BaefDo88EnVFIs2SAl2So3VrmD7dhwu44AJ44IGoKxJpdhTokjwFBT713QknwCWXwKOPRl2RSLOiQJfkysvzdvRjjoELL1SbukgTUqBL8uXl+byl3/62jwGj3i8iTUKBLo2joMCDvH9/OO00HzZARBqVAl0aT+vWHuqdOsFxx8HHH0ddkUhGU6BL4+rY0Xu/7NgBI0Zo+F2RRqRAl8Z34IHe++Xjj31AL81XKtIoFOjSNIYMgYcf9rHUr7wy6mpEMlJ21AVIM3L22T4R9d13w8CB3gNGRJJGR+jStG6/HYYOhYsugtmzo65GJKMo0KVp5eT4eOp77gknnwzr10ddkUjGUKBL0+vY0UN92TK4+GJNZSeSJAp0icahh8Itt8DTT/vJUhFpMAW6ROeKK+Doo+Gyy2Du3KirEUl7CnSJTiwGTzwBbdrAqFGwbVvUFYmkNQW6RKuoCKZM8SP0G26IuhqRtKZAl+gNH+4nR2+/3ecoFZGvRYEuqeGOO6BbNxgzBkpLo65GJC0p0CU1FBZ608vixXDVVVFXI5KWFOiSOoYOhZ/8BO6/H958M+pqRNKOAl1Sy8SJ3vRy8cWwfXvU1YikFQW6pJZWreCBB2DePLjttqirEUkrCnRJPSNHwujRfiXp/PlRVyOSNhTokpruvhtattRYLyL1oECX1FRU5P3S33gDnnwy6mpE0oICXVLXeefBoEE+5svGjVFXI5LyFOiSumIxuO8+WL0abrwx6mpEUp4CXVLbN78JF14I99yjERlFdkOBLqlv0iRo3RouvVQnSEV2QYEuqa9DB7/g6NVX4Y9/jLoakZSlQJf0cPHF0KsXXHkllJVFXY1ISlKgS3rIzvYRGRcv9itJReQ/KNAlfQwfDsccAzfdBCUlUVcjknIU6JI+zPwofcMGuPnmqKsRSTkKdEkvffrABRd4//TFi6OuRiSlKNAl/dx0E+TmwnXXRV2JSEpRoEv66dQJLr8cpk2D2bOjrkYkZSjQJT1dcQW0awfXXBN1JSIpQ4Eu6alNGxg/Hl580S84EhEFuqSxSy6Brl3h6qs1JIAICnRJZ/n5PgrjO+/Ac89FXY1I5BTokt7OPhsOOACuvx4qK6OuRiRSCnRJb9nZMGECvP8+PPNM1NWIREqBLunvtNN84K4JE6CiIupqRCKjQJf0l5Xlbenz5sHUqVFXIxIZBbpkhpNPhr59PdjLy6OuRiQSCnTJDLGYDwmweDE88UTU1YhEQoEumeO442DAALjlFh2lS7OkQJfMYebdF5csUVu6NEsKdMksxx/vbekTJ6rHizQ7CnTJLFVH6QsX+miMIs2IAl0yz4kn+kQYOkqXZkaBLpknFvPJL+bPh6efjroakSajQJfMdMopcPDBMGmSxniRZkOBLpkpFvPx0t9/H154IepqRJqEAl0y1+jR0KOHt6VrvHRpBhTokrmys+Gqq2DmTPjb36KuRqTRKdAls40ZA3vv7VePimQ4Bbpktrw8n1D6H/+AN9+MuhqRRqVAl8x30UWw557e40UkgynQJfMVFMC4cTB9OsyZE3U1Io1GgS7Nw49/DIWFMHly1JWINBoFujQPbdt6qD/9NCxaFHU1Io1CgS7Nx7hxkJMDv/hF1JWINAoFujQfnTrB+efDY4/BZ59FXY1I0inQpXm54gof2+WXv4y6EpGkU6BL87Lvvj4kwK9/DSUlUVcjklQKdGl+rroKSkvh/vujrkQkqRTo0vz06QPf/z7cc48Hu0iGqFOgm9lwM1tgZovN7Ooa1p9rZl+Y2Zz4cmHySxVJomuugbVr4ZFHoq5EJGl2G+hmlgXcD4wAegGnm1mvGjZ9KoTQL748nOQ6RZLr29+Gww+HO+6AHTuirkYkKepyhD4IWBxC+CiEUAZMA05o3LJEmsDVV8Onn8LUqVFXIpIU2XXYZm9gWcL95cDgGrY7xcyOABYCl4cQlu28gZldDFwMUFRUxGuvvVbvgkWSpqCA4h49sOuvZ2aXLj7LkUgaq0ug18XzwNQQwnYz+y/gMeA7O28UQngIeAiguLg4DBs2LEkvL/I13XQTnHUWwzZvhuOPj7oakQapyyHJZ0DXhPtd4o99KYSwNoSwPX73YWBgcsoTaWSjRkH37nDrrZqmTtJeXQJ9JtDTzPY1s1xgNPCnxA3MrHPC3eOBeckrUaQRZWfDz34Gb78Nb7wRdTUiDbLbQA8hlANjgRfxoP59CGGumd1kZlXfUS8zs7lm9h5wGXBuYxUsknTnn+8TYNx6a9SViDSIhYi+ZhYXF4dZs2ZF8toi/2HSJLj2Wpg9G/r1i7oakVqZ2bshhOKa1um0vghoAgzJCAp0EfAJMH74Q58AY/HiqKsR+VoU6CJVLr9cE2BIWlOgi1Tp3BnOO88nwPj886irEak3BbpIoiuugPJyTYAhaUmBLpKoRw+fAONXv9IEGJJ2FOgiO7v6ah8n/b77oq5EpF4U6CI769MHjjvOJ8DYvDnqakTqTIEuUpNrr/Uml1/9KupKROpMgS5Sk8GD4bvf9Qkwtm6NuhqROlGgi9Tm2mth1SpNUydpQ4EuUpuhQ32quttug7KyqKsR2S0FukhtzPwofdkyePLJqKsR2S0FusiuDB8OAwb4aIzl5VFXI7JLCnSRXTGD666DJUs0mbSkPAW6yO6ccAL07QsTJ0JFRdTViNRKgS6yO1VH6QsXwlNPRV2NSK0U6CJ1cdJJ0Ls33HyzjtIlZSnQReoiFoPrr4f5830SDJEUpEAXqatTToFevXSULilLgS5SV7EY3HADfPih2tIlJSnQRerj1FPhG9+ACRPUL11SjgJdpD5iMbjxRli0SFePSspRoIvU1wkn+NWjN90EO3ZEXY3IlxToIvVl5mH+8cfw6KNRVyPyJQW6yNcxcqSPmT5xImzbFnU1IoACXeTrMfMBu5YtgwceiLoaEUCBLvL1fec7cPTRcMstsGFD1NWIKNBFGmTyZJ979Pbbo65ERIEu0iADBsCoUXDXXbBiRdTVSDOnQBdpqIkTfYq6m2+OuhJp5hToIg21//5w0UXwm9/4ELsiEVGgiyTDDTdAfj5ceWXUlUgzpkAXSYaiIhg/Hp57Dl59NepqpJlSoIsky7hxsM8+8NOfanhdiYQCXSRZWrTwboxz5sATT0RdjTRDCnSRZBo92ocEGD8eNm+OuhppZhToIslkVt0n/ZZboq5GmhkFukiyHXYYjBkDd96pbozSpBToIo1h8mRvU7/sMggh6mqkmVCgizSGTp18ZqMXX/SujCJNQIEu0ljGjoXevb0745YtUVcjzYACXaSxZGfDfffB0qU+w5FII1OgizSmoUPh/PPhjjvgvfeirkYynAJdpLHdfju0b+8DeOkKUmlECnSRxtauHdxzD8yc6U0wIo1EgS7SFEaNghEj4Npr4ZNPoq5GMpQCXaQpmMGDD/rP88+HysqoK5IMpEAXaSrduvmwAK++CvffH3U1koEU6CJN6YILYORIuOoqDQsgSadAF2lKZj5VXX4+nHMOlJdHXZFkEAW6SFPbay944AGYMUMjMkpSKdBFojB6NJx9tl9B+o9/RF2NZAgFukhU7r8f9tsPzjgD1qyJuhrJAAp0kagUFsJTT3mYn3uuhtmVBlOgi0Spf38f5+XPf/YhAkQaQIEuErWxY+HUU+Gaa+Dll6OuRtKYAl0kambw6KPQq5efLP3446grkjSlQBdJBa1awR//6EMCnHgilJZGXZGkIQW6SKrYbz+YOhXef9+7NGqoXaknBbpIKhk+HH75S3j2WfjZz6KuRtJMdtQFiMhOxo3zIXbvvhu6d4ef/CTqiiRNKNBFUtGdd8Knn8Lll0OXLnDKKVFXJGlATS4iqSgrC558Eg49FE4/Hf7616grkjSgQBdJVQUF8Je/QO/ecNJJ8NprUVckKU6BLpLK2raFl16CHj3g2GPhX/+KuiJJYQp0kVTXoQP87W8+7O4xx8Df/x51RZKiFOgi6aBzZx9mt3t3n/Hoz3+OuiJJQQp0kXTRubO3o/fu7VeTTpsWdUWSYhToIumkQwd45RU47DDv/XLrrRp2V76kQBdJN23a+InS00+H8ePhwguhrCzqqiQF6MIikXSUnw+//S307OnT2C1Z4pNlFBVFXZlESEfoIunKDG680S9AeucdnyzjzTejrkoipEAXSXdnnglvvw0tW8KwYT5sQGVl1FVJBBToIpngG9+AWbPg+ON9lMajj/axYKRZUaCLZIo2beAPf4Df/AZmzIA+feDxx9ULphlRoItkEjPv9fLeex7oY8b41aWLFkVdmTQBBbpIJtpvP7+y9L77/IRpnz4wYQJs2RJ1ZdKIFOgimSorCy65BObP9ytLb7wRDjgApkzR9HYZSoEukuk6d/ZhAl5/3Qf4Ou88GDiwelJqyRgKdJHmYsgQ7944dSqUlvoY6/37wzPP6Ig9QyjQRZqTWAxGj4Z58+CJJ2D7dvjBD7wp5p57YOPGqCuUBlCgizRH2dlw1lkwdy48/bQ3y4wb5/OX/uhH8O676u6YhhToIs1ZVhaceqoPGTBjhp88fewxKC725pg77oBly6KuUupIgS4ibtAgvxDp88/hgQcgNxeuuAL22cfb3++6ywcBk5RlIaKvVcXFxWHWrFmRvLaI1NGSJd5D5qmn4P33/bFDDoERI3x4gSFDoEWLaGtsZszs3RBCcY3rFOgiUicffQTPPw9/+pM30ZSVQV4eHHqoB/vhh/vEG61bR11pRlOgi0hylZbCG2/Ayy97//bZs73roxkcdJA33wwcCH37+tKmTdQVZwwFuog0rk2bvI/7jBk+1MA778CqVdXr99kHevXy5aCDfGKO/ff3C51iOpVXH7sKdM1YJCINV1jobepHH+33Q4CVK2HOHF/mzoUPP/TxZbZurX5efj506wbdu/vSpQt07eo/O3f2pW1bP/KX3apToJvZcOAeIAt4OIQweaf1ecDjwEBgLTAqhPBJcksVkbRhVh3II0ZUP15Z6d0gFy3yZckSWLoUPvnE+76vWfOf+8rLg44dfdlzT58ou317X/bYo3pp06Z6KSz0JSuryX7lVLDbQDezLOB+4GhgOTDTzP4UQvgwYbMLgHUhhP3NbDRwGzCqMQoWSXULFnhW9ewJBx6Y/O1re35WljdjV+2npv1Onw5vveXnLpcvh7//3VtB5s/3EXf33dcPmEPw9bNnez4OHQq9e8ORR/q50cmTvXfjoEF+fdLSpfDoo7BwoWfuGWf4HNZVdUyd6vNvLF0aY+3abuTkdKOo6LuAH7CbwYYCyOu6jeP6L+e0by1n48KVzJm+grySFRzCFwxuuYryxavJmjmfnE1ryS/btNv3piy7BaFlK6ywFduyWpLTtiUt2hWwenMBqze3IK9NC3LbtKB1x3z26Jzv3xjy8rzLZvznirW5rCzJpWPXXPbulgM5CUt2dvXP7Gz/IyTermFZtCTGws9asn+v3K/1996V3bahm9lhwIQQwvfi968BCCHcmrDNi/Ft3jKzbGAlsGfYxc7Vhi6ZaMECmDSpOlzHj991SNd3+9qev3mzt2z06wetWvnV/dOmfXW/H30EY8d6k/W6dX5eMxbzQE38n2rmj+88vEunTt4qMneuN5lXKSz0bRNH5s3K8tnw/vu/fUa8f/4Ttm2r++9VWOidaLZvr35s7739w+ett7wjzdYNZfTaaz3rPlrHntnraGsbuPScDRzSZQPPPrGJkqWbKAybaJO9mb3alNI+dzPZ5VvpWFDKmk9LyavcRk7FVlrnbCUvbKNFbDux7fUosgEeP+xBXun5w3r/vaGBJ0XN7FRgeAjhwvj9s4HBIYSxCdt8EN9mefz+kvg2a3ba18XAxQBFRUUDp02bVr/fRCTFbdgA69f7AV5ZmTf/7qqDR323r+35lZUe0C1behjn5MCOHV/db2kplJT4Y6WlHuJZWVBeXrfXysnxfZeV/ecHQE0x0qKFv+769R7m9el/UdM+zfz327LFD4DLy6s/eLKy/D1o2xbatYPPPvM6YzF/PDfXW2XKyqrfq1jMPzAKCvzAvG1baNM6YOXlxHbsILZjB5vXlrGppIL82A4qtpbTpqCcwvxyrLwcq6ggVlHhtysrsYTbVFT4/crKLxcqKti2JbCttJK1ffuzutN+9f57Axx55JGpcVI0hPAQ8BD4EfqwYcOa8uVFGl2qH6Ffd13TH6E/8kjjHKFv3OjnTxct8sCOxeCXv/RONPfe6x1uzDywe/f2jjYVFd5d/q67/AOhpMSbnzp3rvm9b+jfZ2cLFsCdkyBrAVR82PD97UxNLiJJpjb0XbWhw9q1frRfVOQ1fdmGvsHvDxkCZ57p2z74oM91fcQR8Itf+O+zYoWfV+3a1YN49mzvQPOd78BFF1W/J1OnwhdfwLHHQo8eX30vqt6Hqn3s6r1v6N8n2ftraJNLNrAQOAr4DJgJnBFCmJuwzSVAnxDCD+MnRU8OIZy2q/0q0EVE6q9B/dBDCOVmNhZ4Ee+2+EgIYa6Z3QTMCiH8Cfhf4AkzWwyUAKOTV76IiNRFndrQQwh/Af6y02PXJ9zeBvwguaWJiEh96JpbEZEMoUAXEckQCnQRkQyhQBcRyRCRDZ9rZl8AS5O0uw5ADaP6pDzV3fTStXbV3fRStfZuIYQ9a1oRWaAnk5nNqq1fZipT3U0vXWtX3U0vHWtXk4uISIZQoIuIZIhMCfSHoi7ga1LdTS9da1fdTS/tas+INnQREcmcI3QRkWZPgS4ikiHSOtDN7BMze9/M5phZWo3Fa2ZtzewZM5tvZvPi486nNDM7MP5eVy0bzWxc1HXVhZldbmZzzewDM5tqZvlR11QXZvaTeM1zU/29NrNHzGx1fAazqsfamdnLZrYo/nOPKGusSS11/yD+nleaWdp0XUzrQI87MoTQL936iwL3AH8NIRwE9AXmRVzPboUQFsTf6y/6tXUAAAKwSURBVH7AQGAL8GzEZe2Wme0NXAYUhxB648NAp/wQz2bWG7gIGIT/GznWzPaPtqpdmgIM3+mxq4FXQgg9gVfi91PNFP6z7g+Ak4HXm7yaBsiEQE87ZtYGOAIfR54QQlkIYX20VdXbUcCSEEKyrvZtbNlAi/iELQXA5xHXUxcHAzNCCFtCCOXAP/CQSUkhhNfx+RASnQA8Fr/9GHBikxZVBzXVHUKYF0JYEFFJX1u6B3oAXjKzd+MTUKeLfYEvgEfNbLaZPWxmLaMuqp5GA1OjLqIuQgifAXcAnwIrgA0hhJeirapOPgCGmFl7MysARgJdI66pvopCCCvit1cCRVEWk+nSPdAPDyEMAEYAl5jZEVEXVEfZwADgwRBCf6CU1PwqWiMzywWOB56Oupa6iLfbnoB/kO4FtDSzs6KtavdCCPOA24CXgL8Cc4CKXT4phcXnGFY/6UaU1oEeP/IihLAab8sdFG1FdbYcWB5CmBG//wwe8OliBPD/Qgiroi6kjr4LfBxC+CKEsAP4P+BbEddUJyGE/w0hDAwhHAGsw+f3TSerzKwzQPzn6ojryWhpG+hm1tLMCqtuA8fgX1FTXghhJbDMzKrm/D4K+DDCkurrdNKkuSXuU+BQMyswM8Pf75Q/CQ1gZh3jP/fB289/F21F9fYnYEz89hjguQhryXhpe6WomfWguodFNvC7EMItEZZUL2bWD3gYyAU+As4LIayLtqrdi394fgr0CCFsiLqeujKzG4FRQDkwG7gwhLA92qp2z8zeANoDO4CfhhBeibikWpnZVGAYPuzsKuAG4I/A74F98OGyTwsh7HziNFK11F0C3AvsCawH5oQQvhdVjXWVtoEuIiJflbZNLiIi8lUKdBGRDKFAFxHJEAp0EZEMoUAXEckQCnQRkQyhQBcRyRD/H1wiqYWe/7RWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#  helper function for approximation\n",
    "def sigmoid_mu_s(x, mu, s):\n",
    "    return 1 / (1 + np.exp(-(x + mu) / s))\n",
    "\n",
    "plt.scatter(X[:, 0], Y, s=10, c='b', alpha=0.5)\n",
    "sig_x = np.linspace(X[:, 0].min(), X[:, 0].max(), 100)\n",
    "y_flip = 1 if Y[X[:, 0].argmin()] == 0 else -1\n",
    "sig_y = sigmoid_mu_s(sig_x, -X[:, 0].mean(), y_flip * 0.5)\n",
    "plt.plot(sig_x, sig_y, c='r')\n",
    "plt.yticks([0, 0.5, 1])\n",
    "plt.grid(axis='y')\n",
    "plt.title('Approximation')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hypothesis and Prediction\n",
    "\n",
    "We can tweak the sigmoid function by passing the dot product of $x$ (vector of feature) and $\\theta$ (the vector of weights).\n",
    "\n",
    "$$h(x) = \\sigma(x \\cdot \\theta) = \\sigma(\\theta_0 x_0 + \\theta_1 x_1 + \\cdots + \\theta_n x_n)$$\n",
    "$x_0$ being equal to 1.  \n",
    "$\\theta$ is not a parameter of $h$ because it's considered as a constant.\n",
    "\n",
    "$h$ is our hypothesis function. It return a probability of an $x$ to belong to one of the group.  \n",
    "To predict some new $x$ we input it to $h$ with the weights and see if the returned value if above or below 0.5.\n",
    "\n",
    "$$\n",
    "\\text{prediction} =\n",
    "    \\begin{cases} \n",
    "        1, & h(x) \\ge 0.5 \\\\\n",
    "        0, & h(x) \\lt 0.5\n",
    "    \\end{cases}\n",
    "$$\n",
    "\n",
    "This is equivalent to:\n",
    "\n",
    "$$\n",
    "\\text{prediction} =\n",
    "    \\begin{cases} \n",
    "        1, & x \\cdot \\theta \\ge 0 \\\\\n",
    "        0, & x \\cdot \\theta \\lt 0\n",
    "    \\end{cases}\n",
    "$$\n",
    "\n",
    "Indeed, in the sigmoid funcion:  \n",
    "* $x$ is *above* 0 then $e^{-x}$ tend to be small, $\\frac{1}{1 + \\text{small number}} \\approx 1$.  \n",
    "* $x$ is *below* 0 then $e^{-x}$ tend to be big, $\\frac{1}{1 + \\text{big number}} \\approx 0$. \n",
    "\n",
    "Therefore, a prediction can be visualized by projecting $x$ onto $\\theta$.  \n",
    "It will be more relevant the more the two vectors go in the same direction\n",
    "(or opposite direction for negative projection),\n",
    "for exemple if $x \\cdot \\theta = 0$ then $x$ is perpendicular to $\\theta$,\n",
    "$h(x) = 0.5$ right in the middle of the sigmoid function,\n",
    "the prediction become more irelevant as $x \\cdot \\theta$ tend to 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hypothesis(x, theta):\n",
    "    # suppose that x_0 = 1 and isn't mentionned\n",
    "    return sigmoid(x.dot(theta))\n",
    "\n",
    "\n",
    "def predict(x, theta):\n",
    "    return int(hypothesis(x, theta) >= 0.5)  #  a bit more clear\n",
    "    # or\n",
    "    return int(x.dot(theta) >= 0)\n",
    "    # there is a nice vector interpretation for this one\n",
    "    # and it probably compute faster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cost function\n",
    "\n",
    "We have to evaluate our model by introducing a cost function, this one is called the *logloss*\n",
    "\n",
    "For a single $x$:\n",
    "$$\n",
    "j(h(x), y) = \n",
    "    \\begin{cases} \n",
    "        -\\ln(h(x)),     & y = 1 \\\\\n",
    "        -\\ln(1 - h(x)), & y = 0 \n",
    "    \\end{cases}\n",
    "$$\n",
    "It can be rewritten as a single continuous function:\n",
    "$$ j(h(x), y) = -y \\ln(h(x)) - (1 - y) \\ln(1 - h(x))$$\n",
    "\n",
    "Even if it may seem more complicated at first sight, it's just a switch with the values of $y$.  \n",
    "- if $y = 1$ then $- (1 - y) \\ln(1 - h(x)) = 0$ and the $-\\ln(h(x))$ stays.\n",
    "- if $y = 0$ then $-y \\ln(h(x)) = 0$            and the $-\\ln(1 - h(x))$ stays."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEICAYAAABCnX+uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3Rc1bn38e/WqPcuq1qyZMm25Cob3HCJDaaYDg6ETgJJLmmElJt7k8slb5JLAiEJgYTQQiDUUGJ6MxYu4G7LXbYsy1bv3aoz+/3jjOQmW8UzOjOa57PWrNFojs55RMa/bO2zi9JaI4QQwn15mV2AEEKIcyNBLoQQbk6CXAgh3JwEuRBCuDkJciGEcHMS5EII4eYkyMWooZS6WilVopRqVUpNH8Hr3qSU+nikrifEqZSMIxfuQCmVB/xTa/30WY45BPxQa73SiXWkAocBH611j7OuI8RQSItcjCZjgT1mFyHESJMgF06hlEpWSr2plKpRStUppR6zf99LKfVzpdQRpVS1Uup5pVSY/T1/pdQ/7cc3KqU2K6XilFK/Bi4AHrN3mzx2yrX8lFKtgAXIt7fMUUpppVTGCcc9p5T6lf3rRUqpUqXUffY6KpRSd5xwbIBS6vf2OpuUUuuUUgHAGvshjfZa5iilbldKrTvhZ+faa2+yP8894b08pdT/U0qtV0q1KKU+VkpFO/a/vvA0EuTC4ZRSFuBd4AiQCiQCr9jfvt3+WAyMA4KB3mC+DQgDkoEo4FtAu9b6v4G1wHe01sFa6++ceD2tdafWOtj+cqrWOn2QpY6xXy8R+DrwuFIqwv7ew0AuMBeIBH4C2IAF9vfD7bV8ecrvHgm8Bzxq/x0eAd5TSkWdcNjXgDuAWMAX+NEg6xWiXxLkwhnOAxKAH2ut27TWHVrr3hbrTcAjWusirXUr8DPgBqWUN9CNEX4ZWmur1nqr1rrZiXV2A7/UWndrrd8HWoEspZQXcCfwfa11mb2WL7TWnYM452XAQa31C1rrHq31y8B+4PITjvm71vqA1rodeA2Y5thfS3gaCXLhDMnAkTPcDEzAaKn3OgJ4A3HAC8BHwCtKqXKl1O+UUj5OrLPulBqPYfyFEA34A4eGcc5Tfz/srxNPeF3ZzzWFGDYJcuEMJUCKvZV9qnKMm5K9UoAeoMreMn5Aaz0Jo0tjOXCr/bjhDK86BgSe8HrMIH+uFugA+uuiGaiOU38/MH7HskFeW4ghkyAXzrAJqAAeVEoF2W9izrO/9zJwr1IqTSkVDPwGeFVr3aOUWqyUmmzvY2/G6Pqw2X+uCqNPfSh2AF9TSlmUUhcDCwfzQ1prG/As8IhSKsH+83OUUn5Ajb2mM9XyPpCplPqaUspbKfVVYBLGPQMhnEKCXDic1tqK0SecARwFSoGv2t9+FqMLZQ3GeOwO4Lv298YAr2OE+D7gc/uxAH8CrlNKNSilHh1kKd+319GI0Tf/7yH8Gj8CdgGbgXrgt4CX1voY8GtgvX1kzewTf0hrXYfxl8R9QB3GTdLlWuvaIVxbiCGRCUFCCOHmpEUuhBBurr+bUadRShUDLYAV6NFaz3RmUUIIIQZvUEFut1j6+YQQwvVI14oQQri5Qd3sVEodBhowxtD+TWv9ZD/H3A3cDeDv758bn5RCWauNmABFkI9ycNlCDI/NZsPLS9ovwnw+3c34d1TTFpyKTXlz4MCBWq11zHDONdggT9RalymlYoFPgO9qrdec6fisrCz96ZfbueB3q3n4+qlcl5s0nNqEcLi8vDwWLVpkdhlCwMa/wQc/gZ8chsBIlFJbh3v/cVBNE611mf25GngLYy2Ns/LzNk7d2WMdTl1CCDG69XQYz95+53yqAYPcPjMvpPdr4CJg90A/5+dtAaCj2zbAkUII4YG6e4Pc/5xPNZhRK3HAW0qp3uNf0lp/ONAP+flIi1wIIc6opwO8fMDLcs6nGjDItdZFwNShnriva0Va5Kbq7u6mtLSUjo4Os0txCWFhYezbt8/sMkzn7+9PUlISPj7OXFxSnFVPp0Na4zC0ceRDopTC19uLDmmRm6q0tJSQkBBSU1Ox/1Xl0VpaWggJCTG7DFNpramrq6O0tJS0tDSzy/FcPR0O6R8HJ48j9/P2kha5yTo6OoiKipIQF32UUkRFRclfaWbr6XBYi9ypQe7vY6GzR4LcbBLi4lTymXABPR3g4wZB7uftJTc7hRCiPw7sI5euFSGEMIP79JFbpEUuhBD96XabPnIv6SMXDnXnnXcSGxtLTk6OU84fHDzwPsjt7e0sXLgQq9VopKxatYpbbrllwOP609XVxYIFC+jp6en3tRjF3OVmp5+3hY5uaZELx7n99tv58MMB56M51bPPPss111yDxWJM5MjPz2f69OkDHtcfX19flixZwquvvtrvazGKuU0fubTIBbB7927mzp3b93rbtm0sWbJkWOdasGABkZGRjiqtX8XFxUycOJG77rqL7OxsLrroItrb2/vef/HFF7nyyiv7Xufn51NZWcmCBQtISUnh008/7fe4xYsX88knnwDw85//nO9+19iq9KqrruLFF1/sO+7U12KUcmAfudMmBIHc7HQ1D7yzh73lzQ4956SEUO6/PPvsx0yaRFFREVarFYvFwg9/+EMeeeSRk4654IILaGlpOe1nH374YZYuXerQmgfj4MGDvPzyyzz11FOsWLGCN954g5tvvpmuri6KiopITU3tOzY/P5+bbrqJNWvW8NZbb/Hiiy+yYMGC04574IEH+J//+R+qq6vZvn07b7/9NgA5OTls3ry577hTX4tRyoFdK04Ncn8fi8zsFHh5eZGdnc2ePXs4ePAgY8eOZcaMGScds3btWqddf+nSpVRWVgInr0f+61//+qQW84nS0tKYNm0aALm5uRQXFwNQW1tLeHh433Hd3d3U1dVx33339b0ODw8/7Tgw/prQWvPII4+Ql5fX1+VisVjw9fXtm3V66msxSjlwHLm0yD3IQC1nZ5o9ezbr16/nL3/5S7993M5skfd2dcDgp+j7+R3/k9disfR1rQQEBJw0I3Lfvn1MnTq17/8cdu7cSU5OzmnHAezatYuKigqioqJOq6GzsxN/f/8zvhajkNv0kcvwQ2E3e/Zsfv7zn3P11VeTmJh42vtr165lx44dpz0GG+JLliyhrKzM0WWfJiIiAqvV2hfS+fn5TJ16fE25nTt3MmXKlNOOq6io4KabbmLlypUEBwef9H9mdXV1REdH9y1gdeprMUq5zzhyudkpDBMmTMDPz4+f/vSn53SeG2+8kTlz5lBQUEBSUhLPPPMMNpuNwsJCp98E7XXRRRexbt06wAjyKVOm9L23e/fuvqGRvccdO3aMa665ht///vdMnDiRX/ziFzzwwAN9P7N69Wouu+yyM74Wo5DNCtYuh7XI0Vo7/JGZmam11vqhD/frtP98V9tsNi3MsXfvXrNL0Fprfc899+jnnnvOKefetWuXvvfeewd1bHNz8zlfb+vWrfrmm2922HFXX321LigoOONrZ3GVz4ZH6mzT+v5Qrdf+oe9bwBY9zMx1eovcpqHbOvC+oGJ0OnToEBMmTKC9vZ3bbrvNKdfIyck5bRSMM82YMYPFixefdaLPYI/r6uriqquuIjMzs9/XYpTqcdzuQODkm53hgUYfX8OxLuJC5caNJ0pPT2f//v1ml+Fwd955p0OO8/X15dZbbz3jazFKHaszngMiHHI6p7bIEyMCAChtaB/gSCGE8CCNR43n8GSHnM6pQZ4QbgR5WaMEuRBC9GkqMZ7DkhxyOue2yO1BXi5BLoQQxzWVgrJASIJDTufUIA/x9yHU35sy6VoRQojjGksgNAEsjrlN6dQgB0iMCJSuFSGEOFFTCYQ5pn8cRiLIw/2la0UIIU7UVOKw/nEYkSAPkK4VIYToZbNCc7nDRqzAiHStBNDS2UNTe7ezLyWEEK6vpQJsPe7VtZIgI1eEA3344YdkZWWRkZHBgw8+6PDzD2erN0dtP3emLeP6u2Z/TtwmTraMc2FNpcazOwV57xBE6V4R58pqtXLPPffwwQcfsHfvXl5++WX27t074nWcuoWbo7afO9OWcf1dsz8nbhMnW8a5sEb7GHJ361oBmRTkyRy11dumTZvIyMhg3Lhx+Pr6csMNN7By5UpHlgoMfas3R20/d6Yt4/q75mC2jZMt41xUk31WpwNvdjp1rRWA6CA/fL29JMhdwQf/CZW7HHvOMZPhkrN3cThqq7eysjKSk4+3YpKSkti4caMDfonTDWWrN0fpb8u4pUuX9nvNwWwbJ1vGuajGEgiMAt8gh53S6UHu5aVICPOXIPdgo3mrt6Fe/0QnXv9MW8ad6ZqD3TZOtoxzQU2lDm2NwwgEORjdK9JH7gIGaDk7kyO2ektMTKSkpKTvvdLS0n53GzqVM7d6G4wTr38mZ9oy7kzXHOy2cbJlnAtqKoGoDIee0ul95GAfSy4tco/miK3eZs2axcGDBzl8+DBdXV288sorXHHFFYB5W705ypm2jOvvmoPdNk62jHNBWhtdK+EpDj3tiAR5QngANS2dsn+nB3PEVm/e3t489thjLFu2jIkTJ7JixQqys7NN3eoN+t9+bqjOtmXcidccyrZxsmWcC2pvgO42h3etOHWrt16vbT6qx/70XX24ptWRmyWJQXCV7bw8cas3RxrOtnEDbRnnKp8Nj1K+w9jibc/K097CVbd66yVDED2XJ2/1NtLXPHGbONkyzkU5YQw5DOFmp1LKAmwByrTWy4dykaTwQEAmBXkiT9/qbSSveeI2cbJlnIvq21DCvD7y7wP7hnORMWH+KCUtciGEh2sqBe8ACHTs/ZxBBblSKgm4DHh6OBfx9fYiNsRPglwI4dkajxrdKko59LSD7Vr5I/AT4IwDcJVSdwN3A8TExJCXl3fS+yFe3ew5XE5eXsPwKhXDEhYWRnNzM8rBHxx3ZbVa+x2v7mm01nR0dJz271Q414zSvfR4h7DTwf/dBwxypdRyoFprvVUptehMx2mtnwSeBMjKytKLFp186Ovl29hV1sSp3xfO1TvmOioqSsKcwU8IGs201tTV1REeHn7GRbqEk2xugrR5Ds/BwbTI5wFXKKUuBfyBUKXUP7XWNw/lQokRAXy8pwqbTePlJYEyUpKSkigtLaWmpsbsUlxCR0eHzHQE/P39SUpy8FhmcXbd7dBW4/ARKzCIINda/wz4GYC9Rf6joYY4QFJ4AF1WGzWtncSFyj+kkeLj40NaWprZZbiMvLw8aYUKc/StQ+7YESswQjM7QcaSCyE8XJNzxpDDEINca5031DHkvRJkgwkhhCfrnQzk6On5jGCLPCUyEKWgsLp1pC4phBCuo/YAePtDSILDTz1iQR7o6016TDC7y5pG6pJCCOE6yndAXA5YHL96+IgFOcDkxDB2SZALITyNzQYV+ZAwzSmnH9Egz0kMo7qlk+pmx67lLIQQLq2+CLpaIH4UBPnkxDAAaZULITxLxQ7jeTS0yLMTQlFKglwI4WHKt4PFD2ImOOX0IxrkQX7ejIsOkhueQgjPUpEPY3LA4pxt90Y0yEFueAohPEzvjU4n9Y+DCUGekxhGVXMn1S1yw1MI4QEaDkNns9P6x8GkFjkg3StCCM9Qvt14Hk0t8uzEMOOGZ2nzSF9aCCFGXsUO40Zn7ESnXWLEgzzYz5u06CDpJxdCeIbyHRCX7bQbnWBCkIPRvSJdK0KIUc9mg4qdTu0fBxODvLK5g5qWTjMuL4QQI6PhMHQ2ObV/HEwK8hy54SmE8AS9NzpHY4s8OyEUkBmeQohRrmIHWHwhxnk3OsGkIA/x92Gc3PAUQox2vTc6vX2dehlTghxgekoEm4vrsdq0WSUIIYTzdHdA6RZImuX0S5kW5Asyo2k81i2tciHE6HT0S+hph/QlTr+UaUE+PyMapWDNgRqzShBCCOc5tAq8fCB1vtMvZVqQRwX7kZMQJkEuhBidCj+DlNngF+z0S5kW5GB0r2wvaaS5o9vMMoQQwrGaK6B6D2Q4v1sFzA7y8TFYbZovCmvNLEMIIRzr0GfG8wj0j4PJQT5jbATBft58fkCCXAgxihxaBUGxEJczIpczNch9LF7MTY9izYEatJZhiEKIUcBmhUOrjW4Vr5GJWFODHGBBZgxlje0U1baZXYoQQpy7ih3QXj9i3SrgAkG+MDMGkGGIQohRovAzQEH64hG7pOlBnhwZSFp0kAS5EGJ0OLQK4qdCUPSIXdL0IAdYMD6aDUX1dPZYzS5FCCGGr6MJSjaN2LDDXi4R5IsmxNLebWWtjF4RQrizAx+BtsL4i0b0si4R5PMzookM8uXfO8rMLkUIIYZv52sQlgxJ543oZV0iyH0sXlw2OZ5P91XR2tljdjlCCDF0rTXGRKDJ143YsMNeLhHkAFdNT6Cj28ZHuyvNLkUIIYZuz1tGt8rkFSN+aZcJ8hkpESRHBkj3ihDCPe16zZjJGTdpxC89YJArpfyVUpuUUvlKqT1KqQecUYhSiiunJrK+sJbqlg5nXEIIIZyjvghKN8Pk6025/GBa5J3AV7TWU4FpwMVKqdnOKOaq6QnYNLybX+GM0wshhHPseh1QRv+4CQYMcm1otb/0sT+csjBKRmwI2QmhrJTuFSGEu9DaGK0ydh6EJZlSgvdgDlJKWYCtQAbwuNZ6Yz/H3A3cDRATE0NeXt6wCsoJ6ebVgi5eee8zxgS5TBe+GCVaW1uH/dkUoj/BLYXMrDtIQdRFVJj02RpUkGutrcA0pVQ48JZSKkdrvfuUY54EngTIysrSixYtGlZBE6Z38NqDqyj1TuSGRVnDOocQZ5KXl8dwP5tC9Ov998HLh6yrf0xWQIQpJQypyau1bgRWAxc7pxwYE+bPV7JieXnTUZmyL4RwbZ0tsOMlmHQlmBTiMLhRKzH2ljhKqQDgQmC/M4u6fV4qdW1dvCM3PYUQrmzHS9DVArO/bWoZg2mRxwOrlVI7gc3AJ1rrd51Z1PyMaMbHBvP39YdlwwkhhGuy2WDj3yBxJiTNNLWUwYxa2am1nq61nqK1ztFa/9LZRSmluH1eKnvKm9lc3ODsywkhxNAVfgL1h0xvjYMLzew81TXTkwgL8OHv6w+bXYoQQpxuw18hJN7oHzeZywZ5gK+FG85L5qM9lZQ2HDO7HCGEOK56PxSthllfB4uP2dW4bpAD3DonFaUUL3x5xOxShBDiuI1PgMUPcu8wuxLAxYM8MTyAi7PH8NKmozS1d5tdjhBCQEsV5L8CU64f0e3czsalgxzgPxan09LRwzNri8wuRQghYN0fwNoF839odiV9XD7IsxPCuCRnDM+uL6ahrcvscoQQnqy5HLY8C1NvhKh0s6vp4/JBDnDvhZm0dfXwpLTKhRBmWvt7Y/OIhT82u5KTuEWQZ8aFcPmUBJ5bX0xta6fZ5QghPFHjUdj6D5h+C0Skml3NSdwiyAG+v3Q8nT1Wnsg7ZHYpQghPtOYhUAoW/MjsSk7jNkGeHhPM1dOTeGHDEaqaZQchIcQIqi+C7S9C7u2mrTl+Nm4T5AA/WDoereG3Hzp1zS4hhDjZx78Abz+XGqlyIrcK8uTIQL5xQRpvbitj65F6s8sRQniCwlWw/12jSyU03uxq+uVWQQ5wz+IMxoT6c//be7DaZGVEIYQT9XTBBz+FyHEw5ztmV3NGbhfkQX7e/NdlE9ld1syrm0vMLkcIMZptfALqDsLFDxpdKy7K7YIc4PIp8ZyXFslDH+2n8ZhMEhJCOEFLJXz+Wxi/DDKXmV3NWbllkCul+N/Ls2lq7+ahjwrMLkcIMRp9/AtjKv7F/2d2JQNyyyAHmJQQyu1z03hx41G+OFRrdjlCiNGk4APY9RrMv9elpuKfidsGOcCPl2WRGhXIT17fSVtnj9nlCCFGg2P18M73IS4HLnC9yT/9cesgD/C18ND1UylrbOfBD2RsuRDCAT78GbTVwlV/AW9fs6sZFLcOcoBZqZHcMTeNFzYckS4WIcS5KfgAdr4CF9wH8VPNrmbQ3D7I4eQuluYO2YBCCDEMbXXHu1QWuNbqhgMZFUEe4Gvh9yumUtHUwc/e3IXWMlFICDEENhu89U1ob4Cr/uo2XSq9RkWQA+SOjeS+izJ5b2cFL206anY5Qgh38uWfofATWPYbiJ9idjVDNmqCHOBbC9JZkBnDA+/sZW95s9nlCCHcQckm+PQBmHQlzPqG2dUMy6gKci8vxSMrphIe4MN3XtpGqwxJFEKczbF6eP1OY2naK/5srDfuhkZVkANEB/vx6I3TKa5r48f/yscmC2sJIfpj7YE3vmFMxb/+7+AfZnZFwzbqghxg9rgo/uvSiXywu5JHPztodjlCCFf06f1waBVc9ntIzDW7mnPibXYBzvL1+Wnsr2zhj58eJCsuhEsmu+Y6wkIIE2x/Eb58DM77JuTeZnY152xUtsjBWFjr11fnMCMlnB++ls+e8iazSxJCuIKSTfDuD2DcImOUyigwaoMcwM/bwhO35BIe6MPXn9tCeWO72SUJIcxUdwhe+RqEJsJ1fwfL6OiUGNVBDhAb4s8zt82irbOH257dJOuXC+GpWqvhn9eAzQo3/QsCI82uyGFGfZCDseTt327N5UjdMb7xjy10dFvNLkkIMZI6W+DF64wwv+lfED3e7IocyiOCHGBuejSPfHUqW4828N2Xt9NjtZldkhBiJPR0wqs3Q+VuuP4fkDTT7IoczmOCHGD5lATuXz6JT/ZW8aN/5cvmzUKMdtZu+NcdUJRnTPjJvMjsipxidPT0D8Ht89Jo67Ly0EcFeFu8+N21U/Dycs/ZXEKIs7D2wBtfh4L34NKHYfpNZlfkNAMGuVIqGXgeiAM08KTW+k/OLsyZ7lmcQY9V84dPD+DtpfjN1ZMlzIUYTWxWeOtu2LvSGGJ43l1mV+RUg2mR9wD3aa23KaVCgK1KqU+01nudXJtTfW9JBt1WG4+tLkQp+NVVk7FImAvh/qzd8O9vw+43YOkDMOcesytyugGDXGtdAVTYv25RSu0DEgG3DnKlFPddlIlG8/jqQxzrsvLw9VPxsXjUbQMhRpfuDmMRrIL3YMn9MP8HZlc0IobUR66USgWmAxv7ee9u4G6AmJgY8vLyzr26ETDLD64b78PrO8o5Wl7Ft6f64WuRlvlo1dra6jafTTE0XtYOcnb/hsiGfA5m3E2ZdQZ4yP/WarC76SilgoHPgV9rrd8827FZWVm6oKDAAeWNnH98Ucz9b+9hXkYUT9ycS4i/j9klCSfIy8tj0aJFZpchHO1YPbx8A5Ruhisec8sbm0qprVrrYY2NHFQ/glLKB3gDeHGgEHdXt81N5ffXT2VDUT0r/raBquYOs0sSQgxGQzE8cxGUbzem3bthiJ+rAYNcKaWAZ4B9WutHnF+Sea7NTeLZ22dxtK6Nqx9fz4GqFrNLEkKcTfl2ePpCaKuGW/4N2VeZXZEpBtMinwfcAnxFKbXD/rjUyXWZZmFmDK9+cw7dNs21f/2C9YW1ZpckhOhPwYfw98vA2w/u/BhS55ldkWkGDHKt9TqttdJaT9FaT7M/3h+J4sySkxjGm9+eS3yYP7c+u4nnvyw2uyQhRC+tYd0fjT7x6Az4+icQO8HsqkwlY+3OIDkykDe+PZdFmTH8z8o9/Pdbu+iW9VmEMFd3B7z1LWN3n+yr4I4PIVQ2jZEgP4sQfx+evHUm31qYzosbj3LTUxupbpGboEKYorEEnrsUdr4Ci39u3Nj0DTS7KpcgQT4Ai5fiPy+ZwJ9umMbOskYue3Qdmw7Xm12WEJ6lcBX8bQHUHICv/hMW/thtd7x3BgnyQbpyWiL/vmcewX7e3PjUBp5eW8Rgx+ALIYbJZoPPH4J/XgshY+DuPJh4udlVuRwJ8iGYMCaUld+Zx9KJsfzqvX3c9fwW6ttkxyEhnKKlCv55Naz+FUy+Dr7xqXFzU5xGgnyIQv19eOLmXO6/fBJrDtRy6Z/WsqGozuyyhBhdCj+FJ+bB0Y1w+aNwzVPgG2R2VS5LgnwYlFLcMS+NN/9jLgG+Fr721AYe+mg/XT0yqkWIc9LdAR/+l9GVEhRjdKXk3ib94QOQID8HOYlhvPvd+Vw7I4nHVx/i6r/IbFAhhq0iH55cCBseh1l3wV2fefz48MGSID9HQX7ePHT9VP52Sy4VTR0s//M6nl5bJNvICTFY1m5Y8xA8tQTaG+HmN+Cyh8EnwOzK3IbHbfXmLMuyxzAjJYKfvbmTX723j/d3VfC766aSERtsdmlCuK7KXbDyHqM1nn01XPYIBEaaXZXbkRa5A8WE+PHUrTP541enUVTbxqWPruUveYUyI1SIU/V0wur/gycXQXM5rHgern9OQnyYpEXuYEoprpqeyLyMaP5n5W5+92EBb+8o5/+umcz0lAizyxPCfMXr4d0fQO0BmPJVuPhBCfBzJC1yJ4kJ8eOvN+fyt1tyaTzWzTV//YJf/Hs3zR3dZpcmhDmO1RvdKM9dCj0dcNPrcM2TEuIOIC1yJ1uWPYZ5GdE8/FEB//iymA92V/KzSyZwzYxElAypEp7AZoVt/4BVv4SOZpj3A1j4U1knxYGkRT4Cgv28+d8rsnn7nvkkRQRw37/yuf6JL9lT3mR2aUI4V+kWeHoJvHsvxE6Cb62FCx+QEHcwCfIRNDnJWOf8d9dOoai2jeV/XsfP3txJTUun2aUJ4VjN5fDmN40Qb66Aa56G29+DuGyzKxuVpGtlhHl5KVbMSmZZ9hge/ewg//iimHfyK/jOVzK4fW4q/j4Ws0sUYvi62uCLx2D9H8HWA/Pvhfk/BP9Qsysb1STITRIW6MMvlk/ipvNT+M37+3jwg/288OURfrQskyunJuLlJf3nwo1Ye2DHi7D6N9BaCZOuhAt/CRGpZlfmEaRrxWTjYoJ5+rZZvPSN84kI8uHeV/NZ/ud1fH6gRpbJFa5Pa9j/vrHA1Tvfg/BkY9eeFc9LiI8gCXIXMTcjmrfvmc+fbphGc0c3tz27iRue3MDWI7KJhXBRh9fAMxfCKzeCtcsI769/AmPnmF2Zx5GuFRfi5aW4cloiF+eM4ZVNJfz5s0Ku/euXLM6K4YcXZjE5KczsEoWAkk2w+tdQlAchCXD5n2DaTWDxMbsyjyVB7oL8vC66VWsAABYnSURBVC3cNjeV62cm8ff1xTy5pojLH1vH0olx/GDpeHISJdCFCUq3GH3gh1ZBYBRc9GuY9Q3w8Te7Mo8nQe7CAn29uWdxBrfMGctz64t5em0Ry/9cxdKJsXznK+OZlhxudonCExz5AtY8fDzAlz4A590lGz24EAlyNxDq78P3lozn9nmpPLe+mGfXH+aqx9dzwfho7lmcwflpkTJLVDiW1lC02gjwI+uNTR6W/q+xTrifrOjpaiTI3UhvoN85P40XNxzhqbVF3PDkBqanhPOthelcODFOhi2Kc2Ozwt6VsO4PULnT6AO/+Lcw41aZjenCJMjdULCfN99cmM5tc1P515YSnlxbxDdf2Ep6TBB3XTCOq6YnysQiMTRdx4xx4F8+Dg2HISoDrvizsTqht5/Z1YkBKGeMVc7KytIFBQUOP6/oX4/Vxvu7K3ki7xB7K5qJDvbl1jmp3Dx7LJFBvmaX51Ly8vJYtGiR2WW4jpYq2PwUbH4G2ushMddY1GrCZeAljYGRpJTaqrWeOZyflRb5KOBt8eKKqQlcPiWeLw/V8dTaIh755ACPry7kqmmJ3DE/lQljZIq0OEH5dtjwBOx+w5hKP+EymPMdSJktGx27IQnyUUQpxdyMaOZmRHOwqoW/f1HMm9tKeXVLCXPTo7h1TipLJ8bibZF5YB7J2g373oZNT8HRL8E3GGbeAed9E6IzzK5OnAPpWhnlGo918fKmEl74spjypg4Swvy5afZYVsxMJibE8/o+PbJrpbkctj0PW/5urIMSkWqMPplxC/jLnARXcS5dKxLkHqLHamPV/mqe/7KY9YV1+FgUy7LHcPPssR41fNFjgtxmg8N5Rt93wQegrZBxIZx3N2QsBS/5q8zVSB+5GJC3xYtl2WNYlj2GwupWXtp4lNe3lvDuzgrGxQRx46wUrpmRSFSw57XSR5WWKtjxT9j2gjH6JCAS5twDubdDVLrZ1QknkRa5B2vvsvLuznJe3nSUbUcb8bEoLsoew1dnJjMvIxrLKByTPipb5NYeY9bltufhwIfGzcux8yH3NmM5WRk+6BakRS6GJcDXwvUzk7l+ZjIFlS28svkob20v472dFSSGB3BtbhLXzUgiJUomgrik2kKj9b3jZaPvOzAaZn8bZtwG0ePNrk6MoAFb5EqpZ4HlQLXWOmcwJ5UWufvq7LHyyd4qXttSytqDNWgN56VGcm1uIpdOjifE371XuHP7Fnl7I+x5E3a8BKWbQVlg/EUw/WbIXCYrELoxp97sVEotAFqB5yXIPUt5YztvbS/jja2lFNW24e/jxYWTxnD19AQuGB+DjxsOY3TLIO/pgsJPIP8Vo+vE2gUxE2Ha12DKCggZY3aFwgGc2rWitV6jlEodzsmFe0sID+CexRn8x6J0th1t5K3tpby7s4J38suJCvJl+ZR4rpiWyIyUcI8Z9TJibDZjrPeuf8Hef0N7g9F1MvNOY9p8wnSZuCP6DOpmpz3I3z1bi1wpdTdwN0BMTEzua6+95qAShSvpsWl21lj5oryHHTVWemwQE6A4P96b8+O9SQpWLh3qra2tBAe76Op9WhPcWkRs9Vpiq9fi31mL1cuP2ujzqYpbSEPENLSX3NYarRYvXuzcceSDCfITSdeKZ2jp6OajPVWs3FHGF4fqsNo0GbHBLJ8Sz/Ip8WTEhphd4mlcrmtFa6jaY7S6d78J9YfAyxvSvwKTV0DWJbJsrIeQUSvCFCH+PlyXm8R1uUnUtXbywe5K3skv50+rDvLHTw+SGRfMpZPjuXRyPONjg126pT6itIaq3cZysXv+DXUHQXlB6gUw7/sw8XIIjDS7SuFGJMiFQ0QF+3Hz7LHcPHss1c0dfLC7kvd2VfSF+riYIC7NiefinDFkJ4R6XqhrDWXbjLVO9q40JusoL0idD3P+AyZcDsExZlcp3NRgRq28DCwCooEq4H6t9TNn+xnpWhG9qps7+GhPJR/srmRDUR02DYnhAfZZpnHMTI0c0YlHI9q1Yu02tknb/y7sexdayo1uk7QFMPEKmLBcwlv0kbVWhFuoa+1k1b5qPtpTydrCWrp6bEQG+fKVCbFcOCmOC8ZHE+jr3D8SnR7knS1QuAoK3ocDH0FHI3gHQMYSI7gzl0m3ieiX9JELtxAV7MeKWcmsmJVMa2cPnxfU8MneSj7eU8nrW0vx9fZiXnoUSyfFsWRCHGPC3GR39sYSY3x3wQdQvNYY5x0QCVmXwoRLIX2JbJMmnEqCXJgi2M+by6bEc9mUeLqtNjYdrufTfVV8uq+K1W/V8N/sJjshlCUTYlk8IZapSeGusx+pzWrMqjzwkfGo3mN8PzLdWF0w6xJIng0W+eclRoZ0rQiXorXmYHUrq/ZV89n+KrYeacCmISrIl4WZMSyaEMuC8dGEBw5vC7thd6201RpdJgc/Nhaoam8wpsePnWt0l2ReLOubiHMiXSti1FBKkRkXQmZcCN9elE5DWxdrDtawen81qwuqeXN7GV4KpiaHszAzhoWZMUxJCnf8DVNrD5RtgcJPjUf5DkBDUAxkXgLjlxpdJgHhjr2uEMMgLXLhNqw2TX5pI58X1PD5gRrySxvRGsIDfZiXEc2C8dFcMD6GhPCAM57jrC3yhiNw6DOjxV20BjqbjCGCSecZmzFkLIH4abIpg3AKaZELj2DxUsxIiWBGSgT3XphJfVsX6wprWXOghrUHa3hvZwUA6TFBXDA+hvkZ0Zw/LvLMKza2N0LxOihabQR4fZHx/dAkyL7SmF05bhEERIzI7yfEcEmLXIwKWmsOVLWy9mANaw/WsvFwHR3dNixeimnJ4czLiGZeajCWnf9iZngTFH0O5dtA28AnCNIugHGLIX0xRGfKglRixMk4ciFO0dFtZXtxNYfz16GLPietdRu56gB+qhsrXlSHToZxC4mZugzv5PPAe3g3T4VwFOlaEQKMmZTl26F4Hf7F65hzdANzutuMt8bkUBp+Myur41hlm0Z+tQ2qIXh7G7NSdzB7XBSzx0WRnRCKtxuusy48mwS5cF89ncb6JUfWQfF6KNkE9uDu23ghbQGkzscSGMlYYEpeHt9btIja1k42FNXx5aE6NhTVsbqgBjDGt+eOjeD8cZGcnxbF5MQwfL0l2IVrkyAX7qOzxQjro18aa5iUbgFrp/FebLYR3KnzYey8AdcwiQ72Y/mUBJZPSQCguqWDjUX1bCiqY9Phen73odE16O/jxYyUCGalRnJeWiTTU8KdvoyAEEMln0jhuporoGQDHN0IR7+Ayl3GzUllgfgpcN5dkDLHmJRzjuuXxIb4c/nUBC6fagR7bWsnmw/Xs/FwPZuL63n0s4NobYycyUkIZVZqJDNTI8gdG0lMiOxSL8wlQS5cg80K1XuhZKO91b0BGo8Y73kHQNJMuOBHkDIbks8DP+duWhEd7Mclk+O5ZHI8AM0d3Ww90sCW4no2Fzfw/IYjPL3uMACpUYHkjo0kd2wEuWMjGB8b7DrLCQiPIEEuzNHeAKVbjeAu3WR83dVivBccB8nnw/nfNNYsiZ9i+u7wof4+LM6KZXFWLACdPVZ2lzWz9Ug9W4obyCuo5o1tpQCE+HszPSWCGSnh5I6NYGpyOKFnGssuhANIkAvns/YYre3SzVC21Whx1x003lNeEJdt7Abf29oOH+vy47j9vC19LfC7Fxjj2I/UHTNa7Uca2H60gT+tMrpjlIKMmGCmp4QzPSWC6SnhjI8NGdF12MXoJkEuHEtraCo1Artsi9HSrtgB3ceM9wOjjCnv026EpFmQMGNU7EmplCI1OojU6CCuzU0CjD1N80ua2H60gW1HG/h4bxWvbTFa7UG+FiYnhTEtOYJpyeFMSw53n2V7hcuRIBfnpq3OGLtdvs0YCli2FdqqjfcsvjBmMsy4FRJnGv3cEaku39p2lBB/H+aPj2b++GjAaLUX1x1j+9EGdpQ0sv1oI8+sK6LbakzKiwv1Y0qSEepTksKYkhhOWKB0yYiBSZCLwetogop8I7jLthnPvTckUcbU9oylkDgDEnMhLkdmTJ5AKUVadBBp0UFcM8NotXd0W9lX0cyOkkZ2ljaRX9LIJ3ur+n5mbFQgU5LCmZoURk6i8Qj2k3+24mTyiRD9a2+Eyp3G8q0VO4zn+kPH3w9PMbpFZt5pBHf8NPAPNa9eN+XvY7H3mx9fmKupvZtdpU3sLGtkZ0kTW4vreSe/HDD+mEmLDmJyYhiT7cGenRB65oXBhEeQIBfGpgkV+Sc/Gg4ffz8sGeKnGv3aCfbQDooyr95RLizg5C4ZMMa17yprYmdJE7vKmthYVM/KHeV976dFBxkt9oRQshOMcI8Ikr+GPIUEuSfR2ugKqdhpTK6p3Gl83XI8EAhPMYJ6xi0wZiokTIOg6DOfU4yI6GC/k4Y/AtS0dLK7rMl4lDex7UhDX8sdIDE8gInxoWQnGI9JCaEkhgegPOQehSeRIB+tujugZh9U7oaq3fbg3m1slgDGsL/oTGNKe/wUo8U9ZrKsve1GYkL8WGzf07RXQ1sXe8qb2VPe1Pe8an8VvYuchgX4MCk+lInxoUyMD2FifCjj44Lx87aY9FsIR5Agd3daQ0ulEdZVu48Hd+1B0FbjGJ8giJsEk68zwnrMZIidJDu7j0IRQb6ndcsc6+phX0UL+yqa2VvRzJ7yZl7adISObhsA3l6K9JjgvmCfGB/KhPgQYoL9pPXuJiTI3UlnK9Tsh6o9xgSbqj3Go73++DFhycZokQnLj4d2RJpsT+bBAn29+yYv9bLaNMV1bewtb2ZfhfHYUFTPv0/od48K8mVCfAhZcUawTxgTwvjYEAJ8pfXuaiTIXZG1G+oK7WG9F6r3QfUeaCg+foxPEMROhInLIW6yMTsybpJ0jYhBsdhb4ekxwX0LhYHRNbO/soX9lUa4F1S2nNR6VwrGRgaSNSaErDGhZMWFkDUmmNSoIFnH3UQS5GayWaH+sNGXXX3Co64QbN3GMcoCURmQMB2m3WR0icRNgvBUaWULh4sI8mVOehRz0o+PSrLaNEfrj1FQ2cz+yhYK7I9P9lZhs/e9+1q8GBcTRGZcCJlxwfbnEJIjA2UpghEgQT4SrD3GcL6a/fZHAVTvh9oDx9fTBmONkdiJkHWxsTFC3CTjhqS3LJMqzGPxOj6R6eKc+L7vd3RbKaxupaCyhQPVLRyobGHrkQbePmHkjL+PF+kxRrBnxAYzPjZYAt4JJMgdqbvDaE3XFkDNAftzgfE9a9fx48JSIHYCpC8yAjt2AkRnjYo1R4Tn8Pex9M02PVFrZw8Hq1o4WNXKgaoWDlS3srGojre2l/Ud4+vtxbjoIMbHhZARE0xGrPFIjQ6UETTDIEE+HO0NxqiQmgKjVV17wPi68Yix8QEACiLGGgE9/kKIsYd1jAS2GN2C/bxPm60KxiJiB6tbKbQ/DlS1sP3oyWPfLV6KlMhA0u3hnh4TZDzHBstSwGchQX4mNqsRzLWFRlDXHTTCu/YAtNUcP87ia/Rhx0+FydcbQR2dCdHjwSfAvPqFcDEh/j7MSIlgxikBf6yrh6Katr6AP1RjPH9+oLpvQTEwxs2nxwSRHhPMuJjgvq8TwgM8vpvGs4NcazhWb4R0XaER1HWFxqO+6OTukIBII5wzL7YHdSbEZBr92l7yp6AQwxXo691vF02P1UZJQ3tfuB+yP7+7s4Km9u6+43y9vUiLCmJcjNGPPy4mmLToINJjgggP9IxlCjwjyDtbjWCuK4S6Q8biT72h3dF4/DgvH4hMg6jxkLnMeO5tXZ/jnpBCiKHxtnj13WS9kLi+72utqW/roqi2jUPVrRTVtlFU09o3kqbHdrwVHxHo0xfsJz5So4JG1Xj40RPkXceMkSG9QV13yB7eh6C18uRjQxIgOgNyroHIdCOoozKM1rVl9PwnEWI0UkoRFexHVLAfs1JPbmB1W22U1B/jcG0bRTVtFNW2cbi2lTUHanh9a+lJx44J9Sc1OrAv2FPtIZ8SGYi/j3uFvHulVmerEdb1RcdDut7++sSFnwCCYoyQzlgCkeMgKt0I68hx4BtkTv1CCKfysXgxzt6HvmTiye+1dvZQXNvG4do247nOeP5oTxX1bce7UZWC+FB/xkYFkRodaDzbv06JDCTQ1/Vi07Uq0hqO1Rnh3HD45Of6ouM7z/QKijGmn49baIR2ZJoR2JHpsja2EOIkwX7998WDsQZ8cW0bxXVtFNce40idEfSnhjxAbIgfY6OMgB8bGUjKCV+HB/qYsj7NoIJcKXUx8CfAAjyttX5w2Fe0dkNTiTHdvKHYHtbFRmA3HIHO5pOPD0kwAjpzmdGajkwzniPSJKyFEA4RFuDD1ORwpiaHn/ZeU3s3R+uOUVzXxtH6Y32Bv+ZADdUtnScdG+LvbYR8ZBDJkYGMjTJa8SmRgcSH+TttGYMBg1wpZQEeBy4ESoHNSqm3tdZ7z/gz2mpsuttw2BjC1xvaDUeMjXl7V+UDY/he+FhjL8eUOUZAR6Taw3qsDOETQpgqLMCHyUlhTE46vSXf3mXlaL3Rgj9af8z+9TH2VjTz8d7Kk4ZPWrwUieEBpEQGkmwP9+TIgL6gPxeDaZGfBxRqrYsAlFKvAFcCZwzy4NbD8PRXjn8jKMYI56RZMGWF8XXvIyRB1gwRQrilAF+LfQGxkNPes9o0FU3tHK0/RskJIV/S0M5HeypP67I5F4MJ8kSg5ITXpcD5px6klLobuNv+slM90Lz7+LvNwKFTf0QIM0QDtWYXIUQ/sob7gw672am1fhJ4EkAptUVrPdNR5xbCUeSzKVyVUmrLcH92MH0aZUDyCa+T7N8TQgjhAgYT5JuB8UqpNKWUL3AD8LZzyxJCCDFYA3ataK17lFLfAT7CGH74rNZ6zwA/9qQjihPCCeSzKVzVsD+bSms98FFCCCFcloz7E0IINydBLoQQbm7YQa6UulgpVaCUKlRK/Wc/7/sppV61v79RKZV6LoUKMRSD+HzerpSqUUrtsD++YUadwvMopZ5VSlUrpXaf4X2llHrU/tndqZSaMdA5hxXkJ0zbvwSYBNyolJp0ymFfBxq01hnAH4DfDudaQgzVID+fAK9qrafZH0+PaJHCkz0HXHyW9y8BxtsfdwN/HeiEw22R903b11p3Ab3T9k90JfAP+9evA0uUGcuCCU80mM+nEKbQWq8B6s9yyJXA89qwAQhXSsWf7ZzDDfL+pu0nnukYrXUP0AREDfN6QgzFYD6fANfa/3R9XSmV3M/7QphhsJ/fPnKzU3iqd4BUrfUU4BOO//UohNsZbpAPZtp+3zFKKW8gDKgb5vWEGIoBP59a6zqtde9i0k8DuSNUmxADGfKyKMMN8sFM238buM3+9XXAZ1pmH4mRMeDn85Q+xyuAfSNYnxBn8zZwq330ymygSWtdcbYfGNbqh2eatq+U+iWwRWv9NvAM8IJSqhCjY/+G4VxLiKEa5Ofze0qpK4AejM/n7aYVLDyKUuplYBEQrZQqBe4HfAC01k8A7wOXAoXAMeCOAc8pjWQhhHBvcrNTCCHcnAS5EEK4OQlyIYRwcxLkQgjh5iTIhRDCzUmQCyGEm5MgF0IIN/f/Af/YIBo4wyssAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs_log = np.linspace(0.001, 0.999, 100)\n",
    "ys_0 = [-np.log(1 - x) for x in xs_log]\n",
    "ys_1 = [-np.log(x) for x in xs_log]\n",
    "plt.plot(xs_log, ys_1, label=r'$y = 1, -\\ln(h(x))$')\n",
    "plt.plot(xs_log, ys_0, label=r'$y = 0, -\\ln(1 - h(x))$')\n",
    "plt.ylim([0, 5])\n",
    "plt.xlim([0, 1])\n",
    "plt.xticks([0, 0.5, 1])\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.title('cost function')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def logloss(x, y, theta):\n",
    "    if y == 1:\n",
    "        return -np.log(hypothesis(x, theta))\n",
    "    elif y == 0:\n",
    "        return -np.log(1 - hypothesis(x, theta))\n",
    "    #  or (less obvious)\n",
    "    return -y * np.log(hypothesis(x, theta)) - (1 - y) * np.log(1 - hypothesis(x, theta))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that if $y$ was suppose to be 0 but is close to 1, the cost will be high.  \n",
    "Same logic if $y$ was suppose to be 1 but is close to 0.\n",
    "\n",
    "Then we generalize the cost function for all $x$:\n",
    "$$J(\\theta) = \\frac{1}{n} \\sum_{i = 1}^{n} j(h(x_i), y_i)$$\n",
    "$n$ is the number of elements in $X$ and $Y$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(xs, ys, theta):\n",
    "    return sum([logloss(x, y, theta) for x, y in zip(xs, ys)]) / len(xs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient descent\n",
    "\n",
    "Finally, use gradient descent to minimize the cost function:\n",
    "\n",
    "$$\\theta_i := \\theta_i - \\alpha \\frac{\\partial}{\\partial \\theta_i} J(\\theta)\\ \\text{ for } i = 1, ..., n$$\n",
    "\n",
    "Or with gradient notation:\n",
    "\n",
    "$$ \\theta := \\theta - \\alpha \\nabla J(\\theta)$$\n",
    "\n",
    "\n",
    "$\\alpha$ is the learning rate (how fast we want the step to be).  \n",
    "Since $\\theta_i$ is only used in the dot product, all of them will have similar partial derivative.\n",
    "\n",
    "let $z = x \\cdot \\theta_j$\n",
    "\n",
    "$\\newcommand{\\pt}{\\frac{\\partial}{\\partial \\theta_j}}$\n",
    "\n",
    "$$\n",
    "\\begin{align}\n",
    "      & \\pt J(\\theta) \\\\\n",
    "    = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n} j(h(x_i, \\theta), y_i) \\right] \\\\\n",
    "    = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n",
    "        (-y_i \\ln(h(x_i, \\theta)) - (1 - y_i) \\ln(1 - h(x_i, \\theta))) \\right] \\\\\n",
    "    = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n",
    "        (-y_i \\ln(\\sigma(z)) - (1 - y_i) \\ln(1 - \\sigma(z))) \\right] \\\\\n",
    "    = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n",
    "        (-y_i \\ln(\\frac{1}{1 + e^{-z}}) -\n",
    "        (1 - y_i) \\ln(1 - \\frac{1}{1 + e^{-z}})) \\right] \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\pt \\left[ \n",
    "        -y_i \\ln(\\frac{1}{1 + e^{-z}}) -\n",
    "        (1 - y_i) \\ln(1 - \\frac{1}{1 + e^{-z}}) \\right] \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i \\pt \\left[ \\ln(\\frac{1}{1 + e^{-z}}) \\right] -\n",
    "        (1 - y_i) \\pt \\left[ \\ln(1 - \\frac{1}{1 + e^{-z}}) \\right] \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i(1 + e^{-z}) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n",
    "        - (1 - y_i) \\frac{1}{1 - \\frac{1}{1 + e^{-z}}} \\pt \\left[ 1 - \\frac{1}{1 + e^{-z}} \\right]\n",
    "        \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i(1 + e^{-z}) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n",
    "        - (1 - y_i) \\frac{1 + e^{-z}}{e^{-z}} (-1) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n",
    "        \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i(1 + e^{-z}) \\frac{e^{-z}}{(1 + e^{-z})^2} \\pt z\n",
    "        - (1 - y_i) \\frac{1 + e^{-z}}{e^{-z}} (-1) \\frac{e^{-z}}{(1 + e^{-z})^2} \\pt z\n",
    "        \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i \\frac{e^{-z}}{1 + e^{-z}} x_{ij}\n",
    "        + (1 - y_i) \\frac{1}{1 + e^{-z}} x_{ij}\n",
    "        \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        \\frac{-y_i e^{-z}}{1 + e^{-z}}\n",
    "        + \\frac{(1 - y_i)}{1 + e^{-z}} \\right) x_{ij} \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        \\frac{-y_i e^{-z} + 1 - y_i}{1 + e^{-z}} \\right) x_{ij} \\\\\n",
    "\\end{align}\n",
    "$$\n",
    "\n",
    "$$\n",
    "\\boxed{\n",
    "    = \\frac{1}{n} \\sum_{i = 1}^{n} (h(x_i) - y_i) x_{ij}\n",
    "}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient(xs, ys, theta):\n",
    "    g = np.array([])\n",
    "    for j in range(len(theta)):\n",
    "        g = np.append(g, [sum([(hypothesis(x, theta) - y) * x[j]\n",
    "                          for x, y in zip(xs, ys)]) / len(xs)],\n",
    "                      axis=0)\n",
    "    return g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(xs, ys, theta, alpha=1, epoch=100):\n",
    "    for _ in range(epoch):\n",
    "        theta = theta - alpha * gradient(xs, ys, theta)     \n",
    "    return theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weights:   -4.121158543930249 33.10478298104216\n",
      "train cost: 0.0031212648079922676\n",
      "test cost:  0.0017122703295056325\n"
     ]
    }
   ],
   "source": [
    "theta_binary = np.random.randn(len(X_train[0]))\n",
    "theta_binary = gradient_descent(X_train, Y_train, theta_binary, 2, 2000)\n",
    "print('weights:  ', *theta_binary)\n",
    "print('train cost:', cost(X_train, Y_train, theta_binary))\n",
    "print('test cost: ', cost(X_test, Y_test, theta_binary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXhV5bn38e+9MxDCEGZkTgijDDKrOBwoiigUPVba0jqd1qGn2um0ttXXau3V1vacnvetVu3RYx3qgHNrVKqIFaeqCEiVWeZ5hjCEkGHf7x9rJwZIyA7ZYWXv/D7XlWvvvdaz17qzA7+sPOtZzzJ3R0REkl8k7AJERCQxFOgiIilCgS4ikiIU6CIiKUKBLiKSIhToIiIpQoEujZqZ9TSzA2aWdoLvP2BmvRNc0xwzuyaR24xzv+PMbGOcbX9uZo83dE3SuCjQJaHM7Goz+9TMisxsq5n90cza1OH9a83svIrX7r7e3Vu6e/mJ1BN77+oTea9IslGgS8KY2Q+B3wI3ATnAGUAv4HUzywyzNpGmQIEuCWFmrYE7gO+4+6vuXurua4EvA7nA5bF2Pzez58zsaTPbb2YLzOy02LrHgJ7AS7Gukh+bWa6ZuZmlx9rMMbNfmtk/Ym1eMrP2ZvaEme0zs4/MLLdKXW5mfcysa6x9xVeRmXmVdt8ws6VmtsfMXjOzXlXWnW9my8ys0MzuAew4n8PPzexZM3s89v19amb9zOxmM9tuZhvMbGKV9l3NrMDMdpvZSjO7tsq65mb2SKymJcDoo/bV1cyeN7MdZrbGzL5b5x+cpBQFuiTKWCALeKHqQnc/AMwEzq+y+GLgWaAd8CTwVzPLcPcrgPXAF2NdJf9Zw76+ClwBdAPygfeBh2PbWwrcfvQb3H1zbJst3b0l8BfgKQAzuxi4BbgU6Ai8A8yIresQ+55uBToAq4Czavksvgg8BrQFPgZeI/i/1g34BXB/lbZPARuBrsBlwK/N7AuxdbfHvr984ALgqoo3mVkEeAn4Z2y7E4Dvm9kFtdQmKUyBLonSAdjp7mXVrNsSW19hvrs/5+6lwP8l+EVwRh329bC7r3L3QuBvwCp3nx3b97PA8OO92cx+AgwAvhFb9C3gTndfGtvGr4FhsaP0i4DFVer9PbC1lvrecffXqtTTEfhN7P1PAblm1sbMehD8cviJuxe7+0LgQeDK2Ha+DPzK3Xe7+wbg7ir7GA10dPdfuHtJ7DzB/xL8spMmKj3sAiRl7AQ6mFl6NaHeJba+woaKJ+4ejY3c6FqHfW2r8vxQNa9b1vRGM7sQ+B5wursfii3uBdxlZv9dtSnBkW/Xo+p1M9vA8R1dz84qJ3Ur9tkytu3d7r6/Svt1wKjY8yP2HVtXoRfQ1cz2VlmWRvDXhTRROkKXRHkfOEzQbVHJzFoCFwJvVFnco8r6CNAd2Bxb1GDTf5pZf+BR4MuxI94KG4Dr3b1Nla/m7v4Pgr8uqtZrVV/X02agnZm1qrKsJ7Ap9vyIfcfWVa15zVE1t3L3ixJUmyQhBbokRKz74w7gD2Y2ycwyYicnnyHoI36sSvORZnZp7ETn9wl+EXwQW7cNSOi4cag8afsi8H/c/d2jVv8PcLOZDYq1zTGzabF1rwCDqtT7XeCURNQU+6XyD+BOM8sys6HAN4GK8ePPxOpqa2bdge9UeftcYL+Z/SR28jTNzAab2REnTqVpUaBLwsROYt4C/A7YB3xIcCQ5wd0PV2n6IvAVYA/Byc1LY/3LAHcCt5rZXjP7UQLLGwH0B/5f1dEusbr/QjDc8ikz2wcsIvirAnffCUwDfgPsAvoC7yWwrukEo4A2E5yovd3dZ8fW3UHQzbIGmEWVX4qxLpwpwLDY+p0E/e85CaxNkozpBhdyMpnZz4E+7n552LWIpBodoYuIpAgFuohIilCXi4hIitARuohIigjtwqIOHTp4bm5uWLsXEUlK8+fP3+nuHatbF1qg5+bmMm/evLB2LyKSlMxsXU3r1OUiIpIiFOgiIilCgS4ikiI026KIJERpaSkbN26kuLg47FJSQlZWFt27dycjIyPu9yjQRSQhNm7cSKtWrcjNzSWYlFJOlLuza9cuNm7cSF5eXtzvU5eLiCREcXEx7du3V5gngJnRvn37Ov+1o0AXkYRRmCfOiXyWCnQRkRShQBcRqUZubi47d+6sd5uTSYEuIpIiFOgikjLWrl3LgAEDuPrqq+nXrx9f//rXmT17NmeddRZ9+/Zl7ty57N69m0suuYShQ4dyxhln8MknnwCwa9cuJk6cyKBBg7jmmmuoOhPt448/zpgxYxg2bBjXX3895eXlNZUQKg1bFJHEe/VV2Lo1sds85RSYNKnWZitXruTZZ5/loYceYvTo0Tz55JO8++67FBQU8Otf/5oePXowfPhw/vrXv/L3v/+dK6+8koULF3LHHXdw9tlnc9ttt/HKK6/wpz/9CYClS5fy9NNP895775GRkcG3v/1tnnjiCa688srEfn8JoEAXkZSSl5fHkCFDABg0aBATJkzAzBgyZAhr165l3bp1PP/88wB84QtfYNeuXezbt4+3336bF154AYDJkyfTtm1bAN544w3mz5/P6NHB/bcPHTpEp06dQvjOaldroJvZQwQ3o93u7oOrWW/AXcBFQBFwtbsvSHShIpJE4jiSbijNmjWrfB6JRCpfRyIRysrK6nTlJQQX+Vx11VXceeedCa2zIcRzhP4IcA/w5xrWX0hwJ/S+wOnAH2OPDaKwEFasCJ736wc5dbjHeWEhbNkC2dlQVPT5Y5cuwXYKC+HBdwr4cOcs/nXoRABeXjqLKQMnclaHqSxaBIMHB223bIGPiwp4c/0sepZNZNNGeKfofppnwQUdr+fAAVgZnUWnnBxWHljIlv3baJ4FPdp15vpR1zO1/1QAZiwo4JF3ZrFnSw7FbReSnQ1TTrmeoc2msmcP9OwJqzMKePiT+ykrD9Y1bw4vbryfLfu3kZYGzUo7M5LradUKPo7cz+6SbXTK7kz54WxW7lnG0GZT+faAX/HPwwV8cmAWF/SZyNBmUzlwANyhVasjP8vCQvjD6wU8uzr4fq4dcT3vrf2Qt7cVcMnAqfzuol9VfqYzFhRUfkbTR0yt/IwrPtN4fh7xtBVJlHPOOYcnnniCn/3sZ8yZM4cOHTrQunVrzj33XJ588kluvfVW/va3v7Fnzx4AJkyYwMUXX8wPfvADOnXqxO7du9m/fz+9evUK+Ts5Vq2B7u5vm1nucZpcDPzZgzMIH5hZGzPr4u5bElRjpcJCuOeXe1n89i4ARo2Ca68NAqk2+/fD43+Gfftg6VLo3RtWr4YBA42cHLjkX+E3T8/mb3yPaKSYjz/+X8App5SP5v6J/OV30XLHeWRkwJAhsDFzNrPTv0cpxRD9X6AcTyvnIPDEmtfAI3h6aeX+3aAEKFwNP/x4Ns0n30dREdzy+o2URYpxwLfDTuDeVa/TfPYDRNZOImPgLHafey3laSW4wX+vfw2PQDRSTtSgPAJRgyXR1yg/AJ5WDgaL9wAOGGw+uIjZM1ZQ3nsmnlHE86sfps/CGRxaOBUz6NEDxo+HH/0oqPXqOwv4a8Y0SC+BQ/Dh66+BBdv977mLAPjdRb9ixoICrnppOqUU8ezKhykqmsHWt6ZW/rK88caag7qwEO65h7jaiiTSz3/+c77xjW8wdOhQsrOzefTRRwG4/fbbmT59OoMGDWLs2LH07NkTgFNPPZVf/vKXTJw4kWg0SkZGBvfee2+jDPS47ikaC/SXa+hyeRn4jbu/G3v9BvATdz/m7hVmdh1wHUDnzp1HPvXUU3UqtrgY/N1P6fzJfADS0qBFC0iP4++MsjI4dCh4fvgwZGRAaSlU/HWWmQn7DhdSagerfX+kvAXNPIeysmB/0YxCDnv1bePRIrMF0SgcKqt+G1baAivJwZsV4ul12I8Z0bQIHokQjUQoT0sjmh4hGsmkPAOi6emUp6fhdCBa1o2y9EysRRZprbNo0y2LsubZLN+/g8OZNY+tbZbWnMGdTmXN7g3sPry9cnlOeieyS3uQmQklJdCuHWRlVb+N4mLYvZu42kpyyMnJoU+fPmGXkVJWrlxJYWHhEcvGjx8/391HVdf+pJ4UdfcHgAcARo0a5ePGjavT+wsL4Q9/H8En2y8GYMwYuO46aN269vfu2wePPhpsY8lSyO8Nq1bDqQODI8NLL4W7nnqdmdxINFJMGplUHKGnk0X+8ntotf18MjM8OEJv9jqvp32HUiuG8kygHNKCoUxWngYegbRSLHaUDATPgWZpmfzhwrs5VAQ3zf4+ZVaMedDMAMozaP7m70lfP57Mfm+w7/T/wCLBtqw8jQgQsXIiDmlRiDhEytNIc0ijnPQopDmkl0NGGaQfguYbR5Pe/p9kUEJWaQbdN0wnsqk1hpOTA7m5cObYoL6d761gbvNn2N2inD1ZsCsrwvaWUba2ggMZ8MPTb+HGcd9my4ICbn7pW5RSRAbZ/HHCDNa8Na7yqHvy5PiP0I/XVpLD0qVLaRXPn8sSt6ysLIYPHx53+0QE+iagR5XX3WPLEi4nB77zf1qzYlqQ4P36Qes4Q6A1cOWtx+9Dv/PUb5L/Tsc4+9Cv4dyiTifch35erA/9QI8u1fehT6roQx/I6oyedepD3x7rQy+q0od+xZXV9KHvd6zkMK3Ti+jTtYisyAHYv58vjj2H9Hd6sXjNK4wqKuWs5sPYsGkj6w4sp3ePIUwZMBj+8Q+m9xoJk5/g5eWzP+9DHxJfv3hOTtDNoj50kcRJRJfLZOBGglEupwN3u/uY2rY5atQo1z1Fk8ShQ7B9ezCueMsWWL8+6C+BoM+rXz8YMADy8+Pr/5KUtHTpUgYOHBh2GSmlus/UzE68y8XMZgDjgA5mthG4HcgAcPf/AWYShPlKgmGL/1aP+qUxat4cevUKvirs3w/r1sHy5cFZ5o8/DsJ92DAYOTLoFBeRkyqeUS7Ta1nvwA0Jq0iSQ6tWQf/T4MFQXh4MGVqwAN5/H957DwYODIbONNILMERSkf4+lvpLS4O+fYOv/fth3jz44ANYtgyGDg2CvU2bsKsUSXmanEsSq1WrIMC/9z0YOxYWL4b77gtCPo7zNSInau/evdx3330n9N6LLrqIvXv3nvC+W7Zsedz19amtLhTo0jCys+H884OhLN27w8svw+OPB+NHRRrA8UKzrKzsuO+dOXMmbRrwr0gFuqSGNm3giiuCgebr18P998OmBhnVKk3cT3/6U1atWsWwYcO46aabmDNnDueccw5Tp07l1FNPBeCSSy5h5MiRDBo0iAceeKDyvRU3qli7di0DBw7k2muvZdCgQUycOJFDFVckVrFmzRrOPPNMhgwZwq233lq5/MCBA0yYMIERI0YwZMgQXnzxxWprq6ldvbl7KF8jR450aWJ27HD//e/df/lL9yVLwq5GEmzJCfxM9+51X7o0eKyvNWvW+KBBgypfv/nmm56dne2rV6+uXLZr1y53dy8qKvJBgwb5zp073d29V69evmPHDl+zZo2npaX5xx9/7O7u06ZN88cee+yYfX3xi1/0Rx991N3d77nnHm/RooW7u5eWlnphYaG7u+/YscPz8/M9Go0eU1tN7Y5W3WcKzPMaclVH6HLydOgA11wDnTvDM8/Ahx+GXZGEqOJq4cceCx6PusI9IcaMGUNeXl7l67vvvpvTTjuNM844gw0bNvDZZ58d8568vDyGDRsGwMiRI1m7du0xbd577z2mTw8GAF5xxRWVy92dW265haFDh3LeeeexadMmtm3bdsz7421XVxrlIidXixZw1VXwwgvwt78Fk7nU4dJmSR1btgRXavfqFVzSsGVL4q8YbtGiReXzOXPmMHv2bN5//32ys7MZN24cxcXFx7yn6vS7aWlp1Xa5AAQzhx/piSeeYMeOHcyfP5+MjAxyc3Or3Ue87epKR+hy8mVkwGWXBVeWvvRScHGSNDldugTnztetCx67dKnf9lq1asX+/ftrXF9YWEjbtm3Jzs5m2bJlfPDBBye8r7POOouKyQWfeOKJI/bRqVMnMjIyePPNN1m3bl21tdXUrr4U6BKOtDT4yleC/8XPPhv8r5YmpWI+nyuuSMz0ye3bt+ess85i8ODB3HTTTcesnzRpEmVlZQwcOJCf/vSnnHHGGSe8r7vuuot7772XIUOGsKnKSf6vf/3rzJs3jyFDhvDnP/+ZAQMGVFtbTe3qK665XBqC5nIRIPib+6GH4OBB+Pd/j2/qTGmUNJdL4tV1LhcdoUu4srNh+vRg+oAXXoBoNOyKRJKWAl3C1749XHQRrF0L774bdjUiSUuBLo3DaacF9/abMye4AElE6kyBLo2DGUyZEpwZe+GF4P6AIlInCnRpPJo1g0sugb17gyl4RaROFOjSuPTqFXS9vPtuEOwiEjcFujQ+558fdMHMmhV2JZJE6juj4e9//3uKiopqbTdnzhymTJly3DYLFy5k5syZJ1zLiVKgS+PTujWcey4sWRLcCUkkDicr0OOhQBep6swzoW3bYL4XjU2XOBw9RS3Af/3XfzF69GiGDh3K7bffDsDBgweZPHkyp512GoMHD+bpp5/m7rvvZvPmzYwfP57x48cfs+1XX32VAQMGMGLECF544YXK5XPnzuXMM89k+PDhjB07luXLl1NSUsJtt93G008/zbBhw3j66aerbdcgapqGsaG/NH2u1GrxYvfbb3f/9NOwK5E4nMj0uS8ue9FveOUGf3HZi/Xe/9FT1L722mt+7bXXejQa9fLycp88ebK/9dZb/txzz/k111xT2W5vbO7eiil0j3bo0CHv3r27r1ixwqPRqE+bNs0nT57s7u6FhYVeWlrq7u6vv/66X3rppe7u/vDDD/sNN9xQuY2a2tWmrtPnarZFabwGDoSOHeGdd2DQoKBfXVJGwfICpj8/naLSIh5e+DAzvjSDqf2nJmz7s2bNYtasWQyPzeZ54MABPvvsM8455xx++MMf8pOf/IQpU6ZwzjnnHHc7y5YtIy8vj759+wJw+eWXV94co7CwkKuuuorPPvsMM6O0huG28barL3W5SONlBmefDdu2wYoVYVcjCTZr1SyKSoM+66LSImatSuxJcHfn5ptvZuHChSxcuJCVK1fyzW9+k379+rFgwYLKuw394he/OOF9/OxnP2P8+PEsWrSIl156qcYpcONtV18KdGnchgwJ+tLffls3mU4xE/Mnkp2RDUB2RjYT8yfWa3tHT1F7wQUX8NBDD3HgwAEANm3axPbt29m8eTPZ2dlcfvnl3HTTTSxYsKDa91cYMGAAa9euZdWqVQDMmDGjcl1hYSHdunUD4JFHHqmxlpraJZoCXRq3SATOOiu4D6lGvKSUqf2nMuNLM7hh9A0J6W45eoraiRMn8rWvfa3y3p+XXXYZ+/fv59NPP2XMmDEMGzaMO+64o/KeoNdddx2TJk065qRoVlYWDzzwAJMnT2bEiBF06tSpct2Pf/xjbr75ZoYPH37EjajHjx/PkiVLKk+K1tQu0TR9rjR+ZWVw113BJF5XXx12NVIDTZ+beJo+V1JPejqMHRvMxrh5c9jViDRaCnRJDsOHB7eumz8/7EpEGi0FuiSHrKxg6OKnn8Lhw2FXIzUIqws3FZ3IZ6lAl+QxahSUlAShLo1OVlYWu3btUqgngLuza9cusrKy6vQ+XVgkyaNbN+jcOeh2GTlSFxo1Mt27d2fjxo3s2LEj7FJSQlZWFt27d6/TexTokjzMgqP0V14JTo7GxvVK45CRkUFeXl7YZTRp6nKR5DJkiE6OitRAgS7JJSsrCHWdHBU5hgJdks/w4cE9R5ctC7sSkUZFgS7Jp3t3aNMGFi0KuxKRRkWBLsnHDAYPhlWrIEF3mBFJBQp0SU6DBwd3Mlq6NOxKRBoNBbokp86dg8m61O0iUkmBLsmpottl7VqoZg5rkaZIgS7Ja/Dg4KYXS5aEXYlIo6BAl+TVsWPQ9aJuFxFAgS7JbvBg2LAB9u4NuxKR0CnQJbkNGhQ8Ll8ebh0ijYACXZJbu3bQoYMCXQQFuqSC/v1h3TooLg67EpFQKdAl+fXrB+XlwZWjIk2YAl2SX48ekJ2tbhdp8hTokvwiEejbFz77LJgOQKSJUqBLaujXDw4dCoYwijRRCnRJDX36QFoarFgRdiUioVGgS2po1gxyc9WPLk2aAl1SR79+sHMn7NoVdiUioVCgS+ro1y94XLky3DpEQqJAl9TRtm0wR7rGo0sTpUCX1JKfD2vWQFlZ2JWInHQKdEkt+flQWgobN4ZdichJp0CX1JKbG1xopG4XaYIU6JJamjULpgJQoEsTpECX1JOfD1u2wMGDYVciclIp0CX15OcH9xpdsybsSkROKgW6pJ4uXaB5c3W7SJOjQJfUE4lA795BoLuHXY3ISaNAl9SUnw/79gVTAYg0EQp0SU29eweP6naRJkSBLqmpTZvgBtI6MSpNiAJdUldubnDzaN3FSJoIBbqkrrw8KC6GrVvDrkTkpFCgS+rKzQ0e1e0iTYQCXVJXq1bQsaMCXZoMBbqktrw8WL8eysvDrkSkwSnQJbXl5kJJCWzeHHYlIg1OgS6pTf3o0oQo0CW1ZWfDKaco0KVJUKBL6svLgw0bdFs6SXkKdEl9eXlBmOu2dJLiFOiS+nr1AjN1u0jKU6BL6mvWLJgjfe3asCsRaVAKdGkacnODLpfS0rArEWkwCnRpGnJzg4uLNm0KuxKRBqNAl6ahZ8+gH13dLpLCFOjSNGRlqR9dUp4CXZqOin50jUeXFBVXoJvZJDNbbmYrzeyn1ay/2sx2mNnC2Nc1iS9VpJ5yczUeXVJaem0NzCwNuBc4H9gIfGRmBe6+5KimT7v7jQ1Qo0hiVO1Hr5jjRSSFxHOEPgZY6e6r3b0EeAq4uGHLEmkAWVnBvC7qR5cUVesROtAN2FDl9Ubg9GrafcnMzgVWAD9w9w1HNzCz64DrADp37sycOXPqXLBIfbQtLKT18uWs79EDT0sLuxyRhIon0OPxEjDD3Q+b2fXAo8AXjm7k7g8ADwCMGjXKx40bl6Ddi8SpSxeYMYO8Pn2CKQFEUkg8XS6bgB5VXnePLavk7rvc/XDs5YPAyMSUJ5JgFfO6qNtFUlA8gf4R0NfM8swsE/gqUFC1gZl1qfJyKrA0cSWKJJD60SWF1drl4u5lZnYj8BqQBjzk7ovN7BfAPHcvAL5rZlOBMmA3cHUD1ixSP7m58NFHwRDG9ET1OoqEL65/ze4+E5h51LLbqjy/Gbg5saWJNJDcXHj//WBeF/WjSwrRlaLS9Gh+dElRCnRpetSPLilKgS5Nk+Z1kRSkQJemSfO6SApSoEvTpPHokoIU6NI0aX50SUEKdGm61I8uKUaBLk1XRT/6hmPmkRNJSgp0abp0n1FJMQp0aboq+tF1gZGkCAW6NG29ewf96CUlYVciUm8KdGnaeveGaBTWrQu7EpF6U6BL09ajRzDj4urVYVciUm8KdGnaMjKCUFc/uqQABbpIXh5s3QoHD4ZdiUi9KNBFevcOHjV8UZKcAl2ka1do1kz96JL0FOgikUhw1aj60SXJKdBFIOh22b0b9u4NuxKRE6ZAF4HgxCio20WSmgJdBKBjR2jZUoEuSU2BLgLBJF35+UGgR6NhVyNyQhToIhX69oWiIti8OexKRE6IAl2kQu/ewZH6ypVhVyJyQhToIhWys6FbN/jss7ArETkhCnSRqvr2DbpcNA2AJCEFukhVffqAu0a7SFJSoItU1bVr0PWibhdJQgp0karMgqP0VauCI3WRJKJAFzlanz5BH7qGL0qSUaCLHC0/X8MXJSkp0EWO1qJF0Je+YkXYlYjUiQJdpDr9+8OmTbBvX9iViMRNgS5SnYEDg8dly8KtQ6QOFOgi1enYETp0UKBLUlGgi9Rk4MDgPqNFRWFXIhIXBbpITQYMCKbS1clRSRIKdJGadO0KrVvD0qVhVyISFwW6SE3Mgm6XVaugpCTsakRqpUAXOZ4BA6CsTBcZSVJQoIscT69ewWRd6naRJKBAFzmeSCQ4Sl++XN0u0ugp0EVqM3RoEOYaky6NnAJdpDa9ekGbNvDJJ2FXInJcCnSR2pgFR+mrVsH+/WFXI1IjBbpIPE47Lbjhxaefhl2JSI0U6CLxaN8euneHhQt1JyNptBToIvE67TTYvh22bQu7EpFqKdBF4jVoEKSlwT//GXYlItVSoIvEKzsb+vULRruUlYVdjcgxFOgidTFqVHAD6cWLw65E5BgKdJG66N07uPnFBx/o5Kg0Ogp0kbowg9NPhy1bYMOGsKsROYICXaSuhg6FrCz48MOwKxE5ggJdpK4yM2HkyGAGxsLCsKsRqaRAFzkRo0cHfegffRR2JSKVFOgiJ6JNm+BuRvPnw+HDYVcjAijQRU7c2WfDoUPw/vthVyICKNBFTlzXrnDqqUGgFxWFXY2IAl2kXsaPD25+8e67YVciokAXqZeOHYNhjHPnaq50CZ0CXaS+xo0LRry8/XbYlUgTp0AXqa+2bWHEiGDEy/btYVcjTZgCXSQRxo8Prh4tKIBoNOxqpIlSoIskQnY2XHghbNyoi40kNAp0kUQZPBj69oU33oC9e8OuRpogBbpIopjBlCnB85df1vS6ctIp0EUSKScHzjsPVq7UFaRy0inQRRJt9OjgCtLXX4dVq8KuRpoQBbpIopnBJZdAp07w3HOwZ0/YFUkToUAXaQiZmfCVrwTPn3pKMzLKSaFAF2ko7drBZZfBjh3w+OMKdWlwCnSRhpSfD9OmwaZN8NhjUFwcdkWSwhToIg1t4ED48peDG0s/9pim2pUGo0AXORkGDAhCfds2uP/+4IhdJMEU6CInS//+8I1vBKNgHnoI5s3TxUeSUAp0kZOpa1e4/nro3Tu4mvTJJ2H37rCrkhShQBc52Zo3h699DSZNgvXr4b77YM4cKC0NuzJJculhFyDSJJnBGWfAoEEwa1YQ6HPnBstGjw5CX6SOFOgiYWrVCr70JRgzBt55B/7+9+D+pEOGBLe269kzCH+ROCjQRRqDHj2Cbpht24JJvT75JLgDUps2wbDHPn2gVy9I139ZqZl5SGfZR40a5fPmzQtl3yKNXkkJLFsWBPuaNb2FhvcAAAqFSURBVFBeDhkZ0L07dOsWfHXpEszuqCP4JsXM5rv7qOrWxfXr3swmAXcBacCD7v6bo9Y3A/4MjAR2AV9x97X1KVqkScvMDLpchg4NTpauXRtMybthA/zjH5/f5i4zEzp2hPbtg6P5Nm2gdeugK6dly+BOSgr8JqPWQDezNOBe4HxgI/CRmRW4+5Iqzb4J7HH3Pmb2VeC3wFcaomCRxq6wMLgotOIAut7tMzKCOyH17Ru8LisL3rBtG+zYwYJPXmP1R/PpF+lB1/R+tGwZ3N50+c7lrNy7irwup3Jqz5HQvDnvb1nM3B1LGZ47khUHVvHulg/p1r4XRNuzc0drsjsVMbDNWHpFJlASTeeFf77J9lOexnJ20bFVB7522uWcmn0Bf5j5Ov888Dbn5o6j/0Dj3U1vsXVda+bvfhNvuYspvadx+4SfkdMaCvcZj70/k7+sf4QNBz/jQOk+WtOTU5r3YmN0PlnegTbksq1sOV2LJvOdobdz+umwfYfx/bcvZ0nRW5ze6V+Y8eXH2boV3t89k7fWz+aUNjkUU4iV5LB+WyFTBp3H1WMnA/DKile4f+6fKC6Ba4Z/kwt6T2brVjjllOAzfubjV5i54g1OadOaw+xjQu8JTOk3pdqfz8srXuaN1W8ct01dFBbClp0ZdOmRHte/j7qotcvFzM4Efu7uF8Re3wzg7ndWafNarM37ZpYObAU6+nE2ri4XSUWFhXDPPcHV/dnZcOONxw/1urY/WsHyAqY/P52i0iIyo8356sEHGWVn0/v0Av7zjR+RVnSYNt6Mm0d+j7J9JTzy0X2kl5eQVRYhozxKRhTSjr6ndTSdZqsuC+YSO/UZiHzeIM3SyN51JvtzPoC0MohGsIjhlB9TW66dzb8Om8CsBctZwjO4xXfzbFt/Np2LJrCn6zMcbr20cnnb0oF05jSWpz+HW9mx7/N0vph/GS2y4ZnFz1Duwf4ipDHQp9Eu2p+MDGiZu5xXVj9HOZ9vIyMtncsGXkb/Dv2P2Obynct5bulzlJaX1dimLoqLg8FMi3KnsCd/VJ1/3nD8Lpd4Av0yYJK7XxN7fQVwurvfWKXNolibjbHXq2Jtdh61reuA6wA6d+488qmnnqrbdyLSyBUXB9cJZWYG3eDt2gVHy4lqf7QN+zaw/eD2ytetIp1oWd6Dw1kb2H348+WdWnSirIwjllWwaJRIeRQrLycSjRKJOpGD7aA8SiRrB+YO0SjmjrkTKW0GacVY1AEHdwxirwEcc0iLZtItO5/tB7dwOLLn86tiq0SOVX1Rsb4si/T9eZTlfAaRMqyySRrNac0hqptfPmjUMq0dmRmwu/jIi7WaRdvRLrMTpaVQnL6dA+XHXszVLqsdnVp0OmLZ9oPbj9hWdW3qoqQE9u2Dkq6nsD+7XZ1/3gDjx4+vXx96orj7A8ADEByhjxs37mTuXqTBHX3EPXly3Y7Qa2t/tILlBXzr+W9RVFpEBtlcXDqDYc3H0fvCAm57NVienZHNjC/N4OBBuPmlb1FKERFPJ0oZGEEWVn0szab1rBns2wdc9iPIKKncX2akGR1W/JDNPf8AmUVQlkkkzYja4c/fT7Ctf4ncwp8vvYY/PljAs0zD00qOLP6o9hU1RN67haE7v8XqUdPY1+25yuV5hy5jKFfwUrPpRNOKjtlOpDybB8bPoGNH+MYz0yiJBvtL82Z8KfoMfQ5MJTsbTjm3gH9/YzqlfL6N7IxsZkydwcj+U4/5fK+L/QVUU5u6qPx5rzmxn3dt1OUikmAJ70OvRcHyAmatmsVZp0xkePbUyu1ULJ+YP5GpsRCasaCAl5fOYsrAiczf+iEFKwo4teMA0g93ZvuGHNqeUsjodhPpz1RKSuDRDwrY2u1+0nK20bV1Z64fdT3Dmk/l188XsKBwFufnT2TwIHhz/Sw2r87hw52vQcudfKnP1/ntBb8iJyf4/h58p4Bn19zPugPLOVBaSA696No8jw3lc8nyjrQjj61ly+h2cCr/MexXjB0bnCL49pvTWFT0d87s/AX+8rVn2bIF3ttZwN/XzaJL21gf+uEc1m4t5JIhE/nm2VMrP5N7P7if4sPwrVHXc1H+1CM+44rPoWIbVT+jmj7f47Wpi/r+vOvb5ZIOrAAmAJuAj4CvufviKm1uAIa4+7diJ0UvdfcvH2+7CnQRkbqr17BFdy8zsxuB1wiGLT7k7ovN7BfAPHcvAP4EPGZmK4HdwFcTV76IiMQjrj50d58JzDxq2W1VnhcD0xJbmoiI1IVmWxQRSREKdBGRFKFAFxFJEQp0EZEUoUAXEUkRCnQRkRShQBcRSREKdBGRFKFAFxFJEQp0EZEUoUAXEUkRCnQRkRRR6/S5DbZjsx3AugRtrgOws9ZWjY/qPvmStXbVffI11tp7uXvH6laEFuiJZGbzapofuDFT3Sdfstauuk++ZKxdXS4iIilCgS4ikiJSJdAfCLuAE6S6T75krV11n3xJV3tK9KGLiEjqHKGLiDR5CnQRkRSR1IFuZmvN7FMzW2hm88Kupy7MrI2ZPWdmy8xsqZmdGXZNtTGz/rHPuuJrn5l9P+y64mFmPzCzxWa2yMxmmFlW2DXFw8y+F6t5cWP/rM3sITPbbmaLqixrZ2avm9lnsce2YdZYnRrqnhb7zKNmljRDF5M60GPGu/uwZBsvCtwFvOruA4DTgKUh11Mrd18e+6yHASOBIuAvIZdVKzPrBnwXGOXug4E04KvhVlU7MxsMXAuMIfg3MsXM+oRb1XE9Akw6atlPgTfcvS/wRux1Y/MIx9a9CLgUePukV1MPqRDoScfMcoBzgT8BuHuJu+8Nt6o6mwCscvdEXe3b0NKB5maWDmQDm0OuJx4DgQ/dvcjdy4C3CEKmUXL3t4HdRy2+GHg09vxR4JKTWlQcqqvb3Ze6+/KQSjphyR7oDswys/lmdl3YxdRBHrADeNjMPjazB82sRdhF1dFXgRlhFxEPd98E/A5YD2wBCt19VrhVxWURcI6ZtTezbOAioEfINdVVZ3ffEnu+FegcZjGpLtkD/Wx3HwFcCNxgZueGXVCc0oERwB/dfThwkMb5p2i1zCwTmAo8G3Yt8Yj1215M8Iu0K9DCzC4Pt6rauftS4LfALOBVYCFQHmpR9eDBGGmNk25ASR3osSMv3H07QV/umHArittGYKO7fxh7/RxBwCeLC4EF7r4t7ELidB6wxt13uHsp8AIwNuSa4uLuf3L3ke5+LrAHWBF2TXW0zcy6AMQet4dcT0pL2kA3sxZm1qriOTCR4E/URs/dtwIbzKx/bNEEYEmIJdXVdJKkuyVmPXCGmWWbmRF83o3+JDSAmXWKPfYk6D9/MtyK6qwAuCr2/CrgxRBrSXlJe6WomfXm8xEW6cCT7v6rEEuqEzMbBjwIZAKrgX9z9z3hVlW72C/P9UBvdy8Mu554mdkdwFeAMuBj4Bp3PxxuVbUzs3eA9kAp8B/u/kbIJdXIzGYA4wimnd0G3A78FXgG6EkwXfaX3f3oE6ehqqHu3cAfgI7AXmChu18QVo3xStpAFxGRIyVtl4uIiBxJgS4ikiIU6CIiKUKBLiKSIhToIiIpQoEuIpIiFOgiIini/wPHyaIjo3iQ1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X_train[:, 0], Y_train, s=8, c='b', alpha=0.4, label='train data')\n",
    "plt.scatter(X_test[:, 0], Y_test, s=10, c='g', label='test data')\n",
    "sig_x = np.linspace(X[:, 0].min(), X[:, 0].max(), 100)\n",
    "sig_y = [hypothesis(np.array([x, 1]), theta_binary) for x in sig_x]\n",
    "plt.plot(sig_x, sig_y, c='r', alpha=0.5, label='model')\n",
    "plt.yticks([0, 0.5, 1])\n",
    "plt.ylim([-0.1, 1.1])\n",
    "plt.legend()\n",
    "plt.grid(axis='y')\n",
    "plt.title('Optimized model')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One vs all\n",
    "\n",
    "What if we don't have a binary target but many groups (>2) to classify.  \n",
    "It's in this that one vs all is applied.\n",
    "\n",
    "Here, there is 2 features and 3 categories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAPx0lEQVR4nO3dXYhc533H8e9fkQzNavxCtHVd+UWhkpP0JW6sDTaNSJXmwrZaGgIuwgl2Yht00RAsUoSIofGFL9KQEpJibCEcWzENjqBeO2lRQgNtohrXLiujWo5FLRGTjSKDVnaJVpsLI/bfizOyttLszqz2aGbOs98PLDuzz+M5P4TOT4/PnNknMhNJUvOtGHQASVI9LHRJKoSFLkmFsNAlqRAWuiQVYuWgDrxmzZpct27doA4vSY104MCBk5k52mlsYIW+bt06JiYmBnV4SWqkiPjFfGNecpGkQljoklQIC12SCmGhS1IhBvamqJaJ6WnYuxeOHIENG2DrVmi1Bp1KKlLXQo+I64CngKuBBHZn5rfOmxPAt4AtwG+Az2fmy/XHVaM8/zxs2QKzszAzAyMj8KUvwb59sGnToNNJxellhX4G+JvMfDkiWsCBiPhxZr42Z84dwIb21y3AY+3vWq6mp6syn54+97OZmer7li1w/DisXj2YbFKhul5Dz8w3z662M3MaOAysPW/ap4CnsvIicGVEXFN7WjXH3r3VyryT2dlqXFKtFvWmaESsAz4CvHTe0Frgl3OeH+PC0icitkXERERMTE1NLS6pmuXIkXMr8vPNzMDRo/3NIy0DPRd6RKwGngG2Z+apizlYZu7OzLHMHBsd7fjJVZViw4bqmnknIyOwfn1/80jLQE+FHhGrqMr8u5k53mHKr4Dr5jy/tv0zLVdbt8KKef56rVhRjUuqVddCb9/B8m3gcGZ+Y55pPwDuicqtwK8z880ac6ppWq3qbpZW69xKfWTk3M99Q1SqXS93uXwMuBs4FBEH2z97ELgeIDN3Afuoblk8SnXb4r31R1XjbNpU3c2yd291zXz9+mplbplLl0TXQs/M54HoMieBL9QVSgVZvRruv3/QKaRlwY/+S1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEKsHHQADYnpadi7F44cgQ0bYOtWaLUGnWp+Tcsr9UFk5sITIp4A/gI4kZl/2GH8CuAfgeup/oH4+8x8stuBx8bGcmJi4qJCq2bPPw9btsDsLMzMwMgIrFgB+/bBpk2DTnehpuWVahQRBzJzrONYD4X+ceA08NQ8hf4gcEVm7oyIUeB/gN/JzHcWel0LfUhMT8PatdX387VacPw4rF7d/1zzaVpeqWYLFXrXa+iZuR94e6EpQCsiAljdnnvmYoJqAPburVa6nczOVuPDpGl5pT6q403RR4APAceBQ8ADmdnxjIuIbRExERETU1NTNRxaS3bkSHXZopOZGTh6tL95umlaXqmP6ij024CDwO8Cfww8EhGXd5qYmbszcywzx0ZHR2s4tJZsw4bqGnQnIyOwfn1/83TTtLxSH9VR6PcC41k5CrwBfLCG11U/bN1avaHYyYoV1fgwaVpeqY/qKPRJ4JMAEXE18AHg5zW8rvqh1aruDmm1zq18R0bO/XzY3mBsWl6pj7rehx4RTwObgTURcQx4CFgFkJm7gIeBPRFxCAhgZ2aevGSJVb9Nm6q7Q/bura5Br19frXSHtRybllfqk663LV4q3rYoSYu3pNsWJUnNYKFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBWia6FHxBMRcSIiXl1gzuaIOBgRP4uIn9YbUZLUi15W6HuA2+cbjIgrgUeBv8zMPwD+qp5okqTF6FrombkfeHuBKZ8BxjNzsj3/RE3ZJEmLUMc19BuBqyLiJxFxICLumW9iRGyLiImImJiamqrh0JKks+oo9JXARuDPgduAv42IGztNzMzdmTmWmWOjo6M1HFqSdNbKGl7jGPBWZs4AMxGxH7gJeL2G15Yk9aiOFfr3gU0RsTIi3gvcAhyu4XUlSYvQdYUeEU8Dm4E1EXEMeAhYBZCZuzLzcET8CHgFmAUez8x5b3GUJF0aXQs9M+/qYc7Xga/XkkiSdFH8pKgkFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQtTx2xYlqa8u/+rlTL8zPe9467IWp758qo+JhoMrdEmNs1CZ9zJeKgtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl9Q4rctaSxovlR8sktQ4y/FDQ71whS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBWia6FHxBMRcSIiXu0y76MRcSYi7qwvniSpV72s0PcAty80ISLeA3wN+NcaMkmSLkLXQs/M/cDbXaZ9EXgGOFFHKEnS4i35GnpErAU+DTzWw9xtETERERNTU1NLPbQkaY463hT9JrAzM2e7TczM3Zk5lpljo6OjNRxaknRWHRtcjAHfiwiANcCWiDiTmc/V8NqSpB4tudAz8/1nH0fEHuBfLHNJ6r+uhR4RTwObgTURcQx4CFgFkJm7Lmk6SVLPuhZ6Zt7V64tl5ueXlEaSdNH8pKgkFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQtTx2xb7anYWvvMd+NznYIX/HEmq0eVfvZzpd6bnHW9d1uLUl0/1MdHiNK4Sx8fhvvvg2WcHnURSaRYq817GB61RhT47Czt2VI937KieS5IqjSr08XE4ebJ6PDXlKl2S5mpMoZ9dnZ8+XT0/fdpVuiTN1ZhCn7s6P8tVuiSd04hCP391fpardEk6pxGF/txzMDnZeWxyshqXpOWuEfeh33ADbN++8LgkLVXrslbX+9CHWSMKfePG6kuSLqVh/tBQLxpxyUWS1J2FLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSpEIz4pKklN1q+t7bqu0CPiiYg4ERGvzjP+2Yh4JSIORcQLEXHTklNJUkH6tbVdL5dc9gC3LzD+BvCnmflHwMPA7hpySZIWqesll8zcHxHrFhh/Yc7TF4Frlx5LkrRYdb8pej/ww/kGI2JbRExExMTU1FTNh5ak5a22Qo+IT1AV+s755mTm7swcy8yx0dHRug4tSaKmu1wi4sPA48AdmflWHa8pSVqcJa/QI+J6YBy4OzNfX3okSdLF6LpCj4ingc3Amog4BjwErALIzF3AV4D3AY9GBMCZzBy7VIElqWn6tbVdZGYtL7RYY2NjOTExMZBjS1JTRcSB+RbNfvRfkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdL1rdhaefLL6Lql5LHS9a3wc7rsPnn120EkkXQwLXUC1Kt+xo3q8Y4erdKmJLHQB1er85Mnq8dSUq3SpiSx0vbs6P326en76tKt0qYksdP2/1flZrtKl5rHQl7nzV+dnuUqXmsdCX+aeew4mJzuPTU5W45KaoeueoirbDTfA9u0Lj0tqBgt9mdu4sfqS1HxecpGkQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUiMYVuvteSlJnXQs9Ip6IiBMR8eo84xER/xARRyPilYi4uf6Y57jvpSR11ssKfQ9w+wLjdwAb2l/bgMeWHqsz972UpPl1LfTM3A+8vcCUTwFPZeVF4MqIuKaugHO576Ukza+Oa+hrgV/OeX6s/bMLRMS2iJiIiImpqalFHcR9LyVpYX19UzQzd2fmWGaOjY6OLuq/dd9LSVpYHYX+K+C6Oc+vbf+sNu57KUnd1VHoPwDuad/tcivw68x8s4bXfZf7XkpSd123oIuIp4HNwJqIOAY8BKwCyMxdwD5gC3AU+A1wb90h3fdSkrrrWuiZeVeX8QS+UFuiDtz3UpK6a9wnRSVJnVnoklQIC12SCmGhS1IhonpPcwAHjpgCfnGR//ka4GTXWcOjSXmblBWalbdJWaFZeZuUFZaW94bM7PjJzIEV+lJExERmjg06R6+alLdJWaFZeZuUFZqVt0lZ4dLl9ZKLJBXCQpekQjS10HcPOsAiNSlvk7JCs/I2KSs0K2+TssIlytvIa+iSpAs1dYUuSTqPhS5JhRjqQh+2DaoX0kPWz7YzHoqIFyLipn5nPC/PgnnnzPtoRJyJiDv7la1Dhq5ZI2JzRByMiJ9FxE/7ma9Dlm5/F66IiH+OiP9u5639N5T2KiKui4h/j4jX2lke6DBnKM6zHrMOzXnWS945c+s5zzJzaL+AjwM3A6/OM74F+CEQwK3AS0Oc9U+Aq9qP7xhk1l7ytue8B/g3ql+RfOewZgWuBF4Drm8//+1h/rMFHgS+1n48SrVn72UDynoNcHP7cQt4Hfj98+YMxXnWY9ahOc96ydseq+08G+oVeg7RBtXddMuamS9k5v+2n75ItbPTwPTwZwvwReAZ4MSlTzS/HrJ+BhjPzMn2/GHPm0ArIgJY3Z57ph/ZLgiS+WZmvtx+PA0c5sI9gYfiPOsl6zCdZz3+2UKN59lQF3oPet6gesjcT7XiGVoRsRb4NPDYoLP04Ebgqoj4SUQciIh7Bh2oi0eADwHHgUPAA5k58I0UI2Id8BHgpfOGhu48WyDrXENzns2Xt+7zrOsGF6pXRHyC6i/apkFn6eKbwM7MnK0WkkNtJbAR+CTwW8B/RsSLmfn6YGPN6zbgIPBnwO8BP46I/8jMU4MKFBGrqVaJ2weZoxe9ZB2m86xL3lrPs6YX+iXfoLpOEfFh4HHgjsx8a9B5uhgDvtf+S7YG2BIRZzJzGHdwPQa8lZkzwExE7AduorpmOYzuBf4uqwuoRyPiDeCDwH8NIkxErKIqnO9m5niHKUNznvWQdajOsx7y1nqeNf2SyyXfoLouEXE9MA7cPcQrx3dl5vszc11mrgP+CfjrIS1zgO8DmyJiZUS8F7iF6nrlsJqk+r8JIuJq4APAzwcRpH0d/9vA4cz8xjzThuI86yXrMJ1nveSt+zwb6hV6DMEG1b3qIetXgPcBj7b/NT6TA/ztcD3kHRrdsmbm4Yj4EfAKMAs8npkL3o45yLzAw8CeiDhEdefIzswc1K9+/RhwN3AoIg62f/YgcD0M3XnWS9ZhOs96yVsrP/ovSYVo+iUXSVKbhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIK8X82B+xm4+hmuQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs_oa = np.array([[1, 1], [1, 1.3], [1.4, 1.1],\n",
    "               [1.5, 2], [1.6, 1.8], [1.4, 1.8],\n",
    "               [2.4, 1.2], [2.2, 1.5], [2.3, 1.3]])\n",
    "ys_oa = np.array([0, 0, 0,\n",
    "               1, 1, 1,\n",
    "               2, 2, 2])\n",
    "\n",
    "xs_oa = np.hstack([xs_oa, np.ones((xs_oa.shape[0], 1))])\n",
    "\n",
    "xs_plot_spec = [\n",
    "    {'marker': '^', 'c': 'b', 's': 50},\n",
    "    {'marker': 'o', 'c': 'r', 's': 50},\n",
    "    {'marker': 's', 'c': 'g', 's': 50}\n",
    "]\n",
    "\n",
    "def plot_onevsall(xs, ys):\n",
    "    for i, y in enumerate(ys):\n",
    "        plt.scatter(xs[i,0], xs[i,1], **xs_plot_spec[y])\n",
    "        \n",
    "plot_onevsall(xs_oa, ys_oa)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tricks is to create an hypothesis function for every target.  \n",
    "Each of those will compare his target to all the other one.\n",
    "\n",
    "For example the hypothesis function for *circle* will separate it from *triangle* and *square*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3gUVRfH8e9NINRQpIs0pSs9FJGOqCAECC0gIEXAXlBEsGAHG6+goqIIqBhCCRAUVASkSJFQlCpFOgihh4BAkvv+cbMuJWUJm8zM7vk8D0+SnXH3B2ZPbu7cuUdprRFCCOF8AVYHEEII4R1S0IUQwkdIQRdCCB8hBV0IIXyEFHQhhPAR2ax64cKFC+uyZcta9fJCCOFIa9euPaa1LpLSMcsKetmyZYmJibHq5YUQwpGUUntTOyZTLkII4SOkoAshhI+Qgi6EED5CCroQQvgIyy6KCj8RFweRkbBjB1SoAN26QXCw1amE8EnpFnSlVCnga6AYoIHxWusxV52jgDFAG+Ac0Edrvc77cYWjLF8ObdpAUhLEx0OePDB4MMybB40aWZ1OCJ/jyQg9AXhWa71OKRUMrFVKLdBab7nsnNZAheQ/9YFPkz8KfxUXZ4p5XJz7sfh487FNGzh0CPLmtSabED4q3Tl0rfVh12hbax0HbAVKXnVae+BrbawCCiilSng9rXCOyEgzMk9JUpI5LoTwquu6KKqUKgvUAlZfdagksP+yrw9wbdFHKTVQKRWjlIqJjY29vqTCWXbscI/IrxYfDzt3Zm0eIfyAxwVdKZUXmAk8rbU+k5EX01qP11qHaK1DihRJ8c5V4SsqVDBz5inJkwfKl8/aPEL4AY8KulIqO6aYT9FaR6VwykGg1GVf35L8mPBX3bpBQCrfXgEB5rgQwqvSLejJK1gmAFu11qNTOS0a6K2MBsBprfVhL+YUThMcbFazBAe7R+p58rgflwuiQnidJ6tc7gJ6ARuVUhuSHxsOlAbQWn8GzMMsWdyJWbbY1/tRheM0amRWs0RGmjnz8uXNyFyKuRCZIt2CrrVeDqh0ztHAY94KJXxI3rzQv7/VKYTwC3LrvxBC+Agp6EII4SOkoAshhI+Qgi6EED5CCroQQvgIKehCCOEjpKALIYSPkIIuhBA+Qgq6EEL4CCnoQgjhI6SgCyGEj5CCLoQQPkIKuhBC+Agp6EIIkdm0hk2bYMwYaN8evvoqU17Gk/3QhRBCXA+tYfduWLgQFi0yf44eNcfKl4fWrTPlZaWgCyGENxw+7C7eCxfC3r3m8RIl4J57oGVLaNECSpfOtAhS0IUQIiNOnoRff3UX8K1bzeMFC0Lz5vD886aAV6oEKs0eQV4jBV0IITwRHw/Ll7sL+Lp1Zmold25o0gT69TMFvGbN1BukZzIp6EIIkZKLF2H1ancBX7UKLl2C7Nnhzjvh1VdNAa9XD4KCrE4LSEEXQggjMRE2bHAX8GXL4Nw5M11Spw4MHmwKeKNGZlRuQ1LQhRD+SWvYts1dwH/91cyLA1Stapqbt2gBTZuaeXEHkIIuhPAfe/e6C/iiRWZlCkDZshAWZgp4ixZQvLilMTNKCroQwncdPQqLF7sL+K5d5vFixdzFu2VLKFfO2pxeIgVdCOE7Tp+GpUvdBXzjRvN4/vzQrBk8+aQp4FWrZtlSwqwkBV0I4Vznz8OKFe4CHhNjLm7mymUuXvboYQp4rVqQzffLne//DYUQvuPSJVO0XQV8xQq4cMEU63r1YPhwU8AbNIAcOaxOm+WkoAsh7CspyUybuAr40qUQF2emS2rWhMcfNwW8USMIDrY6reWkoAsh7ENr2LnTXcAXL4Zjx8yxihWhZ09TwJs1g0KFLI1qR1LQhRDWOnjwyl0J9+83j99yC9x/vyngzZubr0WapKALIbLW8eNm5O0q4H/9ZR4vVMgsI3zxRfOxfHmfXImSmaSgCyMuDiIjYccOqFABunWz95yk0/L6s7Nnzdy364aeP/4wUyt585q7MAcONKPwatUs29TKVyitddonKPUV0BY4qrW+I4Xj+YFvgdKYHxDva60npvfCISEhOiYmJkOhhZctXw5t2pgLUPHxkCePeWPNm2cuNtmN0/L6mwsXYOVK9wh89WpISDCrTho2dN/MExJiNroS10UptVZrHZLiMQ8KehPgLPB1KgV9OJBfaz1UKVUE+AsorrW+mNbzSkG3ibg4KFnSfLxacDAcOmRGUnbhtLz+IDER1q51j8CXL4d//zU/ZOvWdRfwhg3N+nCHOHvxLIEqkFzZ7ZU5rYKe7pSL1nqpUqpsWqcAwUopBeQFTgAJGcgprBAZaUa6KUlKMsf798/aTGlxWl5fpDVs3uwu4EuWmDs0wUybDBpkCniTJuYOTQdKTEqkx8weHD9/nKV9lhIYEGh1JI94Yw79YyAaOAQEA9201im+45RSA4GBAKUzsQ2TuA47dphpi5TEx5slZHbitLy+4u+/r9zUytUf87bboGtX90qUokWtzeklLy16ibnb5/Jx648dU8zBOwX9XmAD0AK4DViglFqmtT5z9Yla6/HAeDBTLl54bXGjKlQwc9ApFck8ecxKAztxWl6nOnz4yk2t9uwxj5coAa1auftjliljaczMMOXPKYz6bRSD6gzi0bqPWh3nuqQ7hw6QPOXyfSpz6D8Ao7TWy5K/XgS8oLX+Pa3nlDl0m3DanLTT8jrFyZNm6sRVwLdsMY8XKGBG3q4CXrmyTy8lXH1gNU0nNaXBLQ1Y0GsB2QPtd9H2hubQPbAPaAksU0oVAyoBf3vheUVWCA42q0NSWzVit+LotLx2de7ctf0xk5JMJ57GjaFPH3d/zEDnTDnciINnDtIxsiM3B9/MjK4zbFnM05NuQVdKRQDNgMJKqQPACCA7gNb6M+ANYJJSaiOggKFa62OZllh4X6NGZmQbGWnmoMuXN+u67VocnZbXDi5ehN9/dxfwlSvd/TEbNIBXXjEFvH592/THzErnL52nQ2QH4i7GsaDXAgrnLmx1pAzxaMolM8iUixCZKCnp2v6Y8fFmuqR2bfcUSqNG5jccP6a1pkdUDyI3RTInfA7tKrWzOlKaMnvKRQhhNa3NLfSX98c8ccIcq1IF+vY1BbxZM8f0x8wqI5ePZOqmqYxsOdL2xTw9UtCFcKp9+65cSnjokHm8TBno0MHdYq1ECWtz2ticbXN4cdGLPFDtAYbeNdTqODdMCroQThEbe+VSQtea+6JFr+2P6cMrUbzlzyN/8kDUA9S9uS5ftPsC5QP/ZlLQhbCrM2eu7I/555/m8Xz5zNSJq7nD7bdLAb9OsfGxhEaEkj9nfmaHz7bd7f0ZJQVdCLs4f96sPnEV8DVrzD4pOXOai5dvv20KeO3aftEfM7NcTLxIp2mdOBJ/hGV9l3Fz8M1WR/Ia+a4QwioJCVf2x/ztN7NTYWCg6Y85bJi7P2bOnFan9Qlaax6f9zjL9i3ju7DvCLk5xcUijiUFXYiskpQEmza5C/iSJe47XmvWhMceM/PgTZrI3u6Z5OPfP+aLdV8wvNFwulfrbnUcr5OCLkRm0Rp27bqyP2ZsrDlWoQI88IC7P2ZhZ97I4iQLdi3gmZ+eoX2l9rzR4g2r42QKKehCeNPBg+7GDgsXuvtjliwJrVu7V6OUKmVtTj+z/fh2us7oSpUiVfim4zcEKN/sjCQFXYgbceKEuz/mwoVX9sds3tw9D16hgqxEscipf08RGhFKtoBsRIdHE5zDd6ezpKALcT3OnjW30bsK+IYN7v6YTZrAgAGmgFevLv0xbSAxKZHuM7uz6+QuFvZeSLmC5ayOlKmkoAuRlgsXYNUqdwF39ccMCjIt1V57zRTwunWlP6YNPb/geX7c+SPj246nSZkmVsfJdI4r6ElJZpuKFi2sTiJ8UmKi2Ur28v6Y58+b0XZICDz3nLs/Zu7cVqcVaZi0YRKjV43miXpPMKDOAKvjZAnHFfQJE2DgQBg8GN55R+6vEDdIa9PM4fJNrVz9Me+4wz2F0qSJafYgHOG3fb8x6PtB3H3r3Yy+d7TVcbKM48phnz6wcSOMHg3r15stsYsUsTqVcJTdu6/c1OrIEfP4rbdCly7u/pjFilmbU2TIvtP7CJsWRun8pYnsHEm2AMeVuQxz3N80e3YYO9b89jtokPkYFQV16lidTNjWP/9cuanV7t3m8eLFTfF27Q1etqylMcWNi78YT2hEKP8m/MuSPku4KddNVkfKUo4r6C69e5s9icLC4K674PPP4cEHrU4lbOHUqSv7Y27ebB4vUMDcxDN4sCngVarIUkIfkqSTeHD2g2w8upHvu39P5cKVrY6U5Rxb0MGMymNiIDzcTMXExJipGFls4GfOnTP7oLgK+Nq15up5rlymP2bv3qaA16rlN/0x/dHrS15n5taZfHDPB7Su0NrqOJZwdEEHM3/+00/wwgvwwQfwxx8wbZr5bVr4qEuXTH9MVwFfudL0zMyWzWxk9fLL7v6YOXJYnVZkgembp/PaktfoU7MPzzR4xuo4lvGpnqIREdC/v+mwFRVl3s/CByQlmZ/UrgK+dKm7P2atWlf2x5RG0X5n/eH13PXVXdQqUYtFvReRI5tv/xD3m56i3btD1arQsaNZZfbJJ/DQQ1anEtdNa9i+/cpNrVz9MStXNvNrrv6YN/nXRS9xpX/O/kPo1FAK5y5MVNcony/m6fGpgg5Qo4aZS+/e3SwhXrPGrIqR37xtbv9+dwFftMhscgVQujS0b+/e1Opm32lGIG7MhYQLhEWGceL8CZb3XU6xvLLM1OcKOphB27x5Zip15EjTuWvmTKkFtuLqj+laD+7qj1mkyJX9MW+9VVaiiGtorRn0/SBWHljJ9C7TqVWiltWRbMEnCzqYxQxvv226dfXpY1bEzJhhljgKC7j6Y7oKuKs/ZnAwNG3qbu5wxx2yqZVI1+iVo5n8x2Rebfoqnat2tjqObfjURdHUbN5s5tV374YxY+CRR2TQl+n+/dfdH3PhQnd/zBw5zMVL1yg8JET2bxDXZf6O+bSNaEtYlTAiO0f67N7mqfGbi6Kpuf12s8qtZ08zEIyJgXHjpE2jV7n6Y7rmwH/7zRT1wECzE+ELL5gC3rCh/MOLDNsau5XwmeFUL1adSe0n+V0xT49fFHQwNwlGR5vdTl9/3ewHExUljWMyLCnJ/OrjGoFf3h+zRg3za5CrP2a+fNZmFT7hxPkTtItoR85sOZkTPoc8QXmsjmQ7flPQwUzNvvaamU/v2dN8nDbNrH4T6XD1x3SNwBctcvfHLF8eevQwBbx5c9ktTXjdpcRLdJ3elf1n9rP4wcWUzl/a6ki25FcF3SU01EzBdOwId99t7jB98kmZV7/GoUNX7kq4b595/Oab4b773PPgpeXNJTLX4J8Gs3D3Qia2n0jDUg2tjmNbflnQwdyfsnq12dDr6afNNbvx4/28Z8GJE2Y/cFcR37bNPH7TTWbkPXSoWUpYsaL89BNZZvza8Xy85mOevfNZ+tTsY3UcW0u3oCulvgLaAke11nekck4z4EMgO3BMa93UmyEzS758Zn36yJFmzfrmzTBrlh/tohofb/pjukbg69ebqZU8eczcd//+poDXqCFLCYUlluxZwmPzHqN1+da8c/c7VsexvXSXLSqlmgBnga9TKuhKqQLACuA+rfU+pVRRrfXR9F44K5ctemL+fDMNHBAAU6dCq1ZWJ8oEFy6YX0su74956ZLpj3nnne6beerWNY8JYaHdJ3dT94u6FMlThFX9V5E/Z36rI9nCDS1b1FovVUqVTeOUHkCU1npf8vnpFnM7at3aTLt07Gimh0eNMu0jHT2zkJhoRt2uEfiyZe7+mHXqmH3BW7Y0d1v59VyTsJu4C3GETg0lSScRHR4txdxD3phDrwhkV0r9CgQDY7TWX6d0olJqIDAQoLQNL6SVL2/uhenXD55/3iyr/uorMwPhCFrD1q1X9sc8dcocu/12s1NZy5bmzkzpjylsKkkn0XNWT7bGbuXHnj9SoVAFqyM5hjcKejagDtASyAWsVEqt0lpvv/pErfV4YDyYKRcvvLbX5c1r+pS67oXZssXMq5cvb3WyVOzZc+VKlH/+MY+XKwedOrn7Y8oG8cIhXlr0EtF/RTP2vrHcfevdVsdxFG8U9APAca11PBCvlFoK1ACuKehOoRQMGQI1a5puSHXrwnffmWkZyx05cmV/zL//No8XK+aeA2/RwhR0IRzmu43fMXL5SAbWHsjj9R63Oo7jeKOgzwE+VkplA4KA+sD/vPC8lmvVyky7hIXB/ffDG2/AsGFZvODj1CmzqZWrgG/aZB7Pn9/cEfX006aAV63q8Al/4e/WHFxD/+j+NCnThI/afISS7+fr5smyxQigGVBYKXUAGIFZnojW+jOt9Val1I/An0AS8KXWelPmRc5a5cqZbUkGDoSXXjLtKidPNpsEZopz52DFCncBj4lx98ds1Mjc4tqihdlGUvpjCh9x8MxB2k9tT/G8xZnZdSZBgbLKKiP8YrdFb9Da7NT43HPmvppZs6BSJS888aVLZnmNq4CvWOHuj1m/vnsKpUED6dIhfNL5S+dpMqkJ245tY0W/FVQrVs3qSLbm97steoNSZnajZk3o2hXq1YNvvjHbCFyXpCSzF/jl/THPnjUvULOm2YOgRQvTrV76Ywofp7Wmf3R/1h5ay6xus6SY3yAp6NepWTMz7RIWZjqjvfIKjBiRxry61rBjx5X9MY8fN8cqVYLevd39MQsVyqK/hRD2MGr5KCI2RfBWi7doX7m91XEcTwp6BpQqZe7ReeQRsxXvunVmtP7f0u4DB9wFfOFCd3/MUqWgXTv3plYlS1r2dxDCatF/RfPiohfpfkd3hjUaZnUcnyAFPYNy5jQ3HdWtC089palX9SyzG4+m6vopZkQOULjwlf0xb7tNVqIIAWw8spEHoh4g5OYQJoROkBUtXiIFPSPi4mDpUtSiRTy6cCHVE/LS+fAM6k8bzKTa2ek0Opcp4tWqyaZWQlwlNj6W0KmhBAcFM6vbLHJlz2V1JJ8hBd0Trv6YrimU339398ds2JBGb7ZkbbWDdH67Fp1XD2fYvfDGHRAotVyIK1xMvEjn6Z05HHeYpX2XUjKfTDt6kxT0lCQkmCufrgLu6o8ZEGDmWIYOdffHzGVGFyWBX+81i1RGjjTz6t99Z7YSF0KYFS1PzHuCpXuXMiVsCvVK1rM6ks+Rgg5mJcqmTe4CvmQJnDljjlWrBg8/7O6PmT/1Xd9y5IDPPzeN7B9/3HycPRuqV8+iv4cQNvbJmk8Yv248L9z1Aj2q9bA6jk/yzxuLtDZ7oLgK+OLFcDR519/bbnPfzNO8ORQtmqGXWLXK7I116hRMmGD2hBHCXy38eyH3fnsvbSq0YXb4bAKUzEdmlNxYBHD48JW7Eu7dax4vUQLuuce9GqVMGa+8XIMGZtamSxfo3t3cwT9qlLkBVAh/suP4DrpM70KVIlWYEjZFinkm8t3ycuKEmTpxFfCtW83jBQuakfeQIWYkXqlSpi0lLF7cvPyzz5pG1OvXm615CxfOlJcTwnZO/3ua0KmhBKgAosOjCc7hnU2Q8o3MR9zFuFSPBwcFc2bYGa+8lpP4TkGPj4fly90FfN06M7WSO7eZ++7b190fMws3tQoKgo8+MvPpgwaZRkGzZpm9tYTwZYlJiXSf2Z2dJ3byS69fKFfQe1s6p1XMPTnuq5xb0C9eND0xXQV81Sqz0VX27KY/5ogRpoDXq2eL/pgPPmiaBoWFmY5v48dDr15WpxIi8wz9ZSjzd87ns/s/o2lZR/SNdzznFfTly+HNN8299+fOmemSOnXgmWfc/TFt2jMuJMTMq3ftarZwiYmB9983P4OE8CWTN0zmg5Uf8FjdxxgUMsjqOH7DeQU9IQH27zeNP139MQsWtDqVx4oUgQULzFL20aNhwwaYNs00HBLCF6zYv4KB3w+kZbmW/O9en+h14xjOK+jNmsHmzVanuCHZspmLpHXqmL7NdepAVJSZHRLCyfaf3k9YZBil85dmWpdpZA+UXz+zkqwfslCPHqafRfbsZvvzCROsTiRExsVfjKf91PacTzhPdHg0N+WS26SzmhR0i9WsaebSmzY1o/VHHjHXe4VwkiSdRJ85fdjwzwYiOkVQpUgVqyP5JSnoNlCoEMyfb+bVP/vMLJM/dMjqVEJ47o0lbzBjywzebfUubSq0yfTXCw5Kez17esd9lX/e+m9j06ebJfPBwTBjhlm0I4Sdzdwyk87TO9O7Rm8mtZ8ke5tnsrRu/ZcRus106WKW1OfJY0bqn31m7o8Swo42/LOB3rN70+CWBnze9nMp5haTgm5Dd9wBa9ZAq1ZmTv2hh8zuvULYyZGzRwiNCOWmXDcxq9sscmbLaXUkvycF3aYKFoS5c+Hll02ruyZNzPJ7IezgQsIFwqaFcezcMaLDoymet7jVkQRS0G0tIMA0oZ41C7ZtM+vVlyyxOpXwd1prHvnhEVbsX8HkDpOpVaKW1ZFEMinoDtChg+l6d9NN5ubYsWNlXl1Y53+r/sfEDRN5pckrdLm9i9VxxGWkoDtE5cqmqLdtC089ZTb7On/e6lTC38zfMZ8hC4bQqUonRjQbYXUccRUp6A6SL5/ZIuD11+Hbb82Sxj17rE4l/MW2Y9sInxlOtaLVmNxhsjSqsCH5P+IwAQHmQuncuaaLXkiI2UFYiMx08vxJQiNCyZktJ3PC55AnyJ47mvo7KegOdf/9Zmlj8eKmg97778u8usgcCUkJdJ3RlT2n9hDVNYoyBbzTplF4nxR0B6tQwdyEFBZmOup1724aNwnhTc/+9Cy//P0Ln7f9nLtKy63LdiYF3eHy5jX7qY8aZbYNuPNO2LXL6lTCV3yx9gvG/j6WZxo8Q99afa2OI9KRbkFXSn2llDqqlNqUznl1lVIJSqnO3osnPKGU2dhr/nw4cMDMq//4o9WphNMt3buUR+c9yr233cu7rd61Oo7wgCcj9EnAfWmdoJQKBN4BfvZCJpFB99xjtuItUwbatIG335Z5dZExe07todO0TtxW8Damdp5KtgDn9cLxR+kWdK31UuBEOqc9AcwEjnojlMi4W281TTPCw+HFF6FzZ4jzzwboIoPiLsQRGhFKQlIC0d2jKZCzgNWRhIdueA5dKVUS6Ah86sG5A5VSMUqpmNjY2Bt9aZGK3LlhyhTTs3TOHKhfH7ZvtzqVcIIknUSvWb3YEruFaZ2nUbFQRasjievgjYuiHwJDtdZJ6Z2otR6vtQ7RWocUKVLECy8tUqMUPPOMaUgdGwt168L331udStjdK4tfYc5fcxh972ha3dbK6jjiOnmjoIcAU5VSe4DOwDilVAcvPK/wgubNYe1as8SxXTt47TVISvdHr/BHERsjeGvZWzxU6yGeqPeE1XFEBtxwQddal9Nal9ValwVmAI9qrWffcDLhNaVLw7JlZv+XV181m32dPm11KmEnaw6uoV90PxqXbswn938ijSocypNlixHASqCSUuqAUqq/UuphpdTDmR9PeEuuXDBxInz8sVneWK8ebNlidSphB4fiDtEhsgPF8hRjZteZBAUGWR1JZFC6a5G01t09fTKtdZ8bSiMylVLw2GNQvbpZ/VK/PkyebO40Ff7p/KXzdIzsyOl/T7Oi/wqK5JFrW04md4r6ocaNYd06uP126NTJLG9MTLQ6lchqWmsGzB3A7wd/59uwb6lerLrVkcQNkoLup0qWNN2PBgwwNyC1bQsnT1qdSmSld397lykbp/Bm8zfpUFnWMfgCKeh+LEcOGD8ePv/cbMEbEgIbN1qdSmSFuX/NZdjCYYTfEc7wxsOtjiO8RAq6YOBAM1o/fx4aNDCbfQnftenoJnpE9aB2idpMCJ0gK1p8iBR0AZhdGteuhVq1oFs3eP55SEiwOpXwtmPnjhEaEUreoLzMCZ9D7uy5rY4kvEgKuvhPiRKwaJFZCfPee3DffXDsmNWphLdcSrxEl+ldOBR3iNndZlMyX0mrIwkvk4IurhAUZNaqT5wIy5ebefX1661OJbzhyflP8uueX/ky9Evq31Lf6jgiE0hBFynq08cU9MREaNjQNKUWzjVuzTg+W/sZQ+8aSs/qPa2OIzKJFHSRqpAQM69evz706gVPPw2XLlmdSlyvhX8v5Mn5T9K2YlveavGW1XFEJpKCLtJUtKjZsfHpp2HMGGjVCo7KrveOsfPETrpM70LlwpWZEjaFwIBAqyOJTOS4NiRJSeZ29QcfhAD5cZQlsmeH//3PjNgHDIA6dSAqymzJK+zr9L+nCY0IRSlFdPdo8uXIZ3Uk28s3Mh9xF1PvCBMcFMyZYWeyMNH1cVxJjIqCfv1g1iyrk/ifBx6A336DwECzfcDEiVYnEqlJTEqkR1QPdpzYwYwuM7i14K1WR3KEtIq5J8et5qiCnpQEQ4aYz4cMkX29rVCrlulb2rix+cH62GNw8aLVqcTVhi0cxrwd8/io9Uc0L9fc6jgiiziqoEdFuddFx8bKKN0qhQubLXiffx7GjYMWLeDwYatTCZev//ia91a8x6Mhj/JwiOxy7U8cU9Bdo/OzZ83XZ8/KKN1K2bLBO+9AZKRZp16nDqxcaXUqsXL/SgbMHUDzss358L4PrY4jsphjCvrlo3MXGaVbr2tXWLXKNKZu2tRs9KW11an80/7T++kY2ZFS+Uoxvct0sgdmtzqSyGKOKOhXj85dZJRuD9WqwZo1cPfd8PDDZrOvCxesTuVfzl06R4fIDpy7dI7o7tEUyl3I6kjCAo4o6LNnw759KR/bt88cF9YqWBDmzjXNMr780ozWDxywOpV/0FrTd05f1h9eT0SnCKoWqWp1JGERR6xDL1PG3NiS1nFhvcBAePNNM5/eu7f5OH06NGlidTLf9ubSN5m2eRrv3v0u91e83+o4jhYcFJzuOnQ7U9qiCc+QkBAdExNjyWuLzLd1K3TsCLt2wejR8Pjjpqep8K6orVF0mtaJXtV7MbnDZNnb3A8opdZqrUNSOuaIKRfhPFWqwOrV0KYNPPmk2ezr/HmrU/mWDf9soNesXjS4pQHj242XYi6koIvMkz+/WYX02mvw9dfQqBHs3Wt1Kt9wNP4o7ae2p2DOgkR1jSJntpxWR9N/yn8AAA6JSURBVBI2IAVdZKqAAHjlFXPBdOdOsx/MokVWp3K2CwkXCIsMIzY+ljnhcygRXMLqSMImpKCLLNG2rVnaWLSo2bFx9GhZr54RWmse/eFRftv/GxPbT6TOzXWsjiRsRAq6yDIVK5qbkDp2hGefhR49ID7e6lTOMmb1GL7a8BUvNX6Jbnd0szqOsBkp6CJLBQebpYxvv222DWjYEP7+2+pUzvDTzp949udn6Vi5I681f83qOMKGpKCLLKcUDBsG8+bB/v1mXv3nn61OZW9/HfuLbjO6cUfRO/i649cEKHnrimvJd4WwzH33ma14S5Uyn48aJfPqKTl5/iTtItoRFBhEdHg0eYPyWh1J2JQUdGGpW2+FFSugWzczau/aFeLs3UMgSyUkJdBtRjf2nNpDVLcoyhSQ26JF6hxx67/wbXnywHffmZZ2Q4bAli1mf54KFaxOZr3nfn6OBX8v4Mt2X9KodCOr44gMyqrWdumO0JVSXymljiqlNqVy/AGl1J9KqY1KqRVKqRo3nEr4HaVg8GAzl37kiCnuP/xgdSprTVg3gTGrx/BU/afoX7u/1XHEDciq1naeTLlMAu5L4/huoKnWuhrwBjDeC7mEn2rZEtauNVMx7drB66/75/bIy/Yu45EfHuGe2+7h/XvetzqOcIh0C7rWeilwIo3jK7TWJ5O/XAXc4qVswk+VKWOaUffqBSNGmHXrp09bnSrr7Dm1h7BpYZQrWI7IzpFkC5CZUeEZb18U7Q/MT+2gUmqgUipGKRUTGxvr5ZcWviRXLpg0CcaONVMv9eubHRx93dmLZ2k/tT2XEi8RHR5NgZwFrI4kHMRrBV0p1RxT0Iemdo7WerzWOkRrHVKkSBFvvbTwUUrBE0/AwoVw8iTUq+fbzUySdBK9ZvVi09FNTOsyjUqFK1kdSTiMVwq6Uqo68CXQXmt93BvPKYRL06ZmXr1qVTP98vLLkJhodSrvG7F4BLO3zeaDez7gntvusTqOcKAbLuhKqdJAFNBLa739xiMJca1bboElS6B/f9MVqV07M2r3FZGbInlz2Zv0q9mPp+o/ZXUc4VCeLFuMAFYClZRSB5RS/ZVSDyulHk4+5RWgEDBOKbVBKSVtiESmyJkTvvgCPvsMfvnFLG3clOJiWmeJORRDnzl9aFS6EePuHyeNKnxQeq3rvNXaTlrQCUdasQI6dTJ3lU6cCF26WJ0oYw7HHabuF3UJDAhkzYA1FM1T1OpIwuakBZ3wOQ0bwrp1UKOG2S5g6FDnzav/m/AvHSI7cOrfU0SHR0sxFzdMCrpwrBIlYPFieOQRePddaN0ajjvkkrzWmgFzB/D7wd/5puM31CguN1iLGycFXThaUBCMGwcTJpiLpiEhsGGD1anS996K9/j2z295vdnrdKzS0eo4wkdIQRc+oV8/WLYMLl0y0zFTplidKHXfb/+eF355ga63d+WlJi9ZHUf4ECnowmfUq2fWq9etCz17wjPPQEKC1amutPnoZnrM7EGtErWY2H6irGgRXiUFXfiUYsXMksannoIPPzQNqY8etTqVcfzccUKnhpInKA9zwueQO3tuqyMJHyMFXfic7NlNMf/mG9OUOiTEdEay0qXES3Se3pmDZw4yq9ssbskne9gJ75OCLnxWz55m18aAAGjUyGz2ZZWnfnyKX/f8yhftvqDBLQ2sCyJ8mhR04dNq1zaj80aNoG9fePxxuHgxazN8uuZTPo35lCENh9CrRq+sfXHhV6SgC59XuDD8+CM89xx88olpovHPP1nz2ot3L+aJ+U9wf4X7GdlyZNa8qPBbUtDFf5KSzG30vtghKFs2eO89iIgwd5jWqWPm1zPTrhO76Dy9MxULVeS7Tt8RGBCYuS8o/J4UdPGfqCiznnvWLKuTZJ7wcFi50mz01aQJjM+kholnLpyhXUQ7AKK7R5MvR77MeSEhLiMFXQBmVD5kiPl8yBDfHKW7VK8Oa9ZAixYwaBAMHAgXLnjv+ROTEukxswfbj29nepfplL+pvPeeXIg0SEEXgBmdHztmPo+N9e1ROsBNN5nWdsOHmy15mzaFgwe989zDFw7nhx0/MLb1WFqUa+GdJxXCA1LQxX+j87Nnzddnz/r+KB0gMBDeegtmzoTNm828+vLlN/ac3/zxDe+ueJeH6zzMo3Uf9U5QITwkBV1cMTp38YdRuktYGKxeDfnyQfPmZiVMRtoErD6wmgFzB9CsbDPGth7r/aBCpEMKup+7enTu4i+jdJeqVc28euvWZq16375w/rzn//2BMwfoENmBm4NvZnqX6WQPzJ55YYVIhRR0Pzd7Nuzbl/KxffvMcX+RP7/5+776KkyeDI0bp/5vc7lzl87Rfmp7zl48y9zucymcu3CmZxUiJdmsDiCsVaYMPP102sf9SUAAjBgBtWpBr15mXn3aNDMVkxKtNf3m9GP94fVEd4/m9qK3Z21gIS4jBd3P1alj/ogrhYbC779Dx45mx8b33jM/+K7e7fatZW8RuTmSUS1H0bZiW2vCCpFMplyESEWlSuZiafv2MHiw2ezr3Dn38VlbZ/Hy4pfpWb0nz9/1vHVBhUgmBV2INAQHw4wZZnljRITphrR7N/x55E96zepFvZL1+KLdF9KoQtiCFHQh0qGUuQHphx9g716o3fgo90wMJX/O/MzuNpuc2XJaHVEIQAq6EB5r3RpWrL7IxY6dOHL2CJ0uzqF43hJWxxLiP1LQhfCQ1poPtj3KucLLufPIJD4aFkK3bteu4RfCKlLQhfDQ2NVjmbB+Ai82fpHfxnfjvffMtgENGsDOnVanE0IKuhAe+XnXzwz+eTAdKnfg9eavo5RpmPHTT6ZZRkgIzJtndUrh76SgC5GO7ce3021GN24vcjvfdPyGAOV+29x9t2lxV64ctG0Lb77pP9slCPuRgi5EGk6eP0m7iHZkC8hGdPdo8gblveacsmVNM+oHHoCXX4ZOneDMmazPKoQUdCFSkZCUQPjMcP4++Tczu86kbIGyqZ6bOzd8/TWMGQNz50L9+rBtW9ZlFQIcWNB9ue+lsJchPw/h510/8+n9n9KkTJN0z1cKnnwSFi6E48ehXj2YMycLggqRLN2CrpT6Sil1VCm1KZXjSik1Vim1Uyn1p1KqtvdjuvlD30thva/Wf8WHqz/kyXpP8lDth67rv23aFNauNVsHdOgAr7wiAxCRNTwZoU8C7kvjeGugQvKfgcCnNx4rZf7U91JYZ/m+5Tz8/cO0urUVH9z7QYaeo1QpWLbM7Kv+xhtms69Tp7wcVIirpFvQtdZLgRNpnNIe+Fobq4ACSqlMuX3O3/peiqy399RewiLDKFugLJGdI8kWkPENSXPmhAkTYNw4+PlnqFvX7AMjRGbxxhx6SWD/ZV8fSH7sGkqpgUqpGKVUTGxs7HW9iL/2vRRZKygwiNolahPdPZqCuQre8PMpBY88AosXmymY4sW9EFKIVGTpRVGt9XitdYjWOqRIkSLX9d/6e99LkTVKBJfgx54/UrlwZa8+7113wfffQ65cXn1aIa7gjYJ+ECh12de3JD/mNdL3Uggh0ueNgh4N9E5e7dIAOK21PuyF5/2P9L0UQoj0pXvFRykVATQDCiulDgAjgOwAWuvPgHlAG2AncA7o6+2Q0vdSCCHSl25B11p3T+e4Bh7zWqIUSN9LIYRIn+PuFBVCCJEyKehCCOEjpKALIYSPkIIuhBA+Qplrmha8sFKxwN4M/ueFgWPpnmUfTsrrpKzgrLxOygrOyuukrHBjectorVO8M9Oygn4jlFIxWusQq3N4ykl5nZQVnJXXSVnBWXmdlBUyL69MuQghhI+Qgi6EED7CqQV9vNUBrpOT8jopKzgrr5OygrPyOikrZFJeR86hCyGEuJZTR+hCCCGuIgVdCCF8hK0Lut0aVKfFg6wPJGfcqJRaoZSqkdUZr8qTZt7LzqurlEpQSnXOqmwpZEg3q1KqmVJqg1Jqs1JqSVbmSyFLet8L+ZVSc5VSfyTn9foOpZ5SSpVSSi1WSm1JzvJUCufY4n3mYVbbvM88yXvZud55n2mtbfsHaALUBjalcrwNMB9QQANgtY2zNgQKJn/e2sqsnuRNPicQWITZIrmzXbMCBYAtQOnkr4va+d8WGA68k/x5EUzP3iCLspYAaid/HgxsB6pedY4t3mceZrXN+8yTvMnHvPY+s/UIXduoQXV60suqtV6htT6Z/OUqTGcny3jwbwvwBDATOJr5iVLnQdYeQJTWel/y+XbPq4FgpZQC8iafm5AV2a4JovVhrfW65M/jgK1c2xPYFu8zT7La6X3m4b8tePF9ZuuC7gGPG1TbTH/MiMe2lFIlgY7Ap1Zn8UBFoKBS6lel1FqlVG+rA6XjY6AKcAjYCDyltba8kaJSqixQC1h91SHbvc/SyHo527zPUsvr7fdZug0uhHcppZpjvtEaWZ0lHR8CQ7XWSWYgaWvZgDpASyAXsFIptUprvd3aWKm6F9gAtABuAxYopZZprc9YFUgplRczSnzayhye8CSrnd5n6eT16vvM6QU90xtUe5NSqjrwJdBaa33c6jzpCAGmJn+TFQbaKKUStNZ27OB6ADiutY4H4pVSS4EamDlLO+oLjNJmAnWnUmo3UBn43YowSqnsmIIzRWsdlcIptnmfeZDVVu8zD/J69X3m9CmXTG9Q7S1KqdJAFNDLxiPH/2ity2mty2qtywIzgEdtWswB5gCNlFLZlFK5gfqY+Uq72of5bQKlVDGgEvC3FUGS5/EnAFu11qNTOc0W7zNPstrpfeZJXm+/z2w9Qlc2aFDtKQ+yvgIUAsYl/zRO0BbuDudBXttIL6vWeqtS6kfgTyAJ+FJrneZyTCvzAm8Ak5RSGzErR4Zqra3a+vUuoBewUSm1Ifmx4UBpsN37zJOsdnqfeZLXq+TWfyGE8BFOn3IRQgiRTAq6EEL4CCnoQgjhI6SgCyGEj5CCLoQQPkIKuhBC+Agp6EII4SP+D6erSXojXA0lAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_onevsall(xs_oa, ys_oa)\n",
    "plt.plot([1.0, 2.4], [1.5, 1.9], c='r')\n",
    "plt.plot([1.0, 1.8], [1.7, 1.0], c='b')\n",
    "plt.plot([1.7, 2.3], [1.0, 1.8], c='g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_against(xs, ys, theta, one, alpha, epoch):\n",
    "    ys_ally = ys.copy()\n",
    "    ys_ally[ys == one] = 0\n",
    "    ys_ally[ys != one] = 1\n",
    "    return gradient_descent(xs, ys_ally, theta, alpha, epoch)\n",
    "\n",
    "def train_thetas(xs, ys, theta, alpha=1, epoch=1000):\n",
    "    thetas = []\n",
    "    for i in np.unique(ys):\n",
    "        thetas.append(train_against(xs, ys, theta, i, alpha, epoch))\n",
    "    return thetas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([  5.03613532,   6.28604462, -15.81831884]),\n",
       " array([  3.21796205, -10.90131786,  12.4557529 ]),\n",
       " array([-9.3864618 ,  6.35954149,  8.46541456])]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_oa = np.random.randn(len(xs_oa[0]))\n",
    "thetas_oa = train_thetas(xs_oa, ys_oa, theta_oa, alpha=1, epoch=1000)\n",
    "thetas_oa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3hUxdfA8e+kUQMohN4VlSKhhA4BRV8UDE1KIioqig0Ve/kh2LGLDRUJXUM3BEQEFQ0dQm/SO0gvIRDS5v1jsmYX07Obu7s5n+fJQ3bvZfdkWU5m5545o7TWCCGE8Hw+VgcghBDCOSShCyGEl5CELoQQXkISuhBCeAlJ6EII4SX8rHriChUq6Nq1a1v19EII4eDQ+UOcSDiBn48f9YPqE+AbYHVImVq7du0prXVQZscsS+i1a9cmLi7OqqcXQoh/fRv3LY///DgBvgH8fv/vtK/Z3uqQsqSUOpDVMZlyEUIUaX/s+4Mh84cAMOauMW6dzHMiCV0IUWTtOr2LPtP7kKpTeantSwxsMtDqkApEEroQokg6e/ksd0XdxdnEs3S/sTvvdX7P6pAKTBK6EKLISUlLod/Mfuw8vZObK97MlF5T8PXxtTqsApOELoQocp5d8Cy/7f2NiqUqMjdiLoHFAq0OySkkoQshipTRa0bz1ZqvCPAN4Kf+P1GrXC2rQ3IaSehCiCLjt72/8fQvTwMwNmwsbWu0tTgi55KELoQoEnae3knfGX1J1am80u4V7gu+z+qQnE4SuhDC6529fJa7fryLc4nn6HlTT97t/K7VIblEjgldKVVDKbVYKbVNKbVVKfVMJucMUEptUkptVkotV0oFuyZcIYTIm+TUZPrO6MuuM7sIrhTM5F6T8VHeOZbNzdL/FOB5rfU6pVQgsFYptUhrvc3unH1AR631WaXUncAYoJUL4hVCiDwZumAov+/7nYqlKhITEUPpgNJWh+QyOSZ0rfUx4Fj69/FKqe1ANWCb3TnL7f7KSqC6k+MUQog8+3r114yOG00x32JE94+mZtmaVofkUnn63KGUqg00BVZlc9og4Jcs/v5gpVScUiru5MmTeXlqIYTIk0V7FvHMAjNDHNk9kjY12lgckevlOqErpUoDs4ChWusLWZxzCyahv5zZca31GK11iNY6JCgo0+6PQghRYH+f+vvfipbX2r/GgMYDrA6pUOSqfa5Syh+TzH/QWs/O4pzGwFjgTq31aeeFKIQQuXfm8hnCosI4f+U8vW7qxdu3vm11SIUmN1UuCogEtmutP83inJrAbOA+rfVO54YohBC5Y6to2X1mN00qN2FSr0leW9GSmdyM0NsB9wGblVIb0u97DagJoLX+FhgOlAdGm/xPitY6xPnhCiFE5rTWPPXLU/yx7w8qlapETLh3V7RkJjdVLksBlcM5DwMPOysoIYTIq69Wf8V3a7+jmG8x5oTPoUbZGlaHVOiKzmcRIYTXWrhnIUN/HQrAuB7jaFW9aC6DkYQuhPBo209up9+MfqTpNIZ1GMY9N99jdUiWkYQuhPBYpy+d/rei5e76d/PmLW9aHZKlJKELITxSUmoSfWb0Yc/ZPTSt3JSJPScWqYqWzBTtn14I4ZG01gyZP4Q/9/9J5dKViYmIoVRAKavDspwkdCGEx/li1Rd8v+57ivsVZ074HKqXkfZRIAldCOFhFuxewHMLnwNgfI/xtKzW0uKI3IckdCGEx9h2chv9Z/YnTacxPHQ44Y3CrQ7JrUhCF0J4hFOXThEWFcaFKxfo06APIzqNsDoktyMJXQjh9pJSk+gzvQ97z+6leZXmUtGSBXlFhBBuTWvNEz8/wV8H/qJK6SrMCZ9DSf+SVoflliShCyHc2qiVo4hcH/lvRUu1MtWsDsltSUIXQrit+bvm88KiFwCY2HMiLaq1sDgi9yYJXQjhlrae2Er4zHDSdBojOo6gX8N+Vofk9iShCyHcjq2iJT4pnn4N+zG843CrQ/IIktCFEG4lKTWJ3tN6s+/cPkKqhjC+x3ipaMkleZWEEG5Da83j8x5nycElVA2sKhUteZSrTaKFcLn4eJg2DXbtgnr1oH9/CAy0OipRyD5b+RnjNoyjhF8J5oTPoWpgVatD8ig5JnSlVA1gElAJ0MAYrfXnV52jgM+BrsAl4AGt9Trnhyu80tKl0LUrpKVBQgKUKgXPPQfz50P79lZHJwrJvJ3zeGFhRkVLSFXZljivcjNCTwGe11qvU0oFAmuVUou01tvszrkTqJf+1Qr4Jv1PIbIXH2+SeXx8xn0JCebPrl3h6FEoXbQ2+i2KtpzYQsSsCDSaNzu9Sd+Gfa0OySPlOIeutT5mG21rreOB7cDVlf09gEnaWAmUU0pVcXq0wvtMm2ZG5plJSzPHhVc7mXCSsKgwLiZdJLxROK+Hvm51SB4rTxdFlVK1gabAqqsOVQMO2d0+zH+TPkqpwUqpOKVU3MmTJ/MWqfBOu3ZljMivlpAAu3cXbjyiUF1JuULv6b3Zf24/Laq2YFz3cZgZXJEfuU7oSqnSwCxgqNb6Qn6eTGs9RmsdorUOCQoKys9DCG9Tr56ZM89MqVJw/fWFG48oNFprHvv5MZYeXEq1wGrMCZ9DCf8SVofl0XKV0JVS/phk/oPWenYmpxwBatjdrp5+nxDZ698ffLJ4G/r4mOPCK32y4hMmbJhACb8SxETEUCVQZmkLKseEnl7BEgls11p/msVpMcD9ymgNnNdaH3NinMJbBQaaapbAwIyReqlSGffLBVGvNHfHXF5a9BIAk3tNplmVZhZH5B1yU+XSDrgP2KyU2pB+32tATQCt9bfAfEzJ4m5M2eKDOT2o1vkJV3il9u1NNcu0aWbO/PrrzchckrlX2nx8M/fMvgeN5u1b3ubuBndbHZLXUNqizFqyZIiOjY0jREpNhSgyTiScoOX3LTlw/gARjSL4ofcPchE0j5RSa7XWmWZOy5b+X74MrVrByy+b74UQ3u1KyhV6TevFgfMHaFWtFZHdIyWZO5llCb1SJfPnhx9CcDDExloViRDC1bTWDJ43mOWHllO9THWiw6OloiUvjh2D6dPhqaeyPc2yXi7Vq8OcOfDQQ7BtG3TsCE88ASNHQpkyVkUlhHCFj5Z/xKSNkyjpX5KY8Bgql65sdUjuS2vYuxeWLDEj3SVLcr0ew7I59JCQEB0XF8eVK/Dee+YrJQVq1IDvvoM777QkLCGEk8XsiKHn1J5oNLP6zaJ3/d5Wh+Re0tJgyxaTuG1J/NhVRYKlSkHbttChA2r48Czn0C1P6DabNsGgQWC767774LPPoHx5S8ITQjjBpuObaBvZloTkBN699V1e6/Ca1SFZLykJ1q3LGH0vXQrnzjmeU748dOiQ8dW0KfiZCZXsLoq6TUIHM0IfNQpefx0SE6FiRfjqK+jTB+TaiRCe5fjF47Qc25KD5w8y4OYBTO41uWheBE1IgJUrM0bfK1f+txKkRg0IDc1I4DfdlOWCO49J6Da7dsHDD2dcKO3ZE0aPhiqykEwIj5CYksitE29lxeEVtK7emsUDF1Pcr7jVYRWOM2fMqNs2hbJ2rRmt2rvppozkHRoKtWrl+uGzS+huucFFvXqweDGMGQMvvQTR0fDnn/Dpp/DAAzJaF8Kdaa0ZPHcwKw6voEaZGvzU/yfvTuZHjjhewNyyxfG4jw80a5YxAm/f3kw/uIBbjtDtHToEjz1mVoED3HabSfR16rg4QCFEvry/9H1e/f1VSvmXYtlDywiuHGx1SM6jtak4sSXv2FjYt8/xnGLFoGXLjNF3mzZOLd3zuBG6vRo1YN48+PFHeOYZ+O03aNTIlDc++ST4+lodoRDCJvrvaF79/VUApvSe4vnJPDUVNm92HIEfP+54TmAgtGuXMYXSogUUt+YTiduP0O2dOGHq6qdPN7fbtoWxY6F+fRcEKITIkw3/bKD9uPYkJCcwsvNIXmn/itUh5d2VK6bUzjb/vWwZnD/veE5QkOP8d+PG/1agFAaPHqHbq1jR9G+KiDCLkJYvhyZNYPhwM9fu7291hEIUTf9c/IfuUd1JSE7gvsb38XK7l60OKXcuXoQVKzJG36tWmRI7e7VqOVag3Hij217I86gRur1z5+CFFyAy0txu0sR830y6cApRqBJTEuk0oROrjqyiTfU2/DHwD/e9CHrqVEYFSmwsrF9vplXsNWiQMfru0MHM+7oRrxmh2ytXzky3RETAI4/Ahg3mOsSLL5oRewlpEyGEy2mteTjmYVYdWUXNsjXdr6Ll0CHH+e9t2xyP+/qaOW/b6Lt9e6hQwZpYncBjE7pN587mmsWwYfD55/D++zB7thmtt29vdXRCeLeRS0fyw+YfKOVfirkRc6lUupJ1wWgNO3Y4LqE/cMDxnOLFTZtX2+i7TRuv6rvv8QkdTJuDzz6Dfv1M+4Dt282/1ZNPmmqYwECrIxTC+8zePpv//fE/FIof7/6RxpUaF24AqamwcWPG6HvJErh68/kyZczIzjYCDwkxZYVeymPn0LNy5Qq8844ZqaekQM2apm69SxenP5UQRdb6Y+tpP749l5Iv8cFtH/BSu5dc/6SJibBmTcboe/lyiI93PKdSJcf575tv9rraZo9b+u8MGzea0frateb2wIFmpem117rsKYuu+HhTfrRrl1nm27+/NR+L3CUOL3cs/hgtx7bk8IXDDAweyPge413To+XCBZO0baPv1avNiM1e3bqOJYTXX++2FSjOkl1CR2ud7RcwDjgBbMnieFlgLrAR2Ao8mNNjaq1p3ry5drXkZK0/+EDrYsW0Bq0rVdJ65kyXP23RsmSJ1oGBWpcqZV7kUqXM7SVLimYcXu5S0iXd8vuWmjfQ7SLb6cTkROc9+PHjWs+apfXQoVo3a6a1j4/5t7T/atRI6yee0DoqSuvDh5333B4EiNNZ5NUcR+hKqVDgIjBJa90ok+OvAWW11i8rpYKAHUBlrXVSdo/r6hG6vZ07TbOvJUvM7d694euvobL02C+Y+HioVu2/H3vBjIyPHi2cC07uEoeX01ozYPYAorZEUatsLVY/spqKpQrQk+TAAcf577//djzu5wfNm2eMvtu1k4/YFLBsUWsdq5Sqnd0pQKAyn7lKA2eAlGzOL3Q33GCae337rdnDdPZs0/zr00/NVIyXf0JznWnTTHP+zKSlmeODBhWdOLzcu0veJWpLFKUDSjM3Ym7ekrnWplrBvoTw0CHHc0qUgNatM+a/W7c2FQ8i15xR5fIVEAMcBQKB/lrrTP93KaUGA4MBatas6YSnzj0fH7O6tFs30+xrwQJ48EGYOtXskJSH7pXCZtcu0+s5MwkJud42y2vi8GKzts3i9cWvm4qW3j9yc6Wbs/8LKSlm0Y5t9L1kCZw+7XhOuXIZFSihoWZVYECA636IIsAZCb0LsAG4FbgOWKSUWqK1vnD1iVrrMcAYMFMuTnjuPKtVy3RunDIFhg6FX3+Fhg1NVcwTT2TZU15kpl49M4LKLJmWKmUuUBWlOLzUumPruO+n+wD48PYPCbsx7L8nXb5sLlraRt8rVphl9faqVHFcQt+okfyHc7JcVbmkT7nMy2IO/Wfgfa31kvTbfwCvaK1XZ/eYhTmHnpXjx2HIEJg509xu184sSLrxRkvD8hzuMnftLnF4oWPxx2jxfQuOxB/hwSYPEtk90lS0nD9vGlfZRt9r1pit1exdf71jCWHdujK/6QSuXvp/EOgMLFFKVQJuBPY64XFdrlIlmDHDzKk/+aR5fwYHw4gRpk+MNPvKQWCg+bjTtauZq05IMCNiHx9zf2ElUXeJw8tcTr5Mj6k9OBJ/hPaVW/HNldtRzzxjEvjGjWZe3EYp85/Hfh9M2WKs0OWmyiUK6ARUAI4DIwB/AK31t0qpqsAEoAqgMKP1KTk9sTuM0O2dPQvPPw/jx5vbTZua0XrTptbG5REuXjQXHnfvNqOy/v2tSaLuEoen0xq9dy8Rs+9h2qXV1L7ox+rRKQRdsjvH39+surSNvtu1M3PiwuWK5MKi/Fq4EAYPNhVVvr6mKub11y3rVy+E66WlmaZVdiWEb11/hBG3QOkrsCISGl0saTYgsI2+W7WCkiWtjrxIkoSeRxcvwv/+B19+aT5V3nSTGa23bWt1ZEI4QXIyrFuXMf+9dKnZ2DjdjAamL5LSMLfEQ3S75VHzUVXmIN2CV7bPdaXSpU3nRluzr7//NtVVTz0F774rn+KFh7l0CVauzEjgK1aY++xVqwahocS1rsnAC59DaiIfd/mEbm2esyZmkS8yQs9BYiK8/TZ88IFp7la7tmn2dfvtVkcmRBbOns2oQImNNQ2NkpMdz7nhBscSwtq1ORJ/lJZjW3I0/igPNXmIsd3HuqZHiygQmXJxgvXr4aGHzEYaYBYlffIJXHONtXEJwdGjjgt4Nm/+bwVKkyYZJYTt25sSLzuXki/RcUJH4o7GEVorlEX3LSLAVxb5uCOZcnGCpk3NuomPP4Y33zTVML/8AqNHQ69eVkcnigytYc8exyX0e/Y4nhMQYHbhsY3A27aFsmWzfMg0ncYD0Q8QdzSOOuXqMKvfLEnmHkoSeh74+8Orr5oE/vDD5lNt797Qt6+5gFrJws1ahJdKSzMjbvsR+LFjjueULp1RgRIaapJ5HvZgfOuvt5ixbQaBAYHMjZhLhZKeuwVbUSdTLvmUlmZG56+8YtaxXHstjBoF994ri+FEASQlmTlv2+h72TKzI7q9ChUcF/A0aWI6E+bDtC3TCJ8Vjo/yYV7EPO6sd6cTfgjhSjKH7kL798Ojj5r6dYA77zRdHQu595jwVAkJpurENvpeudL0RbFXs6bjEvqbbnLKqGHNkTWETgglMSWRz7p8xtDWQwv8mML1ZA7dhWrXNp0bJ06EZ5818+oNG5qqmMcek95D4ipnzpi6b9sIfN0605nQ3k03OVaguKAV6JELR+gxtQeJKYk83PRhnmn1jNOfQxQ+GaE70T//mJ4ws2eb2x06wNixpkJMFFGHDzvuQr91q+NxHx9zxd2+AiUoyKUhXUq+RIfxHVh3bB0da3Vk4X0L5SKoB5EReiGpXBlmzTJfTz5p/g8HB5uqmOeey/c0p/AUWpve7Pa78Ozb53hOsWJm2bxt9N2mjdmZvpCk6TQGRg9k3bF11L2mrlS0eBkZobvImTMmiU+caG43b27aBwQHWxuXcKLUVNi0ybEC5fhxx3MCA03jKtsUSosWJqlbZMTiEbwV+xZlipVhxaAVNAhqYFksIn9khG6Ba6+FCRMgIsI0+1q71jSne+UVGDbM0v/TIr+uXIG4OMcKlAtX7eMSFJSRvENDoXFj0+XNDURtjuKt2LfwUT5M6zNNkrkXkhF6IYiPN/XrX39tbtevb0brbdpYG5fIQXx8RgVKbKxZWZaY6HhO7dqOFzBvuMEt61ZXH1lN6PhQrqReYVSXUTzTWi6CeioZoVssMBC++sq05374YbNXbrt28PTTptmX7IPrJk6eNBUotumT9evNtIq9hg0dSwirV7cm1jw4fOEwPab24ErqFQY3G8zTrZ62OiThIjJCL2SJieYi6UcfmVxRpw58/z107mx1ZEXQwYOOS+i3b3c87utrNi62Je/27aF8eWtizaeEpAQ6jO/A+n/W06l2JxbeuxB/X2mD68lkhO5GiheHkSNNu4CHHjI7ed12m2nT+/HHsumLy2gNO3Y4VqAcOOB4TvHi0Lp1xgi8dWuP7pWcptO4P/p+1v+znuuuuY6ZfWdKMvdyktAt0qyZ2Vf3o4/MiD0y0mx/+c030KOH1dF5gZQU89vSNgJfutRMqdgrW9aMum3z382be9XV6hGLRzB7+2zKFivLvHvmUb6kZ326EHmXmz1FxwF3ASe01o2yOKcTMAqz1+gprXXHnJ64qE65ZGb7djNCX7HC3O7fH774AipWtDYuj5KYaC5a2kbfy5aZrafsVa7sOP/dqJHbVKA424+bf2TA7AH4Kl/mD5jP/133f1aHJJykQL1clFKhwEVgUmYJXSlVDlgO3KG1PqiUqqi1PpFTUJLQHaWmmiqYV181m8mUL292TbrnHrcsmrDehQuwfHnGFMrq1aaxlb26dR1LCK+7rki8mKsOr6LjhI5cSb3CF3d8wVOtnrI6JOFEBZpD11rHKqVqZ3PKPcBsrfXB9PNzTObiv3x9TdVLWJipW//tN9O5MSrKTMPUqGF1hBY7ccJxCf3Gjablpb2bb3YcgVetak2sFjp0/tC/FS2PNX+MIS2HWB2SKETOmEO/AfBXSv0JBAKfa60nZXaiUmowMBigprQjzFSdOqZz4/jxZqXpzz+bSrkPPzSJvkg0+9LaXLC0v4C5Y4fjOX5+0LJlxvx3u3ZmNVcRdjHpImFRYRxPOM6tdW7lizu/kC3kiphclS2mj9DnZTHl8hUQAnQGSgArgG5a653ZPaZMueTs6FHTEyY62tzu2NGUONarZ21cTpeWZi4k2I/ADx92PKdECbMSyzb6btVKCvjtpOk07p5+N9F/R3P9tdez6uFVXFuiaP+C81auLls8DJzWWicACUqpWCAYyDahi5xVrWo6N86cCUOGwF9/mZXkb78NQ4d6cLOvlBSzaMc2Al+6FE6fdjynXLmM0XdoqCkL8peSu6y8/sfrRP8dbSpaIuZJMi+inJES5gBfKaX8gACgFfCZEx5XYK7h9e0Lt95q+q1PngwvvgjTpsG4cWba2O1dvgyrVmWMvlesMBs72Kta1XH+u2HDIjK/VHBTNk3hvaXv4at8mdF3BjdWuNHqkIRFckzoSqkooBNQQSl1GBiBKU9Ea/2t1nq7UmoBsAlIA8Zqrbe4LuSiqXx5mDTJNPt69FHTI6pZM3jtNfPlVuXT586ZskHbFMqaNZCc7HhOvXqOI/A6dYpEBYqzrTi0gkExgwD4/I7Puf262y2OSFhJlv57oAsXTNfGb74xtxs2NAuTWrWyKKB//nFcQr9pk7mwaaOUmSuyX0JfpYpFwXqPA+cO0HJsS04knOCJkCf4utvXVockCoHsKeqlYmNNs69du8zsxNChZn69ZEkXPqnWZtMG+wqUXbscz/H3N32/baPvtm2lp4GTXUy6SLtx7dh0fBOd63TmlwG/yLL+IkISuhe7fBneeMP0gUlLM2tpxo6FW25x0hOkpZlt0+xH4EePOp5TqtR/K1BKlHBSAOJqaTqN3tN6M2fHHG4ofwMrB63kmhLXWB2WKCTSnMuLlShhNqS2NfvavNlcQH3kEdMnpmzZPD5gcrLZjcM2+l66FM6edTynfPmMHiihodCkiVSgFKL//f4/5uyYQ7ni5ZgbMVeSufiXjNC9SFKSWYD09tvm+6pV4dtvzerTLF26BCtXZoy+V64099mrXt1xE4f69aUCxSKTNk5iYPRAfJUvC+5dwG11b7M6JFHIZMqliNm2zYzWV60ytyMiTF+YoCDMaNu2iUNsrBmNp6Q4PsCNNzqWENaqJRUobmD5oeXcMvEWklKTGN11NI+3eNzqkDJVZmQZ4pPiszweGBDIhVcvZHlcZE+mXIqYBg1M1eCXX8KXrx4lNWoJc3+KpXfQEsoe3oKy/yXu42PqH22j7/btoVIl64IXmTpw7gA9p/YkKTWJIS2GuG0yB7JN5rk5LvJPEro30Rp274YlS/BdsoShsbEMTdxrjiUChyDZJ4C0kJYUu82uAqVMGUvDFtmLvxJPWFQYJy+d5Pa6t/PZHbJuT2ROEronS02FLVscSwj/+cfxnNKl0e3asbZEB15f2IHFl1pSbEdxPnoYHukiMynuLjUtlXt/upfNJzZzY/kbmdZnGn4+8t9WZE7eGZ4kKcksEbXNfy9bBufPO55ToYLj/HdwMMrPjxBg7BF44gmIiTGrTadONc2+rrvOkp9G5MJrv79GzI4Yril+jVS0iBxJQndnFy/+twIlMdHxnFq1HJfQ33hjlsPuatVM58bp0+Gpp2DxYtML5p134JlnvHbzHo81YcMEPlz+IX4+fszsN5N65b2tzaZwNkno7uT0accKlHXrzLSKvfr1HUfgeewrr5TZ4q5zZ7Oy9Icf4PnnTbOvyEizK5uw3tKDSxk8dzAAX935FbfWudXiiIQnkIRupUOHMua+lywxKzLt+fpCSIhjBUpQkFOeukIFmDIlo9nX6tWm2OV//zPb4AUEOOVpRD7sP7efXtN6kZyWzNMtn+bRkEetDkl4CEnohUVr2LnTcQn9/v2O5xQrZpbN20bfbdpAYKBLw+rWzfweefll+O4700Zg5kzTmrdFC5c+tciEraLl1KVTdLmuC590+cTqkPIsMCAwxzp04RqysMhVUlNN10H7CpQTV223WqaM2TrNNoUSEmJpH9w//zTNvvbsMeXpzz0Hb77p4mZf4l+paan0nNaTeTvncVOFm1gxaAXliktTM+FIFhYVhitXTN9v2wh8+XLT59ZexYqOS+gbN3arK5GdOpnfQSNGwKefmoZf0dGm2VfHjlZH5/1e+e0V5u00uw3NjZgryVzkmST0/IqPN0nbNvpetcokdXt16jhewKxXz+0Lv0uWNE29+vaFQYNMmXunTvDYY6YJmKxBco3x68fz8YqP8fPxY1a/WVx/7fVWhyQ8kEy55NbJk6YCxTaFsn69aS1rr1GjjNF3hw6mqZUHS0qCkSPh3XdNE8bq1U2zr27drI7Muyw5sITOkzqTnJbMmLvG8EjzR6wOSbgxac6VHwcOOO5C//ffjsd9faF584wReLt2pq2sF9qyxYzWV682twcMgFGjTKWMKJh9Z/fRcmxLTl06xTOtnmHUHaOsDkm4OUnoOdHaJGz7CpSDBx3PKV7cVJ3YRt+tW0Pp0tbEa4HUVNOxcdgws6lGUJBp/tWvn9vPIrmtC1cu0DayLVtPbuWO6+9gbsRcWdYvclSghK6UGgfcBZzQWme57EQp1QJYAYRrrWfmFJSlCT0lBTZscKwBP3XK8Zxy5cyo2zb/3by5FGdjen898oipiAHo3t3sbVq1qqVheZzUtFS6T+3O/F3zqV+hPisGraBs8bzuRiKKooJWuUwAvgImZfMEvsAHwML8BOhyiYlmvsA2+l6+3Cyrt1eliuMS+kaNZBOHTFx/Pfz+u6l8efFF0xfmr7/gk09MD3YZrefOS4teYv6u+f9WtE1rag8AABsvSURBVEgyF86QY0LXWscqpWrncNpTwCzAPZainD+fUYESG2vKCZOSHM+57jrHCpTrrpNslEs+PjB4MHTtCo8/DvPmmfr1H380zb7q1rU6QvcWuS6ST1d+ip+PH7P7zea6a6U7mnCOAk/YKaWqAb2AW8ghoSulBgODAWrmsQdJto4fd5w+2bjRsQJFKVPzbV+BInMEBVa9uhmhT50KTz8Nf/xhmn29+65p/uVGJfZu46/9f/H4z2Zzim+7fUvH2lLgL5wnVxdF00fo8zKbQ1dKzQA+0VqvVEpNSD/PdXPoWpsl8/YXMHfudDzHz8+surSNvtu1g2uk7agrnTxpOjZGRZnbrVubZl8NGlgblzvZe3YvLb9vyenLp3m29bN82uVTq0MSHsjVK0VDgKnKTFdUALoqpVK01tFOeGwz0t6+PSN5x8bCkSOO55QsmVGBEhpq+qHIevVCFRRkplwiIswipJUroWlTUxXzyivg7291hNY6n3iesKgwTl8+Tdd6Xfno9o+sDkl4oQIndK11Hdv3diP0/Cfz5GSzaMeWvJcuhTNnHM+55hrH6ZNmzSRjuImwMPM79cUXzXz68OEZzb6aN7c6OmukpKUQPiucbSe30SCoAVF3R+HrI/NRwvlyTOhKqSigE1BBKXUYGAH4A2itvy1wBJcumWXztvnvFSsgIcHxnGrVHCtQGjSQChQ3VrYsjBkD4eGmxHHTJmjZEl54wXRzLFHC6ggL14sLX2TB7gWUL1GeuRFzKVNM+icI17BuYVGVKjquTh2zpVpysuPBevUcm1jVqSMVKB4qIcGM0keNMrNn9eqZksfQUKsjKxzfr/2ewfMG4+/jz2/3/0ZorSLygwuXcc+VokrpODCJOjg4Y/Tdvj1UrmxJTMJ1Vq0yderbtpnbTzxh+sR4c7OvP/f/ye2TbyclLYXI7pE81PQhq0MSXsA9E3qVKjouMhLatjWrMoXXu3IF3nvPfKWkQI0aZlONO++0OjLn231mN63GtuLM5TM83+Z5Pv6/j60OSXiJ7BK6dRPR1aqZlSmSzIuMYsXMhhlr15qq0kOHzFvg/vvNdqrewlbRcubyGbrV68YHt31gdUiiiJDmXAWQlgYTJ8LAgXKNNq9SUsy8+uuvm84MFSvCV19Bnz6efbkkJS2Fu368i1/3/Eqjio1Y9tAyuQiaD2VGlslxG7sLr17I8rg3c88RuheYPdvMC//0k9WReB4/P1P1smmTuXRy4oTp3Ni7Nxw7ZnV0+ff8r8/z655fqVCyglS0FEB2yTw3x4sqSej5lJZmaq3B/Hn1Xhcid+rVg8WLTcfGwECz5V2DBjB+vFkU7Em+i/uOL1Z/gb+PPz/1/4na5WpbHZIoYiSh59Ps2Rkdd0+elFF6Qfj4mNWlW7eaOfVz58wnny5dTJcHT7B432KG/DIEgDFhY2hfs73FEYmiSBJ6PthG57YOvBcvyijdGWrUMJ0bp0wxmz8tWmS6GH/xhdlgw13tOr2Lu6ffTUpaCi+2fZEHmjxgdUiiiJKEng/2o3MbGaU7h1Jmi7tt28ycekKCafoVGmpa+ribc4nnCIsK42ziWcJuCGNk55FWhySKMEnoeXT16NxGRunOVbEiTJtmfklWqWLa2zdpkrFhtTtISUuh/8z+7Di9g5sr3swPvX+QHi3CUpLQ8yg6+r/bjdocPGiOC+fp2dOM1gcNMnuUDBtm+sKsW2d1ZPDcr8+xcM9CgkoGERMRQ2CxQKtDEkWc7EibR7VqwdCh2R8XzlWunOn/EhFhmn1t2GCS+osvwogRZv/uwvbNmm/4cvWXBPgGSEWLCwQGBOZYhy7+SxYWCY+SkGBG6Z9/bsoab7jBbKTRvhCLSn7f+ztdpnQhVacysedE7g++v/CeXBR5srBIeI1SpeCzz2DZMqhf32xW1aEDDBkC8YWw1mTn6Z30ndGXVJ3Ky+1elmQu3IokdOGR2rQx+6AMG2ZWnX79tSlx/PVX1z3n2ctn/61o6XFjD97r/J7rnkyIfJCELjxWsWLw9tumpX7z5uai9B13wAMP/HeTq4JKTk2m38x+7Dy9k8aVGjOl9xR8lPz3Ee5F3pHC4wUHmz1MP/jAJPmJE037gFmznPccz/76LL/t/Y2KpSoSEx5D6YDSzntwIZxELooKr7JzJzz8sNnNEODuu00Xx4LsmfL16q8Z8ssQAnwD+HPgn7Sp0cY5wQqX8taOjQW6KKqUGqeUOqGU2pLF8QFKqU1Kqc1KqeVKqeCCBixEft1wA/z5p5lTL13ajNIbNIAJE/LX7GvRnkU8s+AZACK7R0oy9yBFsWNjbqZcJgB3ZHN8H9BRa30z8DYwxglxCZFvPj5mi7stW8yc+tmz8OCDZmekAwdy/zg7Tu2g38x+pOpUXm3/Kvc2vtd1QQvhBDkmdK11LJDlJSat9XKt9dn0myuB6k6KTYgCqVUL5s+HSZPg2mtNBUzDhmYKJqcWDWcunyEsKoxziefoeVNP3rn1ncIJWogCcPZF0UHAL1kdVEoNVkrFKaXiTp486eSnFuK/lIL77jPtA/r0MQuTnnrKNPvasSPzv5OcmkzfGX3ZdWYXwZWCmdxrslS0CI/gtHepUuoWTEJ/OatztNZjtNYhWuuQoKAgZz21EDmqVAlmzDBz6pUrm4VJwcHw/vuOzb601jz9y9P8se8PKpWqREyEVLQIz+GUhK6UagyMBXporb1ou1/hbXr3NqP1Bx+EK1fg1VehVSuzSAng6zVf8+3abynmW4zo8Ghqlq1pbcBC5EGBE7pSqiYwG7hPa72z4CEJ4VrXXAPjxpk59Vq1TDJv0QIihi1k6ALTeS2yeyStq7e2OFIh8iY3ZYtRwArgRqXUYaXUIKXUY0qpx9JPGQ6UB0YrpTYopaS4XHiE//s/Uwnz9NOQes3fTE01FS0D6/yPAY0HWB2eKKCcOjJ6Y8dGWVgkirwzl88Q/GUrDl/eDdt7wfSZPP2UD+++a2rZhXAn0m1RiCwkpybTZ3ofDl/eTXDFJrxw3WR8fXz44gu4+Wazr6kQnkISuiiytNYMmT+ExfsXU7l0ZebeE8NH75ZizRqz3d3+/WZa5qGHzOIkIdydJHRRZH25+kvGrBtjKlr6R1OjbA0AmjaF1avhvfdMs6/x4037ANkEXLg7SejCKdLSTOLzlE2yF+xewLO/PgvA+B7jaVW9lcNxf39T0rhhA7RrB//8Y0oe+/WD48etiFiInElCF04xe7aZmvCEUez2k9vpP7M/aTqN10NfJ+LmiCzPvekmiI2FL780uyXNmGFG65Mn56/ZlxCuJAldFFhamtmwGcyf7jxKP33pNGFRYVy4coG769/NG53eyPHv+PiYLe62bDFz6mfOwP33Q7duZlMNIdyFJHRRYLNnw6lT5vuTJ913lJ6UmsTd0+9mz9k9NKvSjIk9J+apR0vt2rBggZlaKlcOfvnFNPsaPdq9f4mJokMSuigQ2+j84kVz++JF9xyl2ypa/jrwF1VKV2FO+BxKBZTK8+MoZba4277dzKlfvAhPPgmdOpnNNYSwkiR0USD2o3Mbdxylf77qc75f9z3F/YozJ3wO1csUrMtz5cqm0dfMmabx15IlptnXhx9CSoqTghYijyShi3y7enRu426j9F92/cLzC58HYEKPCbSo1sJpj3333abZ18CBkJgIL78MrVvDxo1Oewohck0Susi36OisLwoePGiOW23ria3/VrSM6DiC/o36O/05rr3WbHG3YAHUrAlr10JICLz+uunoKERh8bM6AOG5atWCoUOzP26lU5dOERYVRnxSPH0b9GV4x+Eufb4uXUwlzKuvmj1N33nHTMtERkIb2YpUFAJpziW8UlJqErdPvp3YA7E0r9Kc2AdjKelfstCef8kSePhhc6FUKdPR8d13TS27EAUhzblEkaK15omfnyD2QOy/FS2FmcwBOnQw8+ivvGLq2D//3DT7+v33Qg1DFDGS0IXXGbVyFJHrIynuV5yYiBiqlalmSRzFi8PIkaYvTHAw7NsHt91mRu7nzlkSkvByktCFV/l558+8sOgFACb2nEhI1Uw/mRaqZs1gzRoz5RIQYObUGzSAOXOsjkx4G0nowmtsPbGViFkRpOk03uj4Bv0a9rM6pH/5+8Nrr5lmX23awLFj0LMnhIfDiRNWRye8hST0AvC0DoPe7GTCyX8rWvo37O/yipb8ql/fXDD9/HMoWRKmTTOj9R9+kGZfouBys6foOKXUCaXUliyOK6XUF0qp3UqpTUqpZs4P0z15UodBb3Yl5Qq9p/dm37l9tKjagvE9xqOUsjqsLPn6mqqXLVvMnPrp03DvvRAWBocOWR2d8GS5GaFPAO7I5vidQL30r8HANwUPy/15UodBb6a15vGfH2fpwaVUC6xGdHg0JfxLWB1WrtSpAwsXmjn1smXh559Ns69vv5X3k8ifHBO61joWOJPNKT2ASdpYCZRTSlVxVoDuylM6DHq7T1d8yvgN4ynhV4I54XOoGljV6pDyRCnzKW/bNjOnHh8Pjz8Ot94Ku3ZZHZ3wNM6YQ68G2H9QPJx+338opQYrpeKUUnEnT550wlNbw1M6DHq7eTvn8eIi8zFpUq9JNK/a3OKI8q9qVTNImD4dKlaEv/6Cxo3h44+l2ZfIvUK9KKq1HqO1DtFahwQFBRXmUzuVp3QY9Gabj28mYlYEGs1bnd6iT4M+VodUYEpB375mtH7ffabZ14svmqqYzZutjk54Amck9CNADbvb1dPv80qe0mHQm51IOEH3qd25mHSR8EbhDAsdZnVITlW+PEyaBPPnQ40aEBdnatlHjJBmXyJ7zkjoMcD96dUurYHzWutjTnhct+QJHQa92ZWUK/Se1pv95/bTslpLxnUf59YVLQVx552mEubxx820y1tvQfPmsGqV1ZEJd5Vjt0WlVBTQCaiglDoMjAD8AbTW3wLzga7AbuAS8KCrgnUH7t5h0JtprXl03qMsO7SM6mWqE93fcypa8qtMGbPFXXi4aRmwdSu0bWveg2+/bWrZhbCRbovCY3y07CNe+u0lSvqXZOmDS2lapanVIRWqy5fhjTfMhdK0NKhbF8aOhVtusToyUZik26LweDE7Ynj5t5cBmNxrcpFL5gAlSsAHH5gpl5tvhr17TXnj4MFw/rzV0Ql3IAlduL1Nxzdxz6x70GjeueUdetfvbXVIlgoJMRdK337bNPv6/nvTPmDuXKsjE1aThC7c2vGLxwmLCiMhOYF7br6H1zq8ZnVIbiEgAIYNg/XroVUrOHoUuneHe+4xJbSiaJKELtyWrUfLwfMHaVWtFZHdI722oiW/GjSAZcvgs8/MBdKoKHNfVJQ0+yqKJKELt6S1ZvC8wSw/tJwaZWoQHR5Ncb/iVofllnx9TdXL5s1mTv3UKTNS794dDh+2OjpRmCShC7f04bIPmbRxEiX9SxITEUPl0pWtDsnt1a0Lv/1m5tTLlIF580yzrzFjZLReVEhCF25nzt9zePX3VwGY0msKTSo3sTgiz6GUqVffts2M0C9cgEcfhc6dYc8eq6MTriYJXbiVjf9sZMDsAWg07936Hr3q97I6JI9UrZpZtTx1KgQFweLFptTx008hNdXq6ISrSEIXbuOfi//8W9Fyb+N7eaX9K1aH5NGUgv79zWh9wACzMOn5581K0y2ZblcjPJ0kdOEWElMS6TWtF4cuHKJ19dZ8H/a9VLQ4SYUKMGWKmVOvVg1WrzbNvt58E5KSrI5OOJMkdGE5rTWPzH2ElYdXmoqW/lLR4grdupleMI8+CsnJpo1A8+awZo3VkQlnkYQuLPf+0veZsmkKpfxLMTdiLpVKV7I6JK9VtqzZ4m7xYrjuOjP10rq1af186ZLV0YmCkoQuLPXT9p947Y/XUCh+6P0DwZWDrQ6pSOjUCTZtghdeMLc//hiCg81OScJzSUIXlll/bD33/nQvACM7j6THTT0sjqhoKVkSPvoIVqyARo1g926T6B9/3JQ7Cs8jCV1Y4p+L/9B9ancuJV/i/uD7eandS1aHVGS1bAlr15o5dX9/MyXTsCH8/LPVkYm8koQuCl1iSiI9p/bk8IXDtK3RljF3jZGKFosFBJgt7tatMwn+8GG46y64997/7p8r3JckdFGotNYMihnEqiOrqFm2Jj/1/4lifsWsDkuka9QIli+HTz4x/dd/+ME0+5o2TdoHeAJJ6KJQvbfkPX7c/COlA0ozN2IuFUtVtDokcRVfX3juOXPRtFMn0443PBx69jRteoX7ylVCV0rdoZTaoZTarZT6z/I9pVRNpdRipdR6pdQmpVRX54cqPN2sbbMYtngYCsWPvX+kcaXGVocksnH99fD77/Ddd6bZV0yMGa1HRspo3V3lmNCVUr7A18CdQAMgQinV4KrThgHTtdZNgXBgtLMDFZ5t3bF13B99PwAf3PYBYTeGWRyRyA0fH7PF3datZk79/HnT/Ou228wWeMK95GaE3hLYrbXeq7VOAqYCV9eXaaBM+vdlAflgJv51LP4Y3aNMRcvA4IG80PYFq0MSeVS9uhmh//ijaSXwxx+m2deoUdLsy53kJqFXAw7Z3T6cfp+9N4B7lVKHgfnAU5k9kFJqsFIqTikVd1L2ySoSLidfpue0nhyJP0K7Gu347q7vpKLFQykFERGm2VdEhFlZ+uyz0L69uU9Yz1kXRSOACVrr6kBXYLJS6j+PrbUeo7UO0VqHBAUFOemphbvSWvNQzEOsPrKa2uVqS0WLlwgKMiP1mBioWhVWroSmTc2m1cnJVkdXtOUmoR8Batjdrp5+n71BwHQArfUKoDhQwRkBCs/1Tuw7TN0y9d+KlqBS8kvcm4SFmZH5I4+Yro3Dh0NIiFmkJKyRm4S+BqinlKqjlArAXPSMueqcg0BnAKVUfUxClzmVImzmtpkM/3M4CkXU3VE0qtjI6pCEC5Qta7a4+/13swXepk1mYdLLL5v+66Jw5ZjQtdYpwBDgV2A7ppplq1LqLaVU9/TTngceUUptBKKAB7SWwqaiau3Rtdz/k6lo+ej2j7jrhrssjki42q23mmT+3HPm9ocfmmZfsbHWxlXUKKvybkhIiI6Li7PkuYXrHI0/SovvW3A0/igPNnmQyO6RchG0iFm1Ch56KONC6RNPwMiRppZdFJxSaq3WOiSzY7JSVDjNpeRL9Jjag6PxR+lQswPfdPtGknkR1KqV6QkzfDj4+cHo0aalwC+/WB2Z95OELpxCa82Dcx4k7mgcdcrVYVa/WVLRUoQVK2a2uFu71lwoPXQIunaF+++H06etjs57SUIXTvHWX28xfet0AgMCpaJF/KtxY9Nv/aOPoHhxmDzZtA+YMUPaB7iCJHRRYNO3TueNv97AR/kwtc9UGlZsaHVIwo34+ZmdkTZtgtBQOHEC+vWD3r3h2DGro/MuktBFgaw5soaB0QMB+Pj2j+laT/qyiczVq2f2Mv3mGwgMhOhoM1ofP15G684iCV3k25ELR+gxtQeJKYkMajqIoa2HWh2ScHM+PvDYY6bZV9eucO6cqYjp0gX277c6Os8nCV3ki62i5djFY4TWCmV0t9FS0SJyrUYNmDcPpkyB8uVh0SJTCfPFF9LsqyAkoYs8S9NpPBD9AGuPraXuNXWZ1W8WAb4BVoclPIxSMGCAqVfv3x8SEuCZZ8w8+/btVkfnmSxbWKSUOgkcyOG0CoC772joCTGCxOlsEqdzSZy5V0trnWkZmWUJPTeUUnFZrYhyF54QI0icziZxOpfE6Rwy5SKEEF5CEroQQngJd0/oY6wOIBc8IUaQOJ1N4nQuidMJ3HoOXQghRO65+whdCCFELklCF0IIL2FJQldK3aGU2qGU2q2UeiWT458ppTakf+1USp2zO5Zqd+zqrfCcGeM4pdQJpdSWLI4rpdQX6T/DJqVUM7tjA5VSu9K/BroqxlzGOSA9vs1KqeVKqWC7Y/vT79+glHLpbiO5iLOTUuq83b/tcLtj2b5fCjnOF+1i3JL+frw2/Vhhvp41lFKLlVLblFJblVLPZHKOpe/RXMZo+fszl3G6xfszR1rrQv0CfIE9QF0gANgINMjm/KeAcXa3LxZSnKFAM2BLFse7Ar8ACmgNrEq//1pgb/qf16R/f42Fcba1PT9wpy3O9Nv7gQpu8np2AuYV9P3i6jivOjcM+MOi17MK0Cz9+0Bg59Wvi9Xv0VzGaPn7M5dxusX7M6cvK0boLYHdWuu9WuskYCrQI5vzIzD7lBYqrXUscCabU3oAk7SxEiinlKoCdAEWaa3PaK3PAouAO6yKU2u9PD0OgJVAdVfFkp1cvJ5Zyev7pUDyGKcl700ArfUxrfW69O/jMfv9VrvqNEvfo7mJ0R3en7l8LbNSqO/PnFiR0KsBh+xuHyaLF08pVQuoA/xhd3dxpVScUmqlUqqn68LMUVY/R65/PgsMwozYbDSwUCm1Vik12KKY7LVRSm1USv2ilLI1VXfL11MpVRKTBGfZ3W3J66mUqg00BVZddcht3qPZxGjP8vdnDnG6/fvTz6onzqVwYKbW2r7/Wi2t9RGlVF3gD6XUZq31Hovi8xhKqVsw/2Ha293dPv21rAgsUkr9nT5CtcI6zL/tRaVUVyAaqGdRLLkRBizTWtuP5gv99VRKlcb8Uhmqtb7gyufKr9zE6A7vzxzi9Ij3pxUj9CNADbvb1dPvy0w4V32k1VofSf9zL/An5repFbL6OfLy8xUKpVRjYCzQQ2v9746Odq/lCeAnzMdHS2itL2itL6Z/Px/wV0pVwA1fz3TZvTcL5fVUSvljEtAPWuvZmZxi+Xs0FzG6xfszpzg95v1Z2JP2mE8FezFTKbaLCA0zOe8mzEURZXffNUCx9O8rALtw7QWy2mR9Ea8bjhecVqfffy2wLz3Wa9K/v9bFr2l2cdYEdgNtr7q/FBBo9/1y4A4L46xs+7fG/Mc9mP7a5ur9Ulhxph8vi5lnL2XV65n+2kwCRmVzjqXv0VzGaPn7M5dxus37M7uvQp9y0VqnKKWGAL9irhCP01pvVUq9BcRprW2liOHAVJ3+CqarD3ynlErDfLp4X2u9zRVxKqWiMFe2KyilDgMjAP/0n+FbYD6mimA3cAl4MP3YGaXU28Ca9Id6Szt+LC/sOIcD5YHRymxAkaJNt7hKwE/p9/kBP2qtF1gYZx/gcaVUCnAZCE//t8/0/WJhnAC9gIVa6wS7v1qoryfQDrgP2KyU2pB+32uYBOku79HcxOgO78/cxOkW78+cyNJ/IYTwErJSVAghvIQkdCGE8BKS0IUQwktIQhdCCC8hCV0IIbyEJHQhhPASktCFEMJL/D9MxFWpXoEP6wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_onevsall(xs_oa, ys_oa)\n",
    "\n",
    "def plot_boundaries(xs, thetas):\n",
    "    for i, ti in enumerate(thetas):\n",
    "        xs_ti = np.array([min(xs[:, 1]) - 2, max(xs[:, 1]) + 2])\n",
    "        ys_ti = (-1 / ti[1]) * (ti[0] * xs_ti + ti[2])  # from https://utkuufuk.com/2018/05/19/binary-logistic-regression/\n",
    "        plt.plot(xs_ti, ys_ti, linewidth=2, c=xs_plot_spec[i]['c'])\n",
    "\n",
    "plt.xlim([xs_oa[:, 0].min() - 0.3, xs_oa[:, 0].max() + 0.3])\n",
    "plt.ylim([xs_oa[:, 1].min() - 0.3, xs_oa[:, 1].max() + 0.3])\n",
    "plot_boundaries(xs_oa, thetas_oa)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction\n",
    "\n",
    "To make a prediction we get the hypothesis for each group $h^{(i)}(x)$, the predicted group is then $\\max_i h^{(i)}(x)$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_prediction(x, thetas):\n",
    "    ps = [hypothesis(x, ti) for ti in thetas]\n",
    "    return ps.index(min(ps))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO2dd3gU1deA35uEECABlN5BRWmCQOiIKGJBQocQLIgKlh92RfRTwd4QRRSRDpYA0kFBxUIA6R1EiiBdmpRQAiS53x83a3ZDyiZbZnZz3ufZJ7szd2fOTu6ePXPuKUprjSAIghD4hFgtgCAIguAdRKELgiAECaLQBUEQggRR6IIgCEGCKHRBEIQgIcyqE5csWVJXrVrVqtMLgiC4sO/UPo6cPUJYSBg1S9UkPDTcapEyZc2aNce01qUy22eZQq9atSqrV6+26vSCIAj/MXL1SB797lHCQ8P5+b6faVm5pdUiZYlSak9W+8TlIghCvuaX3b/Q//v+AIxqP8rWyjwnRKELgpBv2XF8B92mdiNFpzCg+QB639DbapE8QhS6IAj5khPnT9A+vj0nkk7Q4boOvN3mbatF8hhR6IIg5DuSU5PpMa0H249v5/rS1/NV568IDQm1WiyPEYUuCEK+4+kFT7Nw10JKFynN3Li5RBWMslokryAKXRCEfMWIVSP4dNWnhIeGMzN2JlWKV7FaJK8hCl0QhHzDwl0LeWL+EwCMiRlD80rNLZbIu4hCFwQhX7D9+Ha6f9udFJ3CwBYDubfevVaL5HUsSywSBMtITIQpU2DHDqheHWJjISo4fKhC5pw4f4L237TnZNJJOtXoxFtt3rJaJJ+Qo0JXSlUCJgFlAA2M0loPyzDmbuAFQAGJwKNa6w3eFzcfIMrGtyxZAu3aQWoqnD0LRYrAM8/A999DS5sllMhc8AqXUi7R/dvu7Ph3B/XK1OPLzl8SooLUOaG1zvYBlAMapD2PArYDtTKMaQ5ckfb8TmBFTsdt2LChFjKweLHWUVFaFymiNZi/UVFmu+A5p0+b6wmXP6KitE5MtFrCdGQueI3H5j2mGYwu/UFpvefkHqvF8Rhgtc5Cr+b4M6W1PqS1Xpv2PBHYClTIMOZ3rfWJtJfLgYqe/9TkMxITjeWYmGgsRzB/HdvPnLFWtjFj4IUXzN/EROtk8YQpU4xlnhmpqWa/HbDzXAgwPlv5GSNWj6BgaEFmxc6icrHKVovkU3J136GUqgrUB1ZkM+xBYH4W7++nlFqtlFp99OjR3Jw6+LGrslmyBCpUgKeegvffN38rVDDbA40dO9IVZEbOnoWdO/0rT1bYdS4EGD/99RNPLngSgLEdxtKsUjOLJfI9bit0pVQkMB14Smt9OosxN2MU+guZ7ddaj9JaR2uto0uVyrT6Y/7Fjsom2CzF6tWNzzwzihSBa67xrzxZYce5EGD8eezP/yJaXmr5EnfXvdtqkfyCWwpdKVUAo8y/1lrPyGJMXWAM0FFrfdx7IuYT7Khsgs1SjI2FkCymfEiI2e8PcnJh2XEuBBD/nv+XmPgYTl04RecanXnjljesFslv5KjQlVIKGAts1VoPzWJMZWAGcK/Wert3Rcwn2EXZOBNslmJUlIlmiYpKV5hFiqRvj4z0vQzuuLDsOBcCBEdEy85/d3JD2RuY1HlS8Ea0ZII7cegtgHuBTUqp9WnbXgIqA2itRwKvAiWAEUb/k6y1jva+uEGMQ6lkDKkLCfGfssmIw1LMTKkHqqXYsiUcPGjuLnbuNJ8hNtY/19fZheXAcW3btTNyRUbacy4EAFprHp//OL/s/oUyRcowp+ccIsPz17VSJgrG/0RHR2vpWJQJZ85Yo2wyIzHRWI+ZRbVERaUrIME9xowxFnlWP5DDhsGDD6Zvs9NcCACGrxjOEwueoGBoQRbdv4gmFZtYLZJPUEqtycpglkxRuxEZ6fqlthKxFL1Lbl1YdpoLNufHv37kqR+eAmBcx3FBq8xzQhS6kD1WuiiCjWB0YdmArUe30uPbHqTqVF6+8WV6Xd/LapEsQ1wuguAvxIXldY6fO06TMU3468RfdK3Zlandpwb9Imh2Lpfg/uSCYCfsEGUTRFxMuUi3b7vx14m/qF+2PhM7TQx6ZZ4T4nKxAim6lH/JzIXVrh189x3MnSvzwU201vT/vj+//f0bZSPLMiduDkXCs4jdz0eIy8XfZFbtz7HIaLdqf4LvkfmQJ4YtH8ZTPzxFRFgEi+5fROMKja0WyW9k53IRhe5PxIcqOCPzIU8s2LmAu765i1SdSnzXeHrW6Wm1SH5FfOh2IdhS6QXPkPmQa/44+gex02JJ1am82urVfKfMc0IUuj8JtlR6wTNkPuSKY+eOERMfw+kLp+lWqxuDWg+yWiTbIQrdn0jRJcEZmQ9uczHlIt2mdmPXiV00LNdQIlqyQK6IP3BU19uyBZKTMx8jRZfyH8FehMtLjVG01jz23WMs2rOIcpHlmN1zNoULFPaysMGBhC36moxRDBERZntEBCQlSSp9fiaYSyt4sXfrx8s/Zuy6sUSERTC752wqFK2Q85vyKRLl4kuyi2KIiIDHHoNatQIvlV7i6L1LsBXh8mL0zvc7vicmPoZUncqUblPoUbuHl4UNPKQ4l1VkF8UQGmqUeaAVX/Ki5SWkEWxFuNyJ3nHj8245soWe03qSqlMZdNMgUeZuID50XxJsUQzB1pJO8A1emPeOiJbEi4n0qN2DV2961ctCBiei0H1JsEUxBGvctJcW74Q0PJz3F1Mu0mVKF3af3E10+WjGdxwvES1uIlfJlwRbFEOw3XGAey3hhNzhwbzXWvPovEdZvHcx5aPKS0RLLhGF7kuCrbpesN1xiAvJN3gw7z9a/hHj1o+jUFghZvecTfmo8n4SOjjIcVFUKVUJmASUATQwSms9LMMYBQwD2gHngPu11mu9L24AEkwNImJjzQJoZgTiHYeXFu+ETMjDvJ+3fR7P/fgcABM7TSS6vLQlzi3uRLkkA89qrdcqpaKANUqpn7TWfziNuROonvZoAnye9leA4IliCLa46WB0IdmJXMz7zUc2Ezc9Do3mtdav0b12dx8LF5zkqNC11oeAQ2nPE5VSW4EKgLNC7whM0iaofblSqrhSqlzae4VgIpjuOKQlXM74Iefg6NmjxMTHcObiGXrW6ckrrV7x6vHzE7lKLFJKVQUSgDpa69NO2+cB72qtl6S9/hl4QWu9OsP7+wH9ACpXrtxwz549nsovCHlHytdmjx9qtV9IvsCtX97Kkr1LaFS+EYvuX0ShAoW8cuxgxSvlc5VSkcB04ClnZZ4btNajtNbRWuvoUqVK5eUQguA9gm3R2pv4YcFYa80j3z3Ckr1LqBBVgdk9Z4sy9xC3MkWVUgUwyvxrrfWMTIYcACo5va6Ytk0Q7E0wuZC8iR8WjD9c9iET1k+gUFgh5sTNoVxUOY+OJ7gX5aKAscBWrfXQLIbNAforpSZjFkNPif9cCBiCZdHam/h4wXjutrkM+GkAAF92/pIG5Rp4dDzB4I6F3gK4F9iklFqftu0loDKA1nok8D0mZHEnJmyxT04HtagmmCAI7uDDBeNNhzfRa0YvNJo3bn6DrrW6eiCo4Ixl1RYLF47WCQmriZZQU0GwHz5aMD5y9giNRzdmz6k9xNWJ4+suX2OcAIK72LKn6Pnz0KSJKZ9x/rxVUgh+RWqmBA4+WDC+kHyBzlM6s+fUHppUaMLYDmNFmXsZyyz0smWj9dGjq0lNNXd3Y8ZAq1aWiCL4Az+EwAk+wEu12rXW3D/7fiZtmETFohVZ1XcVZSPL+kDgIOXQIVi8GBYvRn36aZYWuqUNLj77bDUPPAB/pKUoPfYYvPMOFC1qiUiCr5B473zP+0vf54WFL1C4QGGW9FlC/XL1rRbJvmgNu3YZBZ6QYP46LUIrsJ/LBYzLZe1aePVVCAuDESOgTh2YP99KqQSvE6xldwW3mLNtDgMXDgRMRIso8wykpsLGjfDZZ9CzpzF+rrkG+vSB8eONMi9SBNq2hddfz/ZQlncsKlgQXnsNunY1kWOrV5s783vvhY8+ghIlrJZQ8BipmZJv2Xh4I72mm4iWt255iy41u1gtkvVcvGgsWYf1vWQJnDzpOqZECbjxxvRH/frG6gVjAWeB5QrdQd26sGwZfPwxvPIKfPkl/PADfPopdOsGsnYSwEjNlHzJ4TOHiYmP4eyls9x9/d282PJFq0WyhrNnYfnydBfK8uWXR4JUqmQWER0KvEaNrGvKZ4Mtm0Tv2AEPPWQ+O0CnTsYdU04SyQIT8aHnO5KSk7hl4i0s27+MphWb8mvvX4kIi7BaLP/w77/G6k5bxGTNGkhOdh1To0a68m7VCqpUcfvwAdckunp1+PVXGDUKBgyAWbPgt99g6FC4/36x1gOOYCu7K2SL1pp+c/uxbP8yKhWtxMzYmcGtzA8ccF3A3LzZdX9ICDRokG6Bt2wJpUv7RBRbWujO7NsHjzxivvcAt95qFH21aj4WUPA+XgqBE+zNu0ve5cWfX6RIgSIsfWAp9crWs1ok76G1mb8O5Z2QALt3u44pWBAaN063vps182roXnYWuu0VOphr+M038OSTcPw4FC5swhv/9z8IDfWxoIIguM2sP2fReUpnAGbGzqRTjU4WS+QhKSmwaZOrBX74sOuYqCho0SLdhdKoEUT47o4k4BW6gyNH4PHHYepU87p5c5OQVLOmDwT0Nn5oFCAEMQEwf9b/s56W41py9tJZ3mnzDgNbDrRapNxz4YIJtXP4v5cuhVOnXMeUKuXq/65bNz0CxQ8EjUJ3MGuWSUI6dAjCw00Uz4ABUKCAl4X0FpIlKXhCAMyff878Q+PRjdl3eh/31r2XiZ0mBkZa/5kzJrzOYX2vWAFJSa5jqlRxjUC57jpLF/KCTqGDCdt87jkYO9a8vuEG87yB3apwSoSH4AkBMH+SkpNoPaE1Kw6soFnFZvzS+xf7LoIeO5YegZKQAOvWGbeKM7VqpVvfN95oQgptRMBFubhD8eLG3RIXB337wvr1Zh3i+eeNxV7ILo1PpLO84Ak2nz9aax6a8xArDqygcrHK9oto2bfP1f/9xx+u+0NDjc/bYX23bAklS1ojqxcIWIXuoE0bs2bx8sswbBi8+y7MmGGsdVvcjUqWpOBMbn3hNp8/7yx5h683fU2RAkWYGzeXMpFlrBNGa9i2Ld3/nZAAGfsWR0SYmiMO67tZM8vvcLxJwCt0MC7Fjz6CHj2MsbJ1q/lf/e9/JhrG0rUjyZIMXLy9EJmZL/yZZ7L3hdt4/szYOoP/++X/UCi+6foNdcvU9a8AKSmwYUO69b14MRw96jqmaFFzbR0WeHS0CSsMVrTWljwaNmyofUFSktYvv6x1WJjWoHXlylovWOCTU7nH6dNaR0UZYTI+oqK0Tky0UDghSxYvNv+fIkXM/6pIEfN68eK8HS+v88Cm82ftwbW68FuFNYPR7y15zz8nPX9e64QErd96S+vbb8/8upQpo3W3blp/8onW69ZpnZzsH9n8CLBaZ6FXg06hO1i/XuuGDdP/z717a338uE9PmTXeVg6Cb/GFEh09Ov3/n/FRpIjWY8Zk/V6bzZ+Dpw/qikMragaje8/srVNTU31zolOntJ4/X+uXXtL6xhu1Lljw8mt31VXmyz1mjNbbt2vtK1lsRHYK3Z0m0eOA9sARrXWdTPYXA77C9BgNA4Zorcd79z4i99SrZ2rgDB1qFkknToQFC0yFyq7+bmEoneW9i69jsn2xEOmJL9xG8+f8pfN0mtKJ/af306JSC75o/4X3whOPHHGNQFm//vL/Q506riGEFSp459zBQlaa3vEAWgENgM1Z7H8JeC/teSngXyA8p+P62kJ3Zts28wPv+FHv0kXrQ4f8dnrBm/jDWh0wIHNL2vEYODD3x/TEQrcJqampOm5anGYwuspHVfThM4c9O+Dff2s9aZLWfftqXaPG5dclLEzrJk20fu45refMsfAW217giYWutU5QSlXNbggQpczPdGSaQk/OZrzfufZaU9xr5EjTznLGDFP8a+hQ6N1bin0FDImJZlHROSbbYfW2a+e9mGxfLETGxpoF0MwICTH7bc5bi98ifnM8keGRzI2bS+kiuSgwpbWJVnAOIdy3z3VMoULQtGm6Bd60aXo/U8E9stL0zg+gKllb6FHAr8Ah4AxwVzbH6QesBlZXrlzZT79nrvz9t9Z33JFuBNx+u9kmWMDp08ZyHTDA/D19Ovvx/rJyfbUQaTNfeG6YtmWaZjBaDVZ6zp9zcn7DpUtar1yp9Ycfat2pk9YlSlx+LYsX17p9e63fe0/rZcu0vnDB9x8kCMDTRdEcFHo34CNAAdcAu4GiOR3Tny6XjKSmmju9K69M/14NH651SoplIuU/8qLcfOEK8aZ87pCYaH54Bg40fwMgymnNwTW60JuFNIPRHyz9IPNB585p/dtvWr/+utZt22odGXn5/6dcOa1jY7X+9FOtN2yQL1weyU6heyMOvQ/wbtqJdiqldgM1gJVeOLZPUMq0uLvtNujfH6ZNM0W/Jk82CUnXXWe1hEFOXl0n/ozJ9tVCZGRkQGUGH0o8RIf4DpxPPk+fG/rwbLNnzY5Tp0zhKkf896pVprWaM9dc45pCf9VV4t/0Md5Q6HuBNsBipVQZ4DpglxeO63PKlIFvvzU+9f/9z8zPevVg0CBTJ8a2xb4CnbxGkfjbDx1gytfbnL90no6TO3Ig8QAtyzbh8wttUU8+aRT4hg3G7naglPnyOPfBlBZjfsedsMV4oDVQUim1HxgEFADQWo8E3gAmKKU2YdwuL2itj/lMYh/QpQvcfDM8+6xpsv3SS0bRjx1rerMKXsbdEL7MwhOl85Hv0Rq9axd9ZvRi1blVVD0TxoxnVlDwXK/0MQUKmKxLh/XdooUpsCRYijtRLnE57D8I3OY1iSziiitg3Djo2RP69TNF2Bo1MlExr7zi03r1+Q93XCfZpcnbJCY7aEhNNUWrnFLo37jmAFNuhsgLMHdSMqUoDLc2T7e+mzQxnWYEWxGw5XN9yZkz8H//B8OHm7vKGjWMtd68udWSBQk5lYTdts0sZNi4ZGxAc+kSrF2b7v9essQ0Nk7j21qmLpLSMLfQA9x188PmVlV8kLYgKMvn+pLISFO50VHs688/zRrZ44/DW2+JLvGYnJpGf/edrUvGBhznzpm0aYcCX7bMbHOmQgVo1YrVTSvT+/QwSEliyO0fclezLNYsBFsiCj0bWrQw2cdvvAHvvQeffAJz5pgm1W3bWi2dj/BXq7PsokjmzrV1yVjbc+JEegRKQgKsWWOscmeuvdY1hb5qVQ4kHqTjmMacT0nigRse4OmmT1sjv5BnRKHnQESEscq7dYMHHjAK/rbboE8f+PBD43sPGvJS3tUTsooisXHJWFty8GC69b14sWkQkDECpX799BDCli1NiJcT5y6do9OUThxMPEirKq34vP3ngdFCTnBBfOi54NIlGDIEXnvN9JItWxZGjIDOna2WzAvYqdWZN2UJgObKuUJr+Osv1xT6v/5yHRMeblb0HRZ48+ZQrFiWh0zVqfSc1pNv//iWasWrsbLvSkoWDtyuPcFOdj50tzJFffGwMlPUU7Zu1bpFi/QEuO7dtf7nH6ul8hC7FY/yJFPTUVKgVy9TcrVw4YBLtf+PlBRTC3r4cK179DDZlhn/P5GRWt92m9ZvvKH1okUmazMXDPp1kGYwOurtKL358GYffRDBW+DjTNF8R40axjgaMQIGDjQx6z//DB9/DPfcE6DJcHZrdZbXTE2H2ygl5fKFP18U8vI2Fy8an7fD+l661HREd6ZkSdcEnhtugLC8fZWnbJ7Ca4teI0SFMKXbFGqXru2FDyFYhSj0PBISYsoGtG8PDz8MP/4I990H8fGmqmPlylZLmEvs6LfObaZmZiUFMsNOkTJnz5qoE4f/e/lyOH/edUzlyq4p9DVqeMVqWHVgFffPvh+AD2/7kDur3+nxMQVrEYXuIVWrmsYZEyfC00/D/PlQu7aJinnkEaP4A4IgKO+abUkBZ6yMlPn3X3MX4bDA166F5AzVpmvUcI1AqVLF62IcOH2AjpM7kpScxEP1H+LJJk96/RyC/xGF7gWUgvvvhzvuMDVhHLVhJk+GMWNMhJjtySk23I7uiYxk5zZyxp93HPv3u3ah37LFdX9ICDRs6BqBUqqUT0U6d+kcHSZ34NCZQ9xU5SY+u+sziWgJEkShe5GyZWH6dPP43//Md7hePRMV88wzeXZz+g8btTrLE9m5jZzx1R2H1uZHxbkL/e7drmMKFjRp8w7ru1kz05neT6TqVHrP6s3aQ2u56oqrmN5jOuGh4X47v+BbJGzRR/z7r1HiEyea1w0bmvIB9epZK1dQk124I5jaI6Gh3ourT0mBjRtdY8APH3YdExVlMtQcLpRGjYxSt4hBvw7i9YTXKVqwKMseXEatUrUsk0XIG9mFLYpC9zE//GCKfe3dayz0gQPh5Zct/U4HN5klR6WkmM7gN9/s2R3HhQuwerVrBMrp065jSpVKV96tWkHduuZHxAbEb4qn14xehKgQvuv1HXdcc4fVIgl5QBS6xSQmwosvwmefmdc1axprvVkza+UKWs6c8Y7bKDExPQIlIQFWroSkJNcxVau6LmBee60t41ZXHlhJq/GtuJBygY9v/5gnm8oiaKAiCt0mLF4MDz0E27eb7/wTT5iyAtIH1yYcPWosfIf7ZN06Y907U7u2awhhxYrWyJoL9p/eT6PRjfjnzD/0a9CPke1HyiJoACMK3UYkJZlF0g8+MLqiWjUYPRratLFasnzI3r2uKfRbt7ruDw2FBg3SlXfLllCihDWy5pGzF89y4/gbWffPOlpXbc2P9/xIgVApgxvISPlcGxERAe+8A927m2JfGzbArbeaHJchQ6TpS67ITZ0WrU2ddecIlD17XMdEREDTpukWeNOmgRPhkwmpOpX7Zt3Hun/WcfUVVzOt+zRR5kGOWOgWcumSsdRfe81kfJcrB59/Dh07Wi1ZAJDZ4qcjZr5lS5Oss2FDugW+ZIlxqThTrJgZ6/B/N2wYVKvVr/zyCm8ufpNiBYux/KHl1ChZw2qRBC/gkctFKTUOaA8c0VrXyWJMa+BjTK/RY1rrm3ISShR6Olu3Ggt92TLzOjbW1F4vXdpauWxLduGJ4eHGul6+3CyOOlO2rKv/u04d20SgeJtvNn3D3TPuJlSF8v3d33Pb1QHfJVJIw1OXywTgU2BSFgcvDowA7tBa71VKiRrKJTVrGkPys89MNMyUKbBwoema1KuXLYMmrGXKlMsXKx1cvGguHsBVV7mGEF59db64mCv2r+CB2Q8A8NHtH4kyz0e40yQ6QSlVNZshvYAZWuu9aeOPeEe0/EVoqIl6iYkxcesLF5rKjfHxxg1TqZLVElrMkSPpvu/4+MsrKTrTsaMphVm+vP/kswn7Tu2j4+SOXEi5wCMNH6F/4/5WiyT4EW8sil4LFFBK/QZEAcO01llZ8/2AfgCVA64coX+oVs1Ubhw/3mSafvediZR7/32j6AOm2JcnaG0WLJ0XMLdtc++9RYqYX0W7KHM/Ntg4c/EMMfExHD57mFuq3cInd34i4Yn5DLcWRdMs9HmZ+dCVUp8C0UAboBCwDLhLa709u2OKDz1nDh40NWFmzTKvb7rJhDhWr26tXF4nNdUsJDgXsdq/33VMoUImE6tVK7N4GRd3uY8c/N9dKTtyWrj1Iqk6la5TuzLrz1lcc+U1rHhoBVcWutKr5xDsga/DFvcDx7XWZ4GzSqkEoB6QrUIXcqZ8eVO5cdo0U3t90SKTSf7GG/DUUwFQ7CsrkpNN0o7DAl+yBI4fdx1TvHh69EmrViYevIBTyN38+fauDJlZbXYfNth45ZdXmPXnLIoVLMa8uHmizPMp3lAJs4FPlVJhQDjQBPjIC8fNf2Rye66ioujeHW65xdRb//JLeP55M2zcOLj+equFdoPz52HFinTre9myyysili/vGoFSu3b2/iW7V4bMrja7lxtsfLXxK95e8jahKpRvu3/LdSWv88pxhcAjR4WulIoHWgMllVL7gUGY8ES01iO11luVUguAjUAqMEZrvdl3Igcpmd2eP/PMf7fnJUrApEnG0/Dww6ZGVIMG8NJL5mGr8OmTJ03hKocLZdUqE3TvTPXqrhZ4tWq5j0DJbUcjf+Knln7L9i3jwTnmGgy7Yxhtr27rleMKAUpWzUZ9/QjkJtFe5/Rp07w4swbNUVFaJya6DD91SutHH00fUru21suXWyS71lofOqT11Kla9++vdb16Wivl+hmUMtsff9yMO3jQQmH9hB+abv994m9d+oPSmsHox+Y95gWhhUCAbJpES6aoHRgzxjjFs+rnOWxYppZoQoIp9rVjh/FOPPWU8a8XLuxDWbU2TRucI1B27HAdU6CAqfvtsL6bN89/NQ2yS37ywsLtmYtnaDGuBRsPb6RNtTbMv3u+pPXnE6SWi93J4+15q1Ymu33wYFMHZuhQExEzZowp/e0VUlNN2zTnIlYHD7qOKVIkPQLlxhtNR55ChbwkQIDiw5Z+qTqVe2bcw8bDG7m2xLV82/1bUeYCIArdHmTXOi2H/peFCpmG1I5iX5s2mQXUvn1NnZhixXIpy6VLsGZNuvW9ZAmcOOE6pkSJ9BoorVrBDTe4RqAIBh8t3P7fz//H7G2zKR5RnLlxc7mi0BVeElgIdMTlYge8dHt+8aJJQHrjDfO8fHkYOdLk2WTJuXOm7onD+l6+/PIszIoVXZs41KxpnwwnPybu2IFJGybRe1ZvQlUoC+5ZwK1X3Wq1SIKfkXrogYAXk1D++MNY6ytWmNdxccYNX6oUxtp2NHFISDDWeHKy6wGuu841hLBKFXvWQPFj4o4d+H3f79w88WYuplxkRLsRPNroUatFEixAFHqg4K3WaZjaVcOHw/AXDxKdtJjbIxLoUmoxxfZvRjn/z0NCjMvEYX23bAllynjpA/kQHy862o09J/fQaHQjjp47Sv9G/RnebrjVIgkWIYuigYKncdVamx+DxYsJXbyYpxISeCppl9mXBOyDSyHhpEY3puCtThEoRYt6RXy/4sfEHatJvJBITHwMR88dpe1VbdZGEBMAACAASURBVPnoDsnbEzJHFHogk5ICmze7hhD+84/rmMhIdIsWrCl0I6/8eCO/nmtMwW0RfPAQ9L3dnp4Ut/BT4o6nFH2nKIkXM7mLSCMqPIrTL57Ocn9Kagr3zLyHTUc2cV2J65jSbQphIfK1FTJHZkYgcfGiSRF1+L+XLoVTp1zHlCzp6v+uVw8VFkY0MOYAPPYYzJljsk0nTzbFvq6+2pJP4xkeRAb5k+yUuTv7X/r5JeZsm8MVEVdIRIuQI+JDtzNnzlwegZKU5DqmShXXFPrrrsvW7NYapk6Fxx83HdkKFYI334Qnnwyw5j0B4kNXr+V8C6QHZf4dnLB+An1m9yEsJIwf7vmBW6rd4m3xhABEfOiBwvHjrhEoa9de3pmnZk1XCzyXdeWVMmutbdqYzNKvv4ZnnzUu57FjTVe2gMCHiTt2YMneJfSb2w+AT+/8VJS54Bai0K1k37503/fixSYj05nQUIiOdo1AKVXKK6cuWRK++iq92NfKlabY1//9n2mDFx7uldP4FrtXXMwjf5/8m85TOnMp9RJPNH6Ch6MftlokIUAQl4u/0Bq2b3dNof/7b9cxBQuatHmH9d2smV+SZE6dghdegC++MK/r1DGleRs18vmpg57culwSLyTSfFxzNh/ZzO1X3868XvNkEVRwQVwuVpCSAhs3ukagHMnQbrVoUWjRIt2FEh1tSR3cYsVMRmnPnqbY1+bN0LSpqd772mvuFftKTYWJE6F3b/skkQYaKakp9JrRi81HNlOjZA0md5ssylzIFTJbvMWFC6but8MC//13OJ0hHK10adcU+rp1bbUS2bq1+Q0aNMgU+hoyJL3Y1003Za+0Z8ww2alFi0LXrpaIH/AMXDiQedtNt6G5cXMpHpFzhUpPwyKF4EIUel5JTDRK22F9r1hhlLoz1aq5LmBWr277wO/ChU1Rr+7dTV7O5s1G0T/yiPEAZaa0U1NNFyUwfzt3FivdQVR4VI4KF2D8uvEMWTaEsJAwpveYzjVXuhd26WlYpBBciA/dXY4eNREoDhfKunWXZyrWqZNufd94oylqFcBcvAjvvANvvWWKMIaGGk9StWpmDdKhtKdNgz59TJRlZCRMmCBWem5YvGcxbSa14VLqJUa1H0Xfhn3dfq8nYZFCYCK1XPLCnj2uXej//NN1f2io6T7vsMBbtDBlZYOQzZuN1e1IvgwLg1GjjBJPTTWJSc7ruxkVvpA1u0/spvGYxhw7d4wnmzzJx3d8nKv3i0LPf8iiaE5obRS2cwTK3r2uYyIijM/BYX03bRrw4XHuUquWa0vQ5GSzeBoRYZT7sWOu448ehZkzxUrPidMXThMTH8Oxc8e445o7GHLbEKtFEgIcd5pEjwPaA0e01lmmnSilGgHLgJ5a62neE9EHJCfD+vWuMeAZtVLx4sbqdvi/GzYMkOBs7zNjhsl5ciY1FXr1Mpmm58+77jtzRnzpOZGSmkLc9Di2HN1CzZI1mdxVIloEz3FnBk0APgUmZTVAKRUKvAf86B2xvExSksmccVjfv/9utI4z5cq5ptDXqSPaiPQFz4yXC8z6bkZl7mDvXhMh06WLb+ULVAb8NIDvd3z/X0RLsYjctpYShMvJUaFrrROUUlVzGPY4MB2wRyrKqVPpESgJCSac8OJF1zFXX+0agXL11baPQLGCWbMu9z45U6YMHD5snleqBG3bpveDrlLF9/LlBrvEyo9dO5ahy4cSFhLGjB4zuPrKQKyOJtgRj+/xlFIVgM7AzeSg0JVS/YB+AJVzWYMkWw4fdnWfbNjgGoGilIn5do5AKV/ee+cPYqpUMTVfsiIuzlSyfeIJU8lg8mQTFfP447YKsQfsESu/6O9FPPqd6TQ08q6R3FT1Jo+O525YpLeR+Hd74laUS5qFPi8zH7pS6lvgQ631cqXUhLRxOfrQ8xzlorUJqXBewNy+3XVMWJjJunRY3y1awBVSdtSXHD1qKjbGx5vXTZuaYl+1alkrlwPnaByronB2ndhF49GNOX7+OE83fZqhtw/1rwBeRKJrrMPXUS7RwGRl3BUlgXZKqWSt9SwvHNt8E7duTVfeCQlw4IDrmMKF0yNQWrUy9VDcyVfPx3jb/VCqFHzzjbHYH3nEVPqtXx9efhkGDoQCBTw/hyfMmJG+7m1FFM6ppFPExMdw/Pxx2lVvxwdtP/DfyYV8g8cWeoZxE/DUQr90ySTtOJT3kiXw77+uY664wtV90qCB9RojwJg2zWSDTpvmfcV26pRZSB092ryuW9cU+2rY0LvncRerY+WTU5OJiY9hwc4F1CpVi2UPLqNowQBs++eEWOjW4ZGFrpSKB1oDJZVS+4FBQAEArfVIj6U7d86kzTv838uWXd6FpkIF1wiUWrUkAsUDfJ2qX6yYSTzq2RP69jX1YRo3hueeg8GDTaijP3G2zh3400p//sfnWbBzASUKlWBu3NyAV+aCfbEuU7RcOb26WjXTUs05awVMzRPnIlbVqkkEihfxZ6r+2bPw6qvw8cfmh6R6dVPsq1Ur35wvI5lZ5w78YaWPXjOafvP6USCkAAvvW0irKn764D5GLHTrsGfqv1J6NRhFXa9euvXdsiWULWuJTPkBq9wPK1aYCJM//jCvH3vM1Ikp6mNjdcYM41rKWHYHTBTO1Km+i5X/7e/faPtlW5JTkxnbYSwP1H/ANyeyAFHo1mHP1P+yZU0YRPPm6YHLgs+xyv3QpInpqPf22+YxYgTMnWuaatx5p+tYby7Y5hR26atY+Z3/7qTr1K4kpybzbLNng0qZC/ZFinPlI6x2PzjYuNGU5nX8+++9Fz76KL22mS8XbP3BqaRTNB3blD+P/cld1e9ids/ZhIbYLCjfQyQO3Trs6XIRhe53rHQ/ZCQ52fjVX3nFVGYoXRo+/dSc/5prrI0X94Tk1GTaf9OeH/76gTql67D0gaWyCCp4FXu6XAS/Y5X7ITPCwkzUS8eOpnJjQgL06GH6mDo69XnbFeSP1P9nf3iWH/76gZKFS9omokWs6fyDKPR8RMOG1sWCZ0X16vDrrybMccAAU3bHgberNvo69f+L1V/wycpPKBBSgJmxM6lavKr3T5IHpKtR/iGAbmaFYCUkxGSXvv/+5fVfDh82VrqnZIy9z8zt5Am/7v6V/vP7AzAqZhQtK7f07gkEwQ1EoQu2IDUV3nvPtLhz5tw56Nfv8lSF3JJZ6r+32HF8x38RLc83f577b7jfewcXhFwgCl2wBdmV6f33X7j+elPSJy9krOnucOV4w0o/mXSSmPgYTiSdIObaGN5p847nBxWEPCI+dMEWZLVgu3Mn/PwzbNsGN9xgsk4HDMhd6R5fxd4npyYTOy2Wbce3cX3p6/m6y9dBF54oBBYStijYnpMnTUTM2LHm9Q03mOcNGuT8Xl/G3j8x/wmGrxxOqcKlWNl3pW0WQXOKaskMyeoMHLILWxSXi2B7ihc39V8WLjRKeP16U+zrxRdNDHt2ZOfKcbTJywufr/qc4SuHEx4abquIFpColfyMuFyEgKFNG9i0ydRYHzYM3n3XuFPGjjUlgDLDF7H3P+/6mcfnPw7A6JjRtKjcIvcHsRG+6mok+B9xuQgBybJlpnyAY6H0f/8zxb6ifKybth/fTtMxTTmRdIIXWrzAu7e+69sT5gEpnBXciMtFCDqaNTN9UF5+2WSdfvYZ1KkDP/zgu3OeOH/iv4iWjtd15O02b/vuZIKQB0ShCwFLwYLwxhumyFfDhsYnfscdcP/9lze58pRLKZfoMa0H249vp26ZunzV5StClHx9BHshM1IIeOrVMz1M33vPKPmJE01Tq+nTvXeOp394moW7FlK6SGnm9JxDZHik9w4uCF5CFkWFoCAszMSnd+pkin0tXgzdupk4808/zb5nSk5hfgVDC3Ih5QLhoeHMip1FleJ+rGImCLnAnZ6i44D2wJHMmkQrpe4GXgAUkAg8qrXe4G1BBcEdrr0WfvsNRo6EF14wVvovv8DQoabKYmadDHMK87uQcgGAsR3G0qxSMx9InTO5qZgYFR6V41ghOHHHQp8AfApMymL/buAmrfUJpdSdwCigiXfEE4TcExJiWtzddZcp+rVggemhOnmy6ZCUl1DFF1u+yD117/G+sG6Sm4qJUgo3/5KjD11rnQBkucSktf5da30i7eVyoKKXZBMEj6hSBb7/HiZNgiuvNBEwtWsbF0xu67i8ecubvhFSELyItxdFHwTmZ7VTKdVPKbVaKbX66NGjXj61IFyOUqbF3R9/GJ/62bPw+OOmH/m2be4fRyJahEDAa7NUKXUzRqG/kNUYrfUorXW01jq6VKlS3jq1IORImTLw7bfGp162LCxdaqJj3n0XSJHYACE48MpMVkrVBcYAd2qtj3vjmILgC7p0gZtvhmefhfHjTT0Yyq6Ajg9CufVWixfwSLs7a/HYQldKVQZmAPdqrbd7LpIg+JYrroBx44xPvUoV4J8GMGoVrOsNkhHvEdLuzlrcCVuMB1oDJZVS+4FBQAEArfVI4FWgBDBCmZiw5KzqDAiCnbjtNti8Ga687XMu7bgJas4ywbenKkCxA/+NkzA/IVDIUaFrreNy2P8Q8JDXJBIEPxIZCf/8HEu94U3Yf/4UbO0MU6fxxOMhvPWW2W8HJLZccAdZDRLyNZdSLtFtajf2n99JvdI30Pbil3wUEsInn8CcOTBqFLRta7WUElsuuIfEYgn5Fq01/b/vz69//0rZyLLM7TWHD94qwqpVpivS338bt8wDD8CJEzkeThAsRyx0Id8yfOVwRq0dRcHQgsyKnUWlYpUAqF8fVq6EIUPgtddMNMz8+TBiBHTu7H85JXJEcBex0IV8yYKdC3j6h6cBGN9xPE0qularKFDAhDSuXw8tWsA//5iQxx494PBh/8oqkSOCu4hCFwKK1FRjMec2dd+ZrUe3EjstllSdyiutXiHu+qzX/WvUgIQEGD4cihQxyUm1asGXX4JFzb5sTU6Ls7J461vE5SIEFDNmGJ920aKmNG5uOX7uODHxMZy+cJquNbsyuPXgHN8TEgL9+0P79vDww/Djj3DffRAfb6o6Vq6cezmCFXH9WItY6ELAkJoKzz9vnj//fO6t9IspF+k6tSt/nfiLBuUaMLHTxFzVaKla1VRuHD8eihc3fvXatY1v3ZM7BkHwFqLQhYBhxgw4dsw8P3oUZs50/72OiJZFexZRLrIcs3vOpkh4kVzLoJRpcbd1q/GpnzljGlS3bg3bJU9asBhR6EJA4LDOz5wxr8+cyZ2VPmzFMEavHU1EWASze86mYlHPqjyXLWsKfU2bZgp/LV5sin29/z4kJ3t0aEHIM6LQhYDA2Tp34K6VPn/HfJ798VkAJnScQKMKjbwmV9eupjRv796QlGS6JDVtChukZ5dgAaLQBduT0Tp34I6VvuXIlv8iWgbdNIjYOrFel+/KK2HCBONfr1wZ1qyB6Gh45RW4cMHz40vkiOAuSlsUexUdHa1Xr15tybmFwGLGDOjePXPFHRoKU6caf3ZGjp07RuPRjdl9cjfda3VncrfJPm9UkZho4tc/+8y8rlkTxo6FZta0IhWCEKXUmqwKIErYYpCTmgoTJxqXQEiA3o9VqQJPPZX9/ow4Ilp2n9xNw3INmdBpgl+6DkVFmRZ3sbHw0ENm8bRFC3jiCXjrLRPLLgi+Qiz0IGfaNGPdTpuWfdy2LxS/VT8mWmv6zu3L2HVjKRdZjlV9V1GhaAX/CZBGUpIpHfDBB5CSAtWqwejR0KaN30XxKVKawL9kZ6EHqM0muENu4rYdCTu5CQXMCV8c0x0+Xv4xY9eNJSIsgjlxcyxR5gAREfDOO6YuTL16sHs33HqrsdxPnrREJJ8gpQnsg1joQcy0adCnj1k8jIw0C3eZWempqXD11aa6YLVqsHOn5xa1L47pDt9t/44OkzuQqlOZ0m0KPWr3cOt9vr6buHTJWOqvvQYXL0K5cvD559Cxo9nvLSvXCmtZvaZyHKMHSZ0EbyEWej4kN3HbniTsZIUvjpkTW45sIW56HKk6lcE3DXZbmYPv7yYKFICXXjLFvpo1g0OHoFMn6NkTjhzxnpUr1nL+RhR6kOJu3LanCTuZ4Ytj5sTRs0eJiY8h8WIisbVjefWmV91+r6clBXJDzZomCWnYMChcGKZMMcW+2NhL+pkKHpOjQldKjVNKHVFKbc5iv1JKfaKU2qmU2qiUauB9MYXckJu4bU8SdrLCF8fMjgvJF+gytQu7T+6mUflGjO84nrT+tm7h77uJ0FAT9bJ5s/GpHz8OzPgavpkLpzzLYBXyN+5Y6BOAO7LZfydQPe3RD/jcc7EET5g1C/buzXzf3r1mP3iWsJMVvjhmdmitefS7R1mydwkVoiowq+csChUolGd5/XE34aBaNVO5cexYoOBJ2NEePtsCqx6GVPd/kATBgTtNohOUUlWzGdIRmKTN6upypVRxpVQ5rfUhL8ko5BJ347bdUfyZJexkhy+OmR1Dlw1l/PrxFAorxOyesykfVT5X78/ubiIv5Xlzi1LGd//gllrw/WfwZ2f4biRs7gkd+kKJnb4XQggavJFYVAHY5/R6f9q2yxS6UqofxoqnshSR9hkNG5pHTuQlYceKY2bFvO3zeP4n4/ye1HkSDcu78aGdyOluonNnP8bPFz0EsV3gj27w/aewpzV8vhFufgWafgyhKX4SJPdEhUflGFkj+Ae/ZopqrUcBo8CELfrz3MLluKv4rT5mZmw6vIm46XFoNK+3fp1utbrl+hj+vpvIEQXUngbVfoEFH8HG++CnIbAlFjo+4EdBcockDdkHbyj0A0Alp9cV07YJgk84cvYIHSZ34MzFM/Ss05OXW72cp+P4824iJ1ys3ML/QpfecH08zP0CDjaCL9YyqJQJfSxY0M3jZLFfCF7cSixK86HP01rXyWTfXUB/oB3QBPhEa904p2NKYpGQFy4kX6DNpDYs3beUxhUa81vv33K1CBponD4NAweaJCQwHZLGjoUmTbJ/nxC8eJRYpJSKB5YB1yml9iulHlRKPaKUeiRtyPfALmAnMBp4zEtyC4ILWmsenvcwS/ctpWLRisyKzV1ESyBStKhpcbdoEVSvDlu2QPPm8OyzcO6c1dIJdkNS/4WA4YOlHzBg4QAKFyjMkj5LqF+uvtUi+ZXz52HwYBgyxCzoXnUVjBkDN99stWSCP5HUfyHgmbNtDi8sfAGALzt/me+UOUChQvDee7BiBVx/PezaBbfcAv36walTVksn2AFR6ILt2Xh4I72m90KjefPmN+lS05+hJ/YjOhpWr4Y33oDwcFOSt1YtmDvXaskEqxGFLtiaw2cOExMfw9lLZ+l1fS9euvElq0WyBeHh8PLLsG6dWSA9eBA6dIBevUxilJA/EYUu2BZHjZa9p/bSpEITxnYYm6saLfmBWrVg6VL46CNT7Cs+3myLjweLlscECxGFLtgSrTX95vXj932/U6loJWb1nEVEWITVYtmS0FATT79pk/GpHztmLPUOHWD/fqulE/yJKHTBlry/9H0mbZhE4QKFmRM3h7KRZa0WyfZcdRUsXGh86kWLwrx5Jm591Cix1vMLotAF2zH7z9m8+POLAHzV+StuKHuDxRIFDkqZFnd//GEs9NOn4eGHTR/Tv/6yWjrB14hCF2zFhn82cPeMu9Fo3r7lbTrX7Gy1SAFJhQqmFs3kyVCqFPz6qwl1HDrUNKwWghNR6IJt+OfMP/9FtNxT9x4GthxotUgBjVIQG2us9bvvNolJzz5rMk03Z9quRgh0RKELtiApOYnOUzqz7/Q+mlZsyuiY0RLR4iVKloSvvjI+9QoVYOVKaNAgvWG1EDyIQhcsR2tN37l9Wb5/uYloiZWIFl9w112mFszDD8OlS6aMQMOGsGqV1ZIJ3kIUumA57y55l682fkWRAkWYGzeXMpFlrBYpaClWDEaOND71q682rpemTU1DDyn2FfiIQhcsZebWmbz0y0soFF93+Zp6ZetZLVK+oHVr2LgRnnvOvB4yBOrVM1UdhcBFFLpgGesOreOemfcA8E6bd+hYo6PFEuUvCheGDz6AZcugTh3YudMo+kcfNeGOQuAhCl2whH/O/EOHyR04d+kc99W7jwEtBlgtUr6lcWNYs8b41AsUMC6Z2rXhu++slkzILaLQBb+TlJxEp8md2H96P80rNWdU+1ES0WIx4eEwaBCsXWsU/P790L493HOPKSUgBAai0AW/orXmwTkPsuLACioXq8zM2JkUDMumSabgV+rUgd9/hw8/NPXXv/7aFPuaMkXKBwQCotAFv/L24rf5ZtM3RIZHMjduLqWLlLZaJCEDoaHwzDNm0bR1a1OOt2dP6NTJlOkV7ItbCl0pdYdSaptSaqdS6rL0PaVUZaXUr0qpdUqpjUqpdt4XVQh0pv8xnZd/fRmF4psu31C3TF2rRRKy4Zpr4Oef4YsvTLGvOXOMtT52rFjrdsWdJtGhwGfAnUAtIE4pVSvDsJeBqVrr+kBPYIS3BRUCm7WH1nLfrPsAeO/W94i5LsZiiQR3CAkxLe62bDE+9VOnTPGvW281LfAEe+GOhd4Y2Km13qW1vghMBjLGl2mgaNrzYoDcmAn/cSjxEB3iTURL73q9ea75c1aLJOSSihWNhf7NN6aUwC+/mGJfH38sxb7shDsKvQKwz+n1/rRtzgwG7lFK7Qe+Bx7P7EBKqX5KqdVKqdVHpU9WvuD8pfN0mtKJA4kHaFGpBV+0/0IiWgIUpSAuzhT7ioszmaVPPw0tW5ptgvV4a1E0Dpigta4ItAO+VEpddmyt9SitdbTWOrpUqVJeOrVgV7TWPDDnAVYeWEnV4lUloiVIKFXKWOpz5kD58rB8OdSvb5pWX7pktXT5G3cU+gGgktPrimnbnHkQmAqgtV4GRAAlvSGgELi8mfAmkzdP/i+ipVQR+REPJmJijGXet6+p2vjqqxAdbZKUBGtwR6GvAqorpaoppcIxi55zMozZC7QBUErVxCh08ankY6b9MY1Xf3sVhSK+azx1StexWiTBBxQrZlrc/fyzaYG3caNJTHrhBVN/XfAvOSp0rXUy0B/4AdiKiWbZopR6XSnVIW3Ys0BfpdQGIB64X2sJbMqvrDm4hvtmmoiWD9p+QPtr21sskeBrbrnFKPNnnjGv33/fFPtKSLBWrvyGskrvRkdH69WrV1tybsF3HEw8SKPRjTiYeJA+N/RhbIexsgiaz1ixAh54IH2h9LHH4J13TCy74DlKqTVa6+jM9kmmqOA1zl06R8fJHTmYeJAbK9/I53d9Lso8H9KkiakJ8+qrEBYGI0aYkgLz51stWfAjCl3wClpr+szuw+qDq6lWvBrTe0yXiJZ8TMGCpsXdmjVmoXTfPmjXDu67D44ft1q64EUUuuAVXl/0OlO3TCUqPEoiWoT/qFvX1Fv/4AOIiIAvvzTlA779VsoH+AJR6ILHTN0ylcGLBhOiQpjcbTK1S9e2WiTBRoSFmc5IGzdCq1Zw5Aj06AFdusChQ1ZLF1yIQhc8YtWBVfSe1RuAIW2H0K661GUTMqd6ddPL9PPPISoKZs0y1vr48WKtewtR6EKeOXD6AB0ndyQpOYkH6z/IU02fslokweaEhMAjj5hiX+3awcmTJiLm9tvh77+tli7wEYUu5AlHRMuhM4doVaUVI+4aIREtgttUqgTz5sFXX0GJEvDTTyYS5pNPpNiXJ4hCF3JNqk7l/ln3s+bQGq664iqm95hOeGi41WIJAYZScPfdJl49NhbOnoUnnzR+9q1brZYuMLEssUgpdRTYk8OwkoDdOxoGgowgcnobkdO7iJzuU0VrnWkYmWUK3R2UUquzyoiyC4EgI4ic3kbk9C4ip3cQl4sgCEKQIApdEAQhSLC7Qh9ltQBuEAgygsjpbURO7yJyegFb+9AFQRAE97G7hS4IgiC4iSh0QRCEIMESha6UukMptU0ptVMpNTCT/R8ppdanPbYrpU467Utx2pexFZ43ZRynlDqilNqcxX6llPok7TNsVEo1cNrXWym1I+3R21cyuinn3WnybVJK/a6Uque07++07euVUj7tNuKGnK2VUqec/revOu3Ldr74Wc7nnWTcnDYfr0zb58/rWUkp9atS6g+l1Bal1JOZjLF0jropo+Xz0005bTE/c0Rr7dcHEAr8BVwFhAMbgFrZjH8cGOf0+oyf5GwFNAA2Z7G/HTAfUEBTYEXa9iuBXWl/r0h7foWFcjZ3nB+40yFn2uu/gZI2uZ6tgXmezhdfy5lhbAzwi0XXsxzQIO15FLA943Wxeo66KaPl89NNOW0xP3N6WGGhNwZ2aq13aa0vApOBjtmMj8P0KfUrWusE4N9shnQEJmnDcqC4UqoccDvwk9b6X631CeAn4A6r5NRa/54mB8ByoKKvZMkON65nVuR2vnhELuW0ZG4CaK0Paa3Xpj1PxPT7rZBhmKVz1B0Z7TA/3byWWeHX+ZkTVij0CsA+p9f7yeLiKaWqANWAX5w2RyilViulliulOvlOzBzJ6nO4/fks4EGMxeZAAz8qpdYopfpZJJMzzZRSG5RS85VSjqLqtryeSqnCGCU43WmzJddTKVUVqA+syLDLNnM0GxmdsXx+5iCn7ednmFUndpOewDSttXP9tSpa6wNKqauAX5RSm7TWf1kkX8CglLoZ84Vp6bS5Zdq1LA38pJT6M81CtYK1mP/tGaVUO2AWUN0iWdwhBliqtXa25v1+PZVSkZgflae01qd9ea684o6MdpifOcgZEPPTCgv9AFDJ6XXFtG2Z0ZMMt7Ra6wNpf3cBv2F+Ta0gq8+Rm8/nF5RSdYExQEet9X8dHZ2u5RFgJub20RK01qe11mfSnn8PFFBKlcSG1zON7OamX66nUqoARgF9rbWekckQy+eoGzLaYn7mJGfAzE9/O+0xdwW7MK4UxyJC7UzG1cAsiiinbVcABdOelwR24NsFsqpkvYh3F64LTivTtl8JopMRKgAAAUdJREFU7E6T9Yq051f6+JpmJ2dlYCfQPMP2IkCU0/PfgTsslLOs43+N+eLuTbu2bs0Xf8mZtr8Yxs9exKrrmXZtJgEfZzPG0jnqpoyWz0835bTN/Mzu4XeXi9Y6WSnVH/gBs0I8Tmu9RSn1OrBaa+0IRewJTNZpVzCNmsAXSqlUzN3Fu1rrP3whp1IqHrOyXVIptR8YBBRI+wwjge8xUQQ7gXNAn7R9/yql3gBWpR3qde16W+5vOV8FSgAjlGlAkaxNtbgywMy0bWHAN1rrBRbK2Q14VCmVDJwHeqb97zOdLxbKCdAZ+FFrfdbprX69nkAL4F5gk1Jqfdq2lzAK0i5z1B0Z7TA/3ZHTFvMzJyT1XxAEIUiQTFFBEIQgQRS6IAhCkCAKXRAEIUgQhS4IghAkiEIXBEEIEkShC4IgBAmi0AVBEIKE/weEwaSP6mdHHQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs_pred = 1.2 * np.random.rand(50, 2) + 1\n",
    "xs_pred = np.hstack([xs_pred, np.ones((xs_pred.shape[0], 1))])\n",
    "\n",
    "ys_pred = np.array([make_prediction(x, thetas_oa) for x in xs_pred])\n",
    "plot_onevsall(xs_pred, ys_pred)\n",
    "plt.xlim([xs_oa[:, 0].min() - 0.3, xs_oa[:, 0].max() + 0.3])\n",
    "plt.ylim([xs_oa[:, 1].min() - 0.3, xs_oa[:, 1].max() + 0.3])\n",
    "plot_boundaries(xs_oa, thetas_oa)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}