path: root/logistic_regression_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression\n",
    "\n",
    "Logistic regression is a *classification algorithm*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Data representation')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEWCAYAAABliCz2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFj1JREFUeJzt3Xu4ZXV93/H3Z+YwBw9XK5fKRUdjBa2JA4wgTbwU4zQaHPskVKNAjeapYNI0sbVWoY/x8pCYtklTq0a8YhXxGsOIF8bnISFe4uiAeAGhUkC5KAwaLuPojAPf/rHWyew5nJnZZ9ibPXN+79fzrOfstfbvt9Z3rX3OZ6/92/usnapCkrT4LZl0AZKkh4aBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfWmSSPC3JdZOuQ3seA1+7JclNSX6a5N4kdyX5cpKzkwz1O5VkeZJKMjXuWvcmSZ6Z5JYF9qkkj5udr6ovVNUxo69OezsDXw/G86rqAODRwJuB/wK8ZxKFjPqJwyciLUYGvh60qrq7qtYALwRekuRJAEl+PcnXk9yT5OYkrx/o9nf9z7uSbExycpJfSHJZkh8luTPJhUkO3tF2+zPb30vyXeC7/bJjk3w+yY+TXJfkBQPtL0jyjv7+e5NcnuTRD2J9z01yTb+uW5O8auC+U5NcNfDq55cG7rspyauSfDPJ3Uk+kmTfJPsBnwWO6I/JxiRHJDkxyd/36/pBkrcmWdava/Y4fqNv/8K5rxKSPCHJ3/b9r06yes4xeVuST/f7sS7JL+zqMddeqqqcnBY8ATcBvzrP8u8Dr+hvPxP4RboTi18Cbgf+dX/fcqCAqYG+jwOeDUwDh9I9KfzFTmoo4PPAPwEeBuwH3Ay8FJgCjgPuBJ7Yt78AuBd4er+N/wV88UGs7wfA0/rbDweO728fB9wBnAQsBV7SH6/pgWP3VeCIflvfAc4eOGa3zNnPE4Cn9jUs79v/4Zy6Hzcw/4/rAPYBrgfOAZYBp/TH4JiBY/Ij4MR+/RcCH57075fTeCbP8DVqt9GFGFX1t1X1raq6v6q+CVwEPGNHHavq+qr6fFVtrqoNwJ/vrH3vT6rqx1X1U+BU4Kaqel9Vba2qrwOfAP7NQPtPV9XfVdVm4Fzg5CRH7+b6fg48McmBVfUPVXVlv/zlwPlVta6q7quq9wOb6UJ71luq6raq+jHwKWDFTo7LFVX1lb6Gm4Dzhzgus54K7A+8uaq2VNVlwCXAiwbafLKqvlpVW+kCf4e1aO9m4GvUjgR+DJDkpCR/k2RDkruBs4FDdtQxyeFJPtwPj9wDfHBn7Xs3D9x+NHBSP3RxV5K7gNOBfzpf+6ra2Nd6xG6u7zeB5wLf64eHTh7o95/m9Dt6znZ+OHB7E10ozyvJ45NckuSH/XH5Y3Z9XGYdAdxcVfcPLPse3eO04Fq0dzPwNTJJnkIXJF/sF30IWAMcXVUHAe8A0t8332Va/7hf/otVdSBwxkD7HRlcz83A5VV18MC0f1W9YqDNP57NJ9mf7tXIbbuzvqr6WlU9HzgM+GvgowP9zpvTb6aqLtrFvszd/qy/BK4F/ll/XM5h18dl1m3A0XM+PfUo4NYh+2sRMfD1oCU5MMmpwIeBD1bVt/q7DgB+XFU/S3Ii8OKBbhuA+4HHDiw7ANgI3J3kSOA/L7CUS4DHJzkzyT799JQkTxho89wkv9K/6fkm4CtVdfP8q9vx+pIsS3J6koOq6ufAPf3+ALwLOLt/hZMk+6V7A/uAIfbhduARSQ4aWHZAv/6NSY4FXjFPn8cyv3V0Z+2v7ut/JvA8usdKjTHw9WB8Ksm9dGe059KNub904P7fBd7Yt3kd286AqapNwHnAl/phj6cCbwCOB+4GPg381UKKqap7gVXAb9Gd2f4Q+FO6N2hnfQj4I7qhnBPoXkXs7vrOBG7qh1nOphvuoarWA/8OeCvwD3Rvmv72kPtwLd17HTf0x+UI4FV0T5b30j2ZfGROt9cD7+/bv2DO+rbQBfxz6N5wfjvwb/vtqDGp8gtQ1IYkF9B9euW/TroWaRI8w5ekRhj4ktQIh3QkqRGe4UtSI/aoC0QdcsghtXz58kmXIUl7jSuuuOLOqjp0mLZ7VOAvX76c9evXT7oMSdprJPnesG0d0pGkRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqRFjC/wk701yR5Jvj2sbkqThjfMfry6gux74/xnjNgBYswbWroVVq2D16oX3gV33X7MGzj8fbr8dDj8czjrrgW1n13nQQXDVVXDddXDPPXDggXDMMV2fdeu6dqtXw3nnPXD9ACtWwN13b1sPwMwMXH45bNzYze+/PzzjGbBp07aabrgBbrwRHvMYuOsuuOMOmJ6G446DW2+FqamujpmZbntbtsCSJd38pk3wyEfCk5/crf+ss+ADH4DPfa5b9wtfCJdeCnfeCU95Stf+y1/u9u+ww+BlL+tqXrWq28cLL9y2vbPO6tY5u3833ADXXw+HHLJ9v9Wrtx2H666DrVvh9NPnP047exwAzj23a3vssV27hfxuSIvWOL8hHVgOfHvY9ieccEIt1MUXV83MVEH38+KLF9Zn2bKq6emd97/44q4dbJump7dvO7jOHU1Llmw/f8452/rO1rCnTMnu9Zuamn/ZfMsHp5mZ7njMdxwGj9Pcx2HZsgc+ZuecM//6h/ndkPY2wPoaMmMnPoaf5OVJ1idZv2HDhgX3X7u2O9uE7ufatQvrs2ULbN688/5r13btBm3evH3bwXXuyP33bz+/Zs22vrM17ClqNy+iunXr/MvmWz5o06bueMx3HAaP09zHYcuWBz5ms+3nrn+Y3w1pMZt44FfVO6tqZVWtPPTQoa7/s51Vq7ohCeh+zg7RDNtn2bJu2GNn/Vet6toNmp7evu3gOndkyZyjPTvEsGrVthr2FBn2K7LnmJpnkHBqav7lg2ZmuuMx33EYPE5zH4dlyx74mM03dDPs74a0mI31evhJlgOXVNWThmm/cuXK2p2LpzmG7xj+XI7hqxVJrqiqlUO1XQyBL0mtWkjgj/NjmRcBfw8ck+SWJL8zrm1JknZtbB/LrKoXjWvdkqSFm/ibtpKkh4aBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYMFfhJ/iDJgem8J8mVSVaNuzhJ0ugMe4b/sqq6B1gFPBw4E3jz2KqSJI3csIGf/udzgQ9U1dUDyyRJe4FhA/+KJGvpAv/SJAcA94+vLEnSqE0N2e53gBXADVW1KckjgJeOryxJ0qgNFfhVdX+S5cAZSQr4YlV9cpyFSZJGa9hP6bwdOBv4FvBt4KwkbxtnYZKk0Rp2SOcU4AlVVQBJ3g9cM7aqJEkjN+ybttcDjxqYPxr47ujLkSSNy7Bn+AcA30nyVaCAE4H1SdYAVNXqMdUnSRqRYQP/dWOtQpI0dsN+SufycRciSRqvoQI/yb10QzkAy4B9gJ9U1YHjKkySNFrDnuEfMHs7SYDnA08dV1GSpNFb8OWRq/PXwL8aQz2SpDEZdkjnNwZmlwArgZ+NpSJJ0lgM+ymd5w3c3grcRDesI0naSww7hu+F0iRpLzfstXSOSvLJJHf00yeSHDXu4iRJozPsm7bvA9YAR/TTp/plkqS9xLCBf2hVva+qtvbTBcChY6xLkjRiwwb+j5KckWRpP50B/GichUmSRmvoLzEHXgD8EPgBcBp+45Uk7VV2+SmdJEuB3/CKmJK0d9vlGX5V3Qe86CGoRZI0RsP+49WXkrwV+Ajwk9mFVXXlWKqSJI3csIG/ov/5hv5n6K6eecrIK5IkjcWwgX8JXcCnny/gniQrquqqsVQmSRqpYT+lcwJwNvBIun+8OovuapnvSvLqMdUmSRqhYc/wjwKOr6qNAEn+CPg08HTgCuC/jac8SdKoDHuGfxiweWD+58DhVfXTOcslSXuoYc/wLwTWJbm4n38e8KEk+wHXjKUySdJIDXt55Dcl+Szwy/2is6tqfX/79LFUJkkaqWHP8OkDfv0uG0qS9kgL/k5bSdLeycCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEaMNfCT/FqS65Jcn+Q149yWJGnnpsa14iRLgbcBzwZuAb6WZE1VXTOubUp7ujVrYO1aWLUKVq/u5s8/v7vvrLO6ZcP0ha7f7bfD4Yd3fdetg3e9CzZuhP33h2c8o7tvdls7qgHg3HPhwgthagp+8hO46y448siu/5VXwtatkMAjHgHHH99tD+CNb4QNG+CMM+C887p1v/GNcM01sGULLFsGT3wivO513bbOPberEbr6brwRvvtdqILDDoODD+62fdtt3bJTT4Uzz+zqvf12+NrX4M47uxqhq/e++2DffeGVr9xWw2z7a6+FY4/t9uOgg+Duu7v9Xreuazd73+wxXbu2a3fVVd38ihVdn9l1rV7dbWP2OO7osZvvGO/ssdxZ25GqqrFMwMnApQPzrwVeu7M+J5xwQkmL1cUXV83MVEH385xzqqanu3moWrasa7OrvtPTVVNT2/pB1ZIl288PTjMz29Y7t4aLL+7q2FHfHU1TU1VLl26/7LTTHljXYPvTTlv4dna1b3On007btn87q33usunp7vgPs41zzumO22D76emdH+MdPZbLlm37HZiv7TCA9TVkLo9zSOdI4OaB+Vv6ZdtJ8vIk65Os37BhwxjLkSZr7VrYtKm7vWlTd5a3efO2+7ds6drsqu/mzd0Z96D779/xdjdt2rbeuTWsXdvVsVBbt3Zn1oMuu+yBdQ22v+yyhW8Hdr5vc1122bb925H5aty8uTv+w5g9Ox9sv3nzzo/xoMH7t2zZ9jswX9tRm/ibtlX1zqpaWVUrDz300EmXI43NqlUwM9PdnpnpXr5PT2+7f9mybS/xd9Z3erobyhi0ZCd/yTMz29Y7t4bdHUaYmoKlS7dfdsopD6xrsP0ppyx8O7DzfZvrlFO27d+OzFfj9HR3/IexenV33AbbT0/v/BgPGrx/2bJtvwPztR21sY3hA7cCRw/MH9Uvk5q0ejVcdNH247UnnTTcGP7cvrB7Y/jz1TB7n2P4CxvD/9jH5n/s5jvGu3osH6ox/HRDQGNYcTIF/F/gWXRB/zXgxVV19Y76rFy5stavXz+WeiRpMUpyRVWtHKbt2M7wq2prkn8PXAosBd67s7CXJI3XOId0qKrPAJ8Z5zYkScOZ+Ju2kqSHhoEvSY0w8CWpEQa+JDXCwJekRhj4ktSIsf3j1e5IsgH43oTLOAS4c8I1PBTcz8XF/Vw8FrqPj66qoa5Ls0cF/p4gyfph/2ttb+Z+Li7u5+Ixzn10SEeSGmHgS1IjDPwHeuekC3iIuJ+Li/u5eIxtHx3Dl6RGeIYvSY0w8CWpEQY+kGTfJF9N8o0kVyd5w6RrGqckS5N8Pcklk65lXJLclORbSa5Ksmi/VSfJwUk+nuTaJN9JcvKkaxq1JMf0j+PsdE+SP5x0XeOQ5JV9Bn07yUVJ9h3p+h3DhyQB9quqjUn2Ab4I/EFVfWXCpY1Fkv8IrAQOrKpTJ13POCS5CVhZVYv6n3SSvB/4QlW9O8kyYKaq7pp0XeOSZCndN+idVFWT/ifNkUpyJF32PLGqfprko8BnquqCUW3DM3ygOhv72X36aVE+EyY5Cvh14N2TrkUPTpKDgKcD7wGoqi2LOex7zwL+32IL+wFTwMP6r4idAW4b5coN/F4/zHEVcAfw+apaN+maxuQvgFcD90+6kDErYG2SK5K8fNLFjMljgA3A+/ohuncn2W/SRY3ZbwEXTbqIcaiqW4H/AXwf+AFwd1WtHeU2DPxeVd1XVSuAo4ATkzxp0jWNWpJTgTuq6opJ1/IQ+JWqOh54DvB7SZ4+6YLGYAo4HvjLqjoO+AnwmsmWND79kNVq4GOTrmUckjwceD7dE/kRwH5JzhjlNgz8OfqXxH8D/NqkaxmDXwZW9+PbHwZOSfLByZY0Hv3ZElV1B/BJ4MTJVjQWtwC3DLwa/TjdE8Bi9Rzgyqq6fdKFjMmvAjdW1Yaq+jnwV8C/GOUGDHwgyaFJDu5vPwx4NnDtZKsavap6bVUdVVXL6V4aX1ZVIz2D2BMk2S/JAbO3gVXAtydb1ehV1Q+Bm5Mc0y96FnDNBEsatxexSIdzet8Hnppkpv8gybOA74xyA1OjXNle7JHA+/tPACwBPlpVi/Yjiw04HPhk9zfDFPChqvrcZEsam98HLuyHO24AXjrhesaif+J+NnDWpGsZl6pal+TjwJXAVuDrjPgyC34sU5Ia4ZCOJDXCwJekRhj4ktQIA1+SGmHgS1IjDHwtekn+Q38lyQsX2G95khePqy7poWbgqwW/Czy7qk5fYL/lwIIDv/9/DmmPY+BrUUvyDuCxwGeTnJvkvf13H3w9yfP7NsuTfCHJlf00++/sbwae1l+D/ZVJfjvJWwfWfUmSZ/a3Nyb5syTfAE5OckKSy/uLt12a5JEP7Z5LD2Tga1GrqrPpLjH7L4H96C4ncWI//9/7/+C8g+4VwPHAC4G39N1fQ3et+RVV9T93san9gHVV9WRgHfC/gdOq6gTgvcB5I941acG8tIJasoru4nGv6uf3BR5F94Tw1iQrgPuAx+/Guu8DPtHfPgZ4EvD5/vIOS+kudytNlIGvlgT4zaq6bruFyeuB24En073q/dkO+m9l+1fFg18/97Oqum9gO1dX1aL7ukHt3RzSUUsuBX6/vxIhSY7rlx8E/KCq7gfOpDsjB7gXOGCg/03AiiRLkhzNji+5fB1w6Oz3yybZJ8k/H+meSLvBwFdL3kT39ZXfTHJ1Pw/wduAl/Ruux9J9kQjAN4H7+i+3fyXwJeBGuksQv4XuqoYPUFVbgNOAP+3XeRUjvq65tDu8WqYkNcIzfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGvH/AbxuGJjohp/YAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "# create linearly separable data\n",
    "sep = False\n",
    "while not sep:\n",
    "    X, Y = make_blobs(n_samples=200, n_features\n",
    "                      =1, centers=2, cluster_std=0.5, center_box=(0, 10))\n",
    "    sep = True\n",
    "    for x_1 in X[Y == 1]:\n",
    "        for x_0 in X[Y == 0]:\n",
    "            if Y[X.argmin()] == 0 and x_0 > x_1:\n",
    "                sep = False\n",
    "            elif x_0 < x_1:\n",
    "                sep = False\n",
    "                \n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33)\n",
    "\n",
    "plt.scatter(X, Y, s=10, c='b')\n",
    "plt.xlabel('feature')\n",
    "plt.ylabel('groups')\n",
    "plt.yticks([0, 1])\n",
    "plt.title('Data representation')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model\n",
    "\n",
    "We have to fit a **model** trough our data.  \n",
    "A *sigmoid* (also called *logistic*) function is used:\n",
    "$$\\sigma(x) = \\frac{1}{1 + e^{-x}}$$\n",
    "\n",
    "It produce an 'S' shape like function that stays between 0 and 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Sigmoid function')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VPXd9/H3NzuQkECAsBP2VVEWcaUgqOBSeLzrdqNWrXVp1dr2aSt1abWXt21vt7q0PtyK2+1aSilVSxUxdUGRXfZ9C4YlCYSErDPze/6YwQYIkISZnJnJ53Vdc2WWM+d8JiSf/PjNmXPMOYeIiMSXBK8DiIhI+KncRUTikMpdRCQOqdxFROKQyl1EJA6p3EVE4pDKXZqMmU0xs/ejbbtmlmdmNx/jMTOzF81sn5l9GbmUdW77H2b23abcpsQP037uEk5mdi7we2Aw4AfWAHc75xZ6Guw4zCwP+F/n3PN1PHYe8AbQ3zl3MIIZfg30cc5dG6ltSPOS5HUAiR9m1hp4B7gdeBtIAc4DqrzMdZJ6AFsjWewikaBpGQmnfgDOuTecc37nXIVz7n3n3FcAZnaDmX16aGEzu9DM1plZiZn90cz+dWh6JLTsZ2b2hJntN7PNZnZ26P4dZran9pSFmWWa2StmttfMtpnZfWaWcIztXmBma0PbfQawul6MmX0PeB44y8zKzOzBI9cVWs6ZWZ/Q9ZfM7Fkze9fMSs1sgZn1rrXsYDP7wMyKzWy3mf3SzCYAvwSuCm1neWjZb6aLzCwh9Jq2hV77K2aWGXosN5Thu2a23cwKzezeRv8rSlxQuUs4rQf8ZvaymU00szbHWtDM2gEzgKlANrAOOPuIxUYBX4Uefx14ExgJ9AGuBZ4xs/TQsk8DmUAv4FvA9cCNx9juTOA+oB2wCTinrozOuReA24DPnXPpzrlfnegbEHI18CDQBtgIPBzadgYwF5gDdA69jg+dc3OA/wLeCm1naB3rvCF0GRt6jenAM0cscy7QHxgHPGBmA+uZV+KQyl3Cxjl3gGDBOOB/gL1mNtvMcupY/GJglXNupnPOBzwF7DpimS3OuRedc37gLaAb8JBzrso59z5QDfQxs0SChTrVOVfqnNsKPAZcd5ztznDO1QBP1rHdk/VX59yXodf1GnBa6P5LgV3Oucecc5WhrAvquc4pwOPOuc3OuTKCfxSvNrPaU6sPhv63tBxYDtT1R0KaCZW7hJVzbo1z7gbnXFdgCMER6pN1LNoZ2FHreQ7IP2KZ3bWuV4SWO/K+dIIj8GRgW63HtgFd6rndHXUsdzJq/7EoD2WE4B+nTY1cZ2eOfn1JQO0/nMfarjRDKneJGOfcWuAlgiV/pAKg66EbZma1bzdQIVBD8M3PQ7oDO4+x3W5HbLdbHcsdy0GgZa3nd2zAc3cQnFKpy4l2W/uao1+fj8P/AIp8Q+UuYWNmA8zsp2bWNXS7G3AN8EUdi78LnGJmk0NTCz8EGlKU3whN27wNPGxmGWbWA/gJ8L/H2O5gM7s8tN27Grjd5aHnn2ZmacCvG/Dcd4BOZna3maWGso4KPbYbyD30JnAd3gB+bGY9Q+8zHJqj9zVg+9KMqNwlnEoJvgm6wMwOEiz1lcBPj1zQOVcIXEFwn/giYBCwiMbvNnknwVH1ZuBTgm/ATj/Odn8b2m5f4LP6bsQ5tx54iOAboxtC26rvc0uBC4DLCE6hbCD4BinAn0Nfi8xsSR1Pnw68CnwMbAEqCb5mkTrpQ0wSFUIj1nxginPuI6/ziMQ6jdzFM2Z2kZllmVkqwf28jbqncESkgVTu4qWzCO49UkhwqmKyc67C20gi8UHTMiIicUgjdxGROOTZgcPatWvncnNzI7b+gwcP0qpVq4itP9KU3zuxnB2U32uRzr948eJC51z7Ey3nWbnn5uayaNGiiK0/Ly+PMWPGRGz9kab83onl7KD8Xot0fjPbduKlNC0jIhKXVO4iInFI5S4iEoei6kxMNTU15OfnU1lZedLryszMZM2aNWFI5Y2mzJ+WlkbXrl1JTk5uku2JSORFVbnn5+eTkZFBbm4uwYP1NV5paSkZGRlhStb0miq/c46ioiLy8/Pp2bNnxLcnIk3jhNMyZjY9dFqvlcd43MzsKTPbaGZfmdmwxoaprKwkOzv7pItd6s/MyM7ODsv/lkQketRnzv0lYMJxHp9I8Mh6fYFbgD+dTCAVe9PT91wk/pxwWsY597GZ5R5nkUnAK6Ez2nwROhBUJ+dcQZgyiohEBeccvoCj2heg2hegxh+gKvT10P2b9/tJ31pMjd/hDzh8gQA+f/B5h24HnMPndwScwx8Av3MEQo8H73MEHARC9x+63pDDxYRjzr0Lh5+mLD9031Hlbma3EBzdk5OTQ15e3mGPZ2ZmUlpaGoZI4Pf7w7YuLzR1/srKyqP+PU5GWVlZWNfXlGI5Oyj/Ib6A42ANHKxxlPscFTWOCj9U1Dgq/VDpc1T4oMrvqPL/+2u131Hjh6pA8GtNAGpqXa9XvX7x+UnnP1lN+oaqc24aMA1gxIgR7shPca1ZsyZsbyLqDdWGSUtL4/TTTw/b+mL5U4axnB3iN79zjgOVPvYcqGT3gSp2H6hkb1kVRWVVFJZVU1hWxb7yavYdrGFfeTXl1f4Tbis1KYFWqUm0TEmkZUoiLdISaZuSSIvkRNJCl9SkBNKSE0lJSiA1KYGUxARSkv59SU4IfU1MICnRWLNqJcNOG0pSopGUkBD6aiQmBG8HvwZvJyQYiWYkJECiHX5fYoJhBglmoUtwCtV+V7/vYzjKfSeHn4OyK3Wfu1LCaPPmzTz88MOUlJQwY8YMr+OIhEVFtZ9tB/z8Y0UB24vL2V5cTv6+Cr7eH7wcrKOw05ITaJeeSnarFNqnp9KvQwZZLVNo0zKZ1i2SyWyRTOsWSbROSyYjLZmMtCRapSbRKiWRpMTwf9Qnde9aRvc74aFfIi4c5T4buMPM3iR4irUSzbdHXq9evXjhhRf4zne+43UUkQarqPazbncp63eVsnZXKRv2lLJpTxlfl4T22pofPNNgVstkurZpQa/2rTi3bzs6Z7YgJzONnIxUclqn0S4jlVYpidopoA4nLHczewMYA7Qzs3zgV0AygHPuOeA94GJgI1AO3BipsF6rqKhgwoQJzJs3j8TExDqXqa6uZvz48cybN4+kpKj6GIGIJ3z+AGt3lbJ42z6+yi9h5c4SNuwpJRCavE5LTqBvhwxG9cqmV7tWVOzdxiWjR9KtbUtap+mDdY1Vn71lrjnB447gmevj3vTp07n88suPWewAKSkpjBs3jrfeeospU6Y0YTqR6FDjD7B8x37mbyrii81FLNux/5v573bpqZzSpTUXDc5hUOdMBnTMoFvbliQm/HvknZe3k8GdM72KHzc0tKzD8uXLufPOOyksLGTt2rU457j//vuZO3cur7/++jfLjR07ll/+8pdccMEF3HfffZSUlPD0008zefJkpk6dGtFyLyoq4t5772Xp0qU88sgjTJ06NWLbEjmRgpIK5q3dw7w1e/hicxEHq/2YwcCOrblieFeG57ZleI82dM5M0xRKE4nacn/w76tY/fWBRj/f7/cfNcIe1Lk1v7ps8HGfV1lZyVVXXcUrr7zCGWecwf33309lZSX33nsv06ZNo/YJRh588EEeeOAB9uzZw9KlS5k9ezYAQ4YMYeHChUet+7zzzqtz98ZHH32U8ePHN+j1ZWdn89xzzzXoOSLhtLXwIO+uKOC9FQWsCv2udmvbgv8zrAvn9G7Hmb2yadMqxeOUzVfUlrtX5s6dy7BhwzjjjDMAOPXUU5kzZw5FRUVkZWUdtuzo0aNxzvH444+Tl5f3zR+TxMREUlJSjtqd8ZNPPmm6FyISAUVlVfx16U5mLdvJyp3BQh/WPYt7Jg5g3IAO9OmQrpF5lIjacj/RCPtEGruf+MqVKznllFO+ub1kyRKGDRtGixYtjjr+yooVKygoKCA7O/uobVVVVZGWlnbYffUZuTflL4ZOji71EQg4Pt6wlze/3MHcNbvxBRxDu2Zy3yUDufiUTnTOauF1RKlD1Ja7V7Kzs5k3bx4A69evZ+bMmcyfP582bdrg9/uprKwkLS2NgoICpkyZwt/+9jfuuusu5syZw4QJwUPwFBUV0a5du6MOoVufkfuhwg33h5jmzp1LcXExRUVF3H777WFbr8Svg1U+Zi7J58XPtrK58CDZrVK48ZxcrhjRjX45sfsBweZC5X6Ea665htmzZzNkyBDatWvHG2+8QXZ2NgAXXnghn376KWeffTaXX345jz32GAMHDuT+++/nF7/4xTfl/tFHH3HJJZc0avuHPpxUVFTErFmzjnp8xYoVZGZm0r179+OuZ9asWd88f+LEiVx88cVcdNFFfPbZZ43KJc3H/vJqpn+2lZc+28KBSh9Du2byh6tPY+KQTqQk6fw+sULlfoT09HT+/ve/1/nYD3/4Q5544gnGjx/P55//+9gRo0ePPuz266+/zm9/+9tGbf/Qh5MmT55c5+OLFy8mNzf3sHKvqanhV7/6FeXl5QQCAZ566ikmT578zToCgQAPP/wwP/vZz1i0aBEjR45sVDaJb/sOVvP8p5t5ef42yqp8XDQ4h1tG92ZY9yzNo8cglXsDDBs2jLFjx9a5J84h1dXVTJ48mX79+oV126tXr+app55i3bp1ZGRk8Oabb/Lggw+Sk5PDtGnTqKioICsriy1bthz13ISEBO6///6w5pH4UeXz8/L8rTw9byNlVT4uHtKJO8f1YUDH1l5Hk5Ogcm+gm2666biPp6SkcP3114d9u4MGDeK5557jpZdeIjc397ADKy1dupRnn32W1NTUsG9X4pdzjndXFPDbf6wlf18FY/q3Z+rEgfTvqPn0eKAJtChTVFTEbbfdxvLly3nkkUfq9ZxJkyZxww038POf/5w5c+ZEOKHEg+1F5Vw//UvueH0p6alJvPq9M3jpxjNU7HFEI/coc+jDScfaW+aGG2446r7LLruMyy67rAnSSayr8Qd44dMtPDl3PUkJCTz47cFce2aPwz7+L/FB5S7STGwpPMjdby1j+Y79XDQ4h19/ezCdMrWPerxSuYvEOeccby3cwUPvrCY5MYFn/3MYl5zayetYEmEqd5E4VlpZw89nfMU/Vu7i7N7ZPHblUI3Wm4moK3fnnPapbWI6DEF82rinlFteXcy2onKmThzA98/rRYLm1puNqCr3tLQ0ioqKyM7OVsE3EeccRUVFRx0HR2LbnJW7+Onby2iRkshrN4/izF7ZXkeSJhZV5d61a1fy8/PZu3fvSa/r0DFgYlVT5k9LS6Nr165Nsi2JLOcc72yqZsacxQztlsVz1w7TNEwzFVXlnpycTM+ePcOyrry8PE4//fSwrMsLsZ5fmp7PH+BXs1cxY0MNk07rzO+/cyqpScc+a5jEt6gqdxFpnPJqH3e9sZS5a/ZwSc9knrjyNM2vN3Mqd5EYV1pZw40vLmTJ9n38ZtJgulVtVbGLDj8gEssOVNZw/fQvWbZjP09fM4zrzsr1OpJECY3cRWJUSXkN109fwOqCAzw7ZRgXDe7odSSJIip3kRh0oLKGa19YwLpdpTx37XDGDczxOpJEGZW7SIyprPFz80uLWLvrANOuG8HYAR28jiRRSOUuEkNq/AHueH0JC7cV89TVp6vY5Zj0hqpIjAgEHL+Y8RVz1+zhoUlDuGxoZ68jSRRTuYvEiP9+fx0zl+7kpxf047oze3gdR6Kcyl0kBsxYnM+f8jZxzRndueP8Pl7HkRigcheJcgu3FjN15lec3TubhyYN1kH1pF5U7iJRbEdxObe+upiubVryxynDSE7Ur6zUj35SRKJURbWf77+yCJ8/wAvfHUFWyxSvI0kM0a6QIlHIOce9s1awbncpL94wkl7t072OJDFGI3eRKPTWwh3MXLKTO8/vy5j+2pddGk7lLhJlVu4s4YHZqzivbzt+NK6v13EkRqncRaLIgcoafvDaErJbpfDkVaeRqEP3SiNpzl0kijwwayU791fw9q1nkp2e6nUciWEauYtEib8t28msZV9z5/l9GN6jrddxJMap3EWiwM79Fdw3ayXDumdxx1h9AlVOnspdxGP+gOMnby0jEHA8edXpJOmDShIGmnMX8djzn2xmwZZiHr1iKN2zW3odR+KEhggiHtq0t4zHPljPRYNz+I9hXbyOI3FE5S7ikUPHZ2+RnMhvJg/RAcEkrFTuIh555fOtLNq2jwcuHUSHjDSv40icUbmLeGBHcTm/m7OOb/Vrz+WajpEIULmLNDHnHPfM/IrEBOO/Lj9F0zESESp3kSY2e/nXfLaxiF9M6E+XrBZex5E4pXIXaUIlFTX85p01DO2ayX+O0nlQJXK0n7tIE3r8/XUUH6zixRtG6qBgElEauYs0kRX5Jbz6xTauO7MHp3TN9DqOxDmVu0gT8Acc981aQdtWqfzkwv5ex5FmQOUu0gTeXrSD5fkl3H/pQDJbJHsdR5oBlbtIhB2orOHRf67jjNy2fHtoZ6/jSDOhN1RFIuzpDzdQXF7Ny5cN0j7t0mQ0cheJoC2FB3lp/lauHN6NIV30Jqo0HZW7SAQ9/O5qUpMS+b8X6U1UaVoqd5EI+Xj9Xuau2cMd5/ehfYbOhypNS+UuEgH+gOPhd9fQI7slN56T63UcaYZU7iIR8Jcl+azbXcovJgwgNSnR6zjSDKncRcKsotrP4++v57RuWUwc0tHrONJMqdxFwmz6Z1vYdaCSqRMHaNdH8YzKXSSMisqq+FPeJsYPzGFUr2yv40gzpnIXCaOn522kvNrHPRO166N4S+UuEiY7ist5bcE2rhrZjT4dMryOI82cyl0kTJ6Yu54EM340rp/XUURU7iLhsGF3KX9dupPvnp1Lx8w0r+OIqNxFwuHxD9bTKiWJ277V2+soIoDKXeSkfZW/n3+s3MXN5/WkbasUr+OIACp3kZP26PvradMyme+d29PrKCLfULmLnIQFm4v4eP1efjCmDxlpOsOSRA+Vu0gjOed47IP1dMhI5bqzengdR+QwKneRRvp8UxFfbinmB2N6k5asg4NJdFG5izSCc44n5q6nY+s0rj6ju9dxRI6ichdphE83FrJw6z5+OFajdolOKneRBnLO8cQH6+mcmcaVI7t5HUekTip3kQb6eEMhS7bv5wdj++hEHBK1VO4iDXBo1N4lqwVXjtCoXaKXyl2kAT7eUMiyHfv5wdjepCTp10eil346RerJOccf5gbn2r8zvKvXcUSOS+UuUk/zNxWxZPt+bh/TW3PtEvVU7iL19IcPN5DTOpUrNNcuMUDlLlIPX2wOfhr19m9pv3aJDSp3kXp46sMNtM9I1adRJWao3EVOYNHWYuZvKuLW0b00apeYoXIXOYGn520ku1UKU0bpyI8SO1TuIsexfMd+/rV+Lzef14sWKRq1S+xQuYscxzMfbSSzRbKO1y4xR+UucgxrCg7wwerd3HROT9JTk7yOI9IgKneRY3jmo42kpyZxw9m5XkcRaTCVu0gdNu4p470VBVx/Vg8yW+rcqBJ7VO4idfhj3kbSkhL53rk9vY4i0ij1Knczm2Bm68xso5ndU8fjN5jZXjNbFrrcHP6oIk1jT3mAvy37mimjupOdnup1HJFGOeG7RGaWCDwLXADkAwvNbLZzbvURi77lnLsjAhlFmtR7m2tINOP7o3t5HUWk0eozcj8D2Oic2+ycqwbeBCZFNpaINwpKKvh0p48rR3Ylp3Wa13FEGq0++3d1AXbUup0PjKpjuf8ws9HAeuDHzrkdRy5gZrcAtwDk5OSQl5fX4MD1VVZWFtH1R5rye+O1NVUEnGNoyt6YzA+x+70/RPnDI1w77/4deMM5V2VmtwIvA+cfuZBzbhowDWDEiBFuzJgxYdr80fLy8ojk+iNN+ZteYVkVn3w4j7O7JHPFxUf9+MaMWPze16b84VGfaZmdQO0DWHcN3fcN51yRc64qdPN5YHh44ok0nRc+3UKVL8ClvbTro8S++pT7QqCvmfU0sxTgamB27QXMrFOtm98G1oQvokjk7S+v5tXPt3HJKZ3o2Ep7CEvsO+FPsXPOB9wB/JNgab/tnFtlZg+Z2bdDi91lZqvMbDlwF3BDpAKLRMJL87dSVuXjjvP7eB1FJCzqNefunHsPeO+I+x6odX0qMDW80USaRmllDdM/3cKFg3IY0LE1u9Z6nUjk5On/n9LsvfrFNg5U+rjz/L5eRxEJG5W7NGvl1T6e/2QLY/q355SumV7HEQkblbs0a68v2E7xwWru1Fy7xBmVuzRblTV+pn28mbN7ZzO8R1uv44iElcpdmq0/L9rBntIq7SEjcUnlLs1Slc/PH/M2MTK3DWf1yvY6jkjYqdylWZqxOJ+CkkruGtcXM/M6jkjYqdyl2an2BfjjR5s4vXsW5/Zp53UckYhQuUuz89el+ezcX6FRu8Q1lbs0KzX+AM98tJFTu2Yypl97r+OIRIzKXZqVWUt3sqO4grvO16hd4pvKXZoNX2jUPqhTa8YN7OB1HJGIUrlLszFz6U62FZXz4wv6adQucU/lLs1CjT/A0/M2cEqXTMZr1C7NgMpdmoW/LM5nR3EFP75Ac+3SPKjcJe5V+wI8PW8jQ7tlMba/Ru3SPKjcJe7NWBzcr/3u8Rq1S/Ohcpe4Vu0L8OxHGzm9e5b2a5dmReUuce3NhdvZub+Cn2gPGWlmVO4St8qrfTz14UbO7NVWx5CRZkflLnHr5fnbKCyr4mcX9deoXZodlbvEpZKKGp771ybOH9BBZ1mSZknlLnHp+U82U1JRw08v7Od1FBFPqNwl7hSWVfHCp1u49NRODO6c6XUcEU+o3CXuPDNvI5U1fn58gUbt0nyp3CWubCs6yGsLtnHVyO70bp/udRwRz6jcJa789z/XkZSQwI/H9/U6ioinVO4SN5bt2M87XxXw/dG96NA6zes4Ip5SuUtccM7xyHtraJeewi2je3kdR8RzKneJCx+t28OCLcX8aHw/0lOTvI4j4jmVu8Q8nz/AI++tpVe7Vlw9spvXcUSigspdYt7rX25nw54y7pk4gORE/UiLgMpdYtz+8moe/2A95/TJ5oJBOV7HEYkaKneJaU/O3cCBihruv3SQDg4mUovKXWLW+t2lvPrFNqaM6sGAjq29jiMSVVTuEpOcc/zmndW0SknUYQZE6qByl5j0werdfLKhkLvH96NtqxSv44hEHZW7xJzyah8P/n01/XMyuO6sHl7HEYlK+rSHxJynPtzIzv0V/Pm2s7Tro8gx6DdDYsr63aU8/8lmrhjelZG5OsOSyLGo3CVmOOe4b9ZKWqUmcc/EAV7HEYlqKneJGTOX7OTLLcXcM3EA2empXscRiWoqd4kJe0ur+M27qzm9exZXjdDxY0ROROUuMeHXs1dRXuXn9/9xKgkJ+iSqyImo3CXqzVlZwLsrCrhrXB/65mR4HUckJqjcJartL6/mvlmrGNSpNbd+q7fXcURihvZzl6j20Dur2V9ezcs3jdQ+7SINoN8WiVpzVu5i5pKd3Pat3gzunOl1HJGYonKXqLT7QCVTZ37FkC6tuWtcX6/jiMQclbtEnUDA8X//vJyKGj9PXnU6KUn6MRVpKP3WSNR5cf5WPtlQyH2XDKJPh3Sv44jEJJW7RJU1BQf43Zy1jBvQgSmjunsdRyRmqdwlapRW1vCD15bQOi2Z333nVJ02T+QkaFdIiQrOOe75ywq2F5fz2s2jaKdjx4icFI3cJSq8NH8r764o4GcX9efMXtlexxGJeSp38dzibft4+N01jB+Yw62je3kdRyQuqNzFU7tKKvnBa4vplJXGY1cO1Ty7SJio3MUz5dU+bn5lIWWVPv7n+hFktkj2OpJI3NAbquKJQMDxk7eWs+rrAzx//QgGdGztdSSRuKKRu3jisQ/WMWfVLu69eCDjBuZ4HUck7qjcpcm9vmA7z360iatHduN75/b0Oo5IXFK5S5N6b0UB985awZj+7Xlo0hC9gSoSISp3aTKfbijk7jeXMbx7G/40ZbgOCCYSQfrtkiaxdPs+bnl1Eb3at+KF746kRUqi15FE4prKXSJu8bZ9XP/Cl7RLT+WVm84gs6V2eRSJNJW7RNTCrcVc/8ICstNTeOvWM+nQOs3rSCLNgvZzl4j5YnMRN720kI6Zabzx/TPJUbGLNBmVu0TE4t0+ps39ku5tW/La90fRIUPFLtKUNC0jYffy/K08s7SKgZ1a8+YtZ6rYRTygkbuETSDg+N2ctfy/jzdzeodEXv/+mdorRsQjKncJiwOVNfzkreXMXbOba8/szvmZhSp2EQ9pWkZO2obdpUx+5jM+WreHBy4dxG8mDSFBnzwV8ZRG7nJS3vnqa34+4ytapiTy+s2jGKWzKIlEBZW7NEppZQ2/nr2avyzJ5/TuWfxpynA6ZuqNU5FooXKXBlu8rZi731rGzn0V3Hl+H+4a15fkRM3wiUQTlbvUW1mVj8feX8fL87fSOasFb996FiNy23odS0TqoHKXepm7ejcP/G0lBQcqmTKqO7+YMICMNB0jRiRaqdzluDbuKeWR99by4do99M/J4On/HMbwHm28jiUiJ6BylzoVlVXx5NwNvP7ldlomJzJ14gBuOren5tZFYoTKXQ5TVFbFtE828+rn26jyBZgyqjs/GteX7PRUr6OJSAOo3AWAgpIKXvxsK69+vo1Kn5/LTu3MXeP60qdDutfRRKQRVO7N3NLt+5j+2VbeW1GAc45Jp3Xhh2P7qNRFYpzKvRkqqahh9rKdvL0onxU7S8hITeKmc3K5/qxcurVt6XU8EQkDlXszUeXz8/H6Qv6+/Gv+uWoXVb4AAzu15qFJg7l8WFfSU/WjIBJP9Bsdx8qqfHy6YS/vr97NB6t2U1rlI6tlMleO6MZVI7sxpEum1xFFJEJU7nEkEHCs213KZxsL+df6vXyxuYgav6N1WhIThnTk0qGdObt3tnZnFGkGVO4xzOcPsKaglEXbilm0dR9fbC6i6GA1AL3at+LGc3py/oAODO/RRoUu0syo3GNEjT/AlsKDrNxZwsqdB4Jfvy6hvNoPQOfMNL7Vrz1n92nH2b2z6ZzVwuPEIuIllXuUKa/2sbWwnC8LfHyfS+NHAAAG/ElEQVT14QY27ilj/e5SNu0to8bvAEhLTmBQp9ZcOaIbw3u0YXiPNipzETmMyr0JBQKOooPV7D5Qye4DlXy9v4KvS4JfdxSXs724gsKyqlrPWE+XrBb0y0lnTP8O9O+YzqBOmfRu34okTbOIyHHUq9zNbALwByAReN4599sjHk8FXgGGA0XAVc65reGNGl2qfQHKqnyUVtZQUlHDgQofJRU17K+oZt/BavaV17DvYDWFB6spLK2i6GAVhWXV+APusPUkJxodM9Po1qYl4wd2oFvblvTIbknx1jVcMWGMzkMqIo1ywnI3s0TgWeACIB9YaGaznXOray32PWCfc66PmV0N/A64KhKBAZxz+AMOX+DfX33+AP6AoyZ0/euyAGsKDuDzO6r9Aap9AWpCX6v9Aap8fqp9Aap8AapqgrcrawJU1PipqPFTWe2nvNpPeY2fimofB6v8HKz2UVbpo6zKR5UvcNyMLVMSadMyhXbpKXTKTGNIl9a0z0glp3UaHTLSyGmdSpesFrRLTyUh4ejzjeYVr1exi0ij1Wfkfgaw0Tm3GcDM3gQmAbXLfRLw69D1GcAzZmbOucOHqbWs313K+Y/mEXCOgCP4NRC87g9d94dK/ND1QIBv7quXTz+p33IhCQYtkhNpkZJIalIiLVMOXZLonJVMq9QkWqUmkR66ZKQlkZGWTOu0JDJbJJPZMpmsFilktUwmLVnFLCLeqU+5dwF21LqdD4w61jLOOZ+ZlQDZQGHthczsFuAWgPScXNonVWIGBiSY1bp+xAUjISGBhCMeS0yARLN/3w7dl2RQU11FqxZpJCUE709OMJISCF2M5ARCFyM5MXg90cCs9ijaAb7QpQ41oUspVBC87KrHN7Q+ysrKyMvLC9Paml4s54/l7KD8XouW/E36hqpzbhowDWDEiBHurbsnRGxbeXl5jBkzJmLrjzTl904sZwfl91q05K/PLhc7gW61bncN3VfnMmaWBGQSfGNVREQ8UJ9yXwj0NbOeZpYCXA3MPmKZ2cB3Q9e/A8w73ny7iIhE1gmnZUJz6HcA/yS4K+R059wqM3sIWOScmw28ALxqZhuBYoJ/AERExCP1mnN3zr0HvHfEfQ/Uul4JXBHeaCIi0lj6mKOISBxSuYuIxCGVu4hIHFK5i4jEIfNqj0Uz2wtsi+Am2nHEJ2RjjPJ7J5azg/J7LdL5ezjn2p9oIc/KPdLMbJFzboTXORpL+b0Ty9lB+b0WLfk1LSMiEodU7iIicSiey32a1wFOkvJ7J5azg/J7LSryx+2cu4hIcxbPI3cRkWZL5S4iEofivtzN7E4zW2tmq8zs917naQwz+6mZOTNr53WW+jKz/w59378ys7+aWZbXmerDzCaY2Toz22hm93idpyHMrJuZfWRmq0M/7z/yOlNDmVmimS01s3e8ztIYZpZlZjNCP/trzOwsr7LEdbmb2ViC53cd6pwbDDzqcaQGM7NuwIXAdq+zNNAHwBDn3KnAemCqx3lOqNbJ4CcCg4BrzGyQt6kaxAf81Dk3CDgT+GGM5Qf4EbDG6xAn4Q/AHOfcAGAoHr6WuC534Hbgt865KgDn3B6P8zTGE8DPCZ7QNWY45953zh06+ewXBM/gFe2+ORm8c64aOHQy+JjgnCtwzi0JXS8lWCxdvE1Vf2bWFbgEeN7rLI1hZpnAaILnt8A5V+2c2+9Vnngv937AeWa2wMz+ZWYjvQ7UEGY2CdjpnFvudZaTdBPwD69D1ENdJ4OPmXKszcxygdOBBd4maZAnCQ5kAl4HaaSewF7gxdDU0vNm1sqrME16guxIMLO5QMc6HrqX4OtrS/C/qCOBt82sVzSdAvAE+X9JcEomKh0vu3Pub6Fl7iU4XfBaU2ZrzswsHfgLcLdz7oDXeerDzC4F9jjnFpvZGK/zNFISMAy40zm3wMz+ANwD3O9VmJjmnBt/rMfM7HZgZqjMvzSzAMGD+uxtqnwncqz8ZnYKwZHAcjOD4LTGEjM7wzm3qwkjHtPxvvcAZnYDcCkwLpr+oB5HfU4GH9XMLJlgsb/mnJvpdZ4GOAf4tpldDKQBrc3sf51z13qcqyHygXzn3KH/Lc0gWO6eiPdpmVnAWAAz6wekECNHm3POrXDOdXDO5Trncgn+4AyLlmI/ETObQPC/2N92zpV7naee6nMy+KhlwVHAC8Aa59zjXudpCOfcVOdc19DP+tXAvBgrdkK/mzvMrH/ornHAaq/yxPzI/QSmA9PNbCVQDXw3RkaQ8eAZIBX4IPQ/jy+cc7d5G+n4jnUyeI9jNcQ5wHXACjNbFrrvl6FzIEvTuBN4LTQ42Azc6FUQHX5ARCQOxfu0jIhIs6RyFxGJQyp3EZE4pHIXEYlDKncRkTikchcRiUMqdxGROPT/AfZTssY3+RLaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "xs = np.linspace(-7, 7, 100)\n",
    "ys = sigmoid(xs)\n",
    "plt.plot(xs, ys, label=r'$\\sigma(x) = {\\frac{1}{1 + e^{-x}}}$')\n",
    "plt.yticks([0, 0.5, 1])\n",
    "plt.xlim([-7, 7])\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.title('Sigmoid function')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Approximation')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcFNW5//HPMwszwLDP4LCDOhAJ7qOIevnFXSIBjfdGRE3UeN011xg3cImJoqhRMRqNS4waxbgmbrglmKCiAi6oIKIissq+DDAwy/n98XRnmnHYu6emu7/v16tfTXdXVz1drd8+c6rqHAshICIimSUn6gJERCT5FO4iIhlI4S4ikoEU7iIiGUjhLiKSgRTuIiIZSOEuksDMKsxs50ba1j1mdlVjbEuyj+k8d2lMZvYGsCdQGkJYH3E5jcbMTgXOCCEcHHUtkh3UcpdGY2Y9gf8CAjAkRdvIS8V6RdKNwl0a00+Bd4A/Az+LP2lmf451UbxmZqvN7F9m1iPh9WBmF5rZV2a2xMxuNrOc2GunmtlbZnabmS0Ffm1mOWZ2pZnNNrNFZvawmbWJLX+Cmc0ys9axx4PMbKGZlSRsa9eEuv5gZuNi3TVvmVmpmd1uZsvN7DMz2zuhzsvN7MvYZ5hmZsfFnt8NuAcYEFvPioT1X5fw/v81sy/MbJmZPWdmnevtg7PNbKaZrTCzu8zMkvz9SAZRuEtj+inwaOx2lJntlPDaScBvgWLgw9gyiY4DyoF9gKHA6Qmv9Qe+AnYCrgdOjd0OAXYGioA7AUIIfwXeBu4wsw7AA3h3yeJN1PwT4MpYXeuBicD7scdPAbcmLPsl/pdJG+Ba4C9m1imEMB04G5gYQigKIbStvxEzOxS4Iba9TsBs4PF6iw0G9gP2iC131CZqFlG4S+Mws4OBHsATIYQpeBAOT1jkxRDCv2P98CPxVm63hNdHhxCWhRC+AW4HTkx4bX4I4fchhOoQwjr8h+LWEMJXIYQK4ApgWEKXzXnAocAbwPMhhBc2U/qzIYQpIYRK4FmgMoTwcAihBvgr8J+WewjhyRDC/BBCbexHZCaw/1buopOAP4UQ3o/tgyti+6BnwjI3hhBWxPbBeGCvrVy3ZCGFuzSWnwGvhhCWxB4/RkLXDDAn/o9YIC8DOjf0Ot6q3dRrxF6bXW/5PLxlTwhhBfAk0A/43Rbq/jbh3+saeFwUf2BmPzWzD2PdJiti6y/ewvobrDm2D5YCXRKWWZjw77WJ2xapTwefJOXMrDnejZBrZvGAKgDamtmescfdEpYvAtoD8xNW0w34NPbv7vVeq3/K13z8r4S47kA1sWA2s73wbp2xwB3A0dv1wRLEjhHcBxyGd7/UmNmHQLxffEunpW1Us5m1BDoA83a0NslOarlLYzgWqAH64l0JewG7ARPwfniAH5rZwWbWDO97fyeEkNgiv8TM2sW6an6Bd4lsyljgIjPrFfuhGAX8NYRQbWaFwF+AEcBpQBczOzcJn7ElHuCLAczsNLzlHvct0DX2+TZV82lmtpeZFcRqfjeE8HUSapMspHCXxvAz4MEQwjchhIXxG36Q8yT8L8jHgGvw7ph9gZPrrePvwBT8YOuL+IHQTfkT8Ajwb2AWUAlcEHvtBmBOCOHuWN/2ycB1Zla2Ix8whDAN7+KZiAf57sBbCYv8E//LY6GZLWng/a8DVwFPAwuAXYBhO1KTZDddxCSRM7M/A3NDCFdu4vUAlIUQvmjUwkTSmFruIiIZSOEuIpKB1C0jIpKB1HIXEclAkZ3nXlxcHHr27BnV5kVE0tKUKVOWhBBKtrRcZOHes2dPJk+eHNXmRUTSkpnN3vJS6pYREclICncRkQykcBcRyUAKdxGRDKRwFxHJQAp3EZEMpHAXEclAWwx3M/tTbJLhTzbxupnZHbGJfaea2T7JL1NERLbF1lzE9Gd83O2HN/H6IKAsdusP3B27T5kZM2DmTCgrgz59tm152PJ748vn5sLs2OUChxzy3eXjyy1YAOPHw6pVUF4O/ftDTY1v46uvYOJEGDAABg2qe++4cXXP77zzd7e3eDG88w4sWeLrGjrU1z1xInTrBp06wWuvwVtvQatWEAJUVHiN7dr54732qlvub3+DtWuhZ09fftUq6NvXHw8Y4Nt8/nnIyYF+/byWDz8EM388fbpvq1Mn2HdfKC31fRL/fPn5UFVV9zlnzPB98umn8PHHXs/JJ8PAgRt/F+PH+31uLsyZs/F+2trvIXF/xvfN1v63IZKptmrgsNgkvS+EEPo18NofgTdCCGNjj2cAPwghLNjcOsvLy8P2XKH6+YdreXTkNGqbFVJpzTn7/wrZpV9zaN4cCgs9jRLMmAGjRnlALFvmL7dr54E5YkTDgT1qlAfle+952OXnw+67w4031i0fX27BApgwATZs8ADLz4f27eHAA2HlSl+usBBqa+HOOz24xo2D88/3dVdW+jrbtPHtmcG6dV5rbW1dXTk50Lq135Ytg44dYdYs32Z9OTnQrJnfl5Z6AG9Ku3a+bE2Nb3f9eg//dev89epqKCiANWvq3pOf7wHauTPMnet1LloEJSX+NVx+Obz4on+ehQvraiwogL33ht12889QUQFff+0/OitXQnEx5OX5ftp557rvYdIkf3+zZv5DM3r0xt9bfH9WV/t6Bwzw+hr6fkXSnZlNCSGUb2m5ZAw/0IWNJyieG3vuO+FuZmcCZwJ07959uza2YMJMrn1pv7onnq23QPPm0KKF31q2ZKfqllxRUUROqyLmrSxiXV4r2nVvxaLK1lTf3BoGtPFkbdsW2rZl3gftaFXdnupmbaipySE/38OuosJbkfGwiLco16/3cIv/poTgzxUU1LW6u3SBefO8ZTlokN/n5Pjzn3/uy3Xs6MsWFnoLODG0zXwb69ZB166wfLm37EPw1+oHfAheW22tL7c51dV+v2GDB2tVlf+7ttYfx3+A4nWE4HXm5npw19R42IPf5+TAP//p+6u2tq62nBzf1sKFcNRR8M03/rnj+zYE/+GqrPT9E99GQYG/r7Cw4e8B6vZn69a+b9av9/fWX04kmzTq2DIhhHuBe8Fb7tuzjs4DenLroc9TECrJ27COoUeuo7RtpTf/1q71BFy71puaa9eSt2gNfFZBs2UL6F1ZQfOa1bRYtJrC6gqfsK3eZG2Hxm61GCtz2rE8pwMr5hdT3a6E3g+VwFslUFrKPjU78fG8UkJ1Zz61ziwPRf8J24ICD5jiYli61IO9trau+2PAAHj0UX8+N9eXiwdSdbWHZGJoh+Dh1by5d6eE4D8G8VCsz8zDMSfHW9OrV296f+bleYs4N9d3XW1tXUu+ttZvhYW+O+Pbys311+Mt9/gPRFWVr+/QQ73lnpNT9zlqa32/lJZ6sBcV+Xu+/rpuuVWr/P3xrqqaGt8veXm+jdWroUePui6duPj+jO+bgoK6bjGRbJV23TKQpD73nWvo03m1J8KKFd4vsHw5LF/OohnLWfHVMgoqllK1cCmFFUsoZjGFqxZ7UzieZgnW5rVicWE3qnbqSsEu3VjTsSdtdu/BbOvJhPm70O+ITgz6YV2XkfrcfZvqcxfZNlvbLZOMcD8GOB/4IX4g9Y4Qwv5bWueOhHukamv9R+Dbb73DfcECmD/fm7Bz53pCzZnjrydq0cJTvHdvT50+fTxd+/aFli2j+SwiknaS1uduZmOBHwDFZjYXn6E+HyCEcA/wEh7sXwBrgdO2v+w0kJMDHTr4rW/fTS+3bp33P8yaBV9+6bcvvvCm7HPPbdz679kT9tjDm9p77gn77OP9D/UODouIbK3IptlL25Z7MlRVeX/GtGke9p98AlOnej9E/BSZkpK68yoPOgj239+PGIpIVmvMs2VkW+Xn13XNHHdc3fNr13rQT5ni5/9NmgQvv1x3RHXPPeEHP/DbwIF+ho+ISAPUcm/qVq6Ed9/1o5kTJsDbb9edWtO/v59XePTR3srP0WgSIpkuqQdUU0Hhvp0qKz3sX38dXnkFJk/2ln1pKQwe7KfVHHGEnw8oIhlH4Z4tlizxrpvnn/fzAVev9ouyhg6FE07woI9fZSQiaW9rw11/x6e74mI/gfyvf/Wgf+kl78d/7jk45hi/pPWii/ykdRHJGgr3TNKsmV8B9OCDfp793/8OBx8Md93lg7rsvz888MDGA8WISEZSuGeqZs1gyBB4+mm/0GrMGA/1M87wcQMuvrjusk8RyTgK92zQoQNceKGfZjlhgrfux4zxK2Z/8hN4//2oKxSRJFO4ZxMz76Z5/HG/iOrii/2Mm3339f75iROjrlBEkkThnq26d4ebbvKumeuu89MrDzzQQ14HX0XSnsI927VtCyNH+ti7N97orfe994YTT/RxcUQkLSncxRUVwWWXeXfNiBF+KuVuu3nwV1REXZ2IbCOFu2ysbVu4/nqfIup//sfnuuvd2/vpI7rgTUS2ncJdGtalCzzyiHfTdO7s3TQ//KG6akTShMJdNu+AA/xg65gx8Oab8P3vw223bTx7t4g0OQp32bLcXD9Pfvp0OOww+OUvfb47teJFmiyFu2y9rl39QOuf/gQffOCzRz34oPriRZoghbtsGzM47TSf9bq8HE4/HU45xUejFJEmQ+Eu26dHDx9T/je/gbFj/SpXXfwk0mQo3GX75ebCVVfB+PE+ReCBB8Jjj0VdlYigcJdkGDjQ533dbz846SQ/4FpdHXVVIllN4S7JsdNO3k1zwQV+quSgQT7/q4hEQuEuyZOfD3fc4WfTvPEGHHSQxowXiYjCXZLvtNN8KOG5c6F/f5g0KeqKRLKOwl1S49BDfeiC5s39gqfXX4+6IpGsonCX1NltN3j7bZ/x6ZhjfMo/EWkUCndJrU6d4F//8guefvITuP/+qCsSyQoKd0m9du3g1VfhyCPhf/8X7rkn6opEMp7CXRpHy5bwt79598w558Af/hB1RSIZTeEujaegwPvdf/QjOO88uOuuqCsSyVgKd2lcBQXw1FMwZAicf76PKikiSadwl8bXrBk88QQccQSccYaHvYgklcJdolFQAM8+CwMGwPDhMG5c1BWJZBSFu0SnZUt44QXo1w+OP96n8xORpFC4S7TatoWXX/bz4QcPhpkzo65IJCMo3CV6HTt6wIOPJrloUbT1iGQAhbs0DWVl3kUzf7634NeujboikbSmcJemo39/ePxxmDwZTj0VamujrkgkbSncpWkZMgRuugmefNLnZxWR7ZIXdQEi33HxxTBtGlx7LXzvezBsWNQViaQdtdyl6TGDu++G//ovn/jj/fejrkgk7SjcpWmKj0NTUgI//jEsWRJ1RSJpReEuTVdJCTzzDCxcCCeeCNXVUVckkjYU7tK0lZf78MCvvw5XXhl1NSJpQ+EuTd/pp8PZZ8Po0T4mvIhskcJd0sPtt3sr/rTTYNasqKsRafIU7pIeCgp8mOAQ4IQTYMOGqCsSadIU7pI+evXyyT0mTYJLL426GpEmTeEu6eW44+AXv4AxY+C556KuRqTJUrhL+rnpJthnHz/QOn9+1NWINEkKd0k/zZrBY4/BunXw059qgDGRBijcJT316eNdM//4B/zud1FXI9LkKNwlff385z4934gRMGVK1NWINCkKd0lfZnDvvT6T0ymneDeNiAAKd0l37dv76ZHTp8PIkVFXI9JkKNwl/R15JJx7Ltx2G4wfH3U1Ik2Cwl0yw003+Tysp54KK1dGXY1I5BTukhlatoSHH4a5c+FXv4q6GpHIKdwlcxxwAFxyCdx/P7z6atTViERK4S6Z5de/9nlXzzgDVq2KuhqRyCjcJbMUFvrZM/PmeSteJEsp3CXzHHAAXHyxnwP/+utRVyMSCYW7ZKZrr4XeveHMM2HNmqirEWl0CnfJTM2bw333+axNV18ddTUijU7hLplr4EA46yyfom/SpKirEWlUCnfJbKNHQ2mpnz1TVRV1NSKNRuEuma1NG7j7bpg6FW6+OepqRBqNwl0y35Ah8N//Db/5DXzxRdTViDQKhbtkhzFjfAanc86BEKKuRiTlFO6SHTp3hhtu8PPeH3006mpEUk7hLtnj7LOhf3+46CJYujTqakRSSuEu2SM3F/74R1i+HC67LOpqRFJK4S7ZZc89veX+wAPw1ltRVyOSMgp3yT7XXAPduvnBVZ37LhlK4S7Zp6jIz575+GO4446oqxFJCYW7ZKdjj4VjjvFW/Jw5UVcjknQKd8lOZvD730NtrffBi2QYhbtkr169YORIePppeOWVqKsRSSqFu2S3X/0Kysrgggtg/fqoqxFJGoW7ZLeCArjzTpg5E265JepqRJJG4S5y5JE+sNj118PXX0ddjUhSKNxFAG69FXJy4Be/iLoSkaRQuIuAX9R01VXw3HPw0ktRVyOywxTuInEXXQR9+sCFF0JlZdTViOwQhbtIXLNmfnD1yy91cFXSnsJdJNHhh/vB1VGjYPbsqKsR2W4Kd5H6br3Vr2DVlauSxhTuIvV16wZXXgnPPqsrVyVtbVW4m9nRZjbDzL4ws8sbeP1UM1tsZh/Gbmckv1SRRvTLX/qVqxdeqCtXJS1tMdzNLBe4CxgE9AVONLO+DSz61xDCXrHb/UmuU6RxFRT4cMCffw633x51NSLbbGta7vsDX4QQvgohbAAeB4amtiyRJuDoo2HoUPjtb2Hu3KirEdkmWxPuXYDEAa/nxp6r73gzm2pmT5lZt4ZWZGZnmtlkM5u8ePHi7ShXpJHddhvU1MDFF0ddicg2SdYB1eeBniGEPYDXgIcaWiiEcG8IoTyEUF5SUpKkTYukUK9ecPnl8MQTMH581NWIbLWtCfd5QGJLvGvsuf8IISwNIcSPOt0P7Juc8kSagEsv9ZA//3zNuSppY2vCfRJQZma9zKwZMAx4LnEBM+uU8HAIMD15JYpErHlzP6g6bZpfwSqSBrYY7iGEauB84BU8tJ8IIXxqZr8xsyGxxS40s0/N7CPgQuDUVBUsEokf/QgGDfI5VxcujLoakS2yEEIkGy4vLw+TJ0+OZNsi22XmTOjXD044AR5+OOpqJEuZ2ZQQQvmWltMVqiJbq6zMp+V75BGYMCHqakQ2S+Eusi1GjIDu3eG886C6OupqRDZJ4S6yLVq29HPfP/4Y7ror6mpENknhLrKtjjsOjjoKrr5aB1elyVK4i2wrMx93Zt06uOSSqKsRaZDCXWR79O7twf6Xv8C//x11NSLfoXAX2V4jR0KPHnDuubpyVZochbvI9mrRAsaMgU8/9XuRJkThLrIjhgyBwYPh17/WsMDSpCjcRXZE/OBqTY3P3iTSRCjcRXZUr17e//7kk/Dyy1FXIwIo3EWS45JLoE8fv3J13bqoqxFRuIskRUEB3H03fPUVjBoVdTUiCneRpDnkEDjlFBg9Gj77LOpqJMsp3EWS6ZZbfPyZc86BiIbTFgGFu0hydezoLfc33tCY7xIphbtIsp1xBhx8sJ8auXhx1NVIllK4iyRbTg7cey+sXq1z3yUyCneRVNhtN7jiCh9Y7NVXo65GspDCXSRVrrjCz30/+2xYuzbqaiTLKNxFUqWw0LtnZs2Cq66KuhrJMgp3kVQaONBb7rffDu++G3U1kkUU7iKpNno0dO4MP/85rF8fdTWSJRTuIqnWujXcc4+P+37DDVFXI1lC4S7SGI45Bk46Ca6/HqZOjboayQIKd5HGMmYMdOgAP/uZpuWTlFO4izSWDh3gj3+EDz/UyJGScgp3kcY0dCicfDJcdx188EHU1UgGU7iLNLYxY6CkxLtndPaMpIjCXaSxtW/vFzd9/LFPrC2SAgp3kSgMHuyjR44eDRMmRF2NZCCFu0hUbrsNdt7ZZ29atSrqaiTDKNxFolJUBI88AnPmwIUXRl2NZBiFu0iUBgyAkSPhoYfgySejrkYyiMJdJGpXXQX9+8OZZ8Ls2VFXIxlC4S4Stfx8GDsWamth+HCoro66IskACneRpqBXL7969e234dpro65GMoDCXaSpGDYMTj/dBxf75z+jrkbSnMJdpCm54w743vfgxBNh/vyoq5E0pnAXaUpatoSnnoKKCm/Jq/9dtpPCXaSp6dsX7rvPr1wdOTLqaiRNKdxFmqLhw33u1Ztugr/9LepqJA0p3EWaqttug/328+EJpk2LuhpJMwp3kaaqsBCeeQZatIBjj4UVK6KuSNKIwl2kKevaFZ5+Gr7+2rtqamqirkjShMJdpKk7+GD4/e9h3Di44oqoq5E0kRd1ASKyFc46Cz76CG6+GXr39rHgRTZD4S6SLu64A778Es45x4crOOywqCuSJkzdMiLpIi8PnngC+vSB44+H6dOjrkiaMIW7SDpp0wZeeAEKCmDQIA1RIJukcBdJNz17wksvwdKlcPTROkVSGqRwF0lH++7r58B/9hkMHQqVlVFXJE2Mwl0kXR1xBDz8MPz73z7IWFVV1BVJE6JwF0lnw4bBnXfC3//uwxToIieJ0amQIunuvPNg7Vq49FI/0Prgg5Cjdlu2U7iLZIJLLoF16+Caa6BZM5+yTwGf1RTuIpniqqtg/XoYNcr73x94AHJzo65KIqJwF8kUZnDddd41c801HvQPPwz5+VFXJhFQuItkEjO4+moP+Msv91Mkx4714YMlq6hTTiQTXXYZjBnjszgddZQudMpCCneRTHXhhfDYYzBxIgwcqKEKsozCXSSTnXgivPgizJoFAwbA1KlRVySNROEukumOOAL+9S+oroaDDoLnn4+6ImkECneRbLDPPvDeez5c8NChcMstEELUVUkKKdxFskWXLj4OzfHH+0VPw4bB6tVRVyUponAXySYtWviEHzfeCE89Bf37a9KPDKVwF8k2Zn6q5GuvwZIlsN9+8NBD6qbJMAp3kWx16KHw/vtQXg6nngrDh+t8+AyicBfJZl27wj/+AddfD08+CXvu6Y8l7SncRbJdbi6MGAFvveXDFBx+OJx1FqxaFXVlsgMU7iLi+veHDz+EX/0K7r8f+vXzSUAkLSncRaRO8+Zw883w9tvQujUceyz86Ed+haukFYW7iHxX//7wwQd+sdP48dC3r48XX1ERdWWylRTuItKw/Hy4+GL47DNvwV93Hey6K9x3nw9lIE2awl1ENq9rVx8T/p13YJdd4Mwz4fvf9+dqa6OuTjZB4S4iW6d/f3jzTXjmGZ+ndfhw2GMPD3m15JschbuIbD0zOO44+OgjePxxb7kPHw69e8Pdd/sk3dIkKNxFZNvl5MAJJ8Ann8Czz0JJCZx7rnfhXH45fPNN1BVmPYW7iGy/nBw/2PrOOz5m/CGH+KmUvXr50MLPP68um4go3EVkx5n5VH5PPeXnxF92Gbz7LgwZAt27+2PNAtWoFO4iklzdu8OoUTBnjk/Qve++8Lvf+bg1u+/u49h89lnUVWY8hbuIpEZ+fl3XzIIFcNddftXrlVfCbrv56ZRXXukTeNfURF1txrEQ0RjO5eXlYfLkyZFsW0QiNHeuH4R9+mk/tbKmBoqL4cgjfb7Xww/3A7PSIDObEkIo3+JyCncRiczy5fDKK/Dii/Dqq7BokT/fu7f34Q8cCAcfDD17er++KNxFJM3U1vqpla+9Bm+8ARMmwMqV/lppKRxwgF9Ite++fmvfPtJyo6JwF5H0VlPjYf/mm37mzTvvwMyZda/36OEHaffYww/Ufv/7UFbmV89mMIW7iGSeZct8tMopU3yKwI8/hhkz6g7I5ub64Ga9e/utrMzHw9l5Z+jWzQ/yprmtDfe8xihGRCQp2reHww7zW1xlJUyfXnebNs1b+K+95q/F5eZCly7e4u/Rww/aduni9507Q6dO3v2TAT8AsJXhbmZHA2OAXOD+EMKN9V4vAB4G9gWWAieEEL5ObqkiIg0oLIS99/ZbotpamDcPvvrKb19+6cMizJ7t/fnz50NV1XfX16EDdOwIO+3kwyoUF/t9hw5+a9/eb23bQrt2ft8Eu4K2GO5mlgvcBRwBzAUmmdlzIYRpCYv9HFgeQtjVzIYBo4ETUlGwSDqZMcMbkWVl0KePPx4/3l875BB/bkvvzc31XocFC/y6oAEDvJdh7FiYPBlCgP3282ONNTV129pUDQDjxsEjj3i+gTda+/b1hu/8+bB2rU/KdPjhnpllZZ6PL7zgOXfiif6+eA2LFn13+T594Le/9Qb03nv7qe0TJsCKFX5stKTEu9Q/+QTWrPH6L7jA1zt+PCxc6HV//rkfV12xwl/r0QOOOaauhvg+mj3b31Na6svU1ORQVtYNSrsxfvr/Y2EelB4Ih4yMvW9GLc0rFrP847m0WLWAbnkLaLZkATXzFhK+XUSnxd/SfsFUWLLEu4M204Vdk19AaN2GvPZtoFUrP5+/VSsoKoKiIpZtKGJJZUtC85ZUNWtJwRmnULZf2234L2nbbbHP3cwGAL8OIRwVe3wFQAjhhoRlXoktM9HM8oCFQEnYzMrV5y6ZbsYMv1AzHs7DhsG993o3MfgUpaNHNxzw8fdWVPi0pqWlfvV++/beIO3QwZeJ9zo0a+avHXig58mIEXU/Jok1jBjhIX366R7I8eHYE88yTPy/NifHg7iw0Hs71qzx1/v18/upUzceCDK+/K67eoP2D3/w5WpqoEUL/9HIzfXnWrb0zxffXm6uHxMtKfFAX7QI1q9veN/m5/sJMzvt5O977z3fxurVnqm5ubD//v5cRYWvb/lyb2j37u37KDcXJk2q+/y1tX5xbXw/5+XBnXfCoEEw7oUarjp/Oe3CMtpUL+Xys5ZTvstyvp2xgjefX06L6lUUbFhF+a4raW2rfXLxVatgzRqqV1ZQtbyCguo15OAf9uIfz+LMUT03++O+Kcnsc+8CzEl4PBfov6llQgjVZrYS6AAsqVfUmcCZAN27d9+KTYukr3iLsnt37w2YONGDplUrf72iwpdp6H/w+HsLCjwwV670AGrdGpYu9RYq+HMh+G39el8+N7duvfVrmDnTg3D9+oYDvf6p5CH4tisq/D0tWsCGDb79EL47V0d8+dxceP11f66gwEO9stLXHw/S9es3/iEJwf86MfNg3dw8ILW18O23/gPRsaOHeG2th378fQUF/gOxZImvLz/f75fEUqljRx/TrLDQH1dVbbyfKyv9Oxs0CCa+l8vK/GKKuhRMK2SWAAAE+0lEQVTz0Tx4rhrKT4JJL8CLc+v279ofw+DBG9f68gs+BH7lusDU9yo5cK+11LRqu8nvPlkadfiBEMK9IYTyEEJ5SUlJY25apNGVlXnofPON3w8Y4C3G1av9VlTky2zuvevXe1i1aePht2qVB2dpqS+XGMoFBb58vGumoRrKyryOgoKNgzUeuvWZ+baLi+tCuqrKt19a6j88DS1fU+NdNFDX+i4srPshitebuE0z7x4qLvbQrb/uRDk53movKvL15+b6c1VVfp+b688XFdWtr6rK74uL696Xl+fP1dT4exL3c22t7yvw+3gXfuLzDe3fTX6XG4x1NGcpHagOuZv87pNF3TIiKaQ+9yj73OvCNr6+0lLf7/XfF19v/f08aFDdfhw3zlvy9Z9vaP9u6bvc3LJbkrTz3GNh/TlwGDAPmAQMDyF8mrDMecDuIYSzYwdUfxxC+Mnm1qtwFxHZdknrc4/1oZ8PvIKfCvmnEMKnZvYbYHII4TngAeARM/sCWAYM27HyRURkR2zVee4hhJeAl+o9d3XCvyuB/0luaSIisr00nruISAZSuIuIZCCFu4hIBlK4i4hkIIW7iEgGUriLiGSgyCbrMLPFwOxINl6nmHrj32Qofc7Mos+ZWbb1c/YIIWxx/JbIwr0pMLPJW3OlV7rT58ws+pyZJVWfU90yIiIZSOEuIpKBsj3c7426gEaiz5lZ9DkzS0o+Z1b3uYuIZKpsb7mLiGQkhbuISAbKynA3s0Ize8/MPjKzT83s2qhrSiUzyzWzD8zshahrSRUz+9rMPjazD80sY2eBMbO2ZvaUmX1mZtNjM6VlFDPrE/se47dVZvZ/UdeVbGZ2USx/PjGzsWZWmNT1Z2Ofu5kZ0DKEUGFm+cCbwC9CCO9EXFpKmNkvgXKgdQhh8JaWT0dm9jVQHkLI6ItezOwhYEII4X4zawa0CCGsiLquVDGzXHwGuP4hhKgvekwaM+uC507fEMI6M3sCeCmE8OdkbSMrW+7BVcQe5sduGfkrZ2ZdgWOA+6OuRXaMmbUBBuIznxFC2JDJwR5zGPBlJgV7gjygeWwq0xbA/GSuPCvDHf7TVfEhsAh4LYTwbtQ1pcjtwKVAbdSFpFgAXjWzKWZ2ZtTFpEgvYDHwYKyb7X4zaxl1USk2DBgbdRHJFkKYB9wCfAMsAFaGEF5N5jayNtxDCDUhhL2ArsD+ZtYv6pqSzcwGA4tCCFOirqURHBxC2AcYBJxnZgOjLigF8oB9gLtDCHsDa4DLoy0pdWLdTkOAJ6OuJdnMrB0wFP/B7gy0NLOTk7mNrA33uNifteOBo6OuJQUOAobE+qMfBw41s79EW1JqxFpChBAWAc8C+0dbUUrMBeYm/JX5FB72mWoQ8H4I4duoC0mBw4FZIYTFIYQq4BngwGRuICvD3cxKzKxt7N/NgSOAz6KtKvlCCFeEELqGEHrif97+M4SQ1NZBU2BmLc2sVfzfwJHAJ9FWlXwhhIXAHDPrE3vqMGBahCWl2olkYJdMzDfAAWbWInaCx2HA9GRuIC+ZK0sjnYCHYkfic4AnQggZe5pgFtgJeNb/HyEPeCyE8HK0JaXMBcCjsS6Lr4DTIq4nJWI/0kcAZ0VdSyqEEN41s6eA94Fq4AOSPAxBVp4KKSKS6bKyW0ZEJNMp3EVEMpDCXUQkAyncRUQykMJdRCQDKdxFRDKQwl1EJAP9f+rFPmA65SVlAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#  helper function for approximation\n",
    "def sigmoid_mu_s(x, mu, s):\n",
    "    return 1 / (1 + np.exp(-(x + mu) / s))\n",
    "\n",
    "plt.scatter(X, Y, s=10, c='b', alpha=0.5)\n",
    "sig_x = np.linspace(X.min(), X.max(), 100)\n",
    "y_flip = 1 if Y[X.argmin()] == 0 else -1\n",
    "sig_y = sigmoid_mu_s(sig_x, -X.mean(), y_flip * 0.5)\n",
    "plt.plot(sig_x, sig_y, c='r')\n",
    "plt.yticks([0, 0.5, 1])\n",
    "plt.title('Approximation')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hypothesis and Prediction\n",
    "\n",
    "We can tweak the sigmoid function by passing the dot product of $x$ (vector of feature) and $\\theta$ (the vector of weights).\n",
    "\n",
    "$$h(x) = \\sigma(x \\cdot \\theta) = \\sigma(\\theta_0 x_0 + \\theta_1 x_1 + \\cdots + \\theta_n x_n)$$\n",
    "$x_0$ being equal to 1.  \n",
    "$\\theta$ is not a parameter of $h$ because it's considered as a constant.\n",
    "\n",
    "$h$ is our hypothesis function. It return a probability of an $x$ to belong to one of the group.  \n",
    "To predict some new $x$ we input it to $h$ with the weights and see if the returned value if above or below 0.5.\n",
    "\n",
    "$$\n",
    "\\text{prediction} =\n",
    "    \\begin{cases} \n",
    "        1, & h(x) \\ge 0.5 \\\\\n",
    "        0, & h(x) \\lt 0.5\n",
    "    \\end{cases}\n",
    "$$\n",
    "\n",
    "This is equivalent to:\n",
    "\n",
    "$$\n",
    "\\text{prediction} =\n",
    "    \\begin{cases} \n",
    "        1, & x \\cdot \\theta \\ge 0 \\\\\n",
    "        0, & x \\cdot \\theta \\lt 0\n",
    "    \\end{cases}\n",
    "$$\n",
    "\n",
    "Indeed, in the sigmoid funcion:  \n",
    "* $x$ is *above* 0 then $e^{-x}$ tend to be small, $\\frac{1}{1 + \\text{small number}} \\approx 1$.  \n",
    "* $x$ is *below* 0 then $e^{-x}$ tend to be big, $\\frac{1}{1 + \\text{big number}} \\approx 0$. \n",
    "\n",
    "Therefore, a prediction can be visualized by projecting $x$ onto $\\theta$.  \n",
    "It will be more relevant the more the two vectors go in the same direction\n",
    "(or opposite direction for negative projection),\n",
    "for exemple if $x \\cdot \\theta = 0$ then $x$ is perpendicular to $\\theta$,\n",
    "$h(x) = 0.5$ right in the middle of the sigmoid function,\n",
    "the prediction become more irelevant as $x \\cdot \\theta$ tend to 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hypothesis(x, theta):\n",
    "    # suppose that x_0 = 1 and isn't mentionned\n",
    "    return sigmoid(theta[0] + x.dot(theta[1:]))\n",
    "\n",
    "\n",
    "def predict(x, theta):\n",
    "    return int(hypothesis(x, theta) >= 0.5)  #  a bit more clear\n",
    "    # or\n",
    "    return int(theta[0] + x.dot(theta[1:]) >= 0)\n",
    "    # there is a nice vector interpretation for this one\n",
    "    # and it probably compute faster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cost function\n",
    "\n",
    "We have to evaluate our model by introducing a cost function, this one is called the *logloss*\n",
    "\n",
    "For a single $x$:\n",
    "$$\n",
    "j(h(x), y) = \n",
    "    \\begin{cases} \n",
    "        -\\ln(h(x)),     & y = 1 \\\\\n",
    "        -\\ln(1 - h(x)), & y = 0 \n",
    "    \\end{cases}\n",
    "$$\n",
    "It can be rewritten as a single continuous function:\n",
    "$$ j(h(x), y) = -y \\ln(h(x)) - (1 - y) \\ln(1 - h(x))$$\n",
    "\n",
    "Even if it may seem more complicated at first sight, it's just a switch with the values of $y$.  \n",
    "- if $y = 1$ then $- (1 - y) \\ln(1 - h(x)) = 0$ and the $-\\ln(h(x))$ stays.\n",
    "- if $y = 0$ then $-y \\ln(h(x)) = 0$            and the $-\\ln(1 - h(x))$ stays."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'cost function')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEICAYAAABCnX+uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd4XNW57/Hv0qj33qsly0XusokLGIwNpncIhE4CJ+dCThLSTs5NwiE3yeEEQhJCEkIPhAAJzUAwphlsDO62bLnIlmVZvfcuzaz7xx7JTbaKZ7SnvJ/nmWc0nj17v8oz/LK09ipKa40QQgj35WN2AUIIIc6MBLkQQrg5CXIhhHBzEuRCCOHmJMiFEMLNSZALIYSbkyAXHkMpdbVSqlwp1aGUmjuB171ZKfXBRF1PiBMpGUcu3IFS6lPgb1rrp09zzCHgfq31KifWkQkcBvy01gPOuo4QYyEtcuFJMoA9ZhchxESTIBdOoZRKU0q9oZSqV0o1KqUet/+7j1LqJ0qpI0qpOqXUC0qpCPt7gUqpv9mPb1FKbVFKJSilfgmcAzxu7zZ5/IRrBSilOgALUGBvmaOU0kqpnGOOe14p9Qv7z+cppSqUUt+z11GtlLrzmGODlFK/sdfZqpT6XCkVBKyzH9Jir2WRUuoOpdTnx3x2sb32Vvvz4mPe+1Qp9f+UUhuUUu1KqQ+UUrGO/V9feBsJcuFwSikL8C5wBMgEUoBX7G/fYX8sAyYBocBgMN8ORABpQAzwTaBba/1/gfXAfVrrUK31fcdeT2vdq7UOtb+crbXOHmWpifbrpQBfB/6olIqyv/cIkA8sBqKBHwI2YKn9/Uh7LV+e8LtHA/8CHrP/Do8C/1JKxRxz2NeAO4F4wB/4/ijrFWJYEuTCGc4CkoEfaK07tdY9WuvBFuvNwKNa6xKtdQfwY+BGpZQv0I8Rfjlaa6vWepvWus2JdfYDP9da92ut3wM6gClKKR/gLuDbWutKey1faK17R3HOS4GDWusXtdYDWuuXgf3A5ccc85zW+oDWuhv4BzDHsb+W8DYS5MIZ0oAjp7gZmIzRUh90BPAFEoAXgTXAK0qpKqXUr5VSfk6ss/GEGrsw/kKIBQKBQ+M454m/H/bXKce8rhnmmkKMmwS5cIZyIN3eyj5RFcZNyUHpwABQa28ZP6i1no7RpXEZcJv9uPEMr+oCgo95nTjKzzUAPcBwXTQj1XHi7wfG71g5ymsLMWYS5MIZNgPVwENKqRD7Tcwl9vdeBr6rlMpSSoUCvwJe1VoPKKWWKaVm2vvY2zC6Pmz2z9Vi9KmPxU7ga0opi1LqIuDc0XxIa20DngUeVUol2z+/SCkVANTbazpVLe8BuUqprymlfJVSXwWmY9wzEMIpJMiFw2mtrRh9wjlAGVABfNX+9rMYXSjrMMZj9wDfsr+XCLyGEeL7gM/sxwL8HrhOKdWslHpslKV8215HC0bf/Ftj+DW+D+wGtgBNwP8CPlrrLuCXwAb7yJqFx35Ia92I8ZfE94BGjJukl2mtG8ZwbSHGRCYECSGEm5MWuRBCuLnhbkadRClVCrQDVmBAaz3fmUUJIYQYvVEFud0y6ecTQgjXI10rQgjh5kZ1s1MpdRhoxhhD+xet9ZPDHHMPcA9ASEhIflZOLgdq20mPDiYiyJlzOoQQwg11NUJLGSTMAIsf27Zta9Bax43nVKMN8hStdaVSKh74EPiW1nrdqY6fP3++fvODdZzz67U8cv1srstPHU9tQgjhuTb9BVb/EH54GIKjUUptG+/9x1F1rWitK+3PdcCbGGtpnFaAr3Hq3gHreOoSQgjPNtBjPPsGnPGpRgxy+8y8sMGfgQuBwpE+F+BrAaCn3zbCkUII4YX6B4M88IxPNZpRKwnAm0qpweP/rrV+f6QPBfhJi1wIIU5poAd8/MDHcsanGjHItdYlwOyxnnioa0Va5Kbq7++noqKCnp4es0sRLiQwMJDU1FT8/GQggmkGeh3SGoexjSMfE6UU/r4+9EiL3FQVFRWEhYWRmZmJ/a8q4eW01jQ2NlJRUUFWVpbZ5XivgR6H9I+Dk8eRB/j6SIvcZD09PcTExEiIiyFKKWJiYuSvNLMN9DisRe7UIA/0s9A7IEFuNglxcSL5TriAgR7wc4MgD/D1kZudQggxHAf2kUvXihBCmMF9+sgt0iIXQojh9LtNH7mP9JELh7rrrruIj49nxowZTjl/aOjI+yB3d3dz7rnnYrUajZSPP/6YW2+9dcTjhtPX18fSpUsZGBgY9rXwYO5yszPA10JPv7TIhePccccdvP/+iPPRnOrZZ5/lmmuuwWIxJnIUFBQwd+7cEY8bjr+/P8uXL+fVV18d9rXwYG7TRy4tcgEUFhayePHiodfbt29n+fLl4zrX0qVLiY6OdlRpwyotLWXatGncfffd5OXlceGFF9Ld3T30/ksvvcSVV1459LqgoICamhqWLl1Keno6H3300bDHLVu2jA8//BCAn/zkJ3zrW8ZWpVdddRUvvfTS0HEnvhYeyoF95E6bEARys9PVPPjOHvZWtTn0nNOTw3ng8rzTHzN9OiUlJVitViwWC/fffz+PPvroccecc845tLe3n/TZRx55hBUrVji05tE4ePAgL7/8Mk899RQ33HADr7/+Orfccgt9fX2UlJSQmZk5dGxBQQE333wz69at48033+Sll15i6dKlJx334IMP8rOf/Yy6ujp27NjB22+/DcCMGTPYsmXL0HEnvhYeyoFdK04N8kA/i8zsFPj4+JCXl8eePXs4ePAgGRkZzJs377hj1q9f77Trr1ixgpqampP+/Ze//OVxLeZjZWVlMWfOHADy8/MpLS0FoKGhgcjIyKHj+vv7aWxs5Hvf+97Q68jIyJOOA+OvCa01jz76KJ9++ulQl4vFYsHf35/29nbCwsJOei08lAPHkUuL3IuM1HJ2poULF7Jhwwb+9Kc/DdvH7cwW+WBXx1gEBBz9k9disQx1rQQFBR03I3Lfvn3Mnj0bHx+jl3LXrl3MmDHjpOMAdu/eTXV1NTExMScFdG9vL4GBgad8LTyQ2/SRy/BDYbdw4UJ+8pOfcPXVV5OSknLS++vXr2fnzp0nPUYb4suXL6eystLRZZ8kKioKq9U6FNIFBQXMnn10Tbldu3Yxa9ask46rrq7m5ptvZtWqVYSGhh73f2aNjY3ExsYOLWB14mvhodxnHLnc7BSGqVOnEhAQwI9+9KMzOs9NN93EokWLKCoqIjU1lWeeeQabzUZxcbHTb4IOuvDCC/n8888BI8hnzZo19F5hYeHQ0MjB47q6urjmmmv4zW9+w7Rp0/jpT3/Kgw8+OPSZtWvXcumll57ytfBANitY+xzWIkdr7fBHfn6+1lrrh9/fr7P+811ts9m0MMfevXvNLkFrrfW9996rn3/+eaece/fu3fq73/2uU849nG3btulbbrnFYcddffXVuqio6JSvncVVvhteqbdT6wfCtV7/26F/ArbqcWau01vkNg391pH3BRWe6dChQ0ydOpXu7m5uv/12p1xjxowZJ42CcaZ58+axbNmy0070Ge1xfX19XHXVVeTm5g77WnioAcftDgROvtkZGWz08TV39ZEQLjduvFF2djb79+83uwyHu+uuuxxynL+/P7fddtspXwsP1dVoPAdFOeR0Tm2Rp0QFAVDR3D3CkUII4UVayoznyDSHnM6pQZ4caQR5ZYsEuRBCDGktN54jUh1yOue2yO1BXiVBLoQQR7VWgLJAWLJDTufUIA8L9CM80JdK6VoRQoijWsohPBksjrlN6dQgB0iJCpauFSGEOFZrOUQ4pn8cJiLIIwOla0UIIY7VWu6w/nGYkCAPkq4VIYQYZLNCW5XDRqzAhHStBNHeO0Brd7+zLyWEEK6vvRpsA+7VtZIsI1eEA73//vtMmTKFnJwcHnroIYeffzxbvTlq+7lTbRk33DWHc+w2cbJlnAtrrTCe3SnIB4cgSveKOFNWq5V7772X1atXs3fvXl5++WX27t074XWcuIWbo7afO9WWccNdczjHbhMnW8a5sBb7GHJ361oBmRTkzRy11dvmzZvJyclh0qRJ+Pv7c+ONN7Jq1SpHlgqMfas3R20/d6ot44a75mi2jZMt41xUq31WpwNvdjp1rRWA2JAA/H19JMhdwer/hJrdjj1n4ky4+PRdHI7a6q2yspK0tKOtmNTUVDZt2uSAX+JkY9nqzVGG2zJuxYoVw15zNNvGyZZxLqqlHIJjwD/EYad0epD7+CiSIwIlyL2YJ2/15qjrn2rLuFNdc7TbxsmWcS6otcKhrXGYgCAHo3tF+shdwAgtZ2dyxFZvKSkplJeXD71XUVEx7G5DJ3LmVm+jMZrrn2rLuFNdc7TbxsmWcS6otRxichx6Sqf3kYN9LLm0yL2aI7Z6W7BgAQcPHuTw4cP09fXxyiuvcMUVVwDmbfXmKKfaMm64a4522zjZMs4FaW10rUSmO/S0ExLkyZFB1Lf3yv6dXswRW735+vry+OOPs3LlSqZNm8YNN9xAXl6eqVu9wfDbz43V6baMO/aaY9k2TraMc0HdzdDf6fCuFadu9TboH1vKdMaP3tWH6zscuVmSGAVX2c7LG7d6M+Oax24TN9KWca7y3fAqVTuNLd72rDrpLVx1q7dBMgTRe3nzVm8Tfc1jt4mTLeNclBPGkMMYbnYqpSzAVqBSa33ZWC6SGhkMyKQgb+TtW71N5DWP3SZOtoxzUUMbSpjXR/5tYN94LpIYEYhS0iIXQni51grwDYJgx97PGVWQK6VSgUuBp8dzEX9fH+LDAiTIhRDeraXM6FZRyqGnHW2L/HfADwHbqQ5QSt2jlNqqlNpaX19/0vuynK15jPsoQhwl3wmTOHhDiUEjBrlS6jKgTmu97XTHaa2f1FrP11rPj4uLO+n95MggqlolyCdaYGAgjY2N8h+uGKK1prGxUSYKmcEJszphdDc7lwBXKKUuAQKBcKXU37TWt4zlQilRQXywpxabTePj49g/K8SppaamUlFRwXB/JQnvFRgYSGqq4wNFnEZ/N3TWO3zECowiyLXWPwZ+DKCUOg/4/lhDHCA1Mog+q436jl4SwqUlMFH8/PzIysoyuwwhxNA65I4dsQITNLMTZCy5EMLLtTpnDDmMMci11p+OdQz5oGTZYEII4c0GJwM5oY98wlrk6dHBKAXFdR0TdUkhhHAdDQfANxDCkh1+6gkL8mB/X7LjQimsbJ2oSwohhOuo2gkJM8Di+NXDJyzIAWamRLBbglwI4W1sNqgugOQ5Tjn9hAb5jJQI6tp7qWtz7FrOQgjh0ppKoK8dkjwgyGemRABIq1wI4V2qdxrPntAiz0sORykJciGEl6naAZYAiJvqlNNPaJCHBPgyKTZEbngKIbxLdQEkzgCLc7bdm9AgB7nhKYTwMoM3Op3UPw4mBPmMlAhq23qpa5cbnkIIL9B8GHrbnNY/Dia1yAHpXhFCeIeqHcazJ7XI81IijBueFW0TfWkhhJh41TuNG53x05x2iQkP8tAAX7JiQ6SfXAjhHap2QkKe0250gglBDkb3inStCCE8ns0G1buc2j8OJgZ5TVsP9e29ZlxeCCEmRvNh6G11av84mBTkM+SGpxDCGwze6PTEFnlecjggMzyFEB6ueidY/CHOeTc6waQgDwv0Y5Lc8BRCeLrBG52+/k69jClBDjA3PYotpU1YbbK7uxDCA/X3QMVWSF3g9EuZFuRLc2Np6eqXVrkQwjOVfQkD3ZC93OmXMi3Iz86JRSlYd6DerBKEEMJ5Dn0MPn6QebbTL2VakMeEBjAjOUKCXAjhmYo/gfSFEBDq9EuZFuRgdK/sKG+hraffzDKEEMKx2qqhbg/kOL9bBcwO8slxWG2aL4obzCxDCCEc69AnxvME9I+DyUE+LyOK0ABfPjsgQS6E8CCHPoaQeEiYMSGXMzXI/Sw+LM6OYd2BerSWYYhCCA9gs8KhtUa3is/ERKypQQ6wNDeOypZuSho6zS5FCCHOXPVO6G6asG4VcIEgPzc3DpBhiEIID1H8CaAge9mEXdL0IE+LDiYrNkSCXAjhGQ59DEmzISR2wi5pepADLJ0cy8aSJnoHrGaXIoQQ49fTCuWbJ2zY4SCXCPLzpsbT3W9lvYxeEUK4swNrQFth8oUTelmXCPKzc2KJDvHnrZ2VZpcihBDjt+sfEJEGqWdN6GVdIsj9LD5cOjOJj/bV0tE7YHY5Qggxdh31xkSgmddN2LDDQS4R5ABXzU2mp9/GmsIas0sRQoix2/Om0a0y84YJv7TLBPm89CjSooOke0UI4Z52/8OYyZkwfcIvPWKQK6UClVKblVIFSqk9SqkHnVGIUoorZ6ewobiBuvYeZ1xCCCGco6kEKrbAzOtNufxoWuS9wPla69nAHOAipdRCZxRz1dxkbBreLah2xumFEMI5dr8GKKN/3AQjBrk2dNhf+tkfTlkYJSc+jLzkcFZJ94oQwl1obYxWyVgCEammlDCqPnKllEUptROoAz7UWm8a5ph7lFJblVJb6+vHP0vzqjkpFFS0UlLfMfLBQghhtuqd0HgQZpnTrQKjDHKttVVrPQdIBc5SSp20NqPW+kmt9Xyt9fy4uLhxF3T57GSUgje2S6tcCOEGdr5sbOk2/UrTShjTqBWtdQuwFrjIOeVAYkQg50+J5+XNZTJlXwjh2nrbYeffjRAPijKtjNGMWolTSkXafw4CLgD2O7OoO5Zk0tjZxzty01MI4cp2/h362mHhv5taxmha5EnAWqXULmALRh/5u84s6uycWCbHh/LchsOy4YQQwjXZbLDpL5AyH1Lnm1rKaEat7NJaz9Vaz9Jaz9Ba/9zZRSmluGNJJnuq2thS2uzsywkhxNgVfwhNh0xvjYMLzew80TVzU4kI8uO5DYfNLkUIIU628c8QlmTqTc5BLhvkQf4WbjwrjTV7aqho7jK7HCGEOKpuP5SshQVfB4uf2dW4bpAD3LYoE6UUL355xOxShBDiqE1PgCUA8u80uxLAxYM8JTKIi/IS+fvmMlq7+80uRwghoL0WCl4xJgBN4HZup+PSQQ7wf5Zl094zwDPrS8wuRQgh4PPfgrUPzr7f7EqGuHyQ5yVHcPGMRJ7dUEpzZ5/Z5QghvFlbFWx9FmbfBDHZZlczxOWDHOC7F+TS2TfAk9IqF0KYaf1vjM0jzv2B2ZUcxy2CPDchjMtnJfP8hlIaOnrNLkcI4Y1aymDbX2HurRCVaXY1x3GLIAf49orJ9A5YeeLTQ2aXIoTwRuseBqVg6ffNruQkbhPk2XGhXD03lRc3HqG2TXYQEkJMoKYS2PES5N9h2prjp+M2QQ7wnRWT0Rr+932nrtklhBDH++Cn4BvgUiNVjuVWQZ4WHcw3zsnije2VbDvSZHY5QghvUPwx7H/X6FIJTzK7mmG5VZAD3Lssh8TwQB54ew9Wm6yMKIRwooE+WP0jiJ4Ei+4zu5pTcrsgDwnw5b8unUZhZRuvbik3uxwhhCfb9ISxjdtFDxldKy7K7YIc4PJZSZyVFc3Da/bT0iWThIQQTtBeA5/9L0xeCbkrza7mtNwyyJVS/PflebR29/PwmiKzyxFCeKIPfmpMxb/of8yuZERuGeQA05PDuWNxFi9tKuOLQw1mlyOE8CRFq2H3P+Ds77rUVPxTcdsgB/jByilkxgTzw9d20dk7YHY5QghP0NUE73wbEmbAOa43+Wc4bh3kQf4WHr5+NpUt3Ty0WsaWCyEc4P0fQ2cDXPUn8PU3u5pRcesgB1iQGc2di7N4ceMR6WIRQpyZotWw6xU453uQNNvsakbN7YMcju9iaeuRDSiEEOPQ2Xi0S2Wpa61uOBKPCPIgfwu/uWE21a09/PiN3WgtE4WEEGNgs8Gb/wbdzXDVn92mS2WQRwQ5QH5GNN+7MJd/7arm75vLzC5HCOFOvvwDFH8IK38FSbPMrmbMPCbIAb65NJuluXE8+M5e9la1mV2OEMIdlG+Gjx6E6VfCgm+YXc24eFSQ+/goHr1hNpFBftz39+10yJBEIcTpdDXBa3cZS9Ne8QdjvXE35FFBDhAbGsBjN82ltLGTH/yzAJssrCWEGI51AF7/hjEV//rnIDDC7IrGzeOCHGDhpBj+65JprC6s4bFPDppdjhDCFX30ABz6GC79DaTkm13NGfE1uwBn+frZWeyvaed3Hx1kSkIYF890zXWEhRAm2PESfPk4nPVvkH+72dWcMY9skYOxsNYvr57BvPRI7v9HAXuqWs0uSQjhCso3w7vfgUnnGaNUPIDHBjlAgK+FJ27NJzLYj68/v5Wqlm6zSxJCmKnxELzyNQhPgeueA4tndEp4dJADxIcF8sztC+jsHeD2ZzfL+uVCeKuOOvjbNWCzws3/hOBosytyGI8PcjCWvP3LbfkcaeziG3/dSk+/1eyShBATqbcdXrrOCPOb/wmxk82uyKG8IsgBFmfH8uhXZ7OtrJlvvbyDAavN7JKEEBNhoBdevQVqCuH6v0LqfLMrcjivCXKAy2Yl88Bl0/lwby3f/2eBbN4shKez9sM/74SST40JP7kXml2RU3hGT/8Y3LEki84+Kw+vKcLX4sOvr52Fj497zuYSQpyGdQBe/zoU/QsueQTm3mx2RU4zYpArpdKAF4AEQANPaq1/7+zCnOneZTkMWDW//egAvj6KX109U8JcCE9is8Kb98DeVcYQw7PuNrsipxpNi3wA+J7WertSKgzYppT6UGu918m1OdV/LM+h32rj8bXFKAW/uGomFglzIdyftR/e+ncofB1WPAiL7jW7IqcbMci11tVAtf3ndqXUPiAFcOsgV0rxvQtz0Wj+uPYQXX1WHrl+Nn4Wr7ptIIRn6e8xFsEq+hcsfwDO/o7ZFU2IMfWRK6UygbnApmHeuwe4ByA9Pd0BpTmfUoofrJxKsL8vD68poqvPyh9umkugn8Xs0oQQY9XXaUz2KfkULn4YvnKP2RVNmFE3P5VSocDrwHe01ict9q21flJrPV9rPT8uLs6RNTrdvctyePCKPD7cW8vX/7qFdtkuTgj30tUEL14Nh9fBlX/yqhCHUQa5UsoPI8Rf0lq/4dySzHH74kx+c/1sNpY0ccNfNlLb1mN2SUKI0WguhWcuhKodxrR7Dx6dciojBrlSSgHPAPu01o86vyTzXJufyrN3LKCssZOr/7iBA7XtZpckhDidqh3w9AXQWQe3vgV5V5ldkSlG0yJfAtwKnK+U2ml/XOLkukxzbm4cr/7bIvptmmv//AUbihvMLkkIMZyi9+G5S8E3AO76ADKXmF2RaUYMcq3151prpbWepbWeY3+8NxHFmWVGSgRv/PtikiICue3ZzbzwZanZJQkhBmkNn/8OXr4RYnPg6x9C/FSzqzKVjLU7hbToYF7/98WclxvHz1bt4f++uZt+WZ9FCHP198Cb3zR298m7Cu58H8Jl0xgJ8tMIC/Tjydvm881zs3lpUxk3P7WJuna5CSqEKVrK4flLYNcrsOwnxo1N/2Czq3IJEuQjsPgo/vPiqfz+xjnsqmzh0sc+Z/PhJrPLEsK7FH8Mf1kK9Qfgq3+Dc3/gtjveO4ME+ShdOSeFt+5dQmiALzc9tZGn15egtayeKIRT2Wzw2cPwt2shLBHu+RSmXW52VS5HgnwMpiaGs+q+JayYFs8v/rWPu1/YSlOn7DgkhFO018Lfroa1v4CZ18E3PjJuboqTSJCPUXigH0/cks8Dl09n3YEGLvn9ejaWNJpdlhCepfgjeGIJlG2Cyx+Da54C/xCzq3JZEuTjoJTiziVZvPF/FhPkb+FrT23k4TX76RuQUS1CnJH+Hnj/v4yulJA4oysl/3bpDx+BBPkZmJESwbvfOptr56Xyx7WHuPpPMhtUiHGrLoAnz4WNf4QFd8Pdn3j9+PDRkiA/QyEBvjx8/Wz+cms+1a09XPaHz3l6fYlsIyfEaFn7Yd3D8NRy6G6BW16HSx8BvyCzK3MbXrfVm7OszEtkXnoUP35jF7/41z7e213Nr6+bTU58qNmlCeG6anbDqnuN1nje1XDpoxAcbXZVbkda5A4UFxbAU7fN53dfnUNJQyeXPLaeP31aLDNChTjRQC+s/R948jxoq4IbXoDrn5cQHydpkTuYUoqr5qawJCeWn60q5NfvF/H2zir+55qZzE2PMrs8IcxXugHe/Q40HIBZX4WLHpIAP0PSIneSuLAA/nxLPn+5NZ+Wrn6u+fMX/PStQtpk0wrhrbqajG6U5y+BgR64+TW45kkJcQeQFrmTrcxLZElOLI+sKeKvX5ayurCGH188lWvmpaBkSJXwBjYrbP8rfPxz6GmDJd+Bc38k66Q4kHLGNPP58+frrVu3Ovy87m53RSs/XVXIzvIW5mdE8eCVeeQlR5hdlhDOU7EV3vu+sQFExhK45GFIyDO7KpeklNqmtZ4/rs9KkE8sm03z2rYKHnp/P81dfdy4II37L5hCXFiA2aUJ4ThtVfDRg8ZKhaGJcKF9mr38FXpKZxLk0rUywXx8FDcsSGNlXiKPfXKQv35RyjsF1dx3fg53LM4k0M9idolCjF9fJ3zxOGz4HdgG4Ozvwtn3Q2C42ZV5NGmRm6ykvoNfvbePj/bVkRIZxPdX5nLl7BR8fKTlItyIdQB2vgRrfwUdNTD9Srjg5xCVaXZlbkO6VjzAF8UN/Gr1Pgor25ieFM6PLp7K0smxckNUuDatoWg1fPwg1O+H1AVwwf+DjEVmV+Z2JMg9hM2meWdXFQ+vKaKiuZuvZEXzw4umkJ8hw7OECzq8zhiJUrEFoifBiv+GaVdIP/g4SZB7mN4BK69sLucPnxTT0NHLsilx3H/BFGamyggX4QLKN8PaX0LJpxCWDOf9CObcDBY/sytzaxLkHqqrb4DnNpTy5LoSWrv7WTEtge+smMyMFAl0YYKKrUYf+KGPITjGuIm54BvgF2h2ZR5BgtzDtfX08/yGUp5eX0JbzwArpsVz3/mTmZMWaXZpwhsc+QLWPXI0wBf/B5x1t2z04GAS5F5iMNCf3XCYlq5+zpkcy73LcvhKVrTcFBWOpTWUrDUC/MgGY5OHRfca64QHyIqeziBB7mU6egd4aeMRnlpfQkNHH3PTI/nmudlcMC1Bhi2KM2Ozwt5V8PnWLjfhAAATy0lEQVRvoWaX0Qe+5Nsw7zaZUu9kEuReqqffyj+3lvPk+hLKm7rJjgvh7nMmcdXcFJlYJMamr8sYB/7lH6H5MMTkGAE+66vgK7OOJ4IEuZcbsNp4r7CGJz49xN7qNmJD/bltUSa3LMwgOsTf7PKEK2uvhS1PwZZnoLsJUvKNRa2mXgo+0hiYSBLkAgCtNV8eauSp9SWsLaonwNeHq+akcOfZmUxNlCnS4hhVO2DjE1D4ujGVfuqlsOg+SF8o48BNImutCMDY1GJxTiyLc2I5WNvOc1+U8sb2Cl7dWs7i7BhuW5TJimnx+FpkGXqvZO2HfW/D5qeg7EvwD4X5d8JZ/waxOWZXJ86AtMg9XEtXHy9vLufFL0upau0hOSKQmxdmcMP8NFlx0Vu0VcH2F2Drc8Y6KFGZxuiTebdCoMxJcBXStSJGNGC18fH+Ol74spQNxY34WRQr8xK5ZWGGDF/0RDYbHP7U6PsuWg3aCjkXwFn3QM4K8JG/ylyNBLkYk+K6Dv6+qYzXtpXT1jPApLgQblqQzjXzUogJlVa6W2uvhZ1/g+0vGqNPgqJh7i2QfwfEZJtdnTgNCXIxLt19Vt7dVcXLm8vYXtaCn0VxYV4iX52fxpKcWCwyJt09WAeMWZfbX4AD7xs3LzPOhvzbjeVkZfigW5AgF2esqKadV7aU8eaOSlq6+kmJDOLa/FSum5dKeoxMBHFJDcVG63vny0bfd3AszLkJ5t0OsZPNrk6MkVODXCn1LHAZUKe1njGak0qQu6/eASsf7q3lH1srWH+wHq3hrMxors1P4ZKZSYQFygp3pupugT1vwM6/G8vHKgtMvtDoPsldKSsQujFnB/lSoAN4QYLcu1S1dPPmjkpe31ZBSUMngX4+XDA9kavnJnPO5Dj8ZBjjxBjog+IPoeAVo+vE2gdx02DO12DWDRCWaHaFwgGc3rWilMoE3pUg905aa7aXtfDmjgre3VVNS1c/MSH+XDYriSvmpDAvPVJGvTiazWaM9d79T9j7FnQ3G10nM68zps0nz5WJOx7GJYJcKXUPcA9Aenp6/pEjR8ZTj3BxfQM2Pi2q462dlXy0r46+ARtp0UFcMTuZy2YlMzUxTEJ9vLSG6gJjtmXhG9BWAX7BMOUSI7yzl0nXiQdziSA/lrTIvUN7Tz9r9tSyamclXxxqxGrT5MSHctmsJC6blUROfJjZJbo+raF2j9HqLnwDmg6Bjy9knw8zb4ApF8uysV5CglyYrrGjl9WFNbxTUMXm0ia0htyEUC6ZmcQlM5OYHB8qLfVBWkNtobFc7J63oPEgKB/IPAdmXAvTLodg2afV20iQC5dS19bD6sIa/rW7mi32UJ8UF8IlM5K4aEYiecnh3hfqWkPldmOtk72rjMk6ygcyz4a8q2Hq5RAaZ3aVwkTOHrXyMnAeEAvUAg9orZ853WckyMWgurYe1uypYXVhDRtLGrFpSIkMYmVeIivzEpifGe25E4+s/cY2afvfhX3vQnuV0W2StdTYbX7qZRLeYohMCBJuobGjl4/31bFmTw3rixvoG7ARHeLP+VPjuWB6AudMjiXY380X5Oxth+KPoeg9OLAGelrANwhylhvBnbtSuk3EsCTIhdvp6B3gs6J6Ptxbwyf762jrGcDf14cl2TGsmJ7A8qkJJEa4ye7sLeXG+O6i1VC63hjnHRQNuRfB1Esge7lskyZGJEEu3Fq/1cbmw018tK+Wj/bVUt7UDUBecjjLp8azbGo8s1MjXWc/UpvVmFV5YI3xqNtj/Ht0tjHKZMrFkLYQLG7+14WYUBLkwmNorTlY18HH++r4ZH8t2440Y9MQE+LPublxnDc1nqWTY4kMnuAt7DobjC6Tgx8YC1R1NxvT4zMWG90luRfJ+ibijEiQC4/V3NnHuoP1rN1fx2cH6mnu6sdHwey0SM7NjePc3DhmpUY6/oapdQAqt0LxR8ajaiegISTOWNd78gqjyyQo0rHXFV5Lglx4BatNU1DRwmdF9Xx2oJ6Ciha0hshgP5bkxLJ0ciznTI4jOTJofBdoPgKHPjFa3CXroLfVGCKYepaxGUPOckiaI5syCKeQIBdeqamzj8+LG1h3oJ71B+upbesFIDsuhHMmx3F2TixfmRR96hUbu1ug9HMoWWsEeFOJ8e/hqZBzvjG7ctJ5EBQ1Ib+P8G4S5MLraa05UNvB+oP1rD/YwKbDjfT027D4KOakRbIkJ5YlmaHMpQj/svVQ8hlUbQdtA78QyDoHJi0z1jOJzZUFqcSEkyAX4gQ9/VZ2lNZxuOBzdMlnZHVsJ18dIED1Y8WHuvCZMOlc4mavxDftLPCd4JunQpzgTIJcxkcJz2Hth6odUPo5gaWfs6hsI4v6O423EmdQEXkLn1un84/6NArqbFAHoTs6WZC5k4WTYlg4KYa85HB8ZZ114WYkyIX7Gug11i858jmUboDyzWAP7qGNF7KWQubZWIKjyQAygJuBho5eNpY08uWhRjaWNLK2qB6A0ABf8jOi+MqkaL6SFcPMlAj8fSXYhWuTrhXhPnrbjbAu+9JYw6RiK1iNG5zE5xljujPPhowlY17DpK69h00lTWwsaWTz4SYO1nUAEOjnw7z0KBZkRnNWVjRz0yPdfxkB4ZKkj1x4prZqKN8IZZug7Auo2W3cnFQWSJplBHb6IiPAHbx+SUNHL1sON7HpcBNbSpvYW92G1mDxUcxIDmdBZjTzM6PIz4gmLkx2qRdnToJcuD+bFer2Qvkme6t7I7TYd5nyDYLU+UZopy+EtLMgYGI3rWjr6WfbkWa2ljaxpbSZneUt9A3YAMiMCSY/I5r8jCjyM6KYHB/qOssJCLchQS7cT3czVGwzgrtis/FzX7vxXmgCpH3FHtoLjda3i21x1jtgpbCyjW1Hmtha2sy2I800dvYBEBboy9z0KOalR5KfEcXstEjCTzWWXQg7CXLh2qwDRmu7YgtUbjNa3I0HjfeUDyTkGbMnB1vbkRluN45ba82Rxi6j1X6kmR1lzRTVtqO18avkxIUyNz2SuelRzE2PZHJ8mOeuwy7GRYJcuA6tobXCCOzKrUZLu3on9HcZ7wfHGKGdtgBSF0DyPI/dk7K9p5+C8lZ2lDWzvayZHeUttHT1AxDib2FmagRz0qKYkxbJnLRI91m2VziFBLkwT2ejMXa7arsxFLByG3TWGe9Z/CFxphHYKfONfu6oTLdrbTuK1prSxi52lBl97DvKWthf00a/1fhvMCE8gFmpRqjPSo1gVkokEcHSJeMtZEKQmBg9rVBdYAR35XbjefCGJMqY2p6zAlLmQUo+JMyQGZPHUEqRFRtCVmwI18xLBYwZqPuq29hZ3sKuilYKylv4cG/t0GcyYoKZlRrJ7NQIZqQYj9AA+c9WHE++EWJ43S1Qs8tYvrV6p/HcdOjo+5HpRrfI/LuM4E6aA4Hh5tXrpgL9LPZ+86MLc7V297O7opVdlS3sKm9lW2kT7xRUAcYfM1mxIcxMiWCmPdjzksNPvTCY8ArStSKMTROqC45/NB8++n5EGiTNhuQ5RngnzYGQGPPq9UINHb3srmxlV3kruytbKaxspaatZ+j9rNgQo8WeHE5eshHuUSHy15A7kT5yMTpaG10h1buMyTU1u4yf26uOHhOZbgR18hxItId3SKx5NYtTqm/vpdAe6oVVrRRWtlHZ0j30fkpkENOSwslLNh7Tk8NJiQxCeek9ClcnfeTiZP09UL8PagqhttAe3IXGZglgDPuLzTWmtCfNMlrciTNl7W03EhcWwDL7nqaDmjv72FPVxp6q1qHnj/fXMtheiwjyY3pSONOSwpmWFMa0pHAmJ4QS4Gsx6bcQjiAtcnenNbTXGGFdW3g0uBsOgrYax/iFQMJ0SJxlhHXiTIifLju7e4muvgH2Vbezr7qNvdVt7Klqo6imjZ5+Y2aqr48iOy50KNinJYUzNSmMuNAAab1PIGmRe4veDqjfD7V7jAk2tXuMR3fT0WMi0ozRIlMvOxraUVmyPZkXC/b3HVo+YJDVpilt7GRvVRv7qo3HxpIm3tp5tJstJsSfqUlhTEkwgn1qYhiT48MI8pfWu6uRFrkrsvZDY7E9rPdC3T6o2wPNpUeP8QuB+GlGSzthpjE7MmG6dI2IM9Lc2cf+mnb21xjhXlTTTlFt+1DrXSnIiA5mSmIYUxLDmZIQxpTEUDJjQmQd9zMkLXJ3ZbNC02GjL7vumEdjMdiMGYAoC8TkQPJcmHOz0SWSMB0iM6WVLRwuKsSfRdkxLMo+OirJatOUNXVRVNPG/pp2I9xr2vlwby02ezvQ3+LDpLgQchPCyE0ItT+HkRYdLEsRTABpkU8E64AxnK9+v/1RBHX7oeHA0fW0wVhjJH6a8Yizt7Zjc8FXlkkVrqen30pxXQdFNe0cqGvnQE07B2o7jhs5E+jnQ3acEew58aFMjg+VgD8FaZG7iv4eozXdUAT1B+zPRca/WfuOHheRDvFTIfs8I7Djp0LsFI9dc0R4pkA/y9Bs02N19A5wsLadg7UdHKht50BdB5tKGnlzR+XQMf6+PkyKDWFyQhg5caHkxBuPzNhgGUEzDhLk49HdbIwKqS8yWtUNB4yfW44YGx8AoCAqwwjoyRdAnD2s4ySwhWcLDfA9abYqGIuIHazroNj+OFDbzo6y5qFZq2Bs3JEeHUy2Pdyz40KM5/hQWQr4NCTIT8VmNYK5odgI6saDRng3HIDO+qPHWfyNPuyk2TDzeiOoY3MhdjL4BZlXvxAuJizQj3npUcw7IeC7+gYoqe8cCvhD9cbzZwfqhhYUA2PcfHZcCNlxoUyKCx36OTkyyOu7abw7yLWGriYjpBuLjaBuLDYeTSXHd4cERRvhnHuRPahzIS7X6Nf2kT8FhRivYH/fYbtoBqw2ypu7h8L9kP353V3VtHb3Dx3n7+tDVkwIk+KMBckmxYWSFRtCdlwIkcHesUyBdwR5b4cRzI3F0HjIWPxpMLR7Wo4e5+MH0VkQMxlyVxrPg61rB+8JKYQ4PV+Lz9BqkReQMPTvWmuaOvsoaejkUF0HJQ2dlNR3DI2kGbAdbcVHBfsNBfuxj8yYEI8aD+85o1b6uoyRIYNB3XjIHt6HoKPm+GPDkiE2x+gSic42gjomx2hdW7zj/9uE8ET9VhvlTV0cbuikpL6TkoZODjd0UFLfSV1773HHJoYHkhkbPBTsmfaQT48OJtBv4kPee0at9HYYYd1UcjSkm+yvj134CSAkzgjpnOUQPQlisu3BPQn8Q8ypXwjhVH4WHybZ+9CXTzv+vY7eAUobOjnc0Gk8NxrPa/bU0tR5tBtVKUgKDyQjJoTM2GDj2f5zenQwwf6uF5uuVZHW0NVohHPz4eOfm0qO7jwzKCTOmH4+6VwjtKOzjMCOzpa1sYUQxwkNGL4vHow14EsbOilt7KS0oYsjjUbQnxjyAPFhAWTEGAGfER1M+jE/Rwb7mbI+zaiCXCl1EfB7wAI8rbV+aNxXtPZDa7kx3by51B7WpUZgNx+B3rbjjw9LNgI6d6XRmo7OMp6jsiSshRAOERHkx+y0SGanRZ70Xmt3P2WNXZQ2dlLW1DUU+OsO1J/UXRMW6GuEfHQIadHBZMQYrfj06GCSIgKdtozBiEGulLIAfwQuACqALUqpt7XWe0/5IduAselu82FjCN9gaDcfMTbmHVyVD4zhe5EZxl6O6YuMgI7KtId1hgzhE0KYKiLIj5mpEcxMPbkl391npazJaMGXNXXZf+5ib3UbH+ytOW74pMVHkRIZRHp0MGn2cE+LDhoK+jMxmhb5WUCx1roEQCn1CnAlcOogr9kNT59/9HVInBHOqQtg1g3Gz4OPsGRZM0QI4ZaC/C32BcTCTnrPatNUt3ZT1tRF+TEhX97czZo9NSd12ZyJ0QR5ClB+zOsK4CsnHqSUuge4x/6yVz3YVnj03Tbg0IkfEcIMsUCD2UUIMYwp4/2gw252aq2fBJ4EUEptHe8wGiGcSb6bwlUppcY9Zns0fRqVQNoxr1Pt/yaEEMIFjCbItwCTlVJZSil/4EbgbeeWJYQQYrRG7FrRWg8ope4D1mAMP3xWa71nhI896YjihHAC+W4KVzXu76ZTpugLIYSYODLuTwgh3JwEuRBCuLlxB7lS6iKlVJFSqlgp9Z/DvB+glHrV/v4mpVTmmRQqxFiM4vt5h1KqXim10/74hhl1Cu+jlHpWKVWnlCo8xftKKfWY/bu7Syk1b6RzjivIj5m2fzEwHbhJKTX9hMO+DjRrrXOA3wL/O55rCTFWo/x+AryqtZ5jfzw9oUUKb/Y8cNFp3r8YmGx/3AP8eaQTjrdFPjRtX2vdBwxO2z/WlcBf7T+/BixXZiwLJrzRaL6fQphCa70OaDrNIVcCL2jDRiBSKZV0unOON8iHm7afcqpjtNYDQCsQM87rCTEWo/l+Alxr/9P1NaVU2jDvC2GG0X5/h8jNTuGt3gEytdazgA85+tejEG5nvEE+mmn7Q8copXyBCKBxnNcTYixG/H5qrRu11oOLST8N5E9QbUKMZMzLoow3yEczbf9t4Hb7z9cBn2iZfSQmxojfzxP6HK8A9k1gfUKcztvAbfbRKwuBVq119ek+MK7VD081bV8p9XNgq9b6beAZ4EWlVDFGx/6N47mWEGM1yu/nfyilrgAGML6fd5hWsPAqSqmXgfOAWKVUBfAA4AegtX4CeA+4BCgGuoA7RzynNJKFEMK9yc1OIYRwcxLkQgjh5iTIhRDCzUmQCyGEm5MgF0IINydBLoQQbk6CXAgh3Nz/B+pXIWE2XuOkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs = np.linspace(0.001, 0.999, 100)\n",
    "ys_0 = [-np.log(1 - x) for x in xs]\n",
    "ys_1 = [-np.log(x) for x in xs]\n",
    "plt.plot(xs, ys_1, label=r'$y = 1, -\\ln(h(x))$')\n",
    "plt.plot(xs, ys_0, label=r'$y = 0, -\\ln(1 - h(x))$')\n",
    "plt.ylim([0, 5])\n",
    "plt.xlim([0, 1])\n",
    "plt.xticks([0, 0.5, 1])\n",
    "plt.legend()\n",
    "plt.title('cost function')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def logloss(x, y, theta):\n",
    "    if y == 1:\n",
    "        return -np.log(hypothesis(x, theta))\n",
    "    elif y == 0:\n",
    "        return -np.log(1 - hypothesis(x, theta))\n",
    "    #  or (less obvious)\n",
    "    return -y * np.log(hypothesis(x, theta)) - (1 - y) * np.log(1 - hypothesis(x, theta))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that if $y$ was suppose to be 0 but is close to 1, the cost will be high.  \n",
    "Same logic if $y$ was suppose to be 1 but is close to 0.\n",
    "\n",
    "Then we generalize the cost function for all $x$:\n",
    "$$J(\\theta) = \\frac{1}{n} \\sum_{i = 1}^{n} j(h(x_i), y_i)$$\n",
    "$n$ is the number of elements in $X$ and $Y$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(xs, ys, theta):\n",
    "    return sum([logloss(x, y, theta) for x, y in zip(xs, ys)]) / len(xs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimization\n",
    "\n",
    "### Gradient descent\n",
    "\n",
    "Finally, use gradient descent to minimize the cost function:\n",
    "\n",
    "$$\\theta_i := \\theta_i - \\alpha \\frac{\\partial}{\\partial \\theta_i} J(\\theta)\\ \\text{ for } i = 1, ..., n$$\n",
    "\n",
    "Or with gradient notation:\n",
    "\n",
    "$$ \\theta := \\theta - \\alpha \\nabla J(\\theta)$$\n",
    "\n",
    "\n",
    "$\\alpha$ is the learning rate (how fast we want the step to be).  \n",
    "Since $\\theta_i$ is only used in the dot product, all of them will have similar partial derivative.\n",
    "\n",
    "let $z = x \\cdot \\theta_j$\n",
    "\n",
    "$\\newcommand{\\pt}{\\frac{\\partial}{\\partial \\theta_j}}$\n",
    "$$\n",
    "\\begin{align}\n",
    "      & \\pt J(\\theta) \\\\\n",
    "    = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n} j(h(x_i, \\theta), y_i) \\right] \\\\\n",
    "    = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n",
    "        (-y_i \\ln(h(x_i, \\theta)) - (1 - y_i) \\ln(1 - h(x_i, \\theta))) \\right] \\\\\n",
    "    = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n",
    "        (-y_i \\ln(\\sigma(z)) - (1 - y_i) \\ln(1 - \\sigma(z))) \\right] \\\\\n",
    "    = & \\pt \\left[ \\frac{1}{n} \\sum_{i = 1}^{n}\n",
    "        (-y_i \\ln(\\frac{1}{1 + e^{-z}}) -\n",
    "        (1 - y_i) \\ln(1 - \\frac{1}{1 + e^{-z}})) \\right] \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\pt \\left[ \n",
    "        -y_i \\ln(\\frac{1}{1 + e^{-z}}) -\n",
    "        (1 - y_i) \\ln(1 - \\frac{1}{1 + e^{-z}}) \\right] \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i \\pt \\left[ \\ln(\\frac{1}{1 + e^{-z}}) \\right] -\n",
    "        (1 - y_i) \\pt \\left[ \\ln(1 - \\frac{1}{1 + e^{-z}}) \\right] \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i(1 + e^{-z}) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n",
    "        - (1 - y_i) \\frac{1}{1 - \\frac{1}{1 + e^{-z}}} \\pt \\left[ 1 - \\frac{1}{1 + e^{-z}} \\right]\n",
    "        \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i(1 + e^{-z}) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n",
    "        - (1 - y_i) \\frac{1 + e^{-z}}{e^{-z}} (-1) \\pt \\left[ \\frac{1}{1 + e^{-z}} \\right]\n",
    "        \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i(1 + e^{-z}) \\frac{e^{-z}}{(1 + e^{-z})^2} \\pt z\n",
    "        - (1 - y_i) \\frac{1 + e^{-z}}{e^{-z}} (-1) \\frac{e^{-z}}{(1 + e^{-z})^2} \\pt z\n",
    "        \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        -y_i \\frac{e^{-z}}{1 + e^{-z}} x_{ij}\n",
    "        + (1 - y_i) \\frac{1}{1 + e^{-z}} x_{ij}\n",
    "        \\right) \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        \\frac{-y_i e^{-z}}{1 + e^{-z}}\n",
    "        + \\frac{(1 - y_i)}{1 + e^{-z}} \\right) x_{ij} \\\\\n",
    "    = & \\frac{1}{n} \\sum_{i = 1}^{n} \\left(\n",
    "        \\frac{-y_i e^{-z} + 1 - y_i}{1 + e^{-z}} \\right) x_{ij} \\\\\n",
    "\\end{align}\n",
    "$$\n",
    "\n",
    "$$\n",
    "\\boxed{\n",
    "    = \\frac{1}{n} \\sum_{i = 1}^{n} \\left( (h(x_i) - y_i) x_{ij} \\right)\n",
    "}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_partial_theta(xs, ys, theta, theta_j):\n",
    "    total = 0\n",
    "    for x_i, y_i in zip(xs, ys):\n",
    "        temp = hypothesis(x_i, theta) - y_i\n",
    "        if theta_j != 0:\n",
    "            temp *= x_i[theta_j - 1]\n",
    "        total += temp\n",
    "    return total / len(xs)\n",
    "\n",
    "\n",
    "def cost_gradient(xs, ys, theta):\n",
    "    return np.array([cost_partial_theta(xs, ys, theta, i)\n",
    "                     for i in range(len(theta))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(xs, ys, theta, alpha=1, epoch=100):\n",
    "    for _ in range(epoch):\n",
    "        theta = theta - alpha * cost_gradient(xs, ys, theta)     \n",
    "    return theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weights:   29.40983934878407 -5.527334155188794\n",
      "train cost: 0.007186648908279271\n",
      "test cost:  0.010992176598402464\n"
     ]
    }
   ],
   "source": [
    "theta = np.random.randn(len(X_train[0]) + 1)\n",
    "theta = gradient_descent(X_train, Y_train, theta, 4, 500)\n",
    "print('weights:  ', *theta)\n",
    "print('train cost:', cost(X_train, Y_train, theta))\n",
    "print('test cost: ', cost(X_test, Y_test, theta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Optimized model')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VfWd//HX5y4hhISwIxACYYewL4paKJQaUTD6c2tRbK221tGOM/Ob2mpt3R5jnfnN/GaqP50ZHSwu1WjdalSs0SrFtm6oVNmRPawBJCwJSW7u9/fHuQkBAoRwb05y7/v5eORxk3u+55zPvYF3zv2ec75fc84hIiLJJeB3ASIiEn8KdxGRJKRwFxFJQgp3EZEkpHAXEUlCCncRkSSkcJdWzcxyzeyAmQWbuf4BMxsQ55oWmtn347nNJu53mpmVNrHt3Wb2m0TXJK2Xwl3iysyuNbMvzKzCzLab2X+ZWadTWH+DmX2z7mfn3CbnXKZzrrY59cTWXdecdUXaMoW7xI2Z/SPwL8CtQDYwGegHvGVmaX7WJpJqFO4SF2bWEbgH+Fvn3O+dczXOuQ3AlUB/YG6s3d1m9oKZPWdm+83sUzMbE1v2FJALvBrrTvmJmfU3M2dmoVibhWb2T2b2l1ibV82sq5k9bWb7zOxjM+vfoC5nZoPMrHesfd1XhZm5Bu2uM7MVZvaVmb1pZv0aLDvPzFaaWbmZPQTYCd6Hu83seTP7Tez1fWFmQ8zsdjPbaWabzaygQfveZlZsZnvM7Esz+0GDZe3N7PFYTcuBSUftq7eZvWhmZWa23sxuOeVfnCQthbvEyzlAOvBSwyedcweABcB5DZ6+GHge6AI8A/zOzMLOuWuATcBFse6U/3OcfX0buAboAwwE3gfmx7a3Arjr6BWcc1tj28x0zmUCLwPPApjZxcDPgEuB7sB7QFFsWbfYa/o50A1YC5x7kvfiIuApoDPwGfAm3v+1PsC9wCMN2j4LlAK9gcuBX5rZN2LL7oq9voHA+cB361YyswDwKvDX2HZnAH9vZuefpDZJEQp3iZduwC7nXKSRZdtiy+t84px7wTlXA/w73h+Fyaewr/nOubXOuXLgDWCtc+7t2L6fB8adaGUz+ykwDLgu9tSNwP3OuRWxbfwSGBs7er8QWNag3l8B209S33vOuTcb1NMd+OfY+s8C/c2sk5n1xftD8VPn3CHn3BJgHvCd2HauBO5zzu1xzm0GHmywj0lAd+fcvc656th5hf/B+8MnQsjvAiRp7AK6mVmokYDvFVteZ3PdN865aOwKkN6nsK8dDb6vbOTnzOOtaGYXAH8HnOWcq4w93Q94wMz+b8OmeEfEvY+q15nZZk7s6Hp2NTghXLfPzNi29zjn9jdovxGYGPv+iH3HltXpB/Q2s70NngvifeoQ0ZG7xM37QBVe10Y9M8sELgD+0ODpvg2WB4AcYGvsqYQNU2pmQ4EngCtjR8J1NgM/dM51avDV3jn3F7xPHQ3rtYY/n6atQBczy2rwXC6wJfb9EfuOLWtY8/qjas5yzl0Yp9qkjVO4S1zEukjuAf6fmc00s3DsxOZv8fqUn2rQfIKZXRo7Sfr3eH8UPogt2wHE9bp0qD/h+wpwh3PuT0ct/m/gdjPLj7XNNrMrYsteB/Ib1HsLcEY8aor9gfkLcL+ZpZvZaOB6oO769N/G6upsZjnA3zZY/SNgv5n9NHbiNWhmI83siJOukroU7hI3sROgPwP+DdgHfIh3hDnDOVfVoOkrwLeAr/BOjF4a648GuB/4uZntNbMfx7G88cBQ4D8aXjUTq/tlvEs4nzWzfcBSvE8bOOd2AVcA/wzsBgYDf45jXXPwribaineS9y7n3NuxZffgdcWsB0po8Acy1s0zGxgbW74Lr78+O461SRtmmqxDWpKZ3Q0Mcs7N9bsWkWSmI3cRkSSkcBcRSULqlhERSUI6chcRSUK+3cTUrVs3179/f792LyLSJn3yySe7nHPdT9bOt3Dv378/ixcv9mv3IiJtkpltPHkrdcuIiCQlhbuISBJSuIuIJCGNCikicVFTU0NpaSmHDh3yu5SkkJ6eTk5ODuFwuFnrK9xFJC5KS0vJysqif//+eINnSnM559i9ezelpaXk5eU1axvqlhGRuDh06BBdu3ZVsMeBmdG1a9fT+hSkcBeRuFGwx8/pvpcnDXcz+3VsYt+lx1luZvZgbHLfz81s/GlVJCIip60pfe6PAw8BTx5n+QV4Y1wPBs4C/iv2mDDl5bBtG/TqBdlNGL26Yfs/bi+mZG0JBQMLKBxa2Gj74lVem3PPKGBQrddmyJBj91W33T/vKmb+54+wu2oHuV16cu2oHzIuo5BevWDB2mJeW1HC7OEFzBl/eH+bNsHSpTBypLfdbdsgIwNeW1PMkv0lcCib1ZvKGd+pgOxthRQUQJ8+3jq5uRAKeXU+ufwRIu12kEVPhlf+kAsGFLI2XMznB0rI7ZHNIcrJrSng3YWwOVzCmd0K6JABS/aXMLBPNjXBcmYPLwDg4Q+8eZtvnvxDAF7+vITMUDahzHK27t3B59tXclanQm4aeh+Zmd57Uvf6JncvYGCkkJEjvfrKy+HpxcX8z+p72bF/F2MCV/OvM++jb9/Dvwvw2izZX8JZ3Qrote/w+g3f388qinl3Uwljswq4emJho7/zok+9Onp19l7ziX6/Ik1Rd6Nlt27dTquNX5o0cFhsRp3XnHMjG1n2CLDQOVc3W/wqYJpzbtuJtjlx4kTXnDtU963cyuK/e4rqGkgLw9nnQPv047evPATv/wWqa2BvaDlLrYiaaA1pwTBzRs5hRPcRR7RfXracoqVFVNfWEHRhum2fQ4eKEeTlwZSph/dVt93NVctZEXwaF6it30bABcmPXo0ZLLMiaq2GIGGuHD6HsTn5fLUXXnjeqykUhJwccA6+LF/Oth7PELUab7I5A1cbJrT8akJ78+nW3QgYHDhodBy0jLUdn4BALQ5wBi4ahA3fINp/IdFQTf3zURfEYdSGIkRdiCiO2nAtUaA2ANFAiEggSk0gSk0AagMBIsEglcEaqoJQFYJDIagMQ2UIgstuI3/P/XT/WjGvp8+hxiqwSAaDlhTR71Ah//Ef8M8vF1NUexlRi02n6iD785/xnZz7yMqCQABWWzEvBeYQsQoCtRkM+byInIOFPPaY9wfvoYdgSWUxr4S8fYRcBt8KFvHwLUcGfNGnxXz31TnUUFH/XEY4g6LLihTwLWjFihUMHz7c7zLipjWEe2PvqZl94pybeJxVDreLQ7i/hjez+59iP/8Bbzb3Y5LbzG4AbgDo2bPnhGefffak+z5aZNcBgh8tIxyGmhrIyoK0tOO3r66G/fshHIbymjIONZhPuFN6J7pnHDlEQ1lFGXsPHW4TjnQiLdKdUAg6dTq8r7rtVlgZh9iLHfU2tg90ojYC1YHD28oKd6JnVjcqKuCrPY5QyNtOOAzt2kF5zS5qQ3s5mh3KhopuhIKOtDSornKQsYvaYHmshQMHhoNIGAtUAw6LOgwg6rC6n53DotH6x0DUYbVRAtHYV20tgUjtMTUcIRomYDlUZR3iYIcaqjLSqerQnkh6X2rDw+jQN4uy2q1U2M4j14u0p+OhEXTuDBUVcCC4mcrA4TbptT0IHexLr17e+7xnj9dmf/RwmwzXg7wufUlv8Ad9/Z7N7Kk6al9Ajw496NsxXtOdyslkZ2czaNAgX2vYuHEjl156KZMmTeLDDz9k/PjxzJ07l1/+8peUlZUxb948BgwYwM0338yGDRto3749Dz74ICNHjmT37t1cd911bNu2jTPPPJN3332XRYsW0bVrV5599ln++7//m5qaGiZOnMi///u/EwwGGTlyJH/84x/p2rVrQl7Pl19+SXl5+RHPTZ8+vUnh3qKXQjrnHgUeBe/Ifdq0aae8jfJyeGjVbCoqICMbfvSjE3fNlJd7R4AVFbAxyztSrIxU1B/ZzTjqyK54VTHfeXEOFTUVhF0GQ74ootvuQqZMgR/fcnhfddv9eF8xxe2uwAWr67cRdGn8r8jzBNLg5YB31BkmgycuKuK88YVs2gS/uN6rKS0NJk4EF4W3txazbLh3JFt35E51BmlvFtF+UyFDhnhHvLt2Qe/pxbx3xhUQOrxfatIIfPhjomf9CsINthFJ874JVUGknfcxIVxdvzzo0nBE64+yA9Eg4WiQUG01abXQrhbSa6B9BNrXQPflcxm552J6Dl7I2j0PkFVVTeahEF22X0525BBTLgjw+uZtvJH5KJs6RtnQCcrT647cbzrqyP3Go47cpzVy5H7jEUfuV82adsTvfNunxdz+6o2NHrlPG3rq/8akeVasWEFWVmyu79//HrZvj+8OzjgDZs48YZPMzEzWrVvHiy++SH5+PpMmTeJ3v/sd77//PsXFxTzwwAP07duXSZMm8dprr/HOO+/wN3/zNyxZsoQ77riDadOmceedd/L666/z5JNPkpmZSWlpKcXFxXzwwQeEw2FuuukmiouL+c53voOZkZmZefh1x1l6ejrjxo1r1rrxCPctHDlDew6HZ2+Pu+xYoDe1z/3I9oVcub3ohH3uhUMLKbqs6HCf+6zG+9wPb7eQi3Y9f9w+90vWFh3T556bC489dmyf+y0Zhby2pujIPvdeBWRf1lifeyHFq54/ts/9ukLWhs9KWJ97fqdCbppT1+d+IQvWfo3XVpRwTqfpDPvqHEZ0L6NncCffXruNER8dZMnmtzm4toL0zLOYdcHX6XLuZrYGcujV24BCvr646HCf++Qj+9zr3t9LK4pO2Ofuva9F6nMXAPLy8hg1ahQA+fn5zJgxAzNj1KhRbNiwgY0bN/Liiy8C8I1vfIPdu3ezb98+Fi1axEsvvQTArFmz6Ny5MwB/+MMf+OSTT5g0yZt7vLKykh49evjwyk5NPLplZgE/Ai7EO5H6oHPuzJNts7l97tKGRKPex4y1a2HNGti4EWproVs3GDcOxoyBzEy/q5Q4aQ197hs2bGD27NksXepd3Hfttdcye/ZsLr/88vpl4XCYF198kQEDBgDQt29fli1bxtSpU3nppZfqn+/SpQurV6+mqKiIrVu3cv/99x+zv9bc596USyGLgPeBoWZWambXm9mNZnZjrMkCYB3wJfA/wE2n+gIkSQUC0KMHnH02fOc78JOfwMUXe5cFvfUW/OpXUFICBw/6XamkkClTpvD0008DsHDhQrp160bHjh2ZOnUqzzzzDABvvPEGX331FQAzZszghRdeYOdO77zOnj172LixSaPu+uqk3TLOuTknWe6Am+NWkSSvdu28I/Zx46CsDP70J3j/fVi8GCZPhqlTvWs8RRLo7rvv5rrrrmP06NFkZGTwxBNPAHDXXXcxZ84c8vPzOeecc8iN9Q+OGDGCf/qnf6KgoIBoNEo4HObhhx+mX79+fr6Mk/JtDlV1ywjgddssXOidTOjeHS65xDu5IG1Oa+iWSTYJ7ZYRSahu3eDyy2HuXKiqgnnz4J13vP56EWk2hbu0DoMGwU03eSdZFy2C55/3bmQQkWZRuEvrkZ7udcvMnAkrV8ITT3g3A4jIKVO4S+szeTJceaV3E8xjj8GBA35XJNLmKNyldRo+3Lt8ct8+eOYZb5wGEWkyhbu0Xrm5cMUV3u27zz+vk6wip0DhLq3bkCEwa5Z3h+vrr3vj4og0Yu/evfznf/5ns9a98MIL2bv32EH7mirzJHdan05tzaVwl9Zv4kSYMgU++QT++le/q5FW6kQBGolETrjuggUL6NSpUyLKAhTuIsc3fTr06wdvvOENySlylNtuu421a9cyduxYbr31VhYuXMiUKVMoLCxkxAhv3oZLLrmECRMmkJ+fz6OPPlq/bv/+/dm1axcbNmxg+PDh/OAHPyA/P5+CggIqKyuP2df69es5++yzGTVqFD//+c/rnz9w4AAzZsxg/PjxjBo1ildeeaXR2o7XLq6cc758TZgwwYmckj17nLvvPucef9y5aNTvauQoy5cvP+V19u51bsUK7/F0rV+/3uXn59f//O6777qMjAy3bt26+ud2797tnHOuoqLC5efnu127djnnnOvXr58rKytz69evd8Fg0H322WfOOeeuuOIK99RTTx2zr4suusg98cQTzjnnHnroIdehQwfnnHM1NTWuvLzcOedcWVmZGzhwoItGo8fUdrx2R2vsPQUWuyZkrI7cpe3o3BnOPx/Wr4ePPvK7GjlNdXMiPPWU95iID2RnnnkmeXl59T8/+OCDjBkzhsmTJ7N582bWrFlzzDp5eXmMHTsWgAkTJrBhw4Zj2vz5z39mzhxv2K1rrrmm/nnnHD/72c8YPXo03/zmN9myZQs7duw4Zv2mtjsdGqVJ2pbx470bnN56CwYPhi5d/K5ImmnbNu8etX79vNGgt21r2pzIp6JDhw713y9cuJC3336b999/n4yMDKZNm8ahQ4eOWaddu3b13weDwUa7ZQDM7Jjnnn76acrKyvjkk08Ih8P079+/0X00td3p0JG7tC1mcNFF3uM77/hdjZyGXr280Z83bvQe6yZNb66srCz2799/3OXl5eV07tyZjIwMVq5cyQcffNDsfZ177rnUTRNaN3xw3T569OhBOBzm3XffrR8a+OjajtcunhTu0vZ07OiNEb90KWxJ2KRfkmB1s5ldc83Jp8tsiq5du3LuuecycuRIbr311mOWz5w5k0gkwvDhw7ntttuYPHlys/f1wAMP8PDDDzNq1Ci2NPg3ePXVV7N48WJGjRrFk08+ybBhwxqt7Xjt4klD/krbVFUFDzwAPXt6d7I28hFZWpaG/I0/DfkrqaddO/j6172Tq19+6Xc1Iq2Owl3arokTvSto3n5bQxOIHEXhLm1XMAgzZsCOHbBihd/ViLQqCndp20aM8C6HPI0rH0SSkcJd2rZAAM46CzZv1pUzIg0o3KXtGzvWO8Gqo3eRegp3afvatYNx42DZMm9yD0lJpzvy4q9+9SsqmjCt48KFC5k9e/YJ2yxZsoQFCxY0u5Z4ULhLcjjrLG+sd907kbJaKtybQuEuEi+dO8PQoV6419T4XY344OhhdQH+9V//lUmTJjF69GjuuusuAA4ePMisWbMYM2YMI0eO5LnnnuPBBx9k69atTJ8+nenTpx+z7d///vcMGzaM8ePH89JLL9U//9FHH3H22Wczbtw4zjnnHFatWkV1dTV33nknzz33HGPHjuW5555rtF3CNWXoyER8achfibv165276y7nYsO1SstqzpC/r6x8xd38+s3ulZWvnPb+jx5W980333Q/+MEPXDQadbW1tW7WrFnuj3/8o3vhhRfc97///fp2e2PjDdcN+3u0yspKl5OT41avXu2i0ai74oor3KxZs5xzzpWXl7uamhrnnHNvvfWWu/TSS51zzs2fP9/dfPPN9ds4XruTOZ0hfzUqpCSPfv28I/jPP/dOskqrVryqmDkvzqGipoL5S+ZTdFkRhUML47b9kpISSkpKGDduHOBNpLFmzRqmTJnCP/7jP/LTn/6U2bNnM2XKlBNuZ+XKleTl5TF48GAA5s6dWz/RR3l5Od/97ndZs2YNZkbNcT41NrVdPKlbRpKHGYwe7Q1JcILRAaV1KFlbQkWN18ddUVNBydqSuG7fOcftt9/OkiVLWLJkCV9++SXXX389Q4YM4dNPP62fRenee+9t9j5+8YtfMH36dJYuXcqrr7563GF7m9ounhTuklxGj/ZOrH7xhd+VyEkUDCwgI5wBQEY4g4KBBae1vaOH1T3//PP59a9/zYEDBwDYsmULO3fuZOvWrWRkZDB37lxuvfVWPv3000bXrzNs2DA2bNjA2rVrASgqKqpfVl5eTp8+fQB4/PHHj1vL8dolksJdkkvXrtCnj9c1I61a4dBCii4r4uZJN8elS+boYXULCgq46qqr6uc6vfzyy9m/fz9ffPEFZ555JmPHjuWee+6pnwP1hhtuYObMmcecUE1PT+fRRx9l1qxZjB8/nh49etQv+8lPfsLtt9/OuHHjjpiEe/r06Sxfvrz+hOrx2iWShvyV5PPhh95E2jfdBA3+I0piacjf+NOQvyIN5ed7wxLo6F1SmMJdkk9mJgwc6PW7+/TJVMRvCndJTqNHQ3m5N0GntBi/unmT0em+lwp3SU5Dh0IopHHeW1B6ejq7d+9WwMeBc47du3eTnp7e7G3oJiZJTmlpMGAArFoFM2dqjtUWkJOTQ2lpKWVlZX6XkhTS09PJyclp9voKd0leQ4fC6tVQVqarZlpAOBwmLy/P7zIkRt0ykryGDPEeW2KQJpFWRuEuySsrC3r3VrhLSlK4S3IbOtSbfi92C7pIqlC4S3IbOtS71n3NGr8rEWlRCndJbj17Qna2umYk5SjcJbmZeSdW166FFhqwSaQ1ULhL8hs61Jt6b/16vysRaTEKd0l+/ftDOKx+d0kpCndJfqGQNwWfjtwlhSjcJTUMGODdqarp9yRFKNwlNQwY4D2uW+dvHSItROEuqaFnT8jIUNeMpAyFu6QGM8jL847cNSStpACFu6SOAQNg3z7YvdvvSkQSTuEuqaNuOFp1zUgKULhL6ujcGTp10klVSQkKd0kddf3uGzZANOp3NSIJpXCX1DJgAFRWwvbtflciklAKd0ktdf3u6pqRJKdwl9SSmQndu8PGjX5XIpJQCndJPbm5sHmz+t0lqSncJfX06weHDnljzYgkKYW7pJ7cXO9RXTOSxBTuknqys6FjR9i0ye9KRBJG4S6px8zrmtm4UePMSNJSuEtqys31xnbfu9fvSkQSQuEuqamu311dM5KkFO6Smnr0gPR0hbskLYW7pCYz7+hdV8xIklK4S+rKzYVdu+DgQb8rEYk7hbukrrp+982b/a1DJAEU7pK6eveGUEhdM5KUFO6SukIhL+BLS/2uRCTuFO6S2nJyYOtWiET8rkQkrhTuktr69oXaWk3eIUlH4S6pLSfHe1TXjCQZhbuktqwsbyAxXTEjSUbhLtK3r47cJeko3EVycqC8HPbt87sSkbhRuIuo312SkMJdpFcv75p3hbskEYW7SDDoBbzCXZKIwl0EDt/MVFvrdyUicaFwFwHviplIRDczSdJQuIvA4ZOqut5dkoTCXQSgY0fvS/3ukiQU7iJ1cnIU7pI0FO4idXJyYO9ezcwkSUHhLlJHNzNJElG4i9Tp1QsCAYW7JAWFu0idcBh69oQtW/yuROS0NSnczWymma0ysy/N7LZGll9rZmVmtiT29f34lyrSAnJyvHCPRv2uROS0nDTczSwIPAxcAIwA5pjZiEaaPuecGxv7mhfnOkVaRp8+UFUFu3b5XYnIaWnKkfuZwJfOuXXOuWrgWeDixJYl4pO6k6rqmpE2LtSENn2AhrftlQJnNdLuMjObCqwG/sE5d8ytfmZ2A3ADQM+ePVm4cOEpFyySUM6Ru3UrB994g93l5X5XI9JsTQn3pngVKHLOVZnZD4EngG8c3cg59yjwKMDEiRPdtGnT4rR7kTgqLYUDB0D/PqUNa0q3zBagb4Ofc2LP1XPO7XbOVcV+nAdMiE95Ij7IyYGdO6G62u9KRJqtKeH+MTDYzPLMLA34NlDcsIGZ9WrwYyGwIn4lirSwnBxwzhsCWKSNOmm3jHMuYmY/At4EgsCvnXPLzOxeYLFzrhi4xcwKgQiwB7g2gTWLJFafPt5jaSn07+9rKSLN1aQ+d+fcAmDBUc/d2eD724Hb41uaiE8yMqBLF10xI22a7lAVaUzdCJHO+V2JSLMo3EUak5MD+/fDvn1+VyLSLAp3kcZohEhp4xTuIo3p2RNCIU27J22Wwl2kMcEg9O6tI3dpsxTuIsfTty9s2waRiN+ViJwyhbvI8eTkQG0tbN/udyUip0zhLnI8OqkqbZjCXeR4srIgO1vhLm2Swl3kRHJydMWMtEkKd5ETycmB8nLvhiaRNkThLnIi6neXNkrhLnIivXp517wr3KWNUbiLnEgo5AW8+t2ljVG4i5xM377exB26mUnaEIW7yMnk5nrBvm2b35WINJnCXeRk+samEFbXjLQhCneRk8nM9GZm2rTJ70pEmkzhLtIUubleuGtmJmkjFO4iTZGbCxUVsHu335WINInCXaQpcnO9R3XNSBuhcBdpiq5dISND4S5thsJdpCnMDve7i7QBCneRpsrNhT174MABvysROSmFu0hT6Xp3aUMU7iJN1auXN9aMumakDVC4izRVKAR9+ijcpU1QuIucin79vDFmqqr8rkTkhBTuIqciLw+iUdi40e9KRE5I4S5yKnJyvO6Z9ev9rkTkhBTuIqciHPaumlG4SyuncBc5VXl5sGOHN9aMSCulcBc5VXl53uiQ6neXVkzhLnKqeveGtDR1zUirpnAXOVXBoHdJpMJdWjGFu0hz9O8PZWWwf7/flYg0SuEu0hx5ed7jhg2+liFyPAp3keY44wxIT1fXjLRaCneR5ggEvK6Zdev8rkSkUQp3keYaOBD27tW8qtIqKdxFmmvwYO9x9Wp/6xBphMJdpLk6dYIePRTu0iop3EVOx5Ah3p2qGgJYWhmFu8jpGDzYGwJ47Vq/KxE5gsJd5HT07Qvt26trRlodhbvI6QgEYNAgWLPGG0xMpJVQuIucriFD4OBB2LrV70pE6incRU7XoEFgpq4ZaVUU7iKnq317r+9d4S6tiMJdJB6GDIFt26C83O9KRACFu0h85Od7j0uX+luHSIzCXSQeOneGnBz44gu/KxEBFO4i8TNyJGzfDrt2+V2JiMJdJG7y872rZnT0Lq2Awl0kXrKyvDHely7VDU3iO4W7SDyNGuWN775tm9+VSIpTuIvE0/DhEAzqqhnxncJdJJ7at/dmaFLXjPhM4S4Sb6NHw759GgZYfKVwF4m34cMhMxM++sjvSiSFKdxF4i0YhAkTvGGAv/rK72okRSncRRJhwgTvmvePP/a7EklRCneRROjY0eue+ewzqKnxuxpJQQp3kUQ580yorNQdq+ILhbtIouTmQs+e3olVXRYpLUzhLpIoZt7R+/btsH6939VIilG4iyTSmDFe//s77+joXVqUwl0kkUIhmDYNSks1DZ+0KIW7SKKNGQNdu+roXVqUwl0k0YJB7+h9xw4NKCYtRuEu0hLkXVWvAAAL2ElEQVRGjvSunHn3Xait9bsaSQEKd5GWYAYzZsCePfDBB35XIylA4S7SUgYPhmHDvKN3zbMqCaZwF2kpZjBrFoTD8MorEI36XZEkMYW7SEvKyoKZM2HzZg0JLAmlcBdpaaNHe100f/iDN9+qSAIo3EVamhlcdJHXPfPMM97gYiJxpnAX8UPHjvCtb8HevfD887o8UuJO4S7il379vCP4devgjTd096rEVcjvAkRS2tix3mWRf/oTZGTA9Olet43IaVK4i/htxgyv333RIqiogAsvhIA+VMvpUbiL+M0MZs/2jtzfe88L+Esv9UaUFGkm/esRaQ3qhifIyIA33/SGKbjsMuje3e/KpI3SZz+R1uTss2HOHNi3Dx55BD78UCdapVkU7iKtzdChcNNNkJfnXUXz2GOwcaPfVUkbo3AXaY0yM+Gqq+Dii6G8HObP9254Ki3Vkbw0ifrcRVorMxg3zhsL/sMPvcsl582DXr1gwgQYNQratfO7SmmlzPl0FDBx4kS3ePFiX/Yt0iZVVcEXX8DHH3uzOgWD0L+/140zaBB07qxr5FOAmX3inJt4snZNOnI3s5nAA0AQmOec++ejlrcDngQmALuBbznnNpxq0SJyAu3awcSJ3lH7li2wfDmsWgULFnjLMzMhNxf69IEePbyZn7KyFPgp6qThbmZB4GHgPKAU+NjMip1zyxs0ux74yjk3yMy+DfwL8K1EFCzSlpSXw7ZtXk9KdjYUryrmtRUljM0q4OqJhWRnn3zdzyqK+fP2EsZ1LKDXvkJGjjSys3OYV/0p82oeJ2Q7uSr8Tb7GGex4+xVGpeUwtNtQbyNpaVSmd2aP60Snfp3ocEYWZGaydV8m977/IIsqXybUrjsXdr+Ti/Mv4Xcrilmyr4QN1Z+xs3YlI3oOY9wZ4ygYWMDBg/Dy5yWc1a2A708p5I/bi3n4g0fYtGcH6ZGeTOCHXP+1Qjp3Pvx6f7zgDp5Z9hvSw2EGdx5Kt9qx7D5QzqxhBUzuXMj27bBmjTfEzqRJMHWqV/bTi4v5aFcJOdUFbNoEX7oSdhzYwY72C0lLM64c+AP+5fz7gMPv0W9WPMKOgzvo2aEnc4f/kHEZhfTq5W1v9WpYtKOYVZESZg8v4OtnFLJtm3fl6Wtrilmyv4TpuQWMyygkI8P7YAQwZAj1v6OiT73f3ezhBcwZX1j/eypeVUzJ2hIKBhZQOPTw8439LjMyvNsY6t6fRDppt4yZnQ3c7Zw7P/bz7QDOufsbtHkz1uZ9MwsB24Hu7gQbV7eMJLvycnjoIe8/c0YGDLigmOvfmENlpIKQy+BbwSIevqXxgK9bd0llMa+E51BDBYHaDIZ8XsQZ5YV0GF/M65mXQSDireAgYCGiROhCe56d8iDnZY3h4KbdlDz3FcF9X5EVLefs8VVUVcH8hX9gb/afDu/QGdFtU6nI/Qs14RqqA1AThEjA+yIUosZBTSACLo3BgYtZFXqZKiJEDRwQjYZI/+hOpg85lx5nBNiTN48X1z6Ns/oScQAG1Laj96p7+OrzKVRUGGbQvj188zzjQJf3WBj4BTVWBbVh7wRyKOKtHNsXwJjotUxwf8NGt4iFoZ8RsZr6lxMkzLTI/fS3qVRVwftbFrF+4B1EA1WkB9oxnfvosn8qH+9cxLr+dxCxKkKuHedW3ceBZVOJRLxer3HjYO5c+HPpIu5+7w4iVBGiHXdPuY/zh01l0cZF3PHOHRyKVJEeasd937iPqf2mHvG7PHAAfvtb2L/f+yMzeDBUDR7Jjf/QvlkB39RumaaE++XATOfc92M/XwOc5Zz7UYM2S2NtSmM/r4212XXUtm4AbgDo2bPnhGefffbUXpVIG3LokHcvUloaVFdDVfpm9lTtrF+e4XqQ16Uv6enHX/dAcDP7o4fXSa/tQWB/XyIdNlMd3nnsijE9OvSgb8e+x9TQtWOEaHkFpbu+IFi7n1BVNaHqGoKRCMFDIYLuIMHaWgKRWgK1UQK1tQSiUQK1Ucw5LPZ9wIXAVWPOQdRhOCzqoLoDGdaRUNBRYWXUuprj1hiIdMAdysa5wz1H4TC4tHJq7OBJ398AYdpHuxMJllPljm3fzjoQqs0mGoXqQDm1gcNtwnQgnWwqasupDR5+Pg2vJjNvBIhgEDp0gP015VRGDrdrH+pA5/bZlFeVc7D68PMd0jqQ3e7IxI5EDo/qXFXl9a6tPu8SOvTv1Ojv/mSmT58evz73eHHOPQo8Ct6R+7Rp01py9yItqrEj91+8ceMRR+5XzZrWhCP3G486cp8WO3K/rtEj94xwBkWXFTFt6LRjapj1I2/bN93zIRv6/ot3FA1gIdKW/YTq8b+GcEX9UXKdtGA7aiOOWqsmUJvB2fw9H4f/L9XRqsONIml0WPA8V00opF+uY/uIO3j4r/cTiB0/GmCx7wM17cn77CnK/jKbgwccZl6IfutKx94er/FK4LtErBIiad4fkNCxfyTOqf0xU93drHavUxy+hlqq65cFSaOw5imG2CwOHYK3NrzOyqHXEg1W0j7Ynouj8+lRPouFW15nxdDvEaGSMO05v3I+5YtnEYl4oz+ccw7ccgt8tv51bljwPWpi7R69cD5XjJ3F66tf53uvfI+Kmkoywu2Zf/F8xgyZdczv8pFHvK6nL76AkXmQWZbBj+YEEto1o24ZkQRKTJ+7t6157xUzb8297K/dxezcq/n6oLP48/Zj+36PrgFg0ya49fd38N7ep8kKdOOSLndyyfBC9bm3gT73eHbLhIDVwAxgC/AxcJVzblmDNjcDo5xzN8ZOqF7qnLvyRNtVuIuInLq4XQrpnIuY2Y+AN/Euhfy1c26Zmd0LLHbOFQOPAU+Z2ZfAHuDbp1e+iIicjib1uTvnFgALjnruzgbfHwKuiG9pIiLSXBpbRkQkCSncRUSSkMJdRCQJKdxFRJKQwl1EJAkp3EVEkpBv47mbWRng99xh3YBdJ23V9ul1Jhe9zuRyqq+zn3PupDOn+xburYGZLW7KnV5tnV5nctHrTC6Jep3qlhERSUIKdxGRJJTq4f6o3wW0EL3O5KLXmVwS8jpTus9dRCRZpfqRu4hIUlK4i4gkoZQMdzNLN7OPzOyvZrbMzO7xu6ZEMrOgmX1mZq/5XUuimNkGM/vCzJaYWdLOAmNmnczsBTNbaWYrYjOlJRUzGxr7PdZ97TOzv/e7rkQws3+IZdBSMysys2bMqnqcbadin7uZGdDBOXfAzMLAn4C/c8594HNpCWFm/xuYCHR0zs32u55EMLMNwMSjJ2VPNmb2BPCec26emaUBGc65vX7XlShmFsSbAe4s55zfNz3GlZn1wcueEc65SjP7LbDAOfd4PLafkkfuznMg9mM49pWUf+XMLAeYBczzuxY5PWaWDUzFm/kM51x1Mgd7zAxgbbIFewMhoH1sOtMMYGu8NpyS4Q71XRVLgJ3AW865D/2uKUF+BfwEiPpdSII5oMTMPjGzG/wuJkHygDJgfqybbZ6ZdfC7qAT7NlDkdxGJ4JzbAvwbsAnYBpQ750ritf2UDXfnXK1zbiyQA5xpZiP9rinezGw2sNM594nftbSArznnxgMXADeb2VS/C0qAEDAe+C/n3DjgIHCbvyUlTqzbqRB43u9aEsHMOgMX4/3R7g10MLO58dp+yoZ7ndjH2neBmX7XkgDnAoWx/uhngW+Y2W/8LSkxYkdBOOd2Ai8DZ/pbUUKUAqUNPmW+gBf2yeoC4FPn3A6/C0mQbwLrnXNlzrka4CXgnHhtPCXD3cy6m1mn2PftgfOAlf5WFX/OududcznOuf54H2/fcc7F7cigtTCzDmaWVfc9UAAs9beq+HPObQc2m9nQ2FMzgOU+lpRoc0jSLpmYTcBkM8uIXeQxA1gRr42H4rWhNqYX8ETsTHwA+K1zLmkvE0wBPYGXvf8fhIBnnHO/97ekhPlb4OlYl8U64Hs+15MQsT/S5wE/9LuWRHHOfWhmLwCfAhHgM+I4FEFKXgopIpLsUrJbRkQk2SncRUSSkMJdRCQJKdxFRJKQwl1EJAkp3EVEkpDCXUQkCf1/I1j+XUdRPI0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X_train, Y_train, s=8, c='b', alpha=0.4, label='train data')\n",
    "plt.scatter(X_test, Y_test, s=10, c='g', label='test data')\n",
    "sig_x = np.linspace(X.min(), X.max(), 100)\n",
    "sig_y = [hypothesis(np.array([x]), theta) for x in sig_x]\n",
    "plt.plot(sig_x, sig_y, c='r', alpha=0.5, label='model')\n",
    "plt.yticks([0, 0.5, 1])\n",
    "plt.legend()\n",
    "plt.grid(axis='y')\n",
    "plt.title('Optimized model')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}