{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "14e55981",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "import numpy as np \n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 전체 입력은 4차원, 출력은 3클래스: 정수(0, 1, 2)로 되어있음\n",
    "iris = load_iris()\n",
    "x = iris.data \n",
    "y = iris.target\n",
    "N,D = x.shape\n",
    "\n",
    "# 입력은 3개만 사용: (bias, x[0], x[1])\n",
    "# 출력은 binary로:  0인지 아닌지\n",
    "x = np.hstack([np.ones((N,1)), x[:,0:2]])\n",
    "N,D = x.shape\n",
    "\n",
    "# class +1 : sample은 black, decision region은 gray\n",
    "# class -1 : sample은 red, decision region은 white\n",
    "t = np.zeros(y.shape)\n",
    "t[y == 0] = +1\n",
    "t[y != 0] = -1\n",
    "\n",
    "def drawPerceptron(w, x, t_pred, t):\n",
    "    res = 512 \n",
    "    \n",
    "    x1_new = np.linspace(np.min(x[:,1]),np.max(x[:,1]),res)\n",
    "    x2_new = np.linspace(np.min(x[:,2]),np.max(x[:,2]),res)\n",
    "    xx1_new,xx2_new = np.meshgrid(x1_new,x2_new)\n",
    "    \n",
    "    y_new = w[0] + w[1]*xx1_new + w[2]*xx2_new\n",
    "    \n",
    "    id_misclass = np.where(t_pred*t < 0)[0]\n",
    "\n",
    "    cmap_sample = matplotlib.colors.ListedColormap(['black', 'red'])\n",
    "    cmap_region = matplotlib.colors.ListedColormap(['gray', 'white'])\n",
    "    \n",
    "    plt.figure()\n",
    "    plt.pcolormesh(xx1_new, xx2_new, y_new > 0, cmap = cmap_region, vmin = 0.0, vmax = 1.0)\n",
    "    plt.scatter(x[:,1], x[:,2], c = t, s = 20, cmap = cmap_sample, vmin = -1, vmax = +1)\n",
    "    plt.scatter(x[id_misclass,1], x[id_misclass,2], s = 50, marker = 'o', facecolor = 'none', edgecolor = 'cyan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cf2e4860",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47300 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "w = np.array([-10, 1.0, 1.5])\n",
    "y_pred = x.dot(w)\n",
    "t_pred = np.sign(y_pred)\n",
    "drawPerceptron(w, x, t_pred, t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d4b47d48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a872ebe550>]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 47300 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 38400 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 36664 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 35344 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 35016 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 34548 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 34492 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 34380 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 34368 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 33864 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 15888 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# perceptron, stochastic gradient descent\n",
    "\n",
    "# hyperparameter\n",
    "eta = 5e-4\n",
    "maxEpoch = 1000\n",
    "\n",
    "cost = []\n",
    "\n",
    "# initialize\n",
    "w = np.array([-10, 1.0, 1.5])\n",
    "\n",
    "for epoch in range(maxEpoch):\n",
    "    # prediction\n",
    "    y_pred = x.dot(w)\n",
    "    t_pred = np.sign(y_pred)\n",
    "\n",
    "    # visualize\n",
    "    if epoch % 100 == 0:\n",
    "        drawPerceptron(w, x, t_pred, t)\n",
    "    \n",
    "    # misclassified patterns\n",
    "    id_misclass = np.where(t_pred*t < 0)[0]\n",
    "    id_misclass = np.random.permutation(id_misclass)\n",
    "    \n",
    "    if id_misclass.size == 0:\n",
    "        break\n",
    "    \n",
    "    # cost\n",
    "    cost.append(-np.sum(y[id_misclass]*t[id_misclass]))\n",
    "    \n",
    "    # gradient descent\n",
    "    for n in range(id_misclass.size):\n",
    "        grad = t[id_misclass[n]]*x[id_misclass[n],:]\n",
    "        w = w + eta*grad\n",
    "    \n",
    "plt.figure()\n",
    "plt.plot(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ac329130",
   "metadata": {},
   "outputs": [],
   "source": [
    "# class +1 : sample은 black, decision region은 gray\n",
    "# class 0: sample은 red, decision region은 white\n",
    "t = np.zeros(y.shape)\n",
    "t[y == 0] = +1\n",
    "t[y != 0] = 0\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 +np.exp(-x))\n",
    "\n",
    "def drawLogistic(w, x, y, t):\n",
    "    res = 512 \n",
    "    \n",
    "    x1_new = np.linspace(np.min(x[:,1]),np.max(x[:,1]),res)\n",
    "    x2_new = np.linspace(np.min(x[:,2]),np.max(x[:,2]),res)\n",
    "    xx1_new,xx2_new = np.meshgrid(x1_new,x2_new)\n",
    "    \n",
    "    a_new = w[0] + w[1]*xx1_new + w[2]*xx2_new    \n",
    "    y_new = sigmoid(a_new)\n",
    "\n",
    "    cmap_sample = matplotlib.colors.ListedColormap(['black', 'red'])\n",
    "    cmap_region = matplotlib.colors.ListedColormap(['gray', 'white'])\n",
    "    \n",
    "    plt.figure()\n",
    "    plt.pcolormesh(xx1_new, xx2_new, y_new > 0.5, cmap = cmap_region, vmin = 0.0, vmax = 1.0)\n",
    "    plt.scatter(x[:,1], x[:,2], c = t, s = 20, cmap = cmap_sample, vmin = 0, vmax = +1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c40e6736",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 33648 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "w = np.array([-10, 1.0, 1.5])\n",
    "a_pred = x.dot(w)\n",
    "y_pred = sigmoid(a_pred)\n",
    "drawLogistic(w, x, y_pred, t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c02a73b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a800422c40>]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 33648 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31460 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31540 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31488 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31520 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31468 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31600 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31444 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31496 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 31500 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 20100 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# logistic regression, batch gradient descent\n",
    "\n",
    "# hyperparameter\n",
    "eta = 1e-3\n",
    "maxEpoch = 1000\n",
    "\n",
    "cost = []\n",
    "\n",
    "# initialize\n",
    "w = np.array([-10, 1.0, 1.5])\n",
    "\n",
    "for epoch in range(maxEpoch):    \n",
    "    # prediction\n",
    "    a_pred = x.dot(w)\n",
    "    y_pred = sigmoid(a_pred)    \n",
    "    \n",
    "    # visualize\n",
    "    if epoch % 100 == 0:\n",
    "        drawLogistic(w, x, y_pred, t)\n",
    "    \n",
    "#     grad = np.zeros(3)\n",
    "#     for n in range(N):\n",
    "#         grad += (y_pred[n] - t[n])*x[n,:].reshape(-1)\n",
    "        \n",
    "    # gradient descent\n",
    "    grad = (y_pred - t).reshape(1,-1).dot(x).reshape(-1)\n",
    "    w = w - eta*grad\n",
    "    \n",
    "    # cost\n",
    "    cost.append( -np.sum(t*np.log(y_pred) + (1 - t)*np.log(1 - y_pred)) )\n",
    "    \n",
    "plt.figure()\n",
    "plt.plot(cost)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
