{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a0f9ada880>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 15632 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 데이터셋을 적당히 생성\n",
    "# 1차원 입력 (bias를 포함 = 2차원) -> 1차원 출력\n",
    "\n",
    "# 샘플의 개수\n",
    "N = 20\n",
    "x = np.random.rand(N) # (N, ) 벡터\n",
    "y = 10 + 3*x + 1e-1*np.random.randn(N)\n",
    "\n",
    "# design matrix X = (N, D)\n",
    "# sample: N, dimension: D\n",
    "X = np.hstack([np.ones( (N,1) ), x.reshape(N,1)])\n",
    "# print(X)\n",
    "\n",
    "# 전체 입력에 대해서 -> 예측을 구해서 plot\n",
    "theta = np.array([5, -3])\n",
    "X_plot = np.hstack([np.ones((5, 1)), np.linspace(0, 1, 5).reshape(5,1)])\n",
    "y_plot = X_plot.dot(theta)\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(X[:,1], y) # 주어진 학습 데이터를 점으로 찍음\n",
    "plt.plot(X_plot[:,1], y_plot) # 전체 입력을 잘게 쪼개서, 전체 예측을 구함"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a0f9b1e8b0>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 25256 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# normal equation\n",
    "# = closed form으로 해를 구함\n",
    "# 1. theta <- 주어진 데이터(X, y)로 구함\n",
    "# 2. 구한 theta에 대해서 예측을 수행\n",
    "# 3. 그 예측에 대해서 SSE를 계산\n",
    "# 4. 현재 예측한 theta를 가지고 plot을 해봄\n",
    "\n",
    "# 1. theta <- 주어진 데이터(X, y)로 구함\n",
    "theta = np.linalg.inv( X.T.dot(X) ).dot(X.T.dot(y))\n",
    "\n",
    "# 2. 구한 theta에 대해서 예측을 수행\n",
    "# 학습 데이터에 들어있는 정답 출력 = y\n",
    "# (실제 입력 X) x (방금 구한 theta) = y_predict\n",
    "y_predict = X.dot(theta)\n",
    "\n",
    "# 3. 그 예측에 대해서 SSE를 계산\n",
    "# SSE cost = sum [ (정답 y) - (예측 y_predict) ]^2\n",
    "SSE = np.sum( (y - y_predict)**2 )\n",
    "\n",
    "# 4. 현재 예측한 theta를 가지고 plot을 해봄\n",
    "y_plot = X_plot.dot(theta)\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(X[:,1], y) # 주어진 학습 데이터를 점으로 찍음\n",
    "plt.plot(X_plot[:,1], y_plot) # 전체 입력을 잘게 쪼개서, 전체 예측을 구함"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a0f9c4e3d0>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 44684 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 20824 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# batch gradient descent\n",
    "# theta <- 랜덤으로 하나 뽑아서 초기화\n",
    "# epoch을 반복\n",
    "## y_predict <= theta, X\n",
    "## gradient  <= y_predict, y, X <= 2* X.T (y_predict - y)\n",
    "## update    <= theta - eta*gradient\n",
    "\n",
    "# hyperparameter: learning rate, maximum epoch\n",
    "eta = 0.01\n",
    "maxEpoch = 10\n",
    "\n",
    "# theta를 랜덤으로 초기화 한번\n",
    "theta = np.random.randn(2)\n",
    "SSE = []\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(X[:,1], y)\n",
    "for epoch in range(maxEpoch):\n",
    "    y_predict = X.dot(theta)\n",
    "    \n",
    "    # SSE cost 구하는거 추가\n",
    "    SSE.append( np.sum( (y - y_predict)**2 ) )\n",
    "    \n",
    "    # 예측값 plot하는거 추가\n",
    "    y_plot = X_plot.dot(theta)\n",
    "    plt.plot(X_plot[:,1], y_plot)\n",
    "    \n",
    "    grad = 2*X.T.dot(y_predict - y)\n",
    "    theta = theta - eta*grad\n",
    "    \n",
    "plt.figure()\n",
    "plt.plot(SSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a0f9d03190>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 20472 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# stochastic gradient descent\n",
    "# batch gradient      <= 2 * X.^T     * (y_predict - y)\n",
    "# stochastic gradient <= 2 * X[n,:].T * (y_predict[n] - y[n])\n",
    "\n",
    "# hyperparameter: learning rate, maximum epoch\n",
    "eta = 0.1\n",
    "maxEpoch = 1\n",
    "\n",
    "# theta를 랜덤으로 초기화 한번\n",
    "theta = np.random.randn(2)\n",
    "SSE = []\n",
    "\n",
    "idsample = np.arange(N)\n",
    "np.random.shuffle(idsample)\n",
    "\n",
    "for epoch in range(maxEpoch):\n",
    "    for n in range(N):\n",
    "        y_predict = X.dot(theta)\n",
    "        SSE.append(np.sum( (y - y_predict)**2 ))\n",
    "    \n",
    "        grad = 2 * X[idsample[n],:].reshape(2) * (y_predict[idsample[n]] - y[idsample[n]])\n",
    "        theta = theta - eta*grad    \n",
    "\n",
    "plt.figure()\n",
    "plt.plot(SSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101.58787094999292\n",
      "[ 9.99565423  3.00421652 -4.99892138  1.0027838 ]\n",
      "110.10993563801085\n",
      "[ 9.90756188  3.05422488 -4.92987314  1.05227967]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 20040 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "N = 10000\n",
    "x1 = np.random.rand(N)\n",
    "x2 = np.random.rand(N)\n",
    "x3 = np.random.rand(N)\n",
    "y = 10 + 3*x1 - 5*x2 + 1*x3 + 1e-1*np.random.randn(N)\n",
    "\n",
    "X = np.hstack([np.ones((N,1)), x1.reshape(N,1), x2.reshape(N,1), x3.reshape(N,1)])\n",
    "\n",
    "# normal equation\n",
    "theta = np.linalg.inv( X.T.dot(X) ).dot(X.T.dot(y))\n",
    "y_predict = X.dot(theta)\n",
    "SSE = np.sum( (y - y_predict)**2 )\n",
    "print(SSE)\n",
    "print(theta)\n",
    "\n",
    "# batch gradient descent\n",
    "maxEpoch = 200\n",
    "eta = 0.00001\n",
    "\n",
    "SSE = []\n",
    "\n",
    "theta = np.random.randn(4)\n",
    "theta = np.array([9, 3, -4.5, 1])\n",
    "for epoch in range(maxEpoch):\n",
    "    y_predict = X.dot(theta)\n",
    "    grad = 2*X.T.dot(y_predict - y)\n",
    "    theta = theta - eta*grad\n",
    "    SSE.append( np.sum( (y - y_predict)**2 ) )\n",
    "\n",
    "plt.plot(SSE)\n",
    "print(SSE[-1])\n",
    "print(theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101.58787094999295\n",
      "[ 3.00421652 -4.99892138  1.0027838 ]\n",
      "9.995654231291406\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "X = np.hstack([x1.reshape(N,1), x2.reshape(N,1), x3.reshape(N,1)])\n",
    "\n",
    "model = LinearRegression()\n",
    "model.fit(X, y)\n",
    "\n",
    "y_predict = model.predict(X)\n",
    "SSE = np.sum( (y - y_predict)**2 )\n",
    "print(SSE)\n",
    "print(model.coef_)\n",
    "print(model.intercept_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
