{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "30ad8848",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 15108 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import numpy.matlib\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "N = 1000\n",
    "D = 2\n",
    "\n",
    "X = np.random.randn(N,D)\n",
    "A = np.random.randn(D,D) # 2x2, 짜부\n",
    "X = X.dot(A)\n",
    "\n",
    "plt.figure()\n",
    "ax = plt.subplot(1,1,1)\n",
    "plt.scatter(X[:,0], X[:,1], c = 'gray', s = 2)\n",
    "ax.set_aspect('equal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5cb7eaa1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 2)\n",
      "[[ 7.2985444  -1.35145074]\n",
      " [-1.35145074  0.26531687]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a4bebdef40>]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 17044 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# X = N x D matrix\n",
    "C = np.cov(X, rowvar = False)\n",
    "print(C.shape)\n",
    "print(C)\n",
    "\n",
    "U, s, V = np.linalg.svd(C)\n",
    "S = np.diag(s)\n",
    "V = V.T\n",
    "\n",
    "u1 = U[:,0]\n",
    "u2 = U[:,1]\n",
    "\n",
    "s1 = s[0]\n",
    "s2 = s[1]\n",
    "\n",
    "plt.figure()\n",
    "ax = plt.subplot(1,1,1)\n",
    "plt.scatter(X[:,0], X[:,1], c = 'gray', s = 2)\n",
    "ax.set_aspect('equal')\n",
    "\n",
    "plt.plot([0, s1*u1[0]], [0, s1*u1[1]])\n",
    "plt.plot([0, s2*u2[0]], [0, s2*u2[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4cf8d901",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 0.6691, test 0.6740\n"
     ]
    }
   ],
   "source": [
    "from sklearn import tree\n",
    "from keras.datasets import mnist\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "((x_train, t_train), (x_test, t_test)) = mnist.load_data()\n",
    "\n",
    "X_train = x_train.reshape(-1,28*28)\n",
    "X_test = x_test.reshape(-1,28*28)\n",
    "\n",
    "N_train,D = X_train.shape\n",
    "N_test,D = X_test.shape\n",
    "\n",
    "model = tree.DecisionTreeClassifier(max_depth = 10, min_samples_split = 1000, min_samples_leaf = 1000)\n",
    "model.fit(X_train,t_train)\n",
    "\n",
    "y_train = model.predict(X_train)\n",
    "y_test = model.predict(X_test)\n",
    "\n",
    "print( 'train %.4f, test %.4f' % (accuracy_score(y_train, t_train), accuracy_score(y_test, t_test)) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "324d642e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dim   2: train 0.4458, test 0.4513\n",
      "dim   3: train 0.4810, test 0.4914\n",
      "dim   4: train 0.5522, test 0.5480\n",
      "dim   5: train 0.6088, test 0.6184\n",
      "dim   6: train 0.6435, test 0.6439\n",
      "dim   7: train 0.6742, test 0.6712\n",
      "dim   8: train 0.6961, test 0.6924\n",
      "dim   9: train 0.6963, test 0.6927\n",
      "dim  10: train 0.6965, test 0.6897\n",
      "dim  11: train 0.6965, test 0.6897\n",
      "dim  12: train 0.7013, test 0.6957\n",
      "dim  13: train 0.7011, test 0.7009\n",
      "dim  14: train 0.7019, test 0.7011\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a4cfdc6760>]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 27248 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "C = np.cov(X_train, rowvar = False)\n",
    "# print(C.shape)\n",
    "\n",
    "U, s, V = np.linalg.svd(C)\n",
    "S = np.diag(s)\n",
    "V = V.T\n",
    "\n",
    "dimlist = np.arange(2, 15)\n",
    "\n",
    "accuracy_train = []\n",
    "accuracy_test = []\n",
    "\n",
    "for dim_reduction in dimlist:\n",
    "\n",
    "    Z_train = X_train.dot(U[:,0:dim_reduction])\n",
    "    Z_test = X_test.dot(U[:,0:dim_reduction])\n",
    "\n",
    "    model = tree.DecisionTreeClassifier(max_depth = 10, min_samples_split = 1000, min_samples_leaf = 1000)\n",
    "    model.fit(Z_train,t_train)\n",
    "\n",
    "    y_train = model.predict(Z_train)\n",
    "    y_test = model.predict(Z_test)\n",
    "\n",
    "    print( 'dim %3d: train %.4f, test %.4f' % (dim_reduction, accuracy_score(y_train, t_train), accuracy_score(y_test, t_test)) )\n",
    "    \n",
    "    accuracy_train.append(accuracy_score(y_train, t_train))\n",
    "    accuracy_test.append(accuracy_score(y_test, t_test))    \n",
    " \n",
    "plt.figure()\n",
    "plt.plot(dimlist, accuracy_train, 'o-')\n",
    "plt.plot(dimlist, accuracy_test, 'o-')   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "08b873d5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dim   2: train 0.4458, test 0.4514\n",
      "dim   3: train 0.4810, test 0.4913\n",
      "dim   4: train 0.5523, test 0.5480\n",
      "dim   5: train 0.6103, test 0.6202\n",
      "dim   6: train 0.6435, test 0.6439\n",
      "dim   7: train 0.6742, test 0.6710\n",
      "dim   8: train 0.6961, test 0.6924\n",
      "dim   9: train 0.6963, test 0.6927\n",
      "dim  10: train 0.6965, test 0.6897\n",
      "dim  11: train 0.6964, test 0.6896\n",
      "dim  12: train 0.7013, test 0.6957\n",
      "dim  13: train 0.7011, test 0.7009\n",
      "dim  14: train 0.7019, test 0.7011\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a48014ac40>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 27812 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "accuracy_train = []\n",
    "accuracy_test = []\n",
    "\n",
    "dimlist = np.arange(2, 15)\n",
    "\n",
    "for dim_reduction in dimlist:\n",
    "    pca = PCA(n_components = dim_reduction)\n",
    "    pca.fit(X_train)\n",
    "\n",
    "    Z_train = pca.transform(X_train)\n",
    "    Z_test = pca.transform(X_test)\n",
    "\n",
    "    model = tree.DecisionTreeClassifier(max_depth = 10, min_samples_split = 1000, min_samples_leaf = 1000)\n",
    "    model.fit(Z_train,t_train)\n",
    "\n",
    "    y_train = model.predict(Z_train)\n",
    "    y_test = model.predict(Z_test)\n",
    "\n",
    "    print( 'dim %3d: train %.4f, test %.4f' % (dim_reduction, accuracy_score(y_train, t_train), accuracy_score(y_test, t_test)) )\n",
    "    \n",
    "    accuracy_train.append(accuracy_score(y_train, t_train))\n",
    "    accuracy_test.append(accuracy_score(y_test, t_test))    \n",
    "    \n",
    "plt.figure()\n",
    "plt.plot(dimlist, accuracy_train, 'o-')\n",
    "plt.plot(dimlist, accuracy_test, 'o-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cad0320b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jacob\\anaconda3\\lib\\site-packages\\sklearn\\decomposition\\_nmf.py:1637: ConvergenceWarning: Maximum number of iterations 100 reached. Increase it to improve convergence.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(784, 10)\n",
      "(10, 60000)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 34192 characters]\n",
      "text/plain": [
       "<Figure size 1600x700 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import NMF \n",
    "\n",
    "D_reduce = 10\n",
    "\n",
    "nmf = NMF(n_components=D_reduce, init='random', random_state=0, max_iter=100)\n",
    "nmf.fit(X_train.T)\n",
    "\n",
    "W = nmf.transform(X_train.T) # basis\n",
    "H = nmf.components_ # weight\n",
    "\n",
    "print(W.shape)\n",
    "print(H.shape)\n",
    "    \n",
    "n = np.random.randint(0,N,5)\n",
    "plt.figure(figsize = (16,7))\n",
    "for i in range(5):\n",
    "    plt.subplot(2,5,i + 1)\n",
    "    plt.imshow(X_train[n[i],:].reshape(28,28), cmap = 'gray')\n",
    "    plt.subplot(2,5,i + 6)\n",
    "    plt.imshow(W.dot(H[:,n[i]]).reshape(28,28), cmap = 'gray')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
