MNISTProperNNClassification.ipynb (16907B)
1 { 2 "cells": [ 3 { 4 "cell_type": "markdown", 5 "metadata": {}, 6 "source": [ 7 "This is how it should be done. One hot encode inputs then train model with softmax function being applied on the output layer. This will calculate probability of the inputs being members of each class. When compiling the model make sure to add the accuracy metric so accuracy is visible at the end not just the loss function calculation which is categorical cross entropy. \n", 8 "\n", 9 "One thing, yes, I should have split the training set one more time so I would have a training, validation, and testing set, but that does not seem too important as this is a proof of concept.\n", 10 "\n", 11 "Current accuracy:\n", 12 "96.4%" 13 ] 14 }, 15 { 16 "cell_type": "code", 17 "execution_count": 1, 18 "metadata": {}, 19 "outputs": [ 20 { 21 "name": "stderr", 22 "output_type": "stream", 23 "text": [ 24 "2024-06-11 10:44:46.272808: I external/local_tsl/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used.\n", 25 "2024-06-11 10:44:46.276173: I external/local_tsl/tsl/cuda/cudart_stub.cc:32] Could not find cuda drivers on your machine, GPU will not be used.\n", 26 "2024-06-11 10:44:46.319606: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", 27 "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", 28 "2024-06-11 10:44:47.042758: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" 29 ] 30 } 31 ], 32 "source": [ 33 "import tensorflow as tf\n", 34 "from sklearn.datasets import fetch_openml\n", 35 "from tensorflow.keras.models import Sequential\n", 36 "from tensorflow.keras import layers\n", 37 "import numpy as np\n", 38 "\n", 39 "np.random.seed(10)\n", 40 "\n", 41 "mnist = fetch_openml(\"mnist_784\", as_frame=False)" 42 ] 43 }, 44 { 45 "cell_type": "code", 46 "execution_count": 2, 47 "metadata": {}, 48 "outputs": [ 49 { 50 "data": { 51 "text/plain": [ 52 "(70000, 784)" 53 ] 54 }, 55 "execution_count": 2, 56 "metadata": {}, 57 "output_type": "execute_result" 58 } 59 ], 60 "source": [ 61 "from tensorflow.keras.utils import to_categorical\n", 62 "y = mnist.target\n", 63 "X = mnist.data\n", 64 "\n", 65 "y = to_categorical(y,10)\n", 66 "X.shape" 67 ] 68 }, 69 { 70 "cell_type": "code", 71 "execution_count": 3, 72 "metadata": {}, 73 "outputs": [], 74 "source": [ 75 "from sklearn.decomposition import PCA\n", 76 "from sklearn.model_selection import train_test_split\n", 77 "\n", 78 "pca = PCA(n_components=.95)\n", 79 "X = pca.fit_transform(X)\n", 80 "\n", 81 "X_train , X_test, y_train, y_test = train_test_split(X,y)" 82 ] 83 }, 84 { 85 "cell_type": "code", 86 "execution_count": 4, 87 "metadata": {}, 88 "outputs": [ 89 { 90 "name": "stderr", 91 "output_type": "stream", 92 "text": [ 93 "2024-06-11 10:45:01.973478: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", 94 "2024-06-11 10:45:01.974054: W tensorflow/core/common_runtime/gpu/gpu_device.cc:2251] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n", 95 "Skipping registering GPU devices...\n" 96 ] 97 } 98 ], 99 "source": [ 100 "model = Sequential()\n", 101 "input_1 = tf.keras.layers.Input([len(X[0])])\n", 102 "hidden_1 = tf.keras.layers.Dense(100 , 'relu')\n", 103 "hidden_2 = tf.keras.layers.Dense(100 , 'relu')\n", 104 "hidden_3 = tf.keras.layers.Dense(100 , 'relu')\n", 105 "output_1 = tf.keras.layers.Dense(10, 'softmax')\n", 106 "\n", 107 "model.add(input_1)\n", 108 "model.add(hidden_1)\n", 109 "model.add(hidden_2)\n", 110 "model.add(hidden_3)\n", 111 "model.add(output_1)\n" 112 ] 113 }, 114 { 115 "cell_type": "code", 116 "execution_count": 5, 117 "metadata": {}, 118 "outputs": [ 119 { 120 "data": { 121 "text/html": [ 122 "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential\"</span>\n", 123 "</pre>\n" 124 ], 125 "text/plain": [ 126 "\u001b[1mModel: \"sequential\"\u001b[0m\n" 127 ] 128 }, 129 "metadata": {}, 130 "output_type": "display_data" 131 }, 132 { 133 "data": { 134 "text/html": [ 135 "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n", 136 "┃<span style=\"font-weight: bold\"> Layer (type) </span>┃<span style=\"font-weight: bold\"> Output Shape </span>┃<span style=\"font-weight: bold\"> Param # </span>┃\n", 137 "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n", 138 "│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">100</span>) │ <span style=\"color: #00af00; text-decoration-color: #00af00\">15,500</span> │\n", 139 "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", 140 "│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">100</span>) │ <span style=\"color: #00af00; text-decoration-color: #00af00\">10,100</span> │\n", 141 "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", 142 "│ dense_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">100</span>) │ <span style=\"color: #00af00; text-decoration-color: #00af00\">10,100</span> │\n", 143 "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", 144 "│ dense_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">10</span>) │ <span style=\"color: #00af00; text-decoration-color: #00af00\">1,010</span> │\n", 145 "└─────────────────────────────────┴────────────────────────┴───────────────┘\n", 146 "</pre>\n" 147 ], 148 "text/plain": [ 149 "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n", 150 "┃\u001b[1m \u001b[0m\u001b[1mLayer (type) \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m Param #\u001b[0m\u001b[1m \u001b[0m┃\n", 151 "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n", 152 "│ dense (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m100\u001b[0m) │ \u001b[38;5;34m15,500\u001b[0m │\n", 153 "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", 154 "│ dense_1 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m100\u001b[0m) │ \u001b[38;5;34m10,100\u001b[0m │\n", 155 "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", 156 "│ dense_2 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m100\u001b[0m) │ \u001b[38;5;34m10,100\u001b[0m │\n", 157 "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", 158 "│ dense_3 (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m10\u001b[0m) │ \u001b[38;5;34m1,010\u001b[0m │\n", 159 "└─────────────────────────────────┴────────────────────────┴───────────────┘\n" 160 ] 161 }, 162 "metadata": {}, 163 "output_type": "display_data" 164 }, 165 { 166 "data": { 167 "text/html": [ 168 "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">36,710</span> (143.40 KB)\n", 169 "</pre>\n" 170 ], 171 "text/plain": [ 172 "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m36,710\u001b[0m (143.40 KB)\n" 173 ] 174 }, 175 "metadata": {}, 176 "output_type": "display_data" 177 }, 178 { 179 "data": { 180 "text/html": [ 181 "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">36,710</span> (143.40 KB)\n", 182 "</pre>\n" 183 ], 184 "text/plain": [ 185 "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m36,710\u001b[0m (143.40 KB)\n" 186 ] 187 }, 188 "metadata": {}, 189 "output_type": "display_data" 190 }, 191 { 192 "data": { 193 "text/html": [ 194 "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n", 195 "</pre>\n" 196 ], 197 "text/plain": [ 198 "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n" 199 ] 200 }, 201 "metadata": {}, 202 "output_type": "display_data" 203 } 204 ], 205 "source": [ 206 "model.summary()" 207 ] 208 }, 209 { 210 "cell_type": "code", 211 "execution_count": 9, 212 "metadata": {}, 213 "outputs": [], 214 "source": [ 215 "optimize = tf.keras.optimizers.Adam()\n", 216 "model.compile(optimizer=optimize,\n", 217 " loss='categorical_crossentropy',\n", 218 " metrics=['accuracy']\n", 219 ")\n", 220 "\n" 221 ] 222 }, 223 { 224 "cell_type": "code", 225 "execution_count": 7, 226 "metadata": {}, 227 "outputs": [ 228 { 229 "name": "stdout", 230 "output_type": "stream", 231 "text": [ 232 "Epoch 1/10\n", 233 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 1ms/step - loss: 6.3211 - val_loss: 0.5532\n", 234 "Epoch 2/10\n", 235 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step - loss: 0.3455 - val_loss: 0.3746\n", 236 "Epoch 3/10\n", 237 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 1ms/step - loss: 0.1859 - val_loss: 0.2747\n", 238 "Epoch 4/10\n", 239 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step - loss: 0.1428 - val_loss: 0.2256\n", 240 "Epoch 5/10\n", 241 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step - loss: 0.1143 - val_loss: 0.2141\n", 242 "Epoch 6/10\n", 243 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step - loss: 0.1031 - val_loss: 0.1912\n", 244 "Epoch 7/10\n", 245 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step - loss: 0.0922 - val_loss: 0.1822\n", 246 "Epoch 8/10\n", 247 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step - loss: 0.0778 - val_loss: 0.1734\n", 248 "Epoch 9/10\n", 249 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step - loss: 0.0781 - val_loss: 0.1852\n", 250 "Epoch 10/10\n", 251 "\u001b[1m1641/1641\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 1ms/step - loss: 0.0636 - val_loss: 0.1755\n" 252 ] 253 }, 254 { 255 "data": { 256 "text/plain": [ 257 "<keras.src.callbacks.history.History at 0x7f9c4306db10>" 258 ] 259 }, 260 "execution_count": 7, 261 "metadata": {}, 262 "output_type": "execute_result" 263 } 264 ], 265 "source": [ 266 "import numpy as np\n", 267 "\n", 268 "X_train = np.asarray(X_train).astype('float32')\n", 269 "y_train = np.asarray(y_train).astype('float32')\n", 270 "X_test = np.asarray(X_test).astype('float32')\n", 271 "y_test = np.asarray(y_test).astype('float32')\n", 272 "\n", 273 "model.fit(epochs=10, x=X_train, y=y_train, validation_data=(X_test, y_test))" 274 ] 275 }, 276 { 277 "cell_type": "code", 278 "execution_count": 10, 279 "metadata": {}, 280 "outputs": [ 281 { 282 "name": "stdout", 283 "output_type": "stream", 284 "text": [ 285 "\u001b[1m547/547\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 580us/step - accuracy: 0.9633 - loss: 0.1776\n" 286 ] 287 }, 288 { 289 "data": { 290 "text/plain": [ 291 "[0.1755135953426361, 0.9642857313156128]" 292 ] 293 }, 294 "execution_count": 10, 295 "metadata": {}, 296 "output_type": "execute_result" 297 } 298 ], 299 "source": [ 300 "model.evaluate(x=X_test, y=y_test)" 301 ] 302 } 303 ], 304 "metadata": { 305 "kernelspec": { 306 "display_name": "myvenv", 307 "language": "python", 308 "name": "python3" 309 }, 310 "language_info": { 311 "codemirror_mode": { 312 "name": "ipython", 313 "version": 3 314 }, 315 "file_extension": ".py", 316 "mimetype": "text/x-python", 317 "name": "python", 318 "nbconvert_exporter": "python", 319 "pygments_lexer": "ipython3", 320 "version": "3.11.2" 321 } 322 }, 323 "nbformat": 4, 324 "nbformat_minor": 2 325 }