From c714d25bd33c33fe886301962c45d0a55126e1db Mon Sep 17 00:00:00 2001 From: Frank Schultz Date: Sat, 19 Oct 2024 13:23:55 +0200 Subject: [PATCH] mods for ws24/24 - torch support - backend K removed - complex valued left inverse by SGD --- .binder/requirements.txt | 15 +- ci/requirements_env.txt | 18 +- exercise12_HyperParameterTuning.ipynb | 7 +- exercise12_MusicGenreClassification.ipynb | 17 +- exercise13_CNN.py | 2 +- ...ent_descent_on_complex_least_squares.ipynb | 238 ++++++++++++++++++ index.ipynb | 5 +- regression_xor_twolayers.ipynb | 19 +- 8 files changed, 284 insertions(+), 37 deletions(-) create mode 100644 gradient_descent_on_complex_least_squares.ipynb diff --git a/.binder/requirements.txt b/.binder/requirements.txt index fc3f9f5..de6992c 100644 --- a/.binder/requirements.txt +++ b/.binder/requirements.txt @@ -1,11 +1,14 @@ -matplotlib +pytorch +torchvision +complextorch +tensorflow +keras-tuner +scikit-learn +statsmodels numpy scipy +matplotlib +ipympl librosa soundfile -ipympl -scikit-learn -tensorflow -keras-tuner -statsmodels pyloudnorm diff --git a/ci/requirements_env.txt b/ci/requirements_env.txt index 5fa8e5b..de6992c 100644 --- a/ci/requirements_env.txt +++ b/ci/requirements_env.txt @@ -1,10 +1,14 @@ -ipympl +pytorch +torchvision +complextorch +tensorflow keras-tuner -librosa -matplotlib -numpy -pyloudnorm -scipy scikit-learn statsmodels -tensorflow +numpy +scipy +matplotlib +ipympl +librosa +soundfile +pyloudnorm diff --git a/exercise12_HyperParameterTuning.ipynb b/exercise12_HyperParameterTuning.ipynb index f867750..4c1443c 100644 --- a/exercise12_HyperParameterTuning.ipynb +++ b/exercise12_HyperParameterTuning.ipynb @@ -64,7 +64,6 @@ "from sklearn.preprocessing import OneHotEncoder, LabelBinarizer\n", "import tensorflow as tf\n", "from tensorflow import keras\n", - "from keras import backend as K\n", "import time\n", "\n", "print(\n", @@ -190,7 +189,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "8b7a07f0-7151-4914-93ca-6efb3c22779c", "metadata": {}, "outputs": [], @@ -202,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "e1eae32c-83b4-4b69-837f-31e3d87a470a", "metadata": {}, "outputs": [], @@ -326,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "e76a4f45-6f5b-4755-9f98-7ad228efdf97", "metadata": {}, "outputs": [], diff --git a/exercise12_MusicGenreClassification.ipynb b/exercise12_MusicGenreClassification.ipynb index 14e77ae..8f57c3c 100644 --- a/exercise12_MusicGenreClassification.ipynb +++ b/exercise12_MusicGenreClassification.ipynb @@ -85,7 +85,6 @@ "from sklearn.preprocessing import OneHotEncoder, LabelBinarizer\n", "import tensorflow as tf\n", "from tensorflow import keras\n", - "from keras import backend as K\n", "import time\n", "\n", "\n", @@ -157,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "ed5b4e1e-c6fe-4363-80bc-919558d6e06b", "metadata": {}, "outputs": [], @@ -269,7 +268,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "23424375-717e-4ca1-bcf7-9bd4a4fe8af8", "metadata": {}, "outputs": [], @@ -281,7 +280,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "4a58498f-2445-4837-a9c0-95b940fd9086", "metadata": {}, "outputs": [], @@ -307,7 +306,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "39305f54-e211-4564-853d-28dfc52119c1", "metadata": {}, "outputs": [], @@ -465,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "id": "49cb1e2f-da96-4b43-8490-f5b96d0c60cb", "metadata": {}, "outputs": [], @@ -527,7 +526,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "id": "dbf6b6d1-dfb7-47ed-a723-967f2d1f52dc", "metadata": {}, "outputs": [], @@ -539,7 +538,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "id": "dabce6d1-009d-470f-8407-788b7478ab70", "metadata": {}, "outputs": [], @@ -660,7 +659,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "id": "4f27ff18-5dd4-42fa-86a1-60926c9b5993", "metadata": {}, "outputs": [], diff --git a/exercise13_CNN.py b/exercise13_CNN.py index 4509f6e..4b35ca8 100644 --- a/exercise13_CNN.py +++ b/exercise13_CNN.py @@ -10,7 +10,7 @@ import numpy as np import tensorflow as tf -import tensorflow.keras as keras +from tensorflow import keras print("element-wise multiplication of matrix slices") diff --git a/gradient_descent_on_complex_least_squares.ipynb b/gradient_descent_on_complex_least_squares.ipynb new file mode 100644 index 0000000..853cd56 --- /dev/null +++ b/gradient_descent_on_complex_least_squares.ipynb @@ -0,0 +1,238 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4ff5888c", + "metadata": {}, + "source": [ + "Sascha Spors,\n", + "Professorship Signal Theory and Digital Signal Processing,\n", + "Institute of Communications Engineering (INT),\n", + "Faculty of Computer Science and Electrical Engineering (IEF),\n", + "University of Rostock,\n", + "Germany\n", + "\n", + "# Data Driven Audio Signal Processing - A Tutorial with Computational Examples\n", + "\n", + "Winter Semester 2023/24 (Master Course #24512)\n", + "\n", + "- lecture: https://github.com/spatialaudio/data-driven-audio-signal-processing-lecture\n", + "- tutorial: https://github.com/spatialaudio/data-driven-audio-signal-processing-exercise\n", + "\n", + "Feel free to contact lecturer frank.schultz@uni-rostock.de" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a4b3cf8b", + "metadata": {}, + "outputs": [], + "source": [ + "# Linear problem y = X w\n", + "# - with complex-valued data\n", + "# - with full column rank F, tall/thin, pure column space matrix X\n", + "# - given feature matrix X and ground truth outcome y\n", + "# - unknown weights w\n", + "# 1. solve for w with left inverse of X (complex valued closed form solution)\n", + "# 2. iteratively solve with complex, linear layer (without bias) and\n", + "# ADAM stochastic gradient descent\n", + "\n", + "import numpy as np\n", + "import torch\n", + "from torch.utils.data import TensorDataset, DataLoader\n", + "from complextorch.nn.modules.linear import CVLinear\n", + "from complextorch.nn.modules.loss import CVQuadError\n", + "\n", + "torch.manual_seed(1)\n", + "rng = np.random.default_rng(1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96c70af9", + "metadata": {}, + "outputs": [], + "source": [ + "N = 2**10 # number of rows for tall/thin X = number of data samples\n", + "F = 3 # number of columns for tall/thin X = number of features\n", + "w_ground_truth = (np.arange(F)+1) - 1j*((np.arange(F)-F)) # nice numbers\n", + "X_train = rng.normal(size=(N, F)) + 1j * rng.normal(size=(N, F))\n", + "print('\\nmatrix rank == F ? ', np.allclose(np.linalg.matrix_rank(X_train), F))\n", + "U, _, _ = np.linalg.svd(X_train)\n", + "X_train = U[:, 0:F] # X is now pure column space\n", + "y_pure_column_space = X_train @ w_ground_truth # linear combination of pure column space\n", + "y_train = y_pure_column_space + np.sqrt(N)*U[:, F+1] # add 'noise' from left null space,\n", + "# such that we precisely know the residual\n", + "\n", + "residual = y_train - y_pure_column_space\n", + "theoretical_empirical_risk = np.inner(np.conj(residual), residual) / N\n", + "print('\\ntheoretical_empirical_risk', theoretical_empirical_risk)\n", + "# theoretical empirical risk -> any optimisation can never get it lower than that,\n", + "# because linear algebra fundamentals cannot be beaten -> keep this in mind if\n", + "# desperately trying to reduce this loss further\n", + "# if the values above are unchanged, theoretical_empirical_risk = 1\n", + "\n", + "# note: CVQuadError loss used below is normalised by 1/2\n", + "# and not averaged by batch_size\n", + "# so only the empirical risk of the finally trained model should be\n", + "# directly compared with the above theoretical_empirical_risk\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57f40877", + "metadata": {}, + "outputs": [], + "source": [ + "# prep for torch / complextorch\n", + "X_train = torch.from_numpy(X_train.astype('complex64'))\n", + "y_train = torch.from_numpy(y_train.astype('complex64'))\n", + "print()\n", + "print(X_train.shape, X_train.dtype)\n", + "print(y_train.shape, y_train.dtype)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7462572a", + "metadata": {}, + "outputs": [], + "source": [ + "# closed form solution w = (X^H X)^-1 X^H y with torch tensor handling\n", + "w_left_inverse = torch.matmul(torch.matmul(torch.inverse(torch.matmul(X_train.H, X_train)), X_train.H), y_train)\n", + "print('\\nweights true vs. weights from left inverse',\n", + " '\\n', w_ground_truth,\n", + " '\\n', w_left_inverse.detach().numpy())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a0ccf52", + "metadata": {}, + "outputs": [], + "source": [ + "# ML parameters\n", + "B = N // 64\n", + "batch_size = N // B\n", + "learning_rate = 1e-3\n", + "num_epochs = 1000 + 1\n", + "log_epoch = 100\n", + "\n", + "\n", + "class Model(torch.nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " # simple layer: 1 perceptron with F inputs\n", + " self.layer1 = CVLinear(F, 1, bias=False)\n", + "\n", + " def predict_train(self, x):\n", + " # backprop/autograd is by default enabled\n", + " # so x.real and x.imag have grad_fn pointers\n", + " # for training\n", + " x = self.layer1(x)\n", + " return x\n", + "\n", + " def predict_test(self, x):\n", + " # we don't need all the backprop stuff in test prediction\n", + " # so x has no grad_fn object assigned\n", + " with torch.no_grad():\n", + " return self.predict_train(x)\n", + "\n", + "\n", + "# data handling, we do no split into train/test\n", + "train_ds = TensorDataset(X_train, y_train)\n", + "train_dl = DataLoader(train_ds, batch_size, shuffle=True)\n", + "# prep model\n", + "model = Model()\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n", + "loss_fn = CVQuadError()\n", + "\n", + "print(model)\n", + "print('batch_size', batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad98512c", + "metadata": {}, + "outputs": [], + "source": [ + "print('\\nlearn / train ...')\n", + "for epoch in range(num_epochs):\n", + " for X_batch, y_batch in train_dl:\n", + " y_pred = model.predict_train(X_batch)\n", + " loss = loss_fn(y_pred, y_batch[:, None])\n", + " loss.backward()\n", + " optimizer.step()\n", + " optimizer.zero_grad()\n", + " if epoch % log_epoch == 0:\n", + " print(f'epoch {epoch} last batch loss {loss.item():.4e}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92d234eb", + "metadata": {}, + "outputs": [], + "source": [ + "print('\\npredict...')\n", + "# model is trained, check it\n", + "residual = (model.predict_test(X_train)[:, 0] - y_train).detach().numpy()\n", + "empirical_risk = np.inner(np.conj(residual), residual) / N\n", + "print('\\nempirical_risk', empirical_risk)\n", + "print('\\ntheoretical_empirical_risk', theoretical_empirical_risk)\n", + "# check the learned weights\n", + "layer = model.layer1.state_dict()\n", + "print('\\nweights true vs. from trained model')\n", + "print('real part')\n", + "print(w_ground_truth.real)\n", + "print(layer['linear_r.weight'].detach().numpy())\n", + "print('imag part')\n", + "print(w_ground_truth.imag)\n", + "print(layer['linear_i.weight'].detach().numpy())\n" + ] + }, + { + "cell_type": "markdown", + "id": "651b1eff", + "metadata": {}, + "source": [ + "## Copyright\n", + "\n", + "- the notebooks are provided as [Open Educational Resources](https://en.wikipedia.org/wiki/Open_educational_resources)\n", + "- feel free to use the notebooks for your own purposes\n", + "- the text is licensed under [Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/)\n", + "- the code of the IPython examples is licensed under the [MIT license](https://opensource.org/licenses/MIT)\n", + "- please attribute the work as follows: *Frank Schultz, Data Driven Audio Signal Processing - A Tutorial Featuring Computational Examples, University of Rostock* ideally with relevant file(s), github URL https://github.com/spatialaudio/data-driven-audio-signal-processing-exercise, commit number and/or version tag, year." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "myddasp", + "language": "python", + "name": "myddasp" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/index.ipynb b/index.ipynb index 09e035e..f577def 100644 --- a/index.ipynb +++ b/index.ipynb @@ -27,7 +27,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Syllabus Winter Semester 2023/24\n", + "# Planned Syllabus Winter Semester 2024/25\n", "\n", "- [Numerical Examples from the Slides](slides/ddasp_exercise_slides.ipynb)\n", "\n", @@ -79,6 +79,8 @@ "- [Gradient Descent 2](gradient_descent2.ipynb) with saddle points, local maximum, local minima and a global minimum\n", "- [Gradient Descent with Momentum](gradient_descent_momentum.ipynb)\n", "- [Stochastic Gradient Descent for Least Squares Error](gradient_descent_on_least_squares.ipynb)\n", + "- [Stochastic Gradient Descent for Complex Data Least Squares Error Using PyTorch](gradient_descent_on_complex_data_least_squares.ipynb)\n", + "\n", "\n", "\n", "## Exercise: XOR as Non-Linear, Two-Layer Model\n", @@ -151,6 +153,7 @@ "## Textbook Recommendations\n", "Machine Learning (ML) using linear / non-linear models is a vivid topic and dozens of textbooks will be released each year.\n", "The following textbook recommendations are very often referenced in the field and brilliant to learn with. \n", + "- Sebastian **Raschka**, Yuxi Liu, Vahid Mirjalili: *Machine Learning with PyTorch and Scikit-Learn*, Packt, 2022, 1st ed.\n", "- Gilbert **Strang**: *Linear Algebra and Learning from Data*, Wellesley, 2019, consider to buy your own copy of this brilliant book\n", "- Gareth **James**, Daniela Witten, Trevor Hastie, Rob Tibshirani: *An Introduction to Statistical Learning* with Applications in R, Springer, 2nd ed., 2021, [free pdf e-book](https://www.statlearning.com/)\n", "- Trevor **Hastie**, Robert Tibshirani, Jerome Friedman: *The Elements of Statistical Learning: Data Mining, Inference, and Prediction*, Springer, 2nd ed., 2009, [free pdf e-book](https://hastie.su.domains/ElemStatLearn/)\n", diff --git a/regression_xor_twolayers.ipynb b/regression_xor_twolayers.ipynb index ddcfc14..2b95a06 100644 --- a/regression_xor_twolayers.ipynb +++ b/regression_xor_twolayers.ipynb @@ -40,11 +40,9 @@ "metadata": {}, "outputs": [], "source": [ - "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import tensorflow as tf\n", - "import tensorflow.keras as keras\n", - "import tensorflow.keras.backend as K\n", + "from tensorflow import keras\n", "\n", "print(\n", " \"TF version\",\n", @@ -56,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "9d77047c", "metadata": {}, "outputs": [], @@ -126,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "1c792c68", "metadata": {}, "outputs": [], @@ -150,7 +148,10 @@ "model.add(keras.layers.Dense(2, activation=\"relu\"))\n", "model.add(keras.layers.Dense(1, activation=\"linear\"))\n", "model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n", - "tw = np.sum([K.count_params(w) for w in model.trainable_weights])\n", + "tw = 0\n", + "for w in model.trainable_weights:\n", + " print(w)\n", + " tw += np.prod(w.shape)\n", "print(\"\\ntrainable_weights:\", tw, \"\\n\")" ] }, @@ -233,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "b7edb9f1", "metadata": {}, "outputs": [], @@ -274,7 +275,7 @@ "kernelspec": { "display_name": "myddasp", "language": "python", - "name": "myddasp" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -286,7 +287,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.12.3" } }, "nbformat": 4,