diff --git a/lecture1/exercise_0_python.ipynb b/lecture1/exercise_0_python.ipynb index abeb08060f243d303f7e607ab60a148eca8ebb61..ec543759c71a5679a1a7cd7864a5ca8d9a41622b 100644 --- a/lecture1/exercise_0_python.ipynb +++ b/lecture1/exercise_0_python.ipynb @@ -155,7 +155,7 @@ "metadata": {}, "source": [ "## Elementary Arithmetics\n", - "You can use the usual **operators** +, -, *, / for elementary arithmetics:" + "You can use the usual **operators** ```+```, ```-```, ```*```, ```/``` for elementary arithmetics:" ] }, { @@ -183,7 +183,7 @@ "id": "32b1985e", "metadata": {}, "source": [ - "You can use the **double star** \\*\\*-operator to raise number to the **n-th power**:" + "You can use the **double star** ```**```-operator to raise number to the **n-th power**:" ] }, { @@ -230,7 +230,7 @@ "metadata": {}, "source": [ "## Comparison Opeartors\n", - "You can use <, >, ==, <=, >= to **compare numbers** (**and objects**), the **keywords** ```and```, ```or``` to compare statements (booleans) and the keyword ```not``` to negate them.\n" + "You can use ```<```, ```>```, ```==```, ```<=```, ```>=``` to **compare numbers** (**and objects**), the **keywords** ```and```, ```or``` to compare statements (booleans) and the keyword ```not``` to negate them.\n" ] }, { @@ -413,7 +413,7 @@ "metadata": {}, "source": [ "## Classes\n", - "Classes are high-level objects for storing and managing primitives. They are initialized with ```class class_name:``` after which the constructor ```__init__(self, args)``` is automatically called. Data in classes are stored using the ```self``` keyword and are publicly accessible by default." + "**Classes** are high-level objects for **storing and managing primitives**. They are initialized with ```class class_name:``` after which the constructor ```__init__(self, args)``` is automatically called. Data in classes are stored using the ```self``` keyword and are publicly accessible by default." ] }, { @@ -451,7 +451,7 @@ "metadata": {}, "source": [ "## Class Inheritance\n", - "A class can inherit the properties of its parent class. These properties can be freely overriden:" + "A class can **inherit** the **properties of its parent** class. These properties can be freely overriden:" ] }, { @@ -504,7 +504,7 @@ "metadata": {}, "source": [ "### Numpy One Dimensional Array Operations\n", - "Numpy operates on arrays element-wise meaning elementary operations are performed in an intuitive way." + "```numpy``` **operates** on arrays **element-wise** thus elementary operations are performed in an intuitive way." ] }, { @@ -540,7 +540,7 @@ "id": "e6057042", "metadata": {}, "source": [ - "For this to work however the arrays must be of the same shape, otherwise a ```ValueError``` will be raised. In order to avoid this error, you can always access the shape of an array using the ```array.shape``` syntax:" + "For this to work however the arrays must be of the **same shape**, otherwise a ```ValueError``` will be raised. In order to avoid this error, you can always access the shape of an array using the ```array.shape``` syntax:" ] }, { @@ -705,6 +705,45 @@ "# Always check the docs before inventing something \"new\"!" ] }, + { + "cell_type": "markdown", + "id": "43f66264", + "metadata": {}, + "source": [ + "### Numpy Matrix Operations\n", + "By default, numpy performs element-wise multiplication using the star-operator ```*```. For matrix-matrix, matrix-vector multiplications the ```@``` can be used:" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "25b44ae6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Manual operation M*v: [-1. -3.]\n", + "Matrix multiplication @: [-1. -3.]\n" + ] + } + ], + "source": [ + "# Define a matrix manually:\n", + "matrix = np.array([[1., 1.], \n", + " [0., 3.]])\n", + "\n", + "# Define a vector manually:\n", + "vector = np.array([0., -1.])\n", + "\n", + "# The dirty and manual way would be:\n", + "print(\"Manual operation M*v: \", (matrix*vector[np.newaxis,:]).sum(axis=1))\n", + "\n", + "# Fortunately, the @-operator exists!\n", + "print(\"Matrix multiplication @:\", matrix@vector)" + ] + }, { "cell_type": "markdown", "id": "7a826bff", @@ -754,7 +793,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 50, "id": "e15a8939", "metadata": {}, "outputs": [ @@ -765,34 +804,7 @@ "[20 21 22 23 24 25 26 27 28 29]\n", "[0 1 2 3 4 5 6 7 8 9]\n", "[95 96 97 98 99]\n", - "[10 13 21]\n", - "[18 19 17 23 13 3 6 12 7 25 0 9 14 27 1 11 2 29 26 8 28 5 30 15\n", - " 10 20 24 21 22 4 16]\n", - "[[False True True False]\n", - " [False False False True]\n", - " [False False False False]\n", - " [False False True True]\n", - " [ True False False False]\n", - " [False False True False]\n", - " [ True False False True]\n", - " [False True False False]\n", - " [ True True False False]\n", - " [False True False False]\n", - " [False False False False]\n", - " [False False False False]\n", - " [False False False True]\n", - " [False True True False]\n", - " [False False False True]\n", - " [ True False False True]\n", - " [False False True False]\n", - " [ True False True True]\n", - " [False False False False]\n", - " [False False True False]\n", - " [False True False False]\n", - " [False True False False]\n", - " [False True True False]\n", - " [False False False True]\n", - " [ True False True False]]\n" + "[10 13 21]\n" ] } ], @@ -809,12 +821,58 @@ "print(array_to_be_masked[-5:])\n", "\n", "# Print the 10th, 13th and 21st elements:\n", - "print(array_to_be_masked[np.array([10, 13, 21])])\n", - "\n", + "print(array_to_be_masked[np.array([10, 13, 21])])" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "184db0da", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[54 49 48 17]\n", + "[17 1 12 22 14 29 3 9 13 21 23 8 20 6 16 2 10 11 4 19 30 18 28 15\n", + " 0 24 5 7 25 26 27]\n", + "[[False False False True]\n", + " [ True True False False]\n", + " [ True False False False]\n", + " [False False False False]\n", + " [False True True True]\n", + " [ True False False True]\n", + " [False False False True]\n", + " [ True False False False]\n", + " [False False False False]\n", + " [False True False False]\n", + " [False True False True]\n", + " [ True False False False]\n", + " [ True False False False]\n", + " [False True True False]\n", + " [ True False True False]\n", + " [False False False False]\n", + " [False False True False]\n", + " [False False False True]\n", + " [False False False False]\n", + " [False False False True]\n", + " [ True False True False]\n", + " [False True False True]\n", + " [False False False True]\n", + " [False True True False]\n", + " [False False True False]]\n" + ] + } + ], + "source": [ "# Let's shuffle the elements and reshape the array to make things more complicated!\n", "np.random.shuffle(array_to_be_masked)\n", "array_to_be_masked = array_to_be_masked.reshape((25, -1)) # \"-1\" means \"do whatever needed to get the right shape\"\n", "\n", + "# Select the first entries in each row (slicing)\n", + "print(array_to_be_masked[0, :])\n", + "\n", "# Select all entries <=30 in the flattened array:\n", "print(array_to_be_masked[(array_to_be_masked<=30)])\n", "\n", @@ -840,6 +898,19 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.12" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": true, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false } }, "nbformat": 4, diff --git a/lecture1/exercise_0_pytorch.ipynb b/lecture1/exercise_0_pytorch.ipynb index ab00ed2acea09c6905ada6d171c38505d88ab9aa..efa48e2b136c9a11bb7f5c534dbf9292e418977c 100644 --- a/lecture1/exercise_0_pytorch.ipynb +++ b/lecture1/exercise_0_pytorch.ipynb @@ -9,12 +9,12 @@ "---------------\n", "Welcome to the pytorch tutorial! Here we will go through some of the basics of pytorch.\n", "\n", - "The library can be imported the usual way:" + "The library can be imported the as ```torch```:" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 97, "id": "e812c4c3", "metadata": {}, "outputs": [], @@ -27,7 +27,208 @@ "id": "232eb87a", "metadata": {}, "source": [ - "## Tensors" + "## Tensors\n", + "Even if name ```pytorch``` is not as telling as ```tensorflow```, ```pytorch``` supports the creation of tensors too:" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "id": "da356762", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([ 1., 5., 9., 15., -24., -13.])\n" + ] + } + ], + "source": [ + "tensor = torch.tensor([1., 5., 9., 15., -24., -13.])\n", + "print(tensor)" + ] + }, + { + "cell_type": "markdown", + "id": "5ab59c06", + "metadata": {}, + "source": [ + "These objects can both **store data and model parameters** (recall that in tensorflow ```tf.Variable``` is a child class of ```tf.Tensor``` and used for storing weights). To check whether a tensor is storing gradients, one can use the ```requires_grad``` attribute:" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "id": "0e83bc87", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n" + ] + } + ], + "source": [ + "print(tensor.requires_grad)" + ] + }, + { + "cell_type": "markdown", + "id": "318a84d9", + "metadata": {}, + "source": [ + "To initialize a **tensor with gradients** one can use the ```requires_grad``` keyword during initialization; this creates the rough equivalent of a ```tf.Variable```. To obtain the gradients, ```.backward()``` has to be called on the output object:" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "id": "2d6a9052", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Input tensor xᵢ: tensor([ 1., 5., 9., 15., -24., -13.], requires_grad=True)\n", + "Gradients of Σᵢ xᵢ²: tensor([ 2., 10., 18., 30., -48., -26.])\n" + ] + } + ], + "source": [ + "# Create a tensor with gradients and print it:\n", + "tensor_grad = torch.tensor([1., 5., 9., 15., -24., -13.], requires_grad=True)\n", + "print(\"Input tensor xᵢ: \", tensor_grad)\n", + "\n", + "# Perform an operation on the tensor itself and sum the output making it a 1D function:\n", + "output = (tensor_grad ** 2).sum() # This defines y = Σᵢ xᵢ² for every xᵢ\n", + "# Evaluating the gradients:\n", + "output.backward()\n", + "# ...and printing 'em:\n", + "print(\"Gradients of Σᵢ xᵢ²: \", tensor_grad.grad)" + ] + }, + { + "cell_type": "markdown", + "id": "546969ea", + "metadata": {}, + "source": [ + "**Conversion** from and to ```numpy``` is also supported in an intuitive way:" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "id": "b15a0f08", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "A tensor created from a numpy array: tensor([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])\n", + "An array created from a torch tensor: [0. 1. 2. 3. 4. 5.]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "tensor_from_np = torch.tensor(np.arange(10)).float()\n", + "print(\"A tensor created from a numpy array: \", tensor_from_np)\n", + "\n", + "tensor_torch = torch.linspace(0, 5, 6).float()\n", + "array_from_torch = tensor_torch.numpy()\n", + "print(\"An array created from a torch tensor:\", array_from_torch)" + ] + }, + { + "cell_type": "markdown", + "id": "2da3a5ce", + "metadata": {}, + "source": [ + "## Fundamental Mathematical Operations\n", + "```pytorch``` supports several basic mathematical operations on tensors too. Its syntax more or less follows that of ```numpy``` for convenience." + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "id": "b164940f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean: tensor(4.5000)\n", + "Std : tensor(3.0277)\n", + "\n", + "Normal Sample Properties:\n", + " Shape: torch.Size([1, 10000])\n", + " Mean: tensor(1.0189)\n", + " Std: tensor(0.4937)\n", + "\n", + "The first row of the normal samples:\n", + "tensor([1.1931, 0.7112, 1.0409, ..., 1.2609, 0.1830, 1.6109])\n" + ] + } + ], + "source": [ + "# A toy tensor:\n", + "tensor = torch.arange(10).float() # Create a tensor from 0 to 9\n", + "print(\"Mean:\", torch.mean(tensor))\n", + "print(\"Std :\", torch.std(tensor))\n", + "\n", + "# Random numbers:\n", + "# A normal sample with mean 1 and std 0.5:\n", + "normal = torch.normal(mean=1., std=0.5, size=[1, 10000])\n", + "print(\"\\nNormal Sample Properties:\")\n", + "print(\" Shape:\", normal.shape)\n", + "print(\" Mean: \", normal.mean())\n", + "print(\" Std: \", normal.std())\n", + "\n", + "# Getting elements along an axis (slicing):\n", + "print(\"\\nThe first row of the normal samples:\")\n", + "print(normal[0,:])" + ] + }, + { + "cell_type": "markdown", + "id": "2046ab07", + "metadata": {}, + "source": [ + " A key **difference in syntax** however is that ```pytorch``` knows the ```axis``` keyword as ```dim```:" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "id": "4d244cd6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Uniform Sample Properties:\n", + " Shape: torch.Size([3, 100000])\n", + " Mean: tensor([0.4991, 0.5005, 0.4996])\n", + " Std: tensor([0.2888, 0.2885, 0.2884])\n" + ] + } + ], + "source": [ + "# A uniform sample from [0, 1)\n", + "uniform = torch.rand([3, 100000])\n", + "print(\"\\nUniform Sample Properties:\")\n", + "print(\" Shape:\", uniform.shape)\n", + "print(\" Mean: \", uniform.mean(dim=1)) # Equals 1/2 ≈ 0.5, the mean of a uniform distribution between [0, 1)\n", + "print(\" Std: \", uniform.std(dim=1)) # Equals 1/12**0.5 ≈ 0.2887, the std of a uniform distribution of width 1" ] } ], @@ -48,6 +249,19 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.4" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": true, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false } }, "nbformat": 4, diff --git a/lecture1/exercise_0_tensorflow.ipynb b/lecture1/exercise_0_tensorflow.ipynb index 88128dd69a0c29eabc4ad1abf889d3470c0dd8d7..55cc5fb465940ed8ae04f35cda45d6feff1715a5 100644 --- a/lecture1/exercise_0_tensorflow.ipynb +++ b/lecture1/exercise_0_tensorflow.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 29, "id": "fb7e3964", "metadata": {}, "outputs": [], @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 30, "id": "e776d989", "metadata": {}, "outputs": [ @@ -49,26 +49,18 @@ "[[1. 1.]\n", " [1. 1.]\n", " [1. 1.]], shape=(3, 2), dtype=float32)\n", - "tf.random_uniform([1, 3]) = tf.Tensor([[0.6821799 0.35027337 0.35916233]], shape=(1, 3), dtype=float32)\n", + "tf.random_uniform([1, 3]) = tf.Tensor(\n", + "[[0.9811375 0.9579929 0.44337857]\n", + " [0.4048884 0.99830174 0.6368449 ]], shape=(2, 3), dtype=float32)\n", "tf.linspace(1.0, 7.0, 4) = tf.Tensor([1. 3. 5. 7.], shape=(4,), dtype=float64)\n", "tf.convert_to_tensor( np.linspace(1, 7, 4) ) = tf.Tensor([1. 3. 5. 7.], shape=(4,), dtype=float64)\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-08-01 14:33:35.064582: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n", - "2022-08-01 14:33:35.067452: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: SSE4.1 SSE4.2 AVX\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2022-08-01 14:33:35.073213: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n" - ] } ], "source": [ "t1 = tf.zeros([5]) # Zeros of length 5 (note the necessary squared brackets)\n", "t2 = tf.ones([3, 2]) # Array of ones of shape (3, 2)\n", - "t3 = tf.random.uniform([1, 3]) # Random sampling from the interval [0, 1).\n", + "t3 = tf.random.uniform([2, 3]) # Random sampling from the interval [0, 1) as a shape (2, 3)\n", "t4 = tf.linspace(1, 7, 4) # Create a tensor of linear spacing from 1 to 7 with 4 entries\n", "t5 = tf.convert_to_tensor(np.linspace(1, 7, 4))\n", "\n", @@ -90,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 31, "id": "0b576f35", "metadata": {}, "outputs": [ @@ -98,7 +90,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "[[0.6821799 0.35027337 0.35916233]]\n", + "[[0.9811375 0.9579929 0.44337857]\n", + " [0.4048884 0.99830174 0.6368449 ]]\n", "<class 'numpy.ndarray'>\n" ] } @@ -114,12 +107,12 @@ "metadata": {}, "source": [ "## Variables\n", - "On the other hand, ```Variables``` (based on tensors) are objects which are **updated during training** through backpropagation. Each tensorflow variable has to be initialized and their values can be changed with using ```variable.assign(new_values)```:" + "On the other hand, ```Variables``` (based on tensors) are objects which are **updated during training** through backpropagation. Each tensorflow variable has to be initialized and their values can be changed using ```variable.assign(new_values)```:" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 32, "id": "a70aa763", "metadata": {}, "outputs": [ @@ -158,7 +151,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 33, "id": "367e4ba0", "metadata": {}, "outputs": [ @@ -172,7 +165,24 @@ "sum(x) tf.Tensor(10.0, shape=(), dtype=float32)\n", "mean(x) tf.Tensor(2.0, shape=(), dtype=float32)\n" ] - }, + } + ], + "source": [ + "x = tf.linspace(0., 4., 5) # Create a tensor array\n", + "\n", + "print(\"x =\", x)\n", + "print(\"(x+1)**2 - 2) =\", (x + 1.)**2 - 2.)\n", + "print(\"sin(x)\", tf.sin(x))\n", + "print(\"sum(x)\", tf.reduce_sum(x))\n", + "print(\"mean(x)\", tf.reduce_mean(x))" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "d8b9f68c", + "metadata": {}, + "outputs": [ { "data": { "text/plain": [ @@ -181,20 +191,12 @@ " [0., 0., 0.]], dtype=float32)>" ] }, - "execution_count": 6, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "x = tf.linspace(0., 4., 5) # Create a tensor array\n", - "\n", - "print(\"x =\", x)\n", - "print(\"(x+1)**2 - 2) =\", (x + 1.)**2 - 2.)\n", - "print(\"sin(x)\", tf.sin(x))\n", - "print(\"sum(x)\", tf.reduce_sum(x))\n", - "print(\"mean(x)\", tf.reduce_mean(x))\n", - "\n", "# Create some other tensors to showcase arithmatic operations:\n", "a = tf.zeros(shape=(2, 3))\n", "b = tf.ones(shape=(2, 3))\n", @@ -206,6 +208,29 @@ "a * b # same as tf.mul(a, b)\n", "a / b # same as tf.division(a, b)" ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "78ba73da", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Means: tf.Tensor([-0.01646124 0.01251054 0.00447996], shape=(3,), dtype=float32)\n", + "Stds: tf.Tensor([1.0073316 0.9962494 1.0039229], shape=(3,), dtype=float32)\n" + ] + } + ], + "source": [ + "# Create a normal distribution with mean 0 and std 1 of shape (3, 10000):\n", + "y = tf.random.normal([3, 10000], mean=0, stddev=1)\n", + "# Evaluate the means and standard deviations along an axis:\n", + "print(\"Means:\", tf.math.reduce_mean(y, axis=1))\n", + "print(\"Stds: \", tf.math.reduce_std(y, axis=1))" + ] } ], "metadata": { @@ -225,6 +250,19 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.12" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": true, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false } }, "nbformat": 4,