diff --git a/as2_Maggioni_Claudio/download.sh b/as2_Maggioni_Claudio/download.sh new file mode 100755 index 0000000..e144668 --- /dev/null +++ b/as2_Maggioni_Claudio/download.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +cd deliverable +/usr/bin/curl -o models.tar.gz "https://drive.switch.ch/index.php/s/F0ubFgS6PBy8UM5/download" +/usr/bin/tar xzvf models.tar.gz +rm models.tar.gz +cd .. diff --git a/as2_Maggioni_Claudio/report_Maggioni_Claudio.pdf b/as2_Maggioni_Claudio/report_Maggioni_Claudio.pdf index 4159e57..d558382 100644 Binary files a/as2_Maggioni_Claudio/report_Maggioni_Claudio.pdf and b/as2_Maggioni_Claudio/report_Maggioni_Claudio.pdf differ diff --git a/as2_Maggioni_Claudio/report_Maggioni_Claudio.tex b/as2_Maggioni_Claudio/report_Maggioni_Claudio.tex index d801b53..ef3261c 100644 --- a/as2_Maggioni_Claudio/report_Maggioni_Claudio.tex +++ b/as2_Maggioni_Claudio/report_Maggioni_Claudio.tex @@ -86,6 +86,23 @@ \begin{document} +\section*{Note on model download from SWITCHdrive} + +Due to space constraints, all the neural network models in this assigment were saved and uploaded on +SWITCHdrive. To download them, make sure you have \texttt{curl}, \texttt{gzip}, and \texttt{tar} +(all installed by default on MacOS) and then run: + +\begin{verbatim} +./download.sh +\end{verbatim} + +having as current working directory (pwd) the root assigment directory \\ \texttt{as2\_Maggioni\_Claudio}. + +The script will download automatically the models. + +Then, in order to run the models with the \texttt{run\_task*.py}, make sure you first \texttt{cd} in the +\texttt{deliverable} directory. + \maketitle In this assignment you are asked to: @@ -148,11 +165,11 @@ Implement a multi-class classifier to identify the subject of the images from \h The network model was built and trained according to the given specification. -The performance on the given test set is of $0.3649$ loss and $86.4\%$ accuracy. In order to assess performance on new and unseen images +The performance on the given test set is of $0.3879$ loss and $85.63\%$ accuracy. In order to assess performance on new and unseen images a statistical confidence interval is necessary. Since the accuracy is by construction a binomial measure (since an image can either be correctly classified or not, and we repeat this Bernoulli process for each test set datapoint), we perform a binomial distribution confidence interval computation for 95\% confidence. The code use to do this is found in the notebook \texttt{src/Assignment 2.ipynb} under the section \textit{Statistical tests on CIFAR classifier}. -We conclude stating that with 95\% confidence the accuracy for new and unseen images will fall between $\approx 85.12\%$ and $\approx 87.59\%$. +We conclude stating that with 95\% confidence the accuracy for new and unseen images will fall between $\approx 84.29\%$ and $\approx 86.82\%$. The training and validation accuracy curves for the network is shown below: diff --git a/as2_Maggioni_Claudio/src/Assignment 2.ipynb b/as2_Maggioni_Claudio/src/Assignment 2.ipynb index 7f75c2b..8fb838c 100644 --- a/as2_Maggioni_Claudio/src/Assignment 2.ipynb +++ b/as2_Maggioni_Claudio/src/Assignment 2.ipynb @@ -646,7 +646,7 @@ "base_uri": "https://localhost:8080/" }, "id": "0pNBVGIh5NKd", - "outputId": "1fcd7c60-f848-4890-cf6f-56936a352286" + "outputId": "2f973b0b-5d41-4b04-e1ec-e55cd349f732" }, "source": [ "# Compute confidence interval for accuracy using binomial distribution\n", @@ -654,24 +654,24 @@ "import scipy.stats as st\n", "from statsmodels.stats.proportion import proportion_confint \n", "\n", - "test_accuracy = 0.864\n", - "n = len(y_test)\n", + "test_accuracy = 0.856333315372467\n", + "n = 3000\n", "\n", "proportion_confint(test_accuracy * n, n, method='binom_test', alpha=0.05)" ], - "execution_count": 16, + "execution_count": 2, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ - "(0.8512018385349547, 0.8758694331900033)" + "(0.8428686580662224, 0.8682027279962743)" ] }, "metadata": { "tags": [] }, - "execution_count": 16 + "execution_count": 2 } ] }, diff --git a/as2_Maggioni_Claudio/src/Assignment 2.pdf b/as2_Maggioni_Claudio/src/Assignment 2.pdf index 23932da..21e3a68 100644 Binary files a/as2_Maggioni_Claudio/src/Assignment 2.pdf and b/as2_Maggioni_Claudio/src/Assignment 2.pdf differ