Commit 51979ea6 authored by Azat Garifullin's avatar Azat Garifullin
Browse files

unsure slides

parents
%% $Id$
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Ylätunnisteet
\newcommand{\lutkansi}[1][\ ]{
\thispagestyle{empty}
\setlength{\unitlength}{1mm}
\begin{picture}(170,20)(0,-10)
\parbox[b][20mm][t]{170mm}{%
\parbox[b][15mm][t]{100mm}{%
\usefont{OT1}{phv}{m}{n} \large LAPPEENRANTA \\
\usefont{OT1}{phv}{m}{n} \large UNIVERSITY OF TECHNOLOGY
}
\parbox[b][15mm][t]{70mm}{%
\usefont{OT1}{phv}{m}{n} #1 \hfill \arabic{page}
} \\
\parbox[b][5mm][b]{100mm}{%
\usefont{OT1}{phv}{m}{n} \normalsize LUT Machine Vision and Pattern
Recognition
}
\parbox[b][5mm][b]{70mm}{%
\usefont{OT1}{phv}{m}{n}
\number\year-\nplpadding{2}\numprint{\number\month}-\nplpadding{2}\numprint{\number\day}
}
}
\end{picture} \\
\begin{picture}(170,15)(0,210)
\parbox[t][15mm][b]{170mm}{%
\usefont{OT1}{phv}{m}{n} \footnotesize
PO BOX 20, FI-53851 LAPPEENRANTA, FINLAND \\
%Puhelin: 040 759 1720 \\
%Telefax: 042 759 1720 \\
E-mail: lasse.lensu@lut.fi
}
\end{picture}
\vspace*{-7em}
}
\newcommand{\lutlogokansi}[1][\ ]{
\thispagestyle{empty}
\setlength{\unitlength}{1mm}
\begin{picture}(170,30)(0,-10)
\parbox[b][32mm][t]{170mm}{%
\parbox[b][25mm][t]{100mm}{%
\resizebox{!}{25mm}{\includegraphics{resources/LUT_logo_new_cmyk.eps}}
}
\parbox[b][25mm][t]{70mm}{%
\usefont{OT1}{phv}{m}{n} #1 \hfill \arabic{page}
} \\
\parbox[b][5mm][b]{100mm}{%
\usefont{OT1}{phv}{m}{n} LUT Machine Vision and Pattern Recognition
}
\parbox[b][5mm][b]{70mm}{%
\usefont{OT1}{phv}{m}{n} \number\year-\nplpadding{2}\numprint{\number\month}-\nplpadding{2}\numprint{\number\day}
}
}
\end{picture} \\
\begin{picture}(170,15)(0,200)
\parbox[t][15mm][b]{170mm}{%
\usefont{OT1}{phv}{m}{n} \footnotesize
PO BOX 20, FI-53851 LAPPEENRANTA, FINLAND \\
%Puhelin: 040 759 1720 \\
%Telefax: 042 759 1720 \\
E-mail: lasse.lensu@lut.fi \\
}
\end{picture}
\vspace*{-7em}
}
%% Otsikon uudelleenmäärittely
\newcommand{\lutotsikko}{%
\begin{center}%
{\large \@title \par}%
{\begin{tabular}[t]{c}%
\@author%\ \number\day.\number\month.\number\year
\end{tabular}\par}%
\end{center}%
\par}
%% Allekirjoitus
\newcommand{\lutallekir}[1][\ ]{
\parbox{170mm}{%
\bigskip
Lappeenranta \number\year-\nplpadding{2}\numprint{\number\month}-\nplpadding{2}\numprint{\number\day} \\
\medskip \\
\rule{50mm}{0.25mm} \\
\@author \\
#1
}
}
%% Sivujen ala- ja ylätunnisteet
\def\ps@myheadings{
\let\@oddfoot\@empty
\let\@evenfoot\@empty
\def\@evenhead{\@title \hfill \thepage}
\def\@oddhead{\@title \hfill \thepage}
\let\@mkboth\@gobbletwo
\let\sectionmark\@gobble
\let\subsectionmark\@gobble
}
@inproceedings{cheng2019bayesian,
title = {A Bayesian Perspective on the Deep Image Prior},
author = {Cheng, Zezhou and Gadelha, Matheus and Maji, Subhransu and Sheldon, Daniel},
booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
pages = {5443--5451},
year = {2019}
}
@inproceedings{ulyanov2018deep,
title = {Deep image prior},
author = {Ulyanov, Dmitry and Vedaldi, Andrea and Lempitsky, Victor},
booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
pages = {9446--9454},
year = {2018}
}
@article{badrinarayanan2017segnet,
title = {Segnet: A deep convolutional encoder-decoder architecture for image segmentation},
author = {Badrinarayanan, Vijay and Kendall, Alex and Cipolla, Roberto},
journal = {IEEE transactions on pattern analysis and machine intelligence},
volume = {39},
number = {12},
pages = {2481--2495},
year = {2017},
publisher = {IEEE}
}
@article{zhang2016understanding,
title = {Understanding deep learning requires rethinking generalization},
author = {Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
journal = {arXiv preprint arXiv:1611.03530},
year = {2016}
}
@article{cybenko1989approximation,
title = {Approximation by superpositions of a sigmoidal function},
author = {Cybenko, George},
journal = {Mathematics of control, signals and systems},
volume = {2},
number = {4},
pages = {303--314},
year = {1989},
publisher = {Springer}
}
@article{wann,
author = {Gaier, Adam and Ha, David},
year = {2019},
month = {06},
pages = {},
title = {Weight Agnostic Neural Networks},
journal = {To appear at NeurIPS 2019, selected for a spotlight presentation}
}
@phdthesis{neal1995bayesian,
title = {BAYESIAN LEARNING FOR NEURAL NETWORKS},
author = {Neal, Radford M},
year = {1995},
school = {Citeseer}
}
@inproceedings{gal2015dropout,
title = {Dropout as a Bayesian approximation: Insights and applications},
author = {Gal, Yarin and Ghahramani, Zoubin},
year = {2015},
booktitle = {Deep Learning Workshop, ICML}
}
@inproceedings{johnson2016perceptual,
title = {Perceptual losses for real-time style transfer and super-resolution},
author = {Johnson, Justin and Alahi, Alexandre and Fei-Fei, Li},
booktitle = {European conference on computer vision},
pages = {694--711},
year = {2016},
organization = {Springer}
}
@inproceedings{welling2011bayesian,
title = {Bayesian learning via stochastic gradient Langevin dynamics},
author = {Welling, Max and Teh, Yee W},
booktitle = {Proceedings of the 28th international conference on machine learning (ICML-11)},
pages = {681--688},
year = {2011}
}
@inproceedings{ma2015complete,
title = {A complete recipe for stochastic gradient MCMC},
author = {Ma, Yi-An and Chen, Tianqi and Fox, Emily},
booktitle = {Advances in Neural Information Processing Systems},
pages = {2917--2925},
year = {2015}
}
@inproceedings{unsure_dip,
author = {Laves, Max-Heinrich
and T{\"o}lle, Malte
and Ortmaier, Tobias},
title = {Uncertainty Estimation in Medical Image Denoising with Bayesian Deep Image Prior},
booktitle = {Uncertainty for Safe Utilization of Machine Learning in Medical Imaging, and Graphs in Biomedical Image Analysis},
year = {2020},
publisher = {Springer International Publishing},
address = {Cham},
pages = {81--96},
abstract = {Uncertainty quantification in inverse medical imaging tasks with deep learning has received little attention. However, deep models trained on large data sets tend to hallucinate and create artifacts in the reconstructed output that are not anatomically present. We use a randomly initialized convolutional network as parameterization of the reconstructed image and perform gradient descent to match the observation, which is known as deep image prior. In this case, the reconstruction does not suffer from hallucinations as no prior training is performed. We extend this to a Bayesian approach with Monte Carlo dropout to quantify both aleatoric and epistemic uncertainty. The presented method is evaluated on the task of denoising different medical imaging modalities. The experimental results show that our approach yields well-calibrated uncertainty. That is, the predictive uncertainty correlates with the predictive error. This allows for reliable uncertainty estimates and can tackle the problem of hallucinations and artifacts in inverse medical imaging tasks.},
isbn = {978-3-030-60365-6}
}
@article{srivastava14a,
author = {Nitish Srivastava and Geoffrey Hinton and Alex Krizhevsky and Ilya Sutskever and Ruslan Salakhutdinov},
title = {Dropout: A Simple Way to Prevent Neural Networks from Overfitting},
journal = {Journal of Machine Learning Research},
year = {2014},
volume = {15},
number = {56},
pages = {1929-1958},
url = {http://jmlr.org/papers/v15/srivastava14a.html}
}
@article{farquhar2020liberty,
title = {Liberty or depth: Deep Bayesian neural nets do not need complex weight posterior approximations},
author = {Farquhar, Sebastian and Smith, Lewis and Gal, Yarin},
journal = {Advances in Neural Information Processing Systems},
volume = {33},
year = {2020}
}
@inproceedings{guo2017calibration,
title = {On Calibration of Modern Neural Networks},
author = {Guo, Chuan and Pleiss, Geoff and Sun, Yu and Weinberger, Kilian Q},
booktitle = {International Conference on Machine Learning},
pages = {1321--1330},
year = {2017}
}
@article{thulasidasan2019mixup,
title = {On mixup training: Improved calibration and predictive uncertainty for deep neural networks},
author = {Thulasidasan, Sunil and Chennupati, Gopinath and Bilmes, Jeff A and Bhattacharya, Tanmoy and Michalak, Sarah},
journal = {Advances in Neural Information Processing Systems},
volume = {32},
pages = {13888--13899},
year = {2019}
}
@inproceedings{seo2019learning,
title = {Learning for single-shot confidence calibration in deep neural networks through stochastic inferences},
author = {Seo, Seonguk and Seo, Paul Hongsuck and Han, Bohyung},
booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
pages = {9030--9038},
year = {2019}
}
@inproceedings{unsure_calib,
author = {Thiagarajan, Jayaraman J.
and Venkatesh, Bindya
and Rajan, Deepta
and Sattigeri, Prasanna},
title = {Improving Reliability of Clinical Models Using Prediction Calibration},
booktitle = {Uncertainty for Safe Utilization of Machine Learning in Medical Imaging, and Graphs in Biomedical Image Analysis},
year = {2020},
publisher = {Springer International Publishing},
address = {Cham},
pages = {71--80},
abstract = {The wide-spread adoption of representation learning technologies in clinical decision making strongly emphasizes the need for characterizing model reliability and enabling rigorous introspection of model behavior. In supervised and semi-supervised learning, prediction calibration has emerged as a key technique to achieve improved generalization and to promote trust in learned models. In this paper, we investigate the effectiveness of different prediction calibration techniques in improving the reliability of clinical models. First, we introduce reliability plots, which measures the trade-off between model autonomy and generalization, to quantify model reliability. Second, we propose to utilize an interval calibration objective in lieu of the standard cross entropy loss to build classification models. Finally, using a lesion classification problem with dermoscopy images, we evaluate the proposed prediction calibration approach against both uncalibrated models as well as existing prediction calibration techniques such as mixup and single-shot calibration.},
isbn = {978-3-030-60365-6}
}
@inproceedings{unsure_measure,
author = {Camarasa, Robin
and Bos, Daniel
and Hendrikse, Jeroen
and Nederkoorn, Paul
and Kooi, Eline
and van der Lugt, Aad
and de Bruijne, Marleen},
title = {Quantitative Comparison of Monte-Carlo Dropout Uncertainty Measures for Multi-class Segmentation},
booktitle = {Uncertainty for Safe Utilization of Machine Learning in Medical Imaging, and Graphs in Biomedical Image Analysis},
year = {2020},
publisher = {Springer International Publishing},
address = {Cham},
pages = {32--41},
abstract = {Over the past decade, deep learning has become the gold standard for automatic medical image segmentation. Every segmentation task has an underlying uncertainty due to image resolution, annotation protocol, etc. Therefore, a number of methods and metrics have been proposed to quantify the uncertainty of neural networks mostly based on Bayesian deep learning, ensemble learning methods or output probability calibration. The aim of our research is to assess how reliable the different uncertainty metrics found in the literature are. We propose a quantitative and statistical comparison of uncertainty measures based on the relevance of the uncertainty map to predict misclassification. Four uncertainty metrics were compared over a set of 144 models. The application studied is the segmentation of the lumen and vessel wall of carotid arteries based on multiple sequences of magnetic resonance (MR) images in multi-center data.},
isbn = {978-3-030-60365-6}
}
\documentclass[aspectratio=169]{beamer}
\usepackage[export]{adjustbox}
\usepackage{float}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{mathtools}
\usepackage{graphicx}
\usepackage[justification=centering]{caption}
\usepackage{subfig}
\usepackage{graphicx}
\usepackage{multirow}
\usepackage{hyperref}
\usepackage{bbm}
\usepackage{bibentry}
\DeclarePairedDelimiter\abs{\lvert}{\rvert}
\newcommand\given[1][]{\:#1\vert\:}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\DeclarePairedDelimiterX{\inp}[2]{\langle}{\rangle}{#1, #2}
\DeclarePairedDelimiterX{\infdivx}[2]{(}{)}{%
#1\;\delimsize\|\;#2%
}
\newcommand{\infdiv}{\infdivx}
\makeatletter
\let\@@magyar@captionfix\relax
\makeatother
\useoutertheme{miniframes}
\useinnertheme{rectangles}
\definecolor{lutpink}{rgb}{0.929,0.090,0.302}
\definecolor{lutorange}{rgb}{1,0.40,0}
\definecolor{lutgreen}{rgb}{0.251,0.835,0.125}
\setbeamercolor{author}{fg=white}
\setbeamercolor{background canvas}{bg=white}
\setbeamercolor{date}{fg=white}
\setbeamercolor{frametitle}{fg=white}
\setbeamercolor{institute}{fg=white}
\setbeamercolor{itemize item}{fg=black}
\setbeamercolor{normal text}{bg=black,fg=black}
\setbeamercolor{structure}{bg=black, fg=lutpink}
\setbeamercolor{title}{fg=white}
\addtobeamertemplate{frametitle}{}{%
\vspace{-30pt}\hspace{342pt}\includegraphics[height=1cm]{LUT-logo.png}}
\addtobeamertemplate{navigation symbols}{}{%
\usebeamerfont{footline}%
\usebeamercolor[fg]{footline}%
\hspace{1em}%
\insertframenumber/\inserttotalframenumber
}
\title{Bayesian Deep Learning and applications for medical image analysis}
\date{12/01/2021}
\author{Azat Garifullin}
\titlegraphic{
\includegraphics[width=5cm]{LUT-logo.png}%
}
\begin{document}
\nobibliography*
\setbeamercolor{background canvas}{bg=black}
\begin{frame}
\titlepage
\end{frame}
\setbeamercolor{background canvas}{bg=white}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Introduction}
I visited two conferences:
\begin{itemize}
\item Uncertainty for Safe Utilization of Machine Learning in Medical Imaging
\item Bayesian NeurIPS workshop
\end{itemize}
Outline:
\begin{itemize}
\item Neural networks are function space priors
\item Image denoising
\item Calibration and reliability
\item Quantify and interpret uncertainties
\end{itemize}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Deep Priors}
History:
\begin{itemize}
\item \cite{neal1995bayesian} first connections between neural nets and GPs
\item \cite{ulyanov2018deep} used neural nets as image prior solving basic inverse problems
\item \cite{cheng2019bayesian} used them in Bayesian context
\item \cite{unsure_dip} provided additional experiments in medical image denoising
\end{itemize}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Deep priors sampling}
\begin{itemize}
\item input -- image, output -- image
\item Encoder yields compressed hidden representation
\item Decoder yields decoded representation
\item Sampling: \(\mathbf{y} = f(\mathbf{z}, \theta), \ \mathbf{z} \sim \mathcal{U}\)
\end{itemize}
\begin{figure}[h] \centering
\includegraphics[width=\linewidth]{figs/hourglass.png}
\caption{
The architecture used in the experiments \cite{ulyanov2018deep}.
}
\end{figure}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Deep priors sampling}
\begin{columns}
\begin{column}{0.5\textwidth}
\begin{figure}[h] \centering
\includegraphics[width=.9\linewidth]{figs/convnet_1d_samples.png}
\caption{ Priors and posterior with 1D convolutional networks \cite{cheng2019bayesian}. }
\end{figure}
\end{column}
\begin{column}{0.5\textwidth}
\begin{figure}[h] \centering
\includegraphics[width=\linewidth]{figs/2d_samples.png}
\caption{ Samples from GPs and deep priors \cite{cheng2019bayesian}. }
\end{figure}
\end{column}
\end{columns}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Denoising with deep neural nets}
\begin{itemize}
\item One can solve the problem using supervised learning
\(\mathcal{L}(\mathcal{D} \given \theta) \rightarrow \max, \
\mathcal{D} = \lbrace \mathbf{y}_i, \mathbf{x}_i \rbrace\)
\item This way we would learn feature extractors from the whole dataset
\item The problem of hallucinations arises
\item We can use neural nets just as a prior \cite{unsure_dip}
\end{itemize}
\begin{figure}[h] \centering
\includegraphics[width=.6\linewidth]{figs/hallucinations_oct.png}
\caption{ Hallucinations in the reconstructed OCT scan \cite{unsure_dip}. }
\end{figure}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Denoising with deep priors \cite{unsure_dip}}
\begin{itemize}
\item Almost like we would do with GPs
\item Observed image: \( \tilde{\mathbf{y}} = \mathbf{y} + \epsilon \)
\item Restored image: \( \hat{\mathbf{y}} = f\left( \mathbf{z}, \pmb{\theta} \right) \)
\item Where \(f\) is a neural net with parameters \(\pmb{\theta}\)
\end{itemize}
\begin{gather}
\tilde{\mathbf{y}} \given \hat{\mathbf{y}} \sim \mathcal{N}(0, \Sigma_{\epsilon}^2)
\\
\hat{\mathbf{y}}, \Sigma_{\epsilon} \sim f\left( \mathbf{z}, \pmb{\theta} \right)
\\
\mathbf{z} \sim \mathcal{U}(0, 0.1)
\\
\pmb{\theta} \sim \prod_{i} \mathcal{N}(0, \lambda^{-1})
\end{gather}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Uncertainties considered}
\begin{itemize}
\item Heteroscedastic aleatoric (data-dependent) is predicted by the neural network
\begin{gather}
\tilde{\mathbf{y}} = \mathbf{y} + \epsilon, \\
\hat{\mathbf{y}}, \Sigma_{\epsilon} \sim f\left( \mathbf{z}, \pmb{\theta} \right)
\end{gather}
\item Aleatoric uncertainty acts like a learned loss attenuation
\item Epistemic (model's ignorance):
\( p\left( \pmb{\theta} \given \tilde{\mathbf{y}} \right) \propto
p\left( \tilde{\mathbf{y}} \given \pmb{\theta} \right) p\left(\pmb{\theta}\right) \)
\item Denoising problems then becomes a problem of evaluating the posterior predictive
distribution of the restored image given the noisy observation:
\begin{equation}
p\left(\hat{\mathbf{y}} \given \tilde{\mathbf{y}}\right) =
\int
p\left(\hat{\mathbf{y}} \given \tilde{\mathbf{y}}, \pmb{\theta} \right)
p\left( \pmb{\theta} \given \tilde{\mathbf{y}} \right)
d \pmb{\theta}
\end{equation}
\end{itemize}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Sampling techniques: Stochastic Gradient Langevin Dynamics}
\begin{itemize}
\item We could use any MCMC scheme but \(\pmb{\theta}\) is very high-dimensional
\item It might have hudreds of thousands dimensions
\item Scalability issues with common techniques
\item \cite{welling2011bayesian}
Stochastic gradient Langevin dynamics -- scalable extension of SGD
for sampling:
\begin{gather}
\pmb{\theta}_{t + 1} = \pmb{\theta}_t - \Delta_{\pmb{\theta}}
\\
\Delta_{\pmb{\theta}} =
\frac{\epsilon}{2} \left(
\nabla_{\pmb{\theta}}
\log p \left( \hat{\mathbf{y}} \given \pmb{\theta} \right)
+ \nabla_{\pmb{\theta}}
\log p \left( \pmb{\theta} \right)
\right) +
\eta,
\\
\eta \sim \mathcal{N}(0, \epsilon),
\\
\sum \epsilon_i = \infty,
\quad
\sum \epsilon_i^2 < \infty
\end{gather}
\end{itemize}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Sampling techniques: Monte-Carlo Dropout}
TL;DR use dropout at test time
\begin{itemize}
\item Dropout as a regularization technique \cite{srivastava14a}:
\begin{equation}
\pmb{\phi}_{t} = \mathbf{M} \pmb{\theta}_{t},
\
\mathbf{M} = \mathrm{diag}(\mathbf{m}_i),
\
\mathbf{m}_i \sim \mathrm{Bernoulli}(p_{\mathrm{dropout}})
\end{equation}
\item Dropout as stochastic variational inference technique \cite{gal2015dropout}
\begin{gather}
D_{KL} \infdiv{ q\left(\pmb{\phi}\right) }{ p\left( \pmb{\phi} \given \tilde{\mathbf{y}} \right) } \rightarrow \min,
\\
p\left(\hat{\mathbf{y}} \given \tilde{\mathbf{y}}\right) =
\int
p\left(\hat{\mathbf{y}} \given \tilde{\mathbf{y}}, \pmb{\phi} \right)
q\left( \pmb{\phi} \right)
d \pmb{\phi}
\end{gather}
\item Mean-field approximation:
\begin{equation}
q\left(\pmb{\phi}\right) = \prod_{i} q \left( \pmb{\phi}_i \right)
\end{equation}
\end{itemize}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Monte-Carlo Dropout: Mean-field approximation}
\begin{itemize}
\item Extremely widely used
\item \cite{farquhar2020liberty} Deep Bayesian neural nets do not need complex weight posterior approximations:
\begin{quotation}
\dots deeper models with mean-field approximate posteriors can act like shallower
models with much richer approximate posteriors\dots
Moreover, we have proved that neural networks with at least two layers of hidden units and mean-field
weight distributions can approximate any posterior distribution over predictive functions.
In sufficiently deep models, the performance gap between mean-field and structured-covariance approximate
posteriors becomes small or non-existent, suggesting that modes of the true posterior in large settings
may be approximately mean-field.
\end{quotation}
\end{itemize}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Back to denoising: experiments}
\begin{itemize}
\item DIP -- baseline with point estimates
\item SGLD -- Bayesian baseline (Adam)
\item SGLD + NLL -- Bayesian baseline with aleatoric uncertainty (Adam)
\item MCDIP -- DIP with MC-Dropout and aleatoric uncertainty
\end{itemize}
\begin{figure}[h] \centering
\includegraphics[width=.8\linewidth]{figs/psnrs_cmp.png}
\caption{Peak signal-to-noise ratios for three modalities \cite{unsure_dip}.}
\end{figure}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Denoising: Experimental results}
\begin{figure}[h] \centering
\includegraphics[width=.6\linewidth]{figs/noise_overfit.png}
\caption{Denoised images after convergence \cite{unsure_dip}. }
\end{figure}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Denoising: Calibration and uncertainty}
\begin{itemize}
\item Calibration: our inferred statistics must be meaningfull
\begin{equation}
\mathbb{E}_{\hat{\sigma}^2}
\left[
\abs{\left(
\norm{\tilde{\mathbf{y}} - \hat{\mathbf{y}}}
\given
\hat{\sigma}^2 = \sigma^2
\right)
- \sigma^2}
\right]
\end{equation}
\item Uncertainty quantification: convergence vs. early stopping
\item Interesting that they also had an effect of degenerate uncertainties
\end{itemize}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Denoising: full convergence}
\begin{figure}[h] \centering
\includegraphics[width=.6\linewidth]{figs/unc_converged.png}
\caption{Calibration plots and uncertainty maps after convergence \cite{unsure_dip}.}
\end{figure}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Denoising: early stopping}
\begin{figure}[h] \centering
\includegraphics[width=.6\linewidth]{figs/unc_eearly_stop.png}
\caption{Calibration plots and uncertainty maps after early stopping \cite{unsure_dip}.}
\end{figure}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}[plain]{Model calibration and reliability}
\begin{itemize}
\item Just predicting uncertainties is not enough