
% Default to the notebook output style

    


% Inherit from the specified cell style.




    
\documentclass{beamer}

    
    
    \usepackage{graphicx} % Used to insert images
    \usepackage{adjustbox} % Used to constrain images to a maximum size 
    \usepackage{color} % Allow colors to be defined
    \usepackage{enumerate} % Needed for markdown enumerations to work
    \usepackage{geometry} % Used to adjust the document margins
    \usepackage{amsmath} % Equations
    \usepackage{amssymb} % Equations
    \usepackage[mathletters]{ucs} % Extended unicode (utf-8) support
    \usepackage[utf8x]{inputenc} % Allow utf-8 characters in the tex document
    \usepackage{fancyvrb} % verbatim replacement that allows latex
    \usepackage{grffile} % extends the file name processing of package graphics 
                         % to support a larger range 
    % The hyperref package gives us a pdf with properly built
    % internal navigation ('pdf bookmarks' for the table of contents,
    % internal cross-reference links, web links for URLs, etc.)
    \usepackage{hyperref}
    \usepackage{longtable} % longtable support required by pandoc >1.10
    \usepackage{booktabs}  % table support for pandoc > 1.12.2
    

    
    
    \definecolor{orange}{cmyk}{0,0.4,0.8,0.2}
    \definecolor{darkorange}{rgb}{.71,0.21,0.01}
    \definecolor{darkgreen}{rgb}{.12,.54,.11}
    \definecolor{myteal}{rgb}{.26, .44, .56}
    \definecolor{gray}{gray}{0.45}
    \definecolor{lightgray}{gray}{.95}
    \definecolor{mediumgray}{gray}{.8}
    \definecolor{inputbackground}{rgb}{.95, .95, .85}
    \definecolor{outputbackground}{rgb}{.95, .95, .95}
    \definecolor{traceback}{rgb}{1, .95, .95}
    % ansi colors
    \definecolor{red}{rgb}{.6,0,0}
    \definecolor{green}{rgb}{0,.65,0}
    \definecolor{brown}{rgb}{0.6,0.6,0}
    \definecolor{blue}{rgb}{0,.145,.698}
    \definecolor{purple}{rgb}{.698,.145,.698}
    \definecolor{cyan}{rgb}{0,.698,.698}
    \definecolor{lightgray}{gray}{0.5}
    
    % bright ansi colors
    \definecolor{darkgray}{gray}{0.25}
    \definecolor{lightred}{rgb}{1.0,0.39,0.28}
    \definecolor{lightgreen}{rgb}{0.48,0.99,0.0}
    \definecolor{lightblue}{rgb}{0.53,0.81,0.92}
    \definecolor{lightpurple}{rgb}{0.87,0.63,0.87}
    \definecolor{lightcyan}{rgb}{0.5,1.0,0.83}
    
    % commands and environments needed by pandoc snippets
    % extracted from the output of `pandoc -s`
    \DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
    % Add ',fontsize=\small' for more characters per line
    \newenvironment{Shaded}{}{}
    \newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
    \newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}}
    \newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
    \newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
    \newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
    \newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
    \newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
    \newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}}
    \newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
    \newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
    \newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}}
    \newcommand{\RegionMarkerTok}[1]{{#1}}
    \newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
    \newcommand{\NormalTok}[1]{{#1}}
    
    % Define a nice break command that doesn't care if a line doesn't already
    % exist.
    \def\br{\hspace*{\fill} \\* }
    % Math Jax compatability definitions
    \def\gt{>}
    \def\lt{<}
    % Document parameters    
    
    

    % Pygments definitions
    
% \makeatletter
\def\PY@reset{\let\PY@it=\relax \let\PY@bf=\relax%
    \let\PY@ul=\relax \let\PY@tc=\relax%
    \let\PY@bc=\relax \let\PY@ff=\relax}
\def\PY@tok#1{\csname PY@tok@#1\endcsname}
\def\PY@toks#1+{\ifx\relax#1\empty\else%
    \PY@tok{#1}\expandafter\PY@toks\fi}
\def\PY@do#1{\PY@bc{\PY@tc{\PY@ul{%
    \PY@it{\PY@bf{\PY@ff{#1}}}}}}}
\def\PY#1#2{\PY@reset\PY@toks#1+\relax+\PY@do{#2}}

\expandafter\def\csname PY@tok@gd\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}}
\expandafter\def\csname PY@tok@gu\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}}
\expandafter\def\csname PY@tok@gt\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}}
\expandafter\def\csname PY@tok@gs\endcsname{\let\PY@bf=\textbf}
\expandafter\def\csname PY@tok@gr\endcsname{\def\PY@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
\expandafter\def\csname PY@tok@cm\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@vg\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@m\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@mh\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@go\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
\expandafter\def\csname PY@tok@ge\endcsname{\let\PY@it=\textit}
\expandafter\def\csname PY@tok@vc\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@il\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@cs\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@cp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.74,0.48,0.00}{##1}}}
\expandafter\def\csname PY@tok@gi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}}
\expandafter\def\csname PY@tok@gh\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
\expandafter\def\csname PY@tok@ni\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.60,0.60,0.60}{##1}}}
\expandafter\def\csname PY@tok@nl\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.63,0.63,0.00}{##1}}}
\expandafter\def\csname PY@tok@nn\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
\expandafter\def\csname PY@tok@no\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}}
\expandafter\def\csname PY@tok@na\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.49,0.56,0.16}{##1}}}
\expandafter\def\csname PY@tok@nb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@nc\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
\expandafter\def\csname PY@tok@nd\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}}
\expandafter\def\csname PY@tok@ne\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.82,0.25,0.23}{##1}}}
\expandafter\def\csname PY@tok@nf\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
\expandafter\def\csname PY@tok@si\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}}
\expandafter\def\csname PY@tok@s2\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@vi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@nt\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@nv\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@s1\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@sh\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@sc\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@sx\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@bp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@c1\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@kc\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@c\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@mf\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@err\endcsname{\def\PY@bc##1{\setlength{\fboxsep}{0pt}\fcolorbox[rgb]{1.00,0.00,0.00}{1,1,1}{\strut ##1}}}
\expandafter\def\csname PY@tok@kd\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@ss\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@sr\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}}
\expandafter\def\csname PY@tok@mo\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@kn\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@mi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@gp\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
\expandafter\def\csname PY@tok@o\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@kr\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@s\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@kp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@w\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}}
\expandafter\def\csname PY@tok@kt\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.69,0.00,0.25}{##1}}}
\expandafter\def\csname PY@tok@ow\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}}
\expandafter\def\csname PY@tok@sb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@k\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@se\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.13}{##1}}}
\expandafter\def\csname PY@tok@sd\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}

\def\PYZbs{\char`\\}
\def\PYZus{\char`\_}
\def\PYZob{\char`\{}
\def\PYZcb{\char`\}}
\def\PYZca{\char`\^}
\def\PYZam{\char`\&}
\def\PYZlt{\char`\<}
\def\PYZgt{\char`\>}
\def\PYZsh{\char`\#}
\def\PYZpc{\char`\%}
\def\PYZdl{\char`\$}
\def\PYZhy{\char`\-}
\def\PYZsq{\char`\'}
\def\PYZdq{\char`\"}
\def\PYZti{\char`\~}
% for compatibility with earlier versions
\def\PYZat{@}
\def\PYZlb{[}
\def\PYZrb{]}

\setbeamertemplate{navigation symbols}{}

    % Exact colors from NB
    \definecolor{incolor}{rgb}{0.0, 0.0, 0.5}
    \definecolor{outcolor}{rgb}{0.545, 0.0, 0.0}



    
    % Prevent overflowing lines due to hard-to-break entities
    \sloppy 
    % Setup hyperref package
    \hypersetup{
      breaklinks=true,  % so long urls are correctly broken across lines
      colorlinks=true,
      urlcolor=blue,
      linkcolor=darkorange,
      citecolor=darkgreen,
      }
    % Slightly bigger margins than the latex defaults
    
    % \geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in}
    
    

    \begin{document}
    
    \title{CSC 411 Tutorial: Optimization for Machine
Learning}
    \author{Ladislav Rampasek\footnote{\tiny Based on tutorials and slides by Jake Snell, Kevin Swersky, Shenlong Wang and others}}
    \maketitle
    
\begin{frame}\frametitle{Contents}
\begin{itemize}
\item Overview
\item Gradient Descent
\item Convexity
\end{itemize}
\label{overview-of-optimization}
 \end{frame}
 
 \begin{frame}\frametitle{Overview of Optimization}
\label{overview-of-optimization}
 \end{frame}
 
 \begin{frame}\frametitle{An informal definition of
optimization}\label{an-informal-definition-of-optimization}

Minimize (or maximize) some quantity.

    \end{frame}\begin{frame}\frametitle{Applications}\label{applications}

\begin{itemize}
%\itemsep1pt\parskip0pt\parsep0pt
\item
  Engineering: Minimize fuel consumption of an automobile
\item
  Economics: Maximize returns on an investment
\item
  Supply Chain Logistics: Minimize time taken to fulfill an order
\item
  Life: Maximize happiness
\end{itemize}

\end{frame}
    
\begin{frame}\frametitle{More formally}\label{more-formally}
Goal: find $\theta^* = \text{argmin}_\theta f(\theta)$, (possibly
subject to constraints on $\theta$).

\begin{itemize}
\item
  $\theta \in \mathbb{R}^n$: \emph{optimization variable}
\item
  $f: \mathbb{R}^n \rightarrow \mathbb{R}$: \emph{objective function}
\end{itemize}

    Maximizing $f(\theta)$ is equivalent to minimizing $-f(\theta)$, so
we can treat everything as a minimization problem.
\end{frame}
    
\begin{frame}\frametitle{Optimization is a large area of
research}\label{optimization-is-a-large-area-of-research}

The best method for solving the optimization problem depends on which
assumptions we want to make:

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Is $\theta$ discrete or continuous?
\item
  What form do constraints on $\theta$ take? (if any)
\item
  Is $f$ ``well-behaved''? (linear, differentiable, convex, submodular, etc.)
\end{itemize}

    \end{frame}\begin{frame}\frametitle{Optimization for Machine
Learning}\label{optimization-for-machine-learning}

Often in machine learning we are interested in learning the parameters
$\theta$ of a model.

Goal: minimize some loss function

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  For example, if we have some data $(x, y)$, we may want to maximize
  $P(y | x, \theta)$.
\item
  Equivalently, we can minimize $- \log P(y | x, \theta)$.
\item
  We can also minimize other sorts of loss functions
\end{itemize}

    log can help for numerical reasons

    \end{frame}\begin{frame}\frametitle{Gradient Descent}\label{gradient-descent}

    \end{frame}\begin{frame}\frametitle{Gradient Descent:
Motivation}\label{gradient-descent-motivation}

From calculus, we know that the minimum of $f$ must lie at a point
where $\frac{\partial f(\theta^*)}{\partial \theta} = 0$.

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Sometimes, we can solve this equation analytically for $\theta$.
\item
  Most of the time, we are not so lucky and must resort to iterative
  methods.
\end{itemize}

Review

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Gradient:
  $\nabla_{\theta} f = (\frac{\partial f}{\partial \theta_1}, \frac{\partial f}{\partial \theta_2}, ..., \frac{\partial f}{\partial \theta_k} )$
\end{itemize}

    \end{frame}\begin{frame}\frametitle{Outline of Gradient Descent
Algorithm}\label{outline-of-gradient-descent-algorithm}

Where $\eta$ is the learning rate and $T$ is the number of
iterations:

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Initialize $\theta_0$ randomly
\item
  for $t = 1:T$:

  \begin{itemize}
  \itemsep1pt\parskip0pt\parsep0pt
  \item
    $\delta_t \leftarrow -\eta \nabla_{\theta_{t-1}} f$
  \item
    $\theta_t \leftarrow \theta_{t-1} + \delta_t$
  \end{itemize}
\end{itemize}

The learning rate shouldn't be too big (objective function will blow up)
or too small (will take a long time to converge)

\end{frame}\begin{frame}\frametitle{Gradient Descent
with Line-Search}\label{outline-of-gradient-descent-algorithm}

Where $\eta$ is the learning rate and $T$ is the number of
iterations:

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Initialize $\theta_0$ randomly
\item
  for $t = 1:T$:
  \begin{itemize}
  \itemsep1pt\parskip0pt\parsep0pt
  \item
    {\color{red} Finding a step size $\eta_t$ such that $f(\theta_t
    -\eta_t \nabla_{\theta_{t-1}}) < f(\theta_t)$}
  \item
    $\delta_t \leftarrow -\eta_t \nabla_{\theta_{t-1}} f$
  \item
    $\theta_t \leftarrow \theta_{t-1} + \delta_t$
  \end{itemize}
\end{itemize}

Require a line-search step in each iteration.

    \end{frame}\begin{frame}\frametitle{Gradient Descent with
Momentum}\label{gradient-descent-with-momentum}

We can introduce a momentum coefficient $\alpha \in [0,1)$ so that the
updates have ``memory'':

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Initialize $\theta_0$ randomly
\item
  Initialize $\delta_0$ to the zero vector
\item
  for $t = 1:T$:

  \begin{itemize}
  \itemsep1pt\parskip0pt\parsep0pt
  \item
    $\delta_t \leftarrow -\eta \nabla_{\theta_{t-1}} f+${\color{red} $\alpha \delta_{t-1}$}
  \item
    $\theta_t \leftarrow \theta_{t-1} + \delta_t$
  \end{itemize}
\end{itemize}

Momentum is a nice trick that can help speed up convergence. Generally
we choose $\alpha$ between 0.8 and 0.95, but this is problem dependent

    \end{frame}\begin{frame}\frametitle{Outline of Gradient Descent
Algorithm}\label{outline-of-gradient-descent-algorithm}

Where $\eta$ is the learning rate and $T$ is the number of
iterations:

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Initialize $\theta_0$ randomly
\item
  Do:

  \begin{itemize}
  \itemsep1pt\parskip0pt\parsep0pt
  \item
    $\delta_t \leftarrow -\eta \nabla_{\theta_{t-1}} f$
  \item
    $\theta_t \leftarrow \theta_{t-1} + \delta_t$
  \end{itemize}
\item
  {\color{red} Until convergence}
\end{itemize}

Setting a convergence criteria.

    \end{frame}\begin{frame}\frametitle{Some convergence criteria}\label{some-convergence-criteria}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Change in objective function value is close to zero:
  $| f(\theta_{t+1}) - f(\theta_{t}) | < \epsilon$
\item
  Gradient norm is close to zero: $\| \nabla_{\theta} f \| < \epsilon$
\item
  Validation error starts to increase (this is called \emph{early
  stopping})
\end{itemize}

    \end{frame}\begin{frame}\frametitle{Checkgrad}\label{checkgrad}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  When implementing the gradient computation for machine learning
  models, it's often difficult to know if our implementation of $f$
  and $\nabla f$ is correct.
\item
  We can use finite-differences approximation to the gradient to help:
\end{itemize}

\[\frac{\partial f}{\partial \theta_i} \approx \frac{f((\theta_1, \dots, \theta_i + \epsilon, \dots, \theta_n)) - f((\theta_1, \dots, \theta_i - \epsilon, \dots, \theta_n))}{2 \epsilon}\]

Why don't we always just use the finite differences approximation?

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  slow: we need to recompute $f$ twice for each parameter in our
  model.
\item
  numerical issues
\end{itemize}

    \end{frame}\begin{frame}\frametitle{Stochastic Gradient Descent}\label{sgd}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Any iteration of a gradient descent (or quasi-Newton) method
  requires that we sum over the
  entire dataset to compute the gradient.
\item
  SGD idea: at each iteration, sub-sample a small
amount of data (even just 1 point can work) and
use that to estimate the gradient.
\item
  Each update is noisy, but very fast!
\item
  This is the basis of optimizing ML algorithms with
huge datasets (e.g., recent deep learning).
\item
  Computing gradients using the full dataset is
called batch learning, using subsets of data is
called mini-batch learning.
\end{itemize}

    \end{frame}\begin{frame}\frametitle{Stochastic Gradient Descent}\label{sgd2}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
The reason SGD works is because similar data yields
similar gradients, so if there is enough redundancy in
the data, the noise from subsampling won't be so bad.
\item
SGD is very easy to implement compared to
other methods, but the step sizes need to be
tuned to different problems, whereas batch
learning typically ``just works''.
\item
Tip 1: divide the log-likelihood estimate by the
size of your mini-batches. This makes the
learning rate invariant to mini-batch size.
\item
Tip 2: subsample without replacement so that
you visit each point on each pass through the
dataset (this is known as an epoch).
\end{itemize}

    \end{frame}\begin{frame}\frametitle{Demo}\label{demo}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Logistic regression
\end{itemize}

    \end{frame}\begin{frame}\frametitle{Convexity}\label{convexity}

    \end{frame}\begin{frame}\frametitle{Definition of Convexity}\label{definition-of-convexity}

A function $f$ is \textbf{convex} if for any two points $\theta_1$
and $\theta_2$ and any $t \in [0, 1]$,

\[f(t \theta_1 + (1-t) \theta_2) \le t f(\theta_1) + (1-t) f(\theta_2)\]

We can \emph{compose} convex functions such that the resulting function
is also convex:

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  If $f$ is convex, then so is $\alpha f$ for $\alpha \ge 0$
\item
  If $f_1$ and $f_2$ are both convex, then so is $f_1 + f_2$
\item
  \emph{etc.}, see http://www.ee.ucla.edu/ee236b/lectures/functions.pdf
  for more
\end{itemize}

    \end{frame}\begin{frame}\frametitle{Why do we care about
convexity?}\label{why-do-we-care-about-convexity}

\begin{itemize}
\itemsep1pt\parskip0pt\parsep0pt
\item
  Any local minimum is a global minimum.
\item
  This makes optimization a lot easier because we don't have to worry
  about getting stuck in a local minimum.
\end{itemize}

    \end{frame}
    
 \begin{frame}\frametitle{Examples of Convex
Functions}\label{examples-of-convex-functions}

\textbf{Quadratics}
 
    \begin{center}
    \adjustimage{max size={0.8\linewidth}{0.8\paperheight}}{tutorial_files/quadratic.png}
    \end{center}   
     
\end{frame}
    
\begin{frame}\frametitle{Examples of Convex
Functions}\label{examples-of-convex-functions}

\textbf{Negative logarithms}  
    \begin{center}
    \adjustimage{max size={0.8\linewidth}{0.8\paperheight}}{tutorial_files/log.png}
    \end{center}
\end{frame}

\begin{frame}\frametitle{Convexity for logistic
regression}\label{convexity-for-logistic-regression}

\textbf{Cross-entropy} objective function for logistic regression is
also convex!

$f(\theta) = -\sum_n t^{(n)} \log p(y=1 | x^{(n)}, \theta) + (1 - t^{(n)}) \log p(y=0 | x^{(n)}, \theta)$

    Plot of $-\log \sigma(\theta)$
    \begin{center}
    \adjustimage{max size={0.8\linewidth}{0.8\paperheight}}{tutorial_files/sigmoid.png}
    \end{center}
    \end{frame}
    
    \begin{frame}\frametitle{More on optimization}\label{more-on-optimization}

\emph{Convex Optimization} by Boyd \& Vandenberghe

Book available for free online at
http://www.stanford.edu/\textasciitilde{}boyd/cvxbook/

\emph{Numerical Optimization} by Nocedal \& Wright

Electronic version available from UofT Library

    \end{frame}




    % Add a bibliography block to the postdoc
    
    
    
    \end{document}
