%% LyX 1.1 created this file.  For more info, see http://www.lyx.org/.
%% Do not edit unless you really know what you are doing.
\documentclass[10pt,english]{article}
\usepackage[T1]{fontenc}
\usepackage[latin1]{inputenc}
\usepackage{amsmath}
\usepackage{babel}
\usepackage{amssymb}

\makeatletter

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% LyX specific LaTeX commands.
\providecommand{\LyX}{L\kern-.1667em\lower.25em\hbox{Y}\kern-.125emX\@}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% User specified LaTeX commands.
\usepackage{ccfonts}
\usepackage{fullpage}
\usepackage{endnotes}
\renewcommand{\bfseries}{\scshape}
\renewcommand{\footnote}{\endnote}

\makeatother
\begin{document}

\title{CSC165H, Mathematical expression and reasoning for computer science\\
  week 9}

\maketitle
Gary Baumgartner and Danny Heap

heap@cs.toronto.edu

SF4306A

416-978-5899

http://www.cs.toronto.edu/\textasciitilde{}heap/165/S2005/index.shtml


\section*{How large is "sufficiently large?"}
\label{sufficientlyLarge}

Is binary search a better algorithm than linear search?%
                                %
\footnote{Better in the sense of time complexity.}
                                %
It depends on the size of the input.  For example, suppose you
established that linear search has complexity $L(n)=3n$ and binary
search has complexity $B(n)=9 n \log_2 n$.  For the first few $n$,
$L(n)$ is smaller than $B(n)$.  However, certainly for $n>7$, $B(n)$ is
smaller, indicating less ``work'' for binary search.

When we say ``large enough'' $n$, we mean we are discussing the
asymptotic behaviour of the complexity function, and we are prepared
to ignore the behaviour near the origin.

\section*{Making Big-O precise}
\label{preciseBigO}

Here's precise definition of ``The set of functions that, ignoring a
constant, are eventually no more than $f$.''

For any function $f:\mathbb{N}\rightarrow\mathbb{R}^{\geq 0}$, let
\begin{displaymath}
  O(f) = \{g:\mathbb{N}\rightarrow \mathbb{R}^{\geq 0} | \exists c
  \in\mathbb{R}^+, \exists B\in\mathbb{N},\forall n\in\mathbb{N},
  n\geq B\Rightarrow g(n)\leq cf(n)\}.
\end{displaymath}
Saying $g\in O(f)$ says that ``$g$ grows no faster than $f$'' (or
equivalently, ``$f$ is an upper bound for $g$), so long as we modify
our understanding of ``growing no faster'' and being an ``upper
bound'' with the practice of ignoring constant factors.  Now we can
prove some theorems.

Suppose $g(n)=3n^2+2$ and $f(n)=n^2$.  Then $g\in O(f)$.  We need to
prove that $\exists c \in \mathbb{R}^+, \exists B \in\mathbb{N},
\forall n\in \mathbb{N}, n\geq B\Rightarrow 3n^2+2 \leq cn^2$.  It's
enough to find some $c$ and $B$ that ``work'' in order to prove the
theorem.

Finding $c$ means finding a factor that will scale $n^2$ up to the
size of $3n^2+2$.  $c=3$ almost works, but there's that annoying
additional term 2.  Certainly $3n^2+2<4n^2$ so long as $n\geq 2$,
since $n\geq 2\Rightarrow n^2 > 2$.  So pick $c=4$ and $B=2$ (other
values also work, but we like the ones we thought of first).  Now
concoct a proof of
\begin{displaymath}
  \exists c \in \mathbb{R}^+, \exists B \in\mathbb{N},
  \forall n\in \mathbb{N}, n\geq B\Rightarrow 3n^2+2 \leq cn^2.
\end{displaymath}

\noindent Let $c=4$.

\noindent Then $c\in\mathbb{R}^+$.

\begin{quote}
  Let $B=2$.

  Then $B\in\mathbb{N}$.

  \begin{quote}
    Let $n\in\mathbb{N}$.

    Suppose $n\geq B$.

    \begin{quote}
      Then $n^2\geq B^2=4$. (squaring is monotonic on natural
      numbers.)

      So $n^2>2$.

      So $3n^2 +n^2>3n^2+2$. (adding $3n^2$ to both sides of the
      inequality).

      So $3n^2 + 2 \leq 4n^2$.
    \end{quote}
    Thus, $n\geq B\rightarrow 3n^2+2 \leq 4n^2$.

    Since $n$ is an arbitrary natural number, $\forall n\in\mathbb{N},
    n\geq B\rightarrow 3n^2+2\leq 4n^2$.
  \end{quote}
  Since $B$ is a natural number, $\exists B \in\mathbb{N}, \forall
  n\in \mathbb{N}, n\geq B\rightarrow 3n^2+2 \leq cn^2$.
\end{quote}

\noindent Since $c$ is a positive real number, $\exists c \in
\mathbb{R}^+, \exists B \in\mathbb{N}, 
\forall n\in \mathbb{N}, n\geq B\rightarrow 3n^2+2 \leq
cn^2$.\vspace{2\baselineskip}

So, by definition, $g\in O(f)$.  Now suppose that $g(n)=n^4$ and
$f(n)=3n^2$.  Is $g\in O(f)$?  No.  We can see intuitively that any
constant that we multiply times $3n^2$ will be overwhelmed by the
extra factor of $n^2$ in $g(n)$.  But to show this clearly, we negate
the definition and then prove the negation:

\begin{displaymath}
  \forall c\in \mathbb{R}^+, \forall B\in\mathbb{N}, \exists
  n\in\mathbb{N}, n\geq B\wedge n^4 > c3n^2.
\end{displaymath}

The parameter we have some control over is $n$, and we need to pick it
so that $n\geq B$ and $n^4>c3n^2$.  Solve for $n$:
\begin{eqnarray*}
  n^4 &>& c3n^2 \\
  \Rightarrow n^4/n^2 &>& c3n^2/n^2 \\
  \Rightarrow n^2 &>& 3c \\
  \Rightarrow n &>& \sqrt{3c}.
\end{eqnarray*}

To satisfy the conditions, set $n=B+\lceil \sqrt{3c} \rceil+1$.  Since
$\sqrt{3c}$ is not necessarily a natural number, we take its
ceiling.  Now we can generate the proof.

Let $c\in\mathbb{R}^+$.  Let $B\in\mathbb{N}$.

\begin{quote}
  Let $n=B+\lceil \sqrt{3c} \rceil + 1$.

  \begin{quote}
    Then $n\in \mathbb{N}$. (since $B\in\mathbb{N}$, $1\in\mathbb{N}$,
    and $\lceil \sqrt{3c} \rceil\in \mathbb{N}$ (since $c>0$) and
    $\mathbb{N}$ is closed under sums).

    So $n\geq B$ (since it is the sum of $B$ and two other
    non-negative numbers).

    So $n\geq \sqrt{3c}+1$. (since $B\geq 0$)

    So $n^2>(\lceil \sqrt{3c}\rceil + 1)^2$.

    So $n^2>3c$. (ignoring some positive terms).

    So $n^4 > 3cn^2$.
  \end{quote}

  Since $n$ is a natural number, $\exists n\in\mathbb{N}, n\geq
  B\wedge n^4 > c3n^2$.
\end{quote}

Since $c$ is an arbitrary element of $\mathbb{R}^+$ and $B$ is an
arbitrary element of $\mathbb{N}$, $\forall c\in \mathbb{R}^+, \forall
B\in\mathbb{N}, \exists n\in\mathbb{N}, n\geq B\wedge n^4 > c3n^2$.

By definition, this means that $g\not\in O(f)$.

\section*{Other bounds}
\label{otherBounds}

In analogy with $O(f)$, consider two other definitions:
\begin{eqnarray*}
  \Omega(f) &=& \{f:\mathbb{N}\rightarrow \mathbb{R}^{\geq 0}|\exists
  c \in \mathbb{R}^+, \exists B\in\mathbb{N}, \forall n\in\mathbb{N},
  n \geq B\rightarrow g(n) \geq cf(n)\}.
\end{eqnarray*}

To say $g\in\Omega(f)$ expresses the concept that $g$ grows at least as
fast as $f$.'' ($f$ is a lower bound on $g$).

\begin{eqnarray*}
  \Theta(f) &=& \{g:\mathbb{N}\rightarrow\mathbb{R}^{\geq 0}|\exists
  c_1\in\mathbb{R}^+,\exists c_2\in\mathbb{R}^+,\exists
  B\in\mathbb{N}, \forall n\in\mathbb{N}, n\geq B\rightarrow c_1 f(n)
  \leq g(n) \leq c_2 f(n)\}.
\end{eqnarray*}

To say ``$g\in\Theta(f)$'' expresses the concept that ``$g$ grows at
the same rate as $f$.'' ($f$ is a tight bound for $g$).

\section*{Some theorems}
\label{someTheorems}

Here are some general results that we now have the tools to prove.

\begin{itemize}
\item $f\in O(f)$.

\item $(f\in O(g)\wedge g\in O(h))\Rightarrow f\in O(h)$.

\item $g\in\Omega(f)\Leftrightarrow f\in O(g)$.

\item $g\in \Theta(f)\Leftrightarrow g\in O(f)\wedge g\in\Omega(f)$.
\end{itemize}

Test your intuition about Big-O by doing the ``scratch work'' to
answer the following questions:

\begin{itemize}
\item Are there functions $f,g$ such that $f\in O(g)$ and $g\in O(f)$
  but $f\neq g$?%
                                %
  \footnote{Sure, $f=n^2$, $g=3n^2+2$.}
                                %

\item Are there functions such that $f\not\in O(f)$, and $g\not\in
  O(g)$?%
                                %
  \footnote{Sure.  $f$ and $g$ don't need to both be monotonic, so let
    $f(n)=n^2$ and
    \begin{displaymath}
      g(n) = 
      \begin{cases}
        n, & n \text{ even} \\
        n^3, & n \text{ odd}
      \end{cases}
    \end{displaymath}

    So not every pair of functions from $\mathbb{N}\rightarrow
    \mathbb{R}^{\geq 0}$ can be compared using Big-O.}
                                %
\end{itemize}

To show that $(g\in O(f)\wedge g\in O(h))\Rightarrow f\in O(h)$, we
need to find a constant $c\in\mathbb{R}^+$ and a constant
$B\in\mathbb{N}$, that satisfy:
\begin{displaymath}
  \forall n\in\mathbb{N}, n\geq B\Rightarrow f(n) \leq c h(n).
\end{displaymath}
Since we have constants that scale $h$ to $g$ and then $g$ to
$f$, it seems clear that we need their product to scale $g$ to $f$.
And if we take the maximum of the two starting points, we can't go
wrong.  Making this precise.

Assume $f\in O(g)\wedge g\in O(f)$.

\begin{quote}
  So $f\in O(g)$.

  So $g\in O(f)$.

  So $\exists c \in \mathbb{R}^+, \exists B\in \mathbb{N}, \forall
  n\in \mathbb{N}, n>B\Rightarrow f(n) \leq c g(n)$. (by defn. of
  $f\in O(g)$).

  Let $c_g\in\mathbb{R}^+,B_g\in\mathbb{N}$ be such that $\forall
  n\in\mathbb{N}, n\geq B\Rightarrow f(n) \leq c_g g(n)$.

  So $\exists c \in \mathbb{R}^+, \exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow g(n)\leq ch(n)$. (by defn. of
  $g\in O(h)$).

  Let $c_h\in\mathbb{R}^+, B_h\in \mathbb{N}$ be such that $\forall
  n\in\mathbb{N}, n\geq B_h \Rightarrow g(n)\leq c_h h(n)$.

  Let $c=c_g c_h$.  Let $B=\max(B_g, B_h)$.
  \begin{quote}
    Let $n\in\mathbb{N}$.

    Suppose $n\geq B$.
    \begin{quote}
      Then $n\geq B_h$ (definition of $\max$), so $g(n) \leq c_h h(n)$.
      
      Then $n\geq B_g$ (definition of $\max$), so $f(n) \leq c_g
      g(n)\leq c_g c_h h(n)$.

      So $f(n) \leq c h(n)$.
    \end{quote}
    So $n\geq B\Rightarrow f(n)\leq c h(n)$.

    Since $n$ is an arbitrary natural number, $\forall
    n\in\mathbb{N}, n\geq B\Rightarrow f(n)\leq c h(n)$.

  \end{quote}

  Since $c$ is a positive real number, since $B$ is a natural number,
  $\exists c\in\mathbb{R}^+,\exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow f(n) \leq c h(n)$.

  So $f\in O(g)$, by definition.
\end{quote}
So $(f\in O(g)\wedge g\in O(h))\Rightarrow f\in O(g)$

\newpage

\begingroup

\setlength{\parindent}{0pt}

\setlength{\parskip}{2ex}

\renewcommand{\enotesize}{\normalsize}

\theendnotes\endgroup
\end{document}