%% LyX 1.1 created this file.  For more info, see http://www.lyx.org/.
%% Do not edit unless you really know what you are doing.
\documentclass[12pt,english]{article}
\usepackage[T1]{fontenc}
\usepackage[latin1]{inputenc}
\usepackage{amsmath}
\usepackage{babel}
\usepackage{amssymb}

\makeatletter

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% LyX specific LaTeX commands.
\providecommand{\LyX}{L\kern-.1667em\lower.25em\hbox{Y}\kern-.125emX\@}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% User specified LaTeX commands.
\usepackage{ccfonts}
\usepackage{fullpage}
\usepackage{endnotes}
\renewcommand{\bfseries}{\scshape}
\renewcommand{\footnote}{\endnote}

\makeatother
\begin{document}

\title{CSC165H, Mathematical expression and reasoning for computer science\\
  week 10}

\maketitle
Gary Baumgartner and Danny Heap

heap@cs.toronto.edu

SF4306A

416-978-5899

http://www.cs.toronto.edu/\textasciitilde{}heap/165/S2005/index.shtml



\section*{More theorems}
\label{moreTheorems}

To show that $(f\in O(g)\wedge g\in O(h))\Rightarrow f\in O(h)$, we
need to find a constant $c\in\mathbb{R}^+$ and a constant
$B\in\mathbb{N}$, that satisfy:
\begin{displaymath}
  \forall n\in\mathbb{N}, n\geq B\Rightarrow f(n) \leq c h(n).
\end{displaymath}
Since we have constants that scale $h$ to $g$ and then $g$ to
$f$, it seems clear that we need their product to scale $g$ to $f$.
And if we take the maximum of the two starting points, we can't go
wrong.  Making this precise.

\begin{flushleft}
Theorem 1:

Assume $f\in O(g)\wedge g\in O(h)$.

\begin{quote}
  So $f\in O(g)$.

  So $g\in O(h)$.

  So $\exists c \in \mathbb{R}^+, \exists B\in \mathbb{N}, \forall
  n\in \mathbb{N}, n>B\Rightarrow f(n) \leq c g(n)$. (by defn. of
  $f\in O(g)$).

  Let $c_g\in\mathbb{R}^+,B_g\in\mathbb{N}$ be such that $\forall
  n\in\mathbb{N}, n\geq B\Rightarrow f(n) \leq c_g g(n)$.

  So $\exists c \in \mathbb{R}^+, \exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow g(n)\leq ch(n)$. (by defn. of
  $g\in O(h)$).

  Let $c_h\in\mathbb{R}^+, B_h\in \mathbb{N}$ be such that $\forall
  n\in\mathbb{N}, n\geq B_h \Rightarrow g(n)\leq c_h h(n)$.

  Let $c=c_g c_h$.  Let $B=\max(B_g, B_h)$.
  \begin{quote}
    Let $n\in\mathbb{N}$.

    Suppose $n\geq B$.
    \begin{quote}
      Then $n\geq B_h$ (definition of $\max$), so $g(n) \leq c_h h(n)$.
      
      Then $n\geq B_g$ (definition of $\max$), so $f(n) \leq c_g
      g(n)\leq c_g c_h h(n)$.

      So $f(n) \leq c h(n)$.
    \end{quote}
    So $n\geq B\Rightarrow f(n)\leq c h(n)$.

    Since $n$ is an arbitrary natural number, $\forall
    n\in\mathbb{N}, n\geq B\Rightarrow f(n)\leq c h(n)$.

  \end{quote}

  Since $c$ is a positive real number, since $B$ is a natural number,
  $\exists c\in\mathbb{R}^+,\exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow f(n) \leq c h(n)$.

  So $f\in O(g)$, by definition.
\end{quote}
So $(f\in O(g)\wedge g\in O(h))\Rightarrow f\in O(g)$
\end{flushleft}

To show that $g\in\Omega (f) \Leftrightarrow f\in O(g)$, it is enough
to note the the constant, $c$, for one direction is positive, so its
reciprocal will work for the other direction.%
%
\footnote{Let's try the symmetrical presentation of bi-implication.}
%

\begin{flushleft}
Theorem 2:

$g\in\Omega(f)$

$\Leftrightarrow$ (definition)

$\exists c \in \mathbb{R}^+, \exists B \in\mathbb{N}, \forall
n\in\mathbb{N}, n\geq B\Rightarrow g(n) \geq cf(n)$

$\Leftrightarrow$. (by letting $c'=1/c$ and $B'=B$).

$\exists c' \in \mathbb{R}^+, \exists B'\in\mathbb{N}, \forall n
\in\mathbb{N}, n\geq B \Rightarrow f(n) \leq c'g(n)$. 

$\Leftrightarrow$ (definition)

$f \in O(g)$.\vspace{\baselineskip}
\end{flushleft}

To show $g\in\Theta(f) \Leftrightarrow g\in O(f) \wedge
g\in\Omega(f)$, it's really just a matter of unwrapping the
definitions.

Theorem 3:

\begin{flushleft}
$g\in\Theta(f)$

$\Leftrightarrow$ (definition)

$\exists c_1, c_2\in\mathbb{R}^+, B \in\mathbb{N}, \forall
n\in\mathbb{N}, n\geq B\rightarrow c_1 f(n) \leq g(n) \leq c_2 f(n)$.

$\Leftrightarrow$ (combined inequality).

$\exists c_1\in\mathbb{R}^+, \exists B\in\mathbb{N}, \forall
n\in\mathbb{N}, n\geq B\Rightarrow g(n) \geq c_1 f(n)$ $\wedge$
$\exists c_2\in\mathbb{R}^+, \exists B\in\mathbb{N}, \forall
n\in\mathbb{N}, n\geq B\Rightarrow g(n)\leq c_2 f(n)$

$\Leftrightarrow$ (definition)

$g\in\Omega(f)$ $\wedge$ $g\in O(f)$.
\end{flushleft}


\section*{Taxonomy of results}
\label{taxonomy}

A lemma is a small result needed to prove something we really care
about.  A theorem is the main result that we care about (at the
moment).  A corollary is an easy (or said to be easy) consequence of
another result.  A conjecture is something suspected to be true, but
not yet proven.

Here's an example of a conjecture whose proof has evaded the best
minds for over 70 years.  Maybe you'll prove it.

Define $f(n)$, for $n\in\mathbb{N}$ by:
\begin{displaymath}
  f(n) = 
  \begin{cases}
    n/2, & n \text{ even} \\
    3n+1, & n \text{ odd} \\
  \end{cases}
\end{displaymath}
Let's call $f(f(n))$ $f^2(n)$, and $f(f^k(n))$ $f^{k+1}(n)$.  Here's
the conjecture: $\forall n\in\mathbb{N},\exists k\in\mathbb{N},
f^k(n)=1$.  Easy to state, but (so far) hard to prove or disprove.

Here's an example of a corollary that recycles some of the theorems
we've already proven (so we don't have to do the grubby work).  To
show $g\in\Theta(f) \Leftrightarrow f\in\Theta(g)$, I re-use theorems
proved above and the commutativity of $\wedge$:

\begin{flushleft}
  $g\in\Theta(f)$

  $\Leftrightarrow$ (by Theorem 3)

  $g\in O(f) \wedge g\in \Omega(f)$.

  $\Leftrightarrow$ (by Theorem 2)

  $g\in O(f) \wedge f\in O(g)$

  $\Leftrightarrow$ (by commutativity of $\wedge$)

  $f\in O(g) \wedge g\in O(f)$

  $\Leftrightarrow$ (by Theorem 2)

  $f \in O(g) \wedge f\in\Omega(g)$

  $\Leftrightarrow$ (by Theorem 3)
  
  $f\in\Theta (g)$.

\end{flushleft}


\section*{Running time of programs}
\label{runningTime}

For any program $P$ and any input $x$, let $t_P(x)$ denote the number
of ``steps'' $P$ takes on input $x$.  We need to specify what we mean
by a ``step.''  Consider the following (somewhat arbitrary) accounting
for common program steps:
\begin{description}
\item[method call:] 1 step $+$ steps to evaluate each argument, $+$
  steps to execute the method.
\item[return statement:] 1 step $+$ steps to evaluate return value.
\item[if statement:] 1 step $+$ steps to evaluate condition
\item[assignment statement:] 1 $+$ steps to evaluate each side
\item[arithmetic, comparison, boolean operators:] 1 $+$ steps to
  evaluate each operand.
\item[array access:] 1 $+$ steps to evaluate index
\item[member access:] 2 steps
\item[constant, variable evaluation:] 1 step
\end{description}

Example: Linear Search
\begin{tabbing}
  // $A$ is an array, $x$ is an element to search for. \\
  // Return an index $i$ such that $A[i]=x$\\
  // if there is no such index, return $-1$\\
  // Convention: array indices start at 0\\
  \\
  1. \= $i$ \= while \= if \= return i;\qquad\qquad \= \kill
  LS ($A$,$x$) \\
  1. \>  i = 0;  \>\>\>\> // 3 steps (evaluate variable,
  constant, assignment)\\
  2. \>\> while  (i $<$ $A$.length) \{ \>\>\> // 5 steps (while,
  $A$.length (2), $i$, $<$)  \\
  3. \>\>\>if  ($A$[i] == $x$) \{ \>\> // 5 steps ($A$[i], $==$, $x$,
  if)  \\
  4. \>\>\>\> return i; \> // 2 steps (return, $i$) \\
  5. \>\>\> \} \\
  6. \>\>\> i = i$+$1; \>\> // 5 steps ($i$, assignment, i, $+$, 1) \\
  7. \>\>\} \\
  8. \>return -1; \>\>\>\> // 2 steps (return, $-1$)\\
  9. \>\} \\
\end{tabbing}

Let's trace a function call, $t_{LS}([2,4,6,8],4)$:
\begin{enumerate}
\item[1.] 3 steps (i=0)
\item[2.] 5 steps (0 < 4)
\item[3.] 5 steps ($A$[0] == 4)
\item[6.] 5 steps (i = 1)
\item[2.] 5 steps (1 < 4)
\item[3.] 5 steps ($A$[1] == 4)
\item[4.] 2 (return 1)
\end{enumerate}

So $t_{LS}([2,4,6,8],4)=30$.  Notice that if the first index where $x$
is found is $j$, then $t_{LS}(A,x)$ will count lines 2, 3, and 6
for each index from 0 to $j-1$ ($j$ indices), and then count lines
2,3,4 when for index $j$, and so $t_{LS}(A,x)$ will be $3+15j+12$ $=$
$15(j+1)$.

If $x$ doesn't appear in $A$, then $t_{LS}(A,x)$ $=$ $3+15 A.length +
7$ $=$ $15 A.length + 10$, because line 1 executes once, lines 2,3,
and 6 execute for each index from 0 to $A.length-1$ ($A$.length
indices), and then lines 2 and 8 execute.

We want a measure that depends on the size of the input, not the
particular input.  There are three standard ways:

\begin{description}
\item[Best-case complexity:] $\min(t_p(x))$, where $x$ is an input of
  size $n$.  In other words, $\min\{t_p(x)| x\in I \wedge size(x)=n\}$. 

\item[Worst-case complexity:] $max(t_p(x))$, where $x$ is an input of
  size $n$. In other words, $\max\{tp(x)|x\in I\wedge size(x)=n\}$.
\item[Average-case complexity (assuming all inputs equally likely):]
  \begin{displaymath}
    A_p(n) = \frac{\sum_{x \text{ of size }n} t_p(x)}{\text{number of
    inputs of size }n}
  \end{displaymath}
\end{description}

Best-case: useless.  Average-case: difficult to compute.  Worst-case:
easier to compute, and gives a guarantee.

What is meant by ``input size''?  This depends on the algorithm.  For
linear search, the number of elements in the array is a reasonable
parameter.  Technically (CSC363, etc.) the size is the number of bits
required to represent the input in binary.  In practice we use the
number of elements of input (length of array, number of nodes in a
tree, etc.)

Best-case for linear search?%
%
\footnote{15 steps, when $A$[0] == $x$.}
%
Worst-case for linear search?%
%
\footnote{$15n+10$, where $n=A.length$.}
%
Average-case for linear search?%
%
\footnote{Inputs at index 0 through $n-1$, plus missing value equally
  likely $(n+1)$ input categories, so
  \begin{eqnarray*}
  \frac{\left(\sum_{i=0}^{n-1} 15(i+1)\right) + 15n + 10}{n+1} &=&
  \frac{15 n(n+1)/2 + 15n + 10}{n+1} \\
  &=& \frac{7.5 n(n+1) + 15n + 10}{n+1} \\
  &=& \frac{7.5 n(n+1) + 15n+ 10}{n+1} = 7.5 n + \frac{15n+10}{n+1}.
  \end{eqnarray*}
}
%
Now we can use the machinery of big-Oh.  Suppose $U$ is an upper bound
on the worst-case running time of some algorithm $P$, denoted $T_P(n)$:

\begin{flushleft}
  $t_P \in O(U)$

  $\Leftrightarrow$ 

  $\exists c \in\mathbb{R}^+, \exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow T_P(n) \leq cU(n)$

  $\Leftrightarrow$

  $\exists c \in\mathbb{R}^+, \exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow \max\{t_p(x)| x\in I\wedge size(x)=n\}
  \leq cU(n)$

  $\Leftrightarrow$

  $\exists c \in\mathbb{R}^+, \exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow \forall x\in I, size(x)=n
  \Rightarrow t_p(x) 
  \leq cU(n)$

  $\Leftrightarrow$

 $\exists c \in\mathbb{R}^+, \exists B\in\mathbb{N}, \forall x \in I,
  size(x)\geq B \Rightarrow t_p(x)
  \leq cU(size(x))$
\end{flushleft}
($I$ is the set of all inputs for $P$). So to show that $T_P \in
O(U(n))$, you need to find constants $c$ and $B$ and show that for an
arbitrary input $x$ of size $n$, $P$ takes at most $cU(n)$ steps.

In the other direction, $L$ is a lower bound on the worst-case running
time of algorithm $P$:
\begin{flushleft}
  $T_P\in\Omega(L)$

  $\Leftrightarrow$

$\exists c \in\mathbb{R}^+, \exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow \max\{t_p(x)| x\in I\wedge size(x)=n\}
  \geq cL(n)$

  $\Leftrightarrow$

$\exists c \in\mathbb{R}^+, \exists B\in\mathbb{N}, \forall
  n\in\mathbb{N}, n\geq B\Rightarrow \exists x\in I, size(x)=n\wedge t_p(x)
  \geq cL(n)$
\end{flushleft}

So, to prove that $T_p \in\Omega(L)$, we have to find constants $c$,
$B$ and for arbitrary $n$, find an input $x$ of size $n$, for which we
can show that $P$ takes at least $cL(n)$ steps on input $x$



\newpage

\begingroup

\setlength{\parindent}{0pt}

\setlength{\parskip}{2ex}

\renewcommand{\enotesize}{\normalsize}

\theendnotes\endgroup
\end{document}