\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{epsfig}
\usepackage[tight]{subfigure}
\usepackage{algorithmic}
\usepackage{algorithm}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {#1} \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{#1}{#2}{#3}{#4}{#5}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex
%\renewcommand{\baselinestretch}{1.25}

\begin{document}

\lecture{Statistical Techniques in Robotics (16-831, F10)}{Lecture \#14 (Thursday October 7th)}{Lecturer: Drew Bagnell}{Scribe: Robert Fisher \footnotemark}{Support Vector Machines}
\footnotetext{Some content adapted from previous scribe: Alan Kraut}

\section{Introduction}
We begin by considering binary, linear classification. In this problem, we are trying to map elements from our feature space into the set $\{-1,1\}$. When we have two features, $F_1,F_2\in \mathbb R$, we can think of the problem graphically. In the following figure, we represent the points with a label of 1 as O's, we the points with a label of $-1$ as X's. We wish to find a linear decision boundary (a straight line) that separates the points from each class. The decision boundary is shown as the dotted line.
\begin{figure}[htbp]
    \centering
        \includegraphics[width=9cm]{binaryClassification.pdf}
    \caption{Linear binary classification}
    \label{fig:bin}  % you can refer to this figure ase robots
\end{figure}

To represent this notion mathematically, we denote the $i^{th}$ training point as $f_i$, and the class of this point as $y_i$. The linear classification problem requires us to find a set of weights $w$ such that:
    \begin{equation}
   \forall i\ y_iw^Tf_i\geq 0
    \end{equation}

Note that the decision boundary need not pass through the origin. We often use a dummy variable $F_0=c$ for constant $c\neq 0$, which allows us to use $w_0$ as an offset.

One issue with this formulation of the problem is that it has a trivial solution. Specifically, if we set all of our weights to 0, we will have $y_iw^Tf_i=0$ for all points. To address this issue, we modify our constraints to be:
    \begin{equation}
   \forall i\ y_iw^Tf_i\geq margin
    \end{equation}
This can have the added benefit of improving generalization. Once again, we can represent this graphically. In the following picture, the space between the dotted line and the decision boundary represents the margin.
\begin{figure}[htbp]
    \centering
        \includegraphics[width=9cm]{binaryClassification2.pdf}
    \caption{Linear binary classification with margins}
    \label{fig:bin}  % you can refer to this figure ase robots
\end{figure}

We further see that the magnitude of our weight vector does not matter, only the orientation affects classification. Therefore we can restrict $w$ to $||w||^2=1$. Taken together, this yields the following formulation:

\[ \begin{array}{rl}
    \text{{\bf Maximize }}        & margin \\
    \mbox{\text{{\bf Such that }}} & \forall i\ y_iw^Tf_i\geq margin \\
                &  ||w||^2\leq 1
    \end{array}
\]
We use $||w||^2\leq 1$ instead of $||w||^2= 1$ to make sure that our problem remain convex.

This problem is generally solved in the dual, where we hold the margin fixed and minimize the magnitude of the weights. Therefore the final formulation of the hard margin SVM problem is:
\[ \begin{array}{rl}
    \text{{\bf Minimize }}        & ||w||^2\ \\
    \mbox{\text{{\bf Such that }}} & \forall i\ y_iw^Tf_i\geq 1 \\
    &  margin>0
    \end{array}
\]
  
  This last formulation is an example of a quadratic problem, which we are able to solve efficiently. Unfortunately, the hard margin SVM requires that the data be linearly separable, which is almost never the case. To address this issue, we introduce a slack variable, $\xi$. The problem now becomes:
  \[ \begin{array}{rl}
    \text{{\bf Minimize }}        & \lambda ||w||^2 + \displaystyle\sum_i \xi_i \\
    \mbox{\text{{\bf Such that }}} & y_i w^T f_i \geq 1-\xi_i\\
    &   \xi_i \geq 0 \\
     &  margin>0
    \end{array}
\]
There will be one slack variable $\xi_i$ to correspond to each of the $T$ data-points that we are considering.

To make this online, we observe that $\xi = \max (0,1-y_i w^T f_i)$, because the slack variable is 0 if this point is labeled correctly. This allows us to generate the loss function
    \begin{equation}
    l_t = \lambda ||w||^2 + \max (0,1-y_t w^T f_t)
    \end{equation}
    
Our update for $w$ is now as follows.
    \begin{equation}
    w \leftarrow w - 2 \alpha_t \lambda w
    \end{equation}
And if the output for this time step was incorrect,
    \begin{equation}
    w \leftarrow w + \alpha_t y_t f_t
    \end{equation}
    
We note that this loss function will {\bf not} allow us to minimize the number of mistakes made by the algorithm. In fact, solving this problem with a 0-1 loss function (which would minimize the number of mistakes) is known to be NP-hard. We can visualize the difference between these loss functions with the following figure, in which $Z=w^Tf_i$:

\begin{figure}[htbp]
    \centering
        \includegraphics[width=9cm]{lossFunctions.pdf}
    \caption{Loss functions}
    \label{fig:loss}  % you can refer to this figure ase robots
\end{figure}

Our loss function, the hinge loss function, is convex, while the mistake loss (0-1 loss) function is not. However, we do see intuitively that the loss function we are using is the best ``convexification'' of the NP-hard problem.

\section{Implementing Online SVMs}
\subsection{Selecting $\alpha_t$}
\begin{itemize}
\item Stock algorithm would be to set $\alpha_t$ proportional to $\frac{1}{\sqrt{t}}$.
\item If we have $T$ elements, each with a maximum value of $F$, the maximum gradient, $G$, is $\sqrt{TF}$. This results in a regret that is $R\leq \sqrt{FGT}$.
\item This is not as good as we could do.
\item Notice that $l_t$ is an extremely good convex function. It is a quadratic plus a convex function. In the same way all convex functions lie above a line (a subgradient) from every point, $l_t$ lies above a quadratic from every point.
\item Specifically, if it is always the case that
\begin{equation}
f(y) \geq f(x) + \frac{H}{2}(y-x)^2 + \nabla f_x^T(y-x)
\end{equation}
then $f(x)$ is said to be H-strongly convex.
\item In this case $l_t$ is $\lambda$-strongly convex.
\item If $\alpha_t=\frac{G}{Ht}$, then regret$\leq\frac{G^2}{H}(1+\log{t})$. $\log{t}$ is \emph{really} good, and this learning rate and algorithm is essentially the current best for this class of problem.

\end{itemize}

\subsection{SVMs with Multiple Classes}
We can represent problems with more than two classes by having a weight vector, $w_i$ for each class.
\begin{itemize}
\item When we get a classification of a particular example (for example, example $i$ is of class 1), we generate a set of constraints that can be expressed as either
\begin{eqnarray}
w_1^Tf_i &\geq& w_2^Tf_i+1 \nonumber \\
w_1^Tf_i &\geq& w_3^Tf_i+1 \\
w_1^Tf_i &\geq& w_4^Tf_i+1 \nonumber \\
&\ldots& \nonumber
\end{eqnarray}
or
\begin{equation}
w_1^Tf_i \geq \displaystyle{\mathop{\mbox{max}}_{c \neq i}} (w_c^Tf_i + 1)
\end{equation}

\item By the same argument as before
\begin{equation}
\xi = \max (0,\max (w_c^Tf + 1) - w_1^Tf)
\end{equation}

\item We want to update each $w$ by gradient descent on the partial of the cost with respect to that particular $w$.
\item Remember the cost is $l_t = \lambda ||w||^2+\xi$. We want the update step to be
\begin{equation}
w_c \leftarrow w_c - \partial_{w_c}l_t
\end{equation}

\item In the case that the example was classified correctly, $\partial_{w_c} l_t = 0$. If it was misclassified, there are three cases with different partials: the correct class, the class we incorrectly decided this was an example of, and all others.
\begin{eqnarray}
\partial_{w_c} &=& -f_i, \quad y_i = c \\
\partial_{w_c} &=& f_i, \quad c = \displaystyle{\mathop{\mbox{argmax}}_{c}}(w_c^Tf_i + 1) \\
\partial_{w_c} &=& 0, \quad \text{otherwise}
\end{eqnarray}

\item That update is for the $\max$ representation. If we use the multiple constraints representation it is similar, except we update both $w_1$ and $w_c$ for all $c$ which violate the constraint.
\end{itemize}


\end{document}
