\documentclass[11pt]{article}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{bm}
\usepackage{epsfig}
\usepackage[tight]{subfigure}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {#1} \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[5]{\handout{#1}{#2}{#3}{#4}{#5}}

\newtheorem{theorem}{Theorem}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}[theorem]{Assumption}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
\topmargin 0pt
\advance \topmargin by -\headheight
\advance \topmargin by -\headsep
\textheight 8.9in
\oddsidemargin 0pt
\evensidemargin \oddsidemargin
\marginparwidth 0.5in
\textwidth 6.5in

\parindent 0in
\parskip 1.5ex
%\renewcommand{\baselinestretch}{1.25}

\begin{document}
\newcommand{\defeq}[0]{\ensuremath{\stackrel{\triangle}{=}}}
\def\x{\mathbf{x}}
\def\w{\mathbf{w}}
\def\K{\mathbf{K}}
\lecture{Statistical Techniques in Robotics (16-831, F010)}{Lecture \#22 (Nov 11, 2010)}{Lecturer: Drew Bagnell}{Scribe: Varun Ramakrishna$^1$}{Kernel Machines / Functional Gradient Descent}
\let\thefootnote\relax\footnotetext{$^1$Based on the scribe work of Daniel Munoz and Tomas Simon}
\section{The Basic Idea}
We have seen how to use online convex programming to learn linear functions by optimizing costs of the following form:
\begin{equation}
	\underbrace{|y_t - \w^T \x_t|}_{loss} + \underbrace{\lambda \w^T\w^2}_{regularization / prior} \nonumber
\end{equation}

We want generalize this to learn over a space of more general functions $f: \mathbb{R}^n \rightarrow \mathbb{R}$. The high-level idea is to learn non-linear models using the same gradient-based approach used to learn linear models.

\begin{equation}
	|y_t - f(\x_t)| + \lambda ||f||^2 \nonumber
\end{equation}

Up till now we have only considered functions of the form $f(\x)=\w^T \x$, but we will now extend this to a more general space of functions. 

\section{ Review of Kernels}

We observed in Gaussian Processes that the mean function had the form $\mu_{f(\x)| data}= K^TK^{-1}_{data}y$ which can be interpreted as a sum of Kernel functions on the observed data points:

\begin{equation}
	f = \sum_i\alpha_iK(x_i,\cdot)
\end{equation}

Kernels functions are a natural choice of non-linear functions for our task i.e  to learn a function $f: \mathbb{R}^n \rightarrow \mathbb{R}$ that assigns
a meaningful score given a data point.

 E.g. in binary classification, we would like $f(\cdot)$ 
to return positive and negative values, given positive and negative samples, respectively.

 A kernel $K: \mathbb{R}^n \times \mathbb{R}^n \rightarrow \mathbb{R}$ 
intuitively measures the \textit{correlation} between $f(\mathbf{x_i})$ and $f(\mathbf{x_j})$.  Considering
a matrix $\mathbf{K}$ with entries $K_{ij} = K(\mathbf{x_i}, \mathbf{x_j})$, then matrix 
$\mathbf{K}$ must satisfy the properties:
\begin{itemize}
	\item
	$\mathbf{K}$ is symmetric ($K_{ij} = K_{ji}$)
	\item
	$\mathbf{K}$ is positive-definite 
		($\forall \mathbf{x} \in \mathbb{R}^n : \mathbf{x} \neq \mathbf{0}, \mathbf{x^T K x} > 0$)
\end{itemize}
However to do gradient descent on the space of such functions, we need the notion of a distance, norm and an inner product. We formalize this by introducing the Reproducing Kernel Hilbert Space.

\section{Reproducing Kernel Hilbert Space}

The Reproducing Kernel Hilbert Space (RKHS), denoted by $\mathcal{H}_k$, is the space of functions $f(\cdot)$ that can be written as $\sum_i \alpha_i k(\x_i,\cdot)$, where $k(\x_i,\x_j)$ satisfies certain properties described below.

To be able to manipulate objects in this space of functions, we will look at some key properties:

The \emph{inner product} of $f, g \in \mathcal{H}_k$ is defined as
$$\langle f,g \rangle \defeq \sum_i \sum_j \alpha_i \beta_j k(\x_i, \x_j) = \bm{\alpha}^{\top} \mathbf{K} \bm{\beta}$$
where $f(\cdot) = \sum_i \alpha_i k(\x_i,\cdot)$, $g(\cdot) = \sum_j \beta_j k(\x_j,\cdot)$, $\bm{\alpha}$ and $\bm{\beta}$ are vectors comprising, respectively, $\alpha_i$ and $\beta_i$ components, and $\mathbf{K}$ is $n$ by $m$ (where $n$ is the number of $\x_i$ in $f$, and $m$ those in $g$) with $K_{ij}=k(\x_i, \x_j)$.

Note that this will satisfy linearity (in both arguments):
\begin{itemize}
	\item
	   $\langle \lambda f, g\rangle = \lambda \langle f,g\rangle$
	\item
	   $\langle f_1+f_2, g \rangle = \langle f_1,g\rangle + \langle f_2,g\rangle$
\end{itemize}

With this inner product, the \emph{norm} will be: $|| f ||^2 = \langle f,f \rangle = \bm{\alpha}^{\top} \mathbf{K} \bm{\alpha}$.

The \emph{reproducing property} is observed by taking the inner-product of a function with a kernel
\begin{equation}
	\langle f, K(\mathbf{x_j}, \cdot)\rangle = \langle \sum_{i=1}^Q \alpha_i K(\mathbf{x_i},\cdot) , K(\cdot, \mathbf{x_j})\rangle
	 				= \sum_{i=1}^Q \alpha_i \langle K(\mathbf{x_i}, \cdot), K(\cdot, \mathbf{x_j})\rangle
	 				= \sum_{i=1}^Q \alpha_i  K(\mathbf{x_i}, \mathbf{x_j}) = f(\mathbf{x_j})
\end{equation}
An example of a valid kernel for $\x \in \mathbb{R}^n$ is the inner product: $k(\x_i,\x_j) = \x_i^T \x_j$. Intuitively, the kernel measures the \textit{correlation} between $\x_i$ and $\x_j$. 

A very commonly used kernel is the RBF or Radial Basis Function kernel, which takes the form $k(\x_i, \x_j)= e^{-\frac{1}{\gamma}||\x_i - \x_j||^2}$.
With this kernel in mind, a function can be considered as a weighted (by $\alpha_i$) composition of bumps (the kernels) centered at the $Q$ locations $\mathbf{x_i}$: \begin{equation}
%  \label{eq:rkhs-function}
	f(\cdot) = \sum_{i=1}^Q \alpha_i K(\x_i, \cdot), \nonumber
\end{equation}
See figure \ref{fig:kernel_update} for an illustration.



\section{Loss Minimization}
Let us consider again our cost function defined over all functions $f$ in our RKHS, as before our loss is :
\begin{equation}
	|y_t - f(\x_t)| + \lambda ||f||^2 \nonumber
\end{equation}

The purpose of $\langle f, f \rangle$ is to penalize the complexity of the solution $f$. Here it acts like the log of a gaussian prior over functions. Intuitively, the probability can be thought of as being distributed according to $P(f)=\frac{1}{Z} e^{-\frac{1}{2} \langle f,f \rangle}$ (in practice this expression doesn't work because $Z$ becomes infinite).

We want to find the best function $f$ in our RKHS so as to minimize this cost, and we will do this by moving in the direction of the negative gradient: $f - \alpha \nabla L$. To do this, we will first have to be able to express the gradient of a function of functions (ie. a \emph{functional} such as $L[f]$).

%%%%%%%%%%%%
\subsection{Functional gradient}
A gradient can be thought of as:
	\begin{itemize}
		\item
            Vector of partial derivatives
		\item
			Direction of steepest ascent
		\item
			Linear approximation of the function (or functional), ie. $f(x_0 + \epsilon) = f(x_0) + \epsilon \cdot \underbrace{\nabla f(x_0)}_{gradient}  + O(\epsilon^2)$.
	\end{itemize}
We will use the third definition. 
A \textit{functional} $F : f \rightarrow \mathbb{R}$ is a function of functions $f \in \mathcal{H}_K$.
As an example let us write the terms of our loss function from abov as functionals:
	\begin{itemize}
		\item
			$F_{1}[f] = ||f||^2$
		\item
			$F_{2}[f] = (f(x) - y)^2$
		\item
			$F[f] = \frac{\lambda}{2} ||f||^2 + \sum_i (f(x_i) - y_i)^2$
	\end{itemize}
A functional gradient $\nabla F[f]$ is defined implictly
as the linear term of the change in a function due to a small
perturbation $\epsilon$ in its input: $F[f + \epsilon g] = F[f] + \epsilon \langle \nabla F[f], g \rangle + O(\epsilon^2)$.

Before computing the gradients for these functionals, let us look at a few tools that will help us derive the gradient of the loss functional

\subsection {Chain rule for functional gradients}
Consider \textit{differentiable} functions $C : \mathbb{R} \rightarrow \mathbb{R}$ that are functions
of functionals $G$, $C(G[f])$.  Our cost function $L[f]$ from before was such a function, these are precisely the functions that we are interested in minimizing.


The derivative of these functions follows the chain rule:
 \begin{equation}
\nabla C(G[f]) = \frac{\partial C(G[f])}{\partial \lambda}\vert_{G(f)} \nabla G[f]
\end{equation}
	%\item
	Example: If $C = (||f||^2)^3$, then $\nabla C = 3(||f||^2)^2(2f)$

\subsection{Another useful functional gradient}

Another useful functio nthat we come across often  is the evaluation functional.
The \emph{evaluation functional} evaluates $f$ at the specified $x$: $F_x[f] = f(x)$
	\begin{itemize}
		\item
		Gradient is $\nabla F_x = K(x, \cdot)$
		\begin{eqnarray*}
			F_x[f + \epsilon g] &=& f(x) + \epsilon g(x) + 0 \\
			& = & f(x) + \epsilon \langle K(x, \cdot) , g \rangle + 0 \\
			& = & F_x[f] + \epsilon \langle \nabla F_x , g \rangle + O(\epsilon^2)
		\end{eqnarray*}
		
		\item It is called a \textit{linear functional} due to the lack of a multiplier on perturbation $\epsilon$.
	\end{itemize}


%%%%%%%%%%%%%
\subsection{Functional gradient of the regularized least squares loss function}

	\begin{itemize}
		\item Let's look at the functional gradient of the first term of the loss function:
		 \begin{equation}
		\nabla F[f] = \nabla ||f||^2 \\
		\end{equation}
		Expanding it out using a Taylor's series type expansion
		\begin{eqnarray*}
			F[f + \epsilon g] & = &\langle f + \epsilon g, f + \epsilon g\rangle \\
			& = & ||f|| + 2 \langle f, \epsilon g \rangle + \epsilon^ 2	||g|| \\
			& = & ||f|| + \epsilon \langle 2f , g \rangle + O(\epsilon^2)
		\end{eqnarray*}
		We observe that
		 \begin{equation}
		\nabla F[f] = \nabla ||f||^2  = 2f
		\end{equation}

	\item Now for the second term of the loss function
		 \begin{equation}
		F[f] = \sum_i (f(x_i) - y_i)^2 \\
		\end{equation}
		Using the chain rule we have
 		\begin{equation}
		\nabla F[f] = 2(f(x_i)-y_i) \nabla(f(x_i))\\
		\end{equation}
		We observe that $\nabla(f(x_i)) $ is the functional gradient of the evaluation functional. Substituting in the gradient of the evaluaton functional as computed in the previous section we have :
		 \begin{equation}
		\nabla F[f] = 2(f(x_i)-y_i)K(x_i, \cdot)\\
		\end{equation}
	\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Functional gradient descent}
\begin{itemize}
\item
Consider the regularized least squares loss function $L[f]$
\begin{eqnarray*}
L[f] &=& (f(x_i) - y_i)^2 + \lambda ||f||^2 \\
L[f] &=& (F_{x_i}[f] - y_i)^2 + \lambda ||f||^2 \\
\nabla L[f] &=& 2(f(x_i) - y_i)K(x_i, \cdot) + 2 \lambda f
\end{eqnarray*}

\item
Update rule:
\begin{eqnarray*}
f^{t+1} & \leftarrow & f^t - \eta_t \nabla L \\
& \leftarrow & f^t - \eta_t (2(f^t(x_i) - y_i)K(x_i, \cdot) + 2 \lambda f^t) \\
& \leftarrow & f^t(1 - 2 \eta_t \lambda) - \eta_t (2(f^t(x_i) - y_i)K(x_i, \cdot))
\end{eqnarray*}

\item
Need to perform $O(T)$ work at each time step

\item
Example: Figure \ref{fig:kernel_update} shows an update over 3 points $\{(x_1,+), (x_2,-) , (x_3,+)\}$.
The individual kernels centered at the points are \textbf{independently} drawn
with colored lines.  After 3 updates, the function $f$ looks like the solid black line.
\begin{figure}[ht]
\label{fig:kernel_update}
\centering
\includegraphics[width=0.85\linewidth]{kernel_update}
\caption{Illustration of function after 3 updates}
\end{figure}

\item
\textbf{Representer Theorem} (informally): Given a loss function and regularizer objective
with many data points $\{x_i\}$, the minimizing solution $f^*$ can be represented as
\begin{equation}
f^*(\cdot) = \sum_i \alpha_i K(x_i, \cdot)
\end{equation}

\item
Alternate idea from class: perform gradient descent in the space of $\alpha$ coefficients: $\nabla_{\alpha} L$
	\begin{itemize}
	\item Takes $n^2$ iterations to get same performance ($n$ = number of iterations of functional gradient descent)
	\item Every iteration is $O(T^2)$
	\end{itemize}

\end{itemize}


\end{document}
