In [1]:
import math
import numpy as np
import matplotlib.pyplot as plt

In [2]:
mnist_images = np.load("mnist_images.npy")

In [3]:
def plot_mnist(remove_border=False):
""" Plot the first 40 data points in the MNIST train_images. """
import matplotlib.pyplot as plt
plt.figure(figsize=(10, 5))
for i in range(4 * 10):
plt.subplot(4, 10, i+1)
if remove_border:
plt.imshow(mnist_images[i,4:24,4:24])
else:
plt.imshow(mnist_images[i])

plot_mnist() # please comment out this line before submission

In [4]:
mnist_images.shape

Out[4]:
(5000, 28, 28)
In [5]:
A = mnist_images.reshape([5000, 28*28])

In [6]:
A.shape # m = 5000, n = 784

Out[6]:
(5000, 784)
In [8]:
avg = np.mean(A, axis=0)
avg.shape

Out[8]:
(784,)
In [9]:
A = A - avg # centering

In [10]:
A.shape

Out[10]:
(5000, 784)
In [13]:
u, s, v = np.linalg.svd(A) # A = U * S * V

In [14]:
u.shape

Out[14]:
(5000, 5000)
In [15]:
v.shape

Out[15]:
(784, 784)
In [16]:
s.shape  # S should be [5000, 784], but S is diagonal matrix

Out[16]:
(784,)
In [17]:
plt.plot(s) # s contains the singular values (or sqrt(eigenvalues) of A^T A)

Out[17]:
[<matplotlib.lines.Line2D at 0x1208ddfd0>]
In [18]:
for i in range(10):
plt.figure()
plt.imshow(v[i].reshape(28, 28)) # the first column of V --- eigenvector corresponding to the largest eigenvalue

In [19]:
# Demonstrate power iteration on

M = A.T @ A

In [20]:
x = np.random.randn(784)

In [21]:
for i in range(1000):
x = M @ x
x = x / np.linalg.norm(x)

In [22]:
plt.imshow(x.reshape(28, 28))

Out[22]:
<matplotlib.image.AxesImage at 0x12a1d25f8>
In [24]:
plt.imshow((A[0]).reshape(28, 28))

Out[24]:
<matplotlib.image.AxesImage at 0x12a688dd8>
In [28]:
n = 400
recon = (u[:, :n] @ np.diag(s[:n])) @ v[:n, :]
plt.imshow(recon[0].reshape(28, 28))

Out[28]:
<matplotlib.image.AxesImage at 0x129eed518>
In [29]:
v[:n, :].shape

Out[29]:
(400, 784)
In [35]:
(u[0, :n] @ np.diag(s[:n]))

Out[35]:
array([-4.55478549e-01,  1.44640517e+00, -1.26561061e-01,  2.04640698e+00,
-9.06210542e-01,  8.65823507e-01, -7.14304328e-01,  3.84281516e-01,
-1.21694946e+00,  5.73642433e-01, -2.23285139e-01,  1.81757104e+00,
-1.06964767e+00,  2.88461351e+00, -4.34622541e-02,  4.05452549e-01,
-1.35171783e+00,  4.88097101e-01, -7.43918598e-01,  1.25593567e+00,
-4.90995705e-01,  8.97119701e-01, -7.15669572e-01,  7.82275379e-01,
-2.47401431e-01, -3.64219040e-01,  1.31172872e+00,  4.36892539e-01,
-4.51465756e-01, -1.55578479e-01,  5.90545356e-01, -3.17717791e-02,
8.79400969e-02, -1.54493809e-01,  1.75711513e-01, -4.43359107e-01,
7.23667562e-01, -3.56095225e-01,  4.43184942e-01, -1.71119118e+00,
2.67344654e-01, -8.43908191e-01,  3.89415354e-01, -1.52835354e-01,
4.40391488e-02, -5.12474000e-01,  6.19408190e-01, -1.96817964e-01,
2.10924506e-01, -3.99448365e-01,  3.10658127e-01,  9.06638801e-02,
-1.36124969e-01,  5.92600286e-01,  1.47323245e-02, -7.16122448e-01,
4.96965021e-01, -6.16328359e-01,  2.88413376e-01,  1.62665337e-01,
5.68551838e-01, -1.27703428e-01, -6.54533744e-01,  3.40654045e-01,
7.81732321e-01, -2.58569598e-01, -5.66779934e-02, -2.48831183e-01,
3.31237316e-01,  1.06547207e-01,  3.51816006e-02, -6.30600512e-01,
-1.24257179e-02, -2.62386829e-01,  3.62188846e-01,  2.90017456e-01,
2.75611989e-02, -2.36386776e-01, -1.23775043e-01, -3.04301269e-02,
7.01519325e-02,  2.13355035e-01,  9.42880735e-02,  7.14100227e-02,
4.73580062e-01,  3.60519886e-01, -2.18343467e-01,  1.69368550e-01,
-2.33022839e-01,  1.07285976e-01, -2.17119649e-01,  6.78040460e-02,
-4.30201031e-02,  2.50437975e-01, -2.45309100e-02,  1.63109243e-01,
-1.37702674e-02,  2.23139077e-01, -3.50619048e-01,  3.14214528e-02,
-5.39187491e-01,  5.18538579e-02, -2.18868479e-01,  1.75770670e-01,
-3.59771587e-02,  4.88370098e-02, -3.46072644e-01,  7.16489628e-02,
-4.66125039e-03, -1.11945234e-01,  5.54688752e-01, -1.79587856e-01,
3.16594318e-02, -1.85787767e-01, -1.35546410e-03, -7.39166066e-02,
3.02264988e-01, -3.13909531e-01, -1.69744372e-01,  2.18096718e-01,
1.04771867e-01, -2.08010301e-01,  2.62085423e-02,  2.72359140e-02,
5.01496494e-02, -1.43721446e-01, -3.81731950e-02, -2.08706334e-01,
2.15603828e-01,  1.22503052e-02, -6.08255386e-01, -3.89668196e-02,
-1.34328842e-01,  5.69594562e-01, -9.13841836e-03, -2.49004766e-01,
8.87687132e-02,  3.51987034e-02,  2.54112720e-01, -3.50831561e-02,
-2.62510091e-01,  9.73916426e-02,  1.93114638e-01,  1.62793472e-02,
-1.44680038e-01, -7.88379312e-02, -4.08733398e-01,  2.24472880e-02,
1.95195764e-01, -3.43082666e-01, -3.69196087e-02, -8.19294080e-02,
-1.93483397e-01,  5.59696071e-02,  7.18133077e-02,  1.64189469e-02,
2.42368698e-01, -1.51778460e-01, -7.35573098e-02, -3.47919166e-02,
-8.98796320e-02, -8.77071396e-02,  2.44459286e-01, -2.33712327e-02,
-5.56992367e-02, -1.30236998e-01, -2.08050191e-01, -3.10852528e-01,
1.63734332e-01,  9.60184038e-02,  4.55049388e-02,  2.67423004e-01,
-1.29410043e-01, -1.10898033e-01, -7.39086494e-02, -2.70314887e-02,
-2.18479723e-01,  1.79210454e-02,  6.89992728e-03,  1.07667364e-01,
1.95189081e-02,  1.03811920e-01,  2.08852395e-01, -1.37464181e-01,
-9.91296545e-02,  5.65454438e-02, -4.66156900e-02,  3.03496212e-01,
-7.82873258e-02,  6.43257722e-02, -1.62533432e-01,  1.24135368e-01,
-2.44762287e-01,  1.73622414e-01, -2.44327828e-01,  2.22025976e-01,
-2.49803409e-01,  4.13745008e-02, -2.12995633e-01,  5.77087887e-02,
-9.38558877e-02,  2.28245616e-01, -8.89916420e-02,  1.53294697e-01,
-1.00310281e-01,  1.14232361e-01, -4.26537603e-01,  1.94365993e-01,
-9.25412774e-03,  1.06941551e-01, -6.11606687e-02,  2.94950586e-02,
-2.58278757e-01,  2.02425253e-02, -2.39550039e-01, -5.27334958e-02,
8.03919435e-02, -1.59946457e-01,  3.71983349e-01, -2.17928216e-01,
3.66459265e-02,  7.63956085e-02, -5.36324531e-02,  2.28669018e-01,
6.27254993e-02, -7.07053915e-02, -6.07564934e-02,  6.76745549e-02,
-6.13726955e-03, -9.34195071e-02,  5.21260239e-02, -1.91808909e-01,
-1.67751759e-01,  1.96181506e-01, -8.61930475e-02, -1.83737591e-01,
2.40035504e-01, -2.96758525e-02, -1.89198047e-01,  4.98696081e-02,
6.31644279e-02, -8.82010758e-02, -1.31948115e-02,  5.56295700e-02,
2.47956370e-03, -1.51182339e-01,  1.32172080e-02,  1.06509767e-01,
-9.67905000e-02, -1.11154221e-01,  4.53621615e-03,  2.08627045e-01,
-6.93301717e-03, -1.12499772e-02, -1.20006561e-01,  4.61504497e-02,
4.55421321e-02, -9.95150656e-02, -1.40243601e-02,  2.80448884e-01,
6.33695871e-02, -5.04122749e-02, -3.79053280e-02, -1.19889513e-01,
5.82724102e-02,  8.00075661e-03, -1.36183664e-01, -1.63550712e-02,
-1.99281517e-03, -1.61598995e-02,  1.98819712e-01, -8.12733173e-02,
-3.58387269e-02, -1.32877588e-01,  1.16975158e-01,  1.24356128e-01,
-1.14533253e-01, -2.20104456e-01,  1.47130340e-01,  1.50522411e-01,
-7.66807795e-02, -2.43504774e-02, -5.40896133e-02,  7.16419592e-02,
2.17506494e-02,  5.76827191e-02, -1.03735151e-02, -6.24322146e-02,
5.27784154e-02,  2.98679322e-02, -4.66275513e-02, -4.74482067e-02,
-8.51547122e-02, -3.20085846e-02,  2.53157038e-02,  2.71272231e-02,
1.10508978e-01, -4.88457456e-02,  1.38821872e-02,  2.25312542e-02,
1.55577846e-02,  8.00202191e-02,  1.17258057e-02, -1.46938965e-01,
5.79218753e-02,  3.96120623e-02,  1.54027296e-02, -1.51540250e-01,
-1.15365304e-01,  1.22980408e-01,  6.69630691e-02, -5.78529201e-02,
-3.53047363e-02, -3.23274173e-02, -1.65950153e-02, -9.15654376e-03,
-3.24388109e-02,  2.13499814e-01,  7.90050924e-02, -2.47634962e-01,
-3.57196964e-02,  7.02793077e-02,  1.81611702e-01,  9.68229324e-02,
-7.82167837e-02, -3.84227484e-02, -1.52281094e-02,  4.95393947e-03,
9.92308632e-02,  1.94865523e-03, -1.63112834e-01, -1.09934323e-01,
-4.04280759e-02,  9.39346105e-02,  3.74589451e-02,  1.06732868e-01,
3.22796218e-02, -1.57956988e-01, -8.08086023e-02,  3.90712544e-02,
4.43699136e-02,  2.40078364e-02,  3.67829688e-02, -3.51867415e-02,
-5.00396229e-02, -1.72878355e-02,  5.45219108e-02,  1.14026070e-01,
-2.27987655e-02, -9.24210176e-02, -8.11664760e-02,  7.32476637e-02,
5.51053695e-02,  2.46153474e-02,  5.03601646e-03,  5.21797761e-02,
-1.71612844e-01,  5.53996861e-02,  7.05229416e-02,  2.15468984e-02,
1.30488453e-02,  2.09494568e-02,  6.01171926e-02, -7.55025148e-02,
-5.19532189e-02,  1.38722928e-02,  1.57764275e-02,  6.24198653e-02,
5.79645345e-03,  4.12042104e-02, -9.14558843e-02, -7.97056779e-02,
1.01935193e-01,  1.41003042e-01, -8.40005875e-02, -6.40641227e-02,
-5.17347418e-02,  3.95253077e-02,  8.91676173e-02,  9.60582569e-02,
6.05722219e-02, -1.06982030e-01,  3.82130258e-02,  9.22477897e-03,
2.69806646e-02,  4.25643511e-02,  9.14719030e-02, -7.05534369e-02,
-6.61182478e-02,  4.39233258e-02,  1.30742043e-02,  7.70403072e-02,
-2.52834726e-02, -3.23755145e-02, -3.68847772e-02, -1.01946918e-02,
1.63373679e-01,  5.16280085e-02,  4.68082167e-02, -5.99302687e-02],
dtype=float32)