In [1]:
import math
import numpy as np
import matplotlib.pyplot as plt
In [2]:
mnist_images = np.load("mnist_images.npy")
In [3]:
def plot_mnist(remove_border=False):
    """ Plot the first 40 data points in the MNIST train_images. """
    import matplotlib.pyplot as plt
    plt.figure(figsize=(10, 5))
    for i in range(4 * 10):
        plt.subplot(4, 10, i+1)
        if remove_border:
            plt.imshow(mnist_images[i,4:24,4:24])
        else:
            plt.imshow(mnist_images[i])
            
plot_mnist() # please comment out this line before submission
In [4]:
mnist_images.shape
Out[4]:
(5000, 28, 28)
In [5]:
A = mnist_images.reshape([5000, 28*28])
In [6]:
A.shape # m = 5000, n = 784
Out[6]:
(5000, 784)
In [8]:
avg = np.mean(A, axis=0)
avg.shape
Out[8]:
(784,)
In [9]:
A = A - avg # centering
In [10]:
A.shape
Out[10]:
(5000, 784)
In [13]:
u, s, v = np.linalg.svd(A) # A = U * S * V
In [14]:
u.shape
Out[14]:
(5000, 5000)
In [15]:
v.shape
Out[15]:
(784, 784)
In [16]:
s.shape  # S should be [5000, 784], but S is diagonal matrix
Out[16]:
(784,)
In [17]:
plt.plot(s) # s contains the singular values (or sqrt(eigenvalues) of A^T A)
Out[17]:
[<matplotlib.lines.Line2D at 0x1208ddfd0>]
In [18]:
for i in range(10):
    plt.figure()
    plt.imshow(v[i].reshape(28, 28)) # the first column of V --- eigenvector corresponding to the largest eigenvalue
In [19]:
# Demonstrate power iteration on 

M = A.T @ A
In [20]:
x = np.random.randn(784)
In [21]:
for i in range(1000):
    x = M @ x
    x = x / np.linalg.norm(x)
In [22]:
plt.imshow(x.reshape(28, 28))
Out[22]:
<matplotlib.image.AxesImage at 0x12a1d25f8>
In [24]:
plt.imshow((A[0]).reshape(28, 28))
Out[24]:
<matplotlib.image.AxesImage at 0x12a688dd8>
In [28]:
n = 400
recon = (u[:, :n] @ np.diag(s[:n])) @ v[:n, :]
plt.imshow(recon[0].reshape(28, 28))
Out[28]:
<matplotlib.image.AxesImage at 0x129eed518>
In [29]:
v[:n, :].shape
Out[29]:
(400, 784)
In [35]:
(u[0, :n] @ np.diag(s[:n]))
Out[35]:
array([-4.55478549e-01,  1.44640517e+00, -1.26561061e-01,  2.04640698e+00,
       -9.06210542e-01,  8.65823507e-01, -7.14304328e-01,  3.84281516e-01,
       -1.21694946e+00,  5.73642433e-01, -2.23285139e-01,  1.81757104e+00,
       -1.06964767e+00,  2.88461351e+00, -4.34622541e-02,  4.05452549e-01,
       -1.35171783e+00,  4.88097101e-01, -7.43918598e-01,  1.25593567e+00,
       -4.90995705e-01,  8.97119701e-01, -7.15669572e-01,  7.82275379e-01,
       -2.47401431e-01, -3.64219040e-01,  1.31172872e+00,  4.36892539e-01,
       -4.51465756e-01, -1.55578479e-01,  5.90545356e-01, -3.17717791e-02,
        8.79400969e-02, -1.54493809e-01,  1.75711513e-01, -4.43359107e-01,
        7.23667562e-01, -3.56095225e-01,  4.43184942e-01, -1.71119118e+00,
        2.67344654e-01, -8.43908191e-01,  3.89415354e-01, -1.52835354e-01,
        4.40391488e-02, -5.12474000e-01,  6.19408190e-01, -1.96817964e-01,
        2.10924506e-01, -3.99448365e-01,  3.10658127e-01,  9.06638801e-02,
       -1.36124969e-01,  5.92600286e-01,  1.47323245e-02, -7.16122448e-01,
        4.96965021e-01, -6.16328359e-01,  2.88413376e-01,  1.62665337e-01,
        5.68551838e-01, -1.27703428e-01, -6.54533744e-01,  3.40654045e-01,
        7.81732321e-01, -2.58569598e-01, -5.66779934e-02, -2.48831183e-01,
        3.31237316e-01,  1.06547207e-01,  3.51816006e-02, -6.30600512e-01,
       -1.24257179e-02, -2.62386829e-01,  3.62188846e-01,  2.90017456e-01,
        2.75611989e-02, -2.36386776e-01, -1.23775043e-01, -3.04301269e-02,
        7.01519325e-02,  2.13355035e-01,  9.42880735e-02,  7.14100227e-02,
        4.73580062e-01,  3.60519886e-01, -2.18343467e-01,  1.69368550e-01,
       -2.33022839e-01,  1.07285976e-01, -2.17119649e-01,  6.78040460e-02,
       -4.30201031e-02,  2.50437975e-01, -2.45309100e-02,  1.63109243e-01,
       -1.37702674e-02,  2.23139077e-01, -3.50619048e-01,  3.14214528e-02,
       -5.39187491e-01,  5.18538579e-02, -2.18868479e-01,  1.75770670e-01,
       -3.59771587e-02,  4.88370098e-02, -3.46072644e-01,  7.16489628e-02,
       -4.66125039e-03, -1.11945234e-01,  5.54688752e-01, -1.79587856e-01,
        3.16594318e-02, -1.85787767e-01, -1.35546410e-03, -7.39166066e-02,
        3.02264988e-01, -3.13909531e-01, -1.69744372e-01,  2.18096718e-01,
        1.04771867e-01, -2.08010301e-01,  2.62085423e-02,  2.72359140e-02,
        5.01496494e-02, -1.43721446e-01, -3.81731950e-02, -2.08706334e-01,
        2.15603828e-01,  1.22503052e-02, -6.08255386e-01, -3.89668196e-02,
       -1.34328842e-01,  5.69594562e-01, -9.13841836e-03, -2.49004766e-01,
        8.87687132e-02,  3.51987034e-02,  2.54112720e-01, -3.50831561e-02,
       -2.62510091e-01,  9.73916426e-02,  1.93114638e-01,  1.62793472e-02,
       -1.44680038e-01, -7.88379312e-02, -4.08733398e-01,  2.24472880e-02,
        1.95195764e-01, -3.43082666e-01, -3.69196087e-02, -8.19294080e-02,
       -1.93483397e-01,  5.59696071e-02,  7.18133077e-02,  1.64189469e-02,
        2.42368698e-01, -1.51778460e-01, -7.35573098e-02, -3.47919166e-02,
       -8.98796320e-02, -8.77071396e-02,  2.44459286e-01, -2.33712327e-02,
       -5.56992367e-02, -1.30236998e-01, -2.08050191e-01, -3.10852528e-01,
        1.63734332e-01,  9.60184038e-02,  4.55049388e-02,  2.67423004e-01,
       -1.29410043e-01, -1.10898033e-01, -7.39086494e-02, -2.70314887e-02,
       -2.18479723e-01,  1.79210454e-02,  6.89992728e-03,  1.07667364e-01,
        1.95189081e-02,  1.03811920e-01,  2.08852395e-01, -1.37464181e-01,
       -9.91296545e-02,  5.65454438e-02, -4.66156900e-02,  3.03496212e-01,
       -7.82873258e-02,  6.43257722e-02, -1.62533432e-01,  1.24135368e-01,
       -2.44762287e-01,  1.73622414e-01, -2.44327828e-01,  2.22025976e-01,
       -2.49803409e-01,  4.13745008e-02, -2.12995633e-01,  5.77087887e-02,
       -9.38558877e-02,  2.28245616e-01, -8.89916420e-02,  1.53294697e-01,
       -1.00310281e-01,  1.14232361e-01, -4.26537603e-01,  1.94365993e-01,
       -9.25412774e-03,  1.06941551e-01, -6.11606687e-02,  2.94950586e-02,
       -2.58278757e-01,  2.02425253e-02, -2.39550039e-01, -5.27334958e-02,
        8.03919435e-02, -1.59946457e-01,  3.71983349e-01, -2.17928216e-01,
        3.66459265e-02,  7.63956085e-02, -5.36324531e-02,  2.28669018e-01,
        6.27254993e-02, -7.07053915e-02, -6.07564934e-02,  6.76745549e-02,
       -6.13726955e-03, -9.34195071e-02,  5.21260239e-02, -1.91808909e-01,
       -1.67751759e-01,  1.96181506e-01, -8.61930475e-02, -1.83737591e-01,
        2.40035504e-01, -2.96758525e-02, -1.89198047e-01,  4.98696081e-02,
        6.31644279e-02, -8.82010758e-02, -1.31948115e-02,  5.56295700e-02,
        2.47956370e-03, -1.51182339e-01,  1.32172080e-02,  1.06509767e-01,
       -9.67905000e-02, -1.11154221e-01,  4.53621615e-03,  2.08627045e-01,
       -6.93301717e-03, -1.12499772e-02, -1.20006561e-01,  4.61504497e-02,
        4.55421321e-02, -9.95150656e-02, -1.40243601e-02,  2.80448884e-01,
        6.33695871e-02, -5.04122749e-02, -3.79053280e-02, -1.19889513e-01,
        5.82724102e-02,  8.00075661e-03, -1.36183664e-01, -1.63550712e-02,
       -1.99281517e-03, -1.61598995e-02,  1.98819712e-01, -8.12733173e-02,
       -3.58387269e-02, -1.32877588e-01,  1.16975158e-01,  1.24356128e-01,
       -1.14533253e-01, -2.20104456e-01,  1.47130340e-01,  1.50522411e-01,
       -7.66807795e-02, -2.43504774e-02, -5.40896133e-02,  7.16419592e-02,
        2.17506494e-02,  5.76827191e-02, -1.03735151e-02, -6.24322146e-02,
        5.27784154e-02,  2.98679322e-02, -4.66275513e-02, -4.74482067e-02,
       -8.51547122e-02, -3.20085846e-02,  2.53157038e-02,  2.71272231e-02,
        1.10508978e-01, -4.88457456e-02,  1.38821872e-02,  2.25312542e-02,
        1.55577846e-02,  8.00202191e-02,  1.17258057e-02, -1.46938965e-01,
        5.79218753e-02,  3.96120623e-02,  1.54027296e-02, -1.51540250e-01,
       -1.15365304e-01,  1.22980408e-01,  6.69630691e-02, -5.78529201e-02,
       -3.53047363e-02, -3.23274173e-02, -1.65950153e-02, -9.15654376e-03,
       -3.24388109e-02,  2.13499814e-01,  7.90050924e-02, -2.47634962e-01,
       -3.57196964e-02,  7.02793077e-02,  1.81611702e-01,  9.68229324e-02,
       -7.82167837e-02, -3.84227484e-02, -1.52281094e-02,  4.95393947e-03,
        9.92308632e-02,  1.94865523e-03, -1.63112834e-01, -1.09934323e-01,
       -4.04280759e-02,  9.39346105e-02,  3.74589451e-02,  1.06732868e-01,
        3.22796218e-02, -1.57956988e-01, -8.08086023e-02,  3.90712544e-02,
        4.43699136e-02,  2.40078364e-02,  3.67829688e-02, -3.51867415e-02,
       -5.00396229e-02, -1.72878355e-02,  5.45219108e-02,  1.14026070e-01,
       -2.27987655e-02, -9.24210176e-02, -8.11664760e-02,  7.32476637e-02,
        5.51053695e-02,  2.46153474e-02,  5.03601646e-03,  5.21797761e-02,
       -1.71612844e-01,  5.53996861e-02,  7.05229416e-02,  2.15468984e-02,
        1.30488453e-02,  2.09494568e-02,  6.01171926e-02, -7.55025148e-02,
       -5.19532189e-02,  1.38722928e-02,  1.57764275e-02,  6.24198653e-02,
        5.79645345e-03,  4.12042104e-02, -9.14558843e-02, -7.97056779e-02,
        1.01935193e-01,  1.41003042e-01, -8.40005875e-02, -6.40641227e-02,
       -5.17347418e-02,  3.95253077e-02,  8.91676173e-02,  9.60582569e-02,
        6.05722219e-02, -1.06982030e-01,  3.82130258e-02,  9.22477897e-03,
        2.69806646e-02,  4.25643511e-02,  9.14719030e-02, -7.05534369e-02,
       -6.61182478e-02,  4.39233258e-02,  1.30742043e-02,  7.70403072e-02,
       -2.52834726e-02, -3.23755145e-02, -3.68847772e-02, -1.01946918e-02,
        1.63373679e-01,  5.16280085e-02,  4.68082167e-02, -5.99302687e-02],
      dtype=float32)