← All notebooks

December 2, 2021 6 min read

Hopfield Networks

Numpy implementation of Hopfield Networks.

Hopfield NetworksPython

Table of Contents

• 1  Hopfield Network
  • 1.1  Importing Required Libraries
  • 1.2  Download/Import/Preprocess MNIST Dataset
  • 1.3  Hopfield Network
  • 1.4  Experiments with MNIST Data
  • 1.5  Experiments with Small Patterns

Hopfield Network¶

Importing Required Libraries¶

In [1]:

import matplotlib.pyplot as pltimport numpy as npfrom numpy.random import RandomState

Download/Import/Preprocess MNIST Dataset¶

Mnist Util Code: https://github.com/hsjeong5/MNIST-for-Numpy (modified)

In [2]:

import numpy as npfrom urllib import requestimport gzipimport picklefilename = [["training_images","train-images-idx3-ubyte.gz"],["test_images","t10k-images-idx3-ubyte.gz"],["training_labels","train-labels-idx1-ubyte.gz"],["test_labels","t10k-labels-idx1-ubyte.gz"]]def download_mnist():    base_url = "http://yann.lecun.com/exdb/mnist/"    for name in filename:        print("Downloading "+name[1]+"...")        request.urlretrieve(base_url+name[1], name[1])    print("Download complete.")def save_mnist():    mnist = {}    for name in filename[:2]:        with gzip.open(name[1], 'rb') as f:            mnist[name[0]] = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1,28*28)    for name in filename[-2:]:        with gzip.open(name[1], 'rb') as f:            mnist[name[0]] = np.frombuffer(f.read(), np.uint8, offset=8)    with open("mnist.pkl", 'wb') as f:        pickle.dump(mnist,f)    print("Save complete.")def init_mnist():    download_mnist()    save_mnist()def load_mnist():    with open("mnist.pkl",'rb') as f:        mnist = pickle.load(f)    return mnist["training_images"], mnist["training_labels"], mnist["test_images"], mnist["test_labels"]init_mnist()x_train, t_train, x_test, t_test = load_mnist()x_train = x_train.reshape(-1, 28, 28)x_test = x_test.reshape(-1, 28, 28)idx = 0  # <== SET AN INDEXimg = x_train[idx,:] # First image in the training set.plt.imshow(img,cmap='gray')plt.show() # Show the image

Downloading train-images-idx3-ubyte.gz...
Downloading t10k-images-idx3-ubyte.gz...
Downloading train-labels-idx1-ubyte.gz...
Downloading t10k-labels-idx1-ubyte.gz...
Download complete.
Save complete.

In [3]:

np.random.seed(42)  # <== Changing this value would generate different selections# Pre-process the train setx_train = x_train / np.max(x_train)    # normalizationx_train[x_train>=0.5] = 1              # apply threshold for binarizationx_train[x_train<0.5] = -1              # apply threshold for binarization# Shuffle the train setshuffle_mask = np.arange( len(t_train) )np.random.shuffle(shuffle_mask)x_train = x_train[shuffle_mask]t_train = t_train[shuffle_mask]# Select 10 different numbers from the train setuniq, uniq_idx = np.unique(t_train, return_index=True)numbers = x_train[uniq_idx]f, axs = plt.subplots(2, 5)for i, ax in enumerate(axs.flatten()):    ax.imshow(numbers[i], cmap="gray")    ax.set_title("Value = "+ str(i))f.tight_layout()

In [4]:

# Add noise to samplesnoise = 100noisy_numbers = numbers.copy()for number in noisy_numbers:    mask = np.ones((28,28))    for i in range(noise):        r,c = np.random.randint(0,28, size=2)        number[r,c] *= -1f, axs = plt.subplots(2, 5)for i, ax in enumerate(axs.flatten()):    ax.imshow(noisy_numbers[i], cmap="gray")    ax.set_title("Value = "+ str(i))f.tight_layout()

Hopfield Network¶

Note: Threshold parameter can be set as an integer, a float or a vector.

In [5]:

class HopfieldNetwork:    def __init__(self, k, threshold=0):        self.k = k        self.threshold = threshold    def calculate_energy(self, x):        return -0.5*np.sum(np.matmul(x.T, x)*self.w) + np.sum(x*self.threshold)    def train(self, data):        self.w = np.zeros((self.k, self.k))        for x in data:            self.w += self.calculate_w_per_sample(x)        #print("w:", self.w)    def activation(self, z):        if z > self.threshold:            return 1        return -1    def predict(self, x):   #update        x = x.copy()        x = x.reshape(1, -1)        #async update        max_epoch = 1        counter = 0 # counter or not changed states        for e in range(max_epoch):            if (counter > self.k):                break            order = np.arange(self.k)            np.random.shuffle(order)            for i in order:                x_i = x @ self.w[i]                x_i_a = self.activation(x_i)                                if (x_i_a != x[0,i]):                    counter += 1                    x[0,i] = x_i_a                    if (self.k < 101):                        print("State:", x)                    else:                        print("Energy:", self.calculate_energy(x))        return x            def calculate_w_per_sample(self, x):        x = x.reshape(1, -1)        n = x.shape[1]        w = np.matmul(x.T, x) * (1 - np.eye(n))        return wdata = np.array([ [1,1,1,1,-1,-1,-1], [-1,1,-1,1,-1,1,-1], [-1,-1,1,1,-1,1,-1] ])print("Printing only changed states...")hn = HopfieldNetwork(7)hn.train(data)final_state = hn.predict(np.array([-1,-1,-1,1,-1,-1,-1]))print("final state:", final_state)

Printing only changed states...
State: [[-1  1 -1  1 -1 -1 -1]]
State: [[-1  1  1  1 -1 -1 -1]]
State: [[-1  1  1  1 -1  1 -1]]
final state: [[-1  1  1  1 -1  1 -1]]

Experiments with MNIST Data¶

In [6]:

hn = HopfieldNetwork(28*28)hn.train(numbers[[0,1]])f, axs = plt.subplots(1, 2)axs[0].imshow(numbers[0], cmap="gray")axs[0].set_title("Train Pattern 0")axs[1].imshow(numbers[1], cmap="gray")axs[1].set_title("Train Pattern 1")f.tight_layout()

In [7]:

f, axs = plt.subplots(1, 2)out = hn.predict(noisy_numbers[0]).reshape(28, 28)axs[0].imshow(noisy_numbers[0], cmap="gray")axs[0].set_title("Input")axs[1].imshow(out, cmap="gray")axs[1].set_title("Output")f.tight_layout()

Energy: -243386.0
Energy: -245298.0
Energy: -245818.0
Energy: -246346.0
Energy: -248266.0
Energy: -250194.0
Energy: -252130.0
Energy: -254074.0
Energy: -256026.0
Energy: -257986.0
Energy: -259954.0
Energy: -261930.0
Energy: -263914.0
Energy: -265906.0
Energy: -267906.0
Energy: -269914.0
Energy: -271930.0
Energy: -273954.0
Energy: -275986.0
Energy: -276522.0
Energy: -278562.0
Energy: -280610.0
Energy: -282666.0
Energy: -284730.0
Energy: -286802.0
Energy: -288882.0
Energy: -290970.0
Energy: -291514.0
Energy: -292066.0
Energy: -294162.0
Energy: -294722.0
Energy: -295290.0
Energy: -297394.0
Energy: -297970.0
Energy: -300082.0
Energy: -302202.0
Energy: -304330.0
Energy: -306466.0
Energy: -308610.0
Energy: -309194.0
Energy: -309786.0
Energy: -311938.0
Energy: -314098.0
Energy: -316266.0
Energy: -318442.0
Energy: -320626.0
Energy: -322818.0
Energy: -325018.0
Energy: -327226.0
Energy: -329442.0
Energy: -331666.0
Energy: -333898.0
Energy: -334498.0
Energy: -336738.0
Energy: -338986.0
Energy: -341242.0
Energy: -343506.0
Energy: -345778.0
Energy: -348058.0
Energy: -348666.0
Energy: -350954.0
Energy: -353250.0
Energy: -355554.0
Energy: -357866.0
Energy: -360186.0
Energy: -362514.0
Energy: -364850.0
Energy: -367194.0
Energy: -369546.0
Energy: -371906.0
Energy: -374274.0
Energy: -374890.0
Energy: -377266.0
Energy: -379650.0
Energy: -382042.0
Energy: -384442.0
Energy: -386850.0
Energy: -389266.0
Energy: -391690.0
Energy: -394122.0
Energy: -396562.0
Energy: -399010.0
Energy: -401466.0
Energy: -403930.0
Energy: -406402.0
Energy: -407026.0
Energy: -409506.0
Energy: -411994.0
Energy: -412626.0
Energy: -415122.0

In [8]:

f, axs = plt.subplots(1, 2)out = hn.predict(noisy_numbers[1]).reshape(28, 28)axs[0].imshow(noisy_numbers[1], cmap="gray")axs[0].set_title("Input")axs[1].imshow(out, cmap="gray")axs[1].set_title("Output")f.tight_layout()

Energy: -253522.0
Energy: -255490.0
Energy: -257466.0
Energy: -259450.0
Energy: -259914.0
Energy: -261906.0
Energy: -263906.0
Energy: -265914.0
Energy: -267930.0
Energy: -268402.0
Energy: -270426.0
Energy: -272458.0
Energy: -274498.0
Energy: -276546.0
Energy: -278602.0
Energy: -280666.0
Energy: -281146.0
Energy: -281634.0
Energy: -283706.0
Energy: -285786.0
Energy: -287874.0
Energy: -289970.0
Energy: -292074.0
Energy: -294186.0
Energy: -296306.0
Energy: -296802.0
Energy: -298930.0
Energy: -301066.0
Energy: -301570.0
Energy: -303714.0
Energy: -304226.0
Energy: -306378.0
Energy: -308538.0
Energy: -309058.0
Energy: -311226.0
Energy: -313402.0
Energy: -315586.0
Energy: -317778.0
Energy: -319978.0
Energy: -322186.0
Energy: -322714.0
Energy: -324930.0
Energy: -327154.0
Energy: -329386.0
Energy: -331626.0
Energy: -333874.0
Energy: -334410.0
Energy: -336666.0
Energy: -337210.0
Energy: -339474.0
Energy: -341746.0
Energy: -344026.0
Energy: -346314.0
Energy: -348610.0
Energy: -349162.0
Energy: -351466.0
Energy: -352026.0
Energy: -354338.0
Energy: -356658.0
Energy: -358986.0
Energy: -361322.0
Energy: -361890.0
Energy: -364234.0
Energy: -364810.0
Energy: -367162.0
Energy: -367746.0
Energy: -370106.0
Energy: -372474.0
Energy: -374850.0
Energy: -377234.0
Energy: -379626.0
Energy: -380218.0
Energy: -382618.0
Energy: -385026.0
Energy: -387442.0
Energy: -389866.0
Energy: -392298.0
Energy: -392898.0
Energy: -393506.0
Energy: -395946.0
Energy: -398394.0
Energy: -400850.0
Energy: -401466.0
Energy: -403930.0
Energy: -406402.0
Energy: -408882.0
Energy: -409506.0
Energy: -411994.0
Energy: -412626.0
Energy: -415122.0

Note: Unfortunately I was unable to train the model with more than 2 images.

Experiments with Small Patterns¶

In [9]:

sign_T = np.array([[1,1,1],[-1,1,-1],[-1,1,-1]])sign_plus = np.array([[-1,1,-1], [1,1,1], [-1,1,-1]])sign_x = np.array([[1,-1,1],[-1,1,-1],[1,-1,1]])data = np.array([sign_T ,sign_plus, sign_x]).reshape(-1,9)hn = HopfieldNetwork(9)hn.train(data)f, axs = plt.subplots(1, 3)axs[0].imshow(sign_T, cmap="gray")axs[0].set_title("data[0]")axs[1].imshow(sign_plus, cmap="gray")axs[1].set_title("data[1]")axs[2].imshow(sign_x, cmap="gray")axs[2].set_title("data[2]")f.tight_layout()

In [10]:

# TEST-1test_data = np.array([[1,-1,1],[-1,-1,-1],[-1,-1,1]])final_state = hn.predict(test_data)print("final state:", final_state)f, axs = plt.subplots(1, 2)axs[0].imshow(test_data, cmap="gray")axs[0].set_title("Input")axs[1].imshow(final_state.reshape(3,3), cmap="gray")axs[1].set_title("Output")f.tight_layout()

State: [[ 1 -1  1 -1  1 -1 -1 -1  1]]
State: [[ 1 -1  1 -1  1 -1  1 -1  1]]
final state: [[ 1 -1  1 -1  1 -1  1 -1  1]]

In [11]:

# TEST-2test_data = np.array([[1,1,1],[1,1,1],[1,1,1]])final_state = hn.predict(test_data)print("final state:", final_state)f, axs = plt.subplots(1, 2)axs[0].imshow(test_data, cmap="gray")axs[0].set_title("Input")axs[1].imshow(final_state.reshape(3,3), cmap="gray")axs[1].set_title("Output")f.tight_layout()

State: [[ 1  1  1  1  1  1  1 -1  1]]
State: [[-1  1  1  1  1  1  1 -1  1]]
State: [[-1  1  1  1  1  1 -1 -1  1]]
State: [[-1  1  1  1 -1  1 -1 -1  1]]
State: [[-1 -1  1  1 -1  1 -1 -1  1]]
State: [[-1 -1 -1  1 -1  1 -1 -1  1]]
final state: [[-1 -1 -1  1 -1  1 -1 -1  1]]

In [12]:

# TEST-2test_data = np.array([[1,1,1],[-1,1,1],[-1,-1,-1]])final_state = hn.predict(test_data)print("final state:", final_state)f, axs = plt.subplots(1, 2)axs[0].imshow(test_data, cmap="gray")axs[0].set_title("Input")axs[1].imshow(final_state.reshape(3,3), cmap="gray")axs[1].set_title("Output")f.tight_layout()

State: [[ 1  1  1 -1  1  1 -1  1 -1]]
State: [[ 1  1  1 -1  1 -1 -1  1 -1]]
final state: [[ 1  1  1 -1  1 -1 -1  1 -1]]

Note: This model usually works OK but with some inputs (e.g. all -1) can output unrelated patterns.