Self Organizing Map


Reading time: about 25 minutes
In [1]:
get_ipython().ast_node_interactivity = 'all'
import matplotlib.pyplot as plt
import numpy as np
import matplotlib
import math
from sklearn import datasets
import itertools
import random
matplotlib.rcParams['figure.dpi'] = 150

iris = datasets.load_iris()
X = iris.data
Y = iris.target
In [4]:
for x, y in itertools.combinations([0, 1, 2, 3], 2):
    _ = plt.scatter(X[:, x], X[:, y], c=Y)
    _ = plt.figure()
Out:
<Figure size 432x288 with 1 Axes>
Out:
<Figure size 432x288 with 1 Axes>
Out:
<Figure size 432x288 with 1 Axes>
Out:
<Figure size 432x288 with 1 Axes>
Out:
<Figure size 432x288 with 1 Axes>
Out:
<Figure size 432x288 with 1 Axes>
Out:
In [6]:
X
Out [6]:
array([[5.1, 3.5, 1.4, 0.2],
       [4.9, 3. , 1.4, 0.2],
       [4.7, 3.2, 1.3, 0.2],
       [4.6, 3.1, 1.5, 0.2],
       [5. , 3.6, 1.4, 0.2],
       [5.4, 3.9, 1.7, 0.4],
       [4.6, 3.4, 1.4, 0.3],
       [5. , 3.4, 1.5, 0.2],
       [4.4, 2.9, 1.4, 0.2],
       [4.9, 3.1, 1.5, 0.1],
       [5.4, 3.7, 1.5, 0.2],
       [4.8, 3.4, 1.6, 0.2],
       [4.8, 3. , 1.4, 0.1],
       [4.3, 3. , 1.1, 0.1],
       [5.8, 4. , 1.2, 0.2],
       [5.7, 4.4, 1.5, 0.4],
       [5.4, 3.9, 1.3, 0.4],
       [5.1, 3.5, 1.4, 0.3],
       [5.7, 3.8, 1.7, 0.3],
       [5.1, 3.8, 1.5, 0.3],
       [5.4, 3.4, 1.7, 0.2],
       [5.1, 3.7, 1.5, 0.4],
       [4.6, 3.6, 1. , 0.2],
       [5.1, 3.3, 1.7, 0.5],
       [4.8, 3.4, 1.9, 0.2],
       [5. , 3. , 1.6, 0.2],
       [5. , 3.4, 1.6, 0.4],
       [5.2, 3.5, 1.5, 0.2],
       [5.2, 3.4, 1.4, 0.2],
       [4.7, 3.2, 1.6, 0.2],
       [4.8, 3.1, 1.6, 0.2],
       [5.4, 3.4, 1.5, 0.4],
       [5.2, 4.1, 1.5, 0.1],
       [5.5, 4.2, 1.4, 0.2],
       [4.9, 3.1, 1.5, 0.2],
       [5. , 3.2, 1.2, 0.2],
       [5.5, 3.5, 1.3, 0.2],
       [4.9, 3.6, 1.4, 0.1],
       [4.4, 3. , 1.3, 0.2],
       [5.1, 3.4, 1.5, 0.2],
       [5. , 3.5, 1.3, 0.3],
       [4.5, 2.3, 1.3, 0.3],
       [4.4, 3.2, 1.3, 0.2],
       [5. , 3.5, 1.6, 0.6],
       [5.1, 3.8, 1.9, 0.4],
       [4.8, 3. , 1.4, 0.3],
       [5.1, 3.8, 1.6, 0.2],
       [4.6, 3.2, 1.4, 0.2],
       [5.3, 3.7, 1.5, 0.2],
       [5. , 3.3, 1.4, 0.2],
       [7. , 3.2, 4.7, 1.4],
       [6.4, 3.2, 4.5, 1.5],
       [6.9, 3.1, 4.9, 1.5],
       [5.5, 2.3, 4. , 1.3],
       [6.5, 2.8, 4.6, 1.5],
       [5.7, 2.8, 4.5, 1.3],
       [6.3, 3.3, 4.7, 1.6],
       [4.9, 2.4, 3.3, 1. ],
       [6.6, 2.9, 4.6, 1.3],
       [5.2, 2.7, 3.9, 1.4],
       [5. , 2. , 3.5, 1. ],
       [5.9, 3. , 4.2, 1.5],
       [6. , 2.2, 4. , 1. ],
       [6.1, 2.9, 4.7, 1.4],
       [5.6, 2.9, 3.6, 1.3],
       [6.7, 3.1, 4.4, 1.4],
       [5.6, 3. , 4.5, 1.5],
       [5.8, 2.7, 4.1, 1. ],
       [6.2, 2.2, 4.5, 1.5],
       [5.6, 2.5, 3.9, 1.1],
       [5.9, 3.2, 4.8, 1.8],
       [6.1, 2.8, 4. , 1.3],
       [6.3, 2.5, 4.9, 1.5],
       [6.1, 2.8, 4.7, 1.2],
       [6.4, 2.9, 4.3, 1.3],
       [6.6, 3. , 4.4, 1.4],
       [6.8, 2.8, 4.8, 1.4],
       [6.7, 3. , 5. , 1.7],
       [6. , 2.9, 4.5, 1.5],
       [5.7, 2.6, 3.5, 1. ],
       [5.5, 2.4, 3.8, 1.1],
       [5.5, 2.4, 3.7, 1. ],
       [5.8, 2.7, 3.9, 1.2],
       [6. , 2.7, 5.1, 1.6],
       [5.4, 3. , 4.5, 1.5],
       [6. , 3.4, 4.5, 1.6],
       [6.7, 3.1, 4.7, 1.5],
       [6.3, 2.3, 4.4, 1.3],
       [5.6, 3. , 4.1, 1.3],
       [5.5, 2.5, 4. , 1.3],
       [5.5, 2.6, 4.4, 1.2],
       [6.1, 3. , 4.6, 1.4],
       [5.8, 2.6, 4. , 1.2],
       [5. , 2.3, 3.3, 1. ],
       [5.6, 2.7, 4.2, 1.3],
       [5.7, 3. , 4.2, 1.2],
       [5.7, 2.9, 4.2, 1.3],
       [6.2, 2.9, 4.3, 1.3],
       [5.1, 2.5, 3. , 1.1],
       [5.7, 2.8, 4.1, 1.3],
       [6.3, 3.3, 6. , 2.5],
       [5.8, 2.7, 5.1, 1.9],
       [7.1, 3. , 5.9, 2.1],
       [6.3, 2.9, 5.6, 1.8],
       [6.5, 3. , 5.8, 2.2],
       [7.6, 3. , 6.6, 2.1],
       [4.9, 2.5, 4.5, 1.7],
       [7.3, 2.9, 6.3, 1.8],
       [6.7, 2.5, 5.8, 1.8],
       [7.2, 3.6, 6.1, 2.5],
       [6.5, 3.2, 5.1, 2. ],
       [6.4, 2.7, 5.3, 1.9],
       [6.8, 3. , 5.5, 2.1],
       [5.7, 2.5, 5. , 2. ],
       [5.8, 2.8, 5.1, 2.4],
       [6.4, 3.2, 5.3, 2.3],
       [6.5, 3. , 5.5, 1.8],
       [7.7, 3.8, 6.7, 2.2],
       [7.7, 2.6, 6.9, 2.3],
       [6. , 2.2, 5. , 1.5],
       [6.9, 3.2, 5.7, 2.3],
       [5.6, 2.8, 4.9, 2. ],
       [7.7, 2.8, 6.7, 2. ],
       [6.3, 2.7, 4.9, 1.8],
       [6.7, 3.3, 5.7, 2.1],
       [7.2, 3.2, 6. , 1.8],
       [6.2, 2.8, 4.8, 1.8],
       [6.1, 3. , 4.9, 1.8],
       [6.4, 2.8, 5.6, 2.1],
       [7.2, 3. , 5.8, 1.6],
       [7.4, 2.8, 6.1, 1.9],
       [7.9, 3.8, 6.4, 2. ],
       [6.4, 2.8, 5.6, 2.2],
       [6.3, 2.8, 5.1, 1.5],
       [6.1, 2.6, 5.6, 1.4],
       [7.7, 3. , 6.1, 2.3],
       [6.3, 3.4, 5.6, 2.4],
       [6.4, 3.1, 5.5, 1.8],
       [6. , 3. , 4.8, 1.8],
       [6.9, 3.1, 5.4, 2.1],
       [6.7, 3.1, 5.6, 2.4],
       [6.9, 3.1, 5.1, 2.3],
       [5.8, 2.7, 5.1, 1.9],
       [6.8, 3.2, 5.9, 2.3],
       [6.7, 3.3, 5.7, 2.5],
       [6.7, 3. , 5.2, 2.3],
       [6.3, 2.5, 5. , 1.9],
       [6.5, 3. , 5.2, 2. ],
       [6.2, 3.4, 5.4, 2.3],
       [5.9, 3. , 5.1, 1.8]])
In [2]:
#!/usr/bin/env python3
import math
from collections import namedtuple
import random
import numpy as np
np.set_printoptions(precision=3, suppress=True)


# Value

class Value:
    __slots__ = ("value", "grad", "parents")

    def __init__(self, v, g = 0.0, p = tuple()):
        self.value = v
        self.grad = g
        self.parents = p

    def __repr__(self):
        return f"Value({self.value}, grad={self.grad})"

    def __float__(self):
        return self.value

    def backprop(self, bp):
        self.grad += bp
        for parent, grad in self.parents:
            parent.backprop(grad * bp)

    def backward(self):
        return self.backprop(1.0)

    def __add__(self, other):
        v = self.value + other.value
        g = 0.0
        p = ((self, 1.0), (other, 1.0))
        return Value(v, g, p)

    def __sub__(self, other):
        v = self.value - other.value
        g = 0.0
        p = ((self, 1.0), (other, -1.0))
        return Value(v, g, p)

    def __neg__(self):
        v = -self.value
        g = 0.0
        p = ((self, -1.0),)
        return Value(v, g, p)

    def __mul__(self, other):
        v = self.value * other.value
        g = 0.0
        p = ((self, other.value), (other, self.value))
        return Value(v, g, p)

    def __truediv__(self, other):
        v = self.value / other.value
        g = 0.0
        p = ((self, 1 / other.value), (other, (-self.value) / (other.value ** 2)))
        return Value(v, g, p)

    def __pow__(self, pow):
        v = self.value ** pow
        g = 0.0
        p = ((self, pow * self.value ** (pow - 1)),)
        return Value(v, g, p)

    def __lt__(self, other):
        return self.value < other.value

    def sqrt(self):
        v = math.sqrt(self.value)
        g = 0.0
        p = ((self, 1 / (2 * v)),)
        return Value(v, g, p)

    def tanh(self):
        v = float(np.tanh(self.value))
        g = 0.0
        p = ((self, 1 - v ** 2),)
        return Value(v, g, p)

    def relu(self):
        v = max(self.value, 0.0)
        g = 0.0
        p = ((self, 1.0 if self.value > 0.0 else 0.0),)
        return Value(v, g, p)

    def logistic_function(self):
        ex = float(np.exp(self.value))
        v = ex / (1 + ex)
        g = 0.0
        p = ((self, v * (1 - v)),)
        return Value(v, g, p)

    def swish1(self):
        x = self.value
        ex = float(np.exp(x))
        e2x = float(np.exp(2 * x))
        v = x * ex / (1 + ex)
        g = 0.0
        d = (x + ex + 1) * ex / (e2x + 2 * ex + 1)
        p = ((self, d),)
        return Value(v, g, p)
In [3]:
 
 
 In [3]:
def dist(v1, v2):
    distsq = Value(0)
    for x, y in zip(v1, v2):
        distsq += (x - y) ** 2
    return distsq.sqrt()
 
 
 In [19]:
x = [Value(1), Value(2), Value(3)]
y = [Value(1), Value(4), Value(4)]

distsq(x, y)
 
 
 Out [19]:
Value(5, grad=0.0)
 
 
 In [4]:
def normalize(x):
    return (x - np.mean(x)) / np.std(x)

for i in range(4):
    X[:, i] = normalize(X[:, i])
 
 
 In [5]:
for x, y in itertools.combinations([0, 1, 2, 3], 2):
    _ = plt.scatter(X[:, x], X[:, y], c=Y)
    _ = plt.figure()
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
 
 
 In [6]:
features = []

for row in range(150):
    v = []
    for i in range(4):
        val = Value(float(X[row][i]))
        v.append(val)
    features.append(v)

features[0]
X = features
 
 
 Out [6]:
[Value(-0.9006811702978088, grad=0.0),
 Value(1.019004351971607, grad=0.0),
 Value(-1.3402265266227624, grad=0.0),
 Value(-1.3154442950077403, grad=0.0)]
 
 
 In []:
gr
 
 
 In [7]:
GRID_SIZE = 10
MAXDIST = np.linalg.norm(np.array([0, 0]) - np.array([GRID_SIZE, GRID_SIZE]))

grid = []

for y in range(GRID_SIZE):
    for x in range(GRID_SIZE):
        v = [Value(random.uniform(-1, 1)) for _ in range(4)]
        grid.append(v)

grid[0]
 
 
 Out [7]:
[Value(0.10152844980007547, grad=0.0),
 Value(-0.5539298274835704, grad=0.0),
 Value(-0.22458789388172984, grad=0.0),
 Value(-0.17667029856877559, grad=0.0)]
 
 
 In [8]:
def index_to_xy(index):
    return index % GRID_SIZE, index // GRID_SIZE

def xy_to_index(x, y):
    return y * GRID_SIZE + x
 
 
 In [8]:
 
 
 In [9]:
def get_best_index(grid, x):
    d = lambda y: dist(x, grid[y])
    return min(list(range(GRID_SIZE * GRID_SIZE)), key=d)

def get_best_xy(grid, vec):
    return index_to_xy(get_best_index(grid, vec))
 
 
 In [16]:
# Train

def train_samp(grid, features, radius):
    bestx, besty = get_best_xy(grid, features)
    bestindex = xy_to_index(bestx, besty)
    bestPos = np.array([bestx, besty])
    
    loss = Value(0)
    
    for y in range(GRID_SIZE):
        for x in range(GRID_SIZE):
            index = xy_to_index(x, y)
            pos = np.array([x, y])
            d = np.linalg.norm(bestPos - pos)
            dw = np.interp(d, (0, MAXDIST), (1, 0))
            if d < radius:
                loss += dist(grid[index], features) * Value(dw)
            else:
                loss -= dist(grid[index], features) * Value(dw)
    
    return loss

maxiter = 10
indices = list(range(len(Y)))

for iteration in range(maxiter):
    random.shuffle(indices)
    radius = np.interp(iteration, (0, maxiter - 1), (MAXDIST, 0))
    loss = Value(0)
    for i, index in enumerate(indices):
        loss += train_samp(grid, X[index], radius)
        print(iteration, i, round(radius, 4), round(loss.value, 4), end="                                \r")
    for v in grid:
        for p in v:
            p.grad = 0
    loss.backward()

    for v in grid:
        for p in v:
            p.value -= p.grad * 0.0001
 
 
 Out:
9 149 0.0 -19842.8487                                   

 
 
 In [11]:
allxs = []
allys = []

for label in range(3):
    xs = []
    ys = []
    labels = []
    for i, features in enumerate(X):
        l = Y[i]
        if l != label:
            continue
        x, y = get_best_xy(grid, features)
        xs.append(x)
        ys.append(y)
        allxs.append(x)
        allys.append(y)
        labels.append(l)
    _ = plt.gca().set_ylim([0, GRID_SIZE - 1])
    _ = plt.gca().set_xlim([0, GRID_SIZE - 1])
    _ = plt.scatter(xs, ys, c=labels)
    _ = plt.figure()
_ = plt.gca().set_xlim([0, GRID_SIZE - 1])
_ = plt.gca().set_ylim([0, GRID_SIZE - 1])
_ = plt.scatter(allxs, allys, c=Y, alpha=0.7)
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
 Out:
<Figure size 432x288 with 1 Axes>
 
 
  
 
The following pages link here
  
  
 
 
 
Citation
 If you find this work useful, please cite it as: 
@article{yaltirakliwikiselforganizingmap,
  title   = "Self Organizing Map",
  author  = "Yaltirakli, Gokberk",
  journal = "gkbrk.com",
  year    = "2024",
  url     = "https://www.gkbrk.com/wiki/self-organizing-map/"
}
 
 Not using BibTeX? Click here for more citation styles. 
 IEEE Citation
 
Gokberk Yaltirakli, "Self Organizing Map", November, 2024. [Online]. Available: https://www.gkbrk.com/wiki/self-organizing-map/. [Accessed Nov. 12, 2024].
 
 
 APA Style
 
Yaltirakli, G. (2024, November 12). Self Organizing Map. https://www.gkbrk.com/wiki/self-organizing-map/
 
 
 Bluebook Style
 
Gokberk Yaltirakli, Self Organizing Map, GKBRK.COM (Nov. 12, 2024), https://www.gkbrk.com/wiki/self-organizing-map/
 
 
 
 
 
Comments
 
 
 
   
 
   
   
 
  
 
 
  
  
 
 
 
Search
 
      
 
  
 
 
 
More links
  
    
 
This page is valid HTML. Click to validate.
 
It's not actually valid? Send me an email.
 
    
 
 
 
 
 
© 2024 Gokberk Yaltirakli