Neural Networks For Your Dog - 2.2 Perceptron Model

2021-02-01 97 words One minute

Contents

2.2 Perceptron Model

In this lecture, we’ll discuss and code up a Perceptron - a linear binary classifier created by Frank Rosenblatt in 1958.

Code

import numpy as np
import pandas as pd
np.set_printoptions(suppress=True, linewidth=999)
#=== Perceptron class ============================================================================================
class Perceptron():
    Perceptron model for predicting binary target with random guessing fit() procedure
    def __init__(self, w = None, b = None, y_classes = None):
        self.w = w
        self.b = b
        self.y_classes = y_classes
    def fit(self, X, y, MAXGUESSES = 100_000, seed = None):
        """
        Randomly guess hyperplanes until we get one that separates the data,
        or until we exhaust our guesses
        :param X: 2-D array with >= 1 column of real-valued features
        :param y: 1-D array of labels; should have two distinct classes
        :param MAXGUESSES: how many times to guess before we give up
        :param seed: optional random seed
        :return: None; set self.y_clsses, self.w and self.b if a separating hyperplane is found
        """
        # Validate X dimensionality
        if X.ndim != 2:
            raise AssertionError(f"X should have 2 dimensions but it has {X.ndim}")
        # Determine/validate y_classes
        y_classes = np.unique(y)
        if len(y_classes) != 2:
            AssertionError(f"y should have 2 distinct classes, but instead it has {len(y_classes)}")
        # Convert y to 1-d array of {0, 1} where 0 represents class y_classes[0] and 1 represents y_classes[1]
        y01 = (y == y_classes[1]).astype('int64')
        # Set up a random number generator
        gen = np.random.default_rng(seed=seed)
        # In order to guess hyperplanes that have a reasonable chance of separating the data, we
        # 1) Guess random weights in the range [-1000, 1000]
        # 2) Identify a bounding box around the data in X
        # 3) Pick a random point P in the bounding box
        # 4) Calculate b such that the hyper-plane goes passes through it
        # Repeat until we either find a separating hyperplane or we're tired of guessing
        for i in range(MAXGUESSES):
            # Determine X bounds (bounding box)
            X_mins = X.min(axis=0)
            X_maxs = X.max(axis=0)
            # Guess weights
            w = gen.uniform(low=-1000, high=1000, size=X.shape[1])
            # Calculate b such that hyperplane goes through a random point inside X's bounding box
            P = gen.uniform(low=X_mins, high=X_maxs)
            b = -P.dot(w)
            # Check if the hyperplane separates the data
            yhat = (np.sign(X.dot(w) + b) + 1) / 2
            if (np.all(yhat == y01)):
                break
        # Check outcome based on i
        if i == (MAXGUESSES - 1):
            print("Out of guesses. Maybe this data isn't linearly separable..?")
        else:
            print(f"Found a separating hyperplane in {i + 1} guesses!")
            self.w = w
            self.b = b
            self.y_classes = y_classes
    def predict(self, X):
        """
        Predict on X using this object's w and b.
        If w•x + b > 0 we predict y_classes[1], otherwise we predict y_classes[0]
        :param X: 2-D array with >= 1 column of real-valued features
        :return: 1-D array of predicted class labels
        """
        if self.w is None:
            raise AssertionError(f"Need to fit() a before predict()")
        if X.ndim != 2:
            raise AssertionError(f"X should have 2 dimensions but it has {X.ndim}")
        if X.shape[1] != len(self.w):
            raise AssertionError(f"Perceptron was fit on X with {len(self.w)} columns but this X has {X.shape[1]} columns")
        yhat = (X.dot(self.w) + self.b > 0).astype('int64')
        preds = self.y_classes[yhat]
        return preds

view raw 2.2%20perceptron_model.py delivered with ❤ by emgithub

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
#=== Helpers ============================================================================================
def plot_dataset(X, y, w=None, b=None):
    Plot X, y data
    :param X: 2-D array of features (with 1, 2, or 3 columns)
    :param y: 1-D array of lables
    :return: Axes object
    colors = ListedColormap(['r', 'b', 'g'])
    if X.shape[1] == 1:
        scatter = plt.scatter(X[:, 0], np.repeat(0, X.size), c=y, cmap=colors)
        if w is not None and b is not None:
            x1 = -b/w[-1]
            plt.scatter(x1, 0, marker = 'x')
    elif X.shape[1] == 2:
        scatter = plt.scatter(X[:, 0], X[:, 1], c=y, cmap=colors)
        if w is not None and b is not None:
            x1 = np.array([0, 1])
            x2 = -(w[0] * x1 + b)/w[-1]
            plt.axline(xy1=(x1[0], x2[0]), xy2=(x1[1], x2[1]))
    elif X.shape[1] == 3:
        fig = plt.figure()
        ax = fig.add_subplot(1,1,1, projection='3d')
        scatter = ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=y, cmap=colors)
        if w is not None and b is not None:
            x1 = X[:, 0]
            x2 = X[:, 1]
            x3 = -(X[:, [0, 1]].dot(w[[0, 1]]) + b)/w[-1]
            ax.plot_trisurf(x1, x2, x3)
        raise AssertionError("Can't plot data with >3 dimensions")
    # insert legend
    plt.legend(*scatter.legend_elements())
    return plt.gca()
#=== Challenge ============================================================================================
def guess_hyperplane(X, y, MAXGUESSES=100_000):
    Given a dataset of features X and binary labels y which we assume to be linearly separable,
    guess random hyperplanes until we get one that separates the data.
    :param X: 2-D array with >= 1 column of real-valued features
    :param y: 1-D array of labels in {0, 1}
    :param MAXGUESSES: how many times to guess before we give up
    :return: tuple of (w, b) where w is a 1-D array of weights and b is an offset
    ### YOUR CODE HERE ###
#=== Test ============================================================================================
### 1-D Test
df1 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_1d.csv')
X, y = df1.drop(columns='y').to_numpy(), df1.y.to_numpy()
w, b = guess_hyperplane(X, y)
plot_dataset(X, y, w, b)
### 2-D Test
df2 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_2d.csv')
X, y = df2.drop(columns='y').to_numpy(), df2.y.to_numpy()
w, b = guess_hyperplane(X, y)
plot_dataset(X, y, w, b)
### 3-D Test
df3 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_3d.csv')
X, y = df3.drop(columns='y').to_numpy(), df3.y.to_numpy()
w, b = guess_hyperplane(X, y)
plot_dataset(X, y, w, b)
### 99-D Test
df99 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_99d.csv')
X, y = df99.drop(columns='y').to_numpy(), df99.y.to_numpy()
w, b = guess_hyperplane(X, y)

view raw 2.2%20challenge.py delivered with ❤ by emgithub

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
#=== Helpers ============================================================================================
def plot_dataset(X, y, w=None, b=None):
    Plot X, y data
    :param X: 2-D array of features (with 1, 2, or 3 columns)
    :param y: 1-D array of lables
    :return: Axes object
    colors = ListedColormap(['r', 'b', 'g'])
    if X.shape[1] == 1:
        scatter = plt.scatter(X[:, 0], np.repeat(0, X.size), c=y, cmap=colors)
        if w is not None and b is not None:
            x1 = -b/w[-1]
            plt.scatter(x1, 0, marker = 'x')
    elif X.shape[1] == 2:
        scatter = plt.scatter(X[:, 0], X[:, 1], c=y, cmap=colors)
        if w is not None and b is not None:
            x1 = np.array([0, 1])
            x2 = -(w[0] * x1 + b)/w[-1]
            plt.axline(xy1=(x1[0], x2[0]), xy2=(x1[1], x2[1]))
    elif X.shape[1] == 3:
        fig = plt.figure()
        ax = fig.add_subplot(1,1,1, projection='3d')
        scatter = ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=y, cmap=colors)
        if w is not None and b is not None:
            x1 = X[:, 0]
            x2 = X[:, 1]
            x3 = -(X[:, [0, 1]].dot(w[[0, 1]]) + b)/w[-1]
            ax.plot_trisurf(x1, x2, x3)
        raise AssertionError("Can't plot data with >3 dimensions")
    # insert legend
    plt.legend(*scatter.legend_elements())
    return plt.gca()
#=== Challenge ============================================================================================
def guess_hyperplane(X, y, MAXGUESSES=100_000):
    Given a dataset of features X and binary labels y which we assume to be linearly separable,
    guess random hyperplanes until we get one that separates the data.
    :param X: 2-D array with >= 1 column of features
    :param y: 1-D array of labels in {0, 1}
    :param MAXGUESSES: how many times to guess before we give up
    :return: tuple of (w, b) where w is a 1-D array of weights and b is an offset
    # Set up a random number generator
    gen = np.random.default_rng()
    # In order to guess hyperplanes that have a reasonable chance of separating the data, we
    # 1) Guess random weights in the range [-1000, 1000]
    # 2) Identify a bounding box around the data in X
    # 3) Pick a random point P in the bounding box
    # 4) Calculate b such that the hyper-plane goes passes through it
    # Repeat until we either find a separating hyperplane or we're tired of guessing
    for i in range(MAXGUESSES):
        # Determine X bounds (bounding box)
        X_mins = X.min(axis=0)
        X_maxs = X.max(axis=0)
        # Guess weights
        w = gen.uniform(low=-1000, high=1000, size=X.shape[1])
        # Calculate b such that hyperplane goes through a random point inside X's bounding box
        P = gen.uniform(low=X_mins, high=X_maxs)
        b = -P.dot(w)
        # Check if the hyperplane separates the data
        yhat = np.sign(X.dot(w) + b)
        yhat = (yhat + 1) / 2  # transform (1, 0, -1) -> (1, 0.5, 0)
        if (np.all(yhat == y)):
            break
    # Check outcome based on i
    if i == (MAXGUESSES - 1):
        print("Out of guesses. Maybe this data isn't linearly separable..?")
        return None
        print(f"Found a separating hyperplane in {i + 1} guesses!")
        return (w, b)
#=== Test ============================================================================================
### 1-D Test
df1 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_1d.csv')
X, y = df1.drop(columns='y').to_numpy(), df1.y.to_numpy()
w, b = guess_hyperplane(X, y)
plot_dataset(X, y, w, b)
### 2-D Test
df2 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_2d.csv')
X, y = df2.drop(columns='y').to_numpy(), df2.y.to_numpy()
w, b = guess_hyperplane(X, y)
plot_dataset(X, y, w, b)
### 3-D Test
df3 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_3d.csv')
X, y = df3.drop(columns='y').to_numpy(), df3.y.to_numpy()
w, b = guess_hyperplane(X, y)
plot_dataset(X, y, w, b)
### 99-D Test
df99 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_99d.csv')
X, y = df99.drop(columns='y').to_numpy(), df99.y.to_numpy()
w, b = guess_hyperplane(X, y)

view raw 2.2%20solution.py delivered with ❤ by emgithub

Course Curriculum

(See the code on GitHub)

Introduction
1.1 Introduction
Perceptron
2.1 MNIST Dataset
2.2 Perceptron Model
2.3 Perceptron Learning Algorithm
2.4 Pocket Algorithm
2.5 Multiclass Support
2.6 Perceptron To Neural Network
Neural Network
3.1 Simple Images
3.2 Random Weights
3.3 Gradient Descent
3.4 Multiclass Support
3.5 Deep Learning
3.6 Stochastic Gradient Descent
3.7 Going Further

	import numpy as np
	import pandas as pd

	np.set_printoptions(suppress=True, linewidth=999)

	#=== Perceptron class ============================================================================================

	class Perceptron():
	"""
	Perceptron model for predicting binary target with random guessing fit() procedure
	"""

	def __init__(self, w = None, b = None, y_classes = None):
	self.w = w
	self.b = b
	self.y_classes = y_classes

	def fit(self, X, y, MAXGUESSES = 100_000, seed = None):
	"""
	Randomly guess hyperplanes until we get one that separates the data,
	or until we exhaust our guesses

	:param X: 2-D array with >= 1 column of real-valued features
	:param y: 1-D array of labels; should have two distinct classes
	:param MAXGUESSES: how many times to guess before we give up
	:param seed: optional random seed
	:return: None; set self.y_clsses, self.w and self.b if a separating hyperplane is found
	"""

	# Validate X dimensionality
	if X.ndim != 2:
	raise AssertionError(f"X should have 2 dimensions but it has {X.ndim}")

	# Determine/validate y_classes
	y_classes = np.unique(y)
	if len(y_classes) != 2:
	AssertionError(f"y should have 2 distinct classes, but instead it has {len(y_classes)}")

	# Convert y to 1-d array of {0, 1} where 0 represents class y_classes[0] and 1 represents y_classes[1]
	y01 = (y == y_classes[1]).astype('int64')

	# Set up a random number generator
	gen = np.random.default_rng(seed=seed)

	# In order to guess hyperplanes that have a reasonable chance of separating the data, we
	# 1) Guess random weights in the range [-1000, 1000]
	# 2) Identify a bounding box around the data in X
	# 3) Pick a random point P in the bounding box
	# 4) Calculate b such that the hyper-plane goes passes through it

	# Repeat until we either find a separating hyperplane or we're tired of guessing
	for i in range(MAXGUESSES):
	# Determine X bounds (bounding box)
	X_mins = X.min(axis=0)
	X_maxs = X.max(axis=0)

	# Guess weights
	w = gen.uniform(low=-1000, high=1000, size=X.shape[1])

	# Calculate b such that hyperplane goes through a random point inside X's bounding box
	P = gen.uniform(low=X_mins, high=X_maxs)
	b = -P.dot(w)

	# Check if the hyperplane separates the data
	yhat = (np.sign(X.dot(w) + b) + 1) / 2
	if (np.all(yhat == y01)):
	break

	# Check outcome based on i
	if i == (MAXGUESSES - 1):
	print("Out of guesses. Maybe this data isn't linearly separable..?")
	else:
	print(f"Found a separating hyperplane in {i + 1} guesses!")
	self.w = w
	self.b = b
	self.y_classes = y_classes


	def predict(self, X):
	"""
	Predict on X using this object's w and b.
	If w•x + b > 0 we predict y_classes[1], otherwise we predict y_classes[0]

	:param X: 2-D array with >= 1 column of real-valued features
	:return: 1-D array of predicted class labels
	"""

	if self.w is None:
	raise AssertionError(f"Need to fit() a before predict()")
	if X.ndim != 2:
	raise AssertionError(f"X should have 2 dimensions but it has {X.ndim}")
	if X.shape[1] != len(self.w):
	raise AssertionError(f"Perceptron was fit on X with {len(self.w)} columns but this X has {X.shape[1]} columns")

	yhat = (X.dot(self.w) + self.b > 0).astype('int64')
	preds = self.y_classes[yhat]

	return preds

	import numpy as np
	import pandas as pd
	import matplotlib.pyplot as plt
	from matplotlib.colors import ListedColormap

	#=== Helpers ============================================================================================

	def plot_dataset(X, y, w=None, b=None):
	"""
	Plot X, y data

	:param X: 2-D array of features (with 1, 2, or 3 columns)
	:param y: 1-D array of lables
	:return: Axes object
	"""

	colors = ListedColormap(['r', 'b', 'g'])

	if X.shape[1] == 1:
	scatter = plt.scatter(X[:, 0], np.repeat(0, X.size), c=y, cmap=colors)

	if w is not None and b is not None:
	x1 = -b/w[-1]
	plt.scatter(x1, 0, marker = 'x')

	elif X.shape[1] == 2:
	scatter = plt.scatter(X[:, 0], X[:, 1], c=y, cmap=colors)

	if w is not None and b is not None:
	x1 = np.array([0, 1])
	x2 = -(w[0] * x1 + b)/w[-1]
	plt.axline(xy1=(x1[0], x2[0]), xy2=(x1[1], x2[1]))

	elif X.shape[1] == 3:
	fig = plt.figure()
	ax = fig.add_subplot(1,1,1, projection='3d')
	scatter = ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=y, cmap=colors)

	if w is not None and b is not None:
	x1 = X[:, 0]
	x2 = X[:, 1]
	x3 = -(X[:, [0, 1]].dot(w[[0, 1]]) + b)/w[-1]
	ax.plot_trisurf(x1, x2, x3)

	else:
	raise AssertionError("Can't plot data with >3 dimensions")

	# insert legend
	plt.legend(*scatter.legend_elements())

	return plt.gca()

	#=== Challenge ============================================================================================

	def guess_hyperplane(X, y, MAXGUESSES=100_000):
	"""
	Given a dataset of features X and binary labels y which we assume to be linearly separable,
	guess random hyperplanes until we get one that separates the data.

	:param X: 2-D array with >= 1 column of real-valued features
	:param y: 1-D array of labels in {0, 1}
	:param MAXGUESSES: how many times to guess before we give up
	:return: tuple of (w, b) where w is a 1-D array of weights and b is an offset
	"""

	### YOUR CODE HERE ###
	pass


	#=== Test ============================================================================================

	### 1-D Test
	df1 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_1d.csv')
	X, y = df1.drop(columns='y').to_numpy(), df1.y.to_numpy()
	w, b = guess_hyperplane(X, y)
	plot_dataset(X, y, w, b)

	### 2-D Test
	df2 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_2d.csv')
	X, y = df2.drop(columns='y').to_numpy(), df2.y.to_numpy()
	w, b = guess_hyperplane(X, y)
	plot_dataset(X, y, w, b)

	### 3-D Test
	df3 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_3d.csv')
	X, y = df3.drop(columns='y').to_numpy(), df3.y.to_numpy()
	w, b = guess_hyperplane(X, y)
	plot_dataset(X, y, w, b)

	### 99-D Test
	df99 = pd.read_csv('https://raw.githubusercontent.com/ben519/nnets-for-your-dog/master/data/separable_data_99d.csv')
	X, y = df99.drop(columns='y').to_numpy(), df99.y.to_numpy()
	w, b = guess_hyperplane(X, y)

Contents

Neural Networks For Your Dog - 2.2 Perceptron Model

2.2 Perceptron Model

Code

Course Curriculum

Additional Content