Regularized Logistic Regression in Python (Andrew ng Course)

Question

I'm starting the ML journey and I'm having troubles with this coding exercise here is my code

import numpy as np
import pandas as pd
import scipy.optimize as op

# Read the data and give it labels
data = pd.read_csv('ex2data2.txt', header=None, name['Test1', 'Test2', 'Accepted'])

# Separate the features to make it fit into the mapFeature function
X1 = data['Test1'].values.T
X2 = data['Test2'].values.T

# This function makes more features (degree)
def mapFeature(x1, x2):
    degree = 6
    out = np.ones((x1.shape[0], sum(range(degree + 2))))
    curr_column = 1
    for i in range(1, degree + 1):
        for j in range(i+1):
            out[:,curr_column] = np.power(x1, i-j) * np.power(x2, j)
            curr_column += 1
    return out


# Separate the data into training and target, also initialize theta
X = mapFeature(X1, X2)
y = np.matrix(data['Accepted'].values).T
m, n = X.shape
cols = X.shape[1]
theta = np.matrix(np.zeros(cols))

#Initialize the learningRate(sigma)
learningRate = 1


# Define the Sigmoid Function (Output between 0 and 1)
def sigmoid(z):
    return 1 / (1 + np.exp(-z))


def cost(theta, X, y, learningRate):
    # This is require to make the optimize function work
    theta = theta.reshape(-1, 1)
    error = sigmoid(X @ theta)
    first = np.multiply(-y, np.log(error))
    second = np.multiply(1 - y, np.log(1 - error))
    j = np.sum((first - second)) / m + (learningRate * np.sum(np.power(theta, 2)) / 2 * m)
    return j


# Define the gradient of the cost function
def gradient(theta, X, y, learningRate):
    # This is require to make the optimize function work
    theta = theta.reshape(-1, 1)
    error = sigmoid(X @ theta)
    grad =  (X.T @ (error - y)) / m + ((learningRate * theta) / m)
    grad_no = (X.T @ (error - y)) / m
    grad[0] = grad_no[0]
    return grad


Result = op.minimize(fun=cost, x0=theta, args=(X, y, learningRate), method='TNC', jac=gradient)
opt_theta = np.matrix(Result.x)


def predict(theta, X):
    sigValue = sigmoid(X @ theta.T)
    p = sigValue >= 0.5
    return p

p = predict(opt_theta, X)
print('Train Accuracy: {:f}'.format(np.mean(p == y) * 100))

So, when the learningRate = 1, the accuracy should be around 83,05% but I'm getting 80.5% and when the learningRate = 0, the accuracy should be 91.52% but I'm getting 87.28%

So the question is What am I doing wrong? Why my accuracy is below the problem default answer?

Hope someone can guide me in the right direction. Thanks!

P.D: Here is the dataset, maybe it can help

https://raw.githubusercontent.com/TheGirlWhiteWithBandages/Machine-Learning-Algorithms/master/Logistic%20Regression/ex2data2.txt

user_10350726 · Answer

Hey guys I found a way to make it even better! Here is the code

import numpy as np
import pandas as pd
import scipy.optimize as op
from sklearn.preprocessing import PolynomialFeatures

# Read the data and give it labels
data = pd.read_csv('ex2data2.txt', header=None, names=['Test1', 'Test2', 'Accepted'])
# Separate the data into training and target
X = (data.iloc[:, 0:2]).values
y = (data.iloc[:, 2:3]).values
# Modify the features to a certain degree (Polynomial)
poly = PolynomialFeatures(6)
m = y.size
XX = poly.fit_transform(data.iloc[:, 0:2].values)
# Initialize Theta
theta = np.zeros(XX.shape[1])


# Define the Sigmoid Function (Output between 0 and 1)
def sigmoid(z):
    return(1 / (1 + np.exp(-z)))


# Define the Regularized cost function
def costFunctionReg(theta, reg, *args):
    # This is require to make the optimize function work
    h = sigmoid(XX @ theta)
    first = np.log(h).T @ - y
    second = np.log(1 - h).T @ (1 - y)
    J = (1 / m) * (first - second) + (reg / (2 * m)) * np.sum(np.square(theta[1:]))
    return J


# Define the Regularized gradient function
def gradientReg(theta, reg, *args):
    theta = theta.reshape(-1, 1)
    h = sigmoid(XX @ theta)
    grad = (1 / m) * (XX.T @ (h - y)) + (reg / m) * np.r_[[[0]], theta[1:]]
    return grad.flatten()


# Define the predict Function
def predict(theta, X):
    sigValue = sigmoid(X @ theta.T)
    p = sigValue >= 0.5
    return p


# A loop to test between different values for sigma (reg parameter)
for i, Sigma in enumerate([0, 1, 100]):
    # Optimize costFunctionReg 
    res2 = op.minimize(costFunctionReg, theta, args=(Sigma, XX, y), method=None, jac=gradientReg)

    # Get the accuracy of the model
    accuracy = 100 * sum(predict(res2.x, XX) == y.ravel()) / y.size

    # Get the Error between different weights
    error1 = costFunctionReg(res2.x, Sigma, XX, y)

    # print the accuracy and error
    print('Train accuracy {}% with Lambda = {}'.format(np.round(accuracy, decimals=4), Sigma))
    print(error1)

Thanks for all your help!

user_7606121 · Answer

try out this:

# import library
import pandas as pd
import numpy as np
dataset = pd.read_csv('ex2data2.csv',names = ['Test #1','Test #2','Accepted'])


# splitting to x and y variables for features and target variable
x = dataset.iloc[:,:-1].values
y = dataset.iloc[:,-1].values
print('x[0] ={}, y[0] ={}'.format(x[0],y[0]))
m, n = x.shape
print('#{} Number of training samples, #{} features per sample'.format(m,n))


# import library FeatureMapping
from sklearn.preprocessing import PolynomialFeatures

# We also add one column of ones to interpret theta 0 (x with power of 0 = 1) by 
include_bias as True
pf = PolynomialFeatures(degree = 6, include_bias = True)
x_poly = pf.fit_transform(x)
pd.DataFrame(x_poly).head(5)


m,n = x_poly.shape

# define theta as zero
theta = np.zeros(n)

# define hyperparameter λ
lambda_ = 1

# reshape (-1,1) because we just have one feature in y column
y = y.reshape(-1,1)


def sigmoid(z):
    return 1/(1+np.exp(-z))
def lr_hypothesis(x,theta):
    return np.dot(x,theta)

def compute_cost(theta,x,y,lambda_):
    theta = theta.reshape(n,1)
    infunc1 = -y*(np.log(sigmoid(lr_hypothesis(x,theta)))) - ((1-y)*(np.log(1 - sigmoid(lr_hypothesis(x,theta)))))
    infunc2 = (lambda_*np.sum(theta[1:]**2))/(2*m)
    j = np.sum(infunc1)/m+ infunc2
    return j

# gradient[0] correspond to gradient for theta(0)
# gradient[1:] correspond to gradient for theta(j) j>0
def compute_gradient(theta,x,y,lambda_):
    gradient = np.zeros(n).reshape(n,)
    theta = theta.reshape(n,1)
    infunc1 = sigmoid(lr_hypothesis(x,theta))-y
    gradient_in = np.dot(x.transpose(),infunc1)/m
    gradient[0] = gradient_in[0,0] # theta(0)
    gradient[1:] = gradient_in[1:,0]+(lambda_*theta[1:,]/m).reshape(n-1,) # theta(j) ; j>0
    gradient = gradient.flatten()
    return gradient

You can now test your cost and gradient without optimization. Th below code will optimize the model:

# hyperparameters
m,n = x_poly.shape

# define theta as zero
theta = np.zeros(n)

# define hyperparameter λ
lambda_array = [0, 1, 10, 100]


import scipy.optimize as opt

for i in range(0,len(lambda_array)):
    # Train
    print('======================================== Iteration {} ===================================='.format(i))
    optimized = opt.minimize(fun = compute_cost, x0 = theta, args = (x_poly, y,lambda_array[i]),
                         method = 'TNC', jac = compute_gradient)
    new_theta = optimized.x

    # Prediction
    y_pred_train = predictor(x_poly,new_theta)
    cm_train = confusion_matrix(y,y_pred_train)
    t_train,f_train,acc_train = acc(cm_train)
    print('With lambda = {}, {} correct, {} wrong ==========> accuracy = {}%'
      .format(lambda_array[i],t_train,f_train,acc_train*100))

Now you should see output like this :

 === Iteration 0 === With lambda = 0, 104 correct, 14 wrong ==========> accuracy = 88.13559322033898%
 === Iteration 1 === With lambda = 1, 98 correct, 20 wrong ==========> accuracy = 83.05084745762711%
 === Iteration 2 === With lambda = 10, 88 correct, 30 wrong ==========> accuracy = 74.57627118644068%
 === Iteration 3 === With lambda = 100, 72 correct, 46 wrong ==========> accuracy = 61.016949152542374%

Regularized Logistic Regression in Python (Andrew ng Course)

2 Answers