import numpy as np
import matplotlib.pyplot as plt
def ols(X, y):
'''
input:
X: n, m+1
y: n, 1
output:
beta_hat: m+1, 1
'''
return np.dot(np.linalg.inv(np.dot(X.T, X)), X.T).dot(y)
def pred(X, beta_hat):
'''
input:
X: (k, m+1)
output:
y_hat: (k, 1)
'''
return np.dot(X, beta_hat)
x_data = [55, 53, 99, 74, 66]
y_data = [60, 68, 95, 72, 69]
X = np.array(x_data).reshape(len(x_data), -1)
X = np.concatenate((np.ones((len(x_data), 1)), X), axis=1)
y = np.array(y_data).reshape(len(y_data), -1)
beta_hat = ols(X, y)
print(beta_hat)
[[26.23606699]
[ 0.67095004]]
x_data_new = [64]
X_new = np.array(x_data_new).reshape(len(x_data_new), -1)
X_new = np.concatenate((np.ones((len(x_data_new), 1)), X_new), axis=1)
y_hat = pred(X_new, beta_hat)
print(y_hat)
[[69.17686977]]
X_data = [[59, 20, 22], [55, 22, 20], [62, 20, 16], [55, 18, 32], [52, 30, 27],
[60, 24, 28], [59, 16, 24], [54, 22, 24], [83, 18, 14], [50, 21, 29], [73, 22, 21],
[61, 25, 35], [42, 34, 21], [46, 25, 7], [54, 20, 30]]
y_data = [195, 203, 214, 183, 130, 200, 212, 164, 108, 285, 207, 336, 280, 207, 264]
assert len(X_data) == len(y_data)
X = np.array(X_data)
X = np.concatenate((np.ones((len(X_data), 1)), X), axis=1)
y = np.array(y_data).reshape(len(y_data), -1)
beta_hat = ols(X, y)
print(beta_hat)
[[233.55075022]
[ -1.8790613 ]
[ 0.77368823]
[ 2.99826812]]
x_data_new = [53, 20, 25]
X_new = np.matrix(x_data_new)
X_new = np.concatenate( ( np.ones(X_new.shape[0]).reshape(X_new.shape[0],1), X_new), axis=1)
y_hat = pred(X_new, beta_hat)
print(y_hat)
[[224.39096865]]
def sigmoid(z):
return 1/(1+np.exp(-z))
def initialize_params(m):
w = np.zeros((m, 1))
b = 0
return w, b
def train_logistic(w, b, X, Y):
'''
此函数使用行向量形式;
forward+back 整合
input:
w -- weights (m, 1)
b -- bias, scalar
X: (m, n)
Y: true labels (1, n)
return:
cost: cross-entropy loss
dw: gradient over w (dim same as w)
db: ..
'''
n = X.shape[1]
# forward
A = sigmoid(np.dot(w.T, X) + b)
cost = (-1/n) * np.sum( Y*np.log(A) + (1-Y)*np.log(1-A) )
# back
dw = (1/n) * np.dot(X, (A-Y).T)
db = (1/n) * np.sum(A-Y)
assert(dw.shape == w.shape)
assert(db.dtype == float)
cost = np.squeeze(cost)
grads = {"dw":dw, "db":db}
return grads, cost
def optimizer(w, b, X, Y, num_iterations, learning_rate, print_cost = False):
'''
inputs:
same as above
returns:
params: w and b
grads
costs
'''
costs = []
# training
for i in range(num_iterations):
# cost and gradient
grads, cost = train_logistic(w, b, X, Y)
# retrieve
dw = grads["dw"]
db = grads["db"]
# update params
w -= learning_rate*dw
b -= learning_rate*db
# record costs
if i % 100 == 0:
costs.append(cost)
# print cost every 100 iter
if print_cost and i%1000 == 0:
print("cost after iter %i: %f"%(i, cost))
# record
params = {"w": w,
"b": b}
grads = {"dw": dw,
"db": db}
return params, grads, costs
def predict(w, b, X):
'''
X: new data ; (m, n)
'''
n = X.shape[1]
Y_prediction = np.zeros((1, n)) # initialze
w = w.reshape(X.shape[0], 1)
# prediction
A = sigmoid(np.dot(w.T, X) + b)
Y_prediction = np.where(A <= 0.5, 0 ,1)
assert(Y_prediction.shape == (1, n))
return A
X_possible = [26.8, 27.3, 28.5, 30.3, 30.7]
y_possible = [[1, 10], [2, 9], [8, 5], [9, 2], [10, 1]]
y = []
X = []
for i, x in enumerate(X_possible):
for j, n_y in enumerate(y_possible[i]):
y1 = [j]*n_y
X1 = [x]*n_y
X += X1
y += y1
assert(len(X) == len(y))
m = 1
# X (m, n); y (1, n)
X_train = np.array(X).reshape(m, -1)
Y = np.array(y).reshape(1, -1)
Y_train = np.where(Y==1, 0, 1) # male = 1; female = 0
w, b = initialize_params(m)
parameters, grads, costs = optimizer(w, b, X_train, Y_train, 200000, 0.015, True)
cost after iter 0: 13.849085
cost after iter 1000: 1.187040
cost after iter 2000: 1.182306
cost after iter 3000: 1.177617
cost after iter 4000: 1.172971
cost after iter 5000: 1.168369
cost after iter 6000: 1.163810
cost after iter 7000: 1.159294
cost after iter 8000: 1.154821
cost after iter 9000: 1.150389
cost after iter 10000: 1.145999
cost after iter 11000: 1.141650
cost after iter 12000: 1.137341
cost after iter 13000: 1.133071
cost after iter 14000: 1.128841
cost after iter 15000: 1.124648
cost after iter 16000: 1.120493
cost after iter 17000: 1.116375
cost after iter 18000: 1.112293
cost after iter 19000: 1.108246
cost after iter 20000: 1.104234
cost after iter 21000: 1.100255
cost after iter 22000: 1.096309
cost after iter 23000: 1.092395
cost after iter 24000: 1.088511
cost after iter 25000: 1.084658
cost after iter 26000: 1.080833
cost after iter 27000: 1.077037
cost after iter 28000: 1.073268
cost after iter 29000: 1.069525
cost after iter 30000: 1.065807
cost after iter 31000: 1.062113
cost after iter 32000: 1.058443
cost after iter 33000: 1.054794
cost after iter 34000: 1.051167
cost after iter 35000: 1.047560
cost after iter 36000: 1.043971
cost after iter 37000: 1.040401
cost after iter 38000: 1.036847
cost after iter 39000: 1.033309
cost after iter 40000: 1.029786
cost after iter 41000: 1.026276
cost after iter 42000: 1.022779
cost after iter 43000: 1.019293
cost after iter 44000: 1.015818
cost after iter 45000: 1.012352
cost after iter 46000: 1.008893
cost after iter 47000: 1.005442
cost after iter 48000: 1.001997
cost after iter 49000: 0.998556
cost after iter 50000: 0.995120
cost after iter 51000: 0.991685
cost after iter 52000: 0.988253
cost after iter 53000: 0.984820
cost after iter 54000: 0.981387
cost after iter 55000: 0.977953
cost after iter 56000: 0.974515
cost after iter 57000: 0.971073
cost after iter 58000: 0.967627
cost after iter 59000: 0.964174
cost after iter 60000: 0.960715
cost after iter 61000: 0.957247
cost after iter 62000: 0.953769
cost after iter 63000: 0.950282
cost after iter 64000: 0.946783
cost after iter 65000: 0.943272
cost after iter 66000: 0.939748
cost after iter 67000: 0.936209
cost after iter 68000: 0.932654
cost after iter 69000: 0.929084
cost after iter 70000: 0.925496
cost after iter 71000: 0.921890
cost after iter 72000: 0.918264
cost after iter 73000: 0.914618
cost after iter 74000: 0.910951
cost after iter 75000: 0.907262
cost after iter 76000: 0.903550
cost after iter 77000: 0.899814
cost after iter 78000: 0.896053
cost after iter 79000: 0.892266
cost after iter 80000: 0.888453
cost after iter 81000: 0.884612
cost after iter 82000: 0.880743
cost after iter 83000: 0.876844
cost after iter 84000: 0.872916
cost after iter 85000: 0.868957
cost after iter 86000: 0.864967
cost after iter 87000: 0.860944
cost after iter 88000: 0.856888
cost after iter 89000: 0.852798
cost after iter 90000: 0.848674
cost after iter 91000: 0.844515
cost after iter 92000: 0.840320
cost after iter 93000: 0.836088
cost after iter 94000: 0.831819
cost after iter 95000: 0.827512
cost after iter 96000: 0.823167
cost after iter 97000: 0.818782
cost after iter 98000: 0.814359
cost after iter 99000: 0.809895
cost after iter 100000: 0.805390
cost after iter 101000: 0.800845
cost after iter 102000: 0.796258
cost after iter 103000: 0.791630
cost after iter 104000: 0.786959
cost after iter 105000: 0.782245
cost after iter 106000: 0.777489
cost after iter 107000: 0.772690
cost after iter 108000: 0.767847
cost after iter 109000: 0.762961
cost after iter 110000: 0.758032
cost after iter 111000: 0.753058
cost after iter 112000: 0.748041
cost after iter 113000: 0.742981
cost after iter 114000: 0.737877
cost after iter 115000: 0.732730
cost after iter 116000: 0.727540
cost after iter 117000: 0.722307
cost after iter 118000: 0.717032
cost after iter 119000: 0.711715
cost after iter 120000: 0.706357
cost after iter 121000: 0.700959
cost after iter 122000: 0.695521
cost after iter 123000: 0.690044
cost after iter 124000: 0.684530
cost after iter 125000: 0.678979
cost after iter 126000: 0.673393
cost after iter 127000: 0.667774
cost after iter 128000: 0.662123
cost after iter 129000: 0.656442
cost after iter 130000: 0.650733
cost after iter 131000: 0.644998
cost after iter 132000: 0.639240
cost after iter 133000: 0.633461
cost after iter 134000: 0.627664
cost after iter 135000: 0.621852
cost after iter 136000: 0.616030
cost after iter 137000: 0.610200
cost after iter 138000: 0.604366
cost after iter 139000: 0.598534
cost after iter 140000: 0.592706
cost after iter 141000: 0.586889
cost after iter 142000: 0.581088
cost after iter 143000: 0.575307
cost after iter 144000: 0.569554
cost after iter 145000: 0.563834
cost after iter 146000: 0.558154
cost after iter 147000: 0.552521
cost after iter 148000: 0.546941
cost after iter 149000: 0.541423
cost after iter 150000: 0.535974
cost after iter 151000: 0.530602
cost after iter 152000: 0.525315
cost after iter 153000: 0.520120
cost after iter 154000: 0.515027
cost after iter 155000: 0.510044
cost after iter 156000: 0.505178
cost after iter 157000: 0.500438
cost after iter 158000: 0.495832
cost after iter 159000: 0.491369
cost after iter 160000: 0.487058
cost after iter 161000: 0.482912
cost after iter 162000: 0.478948
cost after iter 163000: 0.475201
cost after iter 164000: 0.471752
cost after iter 165000: 0.468824
cost after iter 166000: 0.466944
cost after iter 167000: 0.466392
cost after iter 168000: 0.466334
cost after iter 169000: 0.466307
cost after iter 170000: 0.466282
cost after iter 171000: 0.466256
cost after iter 172000: 0.466231
cost after iter 173000: 0.466206
cost after iter 174000: 0.466182
cost after iter 175000: 0.466157
cost after iter 176000: 0.466133
cost after iter 177000: 0.466109
cost after iter 178000: 0.466086
cost after iter 179000: 0.466063
cost after iter 180000: 0.466040
cost after iter 181000: 0.466017
cost after iter 182000: 0.465994
cost after iter 183000: 0.465972
cost after iter 184000: 0.465950
cost after iter 185000: 0.465928
cost after iter 186000: 0.465907
cost after iter 187000: 0.465885
cost after iter 188000: 0.465864
cost after iter 189000: 0.465844
cost after iter 190000: 0.465823
cost after iter 191000: 0.465803
cost after iter 192000: 0.465782
cost after iter 193000: 0.465762
cost after iter 194000: 0.465743
cost after iter 195000: 0.465723
cost after iter 196000: 0.465704
cost after iter 197000: 0.465685
cost after iter 198000: 0.465666
cost after iter 199000: 0.465647
print ("w = " + str(parameters["w"]))
print ("b = " + str(parameters["b"]))
w = [[0.96751536]]
b = -27.577455634477523
X = np.arange(25.5, 31.1, 0.5).reshape(m,-1)
X.shape
w = parameters["w"]
b = parameters["b"]
Y_predict = predict(w, b, X)
print(Y_predict)
[[0.0518669 0.08150618 0.12583489 0.18930369 0.27472441 0.38059451
0.49918302 0.61786352 0.72397141 0.80969123 0.87344441 0.91800319]]
sig_x = np.linspace(20, 36, 100).reshape(m, -1)
sig_y = sigmoid(np.dot(w.T, sig_x) + b)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
plt.scatter(X, Y_predict)
plt.plot(sig_x.reshape(-1,), sig_y.reshape(-1,) , 'r', label="logistic")
plt.legend(loc='upper left')
plt.xlabel("tempreture")
plt.ylabel("prob to be male")
plt.show()
X = np.array([28]).reshape(m,-1)
Y_predict = predict(w, b, X)
print(Y_predict)
[[0.38059451]]
X = np.array([29]).reshape(m,-1)
Y_predict = predict(w, b, X)
print(Y_predict)
[[0.61786352]]
# x: (n,m+1)
x_data = [[630, 35, 1.6], [550, 50, 2.4]]
X = np.array(x_data).reshape(len(x_data), -1)
X = np.concatenate( (X, np.ones((len(x_data), 1))), axis=1)
m = X.shape[1]
# w: (m+1, n_class) 注意顺序⚠️; intercept 在最后面
W = [[ -0.03, 9.4847, -62.9414,54.4545],
[ 0.2075, 12.3057,-27.9694, -244.1814] ,
[ 0.1824, 12.3408, -29.7338, -226.1336]]
W = np.array(W).T
assert(W.shape[0]==m)
# x (n,m+1) w (m+1 , 1)
# row: for a instance; column: n_class f1, f2..fk
f = np.dot(X, W)
f
P = {}
P_k = 1/ (1+ np.sum(np.exp(f), axis=1) )
P["Pk"] = P_k.reshape(2,1)
for i in range(1, f.shape[1]+1):
P["P"+str(i)] = P["Pk"] * np.exp(f[:,i-1].reshape(2,1))
P