import torch

class MyRelu(torch.autograd.Function):
    @staticmethod
    def forward(ctx, input):
        # 我們使用ctx上下文對(duì)象來(lái)緩存，以便在反向傳播中使用，ctx存儲(chǔ)時(shí)候只能存tensor
        # 在正向傳播中，我們接收一個(gè)上下文對(duì)象ctx和一個(gè)包含輸入的張量input；
        # 我們必須返回一個(gè)包含輸出的張量，
        # input.clamp(min = 0)表示講輸入中所有值范圍規(guī)定到0到正無(wú)窮，如input=[-1,-2,3]則被轉(zhuǎn)換成input=[0,0,3]
        ctx.save_for_backward(input)
        
        # 返回幾個(gè)值，backward接受參數(shù)則包含ctx和這幾個(gè)值
        return input.clamp(min = 0)

    @staticmethod
    def backward(ctx, grad_output):
        # 把ctx中存儲(chǔ)的input張量讀取出來(lái)
        input, = ctx.saved_tensors
        
        # grad_output存放反向傳播過(guò)程中的梯度
        grad_input = grad_output.clone()
        
        # 這兒就是ReLu的規(guī)則，表示原始數(shù)據(jù)小于0，則relu為0，因此對(duì)應(yīng)索引的梯度都置為0
        grad_input[input < 0] = 0
        return grad_input

dtype = torch.float
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# 使用torch的generator定義隨機(jī)數(shù)，注意產(chǎn)生的是cpu隨機(jī)數(shù)還是gpu隨機(jī)數(shù)
generator=torch.Generator(device).manual_seed(42)

# N是Batch, H is hidden dimension，
# D_in is input dimension;D_out is output dimension.
N, D_in, H, D_out = 64, 1000, 100, 10

x = torch.randn(N, D_in, device=device, dtype=dtype,generator=generator)
y = torch.randn(N, D_out, device=device, dtype=dtype, generator=generator)

w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True, generator=generator)
w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True, generator=generator)

learning_rate = 1e-6
for t in range(500):
    relu = MyRelu.apply
    # 使用函數(shù)傳入?yún)?shù)運(yùn)算 
    y_pred = relu(x.mm(w1)).mm(w2)
	# 計(jì)算損失
    loss = (y_pred - y).pow(2).sum()
    if t % 100 == 99:
        print(t, loss.item())
    # 傳播
    loss.backward()
    with torch.no_grad():
        w1 -= learning_rate * w1.grad
        w2 -= learning_rate * w2.grad
       	
        w1.grad.zero_()
        w2.grad.zero_()

import torch
import torch.nn as nn
import matplotlib.pyplot as plt
import numpy as np
torch.manual_seed(10)
#========生成數(shù)據(jù)=============
sample_nums = 100
mean_value = 1.7
bias = 1
n_data = torch.ones(sample_nums,2)
x0 = torch.normal(mean_value*n_data,1)+bias#類(lèi)別0數(shù)據(jù)
y0 = torch.zeros(sample_nums)#類(lèi)別0標(biāo)簽
x1 = torch.normal(-mean_value*n_data,1)+bias#類(lèi)別1數(shù)據(jù)
y1 = torch.ones(sample_nums)#類(lèi)別1標(biāo)簽
train_x = torch.cat((x0,x1),0)
train_y = torch.cat((y0,y1),0)
#==========選擇模型===========
class LR(nn.Module):
    def __init__(self):
        super(LR,self).__init__()
        self.features = nn.Linear(2,1)
        self.sigmoid = nn.Sigmoid()

    def forward(self,x):
        x = self.features(x)
        x = self.sigmoid(x)
        return x

lr_net = LR()#實(shí)例化邏輯回歸模型

#==============選擇損失函數(shù)===============
loss_fn = nn.BCELoss()
#==============選擇優(yōu)化器=================
lr = 0.01
optimizer = torch.optim.SGD(lr_net.parameters(),lr = lr,momentum=0.9)

#===============模型訓(xùn)練==================
for iteration in range(1000):
    #前向傳播
    y_pred = lr_net(train_x)#模型的輸出
    #計(jì)算loss
    loss = loss_fn(y_pred.squeeze(),train_y)
    #反向傳播
    loss.backward()
    #更新參數(shù)
    optimizer.step()

    #繪圖
    if iteration % 20 == 0:
        mask = y_pred.ge(0.5).float().squeeze() #以0.5分類(lèi)
        correct = (mask==train_y).sum()#正確預(yù)測(cè)樣本數(shù)
        acc = correct.item()/train_y.size(0)#分類(lèi)準(zhǔn)確率

        plt.scatter(x0.data.numpy()[:,0],x0.data.numpy()[:,1],c='r',label='class0')
        plt.scatter(x1.data.numpy()[:,0],x1.data.numpy()[:,1],c='b',label='class1')

        w0,w1 = lr_net.features.weight[0]
        w0,w1 = float(w0.item()),float(w1.item())
        plot_b = float(lr_net.features.bias[0].item())
        plot_x = np.arange(-6,6,0.1)
        plot_y = (-w0*plot_x-plot_b)/w1

        plt.xlim(-5,7)
        plt.ylim(-7,7)
        plt.plot(plot_x,plot_y)

        plt.text(-5,5,'Loss=%.4f'%loss.data.numpy(),fontdict={'size':20,'color':'red'})
        plt.title('Iteration:{}\nw0:{:.2f} w1:{:.2f} b{:.2f} accuracy:{:2%}'.format(iteration,w0,w1,plot_b,acc))
        plt.legend()
        plt.show()
        plt.pause(0.5)
        if acc > 0.99:
            break