You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
280 lines
12 KiB
280 lines
12 KiB
2 weeks ago
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
Created on 2021-5-3
|
||
|
|
PCA source code
|
||
|
|
@author: rsj zjl
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
import matplotlib.pyplot as plt
|
||
|
|
from numba import jit
|
||
|
|
import time
|
||
|
|
import numpy as np
|
||
|
|
import traceback
|
||
|
|
from smote import smote
|
||
|
|
import config
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
import requests
|
||
|
|
import datetime
|
||
|
|
import jenkspy
|
||
|
|
import xlrd
|
||
|
|
import AANN_Fit
|
||
|
|
|
||
|
|
class Model(object):
|
||
|
|
def __init__(self):
|
||
|
|
self.v1 = []
|
||
|
|
self.v2 = []
|
||
|
|
self.w1 = []
|
||
|
|
self.w2 = []
|
||
|
|
self.sigma = 0
|
||
|
|
self.maxdata = 0
|
||
|
|
self.mindata = 0
|
||
|
|
|
||
|
|
def AANN(training_Sample, Nodes, num_epochs):
|
||
|
|
"""
|
||
|
|
AANN建模
|
||
|
|
:param training_Sample: numpy的ndarray格式
|
||
|
|
:param Nodes: [6,3,6] 三个隐层
|
||
|
|
:param num_epochs: 迭代次数iteration
|
||
|
|
:return:应返回model(内含v1,w1,v2,w2,sigma,maxdata,mindata)
|
||
|
|
"""
|
||
|
|
time1 = int(round(time.time() * 1000))
|
||
|
|
mm, nn = training_Sample.shape
|
||
|
|
# 预处理 数据归一化
|
||
|
|
mx = np.mean(training_Sample, 0)
|
||
|
|
maxdata = np.max(training_Sample, 0)
|
||
|
|
mindata = np.min(training_Sample, 0)
|
||
|
|
mdata = (2 * training_Sample - (maxdata + mindata + np.zeros((mm, nn)))) / (maxdata - mindata + np.zeros((mm, nn)))
|
||
|
|
Nodes = np.array(Nodes).astype(np.int64)
|
||
|
|
model = Model()
|
||
|
|
# np.save('mdata.npy',mdata)
|
||
|
|
print(type(mm))
|
||
|
|
print(type(nn))
|
||
|
|
print(type(num_epochs))
|
||
|
|
print(type(Nodes))
|
||
|
|
print(type(Nodes[0]))
|
||
|
|
count, spe, o, v1, v2, w1, w2, sigma = cur_aann(mm, nn, num_epochs, Nodes, mdata)
|
||
|
|
reconData = AANN_Fit.AANN_Fit(mdata, v1, v2, w1, w2)
|
||
|
|
r2 = 1-np.sum(np.power((mdata-reconData),2),axis=0)/np.sum(np.power((np.tile(np.average(mdata,axis=0), (mm,1))-reconData),2),axis=0)
|
||
|
|
# 预处理 数据反归一化
|
||
|
|
a = maxdata - mindata + np.zeros((mm, nn))
|
||
|
|
b = maxdata + mindata + np.zeros((mm, nn))
|
||
|
|
reconData = np.matrix((np.multiply(mdata,a) + b)/2)
|
||
|
|
Train_X_min = np.min(training_Sample, axis=0) # 训练值最小值
|
||
|
|
Train_X_max = np.max(training_Sample, axis=0) # 训练值最大值
|
||
|
|
Train_X_mean = np.mean(training_Sample, axis=0) # 训练值平均值
|
||
|
|
Train_X_std = np.std(training_Sample, axis=0) # 训练值方差
|
||
|
|
Train_X_bais = training_Sample - reconData # 训练值偏差
|
||
|
|
Train_X_bais_max = np.max(np.abs(Train_X_bais), axis=0) # 训练值偏差最大值 axis=0 对各列求
|
||
|
|
Train_X_bais_min = np.min(np.abs(Train_X_bais), axis=0) # 训练值偏差最小值
|
||
|
|
Train_X_bais_mean = np.mean(np.abs(Train_X_bais), axis=0) # 训练值偏差平均值
|
||
|
|
Train_X_bais_std_upperB95 = np.array(np.abs(1.96 * np.std(Train_X_bais, axis=0) + Train_X_bais_mean))[
|
||
|
|
0] # 训练值偏差标准差
|
||
|
|
Train_X_bais_std_upperB99 = np.array(np.abs(2.58 * np.std(Train_X_bais, axis=0) + Train_X_bais_mean))[0]
|
||
|
|
Train_X_bais_std_lowerB95 = np.array(np.abs(1.96 * np.std(Train_X_bais, axis=0) - Train_X_bais_mean))[
|
||
|
|
0] # 训练值偏差标准差
|
||
|
|
Train_X_bais_std_lowerB99 = np.array(np.abs(2.58 * np.std(Train_X_bais, axis=0) - Train_X_bais_mean))[0]
|
||
|
|
QCUL_95_line = [] # 限值
|
||
|
|
QCUL_99_line = []
|
||
|
|
for index1 in range(len(Train_X_bais_std_upperB95)):
|
||
|
|
QCUL_95_line.append(max(Train_X_bais_std_upperB95[index1], Train_X_bais_std_lowerB95[index1]))
|
||
|
|
QCUL_99_line.append(max(Train_X_bais_std_upperB99[index1], Train_X_bais_std_lowerB99[index1]))
|
||
|
|
QCUL_95_line = np.array(QCUL_95_line)
|
||
|
|
QCUL_99_line = np.array(QCUL_99_line)
|
||
|
|
#####################################################################################################################
|
||
|
|
items = [('Train_X_min', np.around(Train_X_min, decimals=3).tolist()),
|
||
|
|
('Train_X_max', np.around(Train_X_max, decimals=3).tolist()),
|
||
|
|
('Train_X_std', np.around(Train_X_std, decimals=3).tolist()),
|
||
|
|
('Train_X_mean',np.around(Train_X_mean, decimals=3).tolist()),
|
||
|
|
(('Train_X_bais_max',np.around(Train_X_bais_max, decimals=3).tolist())),
|
||
|
|
(('Train_X_bais_min', np.around(Train_X_bais_min, decimals=3).tolist())),
|
||
|
|
(('Train_X_bais_mean', np.around(Train_X_bais_mean, decimals=3).tolist())),
|
||
|
|
('QCUL_95_line', np.around(QCUL_95_line, decimals=3).tolist()),
|
||
|
|
('QCUL_99_line', np.around(QCUL_99_line, decimals=3).tolist()),
|
||
|
|
('r2', r2.tolist()),
|
||
|
|
('count', count),
|
||
|
|
('maxdata', maxdata.tolist()),
|
||
|
|
('mindata', mindata.tolist()),
|
||
|
|
('sigma', sigma),
|
||
|
|
('v1', v1.tolist()),
|
||
|
|
('v2', v2.tolist()),
|
||
|
|
('w1', w1.tolist()),
|
||
|
|
('w2', w2.tolist())
|
||
|
|
]
|
||
|
|
time2 = int(round(time.time() * 1000))
|
||
|
|
tc = time2 - time1
|
||
|
|
res_items = [('Model_info', dict(items)), ('Model_type', 'AANN')]
|
||
|
|
result = dict(res_items) # json.dumps(result)
|
||
|
|
return json.dumps(result)
|
||
|
|
|
||
|
|
@jit(nopython=True, cache=True)
|
||
|
|
def cur_aann(mm, nn, num_epochs, Nodes, mdata):
|
||
|
|
alpha0 = 0.001
|
||
|
|
alfa = 0.5
|
||
|
|
samplenum = mm
|
||
|
|
spe = np.zeros(num_epochs + 1)
|
||
|
|
errorp = np.zeros(samplenum)
|
||
|
|
v1 = 2 * np.random.rand(nn, Nodes[0]) - 1
|
||
|
|
dv1 = np.zeros((nn, Nodes[0]))
|
||
|
|
w1 = 2 * np.random.rand(Nodes[0], Nodes[1]) - 1
|
||
|
|
dw1 = np.zeros((Nodes[0], Nodes[1]))
|
||
|
|
v2 = 2 * np.random.rand(Nodes[1], Nodes[2]) - 1
|
||
|
|
dv2 = np.zeros((Nodes[1], Nodes[2]))
|
||
|
|
w2 = 2 * np.random.rand(Nodes[2], nn) - 1
|
||
|
|
dw2 = np.zeros((Nodes[2], nn))
|
||
|
|
|
||
|
|
xlist = mdata
|
||
|
|
expectlist = mdata
|
||
|
|
count = 0
|
||
|
|
aa = alfa
|
||
|
|
alpha = alpha0
|
||
|
|
y1, y2, y3, o, yitao = np.zeros((samplenum, Nodes[0])), np.zeros((samplenum, Nodes[1])), np.zeros((samplenum,
|
||
|
|
Nodes[
|
||
|
|
2])), np.zeros(
|
||
|
|
(samplenum, nn)), np.zeros((samplenum, nn))
|
||
|
|
yitay1, yitay2, yitay3 = np.zeros((samplenum, Nodes[0])), np.zeros((samplenum, Nodes[1])), np.zeros((samplenum,
|
||
|
|
Nodes[2]))
|
||
|
|
while count < num_epochs:
|
||
|
|
c = 0
|
||
|
|
while c < samplenum:
|
||
|
|
d = expectlist[c, :]
|
||
|
|
x = xlist[c, :]
|
||
|
|
y1[c, :] = (1 / (1 + np.exp(-np.dot(x, v1))))
|
||
|
|
y2[c, :] = np.dot(y1[c, :], w1)
|
||
|
|
y3[c, :] = (1 / (1 + np.exp(-np.dot(y2[c, :], v2))))
|
||
|
|
o[c, :] = np.dot(y3[c, :], w2)
|
||
|
|
yitao[c, :] = (d - o[c, :])
|
||
|
|
errorp[c] = 0.5 * np.sum(yitao[c, :] * yitao[c, :])
|
||
|
|
yitay3[c, :] = (np.dot(yitao[c, :], w2.T) * y3[c, :] * (1 - y3[c, :]))
|
||
|
|
yitay2[c, :] = np.dot(yitay3[c, :], v2.T)
|
||
|
|
yitay1[c, :] = np.dot(yitay2[c, :], w1.T) * y1[c, :] * (1 - y1[c, :])
|
||
|
|
# 调整各层权值
|
||
|
|
deltw2 = np.dot(alpha * y3[c, :].reshape(Nodes[2], 1), yitao[c, :].reshape(1, nn))
|
||
|
|
w2 = w2 + deltw2 + aa * dw2
|
||
|
|
dw2 = deltw2
|
||
|
|
deltv2 = np.dot(alpha * y2[c, :].reshape(Nodes[1], 1), yitay3[c, :].reshape(1, Nodes[2]))
|
||
|
|
v2 = v2 + deltv2 + aa * dv2
|
||
|
|
dv2 = deltv2
|
||
|
|
deltw1 = np.dot(alpha * y1[c, :].reshape(Nodes[0], 1), yitay2[c, :].reshape(1, Nodes[1]))
|
||
|
|
w1 = w1 + deltw1 + aa * dw1
|
||
|
|
dw1 = deltw1
|
||
|
|
deltv1 = np.dot(alpha * x.reshape(nn, 1), yitay1[c, :].reshape(1, Nodes[0]))
|
||
|
|
v1 = v1 + deltv1 + aa * dv1
|
||
|
|
dv1 = deltv1
|
||
|
|
c = c + 1
|
||
|
|
spe[count] = np.sum(errorp) / samplenum
|
||
|
|
if count > 1 and abs(spe[count] - spe[count - 1]) < 0.0000001:
|
||
|
|
break
|
||
|
|
count += 1
|
||
|
|
|
||
|
|
sigma = 3 * np.sum((expectlist - o) * (expectlist - o)) / mm
|
||
|
|
|
||
|
|
|
||
|
|
return count, spe, o, v1, v2, w1, w2, sigma
|
||
|
|
|
||
|
|
def isnumber(limits):
|
||
|
|
flag=True
|
||
|
|
for item in limits:
|
||
|
|
item=item.replace("-","")
|
||
|
|
if(item.isdigit()==False):
|
||
|
|
flag=False
|
||
|
|
break
|
||
|
|
return flag
|
||
|
|
|
||
|
|
|
||
|
|
def clearmain(info):
|
||
|
|
try:
|
||
|
|
Train_Data = info["Train_Data"]
|
||
|
|
times = Train_Data["time"].split(';')
|
||
|
|
points = Train_Data["points"].split(',')
|
||
|
|
interval = Train_Data["interval"]
|
||
|
|
if interval == 10000:
|
||
|
|
DCount = 60
|
||
|
|
elif interval == 100000:
|
||
|
|
DCount = 6
|
||
|
|
elif interval == 300000:
|
||
|
|
DCount = 5
|
||
|
|
else:
|
||
|
|
DCount = 4
|
||
|
|
dead = Train_Data["dead"].split(',')
|
||
|
|
limit = Train_Data["limit"].split(',')
|
||
|
|
uplower = Train_Data["uplow"].split(';')
|
||
|
|
|
||
|
|
condition=info["conditon"].replace("=","==").replace(">=",">").replace("<=","<")
|
||
|
|
#percent = info["Hyper_para"]["percent"]
|
||
|
|
|
||
|
|
count=0
|
||
|
|
ItemsInfo, SamplingTimePeriods = [], []
|
||
|
|
Constraint = ""
|
||
|
|
for i in range(len(points)):
|
||
|
|
iteminfo = {}
|
||
|
|
iteminfo["ItemName"] = points[i] # 加点
|
||
|
|
if (dead[i] == "1"): # 判断是否参与死区清洗
|
||
|
|
iteminfo["ClearDeadZone"] = "true"
|
||
|
|
else:
|
||
|
|
iteminfo["ClearDeadZone"] = "false"
|
||
|
|
if (limit[i] == "1"): # 参与上下限清洗
|
||
|
|
limits = uplower[i].split(',')
|
||
|
|
if (isnumber(limits) == True): # 输入上下限正确
|
||
|
|
count += 1
|
||
|
|
Constraint += "[" + points[i] + "]>" + limits[0] + " and " + "[" + points[i] + "]<" + limits[1] + " and "
|
||
|
|
ItemsInfo.append(iteminfo)
|
||
|
|
if(count!=0):
|
||
|
|
Constraint = Constraint[:len(Constraint) - 4:]
|
||
|
|
else:
|
||
|
|
Constraint="1==1"#没有上下限清洗
|
||
|
|
Constraint+=" and ("+condition+")"
|
||
|
|
Constraint = Constraint.replace("\n", " ")
|
||
|
|
for i in range(len(times)):
|
||
|
|
Eachsampletime = {}
|
||
|
|
timess = times[i].split(',')
|
||
|
|
Eachsampletime["StartingTime"] = timess[0]
|
||
|
|
Eachsampletime["TerminalTime"] = timess[1]
|
||
|
|
SamplingTimePeriods.append(Eachsampletime)
|
||
|
|
url = f"http://{config._CLEAN_IP}/exawebapi/exatime/GetCleaningData?ItemsInfo=%s&SamplingTimePeriods=%s&Constraint=%s&SamplingPeriod=%s&DCount=%d" % (
|
||
|
|
ItemsInfo, SamplingTimePeriods, Constraint, interval, DCount)
|
||
|
|
response = requests.get(url)
|
||
|
|
content = json.loads(response.text)
|
||
|
|
data = np.array([item for item in content["ClearData"]]).T
|
||
|
|
try:
|
||
|
|
smote_data = info["smote"]
|
||
|
|
# smote_data = False
|
||
|
|
except KeyError:
|
||
|
|
smote_data = False
|
||
|
|
if smote_data:
|
||
|
|
try:
|
||
|
|
smote_index = [points.index(item["pointId"]) for item in info["smote_config"] if item["LAY_CHECKED"]]
|
||
|
|
smote_num = [int(item["number"]) for item in info["smote_config"] if item["LAY_CHECKED"]]
|
||
|
|
max_value = [float(item["max"]) for item in info["smote_config"] if item["LAY_CHECKED"]]
|
||
|
|
min_value = [float(item["min"]) for item in info["smote_config"] if item["LAY_CHECKED"]]
|
||
|
|
except KeyError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
if len(smote_num) != 0:
|
||
|
|
data, *_ = smote(data, smote_index, smote_num, max_value, min_value)
|
||
|
|
Nodes=info["layer"]
|
||
|
|
num_epochs=Train_Data["interval"]
|
||
|
|
result =AANN(data, Nodes, num_epochs)#看看nodes和num_epochs怎么传进来
|
||
|
|
#result = pca(data, percent)
|
||
|
|
result = result.replace("NaN", "-1")
|
||
|
|
result=json.loads(result)
|
||
|
|
result["BeforeCleanSamNum"]=content["BeforeCleanSamNum"]
|
||
|
|
result["AfterCleanSamNum"]=content["AfterCleanSamNum"]
|
||
|
|
result["CleanOrNot"] = True
|
||
|
|
return json.dumps(result)
|
||
|
|
except Exception as e:
|
||
|
|
result = [{"CleanOrNot": False, "msg": traceback.format_exc()}]
|
||
|
|
return json.dumps(result, ensure_ascii=False)
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
info_str='{"layer":[5,2,5],"Train_Data":{"time":"2020-10-14 04:10:28,2020-10-17 14:53:00;2021-04-07 07:32:47,2021-04-16 08:39:01;2021-06-01 18:48:17,2021-06-03 14:29:40","points":"DH4_40MAG20CT362,DH4_40MAG20AN002GT,DH4_40MAG20CE102,DH4_40MAG20CT312,DH4_40MAG20CT322,DH4_40MAG20CT332","interval":300000,"dead":"1,1,1,1,1,1","limit":"0,0,0,0,0,0","uplow":"null,null;null,null;null,null;null,null;null,null;null,null"},"type":"AANN","conditon":"[DH4_40MAG20CE102]>20","epoch":"10000"}'
|
||
|
|
info = json.loads(info_str)
|
||
|
|
print(clearmain(info))
|
||
|
|
|
||
|
|
|
||
|
|
|