# -*- coding: utf-8 -*- """ Created on 2021-5-3 PCA source code @author: rsj zjl """ import matplotlib.pyplot as plt from numba import jit import time import numpy as np import traceback from smote import smote import config import json import sys import requests import datetime import jenkspy import xlrd import AANN_Fit class Model(object): def __init__(self): self.v1 = [] self.v2 = [] self.w1 = [] self.w2 = [] self.sigma = 0 self.maxdata = 0 self.mindata = 0 def AANN(training_Sample, Nodes, num_epochs): """ AANN建模 :param training_Sample: numpy的ndarray格式 :param Nodes: [6,3,6] 三个隐层 :param num_epochs: 迭代次数iteration :return:应返回model(内含v1,w1,v2,w2,sigma,maxdata,mindata) """ time1 = int(round(time.time() * 1000)) mm, nn = training_Sample.shape # 预处理 数据归一化 mx = np.mean(training_Sample, 0) maxdata = np.max(training_Sample, 0) mindata = np.min(training_Sample, 0) mdata = (2 * training_Sample - (maxdata + mindata + np.zeros((mm, nn)))) / (maxdata - mindata + np.zeros((mm, nn))) Nodes = np.array(Nodes).astype(np.int64) model = Model() # np.save('mdata.npy',mdata) print(type(mm)) print(type(nn)) print(type(num_epochs)) print(type(Nodes)) print(type(Nodes[0])) count, spe, o, v1, v2, w1, w2, sigma = cur_aann(mm, nn, num_epochs, Nodes, mdata) reconData = AANN_Fit.AANN_Fit(mdata, v1, v2, w1, w2) r2 = 1 - np.sum(np.power((mdata - reconData), 2), axis=0) / np.sum( np.power((np.tile(np.average(mdata, axis=0), (mm, 1)) - reconData), 2), axis=0) # 预处理 数据反归一化 a = maxdata - mindata + np.zeros((mm, nn)) b = maxdata + mindata + np.zeros((mm, nn)) reconData = np.matrix((np.multiply(mdata, a) + b) / 2) Train_X_min = np.min(training_Sample, axis=0) # 训练值最小值 Train_X_max = np.max(training_Sample, axis=0) # 训练值最大值 Train_X_mean = np.mean(training_Sample, axis=0) # 训练值平均值 Train_X_std = np.std(training_Sample, axis=0) # 训练值方差 Train_X_bais = training_Sample - reconData # 训练值偏差 Train_X_bais_max = np.max(np.abs(Train_X_bais), axis=0) # 训练值偏差最大值 axis=0 对各列求 Train_X_bais_min = np.min(np.abs(Train_X_bais), axis=0) # 训练值偏差最小值 Train_X_bais_mean = np.mean(np.abs(Train_X_bais), axis=0) # 训练值偏差平均值 Train_X_bais_std_upperB95 = np.array(np.abs(1.96 * np.std(Train_X_bais, axis=0) + Train_X_bais_mean))[ 0] # 训练值偏差标准差 Train_X_bais_std_upperB99 = np.array(np.abs(2.58 * np.std(Train_X_bais, axis=0) + Train_X_bais_mean))[0] Train_X_bais_std_lowerB95 = np.array(np.abs(1.96 * np.std(Train_X_bais, axis=0) - Train_X_bais_mean))[ 0] # 训练值偏差标准差 Train_X_bais_std_lowerB99 = np.array(np.abs(2.58 * np.std(Train_X_bais, axis=0) - Train_X_bais_mean))[0] QCUL_95_line = [] # 限值 QCUL_99_line = [] for index1 in range(len(Train_X_bais_std_upperB95)): QCUL_95_line.append(max(Train_X_bais_std_upperB95[index1], Train_X_bais_std_lowerB95[index1])) QCUL_99_line.append(max(Train_X_bais_std_upperB99[index1], Train_X_bais_std_lowerB99[index1])) QCUL_95_line = np.array(QCUL_95_line) QCUL_99_line = np.array(QCUL_99_line) ##################################################################################################################### items = [('Train_X_min', np.around(Train_X_min, decimals=3).tolist()), ('Train_X_max', np.around(Train_X_max, decimals=3).tolist()), ('Train_X_std', np.around(Train_X_std, decimals=3).tolist()), ('Train_X_mean', np.around(Train_X_mean, decimals=3).tolist()), (('Train_X_bais_max', np.around(Train_X_bais_max, decimals=3).tolist())), (('Train_X_bais_min', np.around(Train_X_bais_min, decimals=3).tolist())), (('Train_X_bais_mean', np.around(Train_X_bais_mean, decimals=3).tolist())), ('QCUL_95_line', np.around(QCUL_95_line, decimals=3).tolist()), ('QCUL_99_line', np.around(QCUL_99_line, decimals=3).tolist()), ('r2', r2.tolist()), ('count', count), ('maxdata', maxdata.tolist()), ('mindata', mindata.tolist()), ('sigma', sigma), ('v1', v1.tolist()), ('v2', v2.tolist()), ('w1', w1.tolist()), ('w2', w2.tolist()) ] time2 = int(round(time.time() * 1000)) tc = time2 - time1 res_items = [('Model_info', dict(items)), ('Model_type', 'AANN')] result = dict(res_items) # json.dumps(result) return json.dumps(result) @jit(nopython=True, cache=True) def cur_aann(mm, nn, num_epochs, Nodes, mdata): alpha0 = 0.001 alfa = 0.5 samplenum = mm spe = np.zeros(num_epochs + 1) errorp = np.zeros(samplenum) v1 = 2 * np.random.rand(nn, Nodes[0]) - 1 dv1 = np.zeros((nn, Nodes[0])) w1 = 2 * np.random.rand(Nodes[0], Nodes[1]) - 1 dw1 = np.zeros((Nodes[0], Nodes[1])) v2 = 2 * np.random.rand(Nodes[1], Nodes[2]) - 1 dv2 = np.zeros((Nodes[1], Nodes[2])) w2 = 2 * np.random.rand(Nodes[2], nn) - 1 dw2 = np.zeros((Nodes[2], nn)) xlist = mdata expectlist = mdata count = 0 aa = alfa alpha = alpha0 y1, y2, y3, o, yitao = np.zeros((samplenum, Nodes[0])), np.zeros((samplenum, Nodes[1])), np.zeros((samplenum, Nodes[ 2])), np.zeros( (samplenum, nn)), np.zeros((samplenum, nn)) yitay1, yitay2, yitay3 = np.zeros((samplenum, Nodes[0])), np.zeros((samplenum, Nodes[1])), np.zeros((samplenum, Nodes[2])) while count < num_epochs: c = 0 while c < samplenum: d = expectlist[c, :] x = xlist[c, :] y1[c, :] = (1 / (1 + np.exp(-np.dot(x, v1)))) y2[c, :] = np.dot(y1[c, :], w1) y3[c, :] = (1 / (1 + np.exp(-np.dot(y2[c, :], v2)))) o[c, :] = np.dot(y3[c, :], w2) yitao[c, :] = (d - o[c, :]) errorp[c] = 0.5 * np.sum(yitao[c, :] * yitao[c, :]) yitay3[c, :] = (np.dot(yitao[c, :], w2.T) * y3[c, :] * (1 - y3[c, :])) yitay2[c, :] = np.dot(yitay3[c, :], v2.T) yitay1[c, :] = np.dot(yitay2[c, :], w1.T) * y1[c, :] * (1 - y1[c, :]) # 调整各层权值 deltw2 = np.dot(alpha * y3[c, :].reshape(Nodes[2], 1), yitao[c, :].reshape(1, nn)) w2 = w2 + deltw2 + aa * dw2 dw2 = deltw2 deltv2 = np.dot(alpha * y2[c, :].reshape(Nodes[1], 1), yitay3[c, :].reshape(1, Nodes[2])) v2 = v2 + deltv2 + aa * dv2 dv2 = deltv2 deltw1 = np.dot(alpha * y1[c, :].reshape(Nodes[0], 1), yitay2[c, :].reshape(1, Nodes[1])) w1 = w1 + deltw1 + aa * dw1 dw1 = deltw1 deltv1 = np.dot(alpha * x.reshape(nn, 1), yitay1[c, :].reshape(1, Nodes[0])) v1 = v1 + deltv1 + aa * dv1 dv1 = deltv1 c = c + 1 spe[count] = np.sum(errorp) / samplenum if count > 1 and abs(spe[count] - spe[count - 1]) < 0.0000001: break count += 1 sigma = 3 * np.sum((expectlist - o) * (expectlist - o)) / mm return count, spe, o, v1, v2, w1, w2, sigma def isnumber(limits): flag = True for item in limits: item = item.replace("-", "") if (item.isdigit() == False): flag = False break return flag def clearmain(info): try: Train_Data = info["Train_Data"] times = Train_Data["time"].split(';') points = Train_Data["points"].split(',') interval = Train_Data["interval"] if interval == 10000: DCount = 60 elif interval == 100000: DCount = 6 elif interval == 300000: DCount = 5 else: DCount = 4 dead = Train_Data["dead"].split(',') limit = Train_Data["limit"].split(',') uplower = Train_Data["uplow"].split(';') condition = info["conditon"].replace("=", "==").replace(">=", ">").replace("<=", "<") # percent = info["Hyper_para"]["percent"] count = 0 ItemsInfo, SamplingTimePeriods = [], [] Constraint = "" for i in range(len(points)): iteminfo = {} iteminfo["ItemName"] = points[i] # 加点 if (dead[i] == "1"): # 判断是否参与死区清洗 iteminfo["ClearDeadZone"] = "true" else: iteminfo["ClearDeadZone"] = "false" if (limit[i] == "1"): # 参与上下限清洗 limits = uplower[i].split(',') if (isnumber(limits) == True): # 输入上下限正确 count += 1 Constraint += "[" + points[i] + "]>" + limits[0] + " and " + "[" + points[i] + "]<" + limits[ 1] + " and " ItemsInfo.append(iteminfo) if (count != 0): Constraint = Constraint[:len(Constraint) - 4:] else: Constraint = "1==1" # 没有上下限清洗 Constraint += " and (" + condition + ")" Constraint = Constraint.replace("\n", " ") for i in range(len(times)): Eachsampletime = {} timess = times[i].split(',') Eachsampletime["StartingTime"] = timess[0] Eachsampletime["TerminalTime"] = timess[1] SamplingTimePeriods.append(Eachsampletime) url = f"http://{config._CLEAN_IP}/exawebapi/exatime/GetCleaningData?ItemsInfo=%s&SamplingTimePeriods=%s&Constraint=%s&SamplingPeriod=%s&DCount=%d" % ( ItemsInfo, SamplingTimePeriods, Constraint, interval, DCount) response = requests.get(url) content = json.loads(response.text) data = np.array([item for item in content["ClearData"]]).T try: smote_data = info["smote"] # smote_data = False except KeyError: smote_data = False if smote_data: try: smote_index = [points.index(item["pointId"]) for item in info["smote_config"] if item["LAY_CHECKED"]] smote_num = [int(item["number"]) for item in info["smote_config"] if item["LAY_CHECKED"]] max_value = [float(item["max"]) for item in info["smote_config"] if item["LAY_CHECKED"]] min_value = [float(item["min"]) for item in info["smote_config"] if item["LAY_CHECKED"]] except KeyError: pass else: if len(smote_num) != 0: data, *_ = smote(data, smote_index, smote_num, max_value, min_value) Nodes = info["layer"] num_epochs = Train_Data["interval"] result = AANN(data, Nodes, num_epochs) # 看看nodes和num_epochs怎么传进来 # result = pca(data, percent) result = result.replace("NaN", "-1") result = json.loads(result) result["BeforeCleanSamNum"] = content["BeforeCleanSamNum"] result["AfterCleanSamNum"] = content["AfterCleanSamNum"] result["CleanOrNot"] = True return json.dumps(result) except Exception as e: result = [{"CleanOrNot": False, "msg": traceback.format_exc()}] return json.dumps(result, ensure_ascii=False) if __name__ == "__main__": info_str = '{"layer":[5,2,5],"Train_Data":{"time":"2020-10-14 04:10:28,2020-10-17 14:53:00;2021-04-07 07:32:47,2021-04-16 08:39:01;2021-06-01 18:48:17,2021-06-03 14:29:40","points":"DH4_40MAG20CT362,DH4_40MAG20AN002GT,DH4_40MAG20CE102,DH4_40MAG20CT312,DH4_40MAG20CT322,DH4_40MAG20CT332","interval":300000,"dead":"1,1,1,1,1,1","limit":"0,0,0,0,0,0","uplow":"null,null;null,null;null,null;null,null;null,null;null,null"},"type":"AANN","conditon":"[DH4_40MAG20CE102]>20","epoch":"10000"}' info = json.loads(info_str) print(clearmain(info))