You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
301 lines
13 KiB
301 lines
13 KiB
2 weeks ago
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
@Time : 2019/9/12 15:30
|
||
|
|
@Author : 杰森·家乐森
|
||
|
|
@File : ANN_train.py
|
||
|
|
@Software: PyCharm
|
||
|
|
"""
|
||
|
|
import os
|
||
|
|
import json
|
||
|
|
import time
|
||
|
|
import datetime
|
||
|
|
import requests
|
||
|
|
import numpy as np
|
||
|
|
import pandas as pd
|
||
|
|
import tensorflow as tf
|
||
|
|
from tensorflow.keras import backend
|
||
|
|
import matplotlib.pyplot as plt
|
||
|
|
from tensorflow.keras import layers
|
||
|
|
from sklearn.preprocessing import MinMaxScaler
|
||
|
|
from tensorflow.keras.models import load_model
|
||
|
|
from tensorflow.keras.models import model_from_json
|
||
|
|
import config
|
||
|
|
|
||
|
|
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
|
||
|
|
|
||
|
|
def get_history_value(points, time, interval,typedata):
|
||
|
|
# url="http://192.168.1.201:8080/openPlant/getMultiplePointHistorys"
|
||
|
|
url = f"http://{config._EXA_IP}:9000/exawebapi/exatime/GetSamplingValueArrayFloat"
|
||
|
|
headers = {"Content-Type": "application/json;charset=utf-8"} # ,"token":get_token()
|
||
|
|
point_array = points.split(",")
|
||
|
|
time_span = time.split(";")
|
||
|
|
value_array = []
|
||
|
|
for item in point_array:
|
||
|
|
value_group = []
|
||
|
|
for time_piece in time_span:
|
||
|
|
st = time_piece.split(",")[0]
|
||
|
|
et = time_piece.split(",")[1]
|
||
|
|
para = {"ItemName": item, "StartingTime": st, "TerminalTime": et, "SamplingPeriod": interval}
|
||
|
|
response = requests.get(url, headers=headers, params=para)
|
||
|
|
value = eval(str(response.text).replace("\"", "").replace("null", str(np.nan)))
|
||
|
|
for row in value:
|
||
|
|
value_group.append(row[1])
|
||
|
|
value_array.append(value_group)
|
||
|
|
valuetrs=np.array(value_array)
|
||
|
|
typeArr = list(enumerate(typedata.split(",")))
|
||
|
|
data_x = [(valuetrs.T)[:, item[0]].tolist() for item in typeArr if item[1] == "0"]
|
||
|
|
data_y = [(valuetrs.T)[:, item[0]].tolist() for item in typeArr if item[1] == "1"]
|
||
|
|
x_data = np.array(data_x).T
|
||
|
|
y_data = np.array(data_y).T
|
||
|
|
return x_data,y_data,valuetrs
|
||
|
|
|
||
|
|
def rmse(y_true, y_pred):
|
||
|
|
return backend.sqrt(backend.mean(tf.keras.losses.mean_squared_error(y_true, y_pred), axis=-1))
|
||
|
|
|
||
|
|
|
||
|
|
def TrainOffline(x_data,y_data,hidden_layers,epochsdata):
|
||
|
|
#计算原来每一列数据的最大值
|
||
|
|
valuetrs = np.hstack((x_data, y_data))
|
||
|
|
# valuetrs = valuetrs
|
||
|
|
train_max, train_min, train_ave = [], [], []
|
||
|
|
for row in valuetrs.T:
|
||
|
|
train_max.append(np.max(row))
|
||
|
|
train_min.append(np.min(row))
|
||
|
|
train_ave.append(np.mean(row))
|
||
|
|
mms1 = MinMaxScaler()
|
||
|
|
mms2 = MinMaxScaler()
|
||
|
|
mms3 = MinMaxScaler()
|
||
|
|
x_normal = mms1.fit_transform(x_data)
|
||
|
|
y_normal = mms2.fit_transform(y_data)
|
||
|
|
x_train = x_normal
|
||
|
|
y_train = y_normal
|
||
|
|
# 构建网络结构
|
||
|
|
model = tf.keras.Sequential()
|
||
|
|
model.add(layers.Dense(units=hidden_layers[0], input_dim=x_data.shape[1], activation="sigmoid"))
|
||
|
|
for i in range(len(hidden_layers) - 1):
|
||
|
|
model.add(layers.Dense(units=hidden_layers[i + 1], activation="sigmoid"))
|
||
|
|
model.add(layers.Dense(units=y_data.shape[1]))
|
||
|
|
# 选择损失函数，优化方法
|
||
|
|
model.compile(loss="mse", optimizer="adam", metrics=["accuracy"])
|
||
|
|
# 训练模型
|
||
|
|
history = model.fit(x_train, y_train, epochs=epochsdata, batch_size=80)
|
||
|
|
path = os.getcwd() + "\\ModelOline";
|
||
|
|
if not os.path.exists(path):
|
||
|
|
os.makedirs(path)
|
||
|
|
now = datetime.datetime.now()
|
||
|
|
# filepath=path + "\\M"+"_"+str(now).replace(" ","-").replace(".","-").replace(":","-")+".h5"
|
||
|
|
# model.save(filepath)
|
||
|
|
filepath = model.to_json()
|
||
|
|
model_weight = model.get_weights()
|
||
|
|
test_data = model.predict(x_normal, batch_size=400)
|
||
|
|
|
||
|
|
# 反归一化
|
||
|
|
predict_data = mms2.inverse_transform(test_data)
|
||
|
|
with tf.compat.v1.Session():
|
||
|
|
spe = rmse(test_data, y_normal).eval()
|
||
|
|
limit = 3 * spe
|
||
|
|
mimusdata = predict_data - y_data
|
||
|
|
mimusdatares = mimusdata.T
|
||
|
|
pre_min, pre_max, pre_ave, pre_s = [], [], [], []
|
||
|
|
for row in mimusdatares:
|
||
|
|
pre_max.append(np.max(row))
|
||
|
|
pre_min.append(np.min(row))
|
||
|
|
pre_ave.append(np.mean(row))
|
||
|
|
pre_s.append(np.std(row) * 3)
|
||
|
|
result, mms1new, mms2new = {}, {}, {}
|
||
|
|
mms1new["data_max_"] = mms1.data_max_.tolist()
|
||
|
|
mms1new["data_min_"] = mms1.data_min_.tolist()
|
||
|
|
mms1new["data_range_"] = mms1.data_range_.tolist()
|
||
|
|
mms1new["min_"] = mms1.min_.tolist()
|
||
|
|
mms1new["scale_"] = mms1.scale_.tolist()
|
||
|
|
mms2new["data_max_"] = mms2.data_max_.tolist()
|
||
|
|
mms2new["data_min_"] = mms2.data_min_.tolist()
|
||
|
|
mms2new["data_range_"] = mms2.data_range_.tolist()
|
||
|
|
mms2new["min_"] = mms2.min_.tolist()
|
||
|
|
mms2new["scale_"] = mms2.scale_.tolist()
|
||
|
|
result["filename"] = filepath
|
||
|
|
result["mms1"] = mms1new
|
||
|
|
result["mms2"] = mms2new
|
||
|
|
result["train_max"] = np.array(train_max).tolist()
|
||
|
|
result["train_min"] = np.array(train_min).tolist()
|
||
|
|
result["train_ave"] = np.array(train_ave).tolist()
|
||
|
|
result["pre_max"] = np.array(pre_max).tolist()
|
||
|
|
result["pre_min"] = np.array(pre_min).tolist()
|
||
|
|
result["pre_ave"] = np.array(pre_ave).tolist()
|
||
|
|
result["pre_s"] = np.array(pre_s).tolist()
|
||
|
|
result["pre_s_ave"] = np.mean(np.array(pre_s)).tolist()
|
||
|
|
result["limit"] = limit
|
||
|
|
result["weight"] = [model_weight[index].tolist() for index in range(len(model_weight))]
|
||
|
|
return json.dumps(result)
|
||
|
|
|
||
|
|
def Train(x_data,y_data,hidden_layers,valuetrs,epochsdata):
|
||
|
|
# 计算原来每一列数据的最大值
|
||
|
|
#x_data = np.array(x_data)
|
||
|
|
#y_data = np.array(y_data)
|
||
|
|
valuetrs=valuetrs
|
||
|
|
train_max, train_min, train_ave = [], [], []
|
||
|
|
for row in valuetrs.T:
|
||
|
|
train_max.append(np.max(row))
|
||
|
|
train_min.append(np.min(row))
|
||
|
|
train_ave.append(np.mean(row))
|
||
|
|
mms1 = MinMaxScaler()
|
||
|
|
mms2 = MinMaxScaler()
|
||
|
|
mms3=MinMaxScaler()
|
||
|
|
x_normal = mms1.fit_transform(x_data)
|
||
|
|
y_normal = mms2.fit_transform(y_data)
|
||
|
|
x_train = x_normal
|
||
|
|
y_train = y_normal
|
||
|
|
# 构建网络结构
|
||
|
|
model = tf.keras.Sequential()
|
||
|
|
model.add(layers.Dense(units=hidden_layers[0], input_dim=x_data.shape[1], activation="sigmoid"))
|
||
|
|
for i in range(len(hidden_layers) - 1):
|
||
|
|
model.add(layers.Dense(units=hidden_layers[i + 1], activation="sigmoid"))
|
||
|
|
model.add(layers.Dense(units=y_data.shape[1]))
|
||
|
|
model.summary()
|
||
|
|
# 选择损失函数，优化方法
|
||
|
|
model.compile(loss="mse", optimizer="adam", metrics=["accuracy"])
|
||
|
|
# 训练模型
|
||
|
|
history = model.fit(x_train, y_train, epochs=epochsdata, batch_size=80)
|
||
|
|
path=os.getcwd()+"\\ModelOline";
|
||
|
|
if not os.path.exists(path):
|
||
|
|
os.makedirs(path)
|
||
|
|
now=datetime.datetime.now()
|
||
|
|
#filepath=path + "\\M"+"_"+str(now).replace(" ","-").replace(".","-").replace(":","-")+".h5"
|
||
|
|
#model.save(filepath)
|
||
|
|
filepath=model.to_json()
|
||
|
|
model_weight=model.get_weights()
|
||
|
|
test_data = model.predict(x_normal, batch_size=400)
|
||
|
|
|
||
|
|
|
||
|
|
# 反归一化
|
||
|
|
predict_data = mms2.inverse_transform(test_data)
|
||
|
|
with tf.compat.v1.Session():
|
||
|
|
spe = rmse(y_normal, test_data).eval()
|
||
|
|
limit = 3 * spe
|
||
|
|
|
||
|
|
mimusdata = predict_data - y_data
|
||
|
|
mimusdatares = mimusdata.T
|
||
|
|
pre_min, pre_max, pre_ave, pre_s = [], [], [], []
|
||
|
|
for row in mimusdatares:
|
||
|
|
pre_max.append(np.max(row))
|
||
|
|
pre_min.append(np.min(row))
|
||
|
|
pre_ave.append(np.mean(row))
|
||
|
|
pre_s.append(np.std(row) * 3)
|
||
|
|
result,mms1new,mms2new={},{},{}
|
||
|
|
mms1new["data_max_"]=mms1.data_max_.tolist()
|
||
|
|
mms1new["data_min_"]=mms1.data_min_.tolist()
|
||
|
|
mms1new["data_range_"]=mms1.data_range_.tolist()
|
||
|
|
mms1new["min_"]=mms1.min_.tolist()
|
||
|
|
mms1new["scale_"]=mms1.scale_.tolist()
|
||
|
|
mms2new["data_max_"] = mms2.data_max_.tolist()
|
||
|
|
mms2new["data_min_"] = mms2.data_min_.tolist()
|
||
|
|
mms2new["data_range_"] = mms2.data_range_.tolist()
|
||
|
|
mms2new["min_"] = mms2.min_.tolist()
|
||
|
|
mms2new["scale_"] = mms2.scale_.tolist()
|
||
|
|
result["filename"]=filepath
|
||
|
|
result["mms1"]=mms1new
|
||
|
|
result["mms2"]=mms2new
|
||
|
|
result["train_max"] = np.array(train_max).tolist()
|
||
|
|
result["train_min"] = np.array(train_min).tolist()
|
||
|
|
result["train_ave"] = np.array(train_ave).tolist()
|
||
|
|
result["pre_max"] = np.array(pre_max).tolist()
|
||
|
|
result["pre_min"] = np.array(pre_min).tolist()
|
||
|
|
result["pre_ave"] = np.array(pre_ave).tolist()
|
||
|
|
result["pre_s"] = np.array(pre_s).tolist()
|
||
|
|
result["pre_s_ave"] = np.mean(np.array(pre_s)).tolist()
|
||
|
|
result["limit"] = limit
|
||
|
|
result["weight"] = [model_weight[index].tolist() for index in range(len(model_weight))]
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def isnumber(limits):
|
||
|
|
flag=True
|
||
|
|
for item in limits:
|
||
|
|
item=item.replace("-","")
|
||
|
|
if(item.isdigit()==False):
|
||
|
|
flag=False
|
||
|
|
break
|
||
|
|
return flag
|
||
|
|
|
||
|
|
def clearmain(info):
|
||
|
|
try:
|
||
|
|
points = info["point"].split(',')
|
||
|
|
times = info["time"].split(';')
|
||
|
|
epochs = info["iter"]
|
||
|
|
layer = info["layer"]
|
||
|
|
typedata = info["type"]
|
||
|
|
condition=info["condition"].replace("=","==").replace(">=",">").replace("<=","<")
|
||
|
|
interval = 300000
|
||
|
|
dead = info["dead"].split(',')
|
||
|
|
limit = info["limit"].split(',')
|
||
|
|
uplower = info["uplow"].split(';')
|
||
|
|
count=0
|
||
|
|
ItemsInfo, SamplingTimePeriods = [], []
|
||
|
|
Constraint = ""
|
||
|
|
for i in range(len(points)):
|
||
|
|
iteminfo = {}
|
||
|
|
iteminfo["ItemName"] = points[i] # 加点
|
||
|
|
if dead[i] == "1": # 判断是否参与死区清洗
|
||
|
|
iteminfo["ClearDeadZone"] = "true"
|
||
|
|
else:
|
||
|
|
iteminfo["ClearDeadZone"] = "false"
|
||
|
|
if limit[i] == "1": # 参与上下限清洗
|
||
|
|
limits = uplower[i].split(',')
|
||
|
|
if (isnumber(limits) == True): # 输入上下限正确
|
||
|
|
count += 1
|
||
|
|
Constraint += "[" + points[i] + "]>" + limits[0] + " and " + "[" + points[i] + "]<" + limits[1] + " and "
|
||
|
|
ItemsInfo.append(iteminfo)
|
||
|
|
if count!=0:
|
||
|
|
Constraint = Constraint[:len(Constraint) - 4:]
|
||
|
|
else:
|
||
|
|
Constraint="1==1"#没有上下限清洗
|
||
|
|
Constraint+=" and ("+condition+")"
|
||
|
|
Constraint = Constraint.replace("\n", " ")
|
||
|
|
for i in range(len(times)):
|
||
|
|
Eachsampletime = {}
|
||
|
|
timess = times[i].split(',')
|
||
|
|
Eachsampletime["StartingTime"] = timess[0]
|
||
|
|
Eachsampletime["TerminalTime"] = timess[1]
|
||
|
|
SamplingTimePeriods.append(Eachsampletime)
|
||
|
|
url = f"http://{config._CLEAN_IP}/exawebapi/exatime/GetCleaningData?ItemsInfo=%s&SamplingTimePeriods=%s&Constraint=%s&SamplingPeriod=%s&DCount=6" % (
|
||
|
|
ItemsInfo, SamplingTimePeriods, Constraint, interval)
|
||
|
|
response = requests.get(url)
|
||
|
|
content = json.loads(response.text)
|
||
|
|
# data =np.array([item for item in content["ClearData"] if item ]).T
|
||
|
|
valuetrs = np.array([item for item in content["ClearData"]]).T
|
||
|
|
typeArr = list(enumerate(typedata.split(",")))
|
||
|
|
data_x = [valuetrs[:,item[0]].tolist() for item in typeArr if item[1]=="0"]
|
||
|
|
data_y = [valuetrs[:,item[0]].tolist() for item in typeArr if item[1]=="1"]
|
||
|
|
data_x = np.array(data_x).T
|
||
|
|
data_y = np.array(data_y).T
|
||
|
|
result = Train(data_x, data_y, layer, valuetrs, eval(epochs))
|
||
|
|
result["BeforeCleanSamNum"]=content["BeforeCleanSamNum"]
|
||
|
|
result["AfterCleanSamNum"]=content["AfterCleanSamNum"]
|
||
|
|
result["CleanOrNot"] = True
|
||
|
|
return result
|
||
|
|
except Exception as e:
|
||
|
|
result = {"CleanOrNot": False}
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def main(info):
|
||
|
|
points = info["point"]
|
||
|
|
time1 = info["time"]
|
||
|
|
interval = info["interval"]
|
||
|
|
epochs = info["iter"]
|
||
|
|
layer = info["layer"]
|
||
|
|
datatype = info["type"]
|
||
|
|
data_x, data_y, origndata = get_history_value(points, time1, interval, datatype)
|
||
|
|
result = Train(data_x, data_y, layer, origndata, eval(epochs))
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
# info_str = '{"time":"2020-01-19 23:26:04,2020-01-25 20:42:16;2020-01-26 16:23:49,2020-02-03 11:36:42;2020-02-05 20:02:49,2020-02-06 05:51:40","condition":"1=1","interval":300000,"dead":"1,1,1","layer":["6"],"point":"JL_D1_10MILLA:SEP_TEMP.PNT,JL_D1_10FSSS20A:HFC10CT301.PNT,JL_D1_10FSSS20A:HFC10CT302.PNT","type":"1,0,0","limit":"0,0,0","uplow":",;,;,","iter":"100"}'
|
||
|
|
info_str = '{"iter":"800","dead":"1,1,1,1","point":"DH4_40HLF10CE101,DH4_40HLF10CT351,DH4_40HLF10CT352,DH4_40HLF10CY101","limit":"0,0,0,0","layer":["5"],"type":"0,0,0,1","time":"2020-10-06 19:17:46,2020-10-12 14:58:19","condition":"1=1","interval":300000,"uplow":"null,null;null,null;null,null;null,null"}'
|
||
|
|
info = json.loads(info_str)
|
||
|
|
result = clearmain(info)
|
||
|
|
print(result)
|
||
|
|
# info = {"iter":"800","Train_Data_X":[[7.0,7.0,7.0,7.0,7.0,7.0,7.0],[8.0,8.0,8.0,8.0,8.0,8.0,8.0],[1.0,1.0,1.0,1.0,1.0,1.0,1.0],[2.0,2.0,2.0,2.0,2.0,2.0,2.0],[3.0,3.0,3.0,3.0,3.0,3.0,3.0],[4.0,4.0,4.0,4.0,4.0,4.0,4.0],[5.0,5.0,5.0,5.0,5.0,5.0,5.0],[6.0,6.0,6.0,6.0,6.0,6.0,6.0],[7.0,7.0,7.0,7.0,7.0,7.0,7.0],[8.0,8.0,8.0,8.0,8.0,8.0,8.0],[1.0,1.0,1.0,1.0,1.0,1.0,1.0],[2.0,2.0,2.0,2.0,2.0,2.0,2.0]],"hide":["7","5","1"],"Train_Data_Y":[[7.0],[8.0],[1.0],[2.0],[3.0],[4.0],[5.0],[6.0],[7.0],[8.0],[1.0],[2.0]]}
|
||
|
|
# result = TrainOffline(np.array(info["Train_Data_X"]), np.array(info["Train_Data_Y"]),info["hide"], eval(info["iter"]))
|