# -*- coding: utf-8 -*- """ @Time : 2019/9/12 15:30 @Author : 杰森·家乐森 @File : ANN_train.py @Software: PyCharm """ import datetime import json import os import time import config import matplotlib.pyplot as plt import numpy as np import pandas as pd import requests import tensorflow as tf from sklearn.preprocessing import MinMaxScaler from tensorflow.keras import backend from tensorflow.keras import layers from tensorflow.keras.models import load_model from tensorflow.keras.models import model_from_json os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" def get_history_value(points, time, interval, typedata): # url="http://192.168.1.201:8080/openPlant/getMultiplePointHistorys" url = f"http://{config._EXA_IP}:9000/exawebapi/exatime/GetSamplingValueArrayFloat" headers = {"Content-Type": "application/json;charset=utf-8"} # ,"token":get_token() point_array = points.split(",") time_span = time.split(";") value_array = [] for item in point_array: value_group = [] for time_piece in time_span: st = time_piece.split(",")[0] et = time_piece.split(",")[1] para = {"ItemName": item, "StartingTime": st, "TerminalTime": et, "SamplingPeriod": interval} response = requests.get(url, headers=headers, params=para) value = eval(str(response.text).replace("\"", "").replace("null", str(np.nan))) for row in value: value_group.append(row[1]) value_array.append(value_group) valuetrs = np.array(value_array) typeArr = list(enumerate(typedata.split(","))) data_x = [(valuetrs.T)[:, item[0]].tolist() for item in typeArr if item[1] == "0"] data_y = [(valuetrs.T)[:, item[0]].tolist() for item in typeArr if item[1] == "1"] x_data = np.array(data_x).T y_data = np.array(data_y).T return x_data, y_data, valuetrs def rmse(y_true, y_pred): return backend.sqrt(backend.mean(tf.keras.losses.mean_squared_error(y_true, y_pred), axis=-1)) def TrainOffline(x_data, y_data, hidden_layers, epochsdata): # 计算原来每一列数据的最大值 valuetrs = np.hstack((x_data, y_data)) # valuetrs = valuetrs train_max, train_min, train_ave = [], [], [] for row in valuetrs.T: train_max.append(np.max(row)) train_min.append(np.min(row)) train_ave.append(np.mean(row)) mms1 = MinMaxScaler() mms2 = MinMaxScaler() mms3 = MinMaxScaler() x_normal = mms1.fit_transform(x_data) y_normal = mms2.fit_transform(y_data) x_train = x_normal y_train = y_normal # 构建网络结构 model = tf.keras.Sequential() model.add(layers.Dense(units=hidden_layers[0], input_dim=x_data.shape[1], activation="sigmoid")) for i in range(len(hidden_layers) - 1): model.add(layers.Dense(units=hidden_layers[i + 1], activation="sigmoid")) model.add(layers.Dense(units=y_data.shape[1])) # 选择损失函数，优化方法 model.compile(loss="mse", optimizer="adam", metrics=["accuracy"]) # 训练模型 history = model.fit(x_train, y_train, epochs=epochsdata, batch_size=80) path = os.getcwd() + "\\ModelOline"; if not os.path.exists(path): os.makedirs(path) now = datetime.datetime.now() # filepath=path + "\\M"+"_"+str(now).replace(" ","-").replace(".","-").replace(":","-")+".h5" # model.save(filepath) filepath = model.to_json() model_weight = model.get_weights() test_data = model.predict(x_normal, batch_size=400) # 反归一化 predict_data = mms2.inverse_transform(test_data) with tf.compat.v1.Session(): spe = rmse(test_data, y_normal).eval() limit = 3 * spe mimusdata = predict_data - y_data mimusdatares = mimusdata.T pre_min, pre_max, pre_ave, pre_s = [], [], [], [] for row in mimusdatares: pre_max.append(np.max(row)) pre_min.append(np.min(row)) pre_ave.append(np.mean(row)) pre_s.append(np.std(row) * 3) result, mms1new, mms2new = {}, {}, {} mms1new["data_max_"] = mms1.data_max_.tolist() mms1new["data_min_"] = mms1.data_min_.tolist() mms1new["data_range_"] = mms1.data_range_.tolist() mms1new["min_"] = mms1.min_.tolist() mms1new["scale_"] = mms1.scale_.tolist() mms2new["data_max_"] = mms2.data_max_.tolist() mms2new["data_min_"] = mms2.data_min_.tolist() mms2new["data_range_"] = mms2.data_range_.tolist() mms2new["min_"] = mms2.min_.tolist() mms2new["scale_"] = mms2.scale_.tolist() result["filename"] = filepath result["mms1"] = mms1new result["mms2"] = mms2new result["train_max"] = np.array(train_max).tolist() result["train_min"] = np.array(train_min).tolist() result["train_ave"] = np.array(train_ave).tolist() result["pre_max"] = np.array(pre_max).tolist() result["pre_min"] = np.array(pre_min).tolist() result["pre_ave"] = np.array(pre_ave).tolist() result["pre_s"] = np.array(pre_s).tolist() result["pre_s_ave"] = np.mean(np.array(pre_s)).tolist() result["limit"] = limit result["weight"] = [model_weight[index].tolist() for index in range(len(model_weight))] return json.dumps(result) def Train(x_data, y_data, hidden_layers, valuetrs, epochsdata): # 计算原来每一列数据的最大值 # x_data = np.array(x_data) # y_data = np.array(y_data) valuetrs = valuetrs train_max, train_min, train_ave = [], [], [] for row in valuetrs.T: train_max.append(np.max(row)) train_min.append(np.min(row)) train_ave.append(np.mean(row)) mms1 = MinMaxScaler() mms2 = MinMaxScaler() mms3 = MinMaxScaler() x_normal = mms1.fit_transform(x_data) y_normal = mms2.fit_transform(y_data) x_train = x_normal y_train = y_normal # 构建网络结构 model = tf.keras.Sequential() model.add(layers.Dense(units=hidden_layers[0], input_dim=x_data.shape[1], activation="sigmoid")) for i in range(len(hidden_layers) - 1): model.add(layers.Dense(units=hidden_layers[i + 1], activation="sigmoid")) model.add(layers.Dense(units=y_data.shape[1])) model.summary() # 选择损失函数，优化方法 model.compile(loss="mse", optimizer="adam", metrics=["accuracy"]) # 训练模型 history = model.fit(x_train, y_train, epochs=epochsdata, batch_size=80) path = os.getcwd() + "\\ModelOline"; if not os.path.exists(path): os.makedirs(path) now = datetime.datetime.now() # filepath=path + "\\M"+"_"+str(now).replace(" ","-").replace(".","-").replace(":","-")+".h5" # model.save(filepath) filepath = model.to_json() model_weight = model.get_weights() test_data = model.predict(x_normal, batch_size=400) # 反归一化 predict_data = mms2.inverse_transform(test_data) with tf.compat.v1.Session(): spe = rmse(y_normal, test_data).eval() limit = 3 * spe mimusdata = predict_data - y_data mimusdatares = mimusdata.T pre_min, pre_max, pre_ave, pre_s = [], [], [], [] for row in mimusdatares: pre_max.append(np.max(row)) pre_min.append(np.min(row)) pre_ave.append(np.mean(row)) pre_s.append(np.std(row) * 3) result, mms1new, mms2new = {}, {}, {} mms1new["data_max_"] = mms1.data_max_.tolist() mms1new["data_min_"] = mms1.data_min_.tolist() mms1new["data_range_"] = mms1.data_range_.tolist() mms1new["min_"] = mms1.min_.tolist() mms1new["scale_"] = mms1.scale_.tolist() mms2new["data_max_"] = mms2.data_max_.tolist() mms2new["data_min_"] = mms2.data_min_.tolist() mms2new["data_range_"] = mms2.data_range_.tolist() mms2new["min_"] = mms2.min_.tolist() mms2new["scale_"] = mms2.scale_.tolist() result["filename"] = filepath result["mms1"] = mms1new result["mms2"] = mms2new result["train_max"] = np.array(train_max).tolist() result["train_min"] = np.array(train_min).tolist() result["train_ave"] = np.array(train_ave).tolist() result["pre_max"] = np.array(pre_max).tolist() result["pre_min"] = np.array(pre_min).tolist() result["pre_ave"] = np.array(pre_ave).tolist() result["pre_s"] = np.array(pre_s).tolist() result["pre_s_ave"] = np.mean(np.array(pre_s)).tolist() result["limit"] = limit result["weight"] = [model_weight[index].tolist() for index in range(len(model_weight))] return result def isnumber(limits): flag = True for item in limits: item = item.replace("-", "") if (item.isdigit() == False): flag = False break return flag def clearmain(info): try: points = info["point"].split(',') times = info["time"].split(';') epochs = info["iter"] layer = info["layer"] typedata = info["type"] condition = info["condition"].replace("=", "==").replace(">=", ">").replace("<=", "<") interval = 300000 dead = info["dead"].split(',') limit = info["limit"].split(',') uplower = info["uplow"].split(';') count = 0 ItemsInfo, SamplingTimePeriods = [], [] Constraint = "" for i in range(len(points)): iteminfo = {} iteminfo["ItemName"] = points[i] # 加点 if dead[i] == "1": # 判断是否参与死区清洗 iteminfo["ClearDeadZone"] = "true" else: iteminfo["ClearDeadZone"] = "false" if limit[i] == "1": # 参与上下限清洗 limits = uplower[i].split(',') if (isnumber(limits) == True): # 输入上下限正确 count += 1 Constraint += "[" + points[i] + "]>" + limits[0] + " and " + "[" + points[i] + "]<" + limits[ 1] + " and " ItemsInfo.append(iteminfo) if count != 0: Constraint = Constraint[:len(Constraint) - 4:] else: Constraint = "1==1" # 没有上下限清洗 Constraint += " and (" + condition + ")" Constraint = Constraint.replace("\n", " ") for i in range(len(times)): Eachsampletime = {} timess = times[i].split(',') Eachsampletime["StartingTime"] = timess[0] Eachsampletime["TerminalTime"] = timess[1] SamplingTimePeriods.append(Eachsampletime) url = f"http://{config._CLEAN_IP}/exawebapi/exatime/GetCleaningData?ItemsInfo=%s&SamplingTimePeriods=%s&Constraint=%s&SamplingPeriod=%s&DCount=6" % ( ItemsInfo, SamplingTimePeriods, Constraint, interval) response = requests.get(url) content = json.loads(response.text) # data =np.array([item for item in content["ClearData"] if item ]).T valuetrs = np.array([item for item in content["ClearData"]]).T typeArr = list(enumerate(typedata.split(","))) data_x = [valuetrs[:, item[0]].tolist() for item in typeArr if item[1] == "0"] data_y = [valuetrs[:, item[0]].tolist() for item in typeArr if item[1] == "1"] data_x = np.array(data_x).T data_y = np.array(data_y).T result = Train(data_x, data_y, layer, valuetrs, eval(epochs)) result["BeforeCleanSamNum"] = content["BeforeCleanSamNum"] result["AfterCleanSamNum"] = content["AfterCleanSamNum"] result["CleanOrNot"] = True return result except Exception as e: result = {"CleanOrNot": False} return result def main(info): points = info["point"] time1 = info["time"] interval = info["interval"] epochs = info["iter"] layer = info["layer"] datatype = info["type"] data_x, data_y, origndata = get_history_value(points, time1, interval, datatype) result = Train(data_x, data_y, layer, origndata, eval(epochs)) return result if __name__ == "__main__": # info_str = '{"time":"2020-01-19 23:26:04,2020-01-25 20:42:16;2020-01-26 16:23:49,2020-02-03 11:36:42;2020-02-05 20:02:49,2020-02-06 05:51:40","condition":"1=1","interval":300000,"dead":"1,1,1","layer":["6"],"point":"JL_D1_10MILLA:SEP_TEMP.PNT,JL_D1_10FSSS20A:HFC10CT301.PNT,JL_D1_10FSSS20A:HFC10CT302.PNT","type":"1,0,0","limit":"0,0,0","uplow":",;,;,","iter":"100"}' # info_str = '{"iter":"800","dead":"1,1,1,1","point":"DH4_40HLF10CE101,DH4_40HLF10CT351,DH4_40HLF10CT352,DH4_40HLF10CY101","limit":"0,0,0,0","layer":["5"],"type":"0,0,0,1","time":"2020-10-06 19:17:46,2020-10-12 14:58:19","condition":"1=1","interval":300000,"uplow":"null,null;null,null;null,null;null,null"}' # info = json.loads(info_str) # result = clearmain(info) # print(result) info = {"iter":"800","Train_Data_X":[[7.0,7.0,7.0,7.0,7.0,7.0,7.0],[8.0,8.0,8.0,8.0,8.0,8.0,8.0],[1.0,1.0,1.0,1.0,1.0,1.0,1.0],[2.0,2.0,2.0,2.0,2.0,2.0,2.0],[3.0,3.0,3.0,3.0,3.0,3.0,3.0],[4.0,4.0,4.0,4.0,4.0,4.0,4.0],[5.0,5.0,5.0,5.0,5.0,5.0,5.0],[6.0,6.0,6.0,6.0,6.0,6.0,6.0],[7.0,7.0,7.0,7.0,7.0,7.0,7.0],[8.0,8.0,8.0,8.0,8.0,8.0,8.0],[1.0,1.0,1.0,1.0,1.0,1.0,1.0],[2.0,2.0,2.0,2.0,2.0,2.0,2.0]],"hide":["7","5","1"],"Train_Data_Y":[[7.0],[8.0],[1.0],[2.0],[3.0],[4.0],[5.0],[6.0],[7.0],[8.0],[1.0],[2.0]]} result = TrainOffline(np.array(info["Train_Data_X"]), np.array(info["Train_Data_Y"]),info["hide"], eval(info["iter"]))