Ažurirana funkcija manageH5 u newmlservice.py fajlu i dodata funkcija za njeno pozivanje u controller.py fajlu.

2 files changed, 87 insertions, 65 deletions

diff --git a/backend/microservice/api/controller.py b/backend/microservice/api/controller.py
index 8d49fcc4..73f4ed31 100644
--- a/backend/microservice/api/controller.py
+++ b/backend/microservice/api/controller.py
@@ -96,7 +96,7 @@ def train():
         "h5FileId" : fileId,
         "metrics" : m
     }
-    print(predictor)
+    #print(predictor)
     url = config.api_url + "/Predictor/add"
     r = requests.post(url, json=predictor).text
     print(r)
@@ -109,9 +109,32 @@ def predict():
     paramsExperiment = json.loads(request.form["experiment"])
     paramsPredictor = json.loads(request.form["predictor"])
     print("********************************model loaded*******************************")
-    result = newmlservice.predict(paramsExperiment, paramsPredictor, model)
+    result = newmlservice.predict(paramsExperiment, paramsPredictor, model,train_callback)
     return result
 
+@app.route('/manageH5', methods = ['POST'])
+def manageH5():
+    h5 = request.files.get("h5file")
+    model = tf.keras.models.load_model(h5)
+
+    paramsExperiment = json.loads(request.form["experiment"])
+    paramsModel = json.loads(request.form["model"])
+    paramsDataset = json.loads(request.form["dataset"])
+    
+    f = request.files.get("file")
+    if(paramsDataset['delimiter']=='novi red'):
+        separation='\n'
+    elif(paramsDataset['delimiter']=='razmak'):
+        separation=' '
+    else:
+        separation=paramsDataset['delimiter']
+    
+    data = pd.read_csv(f,sep=separation)
+    
+    result = newmlservice.manageH5(data,paramsModel,paramsExperiment,paramsDataset,model,train_callback)
+    return result
+
+
 @app.route('/preprocess',methods=['POST'])
 def returnColumnsInfo():
     print("********************************PREPROCESS*******************************")
diff --git a/backend/microservice/api/newmlservice.py b/backend/microservice/api/newmlservice.py
index 2f08d4b4..560e575f 100644
--- a/backend/microservice/api/newmlservice.py
+++ b/backend/microservice/api/newmlservice.py
@@ -24,7 +24,7 @@ from dataclasses import dataclass
 import statistics as s
 from sklearn.metrics import roc_auc_score
 import matplotlib.pyplot as plt
-#from ann_visualizer.visualize import ann_viz;
+from ann_visualizer.visualize import ann_viz;
 def returnColumnsInfo(dataset):
     dict=[]
     
@@ -301,7 +301,7 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback):
     ###OPTIMIZATORI
    
     if(paramsModel['optimizer']=='Adam'):
-        opt=tf.keras.optimizers.Adam(learning_rate=float(paramsModel['learningRate']))
+        opt=tf.keras.optimizers.Adam(learning_rate=3)
 
     elif(paramsModel['optimizer']=='Adadelta'):
         opt=tf.keras.optimizers.Adadelta(learning_rate=float(paramsModel['learningRate']))
@@ -331,7 +331,7 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback):
     filepath=os.path.join("temp/",paramsExperiment['_id']+"_"+paramsModel['_id']+".h5")
     if(problem_type=='multi-klasifikacioni'):
         #print('multi')
-        
+        print(paramsModel)
         reg=paramsModel['layers'][0]['regularisation']
         regRate=float(paramsModel['layers'][0]['regularisationRate'])
         if(reg=='l1'):
@@ -365,10 +365,10 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback):
 
         
 
-        classifier.compile(loss =paramsModel["lossFunction"] , optimizer = opt, metrics = ['accuracy','mae','mse'])
+        classifier.compile(loss =paramsModel["lossFunction"] , optimizer =opt, metrics = ['accuracy','mae','mse'])
 
         history=classifier.fit(x_train, y_train, epochs = paramsModel['epochs'],batch_size=int(paramsModel['batchSize']),callbacks=callback(x_test, y_test,paramsModel['_id']))
-     
+
         hist=history.history
         #plt.plot(hist['accuracy'])
         #plt.show()
@@ -415,11 +415,11 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback):
                 kernelreg=tf.keras.regularizers.l2(regRate)
                 biasreg=tf.keras.regularizers.l2(regRate)
                 activityreg=tf.keras.regularizers.l2(regRate)
-            classifier.add(tf.keras.layers.Dense(units=paramsModel['layers'][i+1]['neurons'], activation=paramsModel['layers'][i+1]['activationFunction'],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#i-ti skriveni sloj
+            classifier.add(tf.keras.layers.Dense(units=paramsModel['layers'][i+1]['neurons'], activation=paramsModel['layers'][i+1]['activationFunction']))#i-ti skriveni sloj
         
         classifier.add(tf.keras.layers.Dense(units=1, activation=paramsModel['outputLayerActivationFunction']))#izlazni sloj
 
-        classifier.compile(loss =paramsModel["lossFunction"] , optimizer = opt , metrics = ['accuracy','mae','mse'])
+        classifier.compile(loss =paramsModel["lossFunction"] , optimizer =opt , metrics = ['accuracy','mae','mse'])
 
         history=classifier.fit(x_train, y_train, epochs = paramsModel['epochs'],batch_size=int(paramsModel['batchSize']),callbacks=callback(x_test, y_test,paramsModel['_id']))
         hist=history.history
@@ -431,7 +431,22 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback):
         
         scores = classifier.evaluate(x_test, y_test)
         #print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100))
-        #ann_viz(classifier, title="My neural network")
+        #
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # 
+        # ann_viz(classifier, title="My neural network")
         
         classifier.save(filepath, save_format='h5')
         return filepath,hist
@@ -556,32 +571,36 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback):
         macro_averaged_f1=metrics.f1_score(y_test, y_pred, average = 'macro')
         micro_averaged_f1=metrics.f1_score(y_test, y_pred, average = 'micro')
         roc_auc_dict=roc_auc_score_multiclass(y_test, y_pred)
-    '''
+'''   
 def predict(experiment, predictor, model):
     #model.predict()
     # ovo je pre bilo manageH5 
     return "TODO"
 
 
-def manageH5(dataset,params,h5model):
-    problem_type = params["type"]
-    #print(problem_type)
+def manageH5(dataset,paramsModel,paramsExperiment,paramsDataset,h5model,callback):
+    problem_type = paramsModel["type"]
     data = pd.DataFrame()
-    #print(data)
-    for col in params["inputColumns"]:
-        #print(col)
-        data[col]=dataset[col]
-    output_column = params["columnToPredict"]
+    for col in paramsExperiment["inputColumns"]:
+        if(col!=paramsExperiment["outputColumn"]):
+            data[col]=dataset[col]
+    output_column = paramsExperiment["outputColumn"]
     data[output_column] = dataset[output_column]
-    #print(data)
 
-    ###NULL
-    null_value_options = params["nullValues"]
-    null_values_replacers = params["nullValuesReplacers"]
+    kategorijskekolone=[]
+    columnInfo=paramsDataset['columnInfo']
+    columnTypes=paramsExperiment['columnTypes']
+    for i in range(len(columnInfo)):
+        col=columnInfo[i]
+        if(columnTypes[i]=="categorical" and col['columnName'] in paramsExperiment['inputColumns']):
+            data[col['columnName']]=data[col['columnName']].apply(str)
+            kategorijskekolone.append(col['columnName'])
+
+    null_value_options = paramsExperiment["nullValues"]
+    null_values_replacers = paramsExperiment["nullValuesReplacers"]
     
     if(null_value_options=='replace'):
-        #print("replace null") # TODO
-        dict=params['null_values_replacers']
+        dict=null_values_replacers
         while(len(dict)>0):
             replace=dict.pop()
             col=replace['column']
@@ -592,58 +611,38 @@ def manageH5(dataset,params,h5model):
     elif(null_value_options=='delete_rows'):
         data=data.dropna()
     elif(null_value_options=='delete_columns'):
-        data=data.dropna()
-    
-    #print(data.shape)
-    
-    #
-    # Brisanje kolona koje ne uticu na rezultat
-    #
-    num_rows=data.shape[0]
-    for col in data.columns:
-        if((data[col].nunique()==(num_rows)) and (data[col].dtype==np.object_)):
-            data.pop(col)
-    #
-    ### Enkodiranje
-    encoding=params["encoding"]
-    if(encoding=='label'):
-        encoder=LabelEncoder()
-        for col in data.columns:
-            if(data[col].dtype==np.object_):
-                data[col]=encoder.fit_transform(data[col])
-    
-    
-    elif(encoding=='onehot'):
-        category_columns=[]
-        for col in data.columns:
-            if(data[col].dtype==np.object_):
-                category_columns.append(col)
-        data=pd.get_dummies(data, columns=category_columns, prefix=category_columns)
-    #print(data)
+        data=data.dropna(axis=1)
+
+    encodings=paramsExperiment["encodings"]
+    for kolonaEncoding in encodings:
+        kolona = kolonaEncoding["columnName"]
+        encoding = kolonaEncoding["encoding"]
+        if(kolona in kategorijskekolone):
+            if(encoding=='label'):
+                encoder=LabelEncoder()
+                for col in data.columns:
+                    if(data[col].dtype==np.object_):
+                        data[col]=encoder.fit_transform(data[col])
+            elif(encoding=='onehot'):
+                category_columns=[]
+                for col in data.columns:
+                    if(data[col].dtype==np.object_):
+                        category_columns.append(col)
+                data=pd.get_dummies(data, columns=category_columns, prefix=category_columns)
 
-    #
-    # Input - output
-    #
     x_columns = []
     for col in data.columns:
         if(col!=output_column):
             x_columns.append(col)
-    #print(x_columns)
     x2 = data[x_columns]
-    #print(x2)
-    #print(x2.values)
     x2 = data[x_columns].values
-    #print(x2)
     y2 = data[output_column].values
-    h5model.summary()
+    #h5model.summary()
     #ann_viz(h5model, title="My neural network")
 
-    h5model.compile(loss=params['lossFunction'], optimizer=params['optimizer'], metrics = ['accuracy','mae','mse'])
 
-    history=h5model.fit(x2, y2, epochs = params['epochs'],batch_size=int(params['batchSize']))
-    
+    history=h5model.fit(x2, y2, epochs = paramsModel['epochs'],batch_size=int(paramsModel['batchSize']),callbacks=callback(x2, y2,paramsModel['_id']))  
     y_pred2=h5model.predict(x2)
-     
     y_pred2=np.argmax(y_pred2,axis=1)
     #y_pred=h5model.predict_classes(x)
     score = h5model.evaluate(x2,y_pred2, verbose=0)