Povezao treniranje modela.

15 files changed, 244 insertions, 17 deletions

diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs
index b997efa3..4bc094cd 100644
--- a/backend/api/api/Controllers/ModelController.cs
+++ b/backend/api/api/Controllers/ModelController.cs
@@ -14,22 +14,28 @@ namespace api.Controllers
     {
 
         private IMlConnectionService _mlService;
+        private readonly IDatasetService _datasetService;
+        private readonly IFileService _fileService;
         private readonly IModelService _modelService;
         private JwtToken jwtToken;
 
 
-        public ModelController(IMlConnectionService mlService, IModelService modelService, IConfiguration configuration)
+        public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration)
         {
             _mlService = mlService;
             _modelService = modelService;
+            _datasetService = datasetService;
+            _fileService = fileService;
             jwtToken = new JwtToken(configuration);
         }
 
         [HttpPost("sendModel")]
         [Authorize(Roles = "User")]
-        public async Task<ActionResult<string>> Test([FromBody] object model)
+        public async Task<ActionResult<string>> Test([FromBody] Model model)
         {
-            var result = await _mlService.SendModelAsync(model);
+            var dataset = _datasetService.GetOneDataset(model.datasetId);
+            var filepath = _fileService.GetFilePath(dataset.fileId, dataset.username);
+            var result = await _mlService.SendModelAsync(model, filepath);
             return Ok(result);
         }
 
diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs
index 5678daaf..2baab1c0 100644
--- a/backend/api/api/Models/Model.cs
+++ b/backend/api/api/Models/Model.cs
@@ -38,10 +38,13 @@ namespace api.Models
         public int batchSize { get; set; }
         // na izlazu je moguce da bude vise neurona (klasifikacioni problem sa vise od 2 klase)
         public int outputNeurons { get; set; }
-        public string inputLayerActivationFunction { get; set; }
-        public string hiddenLayerActivationFunction { get; set; }
+        public string[] hiddenLayerActivationFunctions { get; set; }
         public string outputLayerActivationFunction { get; set; }
 
+        public string[] metrics { get; set; }
+        public int epochs { get; set; }
 
+        public string nullValues { get; set; }
+        public string[] nullValuesReplacers { get; set; }
     }
 }
diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs
index 5e708d11..45ed18a9 100644
--- a/backend/api/api/Services/DatasetService.cs
+++ b/backend/api/api/Services/DatasetService.cs
@@ -61,6 +61,11 @@ namespace api.Services
         }
         //odraditi za pretragu getOne
 
+        public Dataset GetOneDataset(string id)
+        {
+            return _dataset.Find(dataset => dataset._id == id).FirstOrDefault();
+        }
+
         //ako je potrebno da se zameni name  ili ekstenzija
         public void Update(string username, string name, Dataset dataset)
         {
diff --git a/backend/api/api/Services/IDatasetService.cs b/backend/api/api/Services/IDatasetService.cs
index be56f5cb..dbe43321 100644
--- a/backend/api/api/Services/IDatasetService.cs
+++ b/backend/api/api/Services/IDatasetService.cs
@@ -6,6 +6,7 @@ namespace api.Services
     public interface IDatasetService
     {
         Dataset GetOneDataset(string username, string name);
+        Dataset GetOneDataset(string id);
         List<Dataset> SearchDatasets(string name, string username);
         List<Dataset> GetMyDatasets(string username);
         List<Dataset> SortDatasets(string username, bool ascdsc, int latest);
diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs
index f38fb50a..ee839d28 100644
--- a/backend/api/api/Services/IMlConnectionService.cs
+++ b/backend/api/api/Services/IMlConnectionService.cs
@@ -3,6 +3,6 @@ namespace api.Services
 {
     public interface IMlConnectionService
     {
-        Task<string> SendModelAsync(object model);
+        Task<string> SendModelAsync(object model, object dataset);
     }
 }
\ No newline at end of file
diff --git a/backend/api/api/Services/IModelService.cs b/backend/api/api/Services/IModelService.cs
index ee5c279f..637d09a3 100644
--- a/backend/api/api/Services/IModelService.cs
+++ b/backend/api/api/Services/IModelService.cs
@@ -6,6 +6,7 @@ namespace api.Services
 	public interface IModelService
 	{
         Model GetOneModel(string username, string name);
+        Model GetOneModel(string id);
         List<Model> GetMyModels(string username);
         List<Model> GetLatestModels(string username);
         //List<Model> GetPublicModels();
diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs
index 9b167537..9c3b3fd8 100644
--- a/backend/api/api/Services/MlConnectionService.cs
+++ b/backend/api/api/Services/MlConnectionService.cs
@@ -6,13 +6,19 @@ namespace api.Services
 {
     public class MlConnectionService : IMlConnectionService
     {
-        public async Task<string> SendModelAsync(object model)
+        private RestClient client;
+
+        public MlConnectionService()
+        {
+            this.client = new RestClient("http://127.0.0.1:5543");
+        }
+
+        public async Task<string> SendModelAsync(object model, object dataset)
         {
-            RestClient client = new RestClient("http://localhost:5000");
-            var request = new RestRequest("data", Method.Post);
-            request.AddJsonBody(model);
-            var result = await client.ExecuteAsync(request);
-            return result.Content;//Response od ML microservisa
+            var request = new RestRequest("train", Method.Post);
+            request.AddJsonBody(new { model, dataset});
+            var result = await this.client.ExecuteAsync(request);
+            return result.Content; //Response od ML microservisa
         }
     }
 }
diff --git a/backend/api/api/Services/ModelService.cs b/backend/api/api/Services/ModelService.cs
index f42219f5..eae8c78b 100644
--- a/backend/api/api/Services/ModelService.cs
+++ b/backend/api/api/Services/ModelService.cs
@@ -50,6 +50,11 @@ namespace api.Services
             return _model.Find(model => model.username == username && model.name == name).FirstOrDefault();
         }
 
+        public Model GetOneModel(string id)
+        {
+            return _model.Find(model => model._id == id).FirstOrDefault();
+        }
+
         public void Update(string username, string name, Model model)
         {
             _model.ReplaceOne(model => model.username == username && model.name == name, model);
diff --git a/backend/microservice/PythonServer/project/api/socket/client.py b/backend/microservice/PythonServer/project/socket_example/socket/client.py
index d5740e25..d5740e25 100644
--- a/backend/microservice/PythonServer/project/api/socket/client.py
+++ b/backend/microservice/PythonServer/project/socket_example/socket/client.py
diff --git a/backend/microservice/PythonServer/project/api/socket/server.py b/backend/microservice/PythonServer/project/socket_example/socket/server.py
index d6ff3f7c..d6ff3f7c 100644
--- a/backend/microservice/PythonServer/project/api/socket/server.py
+++ b/backend/microservice/PythonServer/project/socket_example/socket/server.py
diff --git a/backend/microservice/PythonServer/project/api/socket2/client.py b/backend/microservice/PythonServer/project/socket_example/socket2/client.py
index 65e76b55..65e76b55 100644
--- a/backend/microservice/PythonServer/project/api/socket2/client.py
+++ b/backend/microservice/PythonServer/project/socket_example/socket2/client.py
diff --git a/backend/microservice/PythonServer/project/api/socket2/server.py b/backend/microservice/PythonServer/project/socket_example/socket2/server.py
index c65dae78..c65dae78 100644
--- a/backend/microservice/PythonServer/project/api/socket2/server.py
+++ b/backend/microservice/PythonServer/project/socket_example/socket2/server.py
diff --git a/backend/microservice/api/controller.py b/backend/microservice/api/controller.py
new file mode 100644
index 00000000..ceed02ad
--- /dev/null
+++ b/backend/microservice/api/controller.py
@@ -0,0 +1,42 @@
+import flask
+from flask import request, jsonify
+import ml_socket
+import ml_service
+import tensorflow as tf
+import pandas as pd
+
+app = flask.Flask(__name__)
+app.config["DEBUG"] = True
+app.config["SERVER_NAME"] = "127.0.0.1:5543"
+  
+class train_callback(tf.keras.callbacks.Callback):
+    def __init__(self, x_test, y_test):
+        self.x_test = x_test
+        self.y_test = y_test
+    #
+    def on_epoch_end(self, epoch, logs=None):
+        print(epoch)
+        #print('Evaluation: ', self.model.evaluate(self.x_test,self.y_test),"\n") #broj parametara zavisi od izabranih metrika loss je default
+
+@app.route('/train', methods = ['POST'])
+def train():
+    print("******************************TRAIN*************************************************")
+    f = request.json["dataset"]
+    dataset = pd.read_csv(f)
+    #
+    result = ml_service.train(dataset, request.json["model"], train_callback)
+    print(result)
+    return jsonify(result)
+
+@app.route('/predict', methods = ['POST'])
+def predict():
+    f = request.json['filepath']
+    dataset = pd.read_csv(f)
+    m = request.json['modelpath']
+    #model = tf.keras.models.load_model(m)
+    #
+    #model.predict?
+
+print("App loaded.")
+ml_socket.start()
+app.run()
\ No newline at end of file
diff --git a/backend/microservice/api/ml_service.py b/backend/microservice/api/ml_service.py
new file mode 100644
index 00000000..efd24fdc
--- /dev/null
+++ b/backend/microservice/api/ml_service.py
@@ -0,0 +1,155 @@
+import pandas as pd
+import tensorflow as tf
+import keras
+import numpy as np
+import csv
+import json
+import h5py
+import sklearn.metrics as sm
+from statistics import mode
+from typing_extensions import Self
+from copyreg import constructor
+from flask import request, jsonify, render_template
+from sklearn.preprocessing import LabelEncoder
+from sklearn.preprocessing import StandardScaler
+from sklearn.model_selection import train_test_split
+from dataclasses import dataclass
+
+@dataclass
+class TrainingResult:
+    accuracy: float
+    precision: float
+    recall: float
+    tn: float
+    fp: float
+    fn: float
+    tp: float
+    specificity: float
+    f1: float
+    mse: float
+    mae: float
+    mape: float
+    rmse: float
+    fpr: float
+    tpr: float
+
+def train(dataset, params, callback):
+    data = pd.DataFrame()
+    for col in params["inputColumns"]:
+        data[col]=dataset[col]
+    output_column = params["columnToPredict"]
+    data[output_column] = dataset[output_column]
+    #
+    # Brisanje null kolona / redova / zamena
+    #nullreplace=[
+    #    {"column":"Embarked","value":"C","deleteRow":false,"deleteCol":true},
+    #    {"column": "Cabin","value":"C123","deleteRow":"0","deleteCol":"0"}]
+        
+    null_value_options = params["nullValues"]
+    null_values_replacers = params["nullValuesReplacers"]
+    
+    if(null_value_options=='replace'):
+        print("replace null") # TODO
+    elif(null_value_options=='delete_rows'):
+        data=data.dropna()
+    elif(null_value_options=='delete_columns'):
+        data=data.dropna()
+    #
+    #print(data.isnull().any())
+    #
+    # Brisanje kolona koje ne uticu na rezultat
+    #
+    num_rows=data.shape[0]
+    for col in data.columns:
+        if((data[col].nunique()==(num_rows)) and (data[col].dtype==np.object_)):
+            data.pop(col)
+    #
+    # Enkodiranje
+    #
+    encoding=params["encoding"]
+    if(encoding=='label'):
+        encoder=LabelEncoder()
+        for col in data.columns:
+            if(data[col].dtype==np.object_):
+                data[col]=encoder.fit_transform(data[col])
+    elif(encoding=='onehot'):
+        category_columns=[]
+        for col in data.columns:
+            if(data[col].dtype==np.object_):
+                category_columns.append(col)
+        data=pd.get_dummies(data, columns=category_columns, prefix=category_columns)
+    #
+    # Input - output
+    #
+    x_columns = []
+    for col in data.columns:
+        if(col!=output_column):
+            x_columns.append(col)
+    x = data[x_columns].values
+    y = data[output_column].values
+    #
+    # Podela na test i trening skupove
+    #
+    test=params["randomTestSetDistribution"]
+    randomOrder = params["randomOrder"]
+    if(randomOrder):
+        random=50
+    else:
+        random=0
+    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=test, random_state=random)
+    #
+    # Skaliranje vrednosti
+    #
+    scaler=StandardScaler()
+    scaler.fit(x_train)
+    x_test=scaler.transform(x_test)
+    x_train=scaler.transform(x_train)
+    #
+    # Treniranje modela
+    #
+    classifier=tf.keras.Sequential()
+    hidden_layer_neurons = params["hiddenLayerNeurons"]
+    for func in params["hiddenLayerActivationFunctions"]:
+        classifier.add(tf.keras.layers.Dense(units=hidden_layer_neurons,activation=func))
+    output_func = params["outputLayerActivationFunction"]
+    classifier.add(tf.keras.layers.Dense(units=1,activation=output_func))
+    optimizer = params["optimizer"]
+    metrics=params['metrics']
+    loss_func=params["lossFunction"]
+    classifier.compile(optimizer=optimizer, loss=loss_func,metrics=metrics)
+    batch_size = params["batchSize"]
+    epochs = params["epochs"]
+    history=classifier.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, callbacks=callback(x_test, y_test), validation_split=0.2) # TODO params["validationSplit"]
+    #
+    # Test
+    #
+    y_pred=classifier.predict(x_test)
+    y_pred=(y_pred>=0.5).astype('int')
+    #y_pred=(y_pred * 100).astype('int')
+    y_pred=y_pred.flatten()
+    result=pd.DataFrame({"Actual":y_test,"Predicted":y_pred})
+    model_name = params['_id']
+    classifier.save("temp/"+model_name, save_format='h5')
+    #
+    # Metrike
+    #
+    print("HELLO???")
+    print(result)
+    print("HELLO???")
+    accuracy = float(sm.accuracy_score(y_test,y_pred))
+    precision = float(sm.precision_score(y_test,y_pred))
+    recall = float(sm.recall_score(y_test,y_pred))
+    tn, fp, fn, tp = sm.confusion_matrix(y_test,y_pred).ravel()
+    specificity = float(tn / (tn+fp))
+    f1 = float(sm.f1_score(y_test,y_pred))
+    mse = float(sm.mean_squared_error(y_test,y_pred))
+    mae = float(sm.mean_absolute_error(y_test,y_pred))
+    mape = float(sm.mean_absolute_percentage_error(y_test,y_pred))
+    rmse = float(np.sqrt(sm.mean_squared_error(y_test,y_pred)))
+    fpr, tpr, _ = sm.roc_curve(y_test,y_pred)
+    # TODO upload trenirani model nazad na backend
+    return TrainingResult(accuracy, precision, recall, float(tn), float(fp), float(fn), float(tp), specificity, f1, mse, mae, mape, rmse, fpr.tolist(), tpr.tolist())
+
+
+
+    
diff --git a/backend/microservice/ml_socket.py b/backend/microservice/api/ml_socket.py
index 5489b787..65dd7321 100644
--- a/backend/microservice/ml_socket.py
+++ b/backend/microservice/api/ml_socket.py
@@ -14,12 +14,15 @@ def get_or_create_eventloop():
 # create handler for each connection
 async def handler(websocket, path):
     #data = json.loads(await websocket.recv())
-    #reply = f"Data recieved as:  {data}!"
     #print(data['test'])
     msg = await websocket.recv()
-    await websocket.send("[" + msg + "]")
+    print(msg)
 
-start_server = websockets.serve(handler, "localhost", 5027)
+async def start():
+    start_server = websockets.serve(handler, "localhost", 5027)
+    print('Websocket starting...')
+    get_or_create_eventloop().run_until_complete(start_server)
+    get_or_create_eventloop().run_forever()
 
-get_or_create_eventloop().run_until_complete(start_server)
-get_or_create_eventloop().run_forever()
\ No newline at end of file
+async def send(msg):
+    await websocket.send(msg)
\ No newline at end of file