From 2ca55fc8c9879a2d06e364ee3c88a9cfffa6f299 Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 27 Mar 2022 16:52:49 +0200 Subject: Gost vidi public datasetove uploadera Igrannonica kao i temp datasetove na stranici add-model. --- backend/api/api/Controllers/DatasetController.cs | 5 ++++- backend/api/api/Services/DatasetService.cs | 7 +++++++ backend/api/api/Services/IDatasetService.cs | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) (limited to 'backend/api') diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index bae05ba9..d9803744 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -24,7 +24,7 @@ namespace api.Controllers // GET: api//mydatasets [HttpGet("mydatasets")] - [Authorize(Roles = "User")] + [Authorize(Roles = "User,Guest")] public ActionResult> Get() { string username; @@ -39,6 +39,9 @@ namespace api.Controllers } else return BadRequest(); + //U slucaju da je korisnik gost vrati dataSetove igrannonice + if (username == "") + return _datasetService.GetGuestDatasets(); //ako bude trebao ID, samo iz baze uzeti diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs index 5e708d11..ab19935d 100644 --- a/backend/api/api/Services/DatasetService.cs +++ b/backend/api/api/Services/DatasetService.cs @@ -36,6 +36,13 @@ namespace api.Services { return _dataset.Find(dataset => dataset.username == username).ToList(); } + public List GetGuestDatasets() + { + //Join Igranonica public datasetove sa svim temp uploadanim datasetovima + List datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true).ToList(); + datasets.AddRange(_dataset.Find(dataset => dataset.username == "").ToList()); + return datasets; + } //poslednji datasetovi public List SortDatasets(string username, bool ascdsc, int latest) diff --git a/backend/api/api/Services/IDatasetService.cs b/backend/api/api/Services/IDatasetService.cs index be56f5cb..9beec400 100644 --- a/backend/api/api/Services/IDatasetService.cs +++ b/backend/api/api/Services/IDatasetService.cs @@ -13,5 +13,6 @@ namespace api.Services Dataset Create(Dataset dataset); void Update(string username, string name, Dataset dataset); void Delete(string username, string name); + public List GetGuestDatasets(); } } -- cgit v1.2.3 From 98d580ca1ba501bc059ff417dd1794e0f6a68407 Mon Sep 17 00:00:00 2001 From: Danijel Andjelkovic Date: Tue, 29 Mar 2022 12:24:38 +0200 Subject: Povezao treniranje modela. --- .gitignore | 4 +- backend/api/api/Controllers/ModelController.cs | 12 +- backend/api/api/Models/Model.cs | 7 +- backend/api/api/Services/DatasetService.cs | 5 + backend/api/api/Services/IDatasetService.cs | 1 + backend/api/api/Services/IMlConnectionService.cs | 2 +- backend/api/api/Services/IModelService.cs | 1 + backend/api/api/Services/MlConnectionService.cs | 18 ++- backend/api/api/Services/ModelService.cs | 5 + .../PythonServer/project/api/socket/client.py | 10 -- .../PythonServer/project/api/socket/server.py | 16 --- .../PythonServer/project/api/socket2/client.py | 16 --- .../PythonServer/project/api/socket2/server.py | 39 ------ .../project/socket_example/socket/client.py | 10 ++ .../project/socket_example/socket/server.py | 16 +++ .../project/socket_example/socket2/client.py | 16 +++ .../project/socket_example/socket2/server.py | 39 ++++++ backend/microservice/api/controller.py | 42 ++++++ backend/microservice/api/ml_service.py | 155 +++++++++++++++++++++ backend/microservice/api/ml_socket.py | 28 ++++ backend/microservice/ml_socket.py | 25 ---- frontend/src/app/_data/Model.ts | 13 +- .../app/_pages/add-model/add-model.component.html | 69 ++++----- .../app/_pages/add-model/add-model.component.ts | 32 +++-- frontend/src/app/_services/models.service.ts | 14 +- frontend/src/app/_services/web-socket.service.ts | 6 +- 26 files changed, 414 insertions(+), 187 deletions(-) delete mode 100644 backend/microservice/PythonServer/project/api/socket/client.py delete mode 100644 backend/microservice/PythonServer/project/api/socket/server.py delete mode 100644 backend/microservice/PythonServer/project/api/socket2/client.py delete mode 100644 backend/microservice/PythonServer/project/api/socket2/server.py create mode 100644 backend/microservice/PythonServer/project/socket_example/socket/client.py create mode 100644 backend/microservice/PythonServer/project/socket_example/socket/server.py create mode 100644 backend/microservice/PythonServer/project/socket_example/socket2/client.py create mode 100644 backend/microservice/PythonServer/project/socket_example/socket2/server.py create mode 100644 backend/microservice/api/controller.py create mode 100644 backend/microservice/api/ml_service.py create mode 100644 backend/microservice/api/ml_socket.py delete mode 100644 backend/microservice/ml_socket.py (limited to 'backend/api') diff --git a/.gitignore b/.gitignore index d094e2e2..25a7cdac 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ sandbox/test-projekat-danijel/backend/.vs/ sandbox/testAppSonja/MiniApkSonja/MiniApkSonja/bin/ sandbox/testAppSonja/MiniApkSonja/MiniApkSonja/obj/ sandbox/TestTamara/TestTamara/TestTamara/obj/ -sandbox/TestTamara/TestTamara/TestTamara/bin/ \ No newline at end of file +sandbox/TestTamara/TestTamara/TestTamara/bin/ +backend/microservice/temp/ +backend/microservice/api/__pycache__/ diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index b997efa3..4bc094cd 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -14,22 +14,28 @@ namespace api.Controllers { private IMlConnectionService _mlService; + private readonly IDatasetService _datasetService; + private readonly IFileService _fileService; private readonly IModelService _modelService; private JwtToken jwtToken; - public ModelController(IMlConnectionService mlService, IModelService modelService, IConfiguration configuration) + public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration) { _mlService = mlService; _modelService = modelService; + _datasetService = datasetService; + _fileService = fileService; jwtToken = new JwtToken(configuration); } [HttpPost("sendModel")] [Authorize(Roles = "User")] - public async Task> Test([FromBody] object model) + public async Task> Test([FromBody] Model model) { - var result = await _mlService.SendModelAsync(model); + var dataset = _datasetService.GetOneDataset(model.datasetId); + var filepath = _fileService.GetFilePath(dataset.fileId, dataset.username); + var result = await _mlService.SendModelAsync(model, filepath); return Ok(result); } diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs index 5678daaf..2baab1c0 100644 --- a/backend/api/api/Models/Model.cs +++ b/backend/api/api/Models/Model.cs @@ -38,10 +38,13 @@ namespace api.Models public int batchSize { get; set; } // na izlazu je moguce da bude vise neurona (klasifikacioni problem sa vise od 2 klase) public int outputNeurons { get; set; } - public string inputLayerActivationFunction { get; set; } - public string hiddenLayerActivationFunction { get; set; } + public string[] hiddenLayerActivationFunctions { get; set; } public string outputLayerActivationFunction { get; set; } + public string[] metrics { get; set; } + public int epochs { get; set; } + public string nullValues { get; set; } + public string[] nullValuesReplacers { get; set; } } } diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs index 5e708d11..45ed18a9 100644 --- a/backend/api/api/Services/DatasetService.cs +++ b/backend/api/api/Services/DatasetService.cs @@ -61,6 +61,11 @@ namespace api.Services } //odraditi za pretragu getOne + public Dataset GetOneDataset(string id) + { + return _dataset.Find(dataset => dataset._id == id).FirstOrDefault(); + } + //ako je potrebno da se zameni name ili ekstenzija public void Update(string username, string name, Dataset dataset) { diff --git a/backend/api/api/Services/IDatasetService.cs b/backend/api/api/Services/IDatasetService.cs index be56f5cb..dbe43321 100644 --- a/backend/api/api/Services/IDatasetService.cs +++ b/backend/api/api/Services/IDatasetService.cs @@ -6,6 +6,7 @@ namespace api.Services public interface IDatasetService { Dataset GetOneDataset(string username, string name); + Dataset GetOneDataset(string id); List SearchDatasets(string name, string username); List GetMyDatasets(string username); List SortDatasets(string username, bool ascdsc, int latest); diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index f38fb50a..ee839d28 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -3,6 +3,6 @@ namespace api.Services { public interface IMlConnectionService { - Task SendModelAsync(object model); + Task SendModelAsync(object model, object dataset); } } \ No newline at end of file diff --git a/backend/api/api/Services/IModelService.cs b/backend/api/api/Services/IModelService.cs index ee5c279f..637d09a3 100644 --- a/backend/api/api/Services/IModelService.cs +++ b/backend/api/api/Services/IModelService.cs @@ -6,6 +6,7 @@ namespace api.Services public interface IModelService { Model GetOneModel(string username, string name); + Model GetOneModel(string id); List GetMyModels(string username); List GetLatestModels(string username); //List GetPublicModels(); diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index 9b167537..9c3b3fd8 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -6,13 +6,19 @@ namespace api.Services { public class MlConnectionService : IMlConnectionService { - public async Task SendModelAsync(object model) + private RestClient client; + + public MlConnectionService() + { + this.client = new RestClient("http://127.0.0.1:5543"); + } + + public async Task SendModelAsync(object model, object dataset) { - RestClient client = new RestClient("http://localhost:5000"); - var request = new RestRequest("data", Method.Post); - request.AddJsonBody(model); - var result = await client.ExecuteAsync(request); - return result.Content;//Response od ML microservisa + var request = new RestRequest("train", Method.Post); + request.AddJsonBody(new { model, dataset}); + var result = await this.client.ExecuteAsync(request); + return result.Content; //Response od ML microservisa } } } diff --git a/backend/api/api/Services/ModelService.cs b/backend/api/api/Services/ModelService.cs index f42219f5..eae8c78b 100644 --- a/backend/api/api/Services/ModelService.cs +++ b/backend/api/api/Services/ModelService.cs @@ -50,6 +50,11 @@ namespace api.Services return _model.Find(model => model.username == username && model.name == name).FirstOrDefault(); } + public Model GetOneModel(string id) + { + return _model.Find(model => model._id == id).FirstOrDefault(); + } + public void Update(string username, string name, Model model) { _model.ReplaceOne(model => model.username == username && model.name == name, model); diff --git a/backend/microservice/PythonServer/project/api/socket/client.py b/backend/microservice/PythonServer/project/api/socket/client.py deleted file mode 100644 index d5740e25..00000000 --- a/backend/microservice/PythonServer/project/api/socket/client.py +++ /dev/null @@ -1,10 +0,0 @@ -import socket - -c = socket.socket() - -c.connect(('localhost', 9999)) - -name = input("Client name:") -c.send(bytes(name, 'utf-8')) - -print(c.recv(1024).decode()) \ No newline at end of file diff --git a/backend/microservice/PythonServer/project/api/socket/server.py b/backend/microservice/PythonServer/project/api/socket/server.py deleted file mode 100644 index d6ff3f7c..00000000 --- a/backend/microservice/PythonServer/project/api/socket/server.py +++ /dev/null @@ -1,16 +0,0 @@ -import socket - -s = socket.socket() -print('Socket Created') - -s.bind(('localhost', 9999)) - -s.listen(3) -print('waiting for connections') - -while True: - c, addr = s.accept() - name = c.recv(1024).decode() - print('Connected with ', addr, name) - c.send(bytes('Welcome', 'utf-8')) - c.close() \ No newline at end of file diff --git a/backend/microservice/PythonServer/project/api/socket2/client.py b/backend/microservice/PythonServer/project/api/socket2/client.py deleted file mode 100644 index 65e76b55..00000000 --- a/backend/microservice/PythonServer/project/api/socket2/client.py +++ /dev/null @@ -1,16 +0,0 @@ -# Import socket module -import socket - -# Create a socket object -s = socket.socket() - -# Define the port on which you want to connect -port = 12345 - -# connect to the server on local computer -s.connect(('127.0.0.1', port)) - -# receive data from the server and decoding to get the string. -print (s.recv(1024).decode()) -# close the connection -s.close() \ No newline at end of file diff --git a/backend/microservice/PythonServer/project/api/socket2/server.py b/backend/microservice/PythonServer/project/api/socket2/server.py deleted file mode 100644 index c65dae78..00000000 --- a/backend/microservice/PythonServer/project/api/socket2/server.py +++ /dev/null @@ -1,39 +0,0 @@ -# first of all import the socket library -import socket - -# next create a socket object -s = socket.socket() -print ("Socket successfully created") - -# reserve a port on your computer in our -# case it is 12345 but it can be anything -port = 12345 - -# Next bind to the port -# we have not typed any ip in the ip field -# instead we have inputted an empty string -# this makes the server listen to requests -# coming from other computers on the network -s.bind(('', port)) -print ("socket binded to %s" %(port)) - -# put the socket into listening mode -s.listen(5) -print ("socket is listening") - -# a forever loop until we interrupt it or -# an error occurs -while True: - -# Establish connection with client. - c, addr = s.accept() - print ('Got connection from', addr ) - - # send a thank you message to the client. encoding to send byte type. - c.send('Thank you for connecting'.encode()) - - # Close the connection with the client - c.close() - - # Breaking once connection closed - break \ No newline at end of file diff --git a/backend/microservice/PythonServer/project/socket_example/socket/client.py b/backend/microservice/PythonServer/project/socket_example/socket/client.py new file mode 100644 index 00000000..d5740e25 --- /dev/null +++ b/backend/microservice/PythonServer/project/socket_example/socket/client.py @@ -0,0 +1,10 @@ +import socket + +c = socket.socket() + +c.connect(('localhost', 9999)) + +name = input("Client name:") +c.send(bytes(name, 'utf-8')) + +print(c.recv(1024).decode()) \ No newline at end of file diff --git a/backend/microservice/PythonServer/project/socket_example/socket/server.py b/backend/microservice/PythonServer/project/socket_example/socket/server.py new file mode 100644 index 00000000..d6ff3f7c --- /dev/null +++ b/backend/microservice/PythonServer/project/socket_example/socket/server.py @@ -0,0 +1,16 @@ +import socket + +s = socket.socket() +print('Socket Created') + +s.bind(('localhost', 9999)) + +s.listen(3) +print('waiting for connections') + +while True: + c, addr = s.accept() + name = c.recv(1024).decode() + print('Connected with ', addr, name) + c.send(bytes('Welcome', 'utf-8')) + c.close() \ No newline at end of file diff --git a/backend/microservice/PythonServer/project/socket_example/socket2/client.py b/backend/microservice/PythonServer/project/socket_example/socket2/client.py new file mode 100644 index 00000000..65e76b55 --- /dev/null +++ b/backend/microservice/PythonServer/project/socket_example/socket2/client.py @@ -0,0 +1,16 @@ +# Import socket module +import socket + +# Create a socket object +s = socket.socket() + +# Define the port on which you want to connect +port = 12345 + +# connect to the server on local computer +s.connect(('127.0.0.1', port)) + +# receive data from the server and decoding to get the string. +print (s.recv(1024).decode()) +# close the connection +s.close() \ No newline at end of file diff --git a/backend/microservice/PythonServer/project/socket_example/socket2/server.py b/backend/microservice/PythonServer/project/socket_example/socket2/server.py new file mode 100644 index 00000000..c65dae78 --- /dev/null +++ b/backend/microservice/PythonServer/project/socket_example/socket2/server.py @@ -0,0 +1,39 @@ +# first of all import the socket library +import socket + +# next create a socket object +s = socket.socket() +print ("Socket successfully created") + +# reserve a port on your computer in our +# case it is 12345 but it can be anything +port = 12345 + +# Next bind to the port +# we have not typed any ip in the ip field +# instead we have inputted an empty string +# this makes the server listen to requests +# coming from other computers on the network +s.bind(('', port)) +print ("socket binded to %s" %(port)) + +# put the socket into listening mode +s.listen(5) +print ("socket is listening") + +# a forever loop until we interrupt it or +# an error occurs +while True: + +# Establish connection with client. + c, addr = s.accept() + print ('Got connection from', addr ) + + # send a thank you message to the client. encoding to send byte type. + c.send('Thank you for connecting'.encode()) + + # Close the connection with the client + c.close() + + # Breaking once connection closed + break \ No newline at end of file diff --git a/backend/microservice/api/controller.py b/backend/microservice/api/controller.py new file mode 100644 index 00000000..ceed02ad --- /dev/null +++ b/backend/microservice/api/controller.py @@ -0,0 +1,42 @@ +import flask +from flask import request, jsonify +import ml_socket +import ml_service +import tensorflow as tf +import pandas as pd + +app = flask.Flask(__name__) +app.config["DEBUG"] = True +app.config["SERVER_NAME"] = "127.0.0.1:5543" + +class train_callback(tf.keras.callbacks.Callback): + def __init__(self, x_test, y_test): + self.x_test = x_test + self.y_test = y_test + # + def on_epoch_end(self, epoch, logs=None): + print(epoch) + #print('Evaluation: ', self.model.evaluate(self.x_test,self.y_test),"\n") #broj parametara zavisi od izabranih metrika loss je default + +@app.route('/train', methods = ['POST']) +def train(): + print("******************************TRAIN*************************************************") + f = request.json["dataset"] + dataset = pd.read_csv(f) + # + result = ml_service.train(dataset, request.json["model"], train_callback) + print(result) + return jsonify(result) + +@app.route('/predict', methods = ['POST']) +def predict(): + f = request.json['filepath'] + dataset = pd.read_csv(f) + m = request.json['modelpath'] + #model = tf.keras.models.load_model(m) + # + #model.predict? + +print("App loaded.") +ml_socket.start() +app.run() \ No newline at end of file diff --git a/backend/microservice/api/ml_service.py b/backend/microservice/api/ml_service.py new file mode 100644 index 00000000..efd24fdc --- /dev/null +++ b/backend/microservice/api/ml_service.py @@ -0,0 +1,155 @@ +import pandas as pd +import tensorflow as tf +import keras +import numpy as np +import csv +import json +import h5py +import sklearn.metrics as sm +from statistics import mode +from typing_extensions import Self +from copyreg import constructor +from flask import request, jsonify, render_template +from sklearn.preprocessing import LabelEncoder +from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import train_test_split +from dataclasses import dataclass + +@dataclass +class TrainingResult: + accuracy: float + precision: float + recall: float + tn: float + fp: float + fn: float + tp: float + specificity: float + f1: float + mse: float + mae: float + mape: float + rmse: float + fpr: float + tpr: float + +def train(dataset, params, callback): + data = pd.DataFrame() + for col in params["inputColumns"]: + data[col]=dataset[col] + output_column = params["columnToPredict"] + data[output_column] = dataset[output_column] + # + # Brisanje null kolona / redova / zamena + #nullreplace=[ + # {"column":"Embarked","value":"C","deleteRow":false,"deleteCol":true}, + # {"column": "Cabin","value":"C123","deleteRow":"0","deleteCol":"0"}] + + null_value_options = params["nullValues"] + null_values_replacers = params["nullValuesReplacers"] + + if(null_value_options=='replace'): + print("replace null") # TODO + elif(null_value_options=='delete_rows'): + data=data.dropna() + elif(null_value_options=='delete_columns'): + data=data.dropna() + # + #print(data.isnull().any()) + # + # Brisanje kolona koje ne uticu na rezultat + # + num_rows=data.shape[0] + for col in data.columns: + if((data[col].nunique()==(num_rows)) and (data[col].dtype==np.object_)): + data.pop(col) + # + # Enkodiranje + # + encoding=params["encoding"] + if(encoding=='label'): + encoder=LabelEncoder() + for col in data.columns: + if(data[col].dtype==np.object_): + data[col]=encoder.fit_transform(data[col]) + elif(encoding=='onehot'): + category_columns=[] + for col in data.columns: + if(data[col].dtype==np.object_): + category_columns.append(col) + data=pd.get_dummies(data, columns=category_columns, prefix=category_columns) + # + # Input - output + # + x_columns = [] + for col in data.columns: + if(col!=output_column): + x_columns.append(col) + x = data[x_columns].values + y = data[output_column].values + # + # Podela na test i trening skupove + # + test=params["randomTestSetDistribution"] + randomOrder = params["randomOrder"] + if(randomOrder): + random=50 + else: + random=0 + x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=test, random_state=random) + # + # Skaliranje vrednosti + # + scaler=StandardScaler() + scaler.fit(x_train) + x_test=scaler.transform(x_test) + x_train=scaler.transform(x_train) + # + # Treniranje modela + # + classifier=tf.keras.Sequential() + hidden_layer_neurons = params["hiddenLayerNeurons"] + for func in params["hiddenLayerActivationFunctions"]: + classifier.add(tf.keras.layers.Dense(units=hidden_layer_neurons,activation=func)) + output_func = params["outputLayerActivationFunction"] + classifier.add(tf.keras.layers.Dense(units=1,activation=output_func)) + optimizer = params["optimizer"] + metrics=params['metrics'] + loss_func=params["lossFunction"] + classifier.compile(optimizer=optimizer, loss=loss_func,metrics=metrics) + batch_size = params["batchSize"] + epochs = params["epochs"] + history=classifier.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, callbacks=callback(x_test, y_test), validation_split=0.2) # TODO params["validationSplit"] + # + # Test + # + y_pred=classifier.predict(x_test) + y_pred=(y_pred>=0.5).astype('int') + #y_pred=(y_pred * 100).astype('int') + y_pred=y_pred.flatten() + result=pd.DataFrame({"Actual":y_test,"Predicted":y_pred}) + model_name = params['_id'] + classifier.save("temp/"+model_name, save_format='h5') + # + # Metrike + # + print("HELLO???") + print(result) + print("HELLO???") + accuracy = float(sm.accuracy_score(y_test,y_pred)) + precision = float(sm.precision_score(y_test,y_pred)) + recall = float(sm.recall_score(y_test,y_pred)) + tn, fp, fn, tp = sm.confusion_matrix(y_test,y_pred).ravel() + specificity = float(tn / (tn+fp)) + f1 = float(sm.f1_score(y_test,y_pred)) + mse = float(sm.mean_squared_error(y_test,y_pred)) + mae = float(sm.mean_absolute_error(y_test,y_pred)) + mape = float(sm.mean_absolute_percentage_error(y_test,y_pred)) + rmse = float(np.sqrt(sm.mean_squared_error(y_test,y_pred))) + fpr, tpr, _ = sm.roc_curve(y_test,y_pred) + # TODO upload trenirani model nazad na backend + return TrainingResult(accuracy, precision, recall, float(tn), float(fp), float(fn), float(tp), specificity, f1, mse, mae, mape, rmse, fpr.tolist(), tpr.tolist()) + + + + diff --git a/backend/microservice/api/ml_socket.py b/backend/microservice/api/ml_socket.py new file mode 100644 index 00000000..65dd7321 --- /dev/null +++ b/backend/microservice/api/ml_socket.py @@ -0,0 +1,28 @@ +import asyncio +import websockets +import json + +def get_or_create_eventloop(): + try: + return asyncio.get_event_loop() + except RuntimeError as ex: + if "There is no current event loop in thread" in str(ex): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return asyncio.get_event_loop() + +# create handler for each connection +async def handler(websocket, path): + #data = json.loads(await websocket.recv()) + #print(data['test']) + msg = await websocket.recv() + print(msg) + +async def start(): + start_server = websockets.serve(handler, "localhost", 5027) + print('Websocket starting...') + get_or_create_eventloop().run_until_complete(start_server) + get_or_create_eventloop().run_forever() + +async def send(msg): + await websocket.send(msg) \ No newline at end of file diff --git a/backend/microservice/ml_socket.py b/backend/microservice/ml_socket.py deleted file mode 100644 index 5489b787..00000000 --- a/backend/microservice/ml_socket.py +++ /dev/null @@ -1,25 +0,0 @@ -import asyncio -import websockets -import json - -def get_or_create_eventloop(): - try: - return asyncio.get_event_loop() - except RuntimeError as ex: - if "There is no current event loop in thread" in str(ex): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - return asyncio.get_event_loop() - -# create handler for each connection -async def handler(websocket, path): - #data = json.loads(await websocket.recv()) - #reply = f"Data recieved as: {data}!" - #print(data['test']) - msg = await websocket.recv() - await websocket.send("[" + msg + "]") - -start_server = websockets.serve(handler, "localhost", 5027) - -get_or_create_eventloop().run_until_complete(start_server) -get_or_create_eventloop().run_forever() \ No newline at end of file diff --git a/frontend/src/app/_data/Model.ts b/frontend/src/app/_data/Model.ts index 48418d51..32247bbd 100644 --- a/frontend/src/app/_data/Model.ts +++ b/frontend/src/app/_data/Model.ts @@ -23,12 +23,14 @@ export default class Model { public hiddenLayerNeurons: number = 1, public hiddenLayers: number = 1, public batchSize: number = 5, - public hiddenLayerActivationFunction = [], + public hiddenLayerActivationFunctions: string[] = ['sigmoid'], //public inputLayerActivationFunction: ActivationFunction = ActivationFunction.Sigmoid, public outputLayerActivationFunction: ActivationFunction = ActivationFunction.Sigmoid, public username: string = '', public nullValues: NullValueOptions = NullValueOptions.DeleteRows, - public nullValuesReplacers = [] + public nullValuesReplacers = [], + public metrics: Metric[] = [], // TODO add to add-model form + public epochs: number = 5 // TODO add to add-model form ) { } } @@ -111,4 +113,11 @@ export enum ReplaceWith { None = 'Popuni...', Mean = 'Srednja vrednost', Median = 'Medijana' +} + +export enum Metric { + MSE = 'mse', + MAE = 'mae', + RMSE = 'rmse' + //... } \ No newline at end of file diff --git a/frontend/src/app/_pages/add-model/add-model.component.html b/frontend/src/app/_pages/add-model/add-model.component.html index 9dde9afe..38b3ed47 100644 --- a/frontend/src/app/_pages/add-model/add-model.component.html +++ b/frontend/src/app/_pages/add-model/add-model.component.html @@ -71,7 +71,8 @@
  + name="cbsNew" [checked]="this.selectedOutputColumnVal != item" + [disabled]="this.selectedOutputColumnVal == item">  @@ -163,7 +164,8 @@
+ [(ngModel)]="newModel.hiddenLayers" + (change)="newModel.hiddenLayerActivationFunctions = [].constructor(newModel.hiddenLayers).fill(newModel.hiddenLayerActivationFunctions[0])">
@@ -258,56 +261,36 @@
- - - -
+
-
-
-
- -
-
-
-
- -
-
- -
@@ -357,7 +340,7 @@
-
diff --git a/frontend/src/app/_pages/add-model/add-model.component.ts b/frontend/src/app/_pages/add-model/add-model.component.ts index 37672d0a..4270a2db 100644 --- a/frontend/src/app/_pages/add-model/add-model.component.ts +++ b/frontend/src/app/_pages/add-model/add-model.component.ts @@ -76,19 +76,28 @@ export class AddModelComponent implements OnInit { addModel() { if (!this.showMyDatasets) - this.saveModelWithNewDataset(); + this.saveModelWithNewDataset(_ => { console.log('MODEL ADDED (with new dataset).') }); else - this.saveModelWithExistingDataset(); + this.saveModelWithExistingDataset(_ => { console.log('MODEL ADDED (with existing dataset).') }); } trainModel() { - this.saveModelWithNewDataset().subscribe((modelId: any) => { - if (modelId) - this.models.trainModel(modelId); - }); //privremeno cuvanje modela => vraca id sacuvanog modela koji cemo da treniramo sad + let saveFunc; + + if (!this.showMyDatasets) + saveFunc = (x: (arg0: any) => void) => { this.saveModelWithNewDataset(x) }; + else + saveFunc = (x: (arg0: any) => void) => { this.saveModelWithExistingDataset(x) }; + + saveFunc(((model: any) => { + console.log('Saved, training model...', model); + this.models.trainModel(model).subscribe(response => { + console.log('Train model complete!', response); + }); + })); //privremeno cuvanje modela => vraca id sacuvanog modela koji cemo da treniramo sad } - saveModelWithNewDataset(): any { + saveModelWithNewDataset(callback: ((arg0: any) => void)) { this.getCheckedInputCols(); this.getCheckedOutputCol(); @@ -117,7 +126,7 @@ export class AddModelComponent implements OnInit { this.newModel.username = shared.username; this.models.addModel(this.newModel).subscribe((response) => { - console.log('ADD MODEL: DONE! REPLY:\n', response); + callback(response); }, (error) => { alert("Model sa unetim nazivom već postoji u Vašoj kolekciji.\nPromenite naziv modela i nastavite sa kreiranim datasetom."); }); //kraj addModel subscribe @@ -133,8 +142,7 @@ export class AddModelComponent implements OnInit { } //kraj prvog ifa } - saveModelWithExistingDataset(): any { - + saveModelWithExistingDataset(callback: ((arg0: any) => void)): any { if (this.selectedDataset) { //dataset je izabran this.getCheckedInputCols(); this.getCheckedOutputCol(); @@ -147,7 +155,7 @@ export class AddModelComponent implements OnInit { this.newModel.username = shared.username; this.models.addModel(this.newModel).subscribe((response) => { - console.log('ADD MODEL: DONE! REPLY:\n', response); + callback(response); }, (error) => { alert("Model sa unetim nazivom već postoji u Vašoj kolekciji.\nPromenite naziv modela i nastavite sa kreiranim datasetom."); }); @@ -226,7 +234,7 @@ export class AddModelComponent implements OnInit { for (let i = this.datasetFile.length - 1; i >= 0; i--) { //moguce da je vise redova na kraju fajla prazno i sl. if (this.datasetFile[i].length != this.datasetFile[0].length) this.datasetFile[i].pop(); - else + else break; //nema potrebe dalje } console.log(this.datasetFile); diff --git a/frontend/src/app/_services/models.service.ts b/frontend/src/app/_services/models.service.ts index 58ddb2e6..6ea4310c 100644 --- a/frontend/src/app/_services/models.service.ts +++ b/frontend/src/app/_services/models.service.ts @@ -35,25 +35,23 @@ export class ModelsService { addDataset(dataset: Dataset): Observable { return this.http.post(`${API_SETTINGS.apiURL}/dataset/add`, dataset, { headers: this.authService.authHeader() }); } - trainModel(modelId: string): Observable { - return this.http.post(`${API_SETTINGS.apiURL}/model/train`, modelId, { headers: this.authService.authHeader() }); + trainModel(model: Model): Observable { + return this.http.post(`${API_SETTINGS.apiURL}/model/sendmodel`, model, { headers: this.authService.authHeader(), responseType: 'text' }); } getMyDatasets(): Observable { return this.http.get(`${API_SETTINGS.apiURL}/dataset/mydatasets`, { headers: this.authService.authHeader() }); } - + getMyModels(): Observable { return this.http.get(`${API_SETTINGS.apiURL}/model/mymodels`, { headers: this.authService.authHeader() }); } - editModel(model:Model) : Observable - { + editModel(model: Model): Observable { return this.http.put(`${API_SETTINGS.apiURL}/model/`, model, { headers: this.authService.authHeader() }); } - deleteModel(model:Model) : Observable - { - return this.http.delete(`${API_SETTINGS.apiURL}/model/`+model.name, { headers: this.authService.authHeader() }); + deleteModel(model: Model): Observable { + return this.http.delete(`${API_SETTINGS.apiURL}/model/` + model.name, { headers: this.authService.authHeader() }); } } diff --git a/frontend/src/app/_services/web-socket.service.ts b/frontend/src/app/_services/web-socket.service.ts index 890ada6b..1a7efa87 100644 --- a/frontend/src/app/_services/web-socket.service.ts +++ b/frontend/src/app/_services/web-socket.service.ts @@ -13,15 +13,15 @@ export class WebSocketService { constructor() { this.ws = new WebsocketBuilder(API_SETTINGS.apiWSUrl) - .withBackoff(new ConstantBackoff(30000)) - .onOpen((i, e) => { console.log('WS: Connected to ' + API_SETTINGS.apiWSUrl) }) + .withBackoff(new ConstantBackoff(120000)) + .onOpen((i, e) => { /*console.log('WS: Connected to ' + API_SETTINGS.apiWSUrl)*/ }) .onMessage((i, e) => { console.log('WS MESSAGE: ', e.data); this.handlers.forEach(handler => { handler(e.data); }) }) - .onClose((i, e) => { console.log('WS: Connection closed!') }) + .onClose((i, e) => { /*console.log('WS: Connection closed!')*/ }) .build(); } -- cgit v1.2.3 From 9a708de97086c898ccb2a585617dcc5a64dcaf64 Mon Sep 17 00:00:00 2001 From: Ivan Ljubisavljevic Date: Tue, 29 Mar 2022 13:41:30 +0200 Subject: Dodat back za pristup predictoru #66 --- backend/api/api/Controllers/PredictorController.cs | 30 ++++++++++++++++++++++ backend/api/api/Services/IPredictorService.cs | 1 + backend/api/api/Services/PredictorService.cs | 5 ++++ 3 files changed, 36 insertions(+) (limited to 'backend/api') diff --git a/backend/api/api/Controllers/PredictorController.cs b/backend/api/api/Controllers/PredictorController.cs index 7f8f1692..63c5d2bf 100644 --- a/backend/api/api/Controllers/PredictorController.cs +++ b/backend/api/api/Controllers/PredictorController.cs @@ -74,6 +74,36 @@ namespace api.Controllers return _predictorService.SearchPredictors(name, username); } + //SEARCH za predictore (public ili private sa ovim imenom ) + // GET api//search/{name} + [HttpGet("{id}")] + [Authorize(Roles = "User")] + public ActionResult GetPredictor(string id) + { + string username; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + username = jwtToken.TokenToUsername(parameter); + if (username == null) + return null; + } + else + return BadRequest(); + + //ako bude trebao ID, samo iz baze uzeti + + Predictor predictor = _predictorService.GetPredictor(username, id); + + if (predictor == null) + return NotFound($"Predictor with id = {id} not found"); + + return predictor; + } + + //da li da se odvoji search za public i posebno za private? // GET api//{name} [HttpGet("{name}")] diff --git a/backend/api/api/Services/IPredictorService.cs b/backend/api/api/Services/IPredictorService.cs index 2017add2..729dd0b6 100644 --- a/backend/api/api/Services/IPredictorService.cs +++ b/backend/api/api/Services/IPredictorService.cs @@ -6,6 +6,7 @@ namespace api.Services public interface IPredictorService { Predictor GetOnePredictor(string username, string name); + Predictor GetPredictor(string username, string GetPredictor); List SearchPredictors(string name, string username); List GetMyPredictors(string username); List SortPredictors(string username, bool ascdsc, int latest); diff --git a/backend/api/api/Services/PredictorService.cs b/backend/api/api/Services/PredictorService.cs index 05860126..01bc8359 100644 --- a/backend/api/api/Services/PredictorService.cs +++ b/backend/api/api/Services/PredictorService.cs @@ -39,6 +39,11 @@ namespace api.Services { return _predictor.Find(predictor => predictor.username == username && predictor.name == name).FirstOrDefault(); + } + public Predictor GetPredictor(string username, string id) + { + return _predictor.Find(predictor => predictor.username == username && predictor._id == id).FirstOrDefault(); + } //last private models public List SortPredictors(string username, bool ascdsc, int latest) -- cgit v1.2.3 From 28e3d25b5d4756361b464e8ca34fbc986fc0e75f Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Tue, 29 Mar 2022 17:10:29 +0200 Subject: Kada korisnik menja username dodata provera da li taj username vec postoji u bazi.Prilikom menjanja username-a updajtaju se sve ostale tabele koje sadrze stari username novim username-om korisnika.Vraca povratnu informaciju da li je operacija uspesna. Dodati mehanizmi za transakcije.(Unkomentarisati ih kada se dozvoli da baza podrzava transakcije). --- backend/api/api/Controllers/UserController.cs | 3 +- backend/api/api/Services/IUserService.cs | 2 +- backend/api/api/Services/UserService.cs | 51 +++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 5 deletions(-) (limited to 'backend/api') diff --git a/backend/api/api/Controllers/UserController.cs b/backend/api/api/Controllers/UserController.cs index 0287f3cb..741382b8 100644 --- a/backend/api/api/Controllers/UserController.cs +++ b/backend/api/api/Controllers/UserController.cs @@ -135,8 +135,7 @@ namespace api.Controllers else return BadRequest(); - userService.Update(username, user); - return NoContent(); + return Ok(userService.Update(username, user)); } // DELETE api//5 diff --git a/backend/api/api/Services/IUserService.cs b/backend/api/api/Services/IUserService.cs index 1cb6a609..e4a23213 100644 --- a/backend/api/api/Services/IUserService.cs +++ b/backend/api/api/Services/IUserService.cs @@ -8,7 +8,7 @@ namespace api.Services List Get();// daje sve korisnike User GetUserUsername(string username); //daje korisnika po korisnickom imenu User Create(User user); // kreira korisnika - void Update(string username, User user); //apdejtuje korisnika po idu + bool Update(string username, User user); //apdejtuje korisnika po idu void Delete(string username);//brise korisnika } } diff --git a/backend/api/api/Services/UserService.cs b/backend/api/api/Services/UserService.cs index f613f923..607bb04b 100644 --- a/backend/api/api/Services/UserService.cs +++ b/backend/api/api/Services/UserService.cs @@ -7,11 +7,22 @@ namespace api.Services public class UserService : IUserService { private readonly IMongoCollection _users; + private readonly IMongoClient _client; + private readonly IMongoCollection _models; + private readonly IMongoCollection _datasets; + private readonly IMongoCollection _fileModels; + private readonly IMongoCollection _predictors; + public UserService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) { var database = mongoClient.GetDatabase(settings.DatabaseName); _users = database.GetCollection(settings.CollectionName); + _models = database.GetCollection(settings.ModelCollectionName); + _datasets= database.GetCollection(settings.DatasetCollectionName); + _fileModels = database.GetCollection(settings.FilesCollectionName); + _predictors= database.GetCollection(settings.PredictorCollectionName); + _client = mongoClient; } public User Create(User user) { @@ -26,10 +37,46 @@ namespace api.Services { return _users.Find(user => user.Username == username).FirstOrDefault(); } - public void Update(string username, User user) + public bool Update(string username, User user) { //username koji postoji u bazi - _users.ReplaceOne(user => user.Username == username, user); + using (var session = _client.StartSession()) + { + + if(_users.Find(u => u.Username == user.Username).FirstOrDefault()!=null) + { + return false; + } + + //Trenutan MongoDB Server ne podrzava transakcije.Omoguciti Podrsku + //session.StartTransaction(); + try + { + _users.ReplaceOne(user => user.Username == username, user); + if (username != user.Username) + { + var builderModel = Builders.Update; + var builderDataset = Builders.Update; + var builderFileModel = Builders.Update; + var builderPredictor = Builders.Update; + _models.UpdateMany(x => x.username == username, builderModel.Set(x => x.username, user.Username)); + _datasets.UpdateMany(x => x.username == username, builderDataset.Set(x => x.username, user.Username)); + _fileModels.UpdateMany(x => x.username == username, builderFileModel.Set(x => x.username, user.Username)); + _predictors.UpdateMany(x => x.username == username, builderPredictor.Set(x => x.username, user.Username)); + } + + //session.AbortTransaction(); + + + //session.CommitTransaction(); + } + catch (Exception e) + { + //session.AbortTransaction(); + return false; + } + return true; + } } public void Delete(string username) { -- cgit v1.2.3