diff options
Diffstat (limited to 'backend')
22 files changed, 343 insertions, 23 deletions
diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index bae05ba9..d9803744 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -24,7 +24,7 @@ namespace api.Controllers // GET: api/<DatasetController>/mydatasets [HttpGet("mydatasets")] - [Authorize(Roles = "User")] + [Authorize(Roles = "User,Guest")] public ActionResult<List<Dataset>> Get() { string username; @@ -39,6 +39,9 @@ namespace api.Controllers } else return BadRequest(); + //U slucaju da je korisnik gost vrati dataSetove igrannonice + if (username == "") + return _datasetService.GetGuestDatasets(); //ako bude trebao ID, samo iz baze uzeti diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index b997efa3..4bc094cd 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -14,22 +14,28 @@ namespace api.Controllers { private IMlConnectionService _mlService; + private readonly IDatasetService _datasetService; + private readonly IFileService _fileService; private readonly IModelService _modelService; private JwtToken jwtToken; - public ModelController(IMlConnectionService mlService, IModelService modelService, IConfiguration configuration) + public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration) { _mlService = mlService; _modelService = modelService; + _datasetService = datasetService; + _fileService = fileService; jwtToken = new JwtToken(configuration); } [HttpPost("sendModel")] [Authorize(Roles = "User")] - public async Task<ActionResult<string>> Test([FromBody] object model) + public async Task<ActionResult<string>> Test([FromBody] Model model) { - var result = await _mlService.SendModelAsync(model); + var dataset = _datasetService.GetOneDataset(model.datasetId); + var filepath = _fileService.GetFilePath(dataset.fileId, dataset.username); + var result = await _mlService.SendModelAsync(model, filepath); return Ok(result); } diff --git a/backend/api/api/Controllers/PredictorController.cs b/backend/api/api/Controllers/PredictorController.cs index 7f8f1692..63c5d2bf 100644 --- a/backend/api/api/Controllers/PredictorController.cs +++ b/backend/api/api/Controllers/PredictorController.cs @@ -74,6 +74,36 @@ namespace api.Controllers return _predictorService.SearchPredictors(name, username); } + //SEARCH za predictore (public ili private sa ovim imenom ) + // GET api/<PredictorController>/search/{name} + [HttpGet("{id}")] + [Authorize(Roles = "User")] + public ActionResult<Predictor> GetPredictor(string id) + { + string username; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + username = jwtToken.TokenToUsername(parameter); + if (username == null) + return null; + } + else + return BadRequest(); + + //ako bude trebao ID, samo iz baze uzeti + + Predictor predictor = _predictorService.GetPredictor(username, id); + + if (predictor == null) + return NotFound($"Predictor with id = {id} not found"); + + return predictor; + } + + //da li da se odvoji search za public i posebno za private? // GET api/<PredictorController>/{name} [HttpGet("{name}")] diff --git a/backend/api/api/Controllers/UserController.cs b/backend/api/api/Controllers/UserController.cs index 0287f3cb..741382b8 100644 --- a/backend/api/api/Controllers/UserController.cs +++ b/backend/api/api/Controllers/UserController.cs @@ -135,8 +135,7 @@ namespace api.Controllers else return BadRequest(); - userService.Update(username, user); - return NoContent(); + return Ok(userService.Update(username, user)); } // DELETE api/<UserController>/5 diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs index 5678daaf..2baab1c0 100644 --- a/backend/api/api/Models/Model.cs +++ b/backend/api/api/Models/Model.cs @@ -38,10 +38,13 @@ namespace api.Models public int batchSize { get; set; } // na izlazu je moguce da bude vise neurona (klasifikacioni problem sa vise od 2 klase) public int outputNeurons { get; set; } - public string inputLayerActivationFunction { get; set; } - public string hiddenLayerActivationFunction { get; set; } + public string[] hiddenLayerActivationFunctions { get; set; } public string outputLayerActivationFunction { get; set; } + public string[] metrics { get; set; } + public int epochs { get; set; } + public string nullValues { get; set; } + public string[] nullValuesReplacers { get; set; } } } diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs index 5e708d11..2ff271f3 100644 --- a/backend/api/api/Services/DatasetService.cs +++ b/backend/api/api/Services/DatasetService.cs @@ -36,6 +36,13 @@ namespace api.Services { return _dataset.Find(dataset => dataset.username == username).ToList(); } + public List<Dataset> GetGuestDatasets() + { + //Join Igranonica public datasetove sa svim temp uploadanim datasetovima + List<Dataset> datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true).ToList(); + datasets.AddRange(_dataset.Find(dataset => dataset.username == "").ToList()); + return datasets; + } //poslednji datasetovi public List<Dataset> SortDatasets(string username, bool ascdsc, int latest) @@ -61,6 +68,11 @@ namespace api.Services } //odraditi za pretragu getOne + public Dataset GetOneDataset(string id) + { + return _dataset.Find(dataset => dataset._id == id).FirstOrDefault(); + } + //ako je potrebno da se zameni name ili ekstenzija public void Update(string username, string name, Dataset dataset) { diff --git a/backend/api/api/Services/IDatasetService.cs b/backend/api/api/Services/IDatasetService.cs index be56f5cb..8e62ba43 100644 --- a/backend/api/api/Services/IDatasetService.cs +++ b/backend/api/api/Services/IDatasetService.cs @@ -6,6 +6,7 @@ namespace api.Services public interface IDatasetService { Dataset GetOneDataset(string username, string name); + Dataset GetOneDataset(string id); List<Dataset> SearchDatasets(string name, string username); List<Dataset> GetMyDatasets(string username); List<Dataset> SortDatasets(string username, bool ascdsc, int latest); @@ -13,5 +14,6 @@ namespace api.Services Dataset Create(Dataset dataset); void Update(string username, string name, Dataset dataset); void Delete(string username, string name); + public List<Dataset> GetGuestDatasets(); } } diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index f38fb50a..ee839d28 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -3,6 +3,6 @@ namespace api.Services { public interface IMlConnectionService { - Task<string> SendModelAsync(object model); + Task<string> SendModelAsync(object model, object dataset); } }
\ No newline at end of file diff --git a/backend/api/api/Services/IModelService.cs b/backend/api/api/Services/IModelService.cs index ee5c279f..637d09a3 100644 --- a/backend/api/api/Services/IModelService.cs +++ b/backend/api/api/Services/IModelService.cs @@ -6,6 +6,7 @@ namespace api.Services public interface IModelService { Model GetOneModel(string username, string name); + Model GetOneModel(string id); List<Model> GetMyModels(string username); List<Model> GetLatestModels(string username); //List<Model> GetPublicModels(); diff --git a/backend/api/api/Services/IPredictorService.cs b/backend/api/api/Services/IPredictorService.cs index 2017add2..729dd0b6 100644 --- a/backend/api/api/Services/IPredictorService.cs +++ b/backend/api/api/Services/IPredictorService.cs @@ -6,6 +6,7 @@ namespace api.Services public interface IPredictorService { Predictor GetOnePredictor(string username, string name); + Predictor GetPredictor(string username, string GetPredictor); List<Predictor> SearchPredictors(string name, string username); List<Predictor> GetMyPredictors(string username); List<Predictor> SortPredictors(string username, bool ascdsc, int latest); diff --git a/backend/api/api/Services/IUserService.cs b/backend/api/api/Services/IUserService.cs index 1cb6a609..e4a23213 100644 --- a/backend/api/api/Services/IUserService.cs +++ b/backend/api/api/Services/IUserService.cs @@ -8,7 +8,7 @@ namespace api.Services List<User> Get();// daje sve korisnike User GetUserUsername(string username); //daje korisnika po korisnickom imenu User Create(User user); // kreira korisnika - void Update(string username, User user); //apdejtuje korisnika po idu + bool Update(string username, User user); //apdejtuje korisnika po idu void Delete(string username);//brise korisnika } } diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index 9b167537..9c3b3fd8 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -6,13 +6,19 @@ namespace api.Services { public class MlConnectionService : IMlConnectionService { - public async Task<string> SendModelAsync(object model) + private RestClient client; + + public MlConnectionService() + { + this.client = new RestClient("http://127.0.0.1:5543"); + } + + public async Task<string> SendModelAsync(object model, object dataset) { - RestClient client = new RestClient("http://localhost:5000"); - var request = new RestRequest("data", Method.Post); - request.AddJsonBody(model); - var result = await client.ExecuteAsync(request); - return result.Content;//Response od ML microservisa + var request = new RestRequest("train", Method.Post); + request.AddJsonBody(new { model, dataset}); + var result = await this.client.ExecuteAsync(request); + return result.Content; //Response od ML microservisa } } } diff --git a/backend/api/api/Services/ModelService.cs b/backend/api/api/Services/ModelService.cs index f42219f5..eae8c78b 100644 --- a/backend/api/api/Services/ModelService.cs +++ b/backend/api/api/Services/ModelService.cs @@ -50,6 +50,11 @@ namespace api.Services return _model.Find(model => model.username == username && model.name == name).FirstOrDefault(); } + public Model GetOneModel(string id) + { + return _model.Find(model => model._id == id).FirstOrDefault(); + } + public void Update(string username, string name, Model model) { _model.ReplaceOne(model => model.username == username && model.name == name, model); diff --git a/backend/api/api/Services/PredictorService.cs b/backend/api/api/Services/PredictorService.cs index 05860126..01bc8359 100644 --- a/backend/api/api/Services/PredictorService.cs +++ b/backend/api/api/Services/PredictorService.cs @@ -40,6 +40,11 @@ namespace api.Services return _predictor.Find(predictor => predictor.username == username && predictor.name == name).FirstOrDefault(); } + public Predictor GetPredictor(string username, string id) + { + return _predictor.Find(predictor => predictor.username == username && predictor._id == id).FirstOrDefault(); + + } //last private models public List<Predictor> SortPredictors(string username, bool ascdsc, int latest) { diff --git a/backend/api/api/Services/UserService.cs b/backend/api/api/Services/UserService.cs index f613f923..607bb04b 100644 --- a/backend/api/api/Services/UserService.cs +++ b/backend/api/api/Services/UserService.cs @@ -7,11 +7,22 @@ namespace api.Services public class UserService : IUserService { private readonly IMongoCollection<User> _users; + private readonly IMongoClient _client; + private readonly IMongoCollection<Model> _models; + private readonly IMongoCollection<Dataset> _datasets; + private readonly IMongoCollection<FileModel> _fileModels; + private readonly IMongoCollection<Predictor> _predictors; + public UserService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) { var database = mongoClient.GetDatabase(settings.DatabaseName); _users = database.GetCollection<User>(settings.CollectionName); + _models = database.GetCollection<Model>(settings.ModelCollectionName); + _datasets= database.GetCollection<Dataset>(settings.DatasetCollectionName); + _fileModels = database.GetCollection<FileModel>(settings.FilesCollectionName); + _predictors= database.GetCollection<Predictor>(settings.PredictorCollectionName); + _client = mongoClient; } public User Create(User user) { @@ -26,10 +37,46 @@ namespace api.Services { return _users.Find(user => user.Username == username).FirstOrDefault(); } - public void Update(string username, User user) + public bool Update(string username, User user) { //username koji postoji u bazi - _users.ReplaceOne(user => user.Username == username, user); + using (var session = _client.StartSession()) + { + + if(_users.Find(u => u.Username == user.Username).FirstOrDefault()!=null) + { + return false; + } + + //Trenutan MongoDB Server ne podrzava transakcije.Omoguciti Podrsku + //session.StartTransaction(); + try + { + _users.ReplaceOne(user => user.Username == username, user); + if (username != user.Username) + { + var builderModel = Builders<Model>.Update; + var builderDataset = Builders<Dataset>.Update; + var builderFileModel = Builders<FileModel>.Update; + var builderPredictor = Builders<Predictor>.Update; + _models.UpdateMany(x => x.username == username, builderModel.Set(x => x.username, user.Username)); + _datasets.UpdateMany(x => x.username == username, builderDataset.Set(x => x.username, user.Username)); + _fileModels.UpdateMany(x => x.username == username, builderFileModel.Set(x => x.username, user.Username)); + _predictors.UpdateMany(x => x.username == username, builderPredictor.Set(x => x.username, user.Username)); + } + + //session.AbortTransaction(); + + + //session.CommitTransaction(); + } + catch (Exception e) + { + //session.AbortTransaction(); + return false; + } + return true; + } } public void Delete(string username) { diff --git a/backend/microservice/PythonServer/project/api/socket/client.py b/backend/microservice/PythonServer/project/socket_example/socket/client.py index d5740e25..d5740e25 100644 --- a/backend/microservice/PythonServer/project/api/socket/client.py +++ b/backend/microservice/PythonServer/project/socket_example/socket/client.py diff --git a/backend/microservice/PythonServer/project/api/socket/server.py b/backend/microservice/PythonServer/project/socket_example/socket/server.py index d6ff3f7c..d6ff3f7c 100644 --- a/backend/microservice/PythonServer/project/api/socket/server.py +++ b/backend/microservice/PythonServer/project/socket_example/socket/server.py diff --git a/backend/microservice/PythonServer/project/api/socket2/client.py b/backend/microservice/PythonServer/project/socket_example/socket2/client.py index 65e76b55..65e76b55 100644 --- a/backend/microservice/PythonServer/project/api/socket2/client.py +++ b/backend/microservice/PythonServer/project/socket_example/socket2/client.py diff --git a/backend/microservice/PythonServer/project/api/socket2/server.py b/backend/microservice/PythonServer/project/socket_example/socket2/server.py index c65dae78..c65dae78 100644 --- a/backend/microservice/PythonServer/project/api/socket2/server.py +++ b/backend/microservice/PythonServer/project/socket_example/socket2/server.py diff --git a/backend/microservice/api/controller.py b/backend/microservice/api/controller.py new file mode 100644 index 00000000..ceed02ad --- /dev/null +++ b/backend/microservice/api/controller.py @@ -0,0 +1,42 @@ +import flask +from flask import request, jsonify +import ml_socket +import ml_service +import tensorflow as tf +import pandas as pd + +app = flask.Flask(__name__) +app.config["DEBUG"] = True +app.config["SERVER_NAME"] = "127.0.0.1:5543" + +class train_callback(tf.keras.callbacks.Callback): + def __init__(self, x_test, y_test): + self.x_test = x_test + self.y_test = y_test + # + def on_epoch_end(self, epoch, logs=None): + print(epoch) + #print('Evaluation: ', self.model.evaluate(self.x_test,self.y_test),"\n") #broj parametara zavisi od izabranih metrika loss je default + +@app.route('/train', methods = ['POST']) +def train(): + print("******************************TRAIN*************************************************") + f = request.json["dataset"] + dataset = pd.read_csv(f) + # + result = ml_service.train(dataset, request.json["model"], train_callback) + print(result) + return jsonify(result) + +@app.route('/predict', methods = ['POST']) +def predict(): + f = request.json['filepath'] + dataset = pd.read_csv(f) + m = request.json['modelpath'] + #model = tf.keras.models.load_model(m) + # + #model.predict? + +print("App loaded.") +ml_socket.start() +app.run()
\ No newline at end of file diff --git a/backend/microservice/api/ml_service.py b/backend/microservice/api/ml_service.py new file mode 100644 index 00000000..efd24fdc --- /dev/null +++ b/backend/microservice/api/ml_service.py @@ -0,0 +1,155 @@ +import pandas as pd +import tensorflow as tf +import keras +import numpy as np +import csv +import json +import h5py +import sklearn.metrics as sm +from statistics import mode +from typing_extensions import Self +from copyreg import constructor +from flask import request, jsonify, render_template +from sklearn.preprocessing import LabelEncoder +from sklearn.preprocessing import StandardScaler +from sklearn.model_selection import train_test_split +from dataclasses import dataclass + +@dataclass +class TrainingResult: + accuracy: float + precision: float + recall: float + tn: float + fp: float + fn: float + tp: float + specificity: float + f1: float + mse: float + mae: float + mape: float + rmse: float + fpr: float + tpr: float + +def train(dataset, params, callback): + data = pd.DataFrame() + for col in params["inputColumns"]: + data[col]=dataset[col] + output_column = params["columnToPredict"] + data[output_column] = dataset[output_column] + # + # Brisanje null kolona / redova / zamena + #nullreplace=[ + # {"column":"Embarked","value":"C","deleteRow":false,"deleteCol":true}, + # {"column": "Cabin","value":"C123","deleteRow":"0","deleteCol":"0"}] + + null_value_options = params["nullValues"] + null_values_replacers = params["nullValuesReplacers"] + + if(null_value_options=='replace'): + print("replace null") # TODO + elif(null_value_options=='delete_rows'): + data=data.dropna() + elif(null_value_options=='delete_columns'): + data=data.dropna() + # + #print(data.isnull().any()) + # + # Brisanje kolona koje ne uticu na rezultat + # + num_rows=data.shape[0] + for col in data.columns: + if((data[col].nunique()==(num_rows)) and (data[col].dtype==np.object_)): + data.pop(col) + # + # Enkodiranje + # + encoding=params["encoding"] + if(encoding=='label'): + encoder=LabelEncoder() + for col in data.columns: + if(data[col].dtype==np.object_): + data[col]=encoder.fit_transform(data[col]) + elif(encoding=='onehot'): + category_columns=[] + for col in data.columns: + if(data[col].dtype==np.object_): + category_columns.append(col) + data=pd.get_dummies(data, columns=category_columns, prefix=category_columns) + # + # Input - output + # + x_columns = [] + for col in data.columns: + if(col!=output_column): + x_columns.append(col) + x = data[x_columns].values + y = data[output_column].values + # + # Podela na test i trening skupove + # + test=params["randomTestSetDistribution"] + randomOrder = params["randomOrder"] + if(randomOrder): + random=50 + else: + random=0 + x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=test, random_state=random) + # + # Skaliranje vrednosti + # + scaler=StandardScaler() + scaler.fit(x_train) + x_test=scaler.transform(x_test) + x_train=scaler.transform(x_train) + # + # Treniranje modela + # + classifier=tf.keras.Sequential() + hidden_layer_neurons = params["hiddenLayerNeurons"] + for func in params["hiddenLayerActivationFunctions"]: + classifier.add(tf.keras.layers.Dense(units=hidden_layer_neurons,activation=func)) + output_func = params["outputLayerActivationFunction"] + classifier.add(tf.keras.layers.Dense(units=1,activation=output_func)) + optimizer = params["optimizer"] + metrics=params['metrics'] + loss_func=params["lossFunction"] + classifier.compile(optimizer=optimizer, loss=loss_func,metrics=metrics) + batch_size = params["batchSize"] + epochs = params["epochs"] + history=classifier.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, callbacks=callback(x_test, y_test), validation_split=0.2) # TODO params["validationSplit"] + # + # Test + # + y_pred=classifier.predict(x_test) + y_pred=(y_pred>=0.5).astype('int') + #y_pred=(y_pred * 100).astype('int') + y_pred=y_pred.flatten() + result=pd.DataFrame({"Actual":y_test,"Predicted":y_pred}) + model_name = params['_id'] + classifier.save("temp/"+model_name, save_format='h5') + # + # Metrike + # + print("HELLO???") + print(result) + print("HELLO???") + accuracy = float(sm.accuracy_score(y_test,y_pred)) + precision = float(sm.precision_score(y_test,y_pred)) + recall = float(sm.recall_score(y_test,y_pred)) + tn, fp, fn, tp = sm.confusion_matrix(y_test,y_pred).ravel() + specificity = float(tn / (tn+fp)) + f1 = float(sm.f1_score(y_test,y_pred)) + mse = float(sm.mean_squared_error(y_test,y_pred)) + mae = float(sm.mean_absolute_error(y_test,y_pred)) + mape = float(sm.mean_absolute_percentage_error(y_test,y_pred)) + rmse = float(np.sqrt(sm.mean_squared_error(y_test,y_pred))) + fpr, tpr, _ = sm.roc_curve(y_test,y_pred) + # TODO upload trenirani model nazad na backend + return TrainingResult(accuracy, precision, recall, float(tn), float(fp), float(fn), float(tp), specificity, f1, mse, mae, mape, rmse, fpr.tolist(), tpr.tolist()) + + + + diff --git a/backend/microservice/ml_socket.py b/backend/microservice/api/ml_socket.py index 5489b787..65dd7321 100644 --- a/backend/microservice/ml_socket.py +++ b/backend/microservice/api/ml_socket.py @@ -14,12 +14,15 @@ def get_or_create_eventloop(): # create handler for each connection async def handler(websocket, path): #data = json.loads(await websocket.recv()) - #reply = f"Data recieved as: {data}!" #print(data['test']) msg = await websocket.recv() - await websocket.send("[" + msg + "]") + print(msg) -start_server = websockets.serve(handler, "localhost", 5027) +async def start(): + start_server = websockets.serve(handler, "localhost", 5027) + print('Websocket starting...') + get_or_create_eventloop().run_until_complete(start_server) + get_or_create_eventloop().run_forever() -get_or_create_eventloop().run_until_complete(start_server) -get_or_create_eventloop().run_forever()
\ No newline at end of file +async def send(msg): + await websocket.send(msg)
\ No newline at end of file |