From f8b9f36958f78a1c7605fad02ba9036ce17c51a0 Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sat, 2 Apr 2022 19:54:58 +0200 Subject: Napravljena Experiment klasa. Dodat Experiment servis. Napravljen dep injection za experiment servis. Napravljena konekcija sa bazom. --- backend/api/api/Data/UserStoreDatabaseSettings.cs | 1 + .../api/Interfaces/IUserStoreDatabaseSettings.cs | 1 + backend/api/api/Models/Experiment.cs | 19 ++++++++++++++++ backend/api/api/Program.cs | 1 + backend/api/api/Services/ExperimentService.cs | 22 +++++++++++++++++++ backend/api/api/Services/IExperimentService.cs | 9 ++++++++ backend/api/api/appsettings.json | 25 +++++++++++----------- 7 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 backend/api/api/Models/Experiment.cs create mode 100644 backend/api/api/Services/ExperimentService.cs create mode 100644 backend/api/api/Services/IExperimentService.cs (limited to 'backend/api') diff --git a/backend/api/api/Data/UserStoreDatabaseSettings.cs b/backend/api/api/Data/UserStoreDatabaseSettings.cs index e83d2b54..822f9bf5 100644 --- a/backend/api/api/Data/UserStoreDatabaseSettings.cs +++ b/backend/api/api/Data/UserStoreDatabaseSettings.cs @@ -13,5 +13,6 @@ namespace api.Data public string PredictorCollectionName { get; set; } = String.Empty; public string ModelCollectionName { get; set; } = String.Empty; public string FilesCollectionName { get; set; } = String.Empty; + public string ExperimentCollectionName { get; set; } = String.Empty; } } diff --git a/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs b/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs index a5b5f5eb..05c41701 100644 --- a/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs +++ b/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs @@ -9,5 +9,6 @@ string PredictorCollectionName { get; set; } string ModelCollectionName { get; set; } string FilesCollectionName { get; set; } + string ExperimentCollectionName { get; set; } } } diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs new file mode 100644 index 00000000..3b435d3f --- /dev/null +++ b/backend/api/api/Models/Experiment.cs @@ -0,0 +1,19 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace api.Models +{ + public class Experiment + { + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string _id { get; set; } + + public string datasetId { get; set; } + public string[] inputColumns { get; set; } + public string outputColumn { get; set; } + public string nullValues { get; set; } + public NullValues[] nullValuesReplacers { get; set; } + + } +} diff --git a/backend/api/api/Program.cs b/backend/api/api/Program.cs index 2bb97e45..7d5d0c45 100644 --- a/backend/api/api/Program.cs +++ b/backend/api/api/Program.cs @@ -34,6 +34,7 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); var mlwss = new MLWebSocketService(); diff --git a/backend/api/api/Services/ExperimentService.cs b/backend/api/api/Services/ExperimentService.cs new file mode 100644 index 00000000..a3b28da4 --- /dev/null +++ b/backend/api/api/Services/ExperimentService.cs @@ -0,0 +1,22 @@ +using api.Interfaces; +using api.Models; +using MongoDB.Driver; + +namespace api.Services +{ + public class ExperimentService : IExperimentService + { + private readonly IMongoCollection _experiment; + public ExperimentService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) + { + var database = mongoClient.GetDatabase(settings.DatabaseName); + _experiment = database.GetCollection(settings.ExperimentCollectionName); + } + + public Experiment Create(Experiment experiment) + { + _experiment.InsertOne(experiment); + return experiment; + } + } +} diff --git a/backend/api/api/Services/IExperimentService.cs b/backend/api/api/Services/IExperimentService.cs new file mode 100644 index 00000000..7c80599a --- /dev/null +++ b/backend/api/api/Services/IExperimentService.cs @@ -0,0 +1,9 @@ +using api.Models; + +namespace api.Services +{ + public interface IExperimentService + { + Experiment Create(Experiment experiment); + } +} \ No newline at end of file diff --git a/backend/api/api/appsettings.json b/backend/api/api/appsettings.json index fdccfb07..f8923a10 100644 --- a/backend/api/api/appsettings.json +++ b/backend/api/api/appsettings.json @@ -9,17 +9,18 @@ } }, "AllowedHosts": "*", - "UserStoreDatabaseSettings": { - /* LocalHost - */ - "ConnectionString": "mongodb://127.0.0.1:27017/", - "DatabaseName": "si_project", - "CollectionName": "users", - "DatasetCollectionName": "Dataset", - "ModelCollectionName": "Model", - "PredictorCollectionName": "Predictor", - "FilesCollectionName": "Files" - /* + "UserStoreDatabaseSettings": { + /* LocalHost + */ + "ConnectionString": "mongodb://127.0.0.1:27017/", + "DatabaseName": "si_project", + "CollectionName": "users", + "DatasetCollectionName": "Dataset", + "ModelCollectionName": "Model", + "PredictorCollectionName": "Predictor", + "FilesCollectionName": "Files", + "ExperimentCollectionName": "Experiment" + /* "ConnectionString": "mongodb+srv://si_user:si_user@sidatabase.twtfm.mongodb.net/myFirstDatabase?retryWrites=true&w=majority", "DatabaseName": "si_db", "CollectionName": "users", @@ -27,5 +28,5 @@ "ModelCollectionName": "Model", "PredictorCollectionName": "Predictor", "FilesCollectionName": "Files"*/ - } + } } \ No newline at end of file -- cgit v1.2.3 From 03852781bb840c2c76e400c4408ddf57896bc95b Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sat, 2 Apr 2022 20:44:25 +0200 Subject: Izmenjeni Dataset,Experiment,Model.Predictor,prosiren experiment servis,ispravljena greska kod model Controllera,dodata klasa ColumnInfo. Potrebno ispraviti greske!!! --- backend/api/api/Controllers/ModelController.cs | 7 +++++-- backend/api/api/Models/ColumnInfo.cs | 13 +++++++++++++ backend/api/api/Models/Dataset.cs | 4 ++++ backend/api/api/Models/Experiment.cs | 3 +++ backend/api/api/Models/Model.cs | 10 +--------- backend/api/api/Models/Predictor.cs | 1 + backend/api/api/Services/ExperimentService.cs | 4 ++++ backend/api/api/Services/IExperimentService.cs | 1 + backend/api/api/Services/TempRemovalService.cs | 2 ++ 9 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 backend/api/api/Models/ColumnInfo.cs (limited to 'backend/api') diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index 210d37c8..ffc99cb3 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -17,15 +17,17 @@ namespace api.Controllers private readonly IDatasetService _datasetService; private readonly IFileService _fileService; private readonly IModelService _modelService; + private readonly IExperimentService _experimentService; private IJwtToken jwtToken; - public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration,IJwtToken token) + public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration,IJwtToken token,IExperimentService experiment) { _mlService = mlService; _modelService = modelService; _datasetService = datasetService; _fileService = fileService; + _experimentService = experiment; jwtToken = token; } @@ -145,7 +147,8 @@ namespace api.Controllers { bool overwrite = false; //username="" ako je GUEST - model.inputNeurons = model.inputColumns.Length; + Experiment e = _experimentService.Get(model.experimentId); + model.inputNeurons = e.inputColumns.Length; if (_modelService.CheckHyperparameters(model.inputNeurons, model.hiddenLayerNeurons, model.hiddenLayers, model.outputNeurons) == false) return BadRequest("Bad parameters!"); diff --git a/backend/api/api/Models/ColumnInfo.cs b/backend/api/api/Models/ColumnInfo.cs new file mode 100644 index 00000000..ee4cee0d --- /dev/null +++ b/backend/api/api/Models/ColumnInfo.cs @@ -0,0 +1,13 @@ +namespace api.Models +{ + public class ColumnInfo + { + public string columnName { get; set; } + public bool isNumber { get; set; } + public int numNulls { get; set; } + public float mean { get; set; } + public float median { get; set; } + public string[] uniqueValues { get; set; } + + } +} diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index dcfde3b1..17974d1d 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -22,6 +22,10 @@ namespace api.Models public DateTime lastUpdated { get; set; } public string delimiter { get; set; } public bool hasHeader { get; set; } + + public ColumnInfo[] columnInfo { get; set; } + public int totalNulls; + } } diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs index 3b435d3f..ec50950d 100644 --- a/backend/api/api/Models/Experiment.cs +++ b/backend/api/api/Models/Experiment.cs @@ -12,6 +12,9 @@ namespace api.Models public string datasetId { get; set; } public string[] inputColumns { get; set; } public string outputColumn { get; set; } + public bool randomOrder { get; set; } + public bool randomTestSet { get; set; } + public float randomTestSetDistribution { get; set; } public string nullValues { get; set; } public NullValues[] nullValuesReplacers { get; set; } diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs index 86541767..e88e7e74 100644 --- a/backend/api/api/Models/Model.cs +++ b/backend/api/api/Models/Model.cs @@ -18,14 +18,8 @@ namespace api.Models public DateTime dateCreated { get; set; } public DateTime lastUpdated { get; set; } //proveriti id - public string datasetId { get; set; } + public string experimentId { get; set; } - //Test set settings - public string[] inputColumns { get; set; } - public string columnToPredict { get; set; } - public bool randomOrder {get;set;} - public bool randomTestSet { get; set; } - public float randomTestSetDistribution { get; set; } //Neural net training public string type { get; set; } @@ -44,8 +38,6 @@ namespace api.Models public string[] metrics { get; set; } public int epochs { get; set; } - public string nullValues { get; set; } - public NullValues[] nullValuesReplacers { get; set; } //public NullValues[] nullValues { get; set; } } diff --git a/backend/api/api/Models/Predictor.cs b/backend/api/api/Models/Predictor.cs index 9c8d362f..5fd2aa09 100644 --- a/backend/api/api/Models/Predictor.cs +++ b/backend/api/api/Models/Predictor.cs @@ -17,6 +17,7 @@ namespace api.Models public bool isPublic { get; set; } public bool accessibleByLink { get; set; } public DateTime dateCreated { get; set; } + public string experimentId { get; set; } } } diff --git a/backend/api/api/Services/ExperimentService.cs b/backend/api/api/Services/ExperimentService.cs index a3b28da4..712568d4 100644 --- a/backend/api/api/Services/ExperimentService.cs +++ b/backend/api/api/Services/ExperimentService.cs @@ -18,5 +18,9 @@ namespace api.Services _experiment.InsertOne(experiment); return experiment; } + public Experiment Get(string id) + { + return _experiment.Find(exp=>exp._id == id).FirstOrDefault(); + } } } diff --git a/backend/api/api/Services/IExperimentService.cs b/backend/api/api/Services/IExperimentService.cs index 7c80599a..f16f3eae 100644 --- a/backend/api/api/Services/IExperimentService.cs +++ b/backend/api/api/Services/IExperimentService.cs @@ -5,5 +5,6 @@ namespace api.Services public interface IExperimentService { Experiment Create(Experiment experiment); + public Experiment Get(string id); } } \ No newline at end of file diff --git a/backend/api/api/Services/TempRemovalService.cs b/backend/api/api/Services/TempRemovalService.cs index 9741ace2..7f723986 100644 --- a/backend/api/api/Services/TempRemovalService.cs +++ b/backend/api/api/Services/TempRemovalService.cs @@ -9,6 +9,7 @@ namespace api.Services private readonly IMongoCollection _file; private readonly IMongoCollection _model; private readonly IMongoCollection _dataset; + private readonly IMongoCollection _experiment; public TempRemovalService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) { @@ -16,6 +17,7 @@ namespace api.Services _file = database.GetCollection(settings.FilesCollectionName); _model= database.GetCollection(settings.ModelCollectionName); _dataset = database.GetCollection(settings.DatasetCollectionName); + _experiment= database.GetCollection(settings.ExperimentCollectionName); } public void DeleteTemps() { -- cgit v1.2.3 From 15f225dd766d93e190581763d8bffa4daf98d6ef Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 02:11:01 +0200 Subject: Prilagodjeno brisanje temp fajlova i kolekcija u bazu.Ispravljena greska --- backend/api/api/Controllers/ModelController.cs | 3 ++- backend/api/api/Models/Experiment.cs | 1 + backend/api/api/Services/TempRemovalService.cs | 17 +++++++++++++---- 3 files changed, 16 insertions(+), 5 deletions(-) (limited to 'backend/api') diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index ffc99cb3..97e7c7a7 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -47,7 +47,8 @@ namespace api.Controllers } else return BadRequest(); - var dataset = _datasetService.GetOneDataset(model.datasetId); + var experiment=_experimentService.Get(model.experimentId); + var dataset = _datasetService.GetOneDataset(experiment.datasetId); var filepath = _fileService.GetFilePath(dataset.fileId, uploaderId); var result = await _mlService.SendModelAsync(model, filepath); return Ok(result); diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs index ec50950d..6de3f169 100644 --- a/backend/api/api/Models/Experiment.cs +++ b/backend/api/api/Models/Experiment.cs @@ -10,6 +10,7 @@ namespace api.Models public string _id { get; set; } public string datasetId { get; set; } + public string uploaderId { get; set; } public string[] inputColumns { get; set; } public string outputColumn { get; set; } public bool randomOrder { get; set; } diff --git a/backend/api/api/Services/TempRemovalService.cs b/backend/api/api/Services/TempRemovalService.cs index 7f723986..d72dc89e 100644 --- a/backend/api/api/Services/TempRemovalService.cs +++ b/backend/api/api/Services/TempRemovalService.cs @@ -31,11 +31,16 @@ namespace api.Services foreach(var dataset in datasets) { DeleteDataset(dataset._id); - List models = _model.Find(model => model.datasetId == dataset._id && model.username=="").ToList(); - foreach(var model in models) + List experiments = _experiment.Find(experiment=>experiment.datasetId== dataset._id && experiment.uploaderId=="").ToList(); + foreach(var experiment in experiments) { - DeleteModel(model._id); - } + DeleteExperiment(experiment._id); + List models = _model.Find(model => model.experimentId == experiment._id && model.username == "").ToList(); + foreach (var model in models) + { + DeleteModel(model._id); + } + } } if (File.Exists(file.path)) File.Delete(file.path); @@ -69,6 +74,10 @@ namespace api.Services { _dataset.DeleteOne(dataset => dataset._id == id); } + public void DeleteExperiment(string id) + { + _experiment.DeleteOne(experiment => experiment._id == id); + } } -- cgit v1.2.3 From 1a2a2e928769158b75dbf6e3366ab8ed312cd131 Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 21:37:11 +0200 Subject: Dodat experiment controller. --- .../api/api/Controllers/ExperimentController.cs | 75 ++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 backend/api/api/Controllers/ExperimentController.cs (limited to 'backend/api') diff --git a/backend/api/api/Controllers/ExperimentController.cs b/backend/api/api/Controllers/ExperimentController.cs new file mode 100644 index 00000000..2a2db31e --- /dev/null +++ b/backend/api/api/Controllers/ExperimentController.cs @@ -0,0 +1,75 @@ +using System.Net.Http.Headers; +using api.Models; +using api.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Net.Http.Headers; + +namespace api.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class ExperimentController : ControllerBase + { + + private readonly IExperimentService _experimentService; + private IJwtToken jwtToken; + + public ExperimentController(IExperimentService experimentService, IConfiguration configuration, IJwtToken Token) + { + _experimentService = experimentService; + jwtToken = Token; + } + + [HttpPost("add")] + [Authorize(Roles = "User,Guest")] + public async Task> Post([FromBody] Experiment experiment) + { + string uploaderId; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + uploaderId = jwtToken.TokenToId(parameter); + if (uploaderId == null) + return null; + } + else + return BadRequest(); + + experiment.uploaderId = uploaderId; + + _experimentService.Create(experiment); + return Ok(experiment); + } + + [HttpGet("get")] + [Authorize(Roles = "User,Guest")] + public async Task> Get(string id) + { + string uploaderId; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + uploaderId = jwtToken.TokenToId(parameter); + if (uploaderId == null) + return null; + } + else + return BadRequest(); + + var experiment = _experimentService.Get(id); + if(experiment.uploaderId!=uploaderId) + return BadRequest("Not your experiment"); + + return Ok(experiment); + } + + + + } +} -- cgit v1.2.3 From 7ba7c0451ef4e5a65cf5a9f7391c520b14f9654f Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 22:21:46 +0200 Subject: Kada se dodaje dataset prosledjuje se zahtev ml api-ju da izracuna columnInfo. Potrebno uskladiti sa ml api-jem. --- backend/api/api/Controllers/DatasetController.cs | 16 ++++++++++++---- backend/api/api/Services/FileService.cs | 4 ++++ backend/api/api/Services/IFileService.cs | 1 + backend/api/api/Services/IMlConnectionService.cs | 4 ++++ backend/api/api/Services/MlConnectionService.cs | 16 +++++++++++++++- 5 files changed, 36 insertions(+), 5 deletions(-) (limited to 'backend/api') diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index 8a622138..8a3485dd 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -14,11 +14,15 @@ namespace api.Controllers public class DatasetController : ControllerBase { private readonly IDatasetService _datasetService; + private readonly IMlConnectionService _mlConnectionService; + private readonly IFileService _fileService; private IJwtToken jwtToken; - public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token) + public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token,IMlConnectionService mlConnectionService, IFileService fileService) { _datasetService = datasetService; + _mlConnectionService = mlConnectionService; + _fileService = fileService; jwtToken = Token; } @@ -159,7 +163,7 @@ namespace api.Controllers // POST api//add [HttpPost("add")] [Authorize(Roles = "User,Guest")] - public ActionResult Post([FromBody] Dataset dataset) + public async Task> Post([FromBody] Dataset dataset) { //da li ce preko tokena da se ubaci username ili front salje //dataset.username = usernameToken; @@ -170,9 +174,13 @@ namespace api.Controllers return NotFound($"Dateset with name = {dataset.name} exisits"); else { - _datasetService.Create(dataset); - return CreatedAtAction(nameof(Get), new { id = dataset._id }, dataset); + FileModel fileModel = _fileService.getFile(dataset.fileId); + Dataset newDataset =await _mlConnectionService.PreProcess(dataset,fileModel.path); + + _datasetService.Create(newDataset); + + return CreatedAtAction(nameof(Get), new { id = newDataset._id }, newDataset); } } diff --git a/backend/api/api/Services/FileService.cs b/backend/api/api/Services/FileService.cs index 24491124..13f596d0 100644 --- a/backend/api/api/Services/FileService.cs +++ b/backend/api/api/Services/FileService.cs @@ -36,6 +36,10 @@ namespace api.Services return null; return file.path; } + public FileModel getFile(string id) + { + return _file.Find(x=>x._id==id).FirstOrDefault(); + } } } diff --git a/backend/api/api/Services/IFileService.cs b/backend/api/api/Services/IFileService.cs index 7446e283..d65deb48 100644 --- a/backend/api/api/Services/IFileService.cs +++ b/backend/api/api/Services/IFileService.cs @@ -6,5 +6,6 @@ namespace api.Services { FileModel Create(FileModel file); string GetFilePath(string id, string username); + public FileModel getFile(string id); } } \ No newline at end of file diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index ee839d28..490f2df7 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -1,8 +1,12 @@  +using api.Models; + namespace api.Services { public interface IMlConnectionService { Task SendModelAsync(object model, object dataset); + Task PreProcess(Dataset dataset, string filePath); + //Task PreProcess(Dataset dataset, byte[] file, string filename); } } \ No newline at end of file diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index 9c3b3fd8..a04a323b 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -1,6 +1,8 @@ -using RestSharp; +using api.Models; +using RestSharp; using System.Net.WebSockets; using System.Text; +using Newtonsoft.Json; namespace api.Services { @@ -20,5 +22,17 @@ namespace api.Services var result = await this.client.ExecuteAsync(request); return result.Content; //Response od ML microservisa } + public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) + { + var request=new RestRequest("preprocess", Method.Post);//USKLADITI SA ML API + request.AddParameter("dataset", JsonConvert.SerializeObject(dataset)); + //request.AddFile("file", file,filename); + request.AddFile("file", filePath); + request.AddHeader("Content-Type", "multipart/form-data"); + var result=await this.client.ExecuteAsync(request); + Dataset newDataset = JsonConvert.DeserializeObject(result.Content); + return newDataset; + + } } } -- cgit v1.2.3 From 5fd7e2e9777ab02fad5b7e1a977fe282a98dc7fc Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Mon, 4 Apr 2022 21:31:01 +0200 Subject: Promenja logika za preprocesiranje. --- backend/api/api/Controllers/DatasetController.cs | 12 +++++------- backend/api/api/Models/Dataset.cs | 1 + backend/api/api/Services/DatasetService.cs | 20 ++++++++++++-------- backend/api/api/Services/IDatasetService.cs | 1 + backend/api/api/Services/IMlConnectionService.cs | 2 +- backend/api/api/Services/MlConnectionService.cs | 11 ++++++++--- 6 files changed, 28 insertions(+), 19 deletions(-) (limited to 'backend/api') diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index 8a3485dd..6eb1b9e6 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -139,9 +139,8 @@ namespace api.Controllers return BadRequest(); var dataset = _datasetService.GetOneDataset(username, name); - if (dataset == null) - return NotFound($"Dataset with name = {name} not found or dataset is not public"); + return NotFound($"Dataset with name = {name} not found or dataset is not public or not preprocessed"); return dataset; } @@ -176,11 +175,10 @@ namespace api.Controllers { FileModel fileModel = _fileService.getFile(dataset.fileId); - Dataset newDataset =await _mlConnectionService.PreProcess(dataset,fileModel.path); - - _datasetService.Create(newDataset); - - return CreatedAtAction(nameof(Get), new { id = newDataset._id }, newDataset); + dataset.isPreProcess = false; + _datasetService.Create(dataset); + _mlConnectionService.PreProcess(dataset,fileModel.path); + return Ok(); } } diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index 17974d1d..2b3efa3c 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -25,6 +25,7 @@ namespace api.Models public ColumnInfo[] columnInfo { get; set; } public int totalNulls; + public bool isPreProcess { get; set; } } } diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs index 2ff271f3..c43bd236 100644 --- a/backend/api/api/Services/DatasetService.cs +++ b/backend/api/api/Services/DatasetService.cs @@ -16,7 +16,7 @@ namespace api.Services public List SearchDatasets(string name, string username) { - return _dataset.Find(dataset => dataset.name == name && dataset.isPublic == true).ToList(); + return _dataset.Find(dataset => dataset.name == name && dataset.isPublic == true && dataset.isPreProcess).ToList(); } //kreiranje dataseta @@ -34,12 +34,12 @@ namespace api.Services public List GetMyDatasets(string username) { - return _dataset.Find(dataset => dataset.username == username).ToList(); + return _dataset.Find(dataset => dataset.username == username && dataset.isPreProcess).ToList(); } public List GetGuestDatasets() { //Join Igranonica public datasetove sa svim temp uploadanim datasetovima - List datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true).ToList(); + List datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true && dataset.isPreProcess).ToList(); datasets.AddRange(_dataset.Find(dataset => dataset.username == "").ToList()); return datasets; } @@ -47,7 +47,7 @@ namespace api.Services //poslednji datasetovi public List SortDatasets(string username, bool ascdsc, int latest) { - List list = _dataset.Find(dataset => dataset.username == username).ToList(); + List list = _dataset.Find(dataset => dataset.username == username && dataset.isPreProcess).ToList(); if(ascdsc) list = list.OrderBy(dataset => dataset.lastUpdated).ToList(); @@ -59,25 +59,29 @@ namespace api.Services public List GetPublicDatasets() { - return _dataset.Find(dataset => dataset.isPublic == true).ToList(); + return _dataset.Find(dataset => dataset.isPublic == true && dataset.isPreProcess).ToList(); } public Dataset GetOneDataset(string username, string name) { - return _dataset.Find(dataset => dataset.username == username && dataset.name == name).FirstOrDefault(); + return _dataset.Find(dataset => dataset.username == username && dataset.name == name && dataset.isPreProcess).FirstOrDefault(); } //odraditi za pretragu getOne public Dataset GetOneDataset(string id) { - return _dataset.Find(dataset => dataset._id == id).FirstOrDefault(); + return _dataset.Find(dataset => dataset._id == id && dataset.isPreProcess).FirstOrDefault(); } //ako je potrebno da se zameni name ili ekstenzija - public void Update(string username, string name, Dataset dataset) + public void Update(string username, string name, Dataset dataset ) { _dataset.ReplaceOne(dataset => dataset.username == username && dataset.name == name, dataset); } + public void Update(Dataset dataset) + { + _dataset.ReplaceOne(x=>x._id==dataset._id, dataset); + } } diff --git a/backend/api/api/Services/IDatasetService.cs b/backend/api/api/Services/IDatasetService.cs index 8e62ba43..89856c5b 100644 --- a/backend/api/api/Services/IDatasetService.cs +++ b/backend/api/api/Services/IDatasetService.cs @@ -15,5 +15,6 @@ namespace api.Services void Update(string username, string name, Dataset dataset); void Delete(string username, string name); public List GetGuestDatasets(); + public void Update(Dataset dataset); } } diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index 490f2df7..ea73fb0f 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -6,7 +6,7 @@ namespace api.Services public interface IMlConnectionService { Task SendModelAsync(object model, object dataset); - Task PreProcess(Dataset dataset, string filePath); + Task PreProcess(Dataset dataset, string filePath); //Task PreProcess(Dataset dataset, byte[] file, string filename); } } \ No newline at end of file diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index a04a323b..66f7882a 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -9,10 +9,12 @@ namespace api.Services public class MlConnectionService : IMlConnectionService { private RestClient client; + private readonly IDatasetService _datasetService; - public MlConnectionService() + public MlConnectionService(IDatasetService datasetService) { this.client = new RestClient("http://127.0.0.1:5543"); + _datasetService=datasetService; } public async Task SendModelAsync(object model, object dataset) @@ -22,7 +24,7 @@ namespace api.Services var result = await this.client.ExecuteAsync(request); return result.Content; //Response od ML microservisa } - public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) + public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) { var request=new RestRequest("preprocess", Method.Post);//USKLADITI SA ML API request.AddParameter("dataset", JsonConvert.SerializeObject(dataset)); @@ -31,7 +33,10 @@ namespace api.Services request.AddHeader("Content-Type", "multipart/form-data"); var result=await this.client.ExecuteAsync(request); Dataset newDataset = JsonConvert.DeserializeObject(result.Content); - return newDataset; + newDataset.isPreProcess = true; + _datasetService.Update(newDataset); + + return; } } -- cgit v1.2.3