From f8b9f36958f78a1c7605fad02ba9036ce17c51a0 Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sat, 2 Apr 2022 19:54:58 +0200 Subject: Napravljena Experiment klasa. Dodat Experiment servis. Napravljen dep injection za experiment servis. Napravljena konekcija sa bazom. --- backend/api/api/Data/UserStoreDatabaseSettings.cs | 1 + .../api/Interfaces/IUserStoreDatabaseSettings.cs | 1 + backend/api/api/Models/Experiment.cs | 19 ++++++++++++++++ backend/api/api/Program.cs | 1 + backend/api/api/Services/ExperimentService.cs | 22 +++++++++++++++++++ backend/api/api/Services/IExperimentService.cs | 9 ++++++++ backend/api/api/appsettings.json | 25 +++++++++++----------- 7 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 backend/api/api/Models/Experiment.cs create mode 100644 backend/api/api/Services/ExperimentService.cs create mode 100644 backend/api/api/Services/IExperimentService.cs (limited to 'backend') diff --git a/backend/api/api/Data/UserStoreDatabaseSettings.cs b/backend/api/api/Data/UserStoreDatabaseSettings.cs index e83d2b54..822f9bf5 100644 --- a/backend/api/api/Data/UserStoreDatabaseSettings.cs +++ b/backend/api/api/Data/UserStoreDatabaseSettings.cs @@ -13,5 +13,6 @@ namespace api.Data public string PredictorCollectionName { get; set; } = String.Empty; public string ModelCollectionName { get; set; } = String.Empty; public string FilesCollectionName { get; set; } = String.Empty; + public string ExperimentCollectionName { get; set; } = String.Empty; } } diff --git a/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs b/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs index a5b5f5eb..05c41701 100644 --- a/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs +++ b/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs @@ -9,5 +9,6 @@ string PredictorCollectionName { get; set; } string ModelCollectionName { get; set; } string FilesCollectionName { get; set; } + string ExperimentCollectionName { get; set; } } } diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs new file mode 100644 index 00000000..3b435d3f --- /dev/null +++ b/backend/api/api/Models/Experiment.cs @@ -0,0 +1,19 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace api.Models +{ + public class Experiment + { + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string _id { get; set; } + + public string datasetId { get; set; } + public string[] inputColumns { get; set; } + public string outputColumn { get; set; } + public string nullValues { get; set; } + public NullValues[] nullValuesReplacers { get; set; } + + } +} diff --git a/backend/api/api/Program.cs b/backend/api/api/Program.cs index 2bb97e45..7d5d0c45 100644 --- a/backend/api/api/Program.cs +++ b/backend/api/api/Program.cs @@ -34,6 +34,7 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); var mlwss = new MLWebSocketService(); diff --git a/backend/api/api/Services/ExperimentService.cs b/backend/api/api/Services/ExperimentService.cs new file mode 100644 index 00000000..a3b28da4 --- /dev/null +++ b/backend/api/api/Services/ExperimentService.cs @@ -0,0 +1,22 @@ +using api.Interfaces; +using api.Models; +using MongoDB.Driver; + +namespace api.Services +{ + public class ExperimentService : IExperimentService + { + private readonly IMongoCollection _experiment; + public ExperimentService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) + { + var database = mongoClient.GetDatabase(settings.DatabaseName); + _experiment = database.GetCollection(settings.ExperimentCollectionName); + } + + public Experiment Create(Experiment experiment) + { + _experiment.InsertOne(experiment); + return experiment; + } + } +} diff --git a/backend/api/api/Services/IExperimentService.cs b/backend/api/api/Services/IExperimentService.cs new file mode 100644 index 00000000..7c80599a --- /dev/null +++ b/backend/api/api/Services/IExperimentService.cs @@ -0,0 +1,9 @@ +using api.Models; + +namespace api.Services +{ + public interface IExperimentService + { + Experiment Create(Experiment experiment); + } +} \ No newline at end of file diff --git a/backend/api/api/appsettings.json b/backend/api/api/appsettings.json index fdccfb07..f8923a10 100644 --- a/backend/api/api/appsettings.json +++ b/backend/api/api/appsettings.json @@ -9,17 +9,18 @@ } }, "AllowedHosts": "*", - "UserStoreDatabaseSettings": { - /* LocalHost - */ - "ConnectionString": "mongodb://127.0.0.1:27017/", - "DatabaseName": "si_project", - "CollectionName": "users", - "DatasetCollectionName": "Dataset", - "ModelCollectionName": "Model", - "PredictorCollectionName": "Predictor", - "FilesCollectionName": "Files" - /* + "UserStoreDatabaseSettings": { + /* LocalHost + */ + "ConnectionString": "mongodb://127.0.0.1:27017/", + "DatabaseName": "si_project", + "CollectionName": "users", + "DatasetCollectionName": "Dataset", + "ModelCollectionName": "Model", + "PredictorCollectionName": "Predictor", + "FilesCollectionName": "Files", + "ExperimentCollectionName": "Experiment" + /* "ConnectionString": "mongodb+srv://si_user:si_user@sidatabase.twtfm.mongodb.net/myFirstDatabase?retryWrites=true&w=majority", "DatabaseName": "si_db", "CollectionName": "users", @@ -27,5 +28,5 @@ "ModelCollectionName": "Model", "PredictorCollectionName": "Predictor", "FilesCollectionName": "Files"*/ - } + } } \ No newline at end of file -- cgit v1.2.3 From 03852781bb840c2c76e400c4408ddf57896bc95b Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sat, 2 Apr 2022 20:44:25 +0200 Subject: Izmenjeni Dataset,Experiment,Model.Predictor,prosiren experiment servis,ispravljena greska kod model Controllera,dodata klasa ColumnInfo. Potrebno ispraviti greske!!! --- backend/api/api/Controllers/ModelController.cs | 7 +++++-- backend/api/api/Models/ColumnInfo.cs | 13 +++++++++++++ backend/api/api/Models/Dataset.cs | 4 ++++ backend/api/api/Models/Experiment.cs | 3 +++ backend/api/api/Models/Model.cs | 10 +--------- backend/api/api/Models/Predictor.cs | 1 + backend/api/api/Services/ExperimentService.cs | 4 ++++ backend/api/api/Services/IExperimentService.cs | 1 + backend/api/api/Services/TempRemovalService.cs | 2 ++ 9 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 backend/api/api/Models/ColumnInfo.cs (limited to 'backend') diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index 210d37c8..ffc99cb3 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -17,15 +17,17 @@ namespace api.Controllers private readonly IDatasetService _datasetService; private readonly IFileService _fileService; private readonly IModelService _modelService; + private readonly IExperimentService _experimentService; private IJwtToken jwtToken; - public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration,IJwtToken token) + public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration,IJwtToken token,IExperimentService experiment) { _mlService = mlService; _modelService = modelService; _datasetService = datasetService; _fileService = fileService; + _experimentService = experiment; jwtToken = token; } @@ -145,7 +147,8 @@ namespace api.Controllers { bool overwrite = false; //username="" ako je GUEST - model.inputNeurons = model.inputColumns.Length; + Experiment e = _experimentService.Get(model.experimentId); + model.inputNeurons = e.inputColumns.Length; if (_modelService.CheckHyperparameters(model.inputNeurons, model.hiddenLayerNeurons, model.hiddenLayers, model.outputNeurons) == false) return BadRequest("Bad parameters!"); diff --git a/backend/api/api/Models/ColumnInfo.cs b/backend/api/api/Models/ColumnInfo.cs new file mode 100644 index 00000000..ee4cee0d --- /dev/null +++ b/backend/api/api/Models/ColumnInfo.cs @@ -0,0 +1,13 @@ +namespace api.Models +{ + public class ColumnInfo + { + public string columnName { get; set; } + public bool isNumber { get; set; } + public int numNulls { get; set; } + public float mean { get; set; } + public float median { get; set; } + public string[] uniqueValues { get; set; } + + } +} diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index dcfde3b1..17974d1d 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -22,6 +22,10 @@ namespace api.Models public DateTime lastUpdated { get; set; } public string delimiter { get; set; } public bool hasHeader { get; set; } + + public ColumnInfo[] columnInfo { get; set; } + public int totalNulls; + } } diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs index 3b435d3f..ec50950d 100644 --- a/backend/api/api/Models/Experiment.cs +++ b/backend/api/api/Models/Experiment.cs @@ -12,6 +12,9 @@ namespace api.Models public string datasetId { get; set; } public string[] inputColumns { get; set; } public string outputColumn { get; set; } + public bool randomOrder { get; set; } + public bool randomTestSet { get; set; } + public float randomTestSetDistribution { get; set; } public string nullValues { get; set; } public NullValues[] nullValuesReplacers { get; set; } diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs index 86541767..e88e7e74 100644 --- a/backend/api/api/Models/Model.cs +++ b/backend/api/api/Models/Model.cs @@ -18,14 +18,8 @@ namespace api.Models public DateTime dateCreated { get; set; } public DateTime lastUpdated { get; set; } //proveriti id - public string datasetId { get; set; } + public string experimentId { get; set; } - //Test set settings - public string[] inputColumns { get; set; } - public string columnToPredict { get; set; } - public bool randomOrder {get;set;} - public bool randomTestSet { get; set; } - public float randomTestSetDistribution { get; set; } //Neural net training public string type { get; set; } @@ -44,8 +38,6 @@ namespace api.Models public string[] metrics { get; set; } public int epochs { get; set; } - public string nullValues { get; set; } - public NullValues[] nullValuesReplacers { get; set; } //public NullValues[] nullValues { get; set; } } diff --git a/backend/api/api/Models/Predictor.cs b/backend/api/api/Models/Predictor.cs index 9c8d362f..5fd2aa09 100644 --- a/backend/api/api/Models/Predictor.cs +++ b/backend/api/api/Models/Predictor.cs @@ -17,6 +17,7 @@ namespace api.Models public bool isPublic { get; set; } public bool accessibleByLink { get; set; } public DateTime dateCreated { get; set; } + public string experimentId { get; set; } } } diff --git a/backend/api/api/Services/ExperimentService.cs b/backend/api/api/Services/ExperimentService.cs index a3b28da4..712568d4 100644 --- a/backend/api/api/Services/ExperimentService.cs +++ b/backend/api/api/Services/ExperimentService.cs @@ -18,5 +18,9 @@ namespace api.Services _experiment.InsertOne(experiment); return experiment; } + public Experiment Get(string id) + { + return _experiment.Find(exp=>exp._id == id).FirstOrDefault(); + } } } diff --git a/backend/api/api/Services/IExperimentService.cs b/backend/api/api/Services/IExperimentService.cs index 7c80599a..f16f3eae 100644 --- a/backend/api/api/Services/IExperimentService.cs +++ b/backend/api/api/Services/IExperimentService.cs @@ -5,5 +5,6 @@ namespace api.Services public interface IExperimentService { Experiment Create(Experiment experiment); + public Experiment Get(string id); } } \ No newline at end of file diff --git a/backend/api/api/Services/TempRemovalService.cs b/backend/api/api/Services/TempRemovalService.cs index 9741ace2..7f723986 100644 --- a/backend/api/api/Services/TempRemovalService.cs +++ b/backend/api/api/Services/TempRemovalService.cs @@ -9,6 +9,7 @@ namespace api.Services private readonly IMongoCollection _file; private readonly IMongoCollection _model; private readonly IMongoCollection _dataset; + private readonly IMongoCollection _experiment; public TempRemovalService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) { @@ -16,6 +17,7 @@ namespace api.Services _file = database.GetCollection(settings.FilesCollectionName); _model= database.GetCollection(settings.ModelCollectionName); _dataset = database.GetCollection(settings.DatasetCollectionName); + _experiment= database.GetCollection(settings.ExperimentCollectionName); } public void DeleteTemps() { -- cgit v1.2.3 From 15f225dd766d93e190581763d8bffa4daf98d6ef Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 02:11:01 +0200 Subject: Prilagodjeno brisanje temp fajlova i kolekcija u bazu.Ispravljena greska --- backend/api/api/Controllers/ModelController.cs | 3 ++- backend/api/api/Models/Experiment.cs | 1 + backend/api/api/Services/TempRemovalService.cs | 17 +++++++++++++---- 3 files changed, 16 insertions(+), 5 deletions(-) (limited to 'backend') diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index ffc99cb3..97e7c7a7 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -47,7 +47,8 @@ namespace api.Controllers } else return BadRequest(); - var dataset = _datasetService.GetOneDataset(model.datasetId); + var experiment=_experimentService.Get(model.experimentId); + var dataset = _datasetService.GetOneDataset(experiment.datasetId); var filepath = _fileService.GetFilePath(dataset.fileId, uploaderId); var result = await _mlService.SendModelAsync(model, filepath); return Ok(result); diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs index ec50950d..6de3f169 100644 --- a/backend/api/api/Models/Experiment.cs +++ b/backend/api/api/Models/Experiment.cs @@ -10,6 +10,7 @@ namespace api.Models public string _id { get; set; } public string datasetId { get; set; } + public string uploaderId { get; set; } public string[] inputColumns { get; set; } public string outputColumn { get; set; } public bool randomOrder { get; set; } diff --git a/backend/api/api/Services/TempRemovalService.cs b/backend/api/api/Services/TempRemovalService.cs index 7f723986..d72dc89e 100644 --- a/backend/api/api/Services/TempRemovalService.cs +++ b/backend/api/api/Services/TempRemovalService.cs @@ -31,11 +31,16 @@ namespace api.Services foreach(var dataset in datasets) { DeleteDataset(dataset._id); - List models = _model.Find(model => model.datasetId == dataset._id && model.username=="").ToList(); - foreach(var model in models) + List experiments = _experiment.Find(experiment=>experiment.datasetId== dataset._id && experiment.uploaderId=="").ToList(); + foreach(var experiment in experiments) { - DeleteModel(model._id); - } + DeleteExperiment(experiment._id); + List models = _model.Find(model => model.experimentId == experiment._id && model.username == "").ToList(); + foreach (var model in models) + { + DeleteModel(model._id); + } + } } if (File.Exists(file.path)) File.Delete(file.path); @@ -69,6 +74,10 @@ namespace api.Services { _dataset.DeleteOne(dataset => dataset._id == id); } + public void DeleteExperiment(string id) + { + _experiment.DeleteOne(experiment => experiment._id == id); + } } -- cgit v1.2.3 From 1a2a2e928769158b75dbf6e3366ab8ed312cd131 Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 21:37:11 +0200 Subject: Dodat experiment controller. --- .../api/api/Controllers/ExperimentController.cs | 75 ++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 backend/api/api/Controllers/ExperimentController.cs (limited to 'backend') diff --git a/backend/api/api/Controllers/ExperimentController.cs b/backend/api/api/Controllers/ExperimentController.cs new file mode 100644 index 00000000..2a2db31e --- /dev/null +++ b/backend/api/api/Controllers/ExperimentController.cs @@ -0,0 +1,75 @@ +using System.Net.Http.Headers; +using api.Models; +using api.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Net.Http.Headers; + +namespace api.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class ExperimentController : ControllerBase + { + + private readonly IExperimentService _experimentService; + private IJwtToken jwtToken; + + public ExperimentController(IExperimentService experimentService, IConfiguration configuration, IJwtToken Token) + { + _experimentService = experimentService; + jwtToken = Token; + } + + [HttpPost("add")] + [Authorize(Roles = "User,Guest")] + public async Task> Post([FromBody] Experiment experiment) + { + string uploaderId; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + uploaderId = jwtToken.TokenToId(parameter); + if (uploaderId == null) + return null; + } + else + return BadRequest(); + + experiment.uploaderId = uploaderId; + + _experimentService.Create(experiment); + return Ok(experiment); + } + + [HttpGet("get")] + [Authorize(Roles = "User,Guest")] + public async Task> Get(string id) + { + string uploaderId; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + uploaderId = jwtToken.TokenToId(parameter); + if (uploaderId == null) + return null; + } + else + return BadRequest(); + + var experiment = _experimentService.Get(id); + if(experiment.uploaderId!=uploaderId) + return BadRequest("Not your experiment"); + + return Ok(experiment); + } + + + + } +} -- cgit v1.2.3 From 7ba7c0451ef4e5a65cf5a9f7391c520b14f9654f Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 22:21:46 +0200 Subject: Kada se dodaje dataset prosledjuje se zahtev ml api-ju da izracuna columnInfo. Potrebno uskladiti sa ml api-jem. --- backend/api/api/Controllers/DatasetController.cs | 16 ++++++++++++---- backend/api/api/Services/FileService.cs | 4 ++++ backend/api/api/Services/IFileService.cs | 1 + backend/api/api/Services/IMlConnectionService.cs | 4 ++++ backend/api/api/Services/MlConnectionService.cs | 16 +++++++++++++++- 5 files changed, 36 insertions(+), 5 deletions(-) (limited to 'backend') diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index 8a622138..8a3485dd 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -14,11 +14,15 @@ namespace api.Controllers public class DatasetController : ControllerBase { private readonly IDatasetService _datasetService; + private readonly IMlConnectionService _mlConnectionService; + private readonly IFileService _fileService; private IJwtToken jwtToken; - public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token) + public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token,IMlConnectionService mlConnectionService, IFileService fileService) { _datasetService = datasetService; + _mlConnectionService = mlConnectionService; + _fileService = fileService; jwtToken = Token; } @@ -159,7 +163,7 @@ namespace api.Controllers // POST api//add [HttpPost("add")] [Authorize(Roles = "User,Guest")] - public ActionResult Post([FromBody] Dataset dataset) + public async Task> Post([FromBody] Dataset dataset) { //da li ce preko tokena da se ubaci username ili front salje //dataset.username = usernameToken; @@ -170,9 +174,13 @@ namespace api.Controllers return NotFound($"Dateset with name = {dataset.name} exisits"); else { - _datasetService.Create(dataset); - return CreatedAtAction(nameof(Get), new { id = dataset._id }, dataset); + FileModel fileModel = _fileService.getFile(dataset.fileId); + Dataset newDataset =await _mlConnectionService.PreProcess(dataset,fileModel.path); + + _datasetService.Create(newDataset); + + return CreatedAtAction(nameof(Get), new { id = newDataset._id }, newDataset); } } diff --git a/backend/api/api/Services/FileService.cs b/backend/api/api/Services/FileService.cs index 24491124..13f596d0 100644 --- a/backend/api/api/Services/FileService.cs +++ b/backend/api/api/Services/FileService.cs @@ -36,6 +36,10 @@ namespace api.Services return null; return file.path; } + public FileModel getFile(string id) + { + return _file.Find(x=>x._id==id).FirstOrDefault(); + } } } diff --git a/backend/api/api/Services/IFileService.cs b/backend/api/api/Services/IFileService.cs index 7446e283..d65deb48 100644 --- a/backend/api/api/Services/IFileService.cs +++ b/backend/api/api/Services/IFileService.cs @@ -6,5 +6,6 @@ namespace api.Services { FileModel Create(FileModel file); string GetFilePath(string id, string username); + public FileModel getFile(string id); } } \ No newline at end of file diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index ee839d28..490f2df7 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -1,8 +1,12 @@  +using api.Models; + namespace api.Services { public interface IMlConnectionService { Task SendModelAsync(object model, object dataset); + Task PreProcess(Dataset dataset, string filePath); + //Task PreProcess(Dataset dataset, byte[] file, string filename); } } \ No newline at end of file diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index 9c3b3fd8..a04a323b 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -1,6 +1,8 @@ -using RestSharp; +using api.Models; +using RestSharp; using System.Net.WebSockets; using System.Text; +using Newtonsoft.Json; namespace api.Services { @@ -20,5 +22,17 @@ namespace api.Services var result = await this.client.ExecuteAsync(request); return result.Content; //Response od ML microservisa } + public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) + { + var request=new RestRequest("preprocess", Method.Post);//USKLADITI SA ML API + request.AddParameter("dataset", JsonConvert.SerializeObject(dataset)); + //request.AddFile("file", file,filename); + request.AddFile("file", filePath); + request.AddHeader("Content-Type", "multipart/form-data"); + var result=await this.client.ExecuteAsync(request); + Dataset newDataset = JsonConvert.DeserializeObject(result.Content); + return newDataset; + + } } } -- cgit v1.2.3 From 5fd7e2e9777ab02fad5b7e1a977fe282a98dc7fc Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Mon, 4 Apr 2022 21:31:01 +0200 Subject: Promenja logika za preprocesiranje. --- backend/api/api/Controllers/DatasetController.cs | 12 +++++------- backend/api/api/Models/Dataset.cs | 1 + backend/api/api/Services/DatasetService.cs | 20 ++++++++++++-------- backend/api/api/Services/IDatasetService.cs | 1 + backend/api/api/Services/IMlConnectionService.cs | 2 +- backend/api/api/Services/MlConnectionService.cs | 11 ++++++++--- 6 files changed, 28 insertions(+), 19 deletions(-) (limited to 'backend') diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index 8a3485dd..6eb1b9e6 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -139,9 +139,8 @@ namespace api.Controllers return BadRequest(); var dataset = _datasetService.GetOneDataset(username, name); - if (dataset == null) - return NotFound($"Dataset with name = {name} not found or dataset is not public"); + return NotFound($"Dataset with name = {name} not found or dataset is not public or not preprocessed"); return dataset; } @@ -176,11 +175,10 @@ namespace api.Controllers { FileModel fileModel = _fileService.getFile(dataset.fileId); - Dataset newDataset =await _mlConnectionService.PreProcess(dataset,fileModel.path); - - _datasetService.Create(newDataset); - - return CreatedAtAction(nameof(Get), new { id = newDataset._id }, newDataset); + dataset.isPreProcess = false; + _datasetService.Create(dataset); + _mlConnectionService.PreProcess(dataset,fileModel.path); + return Ok(); } } diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index 17974d1d..2b3efa3c 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -25,6 +25,7 @@ namespace api.Models public ColumnInfo[] columnInfo { get; set; } public int totalNulls; + public bool isPreProcess { get; set; } } } diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs index 2ff271f3..c43bd236 100644 --- a/backend/api/api/Services/DatasetService.cs +++ b/backend/api/api/Services/DatasetService.cs @@ -16,7 +16,7 @@ namespace api.Services public List SearchDatasets(string name, string username) { - return _dataset.Find(dataset => dataset.name == name && dataset.isPublic == true).ToList(); + return _dataset.Find(dataset => dataset.name == name && dataset.isPublic == true && dataset.isPreProcess).ToList(); } //kreiranje dataseta @@ -34,12 +34,12 @@ namespace api.Services public List GetMyDatasets(string username) { - return _dataset.Find(dataset => dataset.username == username).ToList(); + return _dataset.Find(dataset => dataset.username == username && dataset.isPreProcess).ToList(); } public List GetGuestDatasets() { //Join Igranonica public datasetove sa svim temp uploadanim datasetovima - List datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true).ToList(); + List datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true && dataset.isPreProcess).ToList(); datasets.AddRange(_dataset.Find(dataset => dataset.username == "").ToList()); return datasets; } @@ -47,7 +47,7 @@ namespace api.Services //poslednji datasetovi public List SortDatasets(string username, bool ascdsc, int latest) { - List list = _dataset.Find(dataset => dataset.username == username).ToList(); + List list = _dataset.Find(dataset => dataset.username == username && dataset.isPreProcess).ToList(); if(ascdsc) list = list.OrderBy(dataset => dataset.lastUpdated).ToList(); @@ -59,25 +59,29 @@ namespace api.Services public List GetPublicDatasets() { - return _dataset.Find(dataset => dataset.isPublic == true).ToList(); + return _dataset.Find(dataset => dataset.isPublic == true && dataset.isPreProcess).ToList(); } public Dataset GetOneDataset(string username, string name) { - return _dataset.Find(dataset => dataset.username == username && dataset.name == name).FirstOrDefault(); + return _dataset.Find(dataset => dataset.username == username && dataset.name == name && dataset.isPreProcess).FirstOrDefault(); } //odraditi za pretragu getOne public Dataset GetOneDataset(string id) { - return _dataset.Find(dataset => dataset._id == id).FirstOrDefault(); + return _dataset.Find(dataset => dataset._id == id && dataset.isPreProcess).FirstOrDefault(); } //ako je potrebno da se zameni name ili ekstenzija - public void Update(string username, string name, Dataset dataset) + public void Update(string username, string name, Dataset dataset ) { _dataset.ReplaceOne(dataset => dataset.username == username && dataset.name == name, dataset); } + public void Update(Dataset dataset) + { + _dataset.ReplaceOne(x=>x._id==dataset._id, dataset); + } } diff --git a/backend/api/api/Services/IDatasetService.cs b/backend/api/api/Services/IDatasetService.cs index 8e62ba43..89856c5b 100644 --- a/backend/api/api/Services/IDatasetService.cs +++ b/backend/api/api/Services/IDatasetService.cs @@ -15,5 +15,6 @@ namespace api.Services void Update(string username, string name, Dataset dataset); void Delete(string username, string name); public List GetGuestDatasets(); + public void Update(Dataset dataset); } } diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index 490f2df7..ea73fb0f 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -6,7 +6,7 @@ namespace api.Services public interface IMlConnectionService { Task SendModelAsync(object model, object dataset); - Task PreProcess(Dataset dataset, string filePath); + Task PreProcess(Dataset dataset, string filePath); //Task PreProcess(Dataset dataset, byte[] file, string filename); } } \ No newline at end of file diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index a04a323b..66f7882a 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -9,10 +9,12 @@ namespace api.Services public class MlConnectionService : IMlConnectionService { private RestClient client; + private readonly IDatasetService _datasetService; - public MlConnectionService() + public MlConnectionService(IDatasetService datasetService) { this.client = new RestClient("http://127.0.0.1:5543"); + _datasetService=datasetService; } public async Task SendModelAsync(object model, object dataset) @@ -22,7 +24,7 @@ namespace api.Services var result = await this.client.ExecuteAsync(request); return result.Content; //Response od ML microservisa } - public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) + public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) { var request=new RestRequest("preprocess", Method.Post);//USKLADITI SA ML API request.AddParameter("dataset", JsonConvert.SerializeObject(dataset)); @@ -31,7 +33,10 @@ namespace api.Services request.AddHeader("Content-Type", "multipart/form-data"); var result=await this.client.ExecuteAsync(request); Dataset newDataset = JsonConvert.DeserializeObject(result.Content); - return newDataset; + newDataset.isPreProcess = true; + _datasetService.Update(newDataset); + + return; } } -- cgit v1.2.3 From 45201c5c388b4988af3b319d6702c60397929dfc Mon Sep 17 00:00:00 2001 From: Nevena Bojovic Date: Wed, 6 Apr 2022 21:18:27 +0200 Subject: Mala izmena. --- backend/api/api/appsettings.json | 5 +++-- frontend/angular.json | 5 +---- 2 files changed, 4 insertions(+), 6 deletions(-) (limited to 'backend') diff --git a/backend/api/api/appsettings.json b/backend/api/api/appsettings.json index fdccfb07..f8abae88 100644 --- a/backend/api/api/appsettings.json +++ b/backend/api/api/appsettings.json @@ -10,8 +10,8 @@ }, "AllowedHosts": "*", "UserStoreDatabaseSettings": { - /* LocalHost - */ + //LocalHost + "ConnectionString": "mongodb://127.0.0.1:27017/", "DatabaseName": "si_project", "CollectionName": "users", @@ -19,6 +19,7 @@ "ModelCollectionName": "Model", "PredictorCollectionName": "Predictor", "FilesCollectionName": "Files" + /* "ConnectionString": "mongodb+srv://si_user:si_user@sidatabase.twtfm.mongodb.net/myFirstDatabase?retryWrites=true&w=majority", "DatabaseName": "si_db", diff --git a/frontend/angular.json b/frontend/angular.json index f9825281..6653e4b1 100644 --- a/frontend/angular.json +++ b/frontend/angular.json @@ -33,10 +33,7 @@ "./node_modules/@angular/material/prebuilt-themes/indigo-pink.css" ], "scripts": [ - "node_modules/bootstrap/dist/js/bootstrap.bundle.min.js", - "node_modules/jquery/dist/jquery.min.js", - "node_modules/popper.js/dist/popper.min.js", - "node_modules/bootstrap/dist/js/bootstrap.min.js" + "node_modules/bootstrap/dist/js/bootstrap.bundle.min.js" ] }, "configurations": { -- cgit v1.2.3 From 052c97eadc3784306607fab2fd725d3cfa592867 Mon Sep 17 00:00:00 2001 From: Danijel Andjelkovic Date: Wed, 6 Apr 2022 21:31:12 +0200 Subject: json merge fix --- backend/api/api/appsettings.json | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'backend') diff --git a/backend/api/api/appsettings.json b/backend/api/api/appsettings.json index 81255d5c..f8923a10 100644 --- a/backend/api/api/appsettings.json +++ b/backend/api/api/appsettings.json @@ -9,20 +9,6 @@ } }, "AllowedHosts": "*", -<<<<<<< HEAD - "UserStoreDatabaseSettings": { - //LocalHost - - "ConnectionString": "mongodb://127.0.0.1:27017/", - "DatabaseName": "si_project", - "CollectionName": "users", - "DatasetCollectionName": "Dataset", - "ModelCollectionName": "Model", - "PredictorCollectionName": "Predictor", - "FilesCollectionName": "Files" - - /* -======= "UserStoreDatabaseSettings": { /* LocalHost */ @@ -35,7 +21,6 @@ "FilesCollectionName": "Files", "ExperimentCollectionName": "Experiment" /* ->>>>>>> 499f641683767d2da735fb760c89f0c611e37596 "ConnectionString": "mongodb+srv://si_user:si_user@sidatabase.twtfm.mongodb.net/myFirstDatabase?retryWrites=true&w=majority", "DatabaseName": "si_db", "CollectionName": "users", -- cgit v1.2.3 From 7d5af698fdf053cfe6ef674c085022cf8276f56c Mon Sep 17 00:00:00 2001 From: TAMARA JERINIC Date: Wed, 6 Apr 2022 22:49:49 +0200 Subject: Dodate min i max vrednosti kolona u statistiku. Dodata mogućnost zamene null vrednosti. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/microservice/api/newmlservice.py | 77 +++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) (limited to 'backend') diff --git a/backend/microservice/api/newmlservice.py b/backend/microservice/api/newmlservice.py index 50af15f8..2ea31702 100644 --- a/backend/microservice/api/newmlservice.py +++ b/backend/microservice/api/newmlservice.py @@ -33,6 +33,8 @@ def returnColumnsInfo(dataset): uniquevalues=datafront[kolona].unique() mean=0 median=0 + minimum=0 + maximum=0 nullCount=datafront[kolona].isnull().sum() if(nullCount>0): allNullCols=allNullCols+1 @@ -41,10 +43,14 @@ def returnColumnsInfo(dataset): 'uniqueValues':uniquevalues.tolist(), 'median':float(mean), 'mean':float(median), - 'numNulls':float(nullCount) + 'numNulls':float(nullCount), + 'min':float(minimum), + 'max':float(maximum) } dict.append(frontreturn) else: + minimum=min(datafront[kolona]) + maximum=max(datafront[kolona]) mean=datafront[kolona].mean() median=s.median(datafront[kolona]) nullCount=datafront[kolona].isnull().sum() @@ -55,7 +61,9 @@ def returnColumnsInfo(dataset): 'uniqueValues':[], 'mean':float(mean), 'median':float(median), - 'numNulls':float(nullCount) + 'numNulls':float(nullCount), + 'min':float(minimum), + 'max':float(maximum) } dict.append(frontreturn) NullRows = datafront[datafront.isnull().any(axis=1)] @@ -95,27 +103,35 @@ class TrainingResult: def train(dataset, params, callback): problem_type = params["type"] - print(problem_type) + #print(problem_type) data = pd.DataFrame() - print(data) + #print(data) for col in params["inputColumns"]: - print(col) + #print(col) data[col]=dataset[col] output_column = params["columnToPredict"] data[output_column] = dataset[output_column] - print(data) + #print(data) ###NULL null_value_options = params["nullValues"] null_values_replacers = params["nullValuesReplacers"] if(null_value_options=='replace'): - print("replace null") # TODO + #print("replace null") # TODO + dict=params['null_values_replacers'] + while(len(dict)>0): + replace=dict.pop() + col=replace['column'] + opt=replace['option'] + if(opt=='replace'): + replacevalue=replace['value'] + data[col]=data[col].fillna(replacevalue) elif(null_value_options=='delete_rows'): data=data.dropna() elif(null_value_options=='delete_columns'): - data=data.dropna() - print(data.shape) + data=data.dropna(axis=1) + #print(data.shape) # # Brisanje kolona koje ne uticu na rezultat @@ -176,7 +192,7 @@ def train(dataset, params, callback): for col in data.columns: if(col!=output_column): x_columns.append(col) - print(x_columns) + #print(x_columns) x = data[x_columns].values y = data[output_column].values @@ -190,7 +206,7 @@ def train(dataset, params, callback): else: random=0 x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=test, random_state=random) - print(x_train,x_test) + #print(x_train,x_test) # # Treniranje modela @@ -215,7 +231,7 @@ def train(dataset, params, callback): #print(y_pred.flatten()) #print(y_test) scores = classifier.evaluate(x_test, y_test) - print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100)) + #print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100)) classifier.save("temp/"+params['name'], save_format='h5') #vizuelizacija u python-u #from ann_visualizer.visualize import ann_viz; @@ -238,11 +254,11 @@ def train(dataset, params, callback): y_pred=classifier.predict(x_test) y_pred=(y_pred>=0.5).astype('int') - print(y_pred.flatten()) - print(y_test) + #print(y_pred.flatten()) + #print(y_test) scores = classifier.evaluate(x_test, y_test) - print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100)) + #print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100)) #ann_viz(classifier, title="My neural network") classifier.save("temp/"+params['name'], save_format='h5') @@ -260,7 +276,7 @@ def train(dataset, params, callback): history=classifier.fit(x_train, y_train, epochs = params['epochs'],batch_size=params['batchSize']) y_pred=classifier.predict(x_test) - print(classifier.evaluate(x_test, y_test)) + #print(classifier.evaluate(x_test, y_test)) def roc_auc_score_multiclass(actual_class, pred_class, average = "macro"): @@ -346,11 +362,11 @@ def train(dataset, params, callback): def manageH5(dataset,params,h5model): problem_type = params["type"] - print(problem_type) + #print(problem_type) data = pd.DataFrame() #print(data) for col in params["inputColumns"]: - print(col) + #print(col) data[col]=dataset[col] output_column = params["columnToPredict"] data[output_column] = dataset[output_column] @@ -361,12 +377,21 @@ def manageH5(dataset,params,h5model): null_values_replacers = params["nullValuesReplacers"] if(null_value_options=='replace'): - print("replace null") # TODO + #print("replace null") # TODO + dict=params['null_values_replacers'] + while(len(dict)>0): + replace=dict.pop() + col=replace['column'] + opt=replace['option'] + if(opt=='replace'): + replacevalue=replace['value'] + data[col]=data[col].fillna(replacevalue) elif(null_value_options=='delete_rows'): data=data.dropna() elif(null_value_options=='delete_columns'): data=data.dropna() - print(data.shape) + + #print(data.shape) # # Brisanje kolona koje ne uticu na rezultat @@ -402,10 +427,10 @@ def manageH5(dataset,params,h5model): x_columns.append(col) #print(x_columns) x2 = data[x_columns] - print(x2) - print(x2.values) + #print(x2) + #print(x2.values) x2 = data[x_columns].values - print(x2) + #print(x2) y2 = data[output_column].values h5model.summary() ann_viz(h5model, title="My neural network") @@ -419,6 +444,6 @@ def manageH5(dataset,params,h5model): y_pred2=np.argmax(y_pred2,axis=1) #y_pred=h5model.predict_classes(x) score = h5model.evaluate(x2,y_pred2, verbose=0) - print("%s: %.2f%%" % (h5model.metrics_names[1], score[1]*100)) - print(y_pred2) - print( 'done') \ No newline at end of file + #print("%s: %.2f%%" % (h5model.metrics_names[1], score[1]*100)) + #print(y_pred2) + #print( 'done') \ No newline at end of file -- cgit v1.2.3