From f8b9f36958f78a1c7605fad02ba9036ce17c51a0 Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sat, 2 Apr 2022 19:54:58 +0200 Subject: Napravljena Experiment klasa. Dodat Experiment servis. Napravljen dep injection za experiment servis. Napravljena konekcija sa bazom. --- backend/api/api/Data/UserStoreDatabaseSettings.cs | 1 + .../api/Interfaces/IUserStoreDatabaseSettings.cs | 1 + backend/api/api/Models/Experiment.cs | 19 ++++++++++++++++ backend/api/api/Program.cs | 1 + backend/api/api/Services/ExperimentService.cs | 22 +++++++++++++++++++ backend/api/api/Services/IExperimentService.cs | 9 ++++++++ backend/api/api/appsettings.json | 25 +++++++++++----------- 7 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 backend/api/api/Models/Experiment.cs create mode 100644 backend/api/api/Services/ExperimentService.cs create mode 100644 backend/api/api/Services/IExperimentService.cs diff --git a/backend/api/api/Data/UserStoreDatabaseSettings.cs b/backend/api/api/Data/UserStoreDatabaseSettings.cs index e83d2b54..822f9bf5 100644 --- a/backend/api/api/Data/UserStoreDatabaseSettings.cs +++ b/backend/api/api/Data/UserStoreDatabaseSettings.cs @@ -13,5 +13,6 @@ namespace api.Data public string PredictorCollectionName { get; set; } = String.Empty; public string ModelCollectionName { get; set; } = String.Empty; public string FilesCollectionName { get; set; } = String.Empty; + public string ExperimentCollectionName { get; set; } = String.Empty; } } diff --git a/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs b/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs index a5b5f5eb..05c41701 100644 --- a/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs +++ b/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs @@ -9,5 +9,6 @@ string PredictorCollectionName { get; set; } string ModelCollectionName { get; set; } string FilesCollectionName { get; set; } + string ExperimentCollectionName { get; set; } } } diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs new file mode 100644 index 00000000..3b435d3f --- /dev/null +++ b/backend/api/api/Models/Experiment.cs @@ -0,0 +1,19 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace api.Models +{ + public class Experiment + { + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string _id { get; set; } + + public string datasetId { get; set; } + public string[] inputColumns { get; set; } + public string outputColumn { get; set; } + public string nullValues { get; set; } + public NullValues[] nullValuesReplacers { get; set; } + + } +} diff --git a/backend/api/api/Program.cs b/backend/api/api/Program.cs index 2bb97e45..7d5d0c45 100644 --- a/backend/api/api/Program.cs +++ b/backend/api/api/Program.cs @@ -34,6 +34,7 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); var mlwss = new MLWebSocketService(); diff --git a/backend/api/api/Services/ExperimentService.cs b/backend/api/api/Services/ExperimentService.cs new file mode 100644 index 00000000..a3b28da4 --- /dev/null +++ b/backend/api/api/Services/ExperimentService.cs @@ -0,0 +1,22 @@ +using api.Interfaces; +using api.Models; +using MongoDB.Driver; + +namespace api.Services +{ + public class ExperimentService : IExperimentService + { + private readonly IMongoCollection _experiment; + public ExperimentService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) + { + var database = mongoClient.GetDatabase(settings.DatabaseName); + _experiment = database.GetCollection(settings.ExperimentCollectionName); + } + + public Experiment Create(Experiment experiment) + { + _experiment.InsertOne(experiment); + return experiment; + } + } +} diff --git a/backend/api/api/Services/IExperimentService.cs b/backend/api/api/Services/IExperimentService.cs new file mode 100644 index 00000000..7c80599a --- /dev/null +++ b/backend/api/api/Services/IExperimentService.cs @@ -0,0 +1,9 @@ +using api.Models; + +namespace api.Services +{ + public interface IExperimentService + { + Experiment Create(Experiment experiment); + } +} \ No newline at end of file diff --git a/backend/api/api/appsettings.json b/backend/api/api/appsettings.json index fdccfb07..f8923a10 100644 --- a/backend/api/api/appsettings.json +++ b/backend/api/api/appsettings.json @@ -9,17 +9,18 @@ } }, "AllowedHosts": "*", - "UserStoreDatabaseSettings": { - /* LocalHost - */ - "ConnectionString": "mongodb://127.0.0.1:27017/", - "DatabaseName": "si_project", - "CollectionName": "users", - "DatasetCollectionName": "Dataset", - "ModelCollectionName": "Model", - "PredictorCollectionName": "Predictor", - "FilesCollectionName": "Files" - /* + "UserStoreDatabaseSettings": { + /* LocalHost + */ + "ConnectionString": "mongodb://127.0.0.1:27017/", + "DatabaseName": "si_project", + "CollectionName": "users", + "DatasetCollectionName": "Dataset", + "ModelCollectionName": "Model", + "PredictorCollectionName": "Predictor", + "FilesCollectionName": "Files", + "ExperimentCollectionName": "Experiment" + /* "ConnectionString": "mongodb+srv://si_user:si_user@sidatabase.twtfm.mongodb.net/myFirstDatabase?retryWrites=true&w=majority", "DatabaseName": "si_db", "CollectionName": "users", @@ -27,5 +28,5 @@ "ModelCollectionName": "Model", "PredictorCollectionName": "Predictor", "FilesCollectionName": "Files"*/ - } + } } \ No newline at end of file -- cgit v1.2.3 From 03852781bb840c2c76e400c4408ddf57896bc95b Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sat, 2 Apr 2022 20:44:25 +0200 Subject: Izmenjeni Dataset,Experiment,Model.Predictor,prosiren experiment servis,ispravljena greska kod model Controllera,dodata klasa ColumnInfo. Potrebno ispraviti greske!!! --- backend/api/api/Controllers/ModelController.cs | 7 +++++-- backend/api/api/Models/ColumnInfo.cs | 13 +++++++++++++ backend/api/api/Models/Dataset.cs | 4 ++++ backend/api/api/Models/Experiment.cs | 3 +++ backend/api/api/Models/Model.cs | 10 +--------- backend/api/api/Models/Predictor.cs | 1 + backend/api/api/Services/ExperimentService.cs | 4 ++++ backend/api/api/Services/IExperimentService.cs | 1 + backend/api/api/Services/TempRemovalService.cs | 2 ++ 9 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 backend/api/api/Models/ColumnInfo.cs diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index 210d37c8..ffc99cb3 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -17,15 +17,17 @@ namespace api.Controllers private readonly IDatasetService _datasetService; private readonly IFileService _fileService; private readonly IModelService _modelService; + private readonly IExperimentService _experimentService; private IJwtToken jwtToken; - public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration,IJwtToken token) + public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration,IJwtToken token,IExperimentService experiment) { _mlService = mlService; _modelService = modelService; _datasetService = datasetService; _fileService = fileService; + _experimentService = experiment; jwtToken = token; } @@ -145,7 +147,8 @@ namespace api.Controllers { bool overwrite = false; //username="" ako je GUEST - model.inputNeurons = model.inputColumns.Length; + Experiment e = _experimentService.Get(model.experimentId); + model.inputNeurons = e.inputColumns.Length; if (_modelService.CheckHyperparameters(model.inputNeurons, model.hiddenLayerNeurons, model.hiddenLayers, model.outputNeurons) == false) return BadRequest("Bad parameters!"); diff --git a/backend/api/api/Models/ColumnInfo.cs b/backend/api/api/Models/ColumnInfo.cs new file mode 100644 index 00000000..ee4cee0d --- /dev/null +++ b/backend/api/api/Models/ColumnInfo.cs @@ -0,0 +1,13 @@ +namespace api.Models +{ + public class ColumnInfo + { + public string columnName { get; set; } + public bool isNumber { get; set; } + public int numNulls { get; set; } + public float mean { get; set; } + public float median { get; set; } + public string[] uniqueValues { get; set; } + + } +} diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index dcfde3b1..17974d1d 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -22,6 +22,10 @@ namespace api.Models public DateTime lastUpdated { get; set; } public string delimiter { get; set; } public bool hasHeader { get; set; } + + public ColumnInfo[] columnInfo { get; set; } + public int totalNulls; + } } diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs index 3b435d3f..ec50950d 100644 --- a/backend/api/api/Models/Experiment.cs +++ b/backend/api/api/Models/Experiment.cs @@ -12,6 +12,9 @@ namespace api.Models public string datasetId { get; set; } public string[] inputColumns { get; set; } public string outputColumn { get; set; } + public bool randomOrder { get; set; } + public bool randomTestSet { get; set; } + public float randomTestSetDistribution { get; set; } public string nullValues { get; set; } public NullValues[] nullValuesReplacers { get; set; } diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs index 86541767..e88e7e74 100644 --- a/backend/api/api/Models/Model.cs +++ b/backend/api/api/Models/Model.cs @@ -18,14 +18,8 @@ namespace api.Models public DateTime dateCreated { get; set; } public DateTime lastUpdated { get; set; } //proveriti id - public string datasetId { get; set; } + public string experimentId { get; set; } - //Test set settings - public string[] inputColumns { get; set; } - public string columnToPredict { get; set; } - public bool randomOrder {get;set;} - public bool randomTestSet { get; set; } - public float randomTestSetDistribution { get; set; } //Neural net training public string type { get; set; } @@ -44,8 +38,6 @@ namespace api.Models public string[] metrics { get; set; } public int epochs { get; set; } - public string nullValues { get; set; } - public NullValues[] nullValuesReplacers { get; set; } //public NullValues[] nullValues { get; set; } } diff --git a/backend/api/api/Models/Predictor.cs b/backend/api/api/Models/Predictor.cs index 9c8d362f..5fd2aa09 100644 --- a/backend/api/api/Models/Predictor.cs +++ b/backend/api/api/Models/Predictor.cs @@ -17,6 +17,7 @@ namespace api.Models public bool isPublic { get; set; } public bool accessibleByLink { get; set; } public DateTime dateCreated { get; set; } + public string experimentId { get; set; } } } diff --git a/backend/api/api/Services/ExperimentService.cs b/backend/api/api/Services/ExperimentService.cs index a3b28da4..712568d4 100644 --- a/backend/api/api/Services/ExperimentService.cs +++ b/backend/api/api/Services/ExperimentService.cs @@ -18,5 +18,9 @@ namespace api.Services _experiment.InsertOne(experiment); return experiment; } + public Experiment Get(string id) + { + return _experiment.Find(exp=>exp._id == id).FirstOrDefault(); + } } } diff --git a/backend/api/api/Services/IExperimentService.cs b/backend/api/api/Services/IExperimentService.cs index 7c80599a..f16f3eae 100644 --- a/backend/api/api/Services/IExperimentService.cs +++ b/backend/api/api/Services/IExperimentService.cs @@ -5,5 +5,6 @@ namespace api.Services public interface IExperimentService { Experiment Create(Experiment experiment); + public Experiment Get(string id); } } \ No newline at end of file diff --git a/backend/api/api/Services/TempRemovalService.cs b/backend/api/api/Services/TempRemovalService.cs index 9741ace2..7f723986 100644 --- a/backend/api/api/Services/TempRemovalService.cs +++ b/backend/api/api/Services/TempRemovalService.cs @@ -9,6 +9,7 @@ namespace api.Services private readonly IMongoCollection _file; private readonly IMongoCollection _model; private readonly IMongoCollection _dataset; + private readonly IMongoCollection _experiment; public TempRemovalService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) { @@ -16,6 +17,7 @@ namespace api.Services _file = database.GetCollection(settings.FilesCollectionName); _model= database.GetCollection(settings.ModelCollectionName); _dataset = database.GetCollection(settings.DatasetCollectionName); + _experiment= database.GetCollection(settings.ExperimentCollectionName); } public void DeleteTemps() { -- cgit v1.2.3 From 15f225dd766d93e190581763d8bffa4daf98d6ef Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 02:11:01 +0200 Subject: Prilagodjeno brisanje temp fajlova i kolekcija u bazu.Ispravljena greska --- backend/api/api/Controllers/ModelController.cs | 3 ++- backend/api/api/Models/Experiment.cs | 1 + backend/api/api/Services/TempRemovalService.cs | 17 +++++++++++++---- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index ffc99cb3..97e7c7a7 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -47,7 +47,8 @@ namespace api.Controllers } else return BadRequest(); - var dataset = _datasetService.GetOneDataset(model.datasetId); + var experiment=_experimentService.Get(model.experimentId); + var dataset = _datasetService.GetOneDataset(experiment.datasetId); var filepath = _fileService.GetFilePath(dataset.fileId, uploaderId); var result = await _mlService.SendModelAsync(model, filepath); return Ok(result); diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs index ec50950d..6de3f169 100644 --- a/backend/api/api/Models/Experiment.cs +++ b/backend/api/api/Models/Experiment.cs @@ -10,6 +10,7 @@ namespace api.Models public string _id { get; set; } public string datasetId { get; set; } + public string uploaderId { get; set; } public string[] inputColumns { get; set; } public string outputColumn { get; set; } public bool randomOrder { get; set; } diff --git a/backend/api/api/Services/TempRemovalService.cs b/backend/api/api/Services/TempRemovalService.cs index 7f723986..d72dc89e 100644 --- a/backend/api/api/Services/TempRemovalService.cs +++ b/backend/api/api/Services/TempRemovalService.cs @@ -31,11 +31,16 @@ namespace api.Services foreach(var dataset in datasets) { DeleteDataset(dataset._id); - List models = _model.Find(model => model.datasetId == dataset._id && model.username=="").ToList(); - foreach(var model in models) + List experiments = _experiment.Find(experiment=>experiment.datasetId== dataset._id && experiment.uploaderId=="").ToList(); + foreach(var experiment in experiments) { - DeleteModel(model._id); - } + DeleteExperiment(experiment._id); + List models = _model.Find(model => model.experimentId == experiment._id && model.username == "").ToList(); + foreach (var model in models) + { + DeleteModel(model._id); + } + } } if (File.Exists(file.path)) File.Delete(file.path); @@ -69,6 +74,10 @@ namespace api.Services { _dataset.DeleteOne(dataset => dataset._id == id); } + public void DeleteExperiment(string id) + { + _experiment.DeleteOne(experiment => experiment._id == id); + } } -- cgit v1.2.3 From 1a2a2e928769158b75dbf6e3366ab8ed312cd131 Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 21:37:11 +0200 Subject: Dodat experiment controller. --- .../api/api/Controllers/ExperimentController.cs | 75 ++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 backend/api/api/Controllers/ExperimentController.cs diff --git a/backend/api/api/Controllers/ExperimentController.cs b/backend/api/api/Controllers/ExperimentController.cs new file mode 100644 index 00000000..2a2db31e --- /dev/null +++ b/backend/api/api/Controllers/ExperimentController.cs @@ -0,0 +1,75 @@ +using System.Net.Http.Headers; +using api.Models; +using api.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Net.Http.Headers; + +namespace api.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class ExperimentController : ControllerBase + { + + private readonly IExperimentService _experimentService; + private IJwtToken jwtToken; + + public ExperimentController(IExperimentService experimentService, IConfiguration configuration, IJwtToken Token) + { + _experimentService = experimentService; + jwtToken = Token; + } + + [HttpPost("add")] + [Authorize(Roles = "User,Guest")] + public async Task> Post([FromBody] Experiment experiment) + { + string uploaderId; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + uploaderId = jwtToken.TokenToId(parameter); + if (uploaderId == null) + return null; + } + else + return BadRequest(); + + experiment.uploaderId = uploaderId; + + _experimentService.Create(experiment); + return Ok(experiment); + } + + [HttpGet("get")] + [Authorize(Roles = "User,Guest")] + public async Task> Get(string id) + { + string uploaderId; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + uploaderId = jwtToken.TokenToId(parameter); + if (uploaderId == null) + return null; + } + else + return BadRequest(); + + var experiment = _experimentService.Get(id); + if(experiment.uploaderId!=uploaderId) + return BadRequest("Not your experiment"); + + return Ok(experiment); + } + + + + } +} -- cgit v1.2.3 From 7ba7c0451ef4e5a65cf5a9f7391c520b14f9654f Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Sun, 3 Apr 2022 22:21:46 +0200 Subject: Kada se dodaje dataset prosledjuje se zahtev ml api-ju da izracuna columnInfo. Potrebno uskladiti sa ml api-jem. --- backend/api/api/Controllers/DatasetController.cs | 16 ++++++++++++---- backend/api/api/Services/FileService.cs | 4 ++++ backend/api/api/Services/IFileService.cs | 1 + backend/api/api/Services/IMlConnectionService.cs | 4 ++++ backend/api/api/Services/MlConnectionService.cs | 16 +++++++++++++++- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index 8a622138..8a3485dd 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -14,11 +14,15 @@ namespace api.Controllers public class DatasetController : ControllerBase { private readonly IDatasetService _datasetService; + private readonly IMlConnectionService _mlConnectionService; + private readonly IFileService _fileService; private IJwtToken jwtToken; - public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token) + public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token,IMlConnectionService mlConnectionService, IFileService fileService) { _datasetService = datasetService; + _mlConnectionService = mlConnectionService; + _fileService = fileService; jwtToken = Token; } @@ -159,7 +163,7 @@ namespace api.Controllers // POST api//add [HttpPost("add")] [Authorize(Roles = "User,Guest")] - public ActionResult Post([FromBody] Dataset dataset) + public async Task> Post([FromBody] Dataset dataset) { //da li ce preko tokena da se ubaci username ili front salje //dataset.username = usernameToken; @@ -170,9 +174,13 @@ namespace api.Controllers return NotFound($"Dateset with name = {dataset.name} exisits"); else { - _datasetService.Create(dataset); - return CreatedAtAction(nameof(Get), new { id = dataset._id }, dataset); + FileModel fileModel = _fileService.getFile(dataset.fileId); + Dataset newDataset =await _mlConnectionService.PreProcess(dataset,fileModel.path); + + _datasetService.Create(newDataset); + + return CreatedAtAction(nameof(Get), new { id = newDataset._id }, newDataset); } } diff --git a/backend/api/api/Services/FileService.cs b/backend/api/api/Services/FileService.cs index 24491124..13f596d0 100644 --- a/backend/api/api/Services/FileService.cs +++ b/backend/api/api/Services/FileService.cs @@ -36,6 +36,10 @@ namespace api.Services return null; return file.path; } + public FileModel getFile(string id) + { + return _file.Find(x=>x._id==id).FirstOrDefault(); + } } } diff --git a/backend/api/api/Services/IFileService.cs b/backend/api/api/Services/IFileService.cs index 7446e283..d65deb48 100644 --- a/backend/api/api/Services/IFileService.cs +++ b/backend/api/api/Services/IFileService.cs @@ -6,5 +6,6 @@ namespace api.Services { FileModel Create(FileModel file); string GetFilePath(string id, string username); + public FileModel getFile(string id); } } \ No newline at end of file diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index ee839d28..490f2df7 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -1,8 +1,12 @@  +using api.Models; + namespace api.Services { public interface IMlConnectionService { Task SendModelAsync(object model, object dataset); + Task PreProcess(Dataset dataset, string filePath); + //Task PreProcess(Dataset dataset, byte[] file, string filename); } } \ No newline at end of file diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index 9c3b3fd8..a04a323b 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -1,6 +1,8 @@ -using RestSharp; +using api.Models; +using RestSharp; using System.Net.WebSockets; using System.Text; +using Newtonsoft.Json; namespace api.Services { @@ -20,5 +22,17 @@ namespace api.Services var result = await this.client.ExecuteAsync(request); return result.Content; //Response od ML microservisa } + public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) + { + var request=new RestRequest("preprocess", Method.Post);//USKLADITI SA ML API + request.AddParameter("dataset", JsonConvert.SerializeObject(dataset)); + //request.AddFile("file", file,filename); + request.AddFile("file", filePath); + request.AddHeader("Content-Type", "multipart/form-data"); + var result=await this.client.ExecuteAsync(request); + Dataset newDataset = JsonConvert.DeserializeObject(result.Content); + return newDataset; + + } } } -- cgit v1.2.3 From 5fd7e2e9777ab02fad5b7e1a977fe282a98dc7fc Mon Sep 17 00:00:00 2001 From: Ognjen Cirkovic Date: Mon, 4 Apr 2022 21:31:01 +0200 Subject: Promenja logika za preprocesiranje. --- backend/api/api/Controllers/DatasetController.cs | 12 +++++------- backend/api/api/Models/Dataset.cs | 1 + backend/api/api/Services/DatasetService.cs | 20 ++++++++++++-------- backend/api/api/Services/IDatasetService.cs | 1 + backend/api/api/Services/IMlConnectionService.cs | 2 +- backend/api/api/Services/MlConnectionService.cs | 11 ++++++++--- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index 8a3485dd..6eb1b9e6 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -139,9 +139,8 @@ namespace api.Controllers return BadRequest(); var dataset = _datasetService.GetOneDataset(username, name); - if (dataset == null) - return NotFound($"Dataset with name = {name} not found or dataset is not public"); + return NotFound($"Dataset with name = {name} not found or dataset is not public or not preprocessed"); return dataset; } @@ -176,11 +175,10 @@ namespace api.Controllers { FileModel fileModel = _fileService.getFile(dataset.fileId); - Dataset newDataset =await _mlConnectionService.PreProcess(dataset,fileModel.path); - - _datasetService.Create(newDataset); - - return CreatedAtAction(nameof(Get), new { id = newDataset._id }, newDataset); + dataset.isPreProcess = false; + _datasetService.Create(dataset); + _mlConnectionService.PreProcess(dataset,fileModel.path); + return Ok(); } } diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index 17974d1d..2b3efa3c 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -25,6 +25,7 @@ namespace api.Models public ColumnInfo[] columnInfo { get; set; } public int totalNulls; + public bool isPreProcess { get; set; } } } diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs index 2ff271f3..c43bd236 100644 --- a/backend/api/api/Services/DatasetService.cs +++ b/backend/api/api/Services/DatasetService.cs @@ -16,7 +16,7 @@ namespace api.Services public List SearchDatasets(string name, string username) { - return _dataset.Find(dataset => dataset.name == name && dataset.isPublic == true).ToList(); + return _dataset.Find(dataset => dataset.name == name && dataset.isPublic == true && dataset.isPreProcess).ToList(); } //kreiranje dataseta @@ -34,12 +34,12 @@ namespace api.Services public List GetMyDatasets(string username) { - return _dataset.Find(dataset => dataset.username == username).ToList(); + return _dataset.Find(dataset => dataset.username == username && dataset.isPreProcess).ToList(); } public List GetGuestDatasets() { //Join Igranonica public datasetove sa svim temp uploadanim datasetovima - List datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true).ToList(); + List datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true && dataset.isPreProcess).ToList(); datasets.AddRange(_dataset.Find(dataset => dataset.username == "").ToList()); return datasets; } @@ -47,7 +47,7 @@ namespace api.Services //poslednji datasetovi public List SortDatasets(string username, bool ascdsc, int latest) { - List list = _dataset.Find(dataset => dataset.username == username).ToList(); + List list = _dataset.Find(dataset => dataset.username == username && dataset.isPreProcess).ToList(); if(ascdsc) list = list.OrderBy(dataset => dataset.lastUpdated).ToList(); @@ -59,25 +59,29 @@ namespace api.Services public List GetPublicDatasets() { - return _dataset.Find(dataset => dataset.isPublic == true).ToList(); + return _dataset.Find(dataset => dataset.isPublic == true && dataset.isPreProcess).ToList(); } public Dataset GetOneDataset(string username, string name) { - return _dataset.Find(dataset => dataset.username == username && dataset.name == name).FirstOrDefault(); + return _dataset.Find(dataset => dataset.username == username && dataset.name == name && dataset.isPreProcess).FirstOrDefault(); } //odraditi za pretragu getOne public Dataset GetOneDataset(string id) { - return _dataset.Find(dataset => dataset._id == id).FirstOrDefault(); + return _dataset.Find(dataset => dataset._id == id && dataset.isPreProcess).FirstOrDefault(); } //ako je potrebno da se zameni name ili ekstenzija - public void Update(string username, string name, Dataset dataset) + public void Update(string username, string name, Dataset dataset ) { _dataset.ReplaceOne(dataset => dataset.username == username && dataset.name == name, dataset); } + public void Update(Dataset dataset) + { + _dataset.ReplaceOne(x=>x._id==dataset._id, dataset); + } } diff --git a/backend/api/api/Services/IDatasetService.cs b/backend/api/api/Services/IDatasetService.cs index 8e62ba43..89856c5b 100644 --- a/backend/api/api/Services/IDatasetService.cs +++ b/backend/api/api/Services/IDatasetService.cs @@ -15,5 +15,6 @@ namespace api.Services void Update(string username, string name, Dataset dataset); void Delete(string username, string name); public List GetGuestDatasets(); + public void Update(Dataset dataset); } } diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index 490f2df7..ea73fb0f 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -6,7 +6,7 @@ namespace api.Services public interface IMlConnectionService { Task SendModelAsync(object model, object dataset); - Task PreProcess(Dataset dataset, string filePath); + Task PreProcess(Dataset dataset, string filePath); //Task PreProcess(Dataset dataset, byte[] file, string filename); } } \ No newline at end of file diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index a04a323b..66f7882a 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -9,10 +9,12 @@ namespace api.Services public class MlConnectionService : IMlConnectionService { private RestClient client; + private readonly IDatasetService _datasetService; - public MlConnectionService() + public MlConnectionService(IDatasetService datasetService) { this.client = new RestClient("http://127.0.0.1:5543"); + _datasetService=datasetService; } public async Task SendModelAsync(object model, object dataset) @@ -22,7 +24,7 @@ namespace api.Services var result = await this.client.ExecuteAsync(request); return result.Content; //Response od ML microservisa } - public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) + public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) { var request=new RestRequest("preprocess", Method.Post);//USKLADITI SA ML API request.AddParameter("dataset", JsonConvert.SerializeObject(dataset)); @@ -31,7 +33,10 @@ namespace api.Services request.AddHeader("Content-Type", "multipart/form-data"); var result=await this.client.ExecuteAsync(request); Dataset newDataset = JsonConvert.DeserializeObject(result.Content); - return newDataset; + newDataset.isPreProcess = true; + _datasetService.Update(newDataset); + + return; } } -- cgit v1.2.3 From 19fd492a03ea48571d6aaab9e75108a82599ef0a Mon Sep 17 00:00:00 2001 From: Danijel Andjelkovic Date: Wed, 6 Apr 2022 13:07:23 +0200 Subject: Dodao klasu za notifikacije i dizajn. Dodao spinner u datatable komponentu. --- frontend/src/app/_data/Notification.ts | 8 +++ .../_elements/datatable/datatable.component.html | 46 ++++++------ .../app/_elements/datatable/datatable.component.ts | 2 + .../app/_elements/loading/loading.component.css | 81 ++++++++++++++++++++++ .../app/_elements/loading/loading.component.html | 11 +++ .../_elements/loading/loading.component.spec.ts | 25 +++++++ .../src/app/_elements/loading/loading.component.ts | 15 ++++ .../notifications/notifications.component.html | 23 +++++- .../notifications/notifications.component.ts | 12 +++- 9 files changed, 198 insertions(+), 25 deletions(-) create mode 100644 frontend/src/app/_data/Notification.ts create mode 100644 frontend/src/app/_elements/loading/loading.component.css create mode 100644 frontend/src/app/_elements/loading/loading.component.html create mode 100644 frontend/src/app/_elements/loading/loading.component.spec.ts create mode 100644 frontend/src/app/_elements/loading/loading.component.ts diff --git a/frontend/src/app/_data/Notification.ts b/frontend/src/app/_data/Notification.ts new file mode 100644 index 00000000..181bb332 --- /dev/null +++ b/frontend/src/app/_data/Notification.ts @@ -0,0 +1,8 @@ +export default class Notification { + _id: string = ''; + constructor( + public title: string = 'Treniranje u toku...', + public id: string = '042', + public progress: number = 0.5 + ) { } +} \ No newline at end of file diff --git a/frontend/src/app/_elements/datatable/datatable.component.html b/frontend/src/app/_elements/datatable/datatable.component.html index bd9e7a13..8fcd44ac 100644 --- a/frontend/src/app/_elements/datatable/datatable.component.html +++ b/frontend/src/app/_elements/datatable/datatable.component.html @@ -1,25 +1,31 @@
-
- - - - - - - - - - - -
{{item}}
{{col}}
+
+
+ +
+
+ + + + + + + + + + + +
{{item}}
{{col}}
- - - - - - -
{{col}}
+ + + + + + +
{{col}}
+
diff --git a/frontend/src/app/_elements/datatable/datatable.component.ts b/frontend/src/app/_elements/datatable/datatable.component.ts index d3740d83..3343f6f0 100644 --- a/frontend/src/app/_elements/datatable/datatable.component.ts +++ b/frontend/src/app/_elements/datatable/datatable.component.ts @@ -11,6 +11,8 @@ export class DatatableComponent implements OnInit { @Input() data?: any[] = []; + loaded = false; + constructor() { } ngOnInit(): void { diff --git a/frontend/src/app/_elements/loading/loading.component.css b/frontend/src/app/_elements/loading/loading.component.css new file mode 100644 index 00000000..f39f60ee --- /dev/null +++ b/frontend/src/app/_elements/loading/loading.component.css @@ -0,0 +1,81 @@ +.lds-grid { + display: inline-block; + position: relative; + width: 80px; + height: 80px; +} + +.lds-grid div { + position: absolute; + width: 16px; + height: 16px; + border-radius: 50%; + background: #fff; + animation: lds-grid 1.2s linear infinite; +} + +.lds-grid div:nth-child(1) { + top: 8px; + left: 8px; + animation-delay: 0s; +} + +.lds-grid div:nth-child(2) { + top: 8px; + left: 32px; + animation-delay: -0.4s; +} + +.lds-grid div:nth-child(3) { + top: 8px; + left: 56px; + animation-delay: -0.8s; +} + +.lds-grid div:nth-child(4) { + top: 32px; + left: 8px; + animation-delay: -0.4s; +} + +.lds-grid div:nth-child(5) { + top: 32px; + left: 32px; + animation-delay: -0.8s; +} + +.lds-grid div:nth-child(6) { + top: 32px; + left: 56px; + animation-delay: -1.2s; +} + +.lds-grid div:nth-child(7) { + top: 56px; + left: 8px; + animation-delay: -0.8s; +} + +.lds-grid div:nth-child(8) { + top: 56px; + left: 32px; + animation-delay: -1.2s; +} + +.lds-grid div:nth-child(9) { + top: 56px; + left: 56px; + animation-delay: -1.6s; +} + +@keyframes lds-grid { + + 0%, + 100% { + opacity: 1; + } + + 50% { + opacity: 0.5; + } +} \ No newline at end of file diff --git a/frontend/src/app/_elements/loading/loading.component.html b/frontend/src/app/_elements/loading/loading.component.html new file mode 100644 index 00000000..d2c7b74e --- /dev/null +++ b/frontend/src/app/_elements/loading/loading.component.html @@ -0,0 +1,11 @@ +
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file diff --git a/frontend/src/app/_elements/loading/loading.component.spec.ts b/frontend/src/app/_elements/loading/loading.component.spec.ts new file mode 100644 index 00000000..7aacfad9 --- /dev/null +++ b/frontend/src/app/_elements/loading/loading.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { LoadingComponent } from './loading.component'; + +describe('LoadingComponent', () => { + let component: LoadingComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ LoadingComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(LoadingComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/frontend/src/app/_elements/loading/loading.component.ts b/frontend/src/app/_elements/loading/loading.component.ts new file mode 100644 index 00000000..4dff0cdf --- /dev/null +++ b/frontend/src/app/_elements/loading/loading.component.ts @@ -0,0 +1,15 @@ +import { Component, OnInit } from '@angular/core'; + +@Component({ + selector: 'app-loading', + templateUrl: './loading.component.html', + styleUrls: ['./loading.component.css'] +}) +export class LoadingComponent implements OnInit { + + constructor() { } + + ngOnInit(): void { + } + +} diff --git a/frontend/src/app/_elements/notifications/notifications.component.html b/frontend/src/app/_elements/notifications/notifications.component.html index d1da41b4..0b87e4fc 100644 --- a/frontend/src/app/_elements/notifications/notifications.component.html +++ b/frontend/src/app/_elements/notifications/notifications.component.html @@ -1,3 +1,20 @@ -
-

Notifikacije

-
\ No newline at end of file +
+

Notifikacije + +

+ +
+
+
+

{{notification.title}}

+
+
+
+
+
+
+
\ No newline at end of file diff --git a/frontend/src/app/_elements/notifications/notifications.component.ts b/frontend/src/app/_elements/notifications/notifications.component.ts index 6c1d555b..82613448 100644 --- a/frontend/src/app/_elements/notifications/notifications.component.ts +++ b/frontend/src/app/_elements/notifications/notifications.component.ts @@ -1,5 +1,6 @@ import { Component, OnInit } from '@angular/core'; import { WebSocketService } from 'src/app/_services/web-socket.service'; +import Notification from 'src/app/_data/Notification'; @Component({ selector: 'app-notifications', @@ -8,9 +9,16 @@ import { WebSocketService } from 'src/app/_services/web-socket.service'; }) export class NotificationsComponent implements OnInit { - notifications?: any[]; + notifications?: Notification[]; + closed: boolean = false; - constructor(private wsService: WebSocketService) { } + constructor(private wsService: WebSocketService) { + this.notifications = [ + new Notification("Titanik (Preziveli)", "79768456867", 0.2), + new Notification("Test Prediktor 1", "56758768678", 0.4), + new Notification("Test Prediktor 2", "11344556425", 0.7) + ] + } ngOnInit(): void { // this.wsService.send('test'); -- cgit v1.2.3 From af3333a77e254b3268de38ec397921b43f357949 Mon Sep 17 00:00:00 2001 From: Danijel Andjelkovic Date: Wed, 6 Apr 2022 13:08:32 +0200 Subject: Dodao stranicu za eksperimente, ova stranica je zamena za add-model stranicu. --- frontend/src/app/_data/Experiment.ts | 12 +++ frontend/src/app/_data/Model.ts | 41 ++++---- .../app/_elements/annvisual/annvisual.component.ts | 3 +- .../_elements/model-load/model-load.component.css | 0 .../_elements/model-load/model-load.component.html | 1 + .../model-load/model-load.component.spec.ts | 25 +++++ .../_elements/model-load/model-load.component.ts | 15 +++ .../src/app/_elements/navbar/navbar.component.html | 8 +- .../app/_pages/add-model/add-model.component.html | 29 +----- .../app/_pages/add-model/add-model.component.ts | 104 ++++++++++----------- frontend/src/app/app-routing.module.ts | 5 +- frontend/src/app/app.module.ts | 8 +- .../src/app/experiment/experiment.component.css | 39 ++++++++ .../src/app/experiment/experiment.component.html | 13 +++ .../app/experiment/experiment.component.spec.ts | 25 +++++ .../src/app/experiment/experiment.component.ts | 15 +++ 16 files changed, 233 insertions(+), 110 deletions(-) create mode 100644 frontend/src/app/_data/Experiment.ts create mode 100644 frontend/src/app/_elements/model-load/model-load.component.css create mode 100644 frontend/src/app/_elements/model-load/model-load.component.html create mode 100644 frontend/src/app/_elements/model-load/model-load.component.spec.ts create mode 100644 frontend/src/app/_elements/model-load/model-load.component.ts create mode 100644 frontend/src/app/experiment/experiment.component.css create mode 100644 frontend/src/app/experiment/experiment.component.html create mode 100644 frontend/src/app/experiment/experiment.component.spec.ts create mode 100644 frontend/src/app/experiment/experiment.component.ts diff --git a/frontend/src/app/_data/Experiment.ts b/frontend/src/app/_data/Experiment.ts new file mode 100644 index 00000000..706231c7 --- /dev/null +++ b/frontend/src/app/_data/Experiment.ts @@ -0,0 +1,12 @@ +export default class Experiment { + _id: string = ''; + constructor( + public name: string = 'Novi experiment', + public description: string = '', + public datasetId: string = '', + public inputColumns: string[] = [], + public columnToPredict: string = '', + public dateCreated: Date = new Date(), + public lastUpdated: Date = new Date() + ) { } +} \ No newline at end of file diff --git a/frontend/src/app/_data/Model.ts b/frontend/src/app/_data/Model.ts index 58bc3d4f..1ad4fc6d 100644 --- a/frontend/src/app/_data/Model.ts +++ b/frontend/src/app/_data/Model.ts @@ -7,11 +7,9 @@ export default class Model { public description: string = '', public dateCreated: Date = new Date(), public lastUpdated: Date = new Date(), - public datasetId: string = '', + public experimentId: string = '', // Test set settings - public inputColumns: string[] = [], - public columnToPredict: string = '', public randomOrder: boolean = true, public randomTestSet: boolean = true, public randomTestSetDistribution: number = 0.1, //0.1-0.9 (10% - 90%) JESTE OVDE ZAKUCANO 10, AL POSLATO JE KAO 0.1 BACK-U @@ -168,31 +166,28 @@ export enum Metrics { MSE = 'mse', MAE = 'mae', RMSE = 'rmse' - + } -export enum MetricsRegression -{ +export enum MetricsRegression { Mse = 'mse', Mae = 'mae', Mape = 'mape', - Msle='msle', - CosineProximity='cosine' + Msle = 'msle', + CosineProximity = 'cosine' } -export enum MetricsBinaryClassification -{ - Accuracy='binary_accuracy', - Auc="AUC", - Precision='precision_score', - Recall='recall_score', - F1='f1_score', - +export enum MetricsBinaryClassification { + Accuracy = 'binary_accuracy', + Auc = "AUC", + Precision = 'precision_score', + Recall = 'recall_score', + F1 = 'f1_score', + } -export enum MetricsMultiClassification -{ - Accuracy='categorical_accuracy', - Auc="AUC", - Precision='precision_score', - Recall='recall_score', - F1='f1_score', +export enum MetricsMultiClassification { + Accuracy = 'categorical_accuracy', + Auc = "AUC", + Precision = 'precision_score', + Recall = 'recall_score', + F1 = 'f1_score', } diff --git a/frontend/src/app/_elements/annvisual/annvisual.component.ts b/frontend/src/app/_elements/annvisual/annvisual.component.ts index 53e4e2ca..df0a3898 100644 --- a/frontend/src/app/_elements/annvisual/annvisual.component.ts +++ b/frontend/src/app/_elements/annvisual/annvisual.component.ts @@ -18,7 +18,7 @@ export class AnnvisualComponent implements OnInit { let hiddenlayerstring: string = ''; let digraphstring: string = 'digraph {'; - for (let i = 0; i < this.model.inputColumns.length; i++) { + for (let i = 0; i < /*this.model.inputColumns.length*/ 10; i++) { inputlayerstring = inputlayerstring + 'i' + i + ','; } inputlayerstring = inputlayerstring.slice(0, -1); @@ -35,7 +35,6 @@ export class AnnvisualComponent implements OnInit { } digraphstring = digraphstring + 'o}'; - graphviz('#graph').renderDot(digraphstring); } diff --git a/frontend/src/app/_elements/model-load/model-load.component.css b/frontend/src/app/_elements/model-load/model-load.component.css new file mode 100644 index 00000000..e69de29b diff --git a/frontend/src/app/_elements/model-load/model-load.component.html b/frontend/src/app/_elements/model-load/model-load.component.html new file mode 100644 index 00000000..ced9b0d6 --- /dev/null +++ b/frontend/src/app/_elements/model-load/model-load.component.html @@ -0,0 +1 @@ +

model-load works!

diff --git a/frontend/src/app/_elements/model-load/model-load.component.spec.ts b/frontend/src/app/_elements/model-load/model-load.component.spec.ts new file mode 100644 index 00000000..1dafd966 --- /dev/null +++ b/frontend/src/app/_elements/model-load/model-load.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ModelLoadComponent } from './model-load.component'; + +describe('ModelLoadComponent', () => { + let component: ModelLoadComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ ModelLoadComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(ModelLoadComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/frontend/src/app/_elements/model-load/model-load.component.ts b/frontend/src/app/_elements/model-load/model-load.component.ts new file mode 100644 index 00000000..00532bea --- /dev/null +++ b/frontend/src/app/_elements/model-load/model-load.component.ts @@ -0,0 +1,15 @@ +import { Component, OnInit } from '@angular/core'; + +@Component({ + selector: 'app-model-load', + templateUrl: './model-load.component.html', + styleUrls: ['./model-load.component.css'] +}) +export class ModelLoadComponent implements OnInit { + + constructor() { } + + ngOnInit(): void { + } + +} diff --git a/frontend/src/app/_elements/navbar/navbar.component.html b/frontend/src/app/_elements/navbar/navbar.component.html index 82a1ea07..dbee4773 100644 --- a/frontend/src/app/_elements/navbar/navbar.component.html +++ b/frontend/src/app/_elements/navbar/navbar.component.html @@ -8,8 +8,9 @@
  • Početna
    • -
  • Dodaj model
    • +
  • Eksperimentiši +
  • Predvidi
    • @@ -18,7 +19,8 @@
      +

      Nov model:

      @@ -63,12 +62,9 @@
    - -
    -

    Izaberite ulazne kolone:

    @@ -121,7 +117,6 @@
    -
    @@ -198,9 +193,7 @@
    -
    -
    {{column}}  -
    -
    @@ -287,8 +276,6 @@
    - -
    @@ -303,8 +290,6 @@
    - -
    @@ -315,7 +300,6 @@ [(ngModel)]="newModel.hiddenLayerNeurons">
    -
    @@ -364,7 +348,6 @@ type="checkbox" value="" checked>
    -
    @@ -397,8 +380,6 @@
    - -

    Aktivacione funkcije:

    @@ -443,10 +424,7 @@
    -

    - -

    Izaberite metrike:

    @@ -463,7 +441,6 @@
    -
    - - \ No newline at end of file +--> \ No newline at end of file diff --git a/frontend/src/app/_pages/add-model/add-model.component.ts b/frontend/src/app/_pages/add-model/add-model.component.ts index b35b9fa4..d47b24e6 100644 --- a/frontend/src/app/_pages/add-model/add-model.component.ts +++ b/frontend/src/app/_pages/add-model/add-model.component.ts @@ -110,55 +110,55 @@ export class AddModelComponent implements OnInit { } saveModelWithNewDataset(callback: ((arg0: any) => void)) { - - this.getCheckedInputCols(); - this.getCheckedOutputCol(); - this.getMetrics(); - - if (this.validationInputsOutput()) { - console.log('ADD MODEL: STEP 1 - UPLOAD FILE'); - if (this.datasetLoadComponent) { - console.log("this.datasetLoadComponent.files:", this.datasetLoadComponent.files); - this.models.uploadData(this.datasetLoadComponent.files[0]).subscribe((file) => { - console.log('ADD MODEL: STEP 2 - ADD DATASET WITH FILE ID ' + file._id); + /* + this.getCheckedInputCols(); + this.getCheckedOutputCol(); + this.getMetrics(); + + if (this.validationInputsOutput()) { + console.log('ADD MODEL: STEP 1 - UPLOAD FILE'); if (this.datasetLoadComponent) { - this.datasetLoadComponent.dataset.fileId = file._id; - this.datasetLoadComponent.dataset.username = shared.username; - - this.datasets.addDataset(this.datasetLoadComponent.dataset).subscribe((dataset) => { - console.log('ADD MODEL: STEP 3 - ADD MODEL WITH DATASET ID ', dataset._id); - this.newModel.datasetId = dataset._id; - - //da se doda taj dataset u listu postojecih, da bude izabran - this.refreshMyDatasetList(); - this.showMyDatasets = true; - this.selectThisDataset(dataset); - - this.newModel.randomTestSetDistribution = 1 - Math.round(this.tempTestSetDistribution / 100 * 10) / 10; - this.tempTestSetDistribution = 90; - this.newModel.username = shared.username; - - this.newModel.nullValuesReplacers = this.getNullValuesReplacersArray(); - - this.models.addModel(this.newModel).subscribe((response) => { - callback(response); - }, (error) => { - alert("Model sa unetim nazivom već postoji u Vašoj kolekciji.\nPromenite naziv modela i nastavite sa kreiranim datasetom."); - }); //kraj addModel subscribe + console.log("this.datasetLoadComponent.files:", this.datasetLoadComponent.files); + this.models.uploadData(this.datasetLoadComponent.files[0]).subscribe((file) => { + console.log('ADD MODEL: STEP 2 - ADD DATASET WITH FILE ID ' + file._id); + if (this.datasetLoadComponent) { + this.datasetLoadComponent.dataset.fileId = file._id; + this.datasetLoadComponent.dataset.username = shared.username; + + this.datasets.addDataset(this.datasetLoadComponent.dataset).subscribe((dataset) => { + console.log('ADD MODEL: STEP 3 - ADD MODEL WITH DATASET ID ', dataset._id); + this.newModel.datasetId = dataset._id; + + //da se doda taj dataset u listu postojecih, da bude izabran + this.refreshMyDatasetList(); + this.showMyDatasets = true; + this.selectThisDataset(dataset); + + this.newModel.randomTestSetDistribution = 1 - Math.round(this.tempTestSetDistribution / 100 * 10) / 10; + this.tempTestSetDistribution = 90; + this.newModel.username = shared.username; + + this.newModel.nullValuesReplacers = this.getNullValuesReplacersArray(); + + this.models.addModel(this.newModel).subscribe((response) => { + callback(response); + }, (error) => { + alert("Model sa unetim nazivom već postoji u Vašoj kolekciji.\nPromenite naziv modela i nastavite sa kreiranim datasetom."); + }); //kraj addModel subscribe + }, (error) => { + alert("Dataset sa unetim nazivom već postoji u Vašoj kolekciji.\nIzmenite naziv ili iskoristite postojeći dataset."); + }); //kraj addDataset subscribe + } //kraj treceg ifa }, (error) => { - alert("Dataset sa unetim nazivom već postoji u Vašoj kolekciji.\nIzmenite naziv ili iskoristite postojeći dataset."); - }); //kraj addDataset subscribe - } //kraj treceg ifa - }, (error) => { - //alert("greska uploadData"); - }); //kraj uploadData subscribe - - } //kraj drugog ifa - } //kraj prvog ifa + //alert("greska uploadData"); + }); //kraj uploadData subscribe + + } //kraj drugog ifa + } //kraj prvog ifa*/ } saveModelWithExistingDataset(callback: ((arg0: any) => void)): any { - if (this.selectedDataset) { //dataset je izabran + /*if (this.selectedDataset) { //dataset je izabran this.getCheckedInputCols(); this.getCheckedOutputCol(); this.getMetrics(); @@ -180,11 +180,11 @@ export class AddModelComponent implements OnInit { } else { alert("Molimo Vas da izaberete neki dataset iz kolekcije."); - } + }*/ } getCheckedInputCols() { - this.newModel.inputColumns = []; + /*this.newModel.inputColumns = []; let checkboxes: any; checkboxes = document.getElementsByName("cbsNew"); @@ -193,11 +193,11 @@ export class AddModelComponent implements OnInit { let thatCb = checkboxes[i]; if (thatCb.checked == true) // && thatCb.disabled == false ne treba nam ovo vise this.newModel.inputColumns.push(thatCb.value); - } + }*/ //console.log(this.checkedInputCols); } getCheckedOutputCol() { - this.newModel.columnToPredict = ''; + /*this.newModel.columnToPredict = ''; let radiobuttons: any; radiobuttons = document.getElementsByName("rbsNew"); @@ -208,11 +208,11 @@ export class AddModelComponent implements OnInit { this.newModel.columnToPredict = thatRb.value; break; } - } + }*/ //console.log(this.checkedOutputCol); } - validationInputsOutput(): boolean { - if (this.newModel.inputColumns.length == 0 && this.newModel.columnToPredict == '') { + validationInputsOutput() { + /*if (this.newModel.inputColumns.length == 0 && this.newModel.columnToPredict == '') { alert("Molimo Vas da izaberete ulazne i izlazne kolone za mrežu."); return false; } @@ -231,7 +231,7 @@ export class AddModelComponent implements OnInit { return false; } } - return true; + return true;*/ } selectThisDataset(dataset: Dataset) { diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index fd827ac9..93431397 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -2,7 +2,6 @@ import { NgModule } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; import { AuthGuardService } from './_services/auth-guard.service'; -import { AddModelComponent } from './_pages/add-model/add-model.component'; import { HomeComponent } from './_pages/home/home.component'; import { MyDatasetsComponent } from './_pages/my-datasets/my-datasets.component'; import { MyModelsComponent } from './_pages/my-models/my-models.component'; @@ -13,10 +12,12 @@ import { SettingsComponent } from './_pages/settings/settings.component'; import { ProfileComponent } from './_pages/profile/profile.component'; import { PredictComponent } from './_pages/predict/predict.component'; import { FilterDatasetsComponent } from './_pages/filter-datasets/filter-datasets.component'; +import { ExperimentComponent } from './experiment/experiment.component'; const routes: Routes = [ { path: '', component: HomeComponent, data: { title: 'Početna strana' } }, - { path: 'add-model', component: AddModelComponent, data: { title: 'Dodaj model' } }, + /*{ path: 'add-model', component: AddModelComponent, data: { title: 'Dodaj model' } },*/ + { path: 'experiment', component: ExperimentComponent, data: { title: 'Dodaj model' } }, { path: 'my-datasets', component: MyDatasetsComponent, canActivate: [AuthGuardService], data: { title: 'Moji izvori podataka' } }, { path: 'my-models', component: MyModelsComponent, canActivate: [AuthGuardService], data: { title: 'Moji modeli' } }, { path: 'my-predictors', component: MyPredictorsComponent, canActivate: [AuthGuardService], data: { title: 'Moji trenirani modeli' } }, diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index d90de54d..d701f9d7 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -39,6 +39,9 @@ import { FilterDatasetsComponent } from './_pages/filter-datasets/filter-dataset import { ReactiveBackgroundComponent } from './_elements/reactive-background/reactive-background.component'; import { ItemModelComponent } from './_elements/item-model/item-model.component'; import { AnnvisualComponent } from './_elements/annvisual/annvisual.component'; +import { ExperimentComponent } from './experiment/experiment.component'; +import { LoadingComponent } from './_elements/loading/loading.component'; +import { ModelLoadComponent } from './_elements/model-load/model-load.component'; @NgModule({ declarations: [ @@ -67,7 +70,10 @@ import { AnnvisualComponent } from './_elements/annvisual/annvisual.component'; FilterDatasetsComponent, ReactiveBackgroundComponent, ItemModelComponent, - AnnvisualComponent + AnnvisualComponent, + ExperimentComponent, + LoadingComponent, + ModelLoadComponent ], imports: [ BrowserModule, diff --git a/frontend/src/app/experiment/experiment.component.css b/frontend/src/app/experiment/experiment.component.css new file mode 100644 index 00000000..ee4b0448 --- /dev/null +++ b/frontend/src/app/experiment/experiment.component.css @@ -0,0 +1,39 @@ +#header { + background-color: #003459; + padding-top: 30px; + padding-bottom: 20px; +} + +#header h1 { + font-family: 'Lucida Sans', 'Lucida Sans Regular', 'Lucida Grande', 'Lucida Sans Unicode', Geneva, Verdana, sans-serif; + text-align: center; + color: white; +} + +#container { + border-radius: 8px; +} + +#wrapper { + color: #003459; +} + +.btnType1 { + background-color: #003459; + color: white; +} + +.btnType2 { + background-color: white; + color: #003459; + border-color: #003459; +} + +.selectedDatasetClass { + /*border-color: 2px solid #003459;*/ + background-color: lightblue; +} + +ul li:hover { + background-color: lightblue; +} \ No newline at end of file diff --git a/frontend/src/app/experiment/experiment.component.html b/frontend/src/app/experiment/experiment.component.html new file mode 100644 index 00000000..42f2993d --- /dev/null +++ b/frontend/src/app/experiment/experiment.component.html @@ -0,0 +1,13 @@ + +
    +
    +

    1. Izvor podataka

    + +

    2. Preprocesiranje

    +

    3. Podešavanja mreže

    + +

    4. Rezultati treniranja

    +
    +
    \ No newline at end of file diff --git a/frontend/src/app/experiment/experiment.component.spec.ts b/frontend/src/app/experiment/experiment.component.spec.ts new file mode 100644 index 00000000..fd2bbd30 --- /dev/null +++ b/frontend/src/app/experiment/experiment.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ExperimentComponent } from './experiment.component'; + +describe('ExperimentComponent', () => { + let component: ExperimentComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ ExperimentComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(ExperimentComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/frontend/src/app/experiment/experiment.component.ts b/frontend/src/app/experiment/experiment.component.ts new file mode 100644 index 00000000..16de7351 --- /dev/null +++ b/frontend/src/app/experiment/experiment.component.ts @@ -0,0 +1,15 @@ +import { Component, OnInit } from '@angular/core'; + +@Component({ + selector: 'app-experiment', + templateUrl: './experiment.component.html', + styleUrls: ['./experiment.component.css'] +}) +export class ExperimentComponent implements OnInit { + + constructor() { } + + ngOnInit(): void { + } + +} -- cgit v1.2.3 From 093c0d2eeefa6e1a55524727d301753412486284 Mon Sep 17 00:00:00 2001 From: Danijel Andjelkovic Date: Wed, 6 Apr 2022 21:06:06 +0200 Subject: Add model stranica zamenjena sa eksperiment stranicom. --- frontend/angular.json | 5 +- frontend/src/app/_data/Dataset.ts | 18 +- frontend/src/app/_data/Experiment.ts | 22 + frontend/src/app/_data/Model.ts | 5 - .../_elements/model-load/model-load.component.html | 222 ++++++++- .../_elements/model-load/model-load.component.ts | 66 ++- .../app/_pages/add-model/add-model.component.css | 42 -- .../app/_pages/add-model/add-model.component.html | 428 ----------------- .../_pages/add-model/add-model.component.spec.ts | 25 - .../app/_pages/add-model/add-model.component.ts | 508 --------------------- frontend/src/app/app.module.ts | 2 - .../src/app/experiment/experiment.component.html | 192 +++++++- .../src/app/experiment/experiment.component.ts | 105 ++++- 13 files changed, 620 insertions(+), 1020 deletions(-) delete mode 100644 frontend/src/app/_pages/add-model/add-model.component.css delete mode 100644 frontend/src/app/_pages/add-model/add-model.component.html delete mode 100644 frontend/src/app/_pages/add-model/add-model.component.spec.ts delete mode 100644 frontend/src/app/_pages/add-model/add-model.component.ts diff --git a/frontend/angular.json b/frontend/angular.json index f9825281..6653e4b1 100644 --- a/frontend/angular.json +++ b/frontend/angular.json @@ -33,10 +33,7 @@ "./node_modules/@angular/material/prebuilt-themes/indigo-pink.css" ], "scripts": [ - "node_modules/bootstrap/dist/js/bootstrap.bundle.min.js", - "node_modules/jquery/dist/jquery.min.js", - "node_modules/popper.js/dist/popper.min.js", - "node_modules/bootstrap/dist/js/bootstrap.min.js" + "node_modules/bootstrap/dist/js/bootstrap.bundle.min.js" ] }, "configurations": { diff --git a/frontend/src/app/_data/Dataset.ts b/frontend/src/app/_data/Dataset.ts index c5b56957..dd751947 100644 --- a/frontend/src/app/_data/Dataset.ts +++ b/frontend/src/app/_data/Dataset.ts @@ -12,6 +12,22 @@ export default class Dataset { public lastUpdated: Date = new Date(), public username: string = '', public delimiter: string = '', - public hasHeader: boolean = true + public hasHeader: boolean = true, + + public columnInfo: ColumnInfo[] = [], + public preview: string[][] = [[]] + ) { } +} + +export class ColumnInfo { + constructor( + public name: string = '', + public isNumber: boolean = false, + public numNull: number = 0, + public uniqueValues?: string[], + public median?: number, + public mean?: number, + public min?: number, + public max?: number ) { } } \ No newline at end of file diff --git a/frontend/src/app/_data/Experiment.ts b/frontend/src/app/_data/Experiment.ts index 706231c7..10320ab6 100644 --- a/frontend/src/app/_data/Experiment.ts +++ b/frontend/src/app/_data/Experiment.ts @@ -6,7 +6,29 @@ export default class Experiment { public datasetId: string = '', public inputColumns: string[] = [], public columnToPredict: string = '', + public nullValues: NullValueOptions = NullValueOptions.DeleteRows, + public nullValuesReplacers: NullValReplacer[] = [], public dateCreated: Date = new Date(), public lastUpdated: Date = new Date() ) { } +} + +export enum NullValueOptions { + DeleteRows = 'delete_rows', + DeleteColumns = 'delete_columns', + Replace = 'replace' +} + +export enum ReplaceWith { + None = 'Popuni...', + Mean = 'Srednja vrednost', + Median = 'Medijana', + Min = 'Minimum', + Max = 'Maksimum' +} + +export class NullValReplacer { + "column": string; + "option": NullValueOptions; + "value": string; } \ No newline at end of file diff --git a/frontend/src/app/_data/Model.ts b/frontend/src/app/_data/Model.ts index 85b6db2b..9ea437b1 100644 --- a/frontend/src/app/_data/Model.ts +++ b/frontend/src/app/_data/Model.ts @@ -24,11 +24,8 @@ export default class Model { public hiddenLayers: number = 1, public batchSize: number = 5, public hiddenLayerActivationFunctions: string[] = ['sigmoid'], - //public inputLayerActivationFunction: ActivationFunction = ActivationFunction.Sigmoid, public outputLayerActivationFunction: ActivationFunction = ActivationFunction.Sigmoid, public username: string = '', - public nullValues: NullValueOptions = NullValueOptions.DeleteRows, - public nullValuesReplacers: NullValReplacer[] = [], public metrics: string[] = [], // TODO add to add-model form public epochs: number = 5 // TODO add to add-model form ) { } @@ -83,7 +80,6 @@ export enum ActivationFunction { Linear = 'linear', //Sigmoid='sigmoid', Softmax = 'softmax', - } /* export enum ActivationFunctionHiddenLayer @@ -115,7 +111,6 @@ export enum LossFunction { MeanSquaredError = 'mean_squared_error', MeanSquaredLogarithmicError = 'mean_squared_logarithmic_error', HuberLoss = 'Huber' - } export enum LossFunctionRegression { MeanAbsoluteError = 'mean_absolute_error', diff --git a/frontend/src/app/_elements/model-load/model-load.component.html b/frontend/src/app/_elements/model-load/model-load.component.html index ced9b0d6..0c6735a9 100644 --- a/frontend/src/app/_elements/model-load/model-load.component.html +++ b/frontend/src/app/_elements/model-load/model-load.component.html @@ -1 +1,221 @@ -

    model-load works!

    +
    +
    +

    Nov model:

    +
    + + +
    +
    + +
    + +
    +
    +
    +    + +
    +
    +

    Parametri treniranja:

    +
    +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + +
    + +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    +
    +
    + +
    +
    + +
    +
    +
    + + +
    +
    +
    +
    +
    + +
    +
    + trening + + + test +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +

    Aktivacione funkcije:

    + +
    +
    + +
    +
    +
    +
    +
    + #{{i+1}} +
    + +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    +
    +

    +
    +
    +

    Izaberite metrike:

    +
    + +
    + + + +
    +
    +
    +
    +
    + +
    +
    +
    \ No newline at end of file diff --git a/frontend/src/app/_elements/model-load/model-load.component.ts b/frontend/src/app/_elements/model-load/model-load.component.ts index 00532bea..1d38de68 100644 --- a/frontend/src/app/_elements/model-load/model-load.component.ts +++ b/frontend/src/app/_elements/model-load/model-load.component.ts @@ -1,4 +1,7 @@ import { Component, OnInit } from '@angular/core'; +import Shared from 'src/app/Shared'; +import Model, { ActivationFunction, Encoding, LossFunction, LossFunctionBinaryClassification, LossFunctionMultiClassification, LossFunctionRegression, Metrics, MetricsBinaryClassification, MetricsMultiClassification, MetricsRegression, NullValueOptions, Optimizer, ProblemType } from 'src/app/_data/Model'; +import { ModelsService } from 'src/app/_services/models.service'; @Component({ selector: 'app-model-load', @@ -7,9 +10,70 @@ import { Component, OnInit } from '@angular/core'; }) export class ModelLoadComponent implements OnInit { - constructor() { } + newModel: Model = new Model(); + + ProblemType = ProblemType; + Encoding = Encoding; + ActivationFunction = ActivationFunction; + metrics: any = Metrics; + LossFunction = LossFunction; + Optimizer = Optimizer; + Object = Object; + document = document; + shared = Shared; + + term: string = ""; + selectedProblemType: string = ''; + selectedMetrics = []; + tempTestSetDistribution = 90; + lossFunction: any = LossFunction; + + constructor(private models: ModelsService) { } ngOnInit(): void { } + getMetrics() { + this.newModel.metrics = []; + let cb = document.getElementsByName("cbmetrics"); + + for (let i = 0; i < cb.length; i++) { + let chb = cb[i]; + if (chb.checked == true) + this.newModel.metrics.push(chb.value); + } + } + + addModel() { + this.getMetrics(); + + this.newModel.randomTestSetDistribution = 1 - Math.round(this.tempTestSetDistribution / 100 * 10) / 10; + this.newModel.username = Shared.username; + + this.models.addModel(this.newModel).subscribe((response) => { + Shared.openDialog('Model dodat', 'Model je uspešno dodat u bazu.'); + // treba da se selektuje nov model u listi modela + }, (error) => { + Shared.openDialog('Greška', 'Model sa unetim nazivom već postoji u Vašoj kolekciji.\nPromenite naziv modela i nastavite sa kreiranim datasetom.'); + }); + } + + filterOptions() { + switch (this.newModel.type) { + case 'regresioni': + this.lossFunction = LossFunctionRegression; + this.metrics = MetricsRegression; + break; + case 'binarni-klasifikacioni': + this.lossFunction = LossFunctionBinaryClassification; + this.metrics = MetricsBinaryClassification; + break; + case 'multi-klasifikacioni': + this.lossFunction = LossFunctionMultiClassification; + this.metrics = MetricsMultiClassification; + break; + default: + break; + } + } } diff --git a/frontend/src/app/_pages/add-model/add-model.component.css b/frontend/src/app/_pages/add-model/add-model.component.css deleted file mode 100644 index 7f05af0f..00000000 --- a/frontend/src/app/_pages/add-model/add-model.component.css +++ /dev/null @@ -1,42 +0,0 @@ -#header { - background-color: #003459; - padding-top: 30px; - padding-bottom: 20px; -} -#header h1 { - font-family: 'Lucida Sans', 'Lucida Sans Regular', 'Lucida Grande', 'Lucida Sans Unicode', Geneva, Verdana, sans-serif; - text-align: center; - color: white; -} - -#container { - border-radius: 8px; -} - -#wrapper { - color: #003459; -} - -.btnType1 { - background-color: #003459; - color: white; -} -.btnType2 { - background-color: white; - color: #003459; - border-color: #003459; -} -.selectedDatasetClass { - /*border-color: 2px solid #003459;*/ - background-color: lightblue; -} -ul li:hover { - background-color: lightblue; -} - -#divInputs { - margin-left: 20px; -} -#divOutputs { - margin-left: 20px; -} diff --git a/frontend/src/app/_pages/add-model/add-model.component.html b/frontend/src/app/_pages/add-model/add-model.component.html deleted file mode 100644 index 179e9aea..00000000 --- a/frontend/src/app/_pages/add-model/add-model.component.html +++ /dev/null @@ -1,428 +0,0 @@ - \ No newline at end of file diff --git a/frontend/src/app/_pages/add-model/add-model.component.spec.ts b/frontend/src/app/_pages/add-model/add-model.component.spec.ts deleted file mode 100644 index 2926e1c4..00000000 --- a/frontend/src/app/_pages/add-model/add-model.component.spec.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { ComponentFixture, TestBed } from '@angular/core/testing'; - -import { AddModelComponent } from './add-model.component'; - -describe('AddModelComponent', () => { - let component: AddModelComponent; - let fixture: ComponentFixture; - - beforeEach(async () => { - await TestBed.configureTestingModule({ - declarations: [ AddModelComponent ] - }) - .compileComponents(); - }); - - beforeEach(() => { - fixture = TestBed.createComponent(AddModelComponent); - component = fixture.componentInstance; - fixture.detectChanges(); - }); - - it('should create', () => { - expect(component).toBeTruthy(); - }); -}); diff --git a/frontend/src/app/_pages/add-model/add-model.component.ts b/frontend/src/app/_pages/add-model/add-model.component.ts deleted file mode 100644 index ba8f7d01..00000000 --- a/frontend/src/app/_pages/add-model/add-model.component.ts +++ /dev/null @@ -1,508 +0,0 @@ -import { Component, OnInit, ViewChild } from '@angular/core'; -import Model, { LossFunctionBinaryClassification, LossFunctionMultiClassification, LossFunctionRegression, NullValReplacer, ReplaceWith } from 'src/app/_data/Model'; -import { ProblemType, Encoding, ActivationFunction, LossFunction, Optimizer, NullValueOptions, Metrics, MetricsRegression, MetricsBinaryClassification, MetricsMultiClassification } from 'src/app/_data/Model'; -import { DatasetLoadComponent } from 'src/app/_elements/dataset-load/dataset-load.component'; -import { ModelsService } from 'src/app/_services/models.service'; -import shared from 'src/app/Shared'; -import Dataset from 'src/app/_data/Dataset'; -import { DatasetsService } from 'src/app/_services/datasets.service'; -import { CsvParseService } from 'src/app/_services/csv-parse.service'; - - -@Component({ - selector: 'app-add-model', - templateUrl: './add-model.component.html', - styleUrls: ['./add-model.component.css'] -}) -export class AddModelComponent implements OnInit { - - newModel: Model; - - ProblemType = ProblemType; - Encoding = Encoding; - ActivationFunction = ActivationFunction; - metrics: any = Metrics; - LossFunction = LossFunction; - lossFunction: any = LossFunction; - Optimizer = Optimizer; - NullValueOptions = NullValueOptions; - ReplaceWith = ReplaceWith; - Object = Object; - document = document; - shared = shared; - - selectedOutputColumnVal: string = ''; - - showMyDatasets: boolean = true; - myDatasets?: Dataset[]; - existingDatasetSelected: boolean = false; - selectedDataset?: Dataset; - otherDataset?: Dataset; - otherDatasetFile?: any[]; - datasetFile?: any[]; - datasetHasHeader?: boolean = true; - - tempTestSetDistribution: number = 90; - - //accepted: Boolean; - term: string = ""; - - selectedProblemType: string = ''; - selectedMetrics = []; - - trainingResult: string | undefined; - - constructor(private models: ModelsService, private datasets: DatasetsService, private csv: CsvParseService) { - this.newModel = new Model(); - - this.datasets.getMyDatasets().subscribe((datasets) => { - this.myDatasets = datasets; - }); - } - - ngOnInit(): void { - (document.getElementById("btnMyDataset")).focus(); - } - - datasetHasChanged(selectedDataset: Dataset) { - this.selectedDataset = selectedDataset; - this.resetCbsAndRbs(); - this.refreshThreeNullValueRadioOptions(); - } - - /*viewMyDatasetsForm() { - this.showMyDatasets = true; - this.resetSelectedDataset(); - //this.datasetLoaded = false; - this.resetCbsAndRbs(); - } - viewNewDatasetForm() { - this.showMyDatasets = false; - this.resetSelectedDataset(); - this.resetCbsAndRbs(); - }*/ - - addModel() { - if (!this.showMyDatasets) - this.saveModelWithNewDataset(_ => { console.log('MODEL ADDED (with new dataset).') }); - else - this.saveModelWithExistingDataset(_ => { console.log('MODEL ADDED (with existing dataset).') }); - } - - trainModel() { - let saveFunc; - this.trainingResult = undefined; - - if (!this.showMyDatasets) - saveFunc = (x: (arg0: any) => void) => { this.saveModelWithNewDataset(x) }; - else - saveFunc = (x: (arg0: any) => void) => { this.saveModelWithExistingDataset(x) }; - - saveFunc(((model: any) => { - console.log('Saved, training model...', model); - this.models.trainModel(model).subscribe(response => { - console.log('Train model complete!', response); - this.trainingResult = response; - }); - })); //privremeno cuvanje modela => vraca id sacuvanog modela koji cemo da treniramo sad - } - - saveModelWithNewDataset(callback: ((arg0: any) => void)) { - /* - this.getCheckedInputCols(); - this.getCheckedOutputCol(); - this.getMetrics(); - - if (this.validationInputsOutput()) { - console.log('ADD MODEL: STEP 1 - UPLOAD FILE'); - if (this.datasetLoadComponent) { - console.log("this.datasetLoadComponent.files:", this.datasetLoadComponent.files); - this.models.uploadData(this.datasetLoadComponent.files[0]).subscribe((file) => { - console.log('ADD MODEL: STEP 2 - ADD DATASET WITH FILE ID ' + file._id); - if (this.datasetLoadComponent) { - this.datasetLoadComponent.dataset.fileId = file._id; - this.datasetLoadComponent.dataset.username = shared.username; - - this.datasets.addDataset(this.datasetLoadComponent.dataset).subscribe((dataset) => { - console.log('ADD MODEL: STEP 3 - ADD MODEL WITH DATASET ID ', dataset._id); - this.newModel.datasetId = dataset._id; - - //da se doda taj dataset u listu postojecih, da bude izabran - this.refreshMyDatasetList(); - this.showMyDatasets = true; - this.selectThisDataset(dataset); - - this.newModel.randomTestSetDistribution = 1 - Math.round(this.tempTestSetDistribution / 100 * 10) / 10; - this.tempTestSetDistribution = 90; - this.newModel.username = shared.username; - - this.newModel.nullValuesReplacers = this.getNullValuesReplacersArray(); - - this.models.addModel(this.newModel).subscribe((response) => { - callback(response); - }, (error) => { - alert("Model sa unetim nazivom već postoji u Vašoj kolekciji.\nPromenite naziv modela i nastavite sa kreiranim datasetom."); - }); //kraj addModel subscribe - }, (error) => { - alert("Dataset sa unetim nazivom već postoji u Vašoj kolekciji.\nIzmenite naziv ili iskoristite postojeći dataset."); - }); //kraj addDataset subscribe - } //kraj treceg ifa - }, (error) => { - //alert("greska uploadData"); - }); //kraj uploadData subscribe - - } //kraj drugog ifa - } //kraj prvog ifa*/ - } - - saveModelWithExistingDataset(callback: ((arg0: any) => void)): any { - /*if (this.selectedDataset) { //dataset je izabran - this.getCheckedInputCols(); - this.getCheckedOutputCol(); - this.getMetrics(); - if (this.validationInputsOutput()) { - this.newModel.datasetId = this.selectedDataset._id; - - this.newModel.randomTestSetDistribution = 1 - Math.round(this.tempTestSetDistribution / 100 * 10) / 10; - this.tempTestSetDistribution = 90; - this.newModel.username = shared.username; - - this.newModel.nullValuesReplacers = this.getNullValuesReplacersArray(); - - this.models.addModel(this.newModel).subscribe((response) => { - callback(response); - }, (error) => { - shared.openDialog("Neuspeo pokušaj!", "Model sa unetim nazivom već postoji u Vašoj kolekciji. Promenite naziv modela i nastavite sa kreiranim datasetom."); - }); - } - } - else { - alert("Molimo Vas da izaberete neki dataset iz kolekcije."); - }*/ - } - - getCheckedInputCols() { - /*this.newModel.inputColumns = []; - let checkboxes: any; - - checkboxes = document.getElementsByName("cbsNew"); - - for (let i = 0; i < checkboxes.length; i++) { - let thatCb = checkboxes[i]; - if (thatCb.checked == true) // && thatCb.disabled == false ne treba nam ovo vise - this.newModel.inputColumns.push(thatCb.value); - }*/ - //console.log(this.checkedInputCols); - } - getCheckedOutputCol() { - /*this.newModel.columnToPredict = ''; - let radiobuttons: any; - - radiobuttons = document.getElementsByName("rbsNew"); - - for (let i = 0; i < radiobuttons.length; i++) { - let thatRb = radiobuttons[i]; - if (thatRb.checked) { - this.newModel.columnToPredict = thatRb.value; - break; - } - }*/ - //console.log(this.checkedOutputCol); - } - validationInputsOutput() { - /*if (this.newModel.inputColumns.length == 0 && this.newModel.columnToPredict == '') { - alert("Molimo Vas da izaberete ulazne i izlazne kolone za mrežu."); - return false; - } - else if (this.newModel.inputColumns.length == 0) { - shared.openDialog("Neuspeo pokušaj!", "Molimo Vas da izaberete ulaznu kolonu/kolone za mrežu."); - return false; - } - else if (this.newModel.columnToPredict == '') { - shared.openDialog("Neuspeo pokušaj!", "Molimo Vas da izaberete izlaznu kolonu za mrežu."); - return false; - } - for (let i = 0; i < this.newModel.inputColumns.length; i++) { - if (this.newModel.inputColumns[i] == this.newModel.columnToPredict) { - let colName = this.newModel.columnToPredict; - shared.openDialog("Neuspeo pokušaj!", "Izabrali ste istu kolonu (" + colName + ") kao ulaznu i izlaznu iz mreže. Korigujte izbor."); - return false; - } - } - return true;*/ - } - - /*selectThisDataset(dataset: Dataset) { - this.selectedDataset = dataset; - //this.selectedDatasetLoaded = false; - this.existingDatasetSelected = true; - this.datasetHasHeader = this.selectedDataset.hasHeader; - - this.datasets.getDatasetFile(dataset.fileId).subscribe((file: string | undefined) => { - if (file) { - this.datasetFile = this.csv.csvToArray(file, (dataset.delimiter == "razmak") ? " " : (dataset.delimiter == "") ? "," : dataset.delimiter); - //for (let i = this.datasetFile.length - 1; i >= 0; i--) { //moguce da je vise redova na kraju fajla prazno i sl. - //if (this.datasetFile[i].length != this.datasetFile[0].length) - //this.datasetFile[i].pop(); - //else - // break; //nema potrebe dalje - //} - //console.log(this.datasetFile); - this.resetCbsAndRbs(); - this.refreshThreeNullValueRadioOptions(); - //this.selectedDatasetLoaded = true; - this.scrollToNextForm(); - } - }); - //this.datasetHasHeader = false; - }*/ - - scrollToNextForm() { - (document.getElementById("selectInAndOuts")).scrollIntoView({ - behavior: "smooth", - block: "start", - inline: "nearest" - }); - } - - /*resetSelectedDataset(): boolean { - const temp = this.selectedDataset; - this.selectedDataset = this.otherDataset; - this.otherDataset = temp; - const tempFile = this.datasetFile; - this.datasetFile = this.otherDatasetFile; - this.otherDatasetFile = tempFile; - return true; - }*/ - resetCbsAndRbs(): boolean { - this.uncheckRbs(); - this.checkAllCbs(); - return true; - } - checkAllCbs() { - let checkboxes: any; - - checkboxes = document.getElementsByName("cbsNew"); - for (let i = 0; i < checkboxes.length; i++) { - (checkboxes[i]).checked = true; - (checkboxes[i]).disabled = false; - } - } - uncheckRbs() { - this.selectedOutputColumnVal = ''; - let radiobuttons: any; - - radiobuttons = document.getElementsByName("rbsNew"); - for (let i = 0; i < radiobuttons.length; i++) - (radiobuttons[i]).checked = false; - } - - refreshMyDatasetList() { - this.datasets.getMyDatasets().subscribe((datasets) => { - this.myDatasets = datasets; - }); - } - - refreshThreeNullValueRadioOptions() { - this.newModel.nullValues = NullValueOptions.DeleteRows; - } - - isChecked(someId: string) { //proveri ako je element sa datim ID-em cek - return (document.getElementById(someId)).checked; - } - - isNumber(value: string | number): boolean { - return ((value != null) && - (value !== '') && - !isNaN(Number(value.toString()))); - } - - findColIndexByName(colName: string): number { - if (this.datasetFile) - for (let i = 0; i < this.datasetFile[0].length; i++) - if (colName === this.datasetFile[0][i]) - return i; - return -1; - } - findColNameByIndex(index: number): string { - if (this.datasetFile) - if (this.datasetHasHeader && index < this.datasetFile[0].length) - return this.datasetFile[0][index]; - return ''; - } - emptyFillTextInput(colName: string) { - (document.getElementById("fillText_" + colName)).value = ""; - } - - checkFillColRadio(colName: string) { - (document.getElementById("fillCol_" + colName)).checked = true; - } - calculateSumOfNullValuesInCol(colName: string): number { - //console.log(this.datasetFile); - if (this.datasetFile) { - let colIndex = this.findColIndexByName(colName); - let sumOfNulls = 0; - - let startValue = (this.selectedDataset!.hasHeader) ? 1 : 0; - for (let i = startValue; i < this.datasetFile.length; i++) { - if (this.datasetFile[i][colIndex] == "" || this.datasetFile[i][colIndex] == undefined) - ++sumOfNulls; - } - return sumOfNulls; - } - return -1; - } - calculateMeanColValue(colName: string): number { - if (this.datasetFile) { - let colIndex = this.findColIndexByName(colName); - let sum = 0; - let n = 0; - - let startValue = (this.selectedDataset!.hasHeader) ? 1 : 0; - for (let i = startValue; i < this.datasetFile.length; i++) - if (this.datasetFile[i][colIndex] != '') { - sum += Number(this.datasetFile[i][colIndex]); - ++n; - } - console.log(sum / n); - return (sum != 0) ? (sum / n) : 0; - } - return 0; - } - calculateMedianColValue(colName: string): number { - if (this.datasetFile) { - let array = []; - let colIndex = this.findColIndexByName(colName); - - let startValue = (this.datasetHasHeader) ? 1 : 0; - for (let i = startValue; i < this.datasetFile.length; i++) - if (this.datasetFile[i][colIndex] != '') - array.push(Number(this.datasetFile[i][colIndex])); - - array.sort(); - if (array.length % 2 == 0) - return array[array.length / 2 - 1] / 2; - else - return array[(array.length - 1) / 2]; - } - return 0; - } - replaceWithSelectedString(event: Event) { - let value = (event.target).value; - let colIndex = Number(((event.target).id).split("replaceOptions")[1]); - let colName = this.findColNameByIndex(colIndex); - - (document.getElementById("fillCol_" + colName)).checked = true; - - if (!this.datasetHasHeader) - (document.getElementById("fillText_" + colName)).value = value; - else { - if (value == colName) - (document.getElementById("fillText_" + colName)).value = ""; - else - (document.getElementById("fillText_" + colName)).value = value; - } - } - replaceWithSelectedNumber(event: Event) { - let option = (event.target).value; - let colIndex = Number(((event.target).id).split("replaceOptions")[1]); - let colName = this.findColNameByIndex(colIndex); - - (document.getElementById("fillCol_" + colName)).checked = true; - - if (option == ReplaceWith.Mean) - (document.getElementById("fillText_" + colName)).value = this.calculateMeanColValue(colName).toString(); - else if (option == ReplaceWith.Median) - (document.getElementById("fillText_" + colName)).value = this.calculateMedianColValue(colName).toString(); - else if (option == ReplaceWith.None) - (document.getElementById("fillText_" + colName)).value = ""; - } - - - getNullValuesReplacersArray(): NullValReplacer[] { - let array: NullValReplacer[] = []; - - if (this.datasetFile) { - - if (this.newModel.nullValues == NullValueOptions.Replace) { - - for (let i = 0; i < this.datasetFile[0].length; i++) { - let column = this.datasetFile[0][i]; - - if (this.calculateSumOfNullValuesInCol(column) > 0) { //ako kolona nema null vrednosti, ne dodajemo je u niz - if ((document.getElementById("delCol_" + column)).checked) { - array.push({ - column: column, - option: NullValueOptions.DeleteColumns, - value: "" - }); - } - else if ((document.getElementById("delRows_" + column)).checked) { - array.push({ - column: column, - option: NullValueOptions.DeleteRows, - value: "" - }); - } - else if (((document.getElementById("fillCol_" + column)).checked)) { - array.push({ - column: column, - option: NullValueOptions.Replace, - value: (document.getElementById("fillText_" + column)).value - }); - } - } - } - } - } - //console.log(array); - return array; - } - - getInputById(id: string): HTMLInputElement { - return document.getElementById(id) as HTMLInputElement; - } - - arrayColumn = (arr: any[][], n: number) => [...this.dropEmptyString(new Set(arr.map(x => x[n])))]; - - dropEmptyString(set: Set): Set { - if (set.has("")) - set.delete(""); - if (set.has(null)) - set.delete(null); - if (set.has(undefined)) - set.delete(undefined); - return set; - } - - filterOptions() { - switch (this.newModel.type) { - case 'regresioni': - this.lossFunction = LossFunctionRegression; - this.metrics = MetricsRegression; - break; - case 'binarni-klasifikacioni': - this.lossFunction = LossFunctionBinaryClassification; - this.metrics = MetricsBinaryClassification; - break; - case 'multi-klasifikacioni': - this.lossFunction = LossFunctionMultiClassification; - this.metrics = MetricsMultiClassification; - break; - default: - break; - } - } - - getMetrics() { - this.newModel.metrics = []; - let cb = document.getElementsByName("cbmetrics"); - - for (let i = 0; i < cb.length; i++) { - let chb = cb[i]; - if (chb.checked == true) - this.newModel.metrics.push(chb.value); - } - - } -} diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 04523989..c3a2ce7a 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -11,7 +11,6 @@ import { Ng2SearchPipeModule } from 'ng2-search-filter'; import { AppComponent } from './app.component'; import { NgbModule } from '@ng-bootstrap/ng-bootstrap'; import { DatasetLoadComponent } from './_elements/dataset-load/dataset-load.component'; -import { AddModelComponent } from './_pages/add-model/add-model.component'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { LoginModalComponent } from './_modals/login-modal/login-modal.component'; import { ReactiveFormsModule } from '@angular/forms'; @@ -49,7 +48,6 @@ import { AddNewDatasetComponent } from './_elements/add-new-dataset/add-new-data declarations: [ AppComponent, DatasetLoadComponent, - AddModelComponent, LoginModalComponent, RegisterModalComponent, HomeComponent, diff --git a/frontend/src/app/experiment/experiment.component.html b/frontend/src/app/experiment/experiment.component.html index 42f2993d..8d2c86b3 100644 --- a/frontend/src/app/experiment/experiment.component.html +++ b/frontend/src/app/experiment/experiment.component.html @@ -1,13 +1,201 @@
    + +

    1. Izvor podataka

    + +

    2. Preprocesiranje

    +

    Biranje ulaznih i izlaznih kolona:

    +
    +
    +
    +

    Izaberite ulazne kolone:

    +
    +
    +
    +   + +
    +
    +
    +
    +

    Izaberite izlaznu kolonu:

    +
    +
    +
    +   + +
    +
    +
    +
    +
    +

    Popunjavanje nedostajućih vrednosti:

    +
    + +
    + +
    + +

    +
    +
    + +
    +
    +
    + + {{column.name}} ( + + TODO BROJ null) + + + + + +
    + +
    +
    +
    + +
    + + +
    + + +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    +
    +
    +
    + + {{column}} (br + null) + + + + + + + + + + +
    +
    + +
    +
    +
    +
    + +

    3. Podešavanja mreže

    -

    4. Rezultati treniranja

    + + +

    4. Treniraj model

    + +

    Rezultati treniranja

    +
    +

    Rezultati treniranja:

    +

    + {{trainingResult}} +

    +
    \ No newline at end of file diff --git a/frontend/src/app/experiment/experiment.component.ts b/frontend/src/app/experiment/experiment.component.ts index 16de7351..2309dcd7 100644 --- a/frontend/src/app/experiment/experiment.component.ts +++ b/frontend/src/app/experiment/experiment.component.ts @@ -1,4 +1,9 @@ import { Component, OnInit } from '@angular/core'; +import Experiment, { NullValReplacer, NullValueOptions, ReplaceWith } from '../_data/Experiment'; +import Model from '../_data/Model'; +import Dataset from '../_data/Dataset'; +import { ModelsService } from '../_services/models.service'; +import Shared from '../Shared'; @Component({ selector: 'app-experiment', @@ -7,9 +12,107 @@ import { Component, OnInit } from '@angular/core'; }) export class ExperimentComponent implements OnInit { - constructor() { } + experiment: Experiment = new Experiment(); + selectedModel?: Model; + selectedDataset?: Dataset; + trainingResult: any; // any za sad, promeni kasnije + + NullValueOptions = NullValueOptions; + ReplaceWith = ReplaceWith; + Object = Object; + + selectedOutputColumnVal: string = ''; + + constructor(private models: ModelsService) { } ngOnInit(): void { } + getInputById(id: string): HTMLInputElement { + return document.getElementById(id) as HTMLInputElement; + } + + arrayColumn = (arr: any[][], n: number) => [...this.dropEmptyString(new Set(arr.map(x => x[n])))]; + + dropEmptyString(set: Set): Set { + if (set.has("")) + set.delete(""); + if (set.has(null)) + set.delete(null); + if (set.has(undefined)) + set.delete(undefined); + return set; + } + + emptyFillTextInput(colName: string) { + (document.getElementById("fillText_" + colName)).value = ""; + } + + checkFillColRadio(colName: string) { + (document.getElementById("fillCol_" + colName)).checked = true; + } + + replace(event: Event) { + let option = (event.target).value; + // TODO + } + + getNullValuesReplacersArray()/*: NullValReplacer[]*/ { + let array: NullValReplacer[] = []; + + // TODO ispravi + /*if (this.datasetFile) { + + if (this.newModel.nullValues == NullValueOptions.Replace) { + + for (let i = 0; i < this.datasetFile[0].length; i++) { + let column = this.datasetFile[0][i]; + + if (this.calculateSumOfNullValuesInCol(column) > 0) { //ako kolona nema null vrednosti, ne dodajemo je u niz + if ((document.getElementById("delCol_" + column)).checked) { + array.push({ + column: column, + option: NullValueOptions.DeleteColumns, + value: "" + }); + } + else if ((document.getElementById("delRows_" + column)).checked) { + array.push({ + column: column, + option: NullValueOptions.DeleteRows, + value: "" + }); + } + else if (((document.getElementById("fillCol_" + column)).checked)) { + array.push({ + column: column, + option: NullValueOptions.Replace, + value: (document.getElementById("fillText_" + column)).value + }); + } + } + } + } + } + //console.log(array); + return array;*/ + } + + trainModel() { + this.trainingResult = undefined; + console.log('Training model...', this.selectedModel); + if (!this.selectedDataset) { + Shared.openDialog('Greška', 'Izvor podataka nije izabran!'); + return; + } + // TODO proveri nullValues + if (!this.selectedModel) { + Shared.openDialog('Greška', 'Model nije izabran!'); + return; + } + this.models.trainModel(this.selectedModel).subscribe((response: any) => { + console.log('Train model complete!', response); + this.trainingResult = response; + }); + } } -- cgit v1.2.3 From 45201c5c388b4988af3b319d6702c60397929dfc Mon Sep 17 00:00:00 2001 From: Nevena Bojovic Date: Wed, 6 Apr 2022 21:18:27 +0200 Subject: Mala izmena. --- backend/api/api/appsettings.json | 5 +++-- frontend/angular.json | 5 +---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/backend/api/api/appsettings.json b/backend/api/api/appsettings.json index fdccfb07..f8abae88 100644 --- a/backend/api/api/appsettings.json +++ b/backend/api/api/appsettings.json @@ -10,8 +10,8 @@ }, "AllowedHosts": "*", "UserStoreDatabaseSettings": { - /* LocalHost - */ + //LocalHost + "ConnectionString": "mongodb://127.0.0.1:27017/", "DatabaseName": "si_project", "CollectionName": "users", @@ -19,6 +19,7 @@ "ModelCollectionName": "Model", "PredictorCollectionName": "Predictor", "FilesCollectionName": "Files" + /* "ConnectionString": "mongodb+srv://si_user:si_user@sidatabase.twtfm.mongodb.net/myFirstDatabase?retryWrites=true&w=majority", "DatabaseName": "si_db", diff --git a/frontend/angular.json b/frontend/angular.json index f9825281..6653e4b1 100644 --- a/frontend/angular.json +++ b/frontend/angular.json @@ -33,10 +33,7 @@ "./node_modules/@angular/material/prebuilt-themes/indigo-pink.css" ], "scripts": [ - "node_modules/bootstrap/dist/js/bootstrap.bundle.min.js", - "node_modules/jquery/dist/jquery.min.js", - "node_modules/popper.js/dist/popper.min.js", - "node_modules/bootstrap/dist/js/bootstrap.min.js" + "node_modules/bootstrap/dist/js/bootstrap.bundle.min.js" ] }, "configurations": { -- cgit v1.2.3 From 052c97eadc3784306607fab2fd725d3cfa592867 Mon Sep 17 00:00:00 2001 From: Danijel Andjelkovic Date: Wed, 6 Apr 2022 21:31:12 +0200 Subject: json merge fix --- backend/api/api/appsettings.json | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/backend/api/api/appsettings.json b/backend/api/api/appsettings.json index 81255d5c..f8923a10 100644 --- a/backend/api/api/appsettings.json +++ b/backend/api/api/appsettings.json @@ -9,20 +9,6 @@ } }, "AllowedHosts": "*", -<<<<<<< HEAD - "UserStoreDatabaseSettings": { - //LocalHost - - "ConnectionString": "mongodb://127.0.0.1:27017/", - "DatabaseName": "si_project", - "CollectionName": "users", - "DatasetCollectionName": "Dataset", - "ModelCollectionName": "Model", - "PredictorCollectionName": "Predictor", - "FilesCollectionName": "Files" - - /* -======= "UserStoreDatabaseSettings": { /* LocalHost */ @@ -35,7 +21,6 @@ "FilesCollectionName": "Files", "ExperimentCollectionName": "Experiment" /* ->>>>>>> 499f641683767d2da735fb760c89f0c611e37596 "ConnectionString": "mongodb+srv://si_user:si_user@sidatabase.twtfm.mongodb.net/myFirstDatabase?retryWrites=true&w=majority", "DatabaseName": "si_db", "CollectionName": "users", -- cgit v1.2.3 From 7d5af698fdf053cfe6ef674c085022cf8276f56c Mon Sep 17 00:00:00 2001 From: TAMARA JERINIC Date: Wed, 6 Apr 2022 22:49:49 +0200 Subject: Dodate min i max vrednosti kolona u statistiku. Dodata mogućnost zamene null vrednosti. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/microservice/api/newmlservice.py | 77 +++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/backend/microservice/api/newmlservice.py b/backend/microservice/api/newmlservice.py index 50af15f8..2ea31702 100644 --- a/backend/microservice/api/newmlservice.py +++ b/backend/microservice/api/newmlservice.py @@ -33,6 +33,8 @@ def returnColumnsInfo(dataset): uniquevalues=datafront[kolona].unique() mean=0 median=0 + minimum=0 + maximum=0 nullCount=datafront[kolona].isnull().sum() if(nullCount>0): allNullCols=allNullCols+1 @@ -41,10 +43,14 @@ def returnColumnsInfo(dataset): 'uniqueValues':uniquevalues.tolist(), 'median':float(mean), 'mean':float(median), - 'numNulls':float(nullCount) + 'numNulls':float(nullCount), + 'min':float(minimum), + 'max':float(maximum) } dict.append(frontreturn) else: + minimum=min(datafront[kolona]) + maximum=max(datafront[kolona]) mean=datafront[kolona].mean() median=s.median(datafront[kolona]) nullCount=datafront[kolona].isnull().sum() @@ -55,7 +61,9 @@ def returnColumnsInfo(dataset): 'uniqueValues':[], 'mean':float(mean), 'median':float(median), - 'numNulls':float(nullCount) + 'numNulls':float(nullCount), + 'min':float(minimum), + 'max':float(maximum) } dict.append(frontreturn) NullRows = datafront[datafront.isnull().any(axis=1)] @@ -95,27 +103,35 @@ class TrainingResult: def train(dataset, params, callback): problem_type = params["type"] - print(problem_type) + #print(problem_type) data = pd.DataFrame() - print(data) + #print(data) for col in params["inputColumns"]: - print(col) + #print(col) data[col]=dataset[col] output_column = params["columnToPredict"] data[output_column] = dataset[output_column] - print(data) + #print(data) ###NULL null_value_options = params["nullValues"] null_values_replacers = params["nullValuesReplacers"] if(null_value_options=='replace'): - print("replace null") # TODO + #print("replace null") # TODO + dict=params['null_values_replacers'] + while(len(dict)>0): + replace=dict.pop() + col=replace['column'] + opt=replace['option'] + if(opt=='replace'): + replacevalue=replace['value'] + data[col]=data[col].fillna(replacevalue) elif(null_value_options=='delete_rows'): data=data.dropna() elif(null_value_options=='delete_columns'): - data=data.dropna() - print(data.shape) + data=data.dropna(axis=1) + #print(data.shape) # # Brisanje kolona koje ne uticu na rezultat @@ -176,7 +192,7 @@ def train(dataset, params, callback): for col in data.columns: if(col!=output_column): x_columns.append(col) - print(x_columns) + #print(x_columns) x = data[x_columns].values y = data[output_column].values @@ -190,7 +206,7 @@ def train(dataset, params, callback): else: random=0 x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=test, random_state=random) - print(x_train,x_test) + #print(x_train,x_test) # # Treniranje modela @@ -215,7 +231,7 @@ def train(dataset, params, callback): #print(y_pred.flatten()) #print(y_test) scores = classifier.evaluate(x_test, y_test) - print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100)) + #print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100)) classifier.save("temp/"+params['name'], save_format='h5') #vizuelizacija u python-u #from ann_visualizer.visualize import ann_viz; @@ -238,11 +254,11 @@ def train(dataset, params, callback): y_pred=classifier.predict(x_test) y_pred=(y_pred>=0.5).astype('int') - print(y_pred.flatten()) - print(y_test) + #print(y_pred.flatten()) + #print(y_test) scores = classifier.evaluate(x_test, y_test) - print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100)) + #print("\n%s: %.2f%%" % (classifier.metrics_names[1], scores[1]*100)) #ann_viz(classifier, title="My neural network") classifier.save("temp/"+params['name'], save_format='h5') @@ -260,7 +276,7 @@ def train(dataset, params, callback): history=classifier.fit(x_train, y_train, epochs = params['epochs'],batch_size=params['batchSize']) y_pred=classifier.predict(x_test) - print(classifier.evaluate(x_test, y_test)) + #print(classifier.evaluate(x_test, y_test)) def roc_auc_score_multiclass(actual_class, pred_class, average = "macro"): @@ -346,11 +362,11 @@ def train(dataset, params, callback): def manageH5(dataset,params,h5model): problem_type = params["type"] - print(problem_type) + #print(problem_type) data = pd.DataFrame() #print(data) for col in params["inputColumns"]: - print(col) + #print(col) data[col]=dataset[col] output_column = params["columnToPredict"] data[output_column] = dataset[output_column] @@ -361,12 +377,21 @@ def manageH5(dataset,params,h5model): null_values_replacers = params["nullValuesReplacers"] if(null_value_options=='replace'): - print("replace null") # TODO + #print("replace null") # TODO + dict=params['null_values_replacers'] + while(len(dict)>0): + replace=dict.pop() + col=replace['column'] + opt=replace['option'] + if(opt=='replace'): + replacevalue=replace['value'] + data[col]=data[col].fillna(replacevalue) elif(null_value_options=='delete_rows'): data=data.dropna() elif(null_value_options=='delete_columns'): data=data.dropna() - print(data.shape) + + #print(data.shape) # # Brisanje kolona koje ne uticu na rezultat @@ -402,10 +427,10 @@ def manageH5(dataset,params,h5model): x_columns.append(col) #print(x_columns) x2 = data[x_columns] - print(x2) - print(x2.values) + #print(x2) + #print(x2.values) x2 = data[x_columns].values - print(x2) + #print(x2) y2 = data[output_column].values h5model.summary() ann_viz(h5model, title="My neural network") @@ -419,6 +444,6 @@ def manageH5(dataset,params,h5model): y_pred2=np.argmax(y_pred2,axis=1) #y_pred=h5model.predict_classes(x) score = h5model.evaluate(x2,y_pred2, verbose=0) - print("%s: %.2f%%" % (h5model.metrics_names[1], score[1]*100)) - print(y_pred2) - print( 'done') \ No newline at end of file + #print("%s: %.2f%%" % (h5model.metrics_names[1], score[1]*100)) + #print(y_pred2) + #print( 'done') \ No newline at end of file -- cgit v1.2.3 From 588b9528bc39acc04a94ad8d996333bb4e4764e7 Mon Sep 17 00:00:00 2001 From: Sonja Galovic Date: Wed, 6 Apr 2022 23:15:10 +0200 Subject: Dodata funkcionalnost cuvanja dataseta (prvo se cuva pa se bira). Izmenjeni datatable, dataset-load i add-new-dataset-component --- frontend/src/app/Shared.ts | 1 + .../add-new-dataset/add-new-dataset.component.html | 14 ++++++-- .../add-new-dataset/add-new-dataset.component.ts | 42 +++++++++++++++++++--- .../dataset-load/dataset-load.component.html | 25 +++++++------ .../_elements/datatable/datatable.component.html | 8 ++--- .../app/_elements/datatable/datatable.component.ts | 1 + frontend/src/app/_services/auth.service.ts | 2 ++ 7 files changed, 68 insertions(+), 25 deletions(-) diff --git a/frontend/src/app/Shared.ts b/frontend/src/app/Shared.ts index 86e26687..a069ff8b 100644 --- a/frontend/src/app/Shared.ts +++ b/frontend/src/app/Shared.ts @@ -7,6 +7,7 @@ class Shared { constructor( public loggedIn: boolean, public username: string = '', + public userId: string = '', public photoId: string = '1', public dialog?: MatDialog //public alertDialog?: ElementRef diff --git a/frontend/src/app/_elements/add-new-dataset/add-new-dataset.component.html b/frontend/src/app/_elements/add-new-dataset/add-new-dataset.component.html index dfeb4f62..eb68b54c 100644 --- a/frontend/src/app/_elements/add-new-dataset/add-new-dataset.component.html +++ b/frontend/src/app/_elements/add-new-dataset/add-new-dataset.component.html @@ -24,9 +24,9 @@
- - + @@ -38,4 +38,12 @@
-
\ No newline at end of file +
+ +
+ +
+ +
+ +
diff --git a/frontend/src/app/_elements/add-new-dataset/add-new-dataset.component.ts b/frontend/src/app/_elements/add-new-dataset/add-new-dataset.component.ts index fceb53cf..c5e6558a 100644 --- a/frontend/src/app/_elements/add-new-dataset/add-new-dataset.component.ts +++ b/frontend/src/app/_elements/add-new-dataset/add-new-dataset.component.ts @@ -1,6 +1,10 @@ import { Component, EventEmitter, Output, ViewChild } from '@angular/core'; import { NgxCsvParser, NgxCSVParserError } from 'ngx-csv-parser'; import Dataset from 'src/app/_data/Dataset'; +import { DatasetsService } from 'src/app/_services/datasets.service'; +import { ModelsService } from 'src/app/_services/models.service'; +import shared from 'src/app/Shared'; +import { DatatableComponent } from '../datatable/datatable.component'; @Component({ selector: 'app-add-new-dataset', @@ -9,7 +13,8 @@ import Dataset from 'src/app/_data/Dataset'; }) export class AddNewDatasetComponent { - @Output() loaded = new EventEmitter(); + @Output() newDatasetAdded = new EventEmitter(); + @ViewChild(DatatableComponent) datatable?: DatatableComponent; delimiterOptions: Array = [",", ";", "\t", "razmak", "|"]; //podrazumevano "," @@ -23,7 +28,7 @@ export class AddNewDatasetComponent { dataset: Dataset; //dodaj ! potencijalno - constructor(private ngxCsvParser: NgxCsvParser) { + constructor(private ngxCsvParser: NgxCsvParser, private modelsService: ModelsService, private datasetsService: DatasetsService) { this.dataset = new Dataset(); } @@ -48,6 +53,9 @@ export class AddNewDatasetComponent { if (this.files.length < 1) return; + this.datatable!.loaded = false; + this.datatable!.hasInput = this.hasInput; + this.ngxCsvParser.parse(this.files[0], { header: false, delimiter: (this.dataset.delimiter == "razmak") ? " " : (this.dataset.delimiter == "") ? "," : this.dataset.delimiter }) .pipe().subscribe((result) => { @@ -60,10 +68,12 @@ export class AddNewDatasetComponent { this.rowsNumber = this.csvRecords.length; this.colsNumber = this.csvRecords[0].length; - if (this.dataset.hasHeader) //kasnije dodati opciju kada nema header da korisnik rucno unosi header-e + if (this.dataset.hasHeader) this.dataset.header = this.csvRecords[0]; - - this.loaded.emit("loaded"); + + this.datatable!.data = this.csvRecords; + this.datatable!.hasHeader = this.dataset.hasHeader; + this.datatable!.loaded = true; } }, (error: NgxCSVParserError) => { console.log('Error', error); @@ -75,4 +85,26 @@ export class AddNewDatasetComponent { this.dataset.accessibleByLink = true; } + uploadDataset() { + this.modelsService.uploadData(this.files[0]).subscribe((file) => { + //console.log('ADD MODEL: STEP 2 - ADD DATASET WITH FILE ID ' + file._id); + this.dataset.fileId = file._id; + this.dataset.username = shared.username; + + this.datasetsService.addDataset(this.dataset).subscribe((dataset) => { + + this.newDatasetAdded.emit("added"); + //this.refreshMyDatasetList(); refreshuj dataset listu u ds-load i selektuj taj ds + //this.showMyDatasets = true; + //this.selectThisDataset(dataset); + + shared.openDialog("Obaveštenje", "Uspešno ste dodali novi izvor podataka u kolekciju. Molimo sačekajte par trenutaka da se procesira."); + }, (error) => { + shared.openDialog("Neuspeo pokušaj!", "Dataset sa unetim nazivom već postoji u Vašoj kolekciji. Izmenite naziv ili iskoristite postojeći dataset."); + }); //kraj addDataset subscribe + }, (error) => { + + }); //kraj uploadData subscribe + } + } diff --git a/frontend/src/app/_elements/dataset-load/dataset-load.component.html b/frontend/src/app/_elements/dataset-load/dataset-load.component.html index 674e5990..2052c9b5 100644 --- a/frontend/src/app/_elements/dataset-load/dataset-load.component.html +++ b/frontend/src/app/_elements/dataset-load/dataset-load.component.html @@ -3,8 +3,7 @@
-

Izvor podataka:

-
+
-
    -
  • - -
    • -
+
    +
  • + +
    • +
+
+ +
+ (newDatasetAdded)="showMyDatasets = true"> -
- -
\ No newline at end of file diff --git a/frontend/src/app/_elements/datatable/datatable.component.html b/frontend/src/app/_elements/datatable/datatable.component.html index 8fcd44ac..52d50443 100644 --- a/frontend/src/app/_elements/datatable/datatable.component.html +++ b/frontend/src/app/_elements/datatable/datatable.component.html @@ -1,11 +1,11 @@ -
+
- +
@@ -18,7 +18,7 @@
{{item}}
- +
@@ -28,7 +28,7 @@ -
+

{{data.length - 1}} x {{data[0].length}} {{data.length}} x {{data[0].length}} diff --git a/frontend/src/app/_elements/datatable/datatable.component.ts b/frontend/src/app/_elements/datatable/datatable.component.ts index 3343f6f0..19fb204e 100644 --- a/frontend/src/app/_elements/datatable/datatable.component.ts +++ b/frontend/src/app/_elements/datatable/datatable.component.ts @@ -11,6 +11,7 @@ export class DatatableComponent implements OnInit { @Input() data?: any[] = []; + hasInput = false; loaded = false; constructor() { } diff --git a/frontend/src/app/_services/auth.service.ts b/frontend/src/app/_services/auth.service.ts index ccfbe15f..a108f8db 100644 --- a/frontend/src/app/_services/auth.service.ts +++ b/frontend/src/app/_services/auth.service.ts @@ -33,6 +33,7 @@ export class AuthService { var token = this.cookie.get('token'); var property=jwtHelper.decodeToken(this.cookie.get('token')); var username=property['name']; + var userId = property['id']; return !jwtHelper.isTokenExpired(token) && username!=""; } return false; @@ -89,6 +90,7 @@ export class AuthService { console.log("decoded:", decodedToken); this.shared.loggedIn = this.isAuthenticated(); this.shared.username = decodedToken.name; + this.shared.userId = decodedToken.id; this.enableAutoRefresh(); } } -- cgit v1.2.3
{{col}}