diff options
34 files changed, 385 insertions, 208 deletions
diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index f61213c2..a6ebe8ac 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -18,14 +18,14 @@ namespace api.Controllers private readonly IFileService _fileService; private IJwtToken jwtToken; - public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token,IMlConnectionService mlConnectionService, IFileService fileService) + public DatasetController(IDatasetService datasetService, IConfiguration configuration, IJwtToken Token, IMlConnectionService mlConnectionService, IFileService fileService) { _datasetService = datasetService; _mlConnectionService = mlConnectionService; _fileService = fileService; jwtToken = Token; } - + public string getUserId() { string userId; @@ -116,7 +116,7 @@ namespace api.Controllers if (userId == null) return BadRequest(); - var dataset = _datasetService.GetOneDataset(userId, name); + var dataset = _datasetService.GetOneDatasetN(userId, name); if (dataset == null) return NotFound($"Dataset with name = {name} not found or dataset is not public or not preprocessed"); @@ -129,11 +129,13 @@ namespace api.Controllers public async Task<ActionResult<Dataset>> Post([FromBody] Dataset dataset) { string uploaderId = getUserId(); - + + dataset.uploaderId = uploaderId; + //da li ce preko tokena da se ubaci username ili front salje //dataset.username = usernameToken; //username = "" ako je GUEST DODAO - var existingDataset = _datasetService.GetOneDataset(dataset.uploaderId, dataset.name); + var existingDataset = _datasetService.GetOneDatasetN(dataset.uploaderId, dataset.name); if (existingDataset != null) return NotFound($"Dataset with this name already exists"); @@ -142,7 +144,7 @@ namespace api.Controllers FileModel fileModel = _fileService.getFile(dataset.fileId); dataset.isPreProcess = false; _datasetService.Create(dataset); - _mlConnectionService.PreProcess(dataset,fileModel.path,uploaderId); + _mlConnectionService.PreProcess(dataset, fileModel.path, uploaderId); return Ok(); } } diff --git a/backend/api/api/Controllers/ExperimentController.cs b/backend/api/api/Controllers/ExperimentController.cs index eecbe756..6f1bbd42 100644 --- a/backend/api/api/Controllers/ExperimentController.cs +++ b/backend/api/api/Controllers/ExperimentController.cs @@ -53,6 +53,7 @@ namespace api.Controllers return BadRequest(); experiment.uploaderId = uploaderId; + var existingExperiment = _experimentService.Get(uploaderId, experiment.name); if(existingExperiment != null) return NotFound($"Experiment with this name exists"); @@ -99,17 +100,43 @@ namespace api.Controllers if (uploaderId == null) return BadRequest(); - var existingDataset = _experimentService.GetOneExperiment(uploaderId, id); + var existingExperiment = _experimentService.GetOneExperiment(uploaderId, id); //ne mora da se proverava - if (existingDataset == null) + if (existingExperiment == null) return NotFound($"Experiment with ID = {id} or user with ID = {uploaderId} not found"); experiment.lastUpdated = DateTime.UtcNow; - _experimentService.Update(uploaderId, id, experiment); - return Ok($"Experiment with ID = {id} updated"); + return Ok(_experimentService.Update(uploaderId, id, experiment)); } + + // DELETE api/<ExperimentController>/name + [HttpDelete("{id}")] + [Authorize(Roles = "User")] + public ActionResult Delete(string id) + { + string uploaderId = getUserId(); + + if (uploaderId == null) + return BadRequest(); + + var experiment = _experimentService.GetOneExperiment(uploaderId, id); + + if (experiment == null) + return NotFound($"Experiment with ID = {id} or user with ID = {uploaderId} not found"); + + _experimentService.Delete(experiment.uploaderId, experiment._id); + + return Ok($"Experiment with ID = {id} deleted"); + + } + + public void DeleteHelper(string uploaderId, string experimentId) + { + _experimentService.Delete(uploaderId, experimentId); + } + } } diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index 1c1ea364..a0e51e1f 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -91,10 +91,12 @@ namespace api.Controllers return Ok(); } - - - - + // GET: api/<ModelController>/publicmodels + [HttpGet("publicmodels")] + public ActionResult<List<Model>> GetPublicModels() + { + return _modelService.GetPublicModels(); + } // GET: api/<ModelController>/mymodels [HttpGet("mymodels")] diff --git a/backend/api/api/Interfaces/IDatasetService.cs b/backend/api/api/Interfaces/IDatasetService.cs index f493a2ec..2f7d0010 100644 --- a/backend/api/api/Interfaces/IDatasetService.cs +++ b/backend/api/api/Interfaces/IDatasetService.cs @@ -5,7 +5,8 @@ namespace api.Services { public interface IDatasetService { - Dataset GetOneDataset(string userId, string name); + Dataset GetOneDataset(string userId, string id); + Dataset GetOneDatasetN(string userId, string name); Dataset GetOneDataset(string id); List<Dataset> SearchDatasets(string name); List<Dataset> GetMyDatasets(string userId); diff --git a/backend/api/api/Interfaces/IExperimentService.cs b/backend/api/api/Interfaces/IExperimentService.cs index 2a69cff9..7e59ace3 100644 --- a/backend/api/api/Interfaces/IExperimentService.cs +++ b/backend/api/api/Interfaces/IExperimentService.cs @@ -8,8 +8,9 @@ namespace api.Services public Experiment Get(string id); public List<Experiment> GetMyExperiments(string id); public Experiment Get(string uploaderId, string name); - Experiment GetOneExperiment(string userId, string name); - void Update(string userId, string id, Experiment experiment); + Experiment GetOneExperiment(string userId, string id); + Experiment Update(string userId, string id, Experiment experiment); + void Delete(string userId, string id); } }
\ No newline at end of file diff --git a/backend/api/api/Interfaces/IModelService.cs b/backend/api/api/Interfaces/IModelService.cs index 00299979..8c4543de 100644 --- a/backend/api/api/Interfaces/IModelService.cs +++ b/backend/api/api/Interfaces/IModelService.cs @@ -3,14 +3,14 @@ using api.Models; namespace api.Services { - public interface IModelService - { + public interface IModelService + { Model GetOneModel(string userId, string name); Model GetOneModel(string id); List<Model> GetMyModels(string userId); List<Model> GetMyModelsByType(string userId, string problemType); List<Model> GetLatestModels(string userId); - //List<Model> GetPublicModels(); + List<Model> GetPublicModels(); Model Create(Model model); Model Replace(Model model); void Update(string userId, string name, Model model); diff --git a/backend/api/api/Models/ColumnInfo.cs b/backend/api/api/Models/ColumnInfo.cs index dcf5171c..46aeadb6 100644 --- a/backend/api/api/Models/ColumnInfo.cs +++ b/backend/api/api/Models/ColumnInfo.cs @@ -4,10 +4,9 @@ { public ColumnInfo() { } - public ColumnInfo(string columnName, string columnType, bool isNumber, int numNulls, float mean, float min, float max, float median, string[] uniqueValues, int[]uniqueValuesCount, float[] uniqueValuesPercent, float q1, float q3) + public ColumnInfo(string columnName, bool isNumber, int numNulls, float mean, float min, float max, float median, string[] uniqueValues, int[]uniqueValuesCount, float[] uniqueValuesPercent, float q1, float q3) { this.columnName = columnName; - this.columnType = columnType; this.isNumber = isNumber; this.numNulls = numNulls; this.mean = mean; @@ -22,7 +21,6 @@ } public string columnName { get; set; } - public string columnType { get; set; } public bool isNumber { get; set; } public int numNulls { get; set; } public float mean { get; set; } diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index 7acd4382..beb66de9 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -28,6 +28,7 @@ namespace api.Models public int nullRows { get; set; } public bool isPreProcess { get; set; } + public float[][] cMatrix { get; set; } } } diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs index 3af063be..d98e6cfc 100644 --- a/backend/api/api/Models/Experiment.cs +++ b/backend/api/api/Models/Experiment.cs @@ -21,6 +21,6 @@ namespace api.Models public DateTime lastUpdated { get; set; } public NullValues[] nullValuesReplacers { get; set; } public ColumnEncoding[] encodings { get; set; } - + public string[] columnTypes { get; set; } } } diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs index d8921713..a2740ca9 100644 --- a/backend/api/api/Models/Model.cs +++ b/backend/api/api/Models/Model.cs @@ -27,7 +27,8 @@ namespace api.Models public string lossFunction { get; set; } //public int inputNeurons { get; set; } public int hiddenLayers { get; set; } - public int batchSize { get; set; } + public string batchSize { get; set; } + public string learningRate { get; set; } // na izlazu je moguce da bude vise neurona (klasifikacioni problem sa vise od 2 klase) public int outputNeurons { get; set; } public Layer[] layers { get; set; } @@ -40,6 +41,8 @@ namespace api.Models public bool randomOrder { get; set; } public bool randomTestSet { get; set; } public float randomTestSetDistribution { get; set; } + public bool isPublic { get; set; } + public bool accessibleByLink { get; set; } } public class Layer diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs index f39cac29..f38a363b 100644 --- a/backend/api/api/Services/DatasetService.cs +++ b/backend/api/api/Services/DatasetService.cs @@ -1,4 +1,5 @@ -using api.Interfaces; +using api.Controllers; +using api.Interfaces; using api.Models; using MongoDB.Driver; @@ -7,11 +8,15 @@ namespace api.Services public class DatasetService : IDatasetService { private readonly IMongoCollection<Dataset> _dataset; + private readonly IMongoCollection<Experiment> _experiment; + private readonly IExperimentService _experimentService; - public DatasetService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) + public DatasetService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient, IExperimentService experimentService) { var database = mongoClient.GetDatabase(settings.DatabaseName); _dataset = database.GetCollection<Dataset>(settings.DatasetCollectionName); + _experiment = database.GetCollection<Experiment>(settings.ExperimentCollectionName); + _experimentService = experimentService; } public List<Dataset> SearchDatasets(string name) @@ -29,7 +34,13 @@ namespace api.Services //brisanje odredjenog name-a public void Delete(string userId, string id) { - _dataset.DeleteOne(dataset => (dataset.uploaderId == userId && dataset._id == id)); + List<Experiment> experiment = null; + _dataset.DeleteOne(dataset => (dataset.uploaderId == userId && dataset._id == id)); + + experiment = _experiment.Find(experiment => (experiment.datasetId == id && experiment.uploaderId == userId)).ToList(); + + foreach (Experiment experimentItem in experiment) + _experimentService.Delete(userId, experimentItem._id); } public List<Dataset> GetMyDatasets(string userId) @@ -39,7 +50,7 @@ namespace api.Services public List<Dataset> GetGuestDatasets() { //Join Igranonica public datasetove sa svim temp uploadanim datasetovima - List<Dataset> datasets= _dataset.Find(dataset => dataset.uploaderId == "000000000000000000000000" && dataset.isPublic == true && dataset.isPreProcess).ToList(); + List<Dataset> datasets = _dataset.Find(dataset => dataset.uploaderId == "000000000000000000000000" && dataset.isPublic == true && dataset.isPreProcess).ToList(); datasets.AddRange(_dataset.Find(dataset => dataset.uploaderId == "" && dataset.isPreProcess).ToList()); return datasets; } @@ -49,7 +60,7 @@ namespace api.Services { List<Dataset> list = _dataset.Find(dataset => dataset.uploaderId == userId && dataset.isPreProcess).ToList(); - if(ascdsc) + if (ascdsc) list = list.OrderBy(dataset => dataset.lastUpdated).ToList(); else list = list.OrderByDescending(dataset => dataset.lastUpdated).ToList(); @@ -62,7 +73,11 @@ namespace api.Services return _dataset.Find(dataset => dataset.isPublic == true && dataset.isPreProcess).ToList(); } - public Dataset GetOneDataset(string userId, string name) + public Dataset GetOneDataset(string userId, string id) + { + return _dataset.Find(dataset => dataset.uploaderId == userId && dataset._id == id && dataset.isPreProcess).FirstOrDefault(); + } + public Dataset GetOneDatasetN(string userId, string name) { return _dataset.Find(dataset => dataset.uploaderId == userId && dataset.name == name && dataset.isPreProcess).FirstOrDefault(); } @@ -74,13 +89,13 @@ namespace api.Services } //ako je potrebno da se zameni name ili ekstenzija - public void Update(string userId, string id, Dataset dataset ) + public void Update(string userId, string id, Dataset dataset) { _dataset.ReplaceOne(dataset => dataset.uploaderId == userId && dataset._id == id, dataset); } public void Update(Dataset dataset) { - _dataset.ReplaceOne(x=>x._id==dataset._id, dataset); + _dataset.ReplaceOne(x => x._id == dataset._id, dataset); } public string GetDatasetId(string fileId) diff --git a/backend/api/api/Services/ExperimentService.cs b/backend/api/api/Services/ExperimentService.cs index 539e4c08..dde9111d 100644 --- a/backend/api/api/Services/ExperimentService.cs +++ b/backend/api/api/Services/ExperimentService.cs @@ -7,10 +7,13 @@ namespace api.Services public class ExperimentService : IExperimentService { private readonly IMongoCollection<Experiment> _experiment; + private readonly IMongoCollection<Predictor> _predictor; + public ExperimentService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) { var database = mongoClient.GetDatabase(settings.DatabaseName); _experiment = database.GetCollection<Experiment>(settings.ExperimentCollectionName); + _predictor = database.GetCollection<Predictor>(settings.PredictorCollectionName); } public Experiment Create(Experiment experiment) @@ -20,31 +23,38 @@ namespace api.Services } public Experiment Get(string id) { - return _experiment.Find(exp=>exp._id == id).FirstOrDefault(); + return _experiment.Find(exp => exp._id == id).FirstOrDefault(); } public Experiment Get(string uploaderId, string name) { - return _experiment.Find(exp => exp.uploaderId == uploaderId && exp.name == name ).FirstOrDefault(); + return _experiment.Find(exp => exp.uploaderId == uploaderId && exp.name == name).FirstOrDefault(); } public void Update(string id, Experiment experiment) { _experiment.ReplaceOne(experiment => experiment._id == id, experiment); } - public List<Experiment> GetMyExperiments(string id) + public List<Experiment> GetMyExperiments(string userId) { - return _experiment.Find(e=>e.uploaderId==id).ToList(); + return _experiment.Find(experiment => experiment.uploaderId == userId).ToList(); } - public Experiment GetOneExperiment(string userId, string name) + public Experiment GetOneExperiment(string userId, string id) { - return _experiment.Find(experiment => experiment.uploaderId == userId && experiment.name == name).FirstOrDefault(); + return _experiment.Find(experiment => experiment.uploaderId == userId && experiment._id == id).FirstOrDefault(); } - public void Update(string userId, string id, Experiment experiment) + public Experiment Update(string userId, string id, Experiment experiment) { _experiment.ReplaceOne(experiment => experiment.uploaderId == userId && experiment._id == id, experiment); + return _experiment.Find(experiment => experiment.uploaderId == userId && experiment._id == id).FirstOrDefault(); + } + + public void Delete(string userId, string id) + { + _experiment.DeleteOne(experiment => (experiment.uploaderId == userId && experiment._id == id)); + _predictor.DeleteMany(predictor => (predictor.uploaderId == userId && predictor.experimentId == id)); } } } diff --git a/backend/api/api/Services/FillAnEmptyDb.cs b/backend/api/api/Services/FillAnEmptyDb.cs index 99bbb91f..520f3461 100644 --- a/backend/api/api/Services/FillAnEmptyDb.cs +++ b/backend/api/api/Services/FillAnEmptyDb.cs @@ -19,9 +19,9 @@ namespace api.Services var database = mongoClient.GetDatabase(settings.DatabaseName); _fileService = new FileService(settings, mongoClient); - _datasetService = new DatasetService(settings, mongoClient); - _modelService = new ModelService(settings, mongoClient); _experimentService = new ExperimentService(settings, mongoClient); + _datasetService = new DatasetService(settings, mongoClient, _experimentService); + _modelService = new ModelService(settings, mongoClient); _predictorService = new PredictorService(settings, mongoClient); } @@ -66,18 +66,18 @@ namespace api.Services dataset.columnInfo = new[] { - new ColumnInfo( "PassengerId", "columnType", true, 0, 446, 1, 891, 446, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Survived", "columnType", true, 0, 0.38383838534355164f, 0, 1, 0, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Pclass", "columnType", true, 0, 2.3086419105529785f, 1, 3, 3, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Name", "columnType", false, 0, 0, 0, 0, 0, new string[]{"Braund, Mr. Owen Harris", "Boulos, Mr. Hanna", "Frolicher-Stehli, Mr. Maxmillian", "Gilinski, Mr. Eliezer", "Murdlin, Mr. Joseph", "Rintamaki, Mr. Matti", "Stephenson, Mrs. Walter Bertram (Martha Eustis)", "Elsbury, Mr. William James", "Bourke, Miss. Mary", "Chapman, Mr. John Henry"}, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Sex", "columnType", false, 0, 0, 0, 0, 0, new string[]{ "male", "female" }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Age", "columnType", true, 177, 29.69911766052246f, 0.41999998688697815f, 80, 28, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "SibSp", "columnType", true, 0, 0.523007869720459f, 0, 8, 0, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Parch", "columnType", true, 0, 0.3815937042236328f, 0, 6, 0, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Ticket", "columnType", false, 0, 0, 0, 0, 0, new string[]{ "347082", "CA. 2343", "1601", "3101295", "CA 2144", "347088", "S.O.C. 14879", "382652", "LINE", "PC 17757" }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Fare", "columnType", true, 0, 32.20420837402344f, 0, 512.3292236328125f, 14.45419979095459f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Cabin", "columnType", false, 687, 0, 0, 0, 0, new string[]{ "B96 B98", "G6", "C23 C25 C27", "C22 C26", "F33", "F2", "E101", "D", "C78", "C93" }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "Embarked", "columnType", false, 2, 0.3815937042236328f, 0, 6, 0, new string[]{ "S", "C", "Q" }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "PassengerId", true, 0, 446, 1, 891, 446, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Survived", true, 0, 0.38383838534355164f, 0, 1, 0, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Pclass", true, 0, 2.3086419105529785f, 1, 3, 3, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Name", false, 0, 0, 0, 0, 0, new string[]{"Braund, Mr. Owen Harris", "Boulos, Mr. Hanna", "Frolicher-Stehli, Mr. Maxmillian", "Gilinski, Mr. Eliezer", "Murdlin, Mr. Joseph", "Rintamaki, Mr. Matti", "Stephenson, Mrs. Walter Bertram (Martha Eustis)", "Elsbury, Mr. William James", "Bourke, Miss. Mary", "Chapman, Mr. John Henry"}, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Sex", false, 0, 0, 0, 0, 0, new string[]{ "male", "female" }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Age", true, 177, 29.69911766052246f, 0.41999998688697815f, 80, 28, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "SibSp", true, 0, 0.523007869720459f, 0, 8, 0, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Parch", true, 0, 0.3815937042236328f, 0, 6, 0, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Ticket", false, 0, 0, 0, 0, 0, new string[]{ "347082", "CA. 2343", "1601", "3101295", "CA 2144", "347088", "S.O.C. 14879", "382652", "LINE", "PC 17757" }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Fare", true, 0, 32.20420837402344f, 0, 512.3292236328125f, 14.45419979095459f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Cabin", false, 687, 0, 0, 0, 0, new string[]{ "B96 B98", "G6", "C23 C25 C27", "C22 C26", "F33", "F2", "E101", "D", "C78", "C93" }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "Embarked", false, 2, 0.3815937042236328f, 0, 6, 0, new string[]{ "S", "C", "Q" }, new int[] {}, new float[] {}, 0.01f,0.1f ), }; dataset.rowCount = 891; dataset.nullCols = 3; @@ -99,11 +99,13 @@ namespace api.Services model.optimizer = "Adam"; model.lossFunction = "mean_squared_error"; model.hiddenLayers = 5; - model.batchSize = 8; + model.batchSize = "8"; model.outputNeurons = 0; model.outputLayerActivationFunction = "sigmoid"; model.metrics = new string[] { }; model.epochs = 5; + model.isPublic = true; + model.accessibleByLink = true; _modelService.Create(model); @@ -116,7 +118,7 @@ namespace api.Services experiment.ModelIds = new string[] { }.ToList(); experiment.datasetId = _datasetService.GetDatasetId(dataset.fileId); experiment.uploaderId = "000000000000000000000000"; - experiment.inputColumns = new string[] { "Embarked" }; + experiment.inputColumns = new string[] { "Embarked", "Survived" }; experiment.outputColumn = "Survived"; experiment.dateCreated = DateTime.Now; experiment.lastUpdated = DateTime.Now; @@ -127,6 +129,7 @@ namespace api.Services new ColumnEncoding( "Survived", "label" ), new ColumnEncoding("Embarked", "label" ) }; + experiment.columnTypes = new string[] {"categorical"}; _experimentService.Create(experiment); @@ -177,17 +180,17 @@ namespace api.Services dataset.delimiter = ""; dataset.columnInfo = new[] { - new ColumnInfo( "Unnamed: 0", "columnType", true, 0, 26969.5f, 0, 53939, 26969.5f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "carat", "columnType", true, 0, 0.7979397773742676f, 0.20000000298023224f, 5.010000228881836f, 0.699999988079071f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "cut", "columnType", false, 0, 0, 0, 0, 0, new string[]{ "Ideal", "Premium", "Very Good", "Good", "Fair" }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "color", "columnType", false, 0, 0, 0, 0, 0, new string[]{"G", "E", "F", "H", "D", "I", "I", "J"}, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "clarity", "columnType", false, 0, 0, 0, 0, 0, new string[]{ "SI1", "VS2","SI2", "VS1", "VVS2", "VVS1", "IF", "I1" }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "depth", "columnType", true, 0, 61.74940490722656f, 43, 79, 61.79999923706055f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "table", "columnType", true, 0, 57.457183837890625f, 43, 95, 57, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "price", "columnType", true, 0, 3932.7998046875f, 326, 18823, 2401, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "x", "columnType", true, 0, 5.731157302856445f, 0, 10.739999771118164f, 5.699999809265137f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "y", "columnType", true, 0, 5.73452615737915f, 0, 58.900001525878906f, 5.710000038146973f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "z", "columnType", true, 0, 3.538733720779419f, 0, 31.799999237060547f, 3.5299999713897705f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ) + new ColumnInfo( "Unnamed: 0", true, 0, 26969.5f, 0, 53939, 26969.5f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "carat", true, 0, 0.7979397773742676f, 0.20000000298023224f, 5.010000228881836f, 0.699999988079071f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "cut", false, 0, 0, 0, 0, 0, new string[]{ "Ideal", "Premium", "Very Good", "Good", "Fair" }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "color", false, 0, 0, 0, 0, 0, new string[]{"G", "E", "F", "H", "D", "I", "I", "J"}, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "clarity", false, 0, 0, 0, 0, 0, new string[]{ "SI1", "VS2","SI2", "VS1", "VVS2", "VVS1", "IF", "I1" }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "depth", true, 0, 61.74940490722656f, 43, 79, 61.79999923706055f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "table", true, 0, 57.457183837890625f, 43, 95, 57, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "price", true, 0, 3932.7998046875f, 326, 18823, 2401, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "x", true, 0, 5.731157302856445f, 0, 10.739999771118164f, 5.699999809265137f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "y", true, 0, 5.73452615737915f, 0, 58.900001525878906f, 5.710000038146973f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "z", true, 0, 3.538733720779419f, 0, 31.799999237060547f, 3.5299999713897705f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ) }; dataset.rowCount = 53940; dataset.nullCols = 0; @@ -210,11 +213,13 @@ namespace api.Services model.optimizer = "Adam"; model.lossFunction = "mean_absolute_error"; model.hiddenLayers = 4; - model.batchSize = 5; + model.batchSize = "8"; model.outputNeurons = 0; model.outputLayerActivationFunction = "relu"; model.metrics = new string[] { }; model.epochs = 5; + model.isPublic = true; + model.accessibleByLink = true; _modelService.Create(model); @@ -227,7 +232,7 @@ namespace api.Services experiment.ModelIds = new string[] { }.ToList(); experiment.datasetId = _datasetService.GetDatasetId(dataset.fileId); experiment.uploaderId = "000000000000000000000000"; - experiment.inputColumns = new string[] { "Unnamed: 0", "carat", "cut", "color", "clarity", "depth", "table", "x", "y", "z" }; + experiment.inputColumns = new string[] { "Unnamed: 0", "carat", "cut", "color", "clarity", "depth", "table", "x", "y", "z", "price" }; experiment.outputColumn = "price"; experiment.dateCreated = DateTime.Now; experiment.lastUpdated = DateTime.Now; @@ -247,6 +252,7 @@ namespace api.Services new ColumnEncoding( "y", "label" ), new ColumnEncoding( "z", "label" ) }; + experiment.columnTypes = new string[] { "categorical", "numerical", "numerical", "categorical", "categorical", "categorical", "numerical", "numerical", "numerical", "numerical", "numerical" }; _experimentService.Create(experiment); /* @@ -291,11 +297,11 @@ namespace api.Services dataset.delimiter = ""; dataset.columnInfo = new[] { - new ColumnInfo( "sepal_length", "columnType", true, 0, 5.8433332443237305f, 4.300000190734863f, 7.900000095367432f, 5.800000190734863f, new string[]{ }, new int[] {}, new float[] {}, 0.01f, 0.1f ), - new ColumnInfo( "sepal_width", "columnType", true, 0, 3.053999900817871f, 2, 4.400000095367432f, 3, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "petal_length", "columnType", true, 0, 3.758666753768921f, 1, 6.900000095367432f, 4.349999904632568f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "petal_width", "columnType", true, 0, 1.1986666917800903f, 0.10000000149011612f, 2.5f, 1.2999999523162842f, new string[]{}, new int[] {}, new float[] {}, 0.01f,0.1f ), - new ColumnInfo( "class", "columnType", false, 0, 0, 0, 0, 0, new string[]{ "Iris-setosa", "Iris-versicolor", "Iris-virginica" }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "sepal_length", true, 0, 5.8433332443237305f, 4.300000190734863f, 7.900000095367432f, 5.800000190734863f, new string[]{ }, new int[] {}, new float[] {}, 0.01f, 0.1f ), + new ColumnInfo( "sepal_width", true, 0, 3.053999900817871f, 2, 4.400000095367432f, 3, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "petal_length", true, 0, 3.758666753768921f, 1, 6.900000095367432f, 4.349999904632568f, new string[]{ }, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "petal_width", true, 0, 1.1986666917800903f, 0.10000000149011612f, 2.5f, 1.2999999523162842f, new string[]{}, new int[] {}, new float[] {}, 0.01f,0.1f ), + new ColumnInfo( "class", false, 0, 0, 0, 0, 0, new string[]{ "Iris-setosa", "Iris-versicolor", "Iris-virginica" }, new int[] {}, new float[] {}, 0.01f,0.1f ), }; dataset.nullCols = 150; dataset.nullRows = 0; @@ -316,11 +322,13 @@ namespace api.Services model.optimizer = "Adam"; model.lossFunction = "sparse_categorical_crossentropy"; model.hiddenLayers = 3; - model.batchSize = 4; + model.batchSize = "64"; model.outputNeurons = 0; model.outputLayerActivationFunction = "softmax"; model.metrics = new string[] { }; model.epochs = 1; + model.isPublic = true; + model.accessibleByLink = true; _modelService.Create(model); @@ -333,7 +341,7 @@ namespace api.Services experiment.ModelIds = new string[] { }.ToList(); experiment.datasetId = _datasetService.GetDatasetId(dataset.fileId); experiment.uploaderId = "000000000000000000000000"; - experiment.inputColumns = new string[] { "sepal_length", "sepal_width", "petal_length", "petal_width" }; + experiment.inputColumns = new string[] { "sepal_length", "sepal_width", "petal_length", "petal_width", "class" }; experiment.outputColumn = "class"; experiment.dateCreated = DateTime.Now; experiment.lastUpdated = DateTime.Now; @@ -347,6 +355,8 @@ namespace api.Services new ColumnEncoding( "petal_width", "label" ), new ColumnEncoding( "class", "label" ) }; + experiment.columnTypes = new string[] { "categorical", "numerical", "numerical", "numerical", "categorical" }; + _experimentService.Create(experiment); /* diff --git a/backend/api/api/Services/ModelService.cs b/backend/api/api/Services/ModelService.cs index 12297635..e852d71f 100644 --- a/backend/api/api/Services/ModelService.cs +++ b/backend/api/api/Services/ModelService.cs @@ -5,14 +5,16 @@ using MongoDB.Driver; namespace api.Services { - public class ModelService : IModelService + public class ModelService : IModelService { private readonly IMongoCollection<Model> _model; + private readonly IMongoCollection<Predictor> _predictor; public ModelService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) { var database = mongoClient.GetDatabase(settings.DatabaseName); _model = database.GetCollection<Model>(settings.ModelCollectionName); + _predictor = database.GetCollection<Predictor>(settings.PredictorCollectionName); } public Model Create(Model model) @@ -28,7 +30,11 @@ namespace api.Services public void Delete(string userId, string name) { + Model model = _model.Find(model => model.uploaderId == userId && model.name == name).FirstOrDefault(); + _model.DeleteOne(model => (model.uploaderId == userId && model.name == name)); + _predictor.DeleteMany(predictor => (predictor.uploaderId == userId && predictor.modelId == model._id)); + } public List<Model> GetMyModels(string userId) @@ -48,12 +54,12 @@ namespace api.Services return list; } - /* + public List<Model> GetPublicModels() { return _model.Find(model => model.isPublic == true).ToList(); } - */ + public Model GetOneModel(string userId, string name) { return _model.Find(model => model.uploaderId == userId && model.name == name).FirstOrDefault(); @@ -93,7 +99,7 @@ namespace api.Services return false; else return true; - + } } } diff --git a/backend/microservice/api/controller.py b/backend/microservice/api/controller.py index 41035cc7..988ad987 100644 --- a/backend/microservice/api/controller.py +++ b/backend/microservice/api/controller.py @@ -118,7 +118,6 @@ def returnColumnsInfo(): ''' preprocess = newmlservice.returnColumnsInfo(data) #samo 10 jedinstvenih posto ih ima previse, bilo bi dobro da promenimo ovo da to budu 10 najzastupljenijih vrednosti - for col in preprocess["columnInfo"]: col["uniqueValues"] = col["uniqueValues"][0:6] col["uniqueValuesCount"] = col["uniqueValuesCount"][0:6] @@ -128,11 +127,9 @@ def returnColumnsInfo(): dataset["nullRows"] = preprocess["allNullRows"] dataset["colCount"] = preprocess["colCount"] dataset["rowCount"] = preprocess["rowCount"] + dataset["cMatrix"]=preprocess['cMatrix'] dataset["isPreProcess"] = True - #print(dataset) - - - + return jsonify(dataset) print("App loaded.") diff --git a/backend/microservice/api/newmlservice.py b/backend/microservice/api/newmlservice.py index 9e26b03a..85d8fb71 100644 --- a/backend/microservice/api/newmlservice.py +++ b/backend/microservice/api/newmlservice.py @@ -27,15 +27,28 @@ import matplotlib.pyplot as plt #from ann_visualizer.visualize import ann_viz; def returnColumnsInfo(dataset): dict=[] + datafront=dataset.copy() + dataMatrix=dataset.copy() + + svekolone=datafront.columns kategorijskekolone=datafront.select_dtypes(include=['object']).columns + allNullCols=0 rowCount=datafront.shape[0]#ukupan broj redova colCount=len(datafront.columns)#ukupan broj kolona for kolona in svekolone: if(kolona in kategorijskekolone): + encoder=LabelEncoder() + dataMatrix[kolona]=encoder.fit_transform(dataMatrix[kolona]) + + #print(dataMatrix.dtypes) + cMatrix=dataMatrix.corr() + + for kolona in svekolone: + if(kolona in kategorijskekolone): unique=datafront[kolona].value_counts() uniquevalues=[] uniquevaluescount=[] @@ -86,7 +99,7 @@ def returnColumnsInfo(dataset): #pretvaranje u kategorijsku datafront = datafront.astype({kolona: str}) - print(datafront.dtypes) + #print(datafront.dtypes) unique=datafront[kolona].value_counts() uniquevaluesn=[] uniquevaluescountn=[] @@ -117,7 +130,9 @@ def returnColumnsInfo(dataset): #print(NullRows) #print(len(NullRows)) allNullRows=len(NullRows) - return {'columnInfo':dict,'allNullColl':int(allNullCols),'allNullRows':int(allNullRows),'rowCount':int(rowCount),'colCount':int(colCount)} + #print(cMatrix.to_json(orient='index')) + #json.loads()['data'] + return {'columnInfo':dict,'allNullColl':int(allNullCols),'allNullRows':int(allNullRows),'rowCount':int(rowCount),'colCount':int(colCount),'cMatrix':json.loads(cMatrix.to_json(orient='split'))['data']} @dataclass class TrainingResultClassification: @@ -165,17 +180,20 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): kategorijskekolone=[] ###PRETVARANJE NUMERICKIH U KATREGORIJSKE AKO JE KORISNIK TAKO OZNACIO columnInfo=paramsDataset['columnInfo'] - for col in columnInfo: - if(col['columnType']=="Kategorijski"): + columnTypes=paramsExperiment['columnTypes'] + for i in range(len(columnInfo)): + col=columnInfo[i] + if(columnTypes[i]=="categorical"): data[col['columnName']]=data[col['columnName']].apply(str) - kategorijskekolone.append(col['coumnName']) - + kategorijskekolone.append(col['columnName']) + #kategorijskekolone=data.select_dtypes(include=['object']).columns + print(kategorijskekolone) ###NULL null_value_options = paramsExperiment["nullValues"] null_values_replacers = paramsExperiment["nullValuesReplacers"] if(null_value_options=='replace'): - #print("replace null") # + #print("replace null") dict=null_values_replacers while(len(dict)>0): replace=dict.pop() @@ -267,8 +285,8 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): # # Podela na test i trening skupove # - test=paramsExperiment["randomTestSetDistribution"] - randomOrder = paramsExperiment["randomOrder"] + test=paramsModel["randomTestSetDistribution"] + randomOrder = paramsModel["randomOrder"] if(randomOrder): random=123 else: @@ -314,8 +332,8 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): if(problem_type=='multi-klasifikacioni'): #print('multi') - reg=paramsModel['regularisation'][0] - regRate=float(paramsModel['regularisationRate'][0]) + reg=paramsModel['layers'][0]['regularisation'] + regRate=float(paramsModel['layers'][0]['regularisationRate']) if(reg=='l1'): kernelreg=tf.keras.regularizers.l1(regRate) biasreg=tf.keras.regularizers.l1(regRate) @@ -326,12 +344,12 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): activityreg=tf.keras.regularizers.l2(regRate) classifier=tf.keras.Sequential() - classifier.add(tf.keras.layers.Dense(units=paramsModel['hiddenLayerNeurons'], activation=paramsModel['hiddenLayerActivationFunctions'][0],input_dim=x_train.shape[1], kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#prvi skriveni + definisanje prethodnog-ulaznog + classifier.add(tf.keras.layers.Dense(units=paramsModel['layers'][0]['neurons'], activation=paramsModel['layers'][0]['activationFunction'],input_dim=x_train.shape[1], kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#prvi skriveni + definisanje prethodnog-ulaznog for i in range(paramsModel['hiddenLayers']-1):#ako postoji vise od jednog skrivenog sloja ###Kernel - reg=paramsModel['regularisation'][i+1] - regRate=float(paramsModel['regularisationRate'][i+1]) + reg=paramsModel['layers'][i+1]['regularisation'] + regRate=float(paramsModel['layers'][i+1]['regularisationRate']) if(reg=='l1'): kernelreg=tf.keras.regularizers.l1(regRate) biasreg=tf.keras.regularizers.l1(regRate) @@ -341,7 +359,7 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): biasreg=tf.keras.regularizers.l2(regRate) activityreg=tf.keras.regularizers.l2(regRate) - classifier.add(tf.keras.layers.Dense(units=paramsModel['hiddenLayerNeurons'], activation=paramsModel['hiddenLayerActivationFunctions'][i+1],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#i-ti skriveni sloj + classifier.add(tf.keras.layers.Dense(units=paramsModel['layers'][i+1]['neurons'], activation=paramsModel['layers'][i+1]['activationFunction'],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#i-ti skriveni sloj classifier.add(tf.keras.layers.Dense(units=5, activation=paramsModel['outputLayerActivationFunction']))#izlazni sloj @@ -349,7 +367,7 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): classifier.compile(loss =paramsModel["lossFunction"] , optimizer = opt, metrics =paramsModel['metrics']) - history=classifier.fit(x_train, y_train, epochs = paramsModel['epochs'],batch_size=float(paramsModel['batchSize']),callbacks=callback(x_test, y_test,paramsModel['_id'])) + history=classifier.fit(x_train, y_train, epochs = paramsModel['epochs'],batch_size=int(paramsModel['batchSize']),callbacks=callback(x_test, y_test,paramsModel['_id'])) hist=history.history #plt.plot(hist['accuracy']) @@ -371,8 +389,8 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): elif(problem_type=='binarni-klasifikacioni'): #print('*************************************************************************binarni') - reg=paramsModel['regularisation'][0] - regRate=float(paramsModel['regularisationRate'][0]) + reg=paramsModel['layers'][0]['regularisation'] + regRate=float(paramsModel['layers'][0]['regularisationRate']) if(reg=='l1'): kernelreg=tf.keras.regularizers.l1(regRate) biasreg=tf.keras.regularizers.l1(regRate) @@ -383,12 +401,12 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): activityreg=tf.keras.regularizers.l2(regRate) classifier=tf.keras.Sequential() - classifier.add(tf.keras.layers.Dense(units=paramsModel['hiddenLayerNeurons'], activation=paramsModel['hiddenLayerActivationFunctions'][0],input_dim=x_train.shape[1],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#prvi skriveni + definisanje prethodnog-ulaznog + classifier.add(tf.keras.layers.Dense(units=paramsModel['layers'][0]['neurons'], activation=paramsModel['layers'][0]['activationFunction'],input_dim=x_train.shape[1],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#prvi skriveni + definisanje prethodnog-ulaznog for i in range(paramsModel['hiddenLayers']-1):#ako postoji vise od jednog skrivenog sloja #print(i) - reg=paramsModel['regularisation'][i+1] - regRate=float(paramsModel['regularisationRate'][i+1]) + reg=paramsModel['layers'][i+1]['regularisation'] + regRate=float(paramsModel['layers'][0]['regularisationRate']) if(reg=='l1'): kernelreg=tf.keras.regularizers.l1(regRate) biasreg=tf.keras.regularizers.l1(regRate) @@ -397,13 +415,13 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): kernelreg=tf.keras.regularizers.l2(regRate) biasreg=tf.keras.regularizers.l2(regRate) activityreg=tf.keras.regularizers.l2(regRate) - classifier.add(tf.keras.layers.Dense(units=paramsModel['hiddenLayerNeurons'], activation=paramsModel['hiddenLayerActivationFunctions'][i+1],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#i-ti skriveni sloj + classifier.add(tf.keras.layers.Dense(units=paramsModel['layers'][i+1]['neurons'], activation=paramsModel['layers'][i+1]['activationFunction'],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#i-ti skriveni sloj classifier.add(tf.keras.layers.Dense(units=1, activation=paramsModel['outputLayerActivationFunction']))#izlazni sloj classifier.compile(loss =paramsModel["lossFunction"] , optimizer = opt , metrics =paramsModel['metrics']) - history=classifier.fit(x_train, y_train, epochs = paramsModel['epochs'],batch_size=float(paramsModel['batchSize']),callbacks=callback(x_test, y_test,paramsModel['_id'])) + history=classifier.fit(x_train, y_train, epochs = paramsModel['epochs'],batch_size=int(paramsModel['batchSize']),callbacks=callback(x_test, y_test,paramsModel['_id'])) hist=history.history y_pred=classifier.predict(x_test) y_pred=(y_pred>=0.5).astype('int') @@ -419,8 +437,8 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): return filepath,hist elif(problem_type=='regresioni'): - reg=paramsModel['regularisation'][0] - regRate=float(paramsModel['regularisationRate'][0]) + reg=paramsModel['layers'][0]['regularisation'] + regRate=float(paramsModel['layers'][0]['regularisationRate']) if(reg=='l1'): kernelreg=tf.keras.regularizers.l1(regRate) biasreg=tf.keras.regularizers.l1(regRate) @@ -431,12 +449,12 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): activityreg=tf.keras.regularizers.l2(regRate) classifier=tf.keras.Sequential() - classifier.add(tf.keras.layers.Dense(units=paramsModel['hiddenLayerNeurons'], activation=paramsModel['hiddenLayerActivationFunctions'][0],input_dim=x_train.shape[1],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#prvi skriveni + definisanje prethodnog-ulaznog + classifier.add(tf.keras.layers.Dense(units=paramsModel['layers'][0]['neurons'], activation=paramsModel['layers'][0]['activationFunction'],input_dim=x_train.shape[1],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#prvi skriveni + definisanje prethodnog-ulaznog for i in range(paramsModel['hiddenLayers']-1):#ako postoji vise od jednog skrivenog sloja #print(i) - reg=paramsModel['regularisation'][i+1] - regRate=float(paramsModel['regularisationRate'][i+1]) + reg=paramsModel['layers'][i+1]['regularisation'] + regRate=float(paramsModel['layers'][i+1]['regularisationRate']) if(reg=='l1'): kernelreg=tf.keras.regularizers.l1(regRate) biasreg=tf.keras.regularizers.l1(regRate) @@ -446,13 +464,13 @@ def train(dataset, paramsModel,paramsExperiment,paramsDataset,callback): biasreg=tf.keras.regularizers.l2(regRate) activityreg=tf.keras.regularizers.l2(regRate) - classifier.add(tf.keras.layers.Dense(units=paramsModel['hiddenLayerNeurons'], activation=paramsModel['hiddenLayerActivationFunctions'][i+1],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#i-ti skriveni sloj + classifier.add(tf.keras.layers.Dense(units=paramsModel['layers'][i+1]['neurons'], activation=paramsModel['layers'][i+1]['activationFunction'],kernel_regularizer=kernelreg, bias_regularizer=biasreg, activity_regularizer=activityreg))#i-ti skriveni sloj - classifier.add(tf.keras.layers.Dense(units=1),activation=paramsModel['outputLayerActivationFunction']) + classifier.add(tf.keras.layers.Dense(units=1,activation=paramsModel['outputLayerActivationFunction'])) classifier.compile(loss =paramsModel["lossFunction"] , optimizer = opt , metrics =paramsModel['metrics']) - history=classifier.fit(x_train, y_train, epochs = paramsModel['epochs'],batch_size=float(paramsModel['batchSize']),callbacks=callback(x_test, y_test,paramsModel['_id'])) + history=classifier.fit(x_train, y_train, epochs = paramsModel['epochs'],batch_size=int(paramsModel['batchSize']),callbacks=callback(x_test, y_test,paramsModel['_id'])) hist=history.history y_pred=classifier.predict(x_test) #print(classifier.evaluate(x_test, y_test)) @@ -622,7 +640,7 @@ def manageH5(dataset,params,h5model): h5model.compile(loss=params['lossFunction'], optimizer=params['optimizer'], metrics=params['metrics']) - history=h5model.fit(x2, y2, epochs = params['epochs'],batch_size=params['batchSize']) + history=h5model.fit(x2, y2, epochs = params['epochs'],batch_size=int(params['batchSize'])) y_pred2=h5model.predict(x2) diff --git a/frontend/src/app/_data/Dataset.ts b/frontend/src/app/_data/Dataset.ts index 7ae5c4ab..4ff0a471 100644 --- a/frontend/src/app/_data/Dataset.ts +++ b/frontend/src/app/_data/Dataset.ts @@ -18,7 +18,7 @@ export default class Dataset extends FolderFile { public rowCount: number = 0, public nullRows: number = 0, public nullCols: number = 0, - public preview: string[][] = [[]] + public cMatrix: number[][] = [] ) { super(name, dateCreated, lastUpdated); } @@ -27,7 +27,6 @@ export default class Dataset extends FolderFile { export class ColumnInfo { constructor( public columnName: string = '', - public columnType: ColumnType, public isNumber: boolean = false, public numNulls: number = 0, public uniqueValues?: string[], @@ -39,16 +38,12 @@ export class ColumnInfo { public max?: number, public q1?: number, public q3?: number, - ) { + ) { /*if (isNumber) this.columnType = ColumnType.numerical; else this.columnType = ColumnType.categorical;*/ } - + } -export enum ColumnType { - categorical = "Kategorijski", - numerical = "Numerički" -}
\ No newline at end of file diff --git a/frontend/src/app/_data/Experiment.ts b/frontend/src/app/_data/Experiment.ts index 31816c19..cff77535 100644 --- a/frontend/src/app/_data/Experiment.ts +++ b/frontend/src/app/_data/Experiment.ts @@ -14,11 +14,11 @@ export default class Experiment { public dateCreated: Date = new Date(), public lastUpdated: Date = new Date(), public modelIds: string[] = [], - - - + public columnTypes: ColumnType[] = [], public encodings: ColumnEncoding[] = []//[{columnName: "", columnEncoding: Encoding.Label}] ) { } + + _columnsSelected: boolean = false; } export enum NullValueOptions { @@ -44,11 +44,11 @@ export class NullValReplacer { export enum Encoding { Label = 'label', OneHot = 'onehot', - Ordinal = 'ordinal', + /*Ordinal = 'ordinal', Hashing = 'hashing', Binary = 'binary', BaseN = 'baseN' - /* + BackwardDifference = 'backward difference', CatBoost = 'cat boost', Count = 'count', @@ -66,9 +66,13 @@ export enum Encoding { } export class ColumnEncoding { - constructor ( + constructor( public columnName: string, public encoding: Encoding - ) - {} + ) { } +} + +export enum ColumnType { + categorical = "categorical", + numerical = "numerical" }
\ No newline at end of file diff --git a/frontend/src/app/_data/Model.ts b/frontend/src/app/_data/Model.ts index 185e2257..526a8290 100644 --- a/frontend/src/app/_data/Model.ts +++ b/frontend/src/app/_data/Model.ts @@ -28,8 +28,10 @@ export default class Model extends FolderFile { // Test set settings public randomOrder: boolean = true, public randomTestSet: boolean = true, - public randomTestSetDistribution: number = 0.1 //0.1-0.9 (10% - 90%) JESTE OVDE ZAKUCANO 10, AL POSLATO JE KAO 0.1 BACK-U + public randomTestSetDistribution: number = 0.1, //0.1-0.9 (10% - 90%) JESTE OVDE ZAKUCANO 10, AL POSLATO JE KAO 0.1 BACK-U + public isPublic: boolean = false, + public accessibleByLink: boolean = false ) { super(name, dateCreated, lastUpdated); } diff --git a/frontend/src/app/_elements/column-table/column-table.component.html b/frontend/src/app/_elements/column-table/column-table.component.html index 543a0018..efc093d2 100644 --- a/frontend/src/app/_elements/column-table/column-table.component.html +++ b/frontend/src/app/_elements/column-table/column-table.component.html @@ -64,15 +64,15 @@ </tr> </thead> <tbody> - <tr *ngFor="let colInfo of dataset.columnInfo; let i = index"> + <tr *ngFor="let row of dataset.cMatrix; let i = index"> <th [ngClass]="{'header-disabled col-disabled' : !columnsChecked[i]}"> <div class="text-left"> - {{colInfo.columnName}} + {{dataset.columnInfo[i].columnName}} </div> </th> - <td *ngFor="let colInfo of dataset.columnInfo; let j = index" [ngClass]="{'text-disabled col-disabled' : !columnsChecked[j] || !columnsChecked[i]}"> + <td *ngFor="let corrVal of row; let j = index" [ngClass]="{'text-disabled col-disabled' : !columnsChecked[j] || !columnsChecked[i]}"> <div class="text-overflow"> - 0.1 + {{corrVal}} </div> </td> </tr> @@ -103,7 +103,7 @@ <td *ngFor="let colInfo of dataset.columnInfo; let i = index" class="pad-fix" [ngClass]="{'text-disabled' : !columnsChecked[i]}"> <p class="verticalAlign text-left" style="font-size:13px;" *ngIf="!colInfo.isNumber">Kategorijski</p> <mat-form-field *ngIf="colInfo.isNumber"> - <mat-select matNativeControl [(value)]="colInfo.columnType" [disabled]="!columnsChecked[i]" (selectionChange)="columnTypeChanged(colInfo.columnName);"> + <mat-select matNativeControl [(value)]="this.experiment.columnTypes[i]" [disabled]="!columnsChecked[i]" (selectionChange)="columnTypeChanged(colInfo.columnName);"> <mat-option [value]="ColumnType.categorical">Kategorijski</mat-option> <mat-option [value]="ColumnType.numerical">Numerički</mat-option> </mat-select> @@ -113,14 +113,14 @@ <tr class="graphics-row"> <th class="no-pad">Grafik</th> <td class="no-pad" *ngFor="let colInfo of dataset.columnInfo; let i = index" [ngClass]="{'graphic-class' : !columnsChecked[i]}"> - <app-box-plot *ngIf="colInfo.columnType == ColumnType.numerical" [width]="150" [height]="150"></app-box-plot> - <app-pie-chart *ngIf="colInfo.columnType == ColumnType.categorical" [width]="150" [height]="150"></app-pie-chart> + <app-box-plot *ngIf="this.experiment.columnTypes[i] == ColumnType.numerical" [width]="150" [height]="150"></app-box-plot> + <app-pie-chart *ngIf="this.experiment.columnTypes[i] == ColumnType.categorical" [width]="150" [height]="150"></app-pie-chart> </td> </tr> <tr> <th class="border-bottom">Statistika</th> <td *ngFor="let colInfo of dataset.columnInfo; let i = index" [ngClass]="{'text-disabled' : !columnsChecked[i]}" class="text-left"> - <span *ngIf="colInfo.columnType == ColumnType.numerical"> + <span *ngIf="this.experiment.columnTypes[i] == ColumnType.numerical"> Mean: {{colInfo.mean}}<br> Median: {{colInfo.median}}<br> Min: {{colInfo.min}}<br> @@ -128,7 +128,7 @@ Q1: {{colInfo.q1}}<br> Q3: {{colInfo.q3}}<br> </span> - <div class="text-overflow" *ngIf="colInfo.columnType == ColumnType.categorical && colInfo.uniqueValuesPercent"> + <div class="text-overflow" *ngIf="this.experiment.columnTypes[i] == ColumnType.categorical && colInfo.uniqueValuesPercent"> <span *ngFor="let uniqueValue of colInfo.uniqueValues | slice:0:6; let i = index"> ({{(colInfo.uniqueValuesPercent[i] * 100).toFixed(2)}}%) {{uniqueValue}}<br> </span> @@ -186,7 +186,7 @@ </mat-menu> <mat-menu #replaceWith="matMenu"> - <input type="text" id={{colInfo.columnName}} mat-menu-item placeholder="Unesi vrednost..." [value]> + <input type="text" id={{colInfo.columnName}} mat-menu-item placeholder="Unesi vrednost..." [value] #enterAValue> <button [disabled]="getValue(colInfo.columnName) == ''" mat-menu-item value={{getValue(colInfo.columnName)}} (click)="MissValsReplaceClicked($event, colInfo.columnName, i)">Potvrdi unos</button> </mat-menu> </div> @@ -225,15 +225,24 @@ </div> <div class="break-1"></div> <div class="ns-col d-flex align-items-center justify-content-center"> - <button mat-button (click)="saveExperiment()" class="bottom-button text-offwhite rounded-bottom"> - <div class="f-row" style="justify-content: space-around;"> - <div>Potvrdi</div> - <div class="icon-double pt-1"> - <mat-icon>check</mat-icon> - <mat-icon>check</mat-icon> + <button *ngIf="experiment._id == ''" mat-button (click)="saveExperiment()" class="bottom-button text-offwhite rounded-bottom"> + <div class="f-row" style="justify-content: space-around; width: 100%;"> + <div>Sačuvaj</div> + <div class="icon-double pt-1"> + <mat-icon>check</mat-icon> + <mat-icon>check</mat-icon> + </div> </div> - </div> - </button> + </button> + <button *ngIf="experiment._id != ''" mat-button (click)="updateExperiment()" class="bottom-button text-offwhite rounded-bottom"> + <div class="f-row" style="justify-content: space-around; width: 100%;"> + <div>Sačuvaj izmene</div> + <div class="icon-double pt-1"> + <mat-icon>check</mat-icon> + <mat-icon>check</mat-icon> + </div> + </div> + </button> </div> </div> </div> diff --git a/frontend/src/app/_elements/column-table/column-table.component.ts b/frontend/src/app/_elements/column-table/column-table.component.ts index c3d4f206..c200e674 100644 --- a/frontend/src/app/_elements/column-table/column-table.component.ts +++ b/frontend/src/app/_elements/column-table/column-table.component.ts @@ -1,6 +1,6 @@ import { AfterViewInit, Component, ElementRef, EventEmitter, Input, OnInit, Output, ViewChildren } from '@angular/core'; -import Dataset, { ColumnType } from 'src/app/_data/Dataset'; -import Experiment, { ColumnEncoding, Encoding, NullValReplacer, NullValueOptions } from 'src/app/_data/Experiment'; +import Dataset from 'src/app/_data/Dataset'; +import Experiment, { ColumnEncoding, Encoding, ColumnType, NullValueOptions } from 'src/app/_data/Experiment'; import { DatasetsService } from 'src/app/_services/datasets.service'; import { EncodingDialogComponent } from 'src/app/_modals/encoding-dialog/encoding-dialog.component'; import { MatDialog } from '@angular/material/dialog'; @@ -10,6 +10,8 @@ import { CsvParseService } from 'src/app/_services/csv-parse.service'; import { ProblemType } from 'src/app/_data/Model'; import { ExperimentsService } from 'src/app/_services/experiments.service'; import { SaveExperimentDialogComponent } from 'src/app/_modals/save-experiment-dialog/save-experiment-dialog.component'; +import { AlertDialogComponent } from 'src/app/_modals/alert-dialog/alert-dialog.component'; +import Shared from 'src/app/Shared'; @Component({ selector: 'app-column-table', @@ -20,7 +22,6 @@ export class ColumnTableComponent implements AfterViewInit { @Input() dataset?: Dataset; @Input() experiment!: Experiment; - @ViewChildren("nullValMenu") nullValMenus!: ElementRef[]; @Output() okPressed: EventEmitter<string> = new EventEmitter(); @Output() columnTableChanged = new EventEmitter(); @@ -35,7 +36,7 @@ export class ColumnTableComponent implements AfterViewInit { columnsChecked: boolean[] = []; //niz svih kolona loaded: boolean = false; - + constructor(private datasetService: DatasetsService, private experimentService: ExperimentsService, public csvParseService: CsvParseService, public dialog: MatDialog) { //ovo mi nece trebati jer primam dataset iz druge komponente } @@ -44,15 +45,13 @@ export class ColumnTableComponent implements AfterViewInit { this.dataset = dataset; this.setColumnTypeInitial(); - + this.dataset.columnInfo.forEach(column => { this.columnsChecked.push(true); }); - - for (let i = 0; i < this.dataset?.columnInfo.length; i++) { - this.experiment.inputColumns.push(this.dataset.columnInfo[i].columnName); - } - this.experiment.outputColumn = this.experiment.inputColumns[0]; + + this.resetInputColumns(); + this.resetOutputColumn(); this.resetColumnEncodings(Encoding.Label); this.setDeleteRowsForMissingValTreatment(); @@ -70,17 +69,29 @@ export class ColumnTableComponent implements AfterViewInit { } ngAfterViewInit(): void { - + } setColumnTypeInitial() { if (this.dataset != undefined) { for (let i = 0; i < this.dataset.columnInfo.length; i++) { - this.dataset.columnInfo[i].columnType = (this.dataset.columnInfo[i].isNumber) ? ColumnType.numerical : ColumnType.categorical; + this.experiment.columnTypes[i] = (this.dataset.columnInfo[i].isNumber) ? ColumnType.numerical : ColumnType.categorical; } } } + resetInputColumns() { + if (this.dataset != undefined) { + this.experiment.inputColumns = []; + for (let i = 0; i < this.dataset?.columnInfo.length; i++) { + this.experiment.inputColumns.push(this.dataset.columnInfo[i].columnName); + } + } + } + resetOutputColumn() { + this.experiment.outputColumn = this.experiment.inputColumns[0]; + } + setDeleteRowsForMissingValTreatment() { if (this.experiment != undefined) { this.experiment.nullValues = NullValueOptions.DeleteRows; @@ -102,7 +113,7 @@ export class ColumnTableComponent implements AfterViewInit { columnTypeChanged(columnName: string) { if (this.experiment.outputColumn == columnName) this.changeOutputColumn(columnName); - else + else this.columnTableChangeDetected(); } @@ -129,14 +140,14 @@ export class ColumnTableComponent implements AfterViewInit { changeOutputColumn(columnName: string) { if (this.experiment != undefined && this.dataset != undefined) { - let column = this.dataset.columnInfo.filter(x => x.columnName == this.experiment!.outputColumn)[0]; - if (column.columnType == ColumnType.numerical) { + let i = this.dataset.columnInfo.findIndex(x => x.columnName == this.experiment!.outputColumn); + if (this.experiment.columnTypes[i] == ColumnType.numerical) { this.experiment.type = ProblemType.Regression; } else { - if (column.uniqueValues!.length == 2) + if (this.dataset.columnInfo[i].uniqueValues!.length == 2) this.experiment.type = ProblemType.BinaryClassification; - else + else this.experiment.type = ProblemType.MultiClassification; } this.columnTableChangeDetected(); @@ -188,7 +199,7 @@ export class ColumnTableComponent implements AfterViewInit { value: "" }); let numOfRowsToDelete = (this.dataset.columnInfo.filter(x => x.columnName == this.experiment!.inputColumns[i])[0]).numNulls; - this.nullValOption[i] = "Obriši redove (" + numOfRowsToDelete + ")"; + this.nullValOption[i] = "Obriši redove (" + numOfRowsToDelete + ")"; } } this.columnTableChangeDetected(); @@ -212,13 +223,18 @@ export class ColumnTableComponent implements AfterViewInit { this.experiment.name = selectedName; //napravi odvojene dugmice za save i update -> za update nece da se otvara dijalog za ime this.experimentService.addExperiment(this.experiment).subscribe((response) => { - console.log(response); + this.experiment._id = response._id; this.okPressed.emit(); }); }); } - + openUpdateExperimentDialog() { + this.experimentService.updateExperiment(this.experiment).subscribe((response) => { + this.experiment = response; + Shared.openDialog("Izmena eksperimenta", "Uspešno ste izmenili podatke o eksperimentu."); + }); + } MissValsDeleteClicked(event: Event, replacementType: NullValueOptions, index: number) { if (this.experiment != undefined && this.dataset != undefined) { @@ -272,6 +288,9 @@ export class ColumnTableComponent implements AfterViewInit { saveExperiment() { this.openSaveExperimentDialog(); } + updateExperiment() { + this.openUpdateExperimentDialog(); + } tabs = [ diff --git a/frontend/src/app/_elements/folder/folder.component.css b/frontend/src/app/_elements/folder/folder.component.css index 2340ee8a..62324d62 100644 --- a/frontend/src/app/_elements/folder/folder.component.css +++ b/frontend/src/app/_elements/folder/folder.component.css @@ -152,6 +152,14 @@ display: none; } +.hover-show { + display: none; +} + +.list-item:hover>.hover-show { + display: initial; +} + .list-add { display: flex; flex-direction: row; diff --git a/frontend/src/app/_elements/folder/folder.component.html b/frontend/src/app/_elements/folder/folder.component.html index 48b59dc8..bff066be 100644 --- a/frontend/src/app/_elements/folder/folder.component.html +++ b/frontend/src/app/_elements/folder/folder.component.html @@ -56,15 +56,15 @@ <div class="folder-inside bg-blur"> <div class="file-content" [ngClass]="{'form-hidden' : listView}"> <div class="file-bottom-buttons" *ngIf="selectedTab != TabType.NewFile"> - <button *ngIf="this.selectedFile && selectedTab == TabType.File" class="btn-clear file-button" (click)="deleteFile(this.selectedFile)"> + <button *ngIf="this.selectedFile && selectedTab == TabType.File" class="btn-clear file-button" (click)="deleteFile(this.selectedFile, $event)"> <mat-icon>delete</mat-icon> </button> <!-- <button class="btn-clear file-button"> <mat-icon>zoom_out_map</mat-icon> </button> --> </div> - <app-form-model [ngClass]="{'form-hidden': type != FolderType.Model}"></app-form-model> - <app-form-dataset [ngClass]="{'form-hidden': type != FolderType.Dataset}"></app-form-dataset> + <app-form-model [ngClass]="{'form-hidden': type != FolderType.Model}" [forExperiment]="forExperiment"></app-form-model> + <app-form-dataset [ngClass]="{'form-hidden': type != FolderType.Dataset}" [forExperiment]="forExperiment"></app-form-dataset> </div> <div [ngClass]="{'form-hidden' : !listView}" class="list-view"> <div *ngFor="let file of filteredFiles; let i = index" class="list-item force-link" (click)="selectFile(file)"> @@ -74,6 +74,12 @@ <div class="mx-2 hover-hide"> {{file.lastUpdated | date}} </div> + <div class="mx-2 hover-show" *ngIf="selectedTab !== TabType.PublicDatasets && selectedTab !== TabType.PublicModels"> + <button class="btn-clear file-button" (click)="deleteFile(file, $event)"> + <mat-icon>delete</mat-icon> + </button> + </div> + </div> <div class="list-add" [ngSwitch]="type"> <button mat-raised-button *ngSwitchCase="FolderType.Dataset" (click)="selectNewFile()">Dodaj {{privacy == Privacy.Public ? 'javni ' : ' '}}izvor podataka</button> diff --git a/frontend/src/app/_elements/folder/folder.component.ts b/frontend/src/app/_elements/folder/folder.component.ts index 20ca1121..6ca0faa8 100644 --- a/frontend/src/app/_elements/folder/folder.component.ts +++ b/frontend/src/app/_elements/folder/folder.component.ts @@ -53,8 +53,9 @@ export class FolderComponent implements AfterViewInit { if (this.signalRService.hubConnection) { this.signalRService.hubConnection.on("NotifyDataset", (dName: string, dId: string) => { - this.refreshFiles(dId); - + if (this.type == FolderType.Dataset) { + this.refreshFiles(dId); + } }); } else { console.warn("Dataset-Load: No connection!"); @@ -98,8 +99,8 @@ export class FolderComponent implements AfterViewInit { this.newFileSelected = false; this.listView = false; this.selectedFileChanged.emit(this.selectedFile); - this.displayFile(); this.selectTab(TabType.File); + this.displayFile(); } createNewFile() { @@ -117,9 +118,13 @@ export class FolderComponent implements AfterViewInit { _initialized: boolean = false; refreshFiles(selectedDatasetId: string | null) { + this.files = [] + this.filteredFiles.length = 0; + this.folders[TabType.NewFile] = []; + this.folders[TabType.File] = []; this.tabsToShow.forEach(tab => { this.folders[tab] = []; - }) + }); this.datasetsService.getMyDatasets().subscribe((datasets) => { this.folders[TabType.MyDatasets] = datasets; @@ -137,7 +142,6 @@ export class FolderComponent implements AfterViewInit { }); this.modelsService.getMyModels().subscribe((models) => { - console.log(models); this.folders[TabType.MyModels] = models; }); @@ -164,8 +168,9 @@ export class FolderComponent implements AfterViewInit { switch (this.type) { case FolderType.Dataset: this.formDataset!.uploadDataset((dataset: Dataset) => { + this.newFile = undefined; Shared.openDialog("Obaveštenje", "Uspešno ste dodali novi izvor podataka u kolekciju. Molimo sačekajte par trenutaka da se procesira."); - this.refreshFiles(dataset._id); + this.refreshFiles(null); }, () => { Shared.openDialog("Neuspeo pokušaj!", "Izvor podataka sa unetim nazivom već postoji u Vašoj kolekciji. Izmenite naziv ili iskoristite postojeći dataset."); @@ -173,7 +178,7 @@ export class FolderComponent implements AfterViewInit { break; case FolderType.Model: this.modelsService.addModel(this.formModel.newModel).subscribe(model => { - this.formModel.newModel = model; + this.newFile = undefined; Shared.openDialog("Obaveštenje", "Uspešno ste dodali novu konfiguraciju neuronske mreže u kolekciju."); this.refreshFiles(null); // todo select model }, (err) => { @@ -205,8 +210,8 @@ export class FolderComponent implements AfterViewInit { filteredFiles: FolderFile[] = []; searchTermsChanged() { - if (!this.files) return; this.filteredFiles.length = 0; + if (!this.files) return; this.filteredFiles.push(...this.files.filter((file) => file.name.toLowerCase().includes(this.searchTerm.toLowerCase()))); /*if (this.selectedFile) { if (!this.filteredFiles.includes(this.selectedFile)) { @@ -226,17 +231,19 @@ export class FolderComponent implements AfterViewInit { this.listView = !this.listView; } - deleteFile(file: FolderFile) { - console.log('delete'); + deleteFile(file: FolderFile, event: Event) { + event.stopPropagation(); + //console.log('delete'); switch (this.type) { case FolderType.Dataset: this.datasetsService.deleteDataset(<Dataset>file).subscribe((response) => { - console.log(response); + this.filteredFiles.splice(this.filteredFiles.indexOf(file), 1); + this.refreshFiles(null); }); break; case FolderType.Model: this.modelsService.deleteModel(<Model>file).subscribe((response) => { - console.log(response); + this.refreshFiles(null); }); break; case FolderType.Experiment: diff --git a/frontend/src/app/_elements/form-model/form-model.component.html b/frontend/src/app/_elements/form-model/form-model.component.html index c0318012..96a5e1b6 100644 --- a/frontend/src/app/_elements/form-model/form-model.component.html +++ b/frontend/src/app/_elements/form-model/form-model.component.html @@ -108,7 +108,9 @@ </div> <div class="col-sm-9"> - <app-graph [model]="newModel"></app-graph> + <!-- {{forExperiment._columnsSelected}} --> + <app-graph [model]="newModel" *ngIf="forExperiment._columnsSelected" [inputColumns]="forExperiment.inputColumns"></app-graph> + <app-graph [model]="newModel" *ngIf="!forExperiment._columnsSelected" [inputColumns]="['Nisu odabrane ulazne kolone']"></app-graph> </div> </div> </div> diff --git a/frontend/src/app/_elements/graph/graph.component.html b/frontend/src/app/_elements/graph/graph.component.html index 3c01bc5e..35753d40 100644 --- a/frontend/src/app/_elements/graph/graph.component.html +++ b/frontend/src/app/_elements/graph/graph.component.html @@ -1,8 +1,8 @@ <div #graphWrapper class="w-100 position-relative" style="height: 14rem;"> - <ng-container *ngFor="let layer of layers; let i = index"> + <!-- <ng-container *ngFor="let layer of layers; let i = index"> <div [ngClass]="{'inputs': i==0}" class="node-text" *ngFor="let node of layer; let j = index" [style.left.%]="node.x * 99.4" [style.top.%]="node.y * 100"> - {{ i == 0 ? (inputColumns ? inputColumns[j] : 'nepoznato') : (i > 0 && i + {{ i == 0 ? (inputColumns && inputColumns.length >= j ? inputColumns[j] : 'nepoznato') : (i > 0 && i < layers.length - 1 ? model!.layers[i-1].activationFunction : (i==layers.length - 1 ? 'out' : '')) }} </div> - </ng-container> + </ng-container> --> <canvas #graphCanvas></canvas> - </div>
\ No newline at end of file +</div>
\ No newline at end of file diff --git a/frontend/src/app/_elements/graph/graph.component.ts b/frontend/src/app/_elements/graph/graph.component.ts index c7f8d964..da2c7767 100644 --- a/frontend/src/app/_elements/graph/graph.component.ts +++ b/frontend/src/app/_elements/graph/graph.component.ts @@ -28,7 +28,7 @@ export class GraphComponent implements AfterViewInit { @Input() outputNodeColor: string = '#dfd7d7'; private ctx!: CanvasRenderingContext2D; - @Input() inputColumns?: string[] = ['Nije odabran eksperiment']; + @Input() inputColumns?: string[]; constructor() { } @@ -43,7 +43,7 @@ export class GraphComponent implements AfterViewInit { window.addEventListener('resize', () => { this.resize() }); this.update(); this.resize(); - console.log(this.layers); + //console.log(this.layers); } layers: Node[][] = []; diff --git a/frontend/src/app/_elements/metric-view/metric-view.component.ts b/frontend/src/app/_elements/metric-view/metric-view.component.ts index 3840692a..6fd2f320 100644 --- a/frontend/src/app/_elements/metric-view/metric-view.component.ts +++ b/frontend/src/app/_elements/metric-view/metric-view.component.ts @@ -28,7 +28,7 @@ export class MetricViewComponent implements OnInit { myEpochs.push(epoch + 1); for (let key in metrics) { let value = metrics[key]; - console.log(key, ':::', value, epoch); + //console.log(key, ':::', value, epoch); if (key === 'accuracy') { myAcc.push(parseFloat(value)); } diff --git a/frontend/src/app/_elements/notifications/notifications.component.ts b/frontend/src/app/_elements/notifications/notifications.component.ts index d64530b9..5716c1e6 100644 --- a/frontend/src/app/_elements/notifications/notifications.component.ts +++ b/frontend/src/app/_elements/notifications/notifications.component.ts @@ -24,8 +24,8 @@ export class NotificationsComponent implements OnInit { this.signalRService.hubConnection.on("NotifyEpoch", (mName: string, mId: string, stat: string, totalEpochs: number, currentEpoch: number) => { const existingNotification = this.notifications.find(x => x.id === mId) const progress = ((currentEpoch + 1) / totalEpochs); - console.log("Ukupno epoha", totalEpochs, "Trenutna epoha:", currentEpoch); - console.log("stat:", stat); + //console.log("Ukupno epoha", totalEpochs, "Trenutna epoha:", currentEpoch); + //console.log("stat:", stat); if (!existingNotification) this.notifications.push(new Notification(`Treniranje modela: ${mName}`, mId, progress, true)); else { diff --git a/frontend/src/app/_elements/playlist/playlist.component.ts b/frontend/src/app/_elements/playlist/playlist.component.ts index 7529b36b..7f476178 100644 --- a/frontend/src/app/_elements/playlist/playlist.component.ts +++ b/frontend/src/app/_elements/playlist/playlist.component.ts @@ -44,6 +44,6 @@ export class PlaylistComponent implements OnInit { } }); - console.log(this.tableDatas); + //console.log(this.tableDatas); } } diff --git a/frontend/src/app/_modals/save-experiment-dialog/save-experiment-dialog.component.html b/frontend/src/app/_modals/save-experiment-dialog/save-experiment-dialog.component.html index a0b5d771..bac73e0a 100644 --- a/frontend/src/app/_modals/save-experiment-dialog/save-experiment-dialog.component.html +++ b/frontend/src/app/_modals/save-experiment-dialog/save-experiment-dialog.component.html @@ -1,10 +1,11 @@ <h1 mat-dialog-title>Čuvanje eksperimenta</h1> <div mat-dialog-content> - <p>Unesite naziv eksperimenta:</p> + <span>Unesi naziv eksperimenta:</span> <mat-form-field> <input type="text" matInput [(ngModel)]="selectedName"> </mat-form-field> - <p>Da li ste sigurni u izbor?</p> + <br><br> + <p>Sačuvaj eksperiment:</p> </div> <div mat-dialog-actions> <button mat-button [mat-dialog-close]="selectedName" cdkFocusInitial>Da</button> diff --git a/frontend/src/app/_pages/experiment/experiment.component.html b/frontend/src/app/_pages/experiment/experiment.component.html index baae864e..2b32db81 100644 --- a/frontend/src/app/_pages/experiment/experiment.component.html +++ b/frontend/src/app/_pages/experiment/experiment.component.html @@ -32,17 +32,17 @@ </div> <div #steps id="step_2" class="step-content"> <div class="step-content-inside"> - <app-column-table (okPressed)="goToPage(2)" (columnTableChanged)="columnTableChangedEvent()" [experiment]="experiment" [dataset]="dataset"></app-column-table> + <app-column-table (okPressed)="goToPage(2); experiment._columnsSelected = true;" (columnTableChanged)="columnTableChangedEvent()" [experiment]="experiment" [dataset]="dataset"></app-column-table> </div> </div> <div #steps id="step_3" class="step-content"> <div class="step-content-inside"> - <app-folder #folderModel [type]="FolderType.Model" [forExperiment]="experiment" [startingTab]="TabType.NewFile" [tabsToShow]="[TabType.MyModels]" (okPressed)="goToPage(3)"></app-folder> + <app-folder #folderModel [type]="FolderType.Model" [forExperiment]="experiment" [startingTab]="TabType.NewFile" [tabsToShow]="[TabType.MyModels]" (okPressed)="goToPage(3); trainModel();" (selectedFileChanged)="setModel($event)"></app-folder> </div> </div> <div #steps id="step_4" class="step-content"> <div class="step-content-inside"> - <app-metric-view></app-metric-view> + <app-metric-view #metricView></app-metric-view> </div> </div> </div> diff --git a/frontend/src/app/_pages/experiment/experiment.component.ts b/frontend/src/app/_pages/experiment/experiment.component.ts index 28552664..c4d6063c 100644 --- a/frontend/src/app/_pages/experiment/experiment.component.ts +++ b/frontend/src/app/_pages/experiment/experiment.component.ts @@ -10,6 +10,8 @@ import { ModelsService } from 'src/app/_services/models.service'; import Model from 'src/app/_data/Model'; import Dataset from 'src/app/_data/Dataset'; import { ColumnTableComponent } from 'src/app/_elements/column-table/column-table.component'; +import { SignalRService } from 'src/app/_services/signal-r.service'; +import { MetricViewComponent } from 'src/app/_elements/metric-view/metric-view.component'; @Component({ selector: 'app-experiment', @@ -26,11 +28,11 @@ export class ExperimentComponent implements AfterViewInit { experiment: Experiment; dataset?: Dataset; @ViewChild("folderDataset") folderDataset!: FolderComponent; - @ViewChild("folderModel") folderModel!: FolderComponent; @ViewChild(ColumnTableComponent) columnTable!: ColumnTableComponent; + @ViewChild("folderModel") folderModel!: FolderComponent; + @ViewChild("metricView") metricView!: MetricViewComponent; - - constructor(private experimentsService: ExperimentsService, private modelsService: ModelsService) { + constructor(private experimentsService: ExperimentsService, private modelsService: ModelsService, private signalRService: SignalRService) { this.experiment = new Experiment("exp1"); } @@ -43,7 +45,11 @@ export class ExperimentComponent implements AfterViewInit { } trainModel() { - this.modelsService.trainModel((<Model>this.folderModel.selectedFile)._id, this.experiment._id).subscribe(() => { console.log("pocelo treniranje") }) + if (!this.modelToTrain) { + Shared.openDialog('Greška', 'Morate odabrati konfiguraciju neuronske mreže'); + } else { + this.modelsService.trainModel(this.modelToTrain._id, this.experiment._id).subscribe(() => { console.log("pocelo treniranje") }); + } } stepHeight = this.calcStepHeight(); @@ -65,8 +71,23 @@ export class ExperimentComponent implements AfterViewInit { this.stepsContainer.nativeElement.addEventListener('scroll', (event: Event) => { Shared.emitBGScrollEvent(this.stepsContainer.nativeElement.scrollTop); }); + + if (this.signalRService.hubConnection) { + this.signalRService.hubConnection.on("NotifyEpoch", (mName: string, mId: string, stat: string, totalEpochs: number, currentEpoch: number) => { + console.log(this.modelToTrain?._id, mId); + if (this.modelToTrain?._id == mId) { + stat = stat.replace(/'/g, '"'); + //console.log('JSON', this.trainingResult); + this.history.push(JSON.parse(stat)); + this.metricView.update(this.history); + } + }); + + } } + history: any[] = []; + updatePageIfScrolled() { if (this.scrolling) return; const currentPage = Math.round(this.stepsContainer.nativeElement.scrollTop / this.stepHeight) @@ -120,7 +141,7 @@ export class ExperimentComponent implements AfterViewInit { columnTableChangedEvent() { //sta se desi kad se nesto promeni u column-table komponenti... - console.log("promenio se column-table"); + //console.log("promenio se column-table"); } setDataset(dataset: FolderFile) { @@ -130,4 +151,11 @@ export class ExperimentComponent implements AfterViewInit { this.columnTable.loadDataset(this.dataset); } + + modelToTrain?: Model; + + setModel(model: FolderFile) { + const m = <Model>model; + this.modelToTrain = m; + } } diff --git a/frontend/src/app/_services/models.service.ts b/frontend/src/app/_services/models.service.ts index d79e2781..fc888556 100644 --- a/frontend/src/app/_services/models.service.ts +++ b/frontend/src/app/_services/models.service.ts @@ -53,4 +53,9 @@ export class ModelsService { deleteModel(model: Model) { return this.http.delete(`${Configuration.settings.apiURL}/model/` + model.name, { headers: this.authService.authHeader(), responseType: "text" }); } + + getPublicModels(): Observable<Model[]> { + return this.http.get<Model[]>(`${Configuration.settings.apiURL}/model/publicmodels`, { headers: this.authService.authHeader() }); + } + } |