diff options
Diffstat (limited to 'backend/api')
21 files changed, 247 insertions, 44 deletions
diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index 8a622138..6eb1b9e6 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -14,11 +14,15 @@ namespace api.Controllers public class DatasetController : ControllerBase { private readonly IDatasetService _datasetService; + private readonly IMlConnectionService _mlConnectionService; + private readonly IFileService _fileService; private IJwtToken jwtToken; - public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token) + public DatasetController(IDatasetService datasetService, IConfiguration configuration,IJwtToken Token,IMlConnectionService mlConnectionService, IFileService fileService) { _datasetService = datasetService; + _mlConnectionService = mlConnectionService; + _fileService = fileService; jwtToken = Token; } @@ -135,9 +139,8 @@ namespace api.Controllers return BadRequest(); var dataset = _datasetService.GetOneDataset(username, name); - if (dataset == null) - return NotFound($"Dataset with name = {name} not found or dataset is not public"); + return NotFound($"Dataset with name = {name} not found or dataset is not public or not preprocessed"); return dataset; } @@ -159,7 +162,7 @@ namespace api.Controllers // POST api/<DatasetController>/add [HttpPost("add")] [Authorize(Roles = "User,Guest")] - public ActionResult<Dataset> Post([FromBody] Dataset dataset) + public async Task<ActionResult<Dataset>> Post([FromBody] Dataset dataset) { //da li ce preko tokena da se ubaci username ili front salje //dataset.username = usernameToken; @@ -170,9 +173,12 @@ namespace api.Controllers return NotFound($"Dateset with name = {dataset.name} exisits"); else { - _datasetService.Create(dataset); - return CreatedAtAction(nameof(Get), new { id = dataset._id }, dataset); + FileModel fileModel = _fileService.getFile(dataset.fileId); + dataset.isPreProcess = false; + _datasetService.Create(dataset); + _mlConnectionService.PreProcess(dataset,fileModel.path); + return Ok(); } } diff --git a/backend/api/api/Controllers/ExperimentController.cs b/backend/api/api/Controllers/ExperimentController.cs new file mode 100644 index 00000000..2a2db31e --- /dev/null +++ b/backend/api/api/Controllers/ExperimentController.cs @@ -0,0 +1,75 @@ +using System.Net.Http.Headers; +using api.Models; +using api.Services; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Net.Http.Headers; + +namespace api.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class ExperimentController : ControllerBase + { + + private readonly IExperimentService _experimentService; + private IJwtToken jwtToken; + + public ExperimentController(IExperimentService experimentService, IConfiguration configuration, IJwtToken Token) + { + _experimentService = experimentService; + jwtToken = Token; + } + + [HttpPost("add")] + [Authorize(Roles = "User,Guest")] + public async Task<ActionResult<Experiment>> Post([FromBody] Experiment experiment) + { + string uploaderId; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + uploaderId = jwtToken.TokenToId(parameter); + if (uploaderId == null) + return null; + } + else + return BadRequest(); + + experiment.uploaderId = uploaderId; + + _experimentService.Create(experiment); + return Ok(experiment); + } + + [HttpGet("get")] + [Authorize(Roles = "User,Guest")] + public async Task<ActionResult<Experiment>> Get(string id) + { + string uploaderId; + var header = Request.Headers[HeaderNames.Authorization]; + if (AuthenticationHeaderValue.TryParse(header, out var headerValue)) + { + var scheme = headerValue.Scheme; + var parameter = headerValue.Parameter; + uploaderId = jwtToken.TokenToId(parameter); + if (uploaderId == null) + return null; + } + else + return BadRequest(); + + var experiment = _experimentService.Get(id); + if(experiment.uploaderId!=uploaderId) + return BadRequest("Not your experiment"); + + return Ok(experiment); + } + + + + } +} diff --git a/backend/api/api/Controllers/ModelController.cs b/backend/api/api/Controllers/ModelController.cs index 210d37c8..97e7c7a7 100644 --- a/backend/api/api/Controllers/ModelController.cs +++ b/backend/api/api/Controllers/ModelController.cs @@ -17,15 +17,17 @@ namespace api.Controllers private readonly IDatasetService _datasetService; private readonly IFileService _fileService; private readonly IModelService _modelService; + private readonly IExperimentService _experimentService; private IJwtToken jwtToken; - public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration,IJwtToken token) + public ModelController(IMlConnectionService mlService, IModelService modelService, IDatasetService datasetService, IFileService fileService, IConfiguration configuration,IJwtToken token,IExperimentService experiment) { _mlService = mlService; _modelService = modelService; _datasetService = datasetService; _fileService = fileService; + _experimentService = experiment; jwtToken = token; } @@ -45,7 +47,8 @@ namespace api.Controllers } else return BadRequest(); - var dataset = _datasetService.GetOneDataset(model.datasetId); + var experiment=_experimentService.Get(model.experimentId); + var dataset = _datasetService.GetOneDataset(experiment.datasetId); var filepath = _fileService.GetFilePath(dataset.fileId, uploaderId); var result = await _mlService.SendModelAsync(model, filepath); return Ok(result); @@ -145,7 +148,8 @@ namespace api.Controllers { bool overwrite = false; //username="" ako je GUEST - model.inputNeurons = model.inputColumns.Length; + Experiment e = _experimentService.Get(model.experimentId); + model.inputNeurons = e.inputColumns.Length; if (_modelService.CheckHyperparameters(model.inputNeurons, model.hiddenLayerNeurons, model.hiddenLayers, model.outputNeurons) == false) return BadRequest("Bad parameters!"); diff --git a/backend/api/api/Data/UserStoreDatabaseSettings.cs b/backend/api/api/Data/UserStoreDatabaseSettings.cs index e83d2b54..822f9bf5 100644 --- a/backend/api/api/Data/UserStoreDatabaseSettings.cs +++ b/backend/api/api/Data/UserStoreDatabaseSettings.cs @@ -13,5 +13,6 @@ namespace api.Data public string PredictorCollectionName { get; set; } = String.Empty; public string ModelCollectionName { get; set; } = String.Empty; public string FilesCollectionName { get; set; } = String.Empty; + public string ExperimentCollectionName { get; set; } = String.Empty; } } diff --git a/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs b/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs index a5b5f5eb..05c41701 100644 --- a/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs +++ b/backend/api/api/Interfaces/IUserStoreDatabaseSettings.cs @@ -9,5 +9,6 @@ string PredictorCollectionName { get; set; } string ModelCollectionName { get; set; } string FilesCollectionName { get; set; } + string ExperimentCollectionName { get; set; } } } diff --git a/backend/api/api/Models/ColumnInfo.cs b/backend/api/api/Models/ColumnInfo.cs new file mode 100644 index 00000000..ee4cee0d --- /dev/null +++ b/backend/api/api/Models/ColumnInfo.cs @@ -0,0 +1,13 @@ +namespace api.Models +{ + public class ColumnInfo + { + public string columnName { get; set; } + public bool isNumber { get; set; } + public int numNulls { get; set; } + public float mean { get; set; } + public float median { get; set; } + public string[] uniqueValues { get; set; } + + } +} diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index dcfde3b1..2b3efa3c 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -22,6 +22,11 @@ namespace api.Models public DateTime lastUpdated { get; set; } public string delimiter { get; set; } public bool hasHeader { get; set; } + + public ColumnInfo[] columnInfo { get; set; } + public int totalNulls; + public bool isPreProcess { get; set; } + } } diff --git a/backend/api/api/Models/Experiment.cs b/backend/api/api/Models/Experiment.cs new file mode 100644 index 00000000..6de3f169 --- /dev/null +++ b/backend/api/api/Models/Experiment.cs @@ -0,0 +1,23 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace api.Models +{ + public class Experiment + { + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string _id { get; set; } + + public string datasetId { get; set; } + public string uploaderId { get; set; } + public string[] inputColumns { get; set; } + public string outputColumn { get; set; } + public bool randomOrder { get; set; } + public bool randomTestSet { get; set; } + public float randomTestSetDistribution { get; set; } + public string nullValues { get; set; } + public NullValues[] nullValuesReplacers { get; set; } + + } +} diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs index 86541767..e88e7e74 100644 --- a/backend/api/api/Models/Model.cs +++ b/backend/api/api/Models/Model.cs @@ -18,14 +18,8 @@ namespace api.Models public DateTime dateCreated { get; set; } public DateTime lastUpdated { get; set; } //proveriti id - public string datasetId { get; set; } + public string experimentId { get; set; } - //Test set settings - public string[] inputColumns { get; set; } - public string columnToPredict { get; set; } - public bool randomOrder {get;set;} - public bool randomTestSet { get; set; } - public float randomTestSetDistribution { get; set; } //Neural net training public string type { get; set; } @@ -44,8 +38,6 @@ namespace api.Models public string[] metrics { get; set; } public int epochs { get; set; } - public string nullValues { get; set; } - public NullValues[] nullValuesReplacers { get; set; } //public NullValues[] nullValues { get; set; } } diff --git a/backend/api/api/Models/Predictor.cs b/backend/api/api/Models/Predictor.cs index 9c8d362f..5fd2aa09 100644 --- a/backend/api/api/Models/Predictor.cs +++ b/backend/api/api/Models/Predictor.cs @@ -17,6 +17,7 @@ namespace api.Models public bool isPublic { get; set; } public bool accessibleByLink { get; set; } public DateTime dateCreated { get; set; } + public string experimentId { get; set; } } } diff --git a/backend/api/api/Program.cs b/backend/api/api/Program.cs index 2bb97e45..7d5d0c45 100644 --- a/backend/api/api/Program.cs +++ b/backend/api/api/Program.cs @@ -34,6 +34,7 @@ builder.Services.AddScoped<IModelService, ModelService>(); builder.Services.AddScoped<IPredictorService, PredictorService>(); builder.Services.AddScoped<IFileService, FileService>(); builder.Services.AddScoped<IJwtToken, JwtToken>(); +builder.Services.AddScoped<IExperimentService, ExperimentService>(); var mlwss = new MLWebSocketService(); diff --git a/backend/api/api/Services/DatasetService.cs b/backend/api/api/Services/DatasetService.cs index 2ff271f3..c43bd236 100644 --- a/backend/api/api/Services/DatasetService.cs +++ b/backend/api/api/Services/DatasetService.cs @@ -16,7 +16,7 @@ namespace api.Services public List<Dataset> SearchDatasets(string name, string username) { - return _dataset.Find(dataset => dataset.name == name && dataset.isPublic == true).ToList(); + return _dataset.Find(dataset => dataset.name == name && dataset.isPublic == true && dataset.isPreProcess).ToList(); } //kreiranje dataseta @@ -34,12 +34,12 @@ namespace api.Services public List<Dataset> GetMyDatasets(string username) { - return _dataset.Find(dataset => dataset.username == username).ToList(); + return _dataset.Find(dataset => dataset.username == username && dataset.isPreProcess).ToList(); } public List<Dataset> GetGuestDatasets() { //Join Igranonica public datasetove sa svim temp uploadanim datasetovima - List<Dataset> datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true).ToList(); + List<Dataset> datasets= _dataset.Find(dataset => dataset.username == "Igrannonica" && dataset.isPublic == true && dataset.isPreProcess).ToList(); datasets.AddRange(_dataset.Find(dataset => dataset.username == "").ToList()); return datasets; } @@ -47,7 +47,7 @@ namespace api.Services //poslednji datasetovi public List<Dataset> SortDatasets(string username, bool ascdsc, int latest) { - List<Dataset> list = _dataset.Find(dataset => dataset.username == username).ToList(); + List<Dataset> list = _dataset.Find(dataset => dataset.username == username && dataset.isPreProcess).ToList(); if(ascdsc) list = list.OrderBy(dataset => dataset.lastUpdated).ToList(); @@ -59,25 +59,29 @@ namespace api.Services public List<Dataset> GetPublicDatasets() { - return _dataset.Find(dataset => dataset.isPublic == true).ToList(); + return _dataset.Find(dataset => dataset.isPublic == true && dataset.isPreProcess).ToList(); } public Dataset GetOneDataset(string username, string name) { - return _dataset.Find(dataset => dataset.username == username && dataset.name == name).FirstOrDefault(); + return _dataset.Find(dataset => dataset.username == username && dataset.name == name && dataset.isPreProcess).FirstOrDefault(); } //odraditi za pretragu getOne public Dataset GetOneDataset(string id) { - return _dataset.Find(dataset => dataset._id == id).FirstOrDefault(); + return _dataset.Find(dataset => dataset._id == id && dataset.isPreProcess).FirstOrDefault(); } //ako je potrebno da se zameni name ili ekstenzija - public void Update(string username, string name, Dataset dataset) + public void Update(string username, string name, Dataset dataset ) { _dataset.ReplaceOne(dataset => dataset.username == username && dataset.name == name, dataset); } + public void Update(Dataset dataset) + { + _dataset.ReplaceOne(x=>x._id==dataset._id, dataset); + } } diff --git a/backend/api/api/Services/ExperimentService.cs b/backend/api/api/Services/ExperimentService.cs new file mode 100644 index 00000000..712568d4 --- /dev/null +++ b/backend/api/api/Services/ExperimentService.cs @@ -0,0 +1,26 @@ +using api.Interfaces; +using api.Models; +using MongoDB.Driver; + +namespace api.Services +{ + public class ExperimentService : IExperimentService + { + private readonly IMongoCollection<Experiment> _experiment; + public ExperimentService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) + { + var database = mongoClient.GetDatabase(settings.DatabaseName); + _experiment = database.GetCollection<Experiment>(settings.ExperimentCollectionName); + } + + public Experiment Create(Experiment experiment) + { + _experiment.InsertOne(experiment); + return experiment; + } + public Experiment Get(string id) + { + return _experiment.Find(exp=>exp._id == id).FirstOrDefault(); + } + } +} diff --git a/backend/api/api/Services/FileService.cs b/backend/api/api/Services/FileService.cs index 24491124..13f596d0 100644 --- a/backend/api/api/Services/FileService.cs +++ b/backend/api/api/Services/FileService.cs @@ -36,6 +36,10 @@ namespace api.Services return null; return file.path; } + public FileModel getFile(string id) + { + return _file.Find(x=>x._id==id).FirstOrDefault(); + } } } diff --git a/backend/api/api/Services/IDatasetService.cs b/backend/api/api/Services/IDatasetService.cs index 8e62ba43..89856c5b 100644 --- a/backend/api/api/Services/IDatasetService.cs +++ b/backend/api/api/Services/IDatasetService.cs @@ -15,5 +15,6 @@ namespace api.Services void Update(string username, string name, Dataset dataset); void Delete(string username, string name); public List<Dataset> GetGuestDatasets(); + public void Update(Dataset dataset); } } diff --git a/backend/api/api/Services/IExperimentService.cs b/backend/api/api/Services/IExperimentService.cs new file mode 100644 index 00000000..f16f3eae --- /dev/null +++ b/backend/api/api/Services/IExperimentService.cs @@ -0,0 +1,10 @@ +using api.Models; + +namespace api.Services +{ + public interface IExperimentService + { + Experiment Create(Experiment experiment); + public Experiment Get(string id); + } +}
\ No newline at end of file diff --git a/backend/api/api/Services/IFileService.cs b/backend/api/api/Services/IFileService.cs index 7446e283..d65deb48 100644 --- a/backend/api/api/Services/IFileService.cs +++ b/backend/api/api/Services/IFileService.cs @@ -6,5 +6,6 @@ namespace api.Services { FileModel Create(FileModel file); string GetFilePath(string id, string username); + public FileModel getFile(string id); } }
\ No newline at end of file diff --git a/backend/api/api/Services/IMlConnectionService.cs b/backend/api/api/Services/IMlConnectionService.cs index ee839d28..ea73fb0f 100644 --- a/backend/api/api/Services/IMlConnectionService.cs +++ b/backend/api/api/Services/IMlConnectionService.cs @@ -1,8 +1,12 @@ +using api.Models; + namespace api.Services { public interface IMlConnectionService { Task<string> SendModelAsync(object model, object dataset); + Task PreProcess(Dataset dataset, string filePath); + //Task<Dataset> PreProcess(Dataset dataset, byte[] file, string filename); } }
\ No newline at end of file diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index 9c3b3fd8..66f7882a 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -1,16 +1,20 @@ -using RestSharp; +using api.Models; +using RestSharp; using System.Net.WebSockets; using System.Text; +using Newtonsoft.Json; namespace api.Services { public class MlConnectionService : IMlConnectionService { private RestClient client; + private readonly IDatasetService _datasetService; - public MlConnectionService() + public MlConnectionService(IDatasetService datasetService) { this.client = new RestClient("http://127.0.0.1:5543"); + _datasetService=datasetService; } public async Task<string> SendModelAsync(object model, object dataset) @@ -20,5 +24,20 @@ namespace api.Services var result = await this.client.ExecuteAsync(request); return result.Content; //Response od ML microservisa } + public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) + { + var request=new RestRequest("preprocess", Method.Post);//USKLADITI SA ML API + request.AddParameter("dataset", JsonConvert.SerializeObject(dataset)); + //request.AddFile("file", file,filename); + request.AddFile("file", filePath); + request.AddHeader("Content-Type", "multipart/form-data"); + var result=await this.client.ExecuteAsync(request); + Dataset newDataset = JsonConvert.DeserializeObject<Dataset>(result.Content); + newDataset.isPreProcess = true; + _datasetService.Update(newDataset); + + return; + + } } } diff --git a/backend/api/api/Services/TempRemovalService.cs b/backend/api/api/Services/TempRemovalService.cs index 9741ace2..d72dc89e 100644 --- a/backend/api/api/Services/TempRemovalService.cs +++ b/backend/api/api/Services/TempRemovalService.cs @@ -9,6 +9,7 @@ namespace api.Services private readonly IMongoCollection<FileModel> _file; private readonly IMongoCollection<Model> _model; private readonly IMongoCollection<Dataset> _dataset; + private readonly IMongoCollection<Experiment> _experiment; public TempRemovalService(IUserStoreDatabaseSettings settings, IMongoClient mongoClient) { @@ -16,6 +17,7 @@ namespace api.Services _file = database.GetCollection<FileModel>(settings.FilesCollectionName); _model= database.GetCollection<Model>(settings.ModelCollectionName); _dataset = database.GetCollection<Dataset>(settings.DatasetCollectionName); + _experiment= database.GetCollection<Experiment>(settings.ExperimentCollectionName); } public void DeleteTemps() { @@ -29,11 +31,16 @@ namespace api.Services foreach(var dataset in datasets) { DeleteDataset(dataset._id); - List<Model> models = _model.Find(model => model.datasetId == dataset._id && model.username=="").ToList(); - foreach(var model in models) + List<Experiment> experiments = _experiment.Find(experiment=>experiment.datasetId== dataset._id && experiment.uploaderId=="").ToList(); + foreach(var experiment in experiments) { - DeleteModel(model._id); - } + DeleteExperiment(experiment._id); + List<Model> models = _model.Find(model => model.experimentId == experiment._id && model.username == "").ToList(); + foreach (var model in models) + { + DeleteModel(model._id); + } + } } if (File.Exists(file.path)) File.Delete(file.path); @@ -67,6 +74,10 @@ namespace api.Services { _dataset.DeleteOne(dataset => dataset._id == id); } + public void DeleteExperiment(string id) + { + _experiment.DeleteOne(experiment => experiment._id == id); + } } diff --git a/backend/api/api/appsettings.json b/backend/api/api/appsettings.json index fdccfb07..f8923a10 100644 --- a/backend/api/api/appsettings.json +++ b/backend/api/api/appsettings.json @@ -9,17 +9,18 @@ } }, "AllowedHosts": "*", - "UserStoreDatabaseSettings": { - /* LocalHost - */ - "ConnectionString": "mongodb://127.0.0.1:27017/", - "DatabaseName": "si_project", - "CollectionName": "users", - "DatasetCollectionName": "Dataset", - "ModelCollectionName": "Model", - "PredictorCollectionName": "Predictor", - "FilesCollectionName": "Files" - /* + "UserStoreDatabaseSettings": { + /* LocalHost + */ + "ConnectionString": "mongodb://127.0.0.1:27017/", + "DatabaseName": "si_project", + "CollectionName": "users", + "DatasetCollectionName": "Dataset", + "ModelCollectionName": "Model", + "PredictorCollectionName": "Predictor", + "FilesCollectionName": "Files", + "ExperimentCollectionName": "Experiment" + /* "ConnectionString": "mongodb+srv://si_user:si_user@sidatabase.twtfm.mongodb.net/myFirstDatabase?retryWrites=true&w=majority", "DatabaseName": "si_db", "CollectionName": "users", @@ -27,5 +28,5 @@ "ModelCollectionName": "Model", "PredictorCollectionName": "Predictor", "FilesCollectionName": "Files"*/ - } + } }
\ No newline at end of file |