diff options
| -rw-r--r-- | backend/api/api/Controllers/DatasetController.cs | 2 | ||||
| -rw-r--r-- | backend/api/api/Models/ColumnInfo.cs | 2 | ||||
| -rw-r--r-- | backend/api/api/Models/Dataset.cs | 4 | ||||
| -rw-r--r-- | backend/api/api/Services/MlConnectionService.cs | 3 | ||||
| -rw-r--r-- | backend/microservice/api/controller.py | 23 | ||||
| -rw-r--r-- | backend/microservice/api/newmlservice.py | 12 | ||||
| -rw-r--r-- | frontend/src/app/_data/Dataset.ts | 6 | ||||
| -rw-r--r-- | frontend/src/app/_elements/dataset-load/dataset-load.component.ts | 5 | ||||
| -rw-r--r-- | frontend/src/app/experiment/experiment.component.html | 106 | ||||
| -rw-r--r-- | frontend/src/app/experiment/experiment.component.ts | 32 |
10 files changed, 121 insertions, 74 deletions
diff --git a/backend/api/api/Controllers/DatasetController.cs b/backend/api/api/Controllers/DatasetController.cs index 6eb1b9e6..5f01c867 100644 --- a/backend/api/api/Controllers/DatasetController.cs +++ b/backend/api/api/Controllers/DatasetController.cs @@ -148,8 +148,6 @@ namespace api.Controllers /*za pretragu vratiti dataset koji je public public ActionResult<Dataset> Get(string name) { - - var dataset = _datasetService.GetOneDataset(username, name); if (dataset == null) diff --git a/backend/api/api/Models/ColumnInfo.cs b/backend/api/api/Models/ColumnInfo.cs index ee4cee0d..99418732 100644 --- a/backend/api/api/Models/ColumnInfo.cs +++ b/backend/api/api/Models/ColumnInfo.cs @@ -6,6 +6,8 @@ public bool isNumber { get; set; } public int numNulls { get; set; } public float mean { get; set; } + public float min { get; set; } + public float max { get; set; } public float median { get; set; } public string[] uniqueValues { get; set; } diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index 2b3efa3c..12dcfa08 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -6,6 +6,7 @@ namespace api.Models { public class Dataset { + public Dataset() { } public string username { get; set; } [BsonId] @@ -24,7 +25,8 @@ namespace api.Models public bool hasHeader { get; set; } public ColumnInfo[] columnInfo { get; set; } - public int totalNulls; + public int nullCols { get; set; } + public int nullRows { get; set; } public bool isPreProcess { get; set; } } diff --git a/backend/api/api/Services/MlConnectionService.cs b/backend/api/api/Services/MlConnectionService.cs index 66f7882a..3df22c4f 100644 --- a/backend/api/api/Services/MlConnectionService.cs +++ b/backend/api/api/Services/MlConnectionService.cs @@ -26,12 +26,13 @@ namespace api.Services } public async Task PreProcess(Dataset dataset,string filePath)//(Dataset dataset,byte[] file,string filename) { - var request=new RestRequest("preprocess", Method.Post);//USKLADITI SA ML API + var request=new RestRequest("preprocess", Method.Post); request.AddParameter("dataset", JsonConvert.SerializeObject(dataset)); //request.AddFile("file", file,filename); request.AddFile("file", filePath); request.AddHeader("Content-Type", "multipart/form-data"); var result=await this.client.ExecuteAsync(request); + Dataset newDataset = JsonConvert.DeserializeObject<Dataset>(result.Content); newDataset.isPreProcess = true; _datasetService.Update(newDataset); diff --git a/backend/microservice/api/controller.py b/backend/microservice/api/controller.py index 1b17f727..ff803358 100644 --- a/backend/microservice/api/controller.py +++ b/backend/microservice/api/controller.py @@ -4,6 +4,7 @@ import ml_socket import newmlservice import tensorflow as tf import pandas as pd +import json app = flask.Flask(__name__) app.config["DEBUG"] = True @@ -41,14 +42,20 @@ def predict(): @app.route('/preprocess',methods=['POST']) def returnColumnsInfo(): - f=request.json['filepathcolinfo'] - dataset=pd.read_csv(f) - - result=newmlservice.returnColumnsInfo(dataset) - - return jsonify(result) - - + print("********************************PREPROCESS*******************************") + dataset = json.loads(request.form["dataset"]) + file = request.files.get("file") + data=pd.read_csv(file) + preprocess = newmlservice.returnColumnsInfo(data) + #samo 10 jedinstvenih posto ih ima previse, bilo bi dobro da promenimo ovo da to budu 10 najzastupljenijih vrednosti + for col in preprocess["columnInfo"]: + col["uniqueValues"] = col["uniqueValues"][0:10] + dataset["columnInfo"] = preprocess["columnInfo"] + dataset["nullCols"] = preprocess["allNullColl"] + dataset["nullRows"] = preprocess["allNullRows"] + dataset["isPreProcess"] = True + print(dataset) + return jsonify(dataset) print("App loaded.") ml_socket.start() diff --git a/backend/microservice/api/newmlservice.py b/backend/microservice/api/newmlservice.py index 2ea31702..02f2ad6d 100644 --- a/backend/microservice/api/newmlservice.py +++ b/backend/microservice/api/newmlservice.py @@ -21,7 +21,7 @@ from sklearn.model_selection import train_test_split from dataclasses import dataclass import statistics as s from sklearn.metrics import roc_auc_score -from ann_visualizer.visualize import ann_viz; +#from ann_visualizer.visualize import ann_viz; def returnColumnsInfo(dataset): dict=[] datafront=dataset.copy() @@ -43,7 +43,7 @@ def returnColumnsInfo(dataset): 'uniqueValues':uniquevalues.tolist(), 'median':float(mean), 'mean':float(median), - 'numNulls':float(nullCount), + 'numNulls':int(nullCount), 'min':float(minimum), 'max':float(maximum) } @@ -52,7 +52,7 @@ def returnColumnsInfo(dataset): minimum=min(datafront[kolona]) maximum=max(datafront[kolona]) mean=datafront[kolona].mean() - median=s.median(datafront[kolona]) + median=s.median(datafront[kolona].copy().dropna()) nullCount=datafront[kolona].isnull().sum() if(nullCount>0): allNullCols=allNullCols+1 @@ -61,7 +61,7 @@ def returnColumnsInfo(dataset): 'uniqueValues':[], 'mean':float(mean), 'median':float(median), - 'numNulls':float(nullCount), + 'numNulls':int(nullCount), 'min':float(minimum), 'max':float(maximum) } @@ -71,7 +71,7 @@ def returnColumnsInfo(dataset): #print(len(NullRows)) allNullRows=len(NullRows) - return {'columnInfo':dict,'allNullColl':allNullCols,'allNullRows':allNullRows} + return {'columnInfo':dict,'allNullColl':int(allNullCols),'allNullRows':int(allNullRows)} @dataclass class TrainingResultClassification: @@ -433,7 +433,7 @@ def manageH5(dataset,params,h5model): #print(x2) y2 = data[output_column].values h5model.summary() - ann_viz(h5model, title="My neural network") + #ann_viz(h5model, title="My neural network") h5model.compile(loss=params['lossFunction'], optimizer=params['optimizer'], metrics=params['metrics']) diff --git a/frontend/src/app/_data/Dataset.ts b/frontend/src/app/_data/Dataset.ts index dd751947..87f27d12 100644 --- a/frontend/src/app/_data/Dataset.ts +++ b/frontend/src/app/_data/Dataset.ts @@ -15,15 +15,17 @@ export default class Dataset { public hasHeader: boolean = true, public columnInfo: ColumnInfo[] = [], + public nullRows: number = 0, + public nullCols: number = 0, public preview: string[][] = [[]] ) { } } export class ColumnInfo { constructor( - public name: string = '', + public columnName: string = '', public isNumber: boolean = false, - public numNull: number = 0, + public numNulls: number = 0, public uniqueValues?: string[], public median?: number, public mean?: number, diff --git a/frontend/src/app/_elements/dataset-load/dataset-load.component.ts b/frontend/src/app/_elements/dataset-load/dataset-load.component.ts index ed71dc3c..0148ac3a 100644 --- a/frontend/src/app/_elements/dataset-load/dataset-load.component.ts +++ b/frontend/src/app/_elements/dataset-load/dataset-load.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit, ViewChild } from '@angular/core'; +import { Component, OnInit, ViewChild, ViewChildren } from '@angular/core'; import { AddNewDatasetComponent } from '../add-new-dataset/add-new-dataset.component'; import { ModelsService } from 'src/app/_services/models.service'; import shared from 'src/app/Shared'; @@ -19,6 +19,7 @@ export class DatasetLoadComponent { @ViewChild(AddNewDatasetComponent) addNewDatasetComponent?: AddNewDatasetComponent; @ViewChild(AddNewDatasetComponent) datatable?: DatatableComponent; + datasetLoaded: boolean = false; selectedDatasetLoaded: boolean = false; @@ -70,6 +71,8 @@ export class DatasetLoadComponent { //this.refreshThreeNullValueRadioOptions(); //TREBA DA SE DESI this.selectedDatasetLoaded = true; //this.scrollToNextForm(); + + this.selectedDatasetChangeEvent.emit(this.selectedDataset); } }); } diff --git a/frontend/src/app/experiment/experiment.component.html b/frontend/src/app/experiment/experiment.component.html index 8d2c86b3..ac0735fb 100644 --- a/frontend/src/app/experiment/experiment.component.html +++ b/frontend/src/app/experiment/experiment.component.html @@ -6,7 +6,7 @@ <h2>1. Izvor podataka</h2> - <app-dataset-load></app-dataset-load> + <app-dataset-load (selectedDatasetChangeEvent)="updateDataset($event)"></app-dataset-load> <h2>2. Preprocesiranje</h2> @@ -17,12 +17,13 @@ <h3>Izaberite ulazne kolone:</h3> <div id="divInputs" class="form-check mt-2"> <br> - <div *ngFor="let item of selectedDataset.header; let i = index"> - <input class="form-check-input" type="checkbox" value="{{item}}" id="cb_{{item}}" - name="cbsNew" [checked]="this.selectedOutputColumnVal != item" - [disabled]="this.selectedOutputColumnVal == item"> - <label class="form-check-label" for="cb_{{item}}"> - {{item}} + <div *ngFor="let item of selectedDataset.columnInfo; let i = index"> + <input class="form-check-input" type="checkbox" value="{{item.columnName}}" + id="cb_{{item.columnName}}" name="cbsNew" + [checked]="this.selectedOutputColumnVal != item.columnName" + [disabled]="this.selectedOutputColumnVal == item.columnName"> + <label class="form-check-label" for="cb_{{item.columnName}}"> + {{item.columnName}} </label> </div> </div> @@ -31,11 +32,12 @@ <h3>Izaberite izlaznu kolonu:</h3> <div id="divOutputs" class="form-check mt-2"> <br> - <div *ngFor="let item of selectedDataset.header; let i = index"> - <input class="form-check-input" type="radio" value="{{item}}" id="rb_{{item}}" name="rbsNew" - (change)="this.selectedOutputColumnVal = item"> - <label class="form-check-label" for="rb_{{item}}"> - {{item}} + <div *ngFor="let item of selectedDataset.columnInfo; let i = index"> + <input class="form-check-input" type="radio" value="{{item.columnName}}" + id="rb_{{item.columnName}}" name="rbsNew" + (change)="this.selectedOutputColumnVal = item.columnName"> + <label class="form-check-label" for="rb_{{item.columnName}}"> + {{item.columnName}} </label> </div> </div> @@ -48,12 +50,12 @@ class="form-check-input" value="deleteRows" name="fillMissing" id="delRows" checked data-bs-toggle="collapse" data-bs-target="#fillMissingCustom.show"> <label for="delRows" class="form-check-label">Obriši sve - redove sa nedostajućim vrednostima</label><br> + redove sa nedostajućim vrednostima ({{selectedDataset.nullRows}} / TODO)</label><br> <input type="radio" [(ngModel)]="experiment.nullValues" [value]="NullValueOptions.DeleteColumns" class="form-check-input" value="deleteCols" name="fillMissing" id="delCols" data-bs-toggle="collapse" data-bs-target="#fillMissingCustom.show"> <label for="delCols" class="form-check-label">Obriši sve - kolone sa nedostajućim vrednostima</label><br> + kolone sa nedostajućim vrednostima ({{selectedDataset.nullCols}} / TODO)</label><br> <input type="radio" [(ngModel)]="experiment.nullValues" [value]="NullValueOptions.Replace" class="form-check-input" name="fillMissing" id="replace" data-bs-toggle="collapse" data-bs-target="#fillMissingCustom:not(.show)"> @@ -64,38 +66,38 @@ <label for="columnReplacers" class="form-label">Unesite zamenu za svaku kolonu:</label> <div id="columnReplacers"> <div *ngFor="let column of selectedDataset.columnInfo; let i = index" class="my-3"> - <div *ngIf="getInputById('cb_'+column).checked" class=""> + <div *ngIf="getInputById('cb_'+column.columnName).checked" class=""> <span class="w-20 mx-3"> - {{column.name}} <span class="small" style="color:gray;">( - <!--{{//column.numNulls}}--> - TODO BROJ null) + {{column.columnName}} <span class="small" style="color:gray;">( + {{column.numNulls}} + null) </span> </span> - <label *ngIf="column.numNull <= 0" class="text-center form-control mx-3 text-secondary"> + <label *ngIf="column.numNulls <= 0" + class="text-center form-control mx-3 text-secondary"> Ova kolona nema nedostajućih vrednosti. </label> - <div *ngIf="column.numNull > 0" class="d-flex flex-row justify-content-end"> - <!-- ngIf colummn.numNulls == 0 --> + <div *ngIf="column.numNulls > 0" class="d-flex flex-row justify-content-end"> <div class="flex-grow-3 mx-3 me-auto"> <div class="input-group"> <div class="input-group-prepend"> - <label [for]="'fillCol_'+column.name" class="form-control"> + <label [for]="'fillCol_'+column.columnName" class="form-control"> Zameni - <input type="radio" [id]="'fillCol_'+column.name" - [name]="'delOp_'+column.name"> + <input type="radio" [id]="'fillCol_'+column.columnName" + [name]="'delOp_'+column.columnName"> </label> </div> - <input type="text" class="form-control" [id]="'fillText_'+column.name" - (keyup)="checkFillColRadio(column.name)" + <input type="text" class="form-control" [id]="'fillText_'+column.columnName" + (keyup)="checkFillColRadio(column.columnName)" placeholder="Unesi vrednost..."> <div class="input-group-append"> <select [id]="'replaceOptions'+i" class="form-control btn-primary" - *ngIf="column.isNumber" (change)="replace($event);"> + *ngIf="column.isNumber" (change)="replace($event, column);"> <option *ngFor="let option of Object.keys(ReplaceWith); let optionName of Object.values(ReplaceWith)" [value]="option"> @@ -104,8 +106,8 @@ </select> <select [id]="'replaceOptions'+i" class="form-control btn-outline-primary" - *ngIf="!column.isNumber && column.numNull > 0" - (change)="replace($event);"> + *ngIf="!column.isNumber && column.numNulls > 0" + (change)="replace($event, column);"> <option *ngFor="let option of column.uniqueValues" [value]="option"> {{ option }} </option> @@ -116,39 +118,39 @@ <div class="flex-shrink-1 mx-3"> <div class="input-group"> - <label class="form-control" [for]="'delCol_'+column.name">Izbriši + <label class="form-control" [for]="'delCol_'+column.columnName">Izbriši kolonu - <input type="radio" [id]="'delCol_'+column" - [name]="'delOp_'+column.name" - (change)="emptyFillTextInput(column.name)"></label> + <input type="radio" [id]="'delCol_'+column.columnName" + [name]="'delOp_'+column.columnName" + (change)="emptyFillTextInput(column.columnName)"></label> </div> </div> <div class="flex-shrink-1 mx-3"> <div class="input-group"> - <label class="form-control" [for]="'delRows_'+column.name">Izbriši + <label class="form-control" [for]="'delRows_'+column.columnName">Izbriši redove - <input type="radio" [id]="'delRows_'+column.name" - [name]="'delOp_'+column.name" checked - (change)="emptyFillTextInput(column.name)"></label> + <input type="radio" [id]="'delRows_'+column.columnName" + [name]="'delOp_'+column.columnName" checked + (change)="emptyFillTextInput(column.columnName)"></label> </div> </div> </div> </div> </div> <div *ngFor="let column of selectedDataset.columnInfo; let i = index" class="my-3"> - <div class="input-group row" *ngIf="getInputById('rb_'+column.name).checked"> + <div class="input-group row" *ngIf="getInputById('rb_'+column.columnName).checked"> <span class="input-group-text col-2 text-center"> - {{column}} <span class="small" style="color:gray;">(br + {{column.columnName}} <span class="small" style="color:gray;">(br null)</span> </span> <label *ngIf="true" class="form-control">Ova kolona nema nedostajućih vrednosti.</label> - <input *ngIf="true" type="radio" [id]="'fillCol_'+column.name" class="col-1 mt-2" - [name]="'delOp_'+column.name"> + <input *ngIf="true" type="radio" [id]="'fillCol_'+column.columnName" class="col-1 mt-2" + [name]="'delOp_'+column.columnName"> <select [id]="'replaceOptions'+i" class="form-control col-2" *ngIf="column.isNumber" - (change)="replace($event);"> + (change)="replace($event, column);"> <option *ngFor="let option of Object.keys(ReplaceWith); let optionName of Object.values(ReplaceWith)" [value]="option"> @@ -156,24 +158,28 @@ </option> </select> <select [id]="'replaceOptions'+i" class="form-control col-2" *ngIf="!column.isNumber" - (change)="replace($event);"> + (change)="replace($event, column);"> <option *ngFor="let option of []" [value]="option"> <!--/*arrayColumn(datasetFile, i)*/--> {{ option }} </option> </select> <input *ngIf="true" type="text" class="form-control col-1" - [id]="'fillText_'+column.name" (keyup)="checkFillColRadio(column.name)" + [id]="'fillText_'+column.columnName" (keyup)="checkFillColRadio(column.columnName)" placeholder="Unesi vrednost..."> - <label *ngIf="true" class="form-control col-2" [for]="'delCol_'+column" .name>Izbriši + <label *ngIf="true" class="form-control col-2" [for]="'delCol_'+column.columnName" + .name>Izbriši kolonu - <input type="radio" [id]="'delCol_'+column.name" [name]="'delOp_'+column.name" - (change)="emptyFillTextInput(column.name)"></label> - <label *ngIf="true" class="form-control col-2" [for]="'delRows_'+column.name">Izbriši + <input type="radio" [id]="'delCol_'+column.columnName" + [name]="'delOp_'+column.columnName" + (change)="emptyFillTextInput(column.columnName)"></label> + <label *ngIf="true" class="form-control col-2" + [for]="'delRows_'+column.columnName">Izbriši redove - <input type="radio" [id]="'delRows_'+column.name" [name]="'delOp_'+column.name" - (change)="emptyFillTextInput(column.name)" checked></label> + <input type="radio" [id]="'delRows_'+column.columnName" + [name]="'delOp_'+column.columnName" + (change)="emptyFillTextInput(column.columnName)" checked></label> </div> </div> diff --git a/frontend/src/app/experiment/experiment.component.ts b/frontend/src/app/experiment/experiment.component.ts index 2309dcd7..ada0484e 100644 --- a/frontend/src/app/experiment/experiment.component.ts +++ b/frontend/src/app/experiment/experiment.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit } from '@angular/core'; import Experiment, { NullValReplacer, NullValueOptions, ReplaceWith } from '../_data/Experiment'; import Model from '../_data/Model'; -import Dataset from '../_data/Dataset'; +import Dataset, { ColumnInfo } from '../_data/Dataset'; import { ModelsService } from '../_services/models.service'; import Shared from '../Shared'; @@ -28,6 +28,11 @@ export class ExperimentComponent implements OnInit { ngOnInit(): void { } + updateDataset(dataset: Dataset) { + console.log(dataset); + this.selectedDataset = dataset; + } + getInputById(id: string): HTMLInputElement { return document.getElementById(id) as HTMLInputElement; } @@ -52,9 +57,30 @@ export class ExperimentComponent implements OnInit { (<HTMLInputElement>document.getElementById("fillCol_" + colName)).checked = true; } - replace(event: Event) { + replace(event: Event, column: ColumnInfo) { let option = (<HTMLInputElement>event.target).value; - // TODO + + const input = (<HTMLInputElement>document.getElementById("fillText_" + column.columnName)); + if (column.isNumber) { + switch (option) { + case ReplaceWith.Max: + input.value = "" + column.max; + break; + case ReplaceWith.Min: + input.value = "" + column.min; + break; + case ReplaceWith.Mean: + input.value = "" + column.mean; + break; + case ReplaceWith.Median: + input.value = "" + column.median; + break; + case ReplaceWith.None: + break; + } + } else { + input.value = option; + } } getNullValuesReplacersArray()/*: NullValReplacer[]*/ { |