From 4858cd15ec093245e5febc39f3176370c9947ab4 Mon Sep 17 00:00:00 2001 From: TAMARA JERINIC Date: Wed, 4 May 2022 21:02:50 +0200 Subject: Dodato računanje korelacione matrice. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/microservice/api/controller.py | 7 ++----- backend/microservice/api/newmlservice.py | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/microservice/api/controller.py b/backend/microservice/api/controller.py index 41035cc7..988ad987 100644 --- a/backend/microservice/api/controller.py +++ b/backend/microservice/api/controller.py @@ -118,7 +118,6 @@ def returnColumnsInfo(): ''' preprocess = newmlservice.returnColumnsInfo(data) #samo 10 jedinstvenih posto ih ima previse, bilo bi dobro da promenimo ovo da to budu 10 najzastupljenijih vrednosti - for col in preprocess["columnInfo"]: col["uniqueValues"] = col["uniqueValues"][0:6] col["uniqueValuesCount"] = col["uniqueValuesCount"][0:6] @@ -128,11 +127,9 @@ def returnColumnsInfo(): dataset["nullRows"] = preprocess["allNullRows"] dataset["colCount"] = preprocess["colCount"] dataset["rowCount"] = preprocess["rowCount"] + dataset["cMatrix"]=preprocess['cMatrix'] dataset["isPreProcess"] = True - #print(dataset) - - - + return jsonify(dataset) print("App loaded.") diff --git a/backend/microservice/api/newmlservice.py b/backend/microservice/api/newmlservice.py index 9e26b03a..f5e5abcc 100644 --- a/backend/microservice/api/newmlservice.py +++ b/backend/microservice/api/newmlservice.py @@ -27,13 +27,26 @@ import matplotlib.pyplot as plt #from ann_visualizer.visualize import ann_viz; def returnColumnsInfo(dataset): dict=[] + datafront=dataset.copy() + dataMatrix=dataset.copy() + + svekolone=datafront.columns kategorijskekolone=datafront.select_dtypes(include=['object']).columns + allNullCols=0 rowCount=datafront.shape[0]#ukupan broj redova colCount=len(datafront.columns)#ukupan broj kolona + for kolona in svekolone: + if(kolona in kategorijskekolone): + encoder=LabelEncoder() + dataMatrix[kolona]=encoder.fit_transform(dataMatrix[kolona]) + + #print(dataMatrix.dtypes) + cMatrix=dataMatrix.corr() + for kolona in svekolone: if(kolona in kategorijskekolone): unique=datafront[kolona].value_counts() @@ -86,7 +99,7 @@ def returnColumnsInfo(dataset): #pretvaranje u kategorijsku datafront = datafront.astype({kolona: str}) - print(datafront.dtypes) + #print(datafront.dtypes) unique=datafront[kolona].value_counts() uniquevaluesn=[] uniquevaluescountn=[] @@ -117,7 +130,7 @@ def returnColumnsInfo(dataset): #print(NullRows) #print(len(NullRows)) allNullRows=len(NullRows) - return {'columnInfo':dict,'allNullColl':int(allNullCols),'allNullRows':int(allNullRows),'rowCount':int(rowCount),'colCount':int(colCount)} + return {'columnInfo':dict,'allNullColl':int(allNullCols),'allNullRows':int(allNullRows),'rowCount':int(rowCount),'colCount':int(colCount),'cMatrix':str(np.matrix(cMatrix))} @dataclass class TrainingResultClassification: -- cgit v1.2.3 From a5365ef62bc89130b4240e124b3a361edfb34391 Mon Sep 17 00:00:00 2001 From: Danijel Anđelković Date: Wed, 4 May 2022 21:06:06 +0200 Subject: Dodao dugmad za brisanje kada korisnik pregleda liste ssvojih podataka, ispravio neke bagove oko grafa kada nema ulaznih kolona. Dodao kor. matricu u dataset. --- backend/api/api/Models/Dataset.cs | 1 + backend/api/api/Models/Model.cs | 3 ++- backend/api/api/Services/FillAnEmptyDb.cs | 6 +++--- frontend/src/app/_data/Dataset.ts | 6 +++--- frontend/src/app/_data/Experiment.ts | 9 ++++---- .../src/app/_elements/folder/folder.component.css | 8 +++++++ .../src/app/_elements/folder/folder.component.html | 12 ++++++++--- .../src/app/_elements/folder/folder.component.ts | 25 +++++++++++++++------- .../_elements/form-model/form-model.component.html | 4 +++- .../src/app/_elements/graph/graph.component.html | 2 +- .../src/app/_elements/graph/graph.component.ts | 2 +- .../_pages/experiment/experiment.component.html | 2 +- 12 files changed, 54 insertions(+), 26 deletions(-) diff --git a/backend/api/api/Models/Dataset.cs b/backend/api/api/Models/Dataset.cs index 7acd4382..beb66de9 100644 --- a/backend/api/api/Models/Dataset.cs +++ b/backend/api/api/Models/Dataset.cs @@ -28,6 +28,7 @@ namespace api.Models public int nullRows { get; set; } public bool isPreProcess { get; set; } + public float[][] cMatrix { get; set; } } } diff --git a/backend/api/api/Models/Model.cs b/backend/api/api/Models/Model.cs index d8921713..494ae9e9 100644 --- a/backend/api/api/Models/Model.cs +++ b/backend/api/api/Models/Model.cs @@ -27,7 +27,8 @@ namespace api.Models public string lossFunction { get; set; } //public int inputNeurons { get; set; } public int hiddenLayers { get; set; } - public int batchSize { get; set; } + public string batchSize { get; set; } + public string learningRate { get; set; } // na izlazu je moguce da bude vise neurona (klasifikacioni problem sa vise od 2 klase) public int outputNeurons { get; set; } public Layer[] layers { get; set; } diff --git a/backend/api/api/Services/FillAnEmptyDb.cs b/backend/api/api/Services/FillAnEmptyDb.cs index 99bbb91f..f05d8cd8 100644 --- a/backend/api/api/Services/FillAnEmptyDb.cs +++ b/backend/api/api/Services/FillAnEmptyDb.cs @@ -99,7 +99,7 @@ namespace api.Services model.optimizer = "Adam"; model.lossFunction = "mean_squared_error"; model.hiddenLayers = 5; - model.batchSize = 8; + model.batchSize = "8"; model.outputNeurons = 0; model.outputLayerActivationFunction = "sigmoid"; model.metrics = new string[] { }; @@ -210,7 +210,7 @@ namespace api.Services model.optimizer = "Adam"; model.lossFunction = "mean_absolute_error"; model.hiddenLayers = 4; - model.batchSize = 5; + model.batchSize = "8"; model.outputNeurons = 0; model.outputLayerActivationFunction = "relu"; model.metrics = new string[] { }; @@ -316,7 +316,7 @@ namespace api.Services model.optimizer = "Adam"; model.lossFunction = "sparse_categorical_crossentropy"; model.hiddenLayers = 3; - model.batchSize = 4; + model.batchSize = "64"; model.outputNeurons = 0; model.outputLayerActivationFunction = "softmax"; model.metrics = new string[] { }; diff --git a/frontend/src/app/_data/Dataset.ts b/frontend/src/app/_data/Dataset.ts index 7ae5c4ab..9efb9f9b 100644 --- a/frontend/src/app/_data/Dataset.ts +++ b/frontend/src/app/_data/Dataset.ts @@ -18,7 +18,7 @@ export default class Dataset extends FolderFile { public rowCount: number = 0, public nullRows: number = 0, public nullCols: number = 0, - public preview: string[][] = [[]] + public cMatrix: string[][] = [[]] ) { super(name, dateCreated, lastUpdated); } @@ -39,13 +39,13 @@ export class ColumnInfo { public max?: number, public q1?: number, public q3?: number, - ) { + ) { /*if (isNumber) this.columnType = ColumnType.numerical; else this.columnType = ColumnType.categorical;*/ } - + } export enum ColumnType { diff --git a/frontend/src/app/_data/Experiment.ts b/frontend/src/app/_data/Experiment.ts index 9ad57fc3..05336495 100644 --- a/frontend/src/app/_data/Experiment.ts +++ b/frontend/src/app/_data/Experiment.ts @@ -16,10 +16,12 @@ export default class Experiment { public lastUpdated: Date = new Date(), public modelIds: string[] = [], - + public encodings: ColumnEncoding[] = []//[{columnName: "", columnEncoding: Encoding.Label}] ) { } + + _columnsSelected: boolean = false; } export enum NullValueOptions { @@ -67,9 +69,8 @@ export enum Encoding { } export class ColumnEncoding { - constructor ( + constructor( public columnName: string, public encoding: Encoding - ) - {} + ) { } } \ No newline at end of file diff --git a/frontend/src/app/_elements/folder/folder.component.css b/frontend/src/app/_elements/folder/folder.component.css index 2340ee8a..62324d62 100644 --- a/frontend/src/app/_elements/folder/folder.component.css +++ b/frontend/src/app/_elements/folder/folder.component.css @@ -152,6 +152,14 @@ display: none; } +.hover-show { + display: none; +} + +.list-item:hover>.hover-show { + display: initial; +} + .list-add { display: flex; flex-direction: row; diff --git a/frontend/src/app/_elements/folder/folder.component.html b/frontend/src/app/_elements/folder/folder.component.html index 48b59dc8..bff066be 100644 --- a/frontend/src/app/_elements/folder/folder.component.html +++ b/frontend/src/app/_elements/folder/folder.component.html @@ -56,15 +56,15 @@
-
- - + +
@@ -74,6 +74,12 @@
{{file.lastUpdated | date}}
+
+ +
+
diff --git a/frontend/src/app/_elements/folder/folder.component.ts b/frontend/src/app/_elements/folder/folder.component.ts index 20ca1121..ffce3fb1 100644 --- a/frontend/src/app/_elements/folder/folder.component.ts +++ b/frontend/src/app/_elements/folder/folder.component.ts @@ -98,8 +98,8 @@ export class FolderComponent implements AfterViewInit { this.newFileSelected = false; this.listView = false; this.selectedFileChanged.emit(this.selectedFile); - this.displayFile(); this.selectTab(TabType.File); + this.displayFile(); } createNewFile() { @@ -117,9 +117,13 @@ export class FolderComponent implements AfterViewInit { _initialized: boolean = false; refreshFiles(selectedDatasetId: string | null) { + this.files = [] + this.filteredFiles.length = 0; + this.folders[TabType.NewFile] = []; + this.folders[TabType.File] = []; this.tabsToShow.forEach(tab => { this.folders[tab] = []; - }) + }); this.datasetsService.getMyDatasets().subscribe((datasets) => { this.folders[TabType.MyDatasets] = datasets; @@ -137,7 +141,6 @@ export class FolderComponent implements AfterViewInit { }); this.modelsService.getMyModels().subscribe((models) => { - console.log(models); this.folders[TabType.MyModels] = models; }); @@ -164,6 +167,7 @@ export class FolderComponent implements AfterViewInit { switch (this.type) { case FolderType.Dataset: this.formDataset!.uploadDataset((dataset: Dataset) => { + this.newFile = undefined; Shared.openDialog("Obaveštenje", "Uspešno ste dodali novi izvor podataka u kolekciju. Molimo sačekajte par trenutaka da se procesira."); this.refreshFiles(dataset._id); }, @@ -173,7 +177,7 @@ export class FolderComponent implements AfterViewInit { break; case FolderType.Model: this.modelsService.addModel(this.formModel.newModel).subscribe(model => { - this.formModel.newModel = model; + this.newFile = undefined; Shared.openDialog("Obaveštenje", "Uspešno ste dodali novu konfiguraciju neuronske mreže u kolekciju."); this.refreshFiles(null); // todo select model }, (err) => { @@ -205,8 +209,8 @@ export class FolderComponent implements AfterViewInit { filteredFiles: FolderFile[] = []; searchTermsChanged() { - if (!this.files) return; this.filteredFiles.length = 0; + if (!this.files) return; this.filteredFiles.push(...this.files.filter((file) => file.name.toLowerCase().includes(this.searchTerm.toLowerCase()))); /*if (this.selectedFile) { if (!this.filteredFiles.includes(this.selectedFile)) { @@ -226,17 +230,22 @@ export class FolderComponent implements AfterViewInit { this.listView = !this.listView; } - deleteFile(file: FolderFile) { + deleteFile(file: FolderFile, event: Event) { + event.stopPropagation(); console.log('delete'); switch (this.type) { case FolderType.Dataset: this.datasetsService.deleteDataset(file).subscribe((response) => { - console.log(response); + console.log(this.files, this.filteredFiles) + this.filteredFiles.splice(this.filteredFiles.indexOf(file), 1); + console.log(this.files, this.filteredFiles) + this.refreshFiles(null); + console.log(this.files, this.filteredFiles) }); break; case FolderType.Model: this.modelsService.deleteModel(file).subscribe((response) => { - console.log(response); + this.refreshFiles(null); }); break; case FolderType.Experiment: diff --git a/frontend/src/app/_elements/form-model/form-model.component.html b/frontend/src/app/_elements/form-model/form-model.component.html index c0318012..bf4aa0e3 100644 --- a/frontend/src/app/_elements/form-model/form-model.component.html +++ b/frontend/src/app/_elements/form-model/form-model.component.html @@ -108,7 +108,9 @@
- + + +
diff --git a/frontend/src/app/_elements/graph/graph.component.html b/frontend/src/app/_elements/graph/graph.component.html index 3c01bc5e..a021431d 100644 --- a/frontend/src/app/_elements/graph/graph.component.html +++ b/frontend/src/app/_elements/graph/graph.component.html @@ -1,7 +1,7 @@
- {{ i == 0 ? (inputColumns ? inputColumns[j] : 'nepoznato') : (i > 0 && i + {{ i == 0 ? (inputColumns && inputColumns.length >= j ? inputColumns[j] : 'nepoznato') : (i > 0 && i < layers.length - 1 ? model!.layers[i-1].activationFunction : (i==layers.length - 1 ? 'out' : '')) }}
diff --git a/frontend/src/app/_elements/graph/graph.component.ts b/frontend/src/app/_elements/graph/graph.component.ts index c7f8d964..73c3b1ab 100644 --- a/frontend/src/app/_elements/graph/graph.component.ts +++ b/frontend/src/app/_elements/graph/graph.component.ts @@ -28,7 +28,7 @@ export class GraphComponent implements AfterViewInit { @Input() outputNodeColor: string = '#dfd7d7'; private ctx!: CanvasRenderingContext2D; - @Input() inputColumns?: string[] = ['Nije odabran eksperiment']; + @Input() inputColumns?: string[]; constructor() { } diff --git a/frontend/src/app/_pages/experiment/experiment.component.html b/frontend/src/app/_pages/experiment/experiment.component.html index baae864e..83c45405 100644 --- a/frontend/src/app/_pages/experiment/experiment.component.html +++ b/frontend/src/app/_pages/experiment/experiment.component.html @@ -32,7 +32,7 @@
- +
-- cgit v1.2.3