Merge branch 'dev' of http://gitlab.pmf.kg.ac.rs/igrannonica/neuronstellar into dev

6 files changed, 421 insertions, 9 deletions

diff --git a/backend/microservice/PythonServer/project/api/api.py b/backend/microservice/PythonServer/project/api/api.py
index 1f4afdeb..b2fb503b 100644
--- a/backend/microservice/PythonServer/project/api/api.py
+++ b/backend/microservice/PythonServer/project/api/api.py
@@ -7,7 +7,8 @@ import pandas as pd
 import keras
 import csv
 import json
-
+import mlservice
+from mlservice import obuka
 
 app = flask.Flask(__name__)
 app.config["DEBUG"] = True
@@ -20,13 +21,8 @@ def index():
 @app.route('/data', methods = ['GET', 'POST'])
 def data():
     if request.method == 'POST':
-        f = request.json['filepath']
-        data = []
-        with open(f) as file:
-            csvfile = csv.reader(file)
-            for row in csvfile:
-                data.append(row)
-        data = pd.DataFrame(data)
+        f = request.json['filepath']  
+        data = pd.read_csv(f)
         print(data)
-        return render_template('data.html', data = data.to_html(header=False, index=False))
+        return obuka(data,request.json)
 app.run()
\ No newline at end of file
diff --git a/backend/microservice/PythonServer/project/api/socket/client.py b/backend/microservice/PythonServer/project/api/socket/client.py
new file mode 100644
index 00000000..d5740e25
--- /dev/null
+++ b/backend/microservice/PythonServer/project/api/socket/client.py
@@ -0,0 +1,10 @@
+import socket
+
+c = socket.socket()
+
+c.connect(('localhost', 9999))
+
+name = input("Client name:")
+c.send(bytes(name, 'utf-8'))
+
+print(c.recv(1024).decode())
\ No newline at end of file
diff --git a/backend/microservice/PythonServer/project/api/socket/server.py b/backend/microservice/PythonServer/project/api/socket/server.py
new file mode 100644
index 00000000..d6ff3f7c
--- /dev/null
+++ b/backend/microservice/PythonServer/project/api/socket/server.py
@@ -0,0 +1,16 @@
+import socket
+
+s = socket.socket()
+print('Socket Created')
+
+s.bind(('localhost', 9999))
+
+s.listen(3)
+print('waiting for connections')
+
+while True:
+    c, addr = s.accept()
+    name = c.recv(1024).decode()
+    print('Connected with ', addr, name)
+    c.send(bytes('Welcome', 'utf-8'))
+    c.close()
\ No newline at end of file
diff --git a/backend/microservice/PythonServer/project/api/socket2/client.py b/backend/microservice/PythonServer/project/api/socket2/client.py
new file mode 100644
index 00000000..65e76b55
--- /dev/null
+++ b/backend/microservice/PythonServer/project/api/socket2/client.py
@@ -0,0 +1,16 @@
+# Import socket module
+import socket            
+ 
+# Create a socket object
+s = socket.socket()        
+ 
+# Define the port on which you want to connect
+port = 12345               
+ 
+# connect to the server on local computer
+s.connect(('127.0.0.1', port))
+ 
+# receive data from the server and decoding to get the string.
+print (s.recv(1024).decode())
+# close the connection
+s.close()    
\ No newline at end of file
diff --git a/backend/microservice/PythonServer/project/api/socket2/server.py b/backend/microservice/PythonServer/project/api/socket2/server.py
new file mode 100644
index 00000000..c65dae78
--- /dev/null
+++ b/backend/microservice/PythonServer/project/api/socket2/server.py
@@ -0,0 +1,39 @@
+# first of all import the socket library
+import socket            
+ 
+# next create a socket object
+s = socket.socket()        
+print ("Socket successfully created")
+ 
+# reserve a port on your computer in our
+# case it is 12345 but it can be anything
+port = 12345               
+ 
+# Next bind to the port
+# we have not typed any ip in the ip field
+# instead we have inputted an empty string
+# this makes the server listen to requests
+# coming from other computers on the network
+s.bind(('', port))        
+print ("socket binded to %s" %(port))
+ 
+# put the socket into listening mode
+s.listen(5)    
+print ("socket is listening")           
+ 
+# a forever loop until we interrupt it or
+# an error occurs
+while True:
+ 
+# Establish connection with client.
+  c, addr = s.accept()    
+  print ('Got connection from', addr )
+ 
+  # send a thank you message to the client. encoding to send byte type.
+  c.send('Thank you for connecting'.encode())
+ 
+  # Close the connection with the client
+  c.close()
+   
+  # Breaking once connection closed
+  break
\ No newline at end of file
diff --git a/backend/microservice/mlservice.py b/backend/microservice/mlservice.py
new file mode 100644
index 00000000..f8e98184
--- /dev/null
+++ b/backend/microservice/mlservice.py
@@ -0,0 +1,335 @@
+from typing_extensions import Self
+import pandas as pd
+import tensorflow as tf
+import keras
+import numpy as np
+
+from copyreg import constructor
+import flask
+from flask import request, jsonify, render_template
+from sklearn.preprocessing import LabelEncoder
+import csv
+import json
+class Response:
+    def __init__(self,history,rezultat,tacnost,preciznost,recall,spec,f1,classificationreport,mse,mae,mape,rmse,cmatrix,fpr,tpr):
+        self.history=history
+        self.rezultat=rezultat
+        self.tacnost=tacnost
+        self.preciznost=preciznost
+        self.recall=recall
+        self.spec=spec
+        self.f1=f1
+        self.classificationreport=classificationreport
+        self.mse=mse
+        self.mae=mae
+        self.mape=mape
+        self.rmse=rmse
+        self.cmatrix=cmatrix
+        self.fpr=fpr
+        self.tpr=tpr
+
+    ### 1)Ucitavanje vrednosti
+def obuka(data,params):
+    import numpy as np
+    import pandas as pd
+    import tensorflow as tf
+    import matplotlib.pyplot as plt   
+    #print(1)
+    #data1=pd.read_csv('titanic.csv')
+    #data=data1.copy()
+    #print(data.head())
+
+    ### U promenjivoj kolone nalaze se nazivi svih kolona seta podataka
+    kolone=data.columns
+    #print(kolone[1])
+    #print(data[kolone[1]].isnull().sum())
+    #print(data[kolone[1]].head(10))
+
+
+    ### 2)Proveravanje svih kolona za null vrednosti i popunjavanje medijanom ili srednjom vrednosti ili birisanje
+
+    #####Part2 #####
+    '''
+    brisanje=input("DA LI ZELITE DA IZBRSETE SVE KOLONE SA NULL VREDNOSTIMA? ")
+    
+    brisanje=True
+    if(brisanje=='da'):
+        data=data.dropna(axis=1)
+    elif(brisanje=='ne'):
+        brisanjer=input("DA LI ZELITE DA IZBRISETE SVE REDOVE SA NULL VREDNOSTINA ")
+        if(brisanjer=='da'):
+            data=data.dropna()
+        elif(brisanjer=='ne'):
+
+            for i in range(len(kolone)):
+                if(isinstance(data[kolone[i]].dtype, pd.CategoricalDtype)):
+                    print('cat')
+
+                if(data[kolone[i]].isnull().any()):
+                    tippodataka=data[kolone[i]].dtype
+                    kolona=data[kolone[i]].copy()
+            
+                    if(tippodataka==np.float64 or tippodataka==np.int64):
+                        popunjavanje=input("UNETI NACIN POPUNJAVANJA PROMENJIVIH SA NULL VREDNOSTIMA ")
+                        if(popunjavanje=='medijana'):
+                            medijana=kolona.mean()
+                            data[kolone[i]]=data[kolone[i]].fillna(medijana)
+                        if(popunjavanje=='srednjavrednost'):
+                            sv=data[kolone[i]].sum()/data[kolone[i]].count()
+                            data[kolone[i]]=sv
+                        if(popunjavanje=='brisanjekolone'):
+                            data=data.dropna(axis=1)
+
+                    elif(tippodataka==np.object_):
+                        najcescavrednost=kolona.value_counts().index[0]
+                        data[kolone[i]]=data[kolone[i]].fillna(najcescavrednost)
+
+    '''  
+    ### 3)Izbacivanje kolona koje ne uticu na rezultat PART2
+    nredova=data.shape[0]
+    for i in range(len(kolone)):
+        if((data[kolone[i]].nunique()>(nredova/2)) and( data[kolone[i]].dtype==np.object_)):
+            data.pop(kolone[i])
+
+
+    print(data.head(10))
+
+    ### 4)izbor tipa enkodiranja
+    kolone=data.columns ### Azuriranje postojecih kolona nakon moguceg brisanja
+
+    #enc=input("UNETI TIP ENKODIRANJA ")
+    enc=params["encoding"]
+    onehot=0
+
+    ### 5)Enkodiranje svih kategorijskih promenjivih label-encode metodom
+
+    if(enc=='label'):
+        from sklearn.preprocessing import LabelEncoder
+        encoder=LabelEncoder()
+        for k in range(len(kolone)):
+            if(data[kolone[k]].dtype==np.object_):
+                data[kolone[k]]=encoder.fit_transform(data[kolone[k]])
+        print(data.head(20))
+
+    ### 6)Enkodiranje svih kategorijskih promenjivih onehot metodom
+
+    elif(enc=='onehot'):
+        ### PART2###
+        onehot==1
+        kategorijskekolone=[]
+        for k in range(len(kolone)):
+            if(data[kolone[k]].dtype==np.object_):
+                
+                kategorijskekolone.append(kolone[k]) ###U kategorijske kolone smestaju se nazivi svih kolona sa kategorijskim podacima
+        
+        print(kategorijskekolone)
+
+        ### Enkodiranje 
+        data=pd.get_dummies(data,columns=kategorijskekolone,prefix=kategorijskekolone)
+        print(data.head(10))
+
+    kolone=data.columns ### Azuriranje kolona nakon moguceg dodavanja
+
+    ### 7)Podela skupa na skup za trening i skup za testiranje
+
+    #predvidetikol=input("UNETI NAZIV KOLONE ČIJU VREDNOST TREBA PREDVIDETI ")
+    ###sta se cuva od promenjivih broj kolone ili naziv kolone???
+    predvidetikol=params["columnToPredict"]
+
+    xkolone=[]
+    for k in range(len(kolone)):
+            if(kolone[k]!=predvidetikol):
+                
+                xkolone.append(kolone[k])###U xkolone se smestaju nazivi kolona cije vrednosti nije potrebno predvideti !!!Prefiks one-hot!!!
+
+    ### 7.1)Podela na x i y
+    ###Dodavanje vrednosti u x
+    x=data[xkolone].values
+    ###Dodavanje vrednosti u y, samo za label enkodiranje, bez prefiksa
+    y=data[predvidetikol].values
+
+    print(data[xkolone].head(10))
+    print(data[predvidetikol].head(10))
+
+    ### 7.2)Unos velicina za trening i test skup
+    #trening=int(input('UNETI VELIČINU TRENING SKUPA '))
+    #test=int(input("UNETI VELICINU TESTNOG SKUPA"))
+    test=params["randomTestSetDistribution"]
+    ###Provera unetih velicina
+    if(test<=0 or test>=100):
+        print("POGREŠAN UNOS VELIČINE SKUPA ZA TRENING")
+    if(test>1):
+        test=test/100
+
+    ### 7.3)Da li korisnik zeli nasumicno rasporedjivanje podataka?
+    #nasumicno=input("DA LI ŽELITE NASUMIČNO RASPOREDJIVANJE PODATAKA U TRENING I TEST SKUP? ")
+    nasumicno=params["randomTestSet"]
+    ###!!!Dugme za nasumici izbor
+    if(nasumicno):
+        random=50
+    else:
+        random=0
+
+    ### 7.4)Podela podataka
+    from sklearn.model_selection import train_test_split
+    x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=test,random_state=random)
+
+    ### 8)Skaliranje podataka
+    from sklearn.preprocessing import StandardScaler
+    scaler=StandardScaler()
+    scaler.fit(x_train)
+    x_test=scaler.transform(x_test)
+    x_train=scaler.transform(x_train)
+
+    #####ZAVRSENA PRIPREMA PODATAKA#####
+
+    #####OBUCAVANJE MODELA#####
+
+    ### 9)Inicijalizacija vestacke neuronske mreze
+
+    classifier=tf.keras.Sequential()
+
+    ### 10)Dodavanje prvog,ulaznog sloja
+    #aktivacijau=input("UNETI ŽELJENU AKTIVACIONU FUNKCIJU ULAZNOG SLOJA ")
+    #brojnu=int(input("UNETI BROJ NEURONA ULAZNOG SLOJA "))
+
+    aktivacijau=params["inputLayerActivationFunction"]
+    brojnu=params["inputNeurons"]
+
+    classifier.add(tf.keras.layers.Dense(units=brojnu,activation=aktivacijau,input_dim=x_train.shape[1]))
+
+    ### 11)Dodavanje drugog, skrivenog sloja
+    #aktivacijas=input("UNETI ŽELJENU AKTIVACIONU FUNKCIJU SKRIVENOG SLOJA ")
+    #brojns=int(input("UNETI BROJ NEURONA SKRIVENOG SLOJA "))
+
+    aktivacijas=params["hiddenLayerActivationFunction"]
+    brojns=params["hiddenLayerNeurons"]
+
+    classifier.add(tf.keras.layers.Dense(units=brojns,activation=aktivacijas))
+
+    ### 12) Dodavanje treceg, izlaznog sloja
+    #aktivacijai=input("UNETI ŽELJENU AKTIVACIONU FUNKCIJU IZLAZNOG SLOJA ")
+
+    aktivacijai=params["outputLayerActivationFunction"]
+
+    classifier.add(tf.keras.layers.Dense(units=1,activation=aktivacijai))
+
+
+    ### 13) Kompajliranje neuronske mreze
+    #gubici=input("UNETI FUNKCIJU OBRADE GUBITAKA ")
+    #optimizator=input("UNETI ŽELJENI OPTIMIZATOR ")
+
+    optimizator=params["optimizer"]
+
+    ### 13.1)Izbor metrike za kompajler PART2
+    metrike=[]
+    while(1):
+        m=params['lossFunction']
+        
+        if(m=='KRAJ'):
+            break   
+        metrike.append(m)
+    classifier.compile(optimizer=optimizator, loss='binary_crossentropy',metrics = metrike)
+
+    ### 14) 
+    #uzorci=int(input("UNETI KOLIKO UZORAKA ĆE BITI UNETO U ISTO VREME "))
+    #epohe=int(input("UNETI BROJ EPOHA"))
+    uzorci=params["batchSize"]
+    epohe=params["epochs"]
+    history=classifier.fit(x_train,y_train,batch_size=uzorci,epochs=epohe)
+
+    ### 14.1)Parametri grafika iz history PART2
+    metrikedf=pd.DataFrame() ###DataFrame u kom se nalaze podaci o rezultatima metrika za iscrtavanje na grafiku. Svaka kolona sadrzi vrednost metrike po epohama
+    for i in range(len(metrike)):
+        metrikedf[metrike[i]]=history.history[metrike[i]]
+        #print(history.history[metrike[i]])
+        plt.plot(history.history[metrike[i]])
+    plt.show()
+
+    #print(metrikedf)
+
+    #metrikedf.to_csv("metrike.csv")
+
+
+    ### 15) Predvidjanje
+    y_pred=classifier.predict(x_test)
+
+    print(y_pred)
+
+    ### 15.1) Formatiranje podataka za metrike PART2
+    y_pred=(y_pred>=0.5).astype('int')
+    y_pred=y_pred.flatten()
+
+    #print(y_pred)
+
+    #print(y_test)
+    ### 15.2) Kreiranje DataFrame-a u kom se nalaze kolone koje predstavljaju stvarne i predvidjene vrednosti, potrebne za iscrtavanje grafika i metrike PART2
+    rezultat=pd.DataFrame({"Stvarna vrednost ":y_test,"Predvidjena vrednost":y_pred})
+    #print(rezultat.head(20))
+
+    #####METRIKE##### PART2
+
+    import  sklearn.metrics as sm
+            
+
+    ### 16)Tacnost
+    tacnost=sm.accuracy_score(y_test,y_pred)
+    print('tacnost ',tacnost)
+
+    ### 17)Preciznost
+    preciznost=sm.precision_score(y_test,y_pred)
+    print('preciznost ',preciznost)
+
+    ### 18)Recall
+    recall=sm.recall_score(y_test,y_pred)
+    print('recall ',recall)
+
+    ### 19)Specificity
+    tn, fp, fn, tp = sm.confusion_matrix(y_test,y_pred).ravel()
+    spec = tn / (tn+fp)
+    print('spec ',spec)
+
+    ### 20)F1
+    f1=sm.f1_score(y_test,y_pred)
+    print('f1 ',f1)
+
+    ### 21)Classification report
+    classificationreport=sm.classification_report(y_test,y_pred)
+    print('classification ',classificationreport)
+
+    ### 22)Mean squared error (mse)
+    mse=sm.mean_squared_error(y_test,y_pred)
+    print('mse ',mse)
+
+    ### 23)Mean absolute error (mae)
+    mae=sm.mean_absolute_error(y_test,y_pred)
+    print('mae ',mae)
+
+    ### 24)Mean absolute percentage error (mape)
+    mape=sm.mean_absolute_percentage_error(y_test,y_pred)
+    print('mape ',mape)
+
+    ### 25)Root mean square error (rmse) *** da bi se iskoristila u history, salje se u metrics preko funkcije
+    import numpy as np
+    rmse=np.sqrt(sm.mean_squared_error(y_test,y_pred))
+    print("rmse ",rmse)
+
+    ### 26)Confusion matrix
+    cmatrix=sm.confusion_matrix(y_test,y_pred)
+    print('cmatrix ',cmatrix)
+
+    ### 27)ROC
+    fpr, tpr, _ = sm.roc_curve(y_test,y_pred)
+    plt.plot(fpr, tpr, color='blue')
+    plt.title('ROC')
+    plt.xlim([0.0, 1.0])
+    plt.xlabel('False Positive Rate')
+    plt.ylim([0.0, 1.0])
+    plt.ylabel('True Positive Rate')
+    plt.show()
+
+    r=Response(history,rezultat,tacnost,preciznost,recall,spec,f1,classificationreport,mse,mae,mape,rmse,cmatrix,fpr,tpr)
+    
+    return "Done"
+
+