Autorzy: Paulina Chmielewska, Mikołaj Macura, Julian Kominiak
import pandas as pd
import numpy as np
import seaborn as sns
import random as rd
from timer import Timer
sns.set_palette("Set3", 10)
sns.set_style("whitegrid")
t = Timer()
data = pd.read_csv("/work/healthcare-dataset-stroke-data.csv")
Czyszczenie Data Setu
Usuwamy puste rekordy, a także kolumny 'index' i 'id'.
data.dropna(inplace=True)
del data['id']
Usuwamy rekord, który wskazuje na 'Other' w kolumnie 'gender' oraz zamieniamy ją na 'is_female' z wartościami liczbowymi.
data["gender"].value_counts()
data = data[data.gender != "Other"]
def gender_value(x):
if x == "Female":
return 1
else:
return 0
data["is_female"] = data["gender"].apply(gender_value)
del data['gender']
Zamieniamy kolumnę 'work_type' na 'is_working' z wartościami liczbowymi.
data["work_type"].value_counts()
def is_working(x):
if x == "Govt_job":
return 1
elif x == 'Private':
return 0.75
elif x == 'Self-employed':
return 0.5
else:
return 0
data["is_working"] = data["work_type"].apply(is_working)
del data["work_type"]
Zamieniamy kolumnę 'Residence_type' na 'is_urban' z wartościami liczbowymi.
data["Residence_type"].value_counts()
def residence_value(x):
if x == "Urban":
return 1
else:
return 0
data["is_urban"] = data["Residence_type"].apply(residence_value)
del data['Residence_type']
Zamieniamy kolumnę 'ever_married' na wartości liczbowe.
data["ever_married"].value_counts()
def ever_married_value(x):
if x == "Yes":
return 1
else:
return 0
data["ever_married"] = data["ever_married"].apply(ever_married_value)
Zamieniamy kolumnę 'smoking_status' na wartości liczbowe.
data["smoking_status"].value_counts()
def smoke_value(x):
if x == "never smoked":
return 0
elif x == "formerly smoked":
return 0.5
elif x == "smokes":
return 1
data["smoking_status"] = data["smoking_status"].apply(smoke_value)
Usuwamy puste rekordy, resetujemy index, ustawiamy kolumny w odpowiedniej kolejności oraz zapisujemy je do nowego pliku.
data.dropna(inplace=True)
data.reset_index(inplace=True)
data = data[['is_female','age','is_urban','is_working','hypertension','heart_disease','ever_married','avg_glucose_level','bmi','smoking_status','stroke']]
data.to_csv("dane.csv")
data
Wykresy
ax = sns.catplot(data=data, x="heart_disease", y="bmi", hue="stroke", kind="swarm")
ax.fig.suptitle('Sprawdzenie czy duża waga oraz występowanie choroby serca, prowadzi do udaru.')
/shared-libs/python3.7/py/lib/python3.7/site-packages/seaborn/categorical.py:1296: UserWarning: 80.3% of the points cannot be placed; you may want to decrease the size of the markers or use stripplot.
warnings.warn(msg, UserWarning)
sns.countplot(data=data, x="is_female", hue="stroke").set_title('Występowanie udaru dla danej płci.')
sns.violinplot(data=data, x="stroke", y="age").set_title('Występowanie udaru dla danego wieku.')
sns.countplot(data=data, x="is_urban", hue="stroke").set_title('Występowanie udaru dla danego miejsca zamieszkania.')
sns.countplot(data=data, x="is_working", hue="stroke").set(xlabel="Never Worked Self-Employed Private Govt_Job ", title="Występowanie udaru w zależności od rodzaju zatrudniania.")
sns.countplot(data=data, x="hypertension", hue="stroke").set_title('Występowanie udaru u osób z nadciśnieniem')
sns.countplot(data=data, x="heart_disease", hue="stroke").set_title("Występowanie udaru u osób z chorobami serca")
sns.countplot(data=data, x="ever_married", hue="stroke").set_title('Występowanie udaru u osób w związku małżeńskim')
sns.violinplot(data=data, x="stroke", y="avg_glucose_level").set_title("Występowanie udaru w zależności od poziomu cukru.")
sns.violinplot(data=data, x="stroke", y="bmi").set_title('Występowanie udaru w zależności od wartości bmi')
sns.countplot(data=data, x="smoking_status", hue="stroke").set(xlabel='Smoking status',ylabel='Quantity',xticklabels=['Never Smoked','Formerly Smoked','Smokes'], title="Występowanie udaru u palących i niepalących.")
Implementacja Algorytmów
Klasa DataProcessing, pomocna przy przygotowywaniu danych do klasyfikacji, tasowania, normalizacji oraz rozdzielania na zbiór treningowy i walidacyjny
class DataProcessing:
@staticmethod
def shuffle(x):
for i in range(len(x)-1,0,-1):
j = rd.randint(0,i)
x.iloc[i], x.iloc[j] = x.iloc[j], x.iloc[i]
return x
@staticmethod
def normalize(df):
normalized = df.drop("stroke", axis=1)
normalized = (normalized - normalized.min())/(normalized.max() - normalized.min())
normalized = normalized.join(df["stroke"])
return normalized
@staticmethod
def split_set(x):
k = int(len(x)*0.7)
train = x[:k]
val = x[k:]
return train, val
Klasyfikatora NaiveBayes
class NaiveBayes:
@staticmethod
def gauss(x, mean, std):
if std == 0.0:
std = 0.0001
exponent = np.exp(-(x-mean)**2/(2*std**2))
return 1/(np.sqrt(2*np.pi*std**2))*exponent
@staticmethod
def classify(trainSet, sample):
stroke = data.query('stroke == 1')
not_stroke = data.query('stroke == 0')
classes = [stroke,not_stroke]
probabilities = []
for c in classes:
means = []
stds = []
for atr in c:
means.append(c[atr].mean())
stds.append(c[atr].std())
prob = 1
for i in range(10):
prob *= NaiveBayes.gauss(sample[i],means[i],stds[i])
probabilities.append(prob)
for i in range(2):
if probabilities[i] == max(probabilities):
return classes[i]['stroke'].iloc[0]
Implementacja klasyfikatora NaiveBayes dla naszych danych
data = pd.read_csv("dane.csv", index_col="Unnamed: 0")
best_acc = 0
DataProcessing.shuffle(data)
trainSet, valuesSet = DataProcessing.split_set(DataProcessing.normalize(data))
acc = 0
for i in range(len(valuesSet)):
sample = valuesSet.iloc[i].to_list()
a = NaiveBayes.classify(trainSet,sample[:10])
if a == sample[10]:
acc+=1
BayesAccuracy = acc/len(valuesSet)
print('Accuracy: {}%'.format(int(BayesAccuracy*100)))
Accuracy: 93%
Klasyfikator KNN
class KNN:
@staticmethod
def minkowski_metric(v1,v2,m):
distance = 0
for i in range(len(v1)-1):
distance+=(abs(v1[i]-v2[i])*m)
distance = distance **(1/m)
return distance
@staticmethod
def clustering(sample, x, k):
distances = []
classes = {'1.0': 0, '0.0': 0}
for i in x.index:
distances.append([KNN.minkowski_metric(sample, x.iloc[i], 2), x.iloc[i].stroke])
distances = sorted(distances)
for i in range(k):
classes[str(distances[i][1])]+=1
return max(classes, key=classes.get)
Implementacja klasyfikatora KNN dla naszych danych
data = pd.read_csv("dane.csv", index_col="Unnamed: 0")
X, Y = DataProcessing.split_set(DataProcessing.shuffle(DataProcessing.normalize(data)))
k = 3
correct = 0
for sample in Y.iloc:
model_guess = KNN.clustering(sample, X, k)
if str(model_guess) == str(sample.stroke):
correct += 1
KNNAccurracy = correct/len(Y)
print("For k = {} | Accuracy: {} % ({} correct)".format(k, round(KNNAccurracy * 100, 3), correct))
For k = 3 | Accuracy: 94.553 % (972 correct)
data = pd.read_csv("dane.csv", index_col="Unnamed: 0")
X, Y = DataProcessing.split_set(DataProcessing.shuffle(DataProcessing.normalize(data)))
k = 4
correct = 0
for sample in Y.iloc:
model_guess = KNN.clustering(sample, X, k)
if str(model_guess) == str(sample.stroke):
correct += 1
KNNAccurracy = correct/len(Y)
print("For k = {} | Accuracy: {} % ({} correct)".format(k, round(KNNAccurracy * 100, 3), correct))
For k = 4 | Accuracy: 93.482 % (961 correct)
data = pd.read_csv("dane.csv", index_col="Unnamed: 0")
X, Y = DataProcessing.split_set(DataProcessing.shuffle(DataProcessing.normalize(data)))
k = 5
correct = 0
for sample in Y.iloc:
model_guess = KNN.clustering(sample, X, k)
if str(model_guess) == str(sample.stroke):
correct += 1
KNNAccurracy = correct/len(Y)
print("For k = {} | Accuracy: {} % ({} correct)".format(k, round(KNNAccurracy * 100, 3), correct))
For k = 5 | Accuracy: 93.872 % (965 correct)
data = pd.read_csv("dane.csv", index_col="Unnamed: 0")
sample = pd.Series(data={'is_female': 0, 'age': 81, 'is_urban': 1, 'is_working': 0.75, 'hypertension': 0, 'heart_disease': 0,
'ever_married': 1, 'avg_glucose_level': 186.21, 'bmi': 29, 'smoking_status': 0.5})
t.start()
BayesPrediction = NaiveBayes.classify(data, sample)
print('Time for BayesPrediction is {}s'.format(round(t.stop(),3)))
t.start()
KNNPrediction = KNN.clustering(sample, data, 5)
print('Time for KNNPrediction is {}s'.format(round(t.stop(),3)))
StrokePrediction = (float(BayesAccuracy) * float(BayesPrediction) + float(KNNAccurracy) * float(KNNPrediction)) / 2
print("Probability of stroke is:", round(StrokePrediction * 100, 1),"%")
Time for BayesPrediction is 0.013s
Time for KNNPrediction is 1.769s
Probability of stroke is: 94.4 %