Lung Cancer

Import Libraries and Dataset

import numpy as np import pandas as pd import seaborn as sns import matplotlib.pyplot as plt import warnings warnings.filterwarnings("ignore")

#Loading the Dataset df = pd.read_csv("cancer patient data sets.csv")

This dataset contains information on patients with lung cancer, including their age, gender, air pollution exposure, alcohol use, dust allergy, occupational hazards, genetic risk, chronic lung disease, balanced diet, obesity, smoking status, passive smoker status, chest pain, coughing of blood, fatigue levels , weight loss , shortness of breath , wheezing , swallowing difficulty , clubbing of finger nails , frequent colds , dry coughs , and snoring. By analyzing this data we can gain insight into what causes lung cancer and how best to treat it

Age: The age of the patient. (Numeric) Gender: The gender of the patient. (Categorical) Air Pollution: The level of air pollution exposure of the patient. (Categorical) Alcohol use: The level of alcohol use of the patient. (Categorical) Dust Allergy: The level of dust allergy of the patient. (Categorical) OccuPational Hazards: The level of occupational hazards of the patient. (Categorical) Genetic Risk: The level of genetic risk of the patient. (Categorical) chronic Lung Disease: The level of chronic lung disease of the patient. (Categorical) Balanced Diet: The level of balanced diet of the patient. (Categorical) Obesity: The level of obesity of the patient. (Categorical) Smoking: The level of smoking of the patient. (Categorical) Passive Smoker: The level of passive smoker of the patient. (Categorical) Chest Pain: The level of chest pain of the patient. (Categorical) Coughing of Blood: The level of coughing of blood of the patient. (Categorical) Fatigue: The level of fatigue of the patient. (Categorical) Weight Loss: The level of weight loss of the patient. (Categorical) Shortness of Breath: The level of shortness of breath of the patient. (Categorical) Wheezing: The level of wheezing of the patient. (Categorical) Swallowing Difficulty: The level of swallowing difficulty of the patient. (Categorical) Clubbing of Finger Nails: The level of clubbing of finger nails of the patient. (Categorical)

Descriptive Statistics

# Preview data df.head()

#Features data-type df.info()

df.drop(["index", "Patient Id"], axis = 1, inplace = True)

# Dataset dimensions - (rows, columns) df.shape

# Count of null values df.isnull().sum()

# Statistical summary df.describe().T

Data Visualization

sns.displot(df, bins = 20)

# Scatter plot matrix pd.plotting.scatter_matrix(df, figsize = (35,35))

# Pairplot sns.pairplot(data = df, hue = 'Level') plt.show()

#map function, convering in numerical values df['Level']=df['Level'].map({'Low':0,'Medium':1, 'High':2}) df

Correlation

df.corr()

corr_matrix = df.corr().abs() plt.figure(figsize=(30,10)) sns.heatmap(corr_matrix, annot=True, cmap='YlGnBu') plt.show()

sns.countplot(x = 'Level', data=df)

Removing Outliers

# Compute the mean & std.deviation of each feature mean = df.mean() std = df.std() # Compute the Z-score for each observation z = (df - mean) / std

Removing observations with Z-scores beyond a certain threshold

# Set a threshold for the Z-score threshold = 3 # Remove observations with Z-scores beyond the threshold df_without_outliers = df[(np.abs(z) < threshold)]

df_without_outliers

Visualize the results

# Plot the orignal data & the data without outliers plt.scatter(df.index, df['Level'], label='Orignal Data') plt.scatter(df_without_outliers.index, df_without_outliers['Level'], label='Without Outliers') # Add a legend & display the plot plt.legend() plt.show()

df = df_without_outliers

df.isnull().sum()

#independent and dependent features x = df.iloc[:,:-1] #leaving last column y = df.iloc[:,-1]

#Train Test Split from sklearn.model_selection import train_test_split x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.20,random_state=45)

x_train

y_train

x_test

y_test

#standardizing the dataset from sklearn.preprocessing import StandardScaler scaler = StandardScaler()

scaler.fit_transform(x_train)

X_train = scaler.fit_transform(x_train)

X_test = scaler.fit_transform(x_test)

#if you want to get the data back (orignal data) #scaler.inverse_transform(X_train)

Model

Multiple Linear Regression:

from sklearn.linear_model import LinearRegression #CrossValidation from sklearn.model_selection import cross_val_score

#estimator regression = LinearRegression() regression.fit(X_train,y_train)

cross_val_score(regression,X_train,y_train,scoring ='neg_mean_squared_error',cv=10)

mse=cross_val_score(regression,X_train,y_train,scoring ='neg_mean_squared_error',cv=10)

np.mean(mse)

#prediction reg_pred = regression.predict(X_test)

reg_pred

import seaborn as sns sns.displot(reg_pred-y_test)

sns.displot (reg_pred-y_test,kind='kde')

from sklearn.metrics import r2_score score = r2_score(reg_pred,y_test) score

Logistic Regression:

from sklearn.linear_model import LogisticRegression

classifier = LogisticRegression()

classifier

from sklearn.model_selection import GridSearchCV parameter = {'penalty':['l1','l2','elasticnet'],'C':[1,2,3,4,5,6,10,20,30,40,50],'max_iter':[100,200,300]}

classifier_regressor = GridSearchCV(classifier,param_grid = parameter,scoring='accuracy',cv = 5)

classifier_regressor.fit(x_train,y_train)

print(classifier_regressor.best_params_)

print(classifier_regressor.best_score_)

#prediction classifier_regressor.predict(x_test)

y_pred =classifier_regressor.predict(x_test) y_pred

# accuracy score from sklearn.metrics import accuracy_score,classification_report

score = accuracy_score(y_pred,y_test) print(score)

print(classification_report(y_pred,y_test))

Decision Tree:

from sklearn.tree import DecisionTreeClassifier

# PostPruning technique treemodel = DecisionTreeClassifier()

treemodel

treemodel.fit(x_train,y_train)

# constructing Tree from sklearn import tree plt.figure(figsize = (15,10)) tree.plot_tree(treemodel,filled = True)

# PostPruning technique treemodel = DecisionTreeClassifier(max_depth = 4) treemodel

treemodel.fit(x_train,y_train)

# constructing Tree from sklearn import tree plt.figure(figsize = (15,10)) tree.plot_tree(treemodel,filled = True)

#prediction y_pred =treemodel.predict(x_test)

y_pred

from sklearn.metrics import accuracy_score,classification_report

score = accuracy_score(y_pred,y_test) print(score)

print(classification_report(y_pred,y_test))

Random Forest:

from sklearn.ensemble import RandomForestClassifier

fit_rf = RandomForestClassifier(random_state=40)

#Hyper-Parameter Optimization Using GridSearchCV param_dist = {'max_depth' : [2,3,4,5], 'bootstrap' : [True,False], 'max_features' : ['auto', 'sqrt', 'log2', None], 'criterion' : ['gini', 'entrophy']} cv_rf = GridSearchCV(fit_rf, cv=10, param_grid=param_dist ,n_jobs=3) cv_rf.fit(x_train,y_train) print("Best parameters using Grid Search: \n", cv_rf.best_params_)

fit_rf.set_params(bootstrap=True, criterion='gini', max_features='auto', max_depth=4)

#Train the Random Forest fit_rf.fit(x_train,y_train)

y_pred

y_test

# accuracy score from sklearn.metrics import accuracy_score, classification_report

score = accuracy_score(y_pred,y_test) print(score)

print(classification_report(y_pred,y_test))

Naïve Bayes:

from sklearn.naive_bayes import GaussianNB

# Initializing the Gaussian Classifier model = GaussianNB() # Train the model model.fit(x_train, y_train)

# Predict Output pred = model.predict(x_test) print()

# accuracy score from sklearn.metrics import accuracy_score, classification_report

score = accuracy_score(y_pred,y_test) print(score)

print(classification_report(y_pred,y_test))

.css-dew5jc{color:var(--chakra-colors-fg-neutral-primary);font-weight:var(--chakra-fontWeights-bold);letter-spacing:-0.09px;-webkit-text-decoration:underline;text-decoration:underline;}Lung Cancer

.css-15w88e5{color:var(--chakra-colors-fg-neutral-primary);font-weight:inherit;letter-spacing:-0.09px;}Import Libraries and Dataset

Descriptive Statistics

Data Visualization

Correlation

Removing Outliers

Model

Multiple Linear Regression:

Logistic Regression:

Decision Tree:

Random Forest:

Naïve Bayes:

Lung Cancer

Import Libraries and Dataset