import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
df=pd.read_csv('data.csv')
df.head(3)
Moderate Positive Skewfloat64
Highly Positive Skewfloat64
0
0.8999903238
2.895073786
1
1.1135538
2.962384883
2
1.156830061
2.966377528
df.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 10000 entries, 0 to 9999
Data columns (total 4 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 Moderate Positive Skew 10000 non-null float64
1 Highly Positive Skew 10000 non-null float64
2 Moderate Negative Skew 10000 non-null float64
3 Highly Negative Skew 10000 non-null float64
dtypes: float64(4)
memory usage: 312.6 KB
fig, axes = plt.subplots(2,2, figsize=(8,7))
plt.suptitle('Data Set Graph', fontsize=16)
# Positive Bias Moderate (right bias)
sns.histplot(ax=axes[0,0], data=df['Moderate Positive Skew'], color='blue', alpha=0.5)
axes[0,0].set_title('Positive Bias Moderate (right bias)')
# High Positive Bias (right bias)
sns.histplot(ax=axes[0,1], data=df['Highly Positive Skew'],color='c', alpha=0.5)
axes[0,1].set_title('Positive Bias Moderate (right bias)')
# title
sns.histplot(ax=axes[1,0], data=df['Moderate Negative Skew'],color='indigo', alpha=0.5)
axes[1,0].set_title('Negative Bias Moderate (left bias)')
# title
sns.histplot(ax=axes[1,1], data=df['Highly Negative Skew'],color='tab:brown', alpha=0.5)
axes[1,1].set_title('High Negative Bias (left bias)')
fig.tight_layout()
plt.show()
# Calculation
df.skew()
def diagnostic_graph(data_set, variable):
plt.figure(figsize=(15,6))
plt.subplot(1,2,1)
plt.title(variable)
sns.histplot(data=data_set, x=variable, kde=True, bins=27);
plt.subplot(1,2,2)
stats.probplot(data_set[variable], dist='norm', plot=plt);
plt.title('Q-Q graph')
plt.show()
diagnostic_graph(df,'Moderate Positive Skew');
df_transformed = df.copy()
# Transformation
df_transformed['Sqrt_transformation'] = np.sqrt(df['Moderate Positive Skew'])
df_transformed.head(3)
Moderate Positive Skewfloat64
Highly Positive Skewfloat64
0
0.8999903238
2.895073786
1
1.1135538
2.962384883
2
1.156830061
2.966377528
# Graph after Square Root transformation
diagnostic_graph(df_transformed, 'Sqrt_transformation')
diagnostic_graph(df_transformed,'Highly Positive Skew')
df_transformed['Log_transformation'] = np.log(df['Highly Positive Skew'])
df_transformed.head(3)
Moderate Positive Skewfloat64
Highly Positive Skewfloat64
0
0.8999903238
2.895073786
1
1.1135538
2.962384883
2
1.156830061
2.966377528
# Graph after Log transformation
diagnostic_graph(df_transformed, 'Log_transformation')
diagnostic_graph(df_transformed,'Moderate Positive Skew')
# Add a new column with the transformation
df_transformed['Cubic_transformation'] = np.cbrt(df['Moderate Positive Skew'])
# After Cubic transformation
diagnostic_graph(df_transformed, 'Cubic_transformation')
from sklearn.preprocessing import PowerTransformer
df_pt = df.copy()
# Model Creation
p_scaler = PowerTransformer(method='yeo-johnson')
# fitting and transforming the model
df_yjt = pd.DataFrame(p_scaler.fit_transform(df_pt), columns=['Moderate Positive Skew', 'Highly Positive Skew','Moderate Negative Skew', 'Highly Negative Skew'],)
# Transformed Data Set
df_yjt.head()
Moderate Positive Skewfloat64
Highly Positive Skewfloat64
0
-3.7649515812625647
-2.7456154688892176
1
-3.4894058699838686
-2.6381286353754665
2
-3.4362183461375007
-2.631854810602758
3
-3.3078164561199515
-2.5789633235465255
4
-3.2383065114725404
-2.5607869615769774
df_yjt.describe()
Moderate Positive Skewfloat64
Highly Positive Skewfloat64
count
10000
10000
mean
9.549694368615746e-16
4.5474735088646414e-17
std
1.0000500037503124
1.0000500037503124
min
-3.7649515812625647
-2.7456154688892176
25%
-0.682663691945428
-0.7351637353765551
50%
-0.011129709065981711
0.009128949240715833
75%
0.6911628526669109
0.701639101962064
max
3.537717563100896
3.015598786292614
diagnostic_graph(df_pt,'Moderate Negative Skew')
diagnostic_graph(df_yjt,'Moderate Negative Skew')
pt = PowerTransformer(method='box-cox', standardize=False)
# Only positive data
df_bxcx = pd.DataFrame(p_scaler.fit_transform(df_pt.iloc[:, 0:2]), columns=['Moderate Positive Skew', 'Highly Positive Skew'])
df_bxcx.head(3)
Moderate Positive Skewfloat64
Highly Positive Skewfloat64
0
-3.7649515812625647
-2.7456154688892176
1
-3.4894058699838686
-2.6381286353754665
2
-3.4362183461375007
-2.631854810602758
diagnostic_graph(df_pt, 'Highly Positive Skew')
diagnostic_graph(df_bxcx, 'Highly Positive Skew')