# toggle to hide code
from IPython.display import HTML
HTML('''<script>
code_show=true;
function code_toggle() {
if (code_show){
$('div.input').hide();
} else {
$('div.input').show();
}
code_show = !code_show
}
$( document ).ready(code_toggle);
</script>
<form action="javascript:code_toggle()"><input type="submit" value="Click here to toggle on/off the raw code."></form>''')
# center all images
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode(connected=True)
HTML("""
<style>
.output_png {
display: table-cell;
text-align: center;
vertical-align: middle;
}
</style>
""")
# Install tensorflow if not installed already
!pip install tensorflow
!pip install tensorflow_datasets
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow_datasets as tfds
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras import layers
from tensorflow.keras import optimizers
from tensorflow.keras import initializers
from tensorflow.keras import callbacks
from tensorflow.keras import backend as K
from tensorflow.keras.regularizers import l2
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from sklearn.base import BaseEstimator
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
print("TensorFlow Version: " + tf.__version__)
print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))
# Provide your credentials to the runtime
# from google.colab import auth
# auth.authenticate_user()
# print('Authenticated')
# %%time
# # This step takes ~1 hour to load the entire dataset
# # Load data into dataframe 'df'
# %%bigquery --project quiet-rigging-307720 df
# SELECT *
# FROM `quiet-rigging-307720.Higgs_Dataset.Higgs_Data_Table`
# from google.colab import drive
# drive.mount('/content/drive')
# Load the given HIGGS data
#df = pd.read_csv("/content/drive/MyDrive/Colab Notebooks/HIGGS.csv", header = None)
# Load this instead if want a small amount for dev testing
# df = pd.read_csv("/content/drive/MyDrive/Colab Notebooks/HIGGS.csv", header = None, nrows=9999)
#%%time
# Load the given HIGGS data
#df = pd.read_csv("../Data/HIGGS.csv", header = None)
#df = pd.read_csv("../Data/HIGGS.csv", header = None, nrows=50000) # use for testing purposes only
%%time
# download the .gz file
# https://www.tensorflow.org/api_docs/python/tf/data/experimental/CsvDataset
gz = tf.keras.utils.get_file('HIGGS.csv.gz', 'https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz')
#df = pd.read_csv(gz, compression='gzip', header = None, nrows=50000) # use for testing purposes only
df = pd.read_csv(gz, compression='gzip', header = None)
df.columns = ['class label', 'lepton pT', 'lepton eta', 'lepton phi',
'missing energy magnitude', 'missing energy phi',
'jet 1 pt', 'jet 1 eta', 'jet 1 phi', 'jet 1 b-tag',
'jet 2 pt', 'jet 2 eta', 'jet 2 phi', 'jet 2 b-tag',
'jet 3 pt', 'jet 3 eta', 'jet 3 phi', 'jet 3 b-tag',
'jet 4 pt', 'jet 4 eta', 'jet 4 phi', 'jet 4 b-tag',
'm_jj', 'm_jjj', 'm_lv', 'm_jlv',
'm_bb', 'm_wbb', 'm_wwbb']
# Make a copy of the original dataframe for future use
df_original = df.copy()
# List the total number of rows and columns in the dataframe
print("Total number of rows in the dataframe: " + str(df.shape[0]))
print("Total number of columns in the dataframe: " + str(df.shape[1]))
# Display first 2 rows in the data frame
print("Listed below are the first 2 rows in the data frame:")
df.head(2)
# Check the data types of all the columns in the dataset
count = df.dtypes.value_counts()
for i in count.index:
print('The datafram has', count[i], 'columns, which are of type', i)
# Determining the number of columns which have null values
print(f'There are {df.isnull().sum().sum()} columns with at least 1 na value.')
%%time
print("Grouping non-duplicates into False bucket and duplicates into True bucket:")
print(df.groupby( [df.duplicated(subset=None, keep='first')] ).size())
# Remove duplicates except for the first occurrence
df.drop_duplicates(inplace=True)
%%time
print("Grouping non-duplicates into False bucket and duplicates into True bucket:")
print(df.groupby( [df.duplicated(subset=None, keep='first')] ).size())
print("Listed below is a Statistical Distribution of all numerical data points in the data frame:")
df.describe()
plt_matrix = df.hist(bins=30, figsize=(25, 25))
plt.figure(figsize=(20, 20))
corrMatrix = df.corr()
sns.heatmap(corrMatrix, annot=True)
plt.show()
# Standardizes a dataframe such that mean is 0 and stdv is 1,
# except for columns with all positive values which have mean 1
def standardize(df):
mins =