In [1]:
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
import os
from tensorflow import keras
import numpy as np
from sklearn.model_selection import train_test_split
import shutil
from sklearn.model_selection import StratifiedKFold
import keras
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from keras.callbacks import EarlyStopping
pd.get_option("display.max_columns", None)
Out[1]:
In [2]:
#set up seed for reproducibility
seed = 5
np.random.seed(seed)
In [3]:
#loading cleaned dataset
coupon_data = pd.read_csv('https://raw.githubusercontent.com/rijinbaby/Statistical-Learning/main/cleaned_data_raw_columns.csv')
coupon_data.columns
Out[3]:
In [4]:
#dropping and transforming columns
coupon_data = coupon_data.drop(columns = ['expiration', 'occupation', 'time', 'direction_opp','age_weightage', 'income_weightage' ])
coupon_data['temperature'] = coupon_data['temperature'].astype(str)
scaler = StandardScaler(with_mean=False)
coupon_data['expiration_weightage'] = scaler.fit_transform(coupon_data['expiration_weightage'].values.reshape(-1,1))
coupon_data.shape
Out[4]:
In [5]:
#creating dummy variables
coupon_data = pd.get_dummies(coupon_data, drop_first = True)
print(coupon_data.shape)
In [6]:
#creating X and Y
X_data = coupon_data.drop('Y', axis =1).to_numpy()
print(X_data.shape)
Y_data = coupon_data['Y'].to_numpy()
print(Y_data.shape)
PCA analysis¶
In [7]:
#PCA check with 20 components
pca = PCA(n_components=20)
pca.fit(X_data)
Out[7]:
In [8]:
print(pca.explained_variance_ratio_)
plt.plot(np.cumsum(pca.explained_variance_ratio_))
Out[8]:
Basic Neural network using Keras¶
In [9]:
#splitting dataset into train and test
X_train, X_test, y_train, y_test = train_test_split(X_data, Y_data, test_size=0.25, random_state= 0)
# Check the dimension of the sets
print('X_train:',np.shape(X_train))
print('y_train:',np.shape(y_train))
print('X_test:',np.shape(X_test))
print('y_test:',np.shape(y_test))
Network parameters:¶
- ADAM as gradient descent optimization algorithm
- Activation function:
- output layer : sigmoid/ hard_sigmoid - binary classification
- hidden layers: relu or tahn
- Metrics: accuracy as the data is balanced
- Loss function:binary cross entropy -- binary classification
- Hyperparameters (can be tuned)
- number of epochs - number times that the learning algorithm will work through the entire training dataset
- batch_size - number of training examples utilized in one iteration
- Number of layers: 2-3 (empirically)
- Number of neurons on each layer (empirically)
- Regularization parameters/ Cross-Validation or Early Stopping to reduce overfitting:
- Validation split 0.2
- CV with 5 folds
Adding more layers to NN can help addressing bias, not variance.
Regularizaition is used to reduce variance. By adding a regularizer, you may enforce the training process to steer towards relatively “simple” weights, which may make your model more generic and thus scalable.
Activation Functions: Sigmoid or Logistic Activation Function: Sigmoid function maps any input to an output ranging from 0 to 1. For small values (<-5), sigmoid returns a value close to zero, and for large values (>5) the result of the function gets close to 1.
Loss Functions: For Binary Classification tasks - BinaryCrossentropy. Computes the cross-entropy loss between true labels and predicted labels. We use this cross-entropy loss when there are only two label classes (assumed to be 0 and 1).
In [10]:
#defining a NN model
def create_model():
model = tf.keras.Sequential([
tf.keras.layers.InputLayer(input_shape = (70)),
tf.keras.layers.Dense(64, activation = "relu", kernel_initializer='normal'),
tf.keras.layers.Dense(32, activation = "sigmoid", kernel_initializer='normal'),
tf.keras.layers.Dense(1, activation='hard_sigmoid', kernel_initializer='normal')
])
model.compile(optimizer = "adam", loss='binary_crossentropy', metrics=['accuracy'])
return model
In [11]:
#define early_stopping
es = EarlyStopping(monitor='val_accuracy', mode='max', verbose=1)
In [12]:
#set up seed for reproducibility
seed = 5
np.random.seed(seed)
model = create_model()
history = model.fit(X_train, y_train, validation_split = 0.2, epochs=50, batch_size=64, verbose =0)
#callbacks=[es]
# plot training history
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()
In [13]:
#running cross validation over a train set
estimator = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=create_model, epochs=50, batch_size=64, validation_split = 0.2, verbose=0)
kfold = StratifiedKFold(n_splits=5, shuffle=True)
results = cross_val_score(estimator, X_train, y_train, cv=kfold)
print(' Training Accuracy: %.3f (%.3f)' % (results.mean(), results.std()))
In [14]:
# predicting accuracy
# Stochastic nature of the algorithms, run multiple time to see the average test accuracy
# setting seed does not help
n = 0
acc_scores = []
while n < 10:
print(n)
model = create_model()
history = model.fit(X_train, y_train, validation_split = 0.2, epochs=50, batch_size=64, verbose =0)
loss_and_metrics = model.evaluate(X_test, y_test)
acc_scores.append(loss_and_metrics[1])
n +=1
In [15]:
acc_scores = pd.Series(acc_scores)
acc_scores.describe()
Out[15]:
Open questions for model improvement
- The test accuracy predicted can further be increased by trying to optimize the epochs, batch_size, learning rate, the number of layers or the number of nodes per layer - add grid search for the best paramerters
- Adding hidden layers with other non-linear activation functions (tahn, relu) -correctness of the model
- Early stopping is not working
- Regulatization does not change anything - should help reducing overfitting.
- Fluctuations in val and train accuracy depend on batch_size (the larger batch_size)
- Contradicting observations - should we remove them (same X and different Y).
- Problem of overlapping features (noticed during clustering and PCA analysis) - data structure itself
Example: we may same 100 observations with similarity higher than 70% based on X - also probably would be in the same cluster. Among these 100 observations 70 accepted coupon (Y=1) and 30 declined (Y=0). There is an unevitable error.