Device Failures - Over-Sampling Regressor Algorithms
Project - Device Failures (Over-Sampling Regression)¶
Purpose/Goal:¶
To use the given data to create a classifcation algorithm that will accurately predict if a device will fail or not.
In this notebook, I manipulate the data for traditionally regression algorithms. I also use over-sampling due to the imbalance of device failures.
In [1]:
import numpy as np
from sklearn.cross_validation import train_test_split as tts
from sklearn import linear_model
import pandas as pd
import matplotlib.pyplot as plt
from imblearn.over_sampling import SMOTE
from imblearn.combine import SMOTEENN
from imblearn.under_sampling import RandomUnderSampler
from sklearn.metrics import recall_score, precision_score, accuracy_score
from sklearn.feature_selection import SelectFromModel
import seaborn as sns
from datetime import datetime
%matplotlib inline
In [2]:
df = pd.read_csv('failures.csv')
In [3]:
df.head()
Out[3]:
In [4]:
df.info()
In [5]:
df.corr()
Out[5]:
In [6]:
corr = df.corr()
fig, ax = plt.subplots(figsize=(10,10))
sns.heatmap(corr, xticklabels=corr.columns.values, yticklabels=corr.columns.values, ax=ax)
Out[6]:
In [7]:
df.describe()
Out[7]:
In [8]:
df.nunique()
# Columns could be categorical data
Out[8]:
In [9]:
fig = plt.figure(figsize=(7,10))
ax1 = plt.subplot(331)
ax2 = plt.subplot(332)
ax3 = plt.subplot(333)
ax4 = plt.subplot(334)
ax5 = plt.subplot(335)
ax6 = plt.subplot(336)
ax7 = plt.subplot(337)
ax8 = plt.subplot(338)
ax9 = plt.subplot(339)
df.boxplot(column='attribute1',by='failure',ax=ax1)
df.boxplot(column='attribute2',by='failure',ax=ax2)
df.boxplot(column='attribute3',by='failure',ax=ax3)
df.boxplot(column='attribute4',by='failure',ax=ax4)
df.boxplot(column='attribute5',by='failure',ax=ax5)
df.boxplot(column='attribute6',by='failure',ax=ax6)
df.boxplot(column='attribute7',by='failure',ax=ax7)
df.boxplot(column='attribute8',by='failure',ax=ax8)
df.boxplot(column='attribute9',by='failure',ax=ax9)
plt.suptitle('')
plt.tight_layout()
In [10]:
df.attribute1.value_counts()
# Continuous variables create problems for Tree Based Algorithms. Creates pure tables too quickly
Out[10]:
In [11]:
# Removing Duplicates
print df.duplicated().sum() # No duplicated rows
df.drop('attribute8',axis=1,inplace=True) #attribute 8 is a duplicate of attribute7
df.head()
Out[11]:
In [12]:
# Have an Imbalanced Data Set
numfailure0 = len(df[df.failure==0]) #len(df.failure==0)
numfailure1 = len(df[df.failure==1])
total = numfailure0 + numfailure1
print numfailure0, numfailure1, total
df.failure.value_counts(normalize=True)
Out[12]:
In [13]:
print df.date.nunique(), df.device.nunique()
print df.date.nunique()*df.device.nunique()/float(total)
df.date.value_counts()
# Does not appear to be an obvious pattern between
Out[13]:
In [14]:
df[df.failure==1].head(20)
Out[14]:
In [15]:
# It appears measurement are taken until the device fails, since the last measurement is the failure
print df[df.failure==1].device.duplicated().sum()
print df[df.device=='S1F0P3G2']
print df[df.device=='W1F0P114']
In [16]:
# We must assume all devices are created equal. Thus a combination of the attibutes and maybe 'time' attibutes to failure
# Creating a column to measure time called 'daysActive
df.date = pd.to_datetime(df.date)
df['daysActive'] = (df.date-df.date[0])
df.daysActive = df.daysActive.dt.days
df.head()
Out[16]:
In [17]:
# Creating a feature called 'season'
# Using Northen Hemisphere Seasonality
season = []
for date in df.date:
if date < datetime(2015,3,1):
season.append('winter')
elif date < datetime(2015,6,1):
season.append('spring')
elif date < datetime(2015,9,1):
season.append('summer')
else:
season.append('fall')
df['season'] = season
df.head()
Out[17]:
In [18]:
df.drop(['date','device'],axis=1,inplace=True)
df.info()
Cleaning Data for Regression Algorithms¶
In [19]:
# Normalize Attributes 1 & 6
attribute1max = float(df.attribute1.max())
attribute1min = float(df.attribute1.min())
attribute6max = float(df.attribute6.max())
attribute6min = float(df.attribute6.min())
df.attribute1 = df.attribute1.apply(lambda x: (x - attribute1min)/(attribute1max - attribute1min))
df.attribute6 = df.attribute6.apply(lambda x: (x - attribute6min)/(attribute6max - attribute6min))
In [20]:
print 'Skew for {a} is: {b}'.format(a='attribute1', b=df.attribute1.skew())
print 'Skew for {a} is: {b}'.format(a='attribute6', b=df.attribute6.skew())
fig = plt.figure(figsize=(10,10))
ax1 = plt.subplot(121)
ax2 = plt.subplot(122)
df.hist(column='attribute1',ax=ax1)
df.hist(column='attribute6',ax=ax2)
Out[20]:
In [21]:
df.info()
In [22]:
df = pd.get_dummies(df,columns=['season','attribute2','attribute3','attribute4','attribute5','attribute7','attribute9'],drop_first=True)
In [23]:
df.head()
Out[23]:
Modeling - Regression Algorithms¶
In [24]:
X = df.drop('failure', axis=1, inplace=False)
y = df.failure
print X.shape, y.shape
In [25]:
from sklearn.cross_validation import train_test_split as tts
X_train, X_test, y_train, y_test = tts(X, y, train_size=0.8,random_state=123, stratify=y) # Stratify Data to Keep Test Balance
print X_train.shape, X_test.shape, y_train.shape, y_test.shape
Over-Sampling¶
SMOTE¶
In [26]:
sm = SMOTE(random_state=123, ratio=1.0)
X_train_smote, y_train_smote = sm.fit_sample(X_train, y_train)
print X_train_smote.shape, y_train_smote.shape
SMOTEENN¶
In [27]:
sm = SMOTEENN(random_state=123, ratio=1.0)
X_train_smoteenn, y_train_smoteenn = sm.fit_sample(X_train, y_train)
print X_train_smoteenn.shape, y_train_smoteenn.shape
Logistic Regression (SMOTE)¶
In [28]:
from sklearn.linear_model import LogisticRegression as LR
In [29]:
lr_smote = LR(random_state=123).fit(X_train_smote, y_train_smote)
pred_lr_smote = lr_smote.predict(X_test)
accScore_lr_smote = accuracy_score(y_test, pred_lr_smote)
precisionScore_lr_smote = precision_score(y_test, pred_lr_smote)
recallScore_lr_smote = recall_score(y_test, pred_lr_smote)
print 'Over-Sampling (SMOTE) Logistic Regression: \nAccuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.\n'.format(a=accScore_lr_smote, b=precisionScore_lr_smote, c=recallScore_lr_smote)
In [44]:
from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = LR()
for train_index, test_index in kf.split(X_train_smote):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X_train_smote[train_index], X_train_smote[test_index]
ytrain, ytest = y_train_smote[train_index], y_train_smote[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Over-Sampling (SMOTE) Logistic Regression - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [45]:
from sklearn.metrics import confusion_matrix
import itertools
def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion Matrix', cmap=plt.cm.Blues):
# This function prints the confusion matrix
# Normalization can be applied by setting it to true
plt.imshow(cm, interpolation= 'nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=0)
plt.yticks(tick_marks, classes)
if normalize==True:
cm = cm.astype(float)/cm.sum(axis=1)[:, np.newaxis]
print 'Normalized Confusion Matrix'
else:
print 'Confusion Matrix without Normalization'
print cm
thresh = cm.max()/2
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, cm[i,j], horizontalalignment='center', color='white' if cm[i,j] > thresh else 'black')
plt.tight_layout()
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
In [46]:
cnf_matrix = confusion_matrix(y_test, pred_lr_smote)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Logistic Regression (SMOTE) - Normalized Confusion Matrix')
plt.show()
In [47]:
from sklearn.metrics import precision_recall_curve
precision, recall, _ = precision_recall_curve(y_test, pred_lr_smote)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Logistic Regression (SMOTE) - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_lr_smote))
Out[47]:
In [48]:
from sklearn import metrics
predsLR_smote = lr_smote.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predsLR_smote)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Logistic Regression (SMOTE) - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
Logistic Regression (SMOTEENN)¶
In [49]:
from sklearn.linear_model import LogisticRegression as LR
In [50]:
lr_smoteenn = LR(random_state=123).fit(X_train_smoteenn, y_train_smoteenn)
pred_lr_smoteenn = lr_smoteenn.predict(X_test)
accScore_lr_smoteenn = accuracy_score(y_test, pred_lr_smoteenn)
precisionScore_lr_smoteenn = precision_score(y_test, pred_lr_smoteenn)
recallScore_lr_smoteenn = recall_score(y_test, pred_lr_smoteenn)
print 'Over-Sampling (SMOTEENN) Logistic Regression: \nAccuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.\n'.format(a=accScore_lr_smoteenn, b=precisionScore_lr_smoteenn, c=recallScore_lr_smoteenn)
In [57]:
from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = LR()
for train_index, test_index in kf.split(X):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X_train_smoteenn[train_index], X_train_smoteenn[test_index]
ytrain, ytest = y_train_smoteenn[train_index], y_train_smoteenn[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Over-Sampling (SMOTEENN) Logistic Regression - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [58]:
cnf_matrix = confusion_matrix(y_test, pred_lr_smoteenn)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Logistic Regression (SMTOEENN) - Normalized Confusion Matrix')
plt.show()
In [59]:
from sklearn.metrics import precision_recall_curve
precision, recall, _ = precision_recall_curve(y_test, pred_lr_smoteenn)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Logistic Regression (SMOTEENN) - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_lr_smoteenn))
Out[59]:
In [60]:
from sklearn import metrics
predsLR_smoteenn = lr_smoteenn.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predsLR_smoteenn)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Logistic Regression (SMOTE) - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
Ridge Regression (SMOTE)¶
In [61]:
from sklearn.linear_model import RidgeClassifier
In [62]:
ridge_smote = RidgeClassifier(random_state=123).fit(X_train_smote, y_train_smote)
pred_ridge_smote = ridge_smote.predict(X_test)
accScore_ridge_smote = accuracy_score(y_test, pred_ridge_smote)
precisionScore_ridge_smote = precision_score(y_test, pred_ridge_smote)
recallScore_ridge_smote = recall_score(y_test, pred_ridge_smote)
print 'Over-Sampling (SMTOE) Ridge Regression: Accuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.'.format(a=accScore_ridge_smote, b=precisionScore_ridge_smote, c=recallScore_ridge_smote)
In [63]:
#from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = RidgeClassifier()
for train_index, test_index in kf.split(X):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X_train_smote[train_index], X_train_smote[test_index]
ytrain, ytest = y_train_smote[train_index], y_train_smote[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Over-Sampling (SMOTE) Ridge Classifier - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [64]:
cnf_matrix = confusion_matrix(y_test, pred_ridge_smote)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Ridge Classifier - Normalized Confusion Matrix')
plt.show()
In [65]:
from sklearn.metrics import precision_recall_curve
precision, recall, _ = precision_recall_curve(y_test, pred_ridge_smote)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Ridge Classifier - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_ridge_smote))
Out[65]:
In [66]:
'''from sklearn import metrics
predsRidge_smote = ridge_smote.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predsRidge_smote)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Ridge Classifier - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()'''
Out[66]:
Ridge Regression (SMTOEENN)¶
In [67]:
ridge_smoteenn = RidgeClassifier(random_state=123).fit(X_train_smoteenn, y_train_smoteenn)
pred_ridge_smoteenn = ridge_smoteenn.predict(X_test)
accScore_ridge_smoteenn = accuracy_score(y_test, pred_ridge_smoteenn)
precisionScore_ridge_smoteenn = precision_score(y_test, pred_ridge_smoteenn)
recallScore_ridge_smoteenn = recall_score(y_test, pred_ridge_smoteenn)
print 'Over-Sampling (SMOTEENN) Ridge Regression: Accuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.'.format(a=accScore_ridge_smoteenn, b=precisionScore_ridge_smoteenn, c=recallScore_ridge_smoteenn)
In [76]:
#from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = RidgeClassifier()
for train_index, test_index in kf.split(X):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X_train_smoteenn[train_index], X_train_smoteenn[test_index]
ytrain, ytest = y_train_smoteenn[train_index], y_train_smoteenn[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Over-Sampling (SMOTEENN) Ridge Classifier - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [77]:
cnf_matrix = confusion_matrix(y_test, pred_ridge_smoteenn)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Ridge Classifier - Normalized Confusion Matrix')
plt.show()
In [78]:
from sklearn.metrics import precision_recall_curve
precision, recall, _ = precision_recall_curve(y_test, pred_ridge_smoteenn)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Ridge Classifier - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_ridge_smoteenn))
Out[78]:
In [79]:
'''from sklearn import metrics
predsRidge_smoteenn = ridge_smoteenn.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predsRidge_smoteenn)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Ridge Classifier - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()'''
Out[79]:
Stochastic Gradient Descent (SMOTE)¶
In [80]:
from sklearn.linear_model import SGDClassifier
In [81]:
sgd_smote = SGDClassifier(random_state=123).fit(X_train_smote, y_train_smote)
pred_sgd_smote = sgd_smote.predict(X_test)
accScore_sgd_smote = accuracy_score(y_test, pred_sgd_smote)
precisionScore_sgd_smote = precision_score(y_test, pred_sgd_smote)
recallScore_sgd_smote = recall_score(y_test, pred_sgd_smote)
In [82]:
print 'Over-Sampling (SMOTE) Stochastic Gradient Descent: Accuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.'.format(a=accScore_sgd_smote, b=precisionScore_sgd_smote, c=recallScore_sgd_smote)
In [83]:
#from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = SGDClassifier()
for train_index, test_index in kf.split(X):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X_train_smote[train_index], X_train_smote[test_index]
ytrain, ytest = y_train_smote[train_index], y_train_smote[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Over-Sampling (SMOTE) Stochastic Gradient Descent - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [84]:
cnf_matrix = confusion_matrix(y_test, pred_sgd_smote)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Stochastic Gradient Descent - Normalized Confusion Matrix')
plt.show()
In [85]:
precision, recall, _ = precision_recall_curve(y_test, pred_sgd_smote)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Stochastic Gradient Descent - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_sgd_smote))
Out[85]:
In [88]:
'''predsSGD_smote = sgd_smote.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predSGD_smote)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Stochastic Gradient Descent - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()'''
Out[88]:
Stochastic Gradient Descent (SMOTEENN)¶
In [89]:
sgd_smoteenn = SGDClassifier(random_state=123).fit(X_train_smoteenn, y_train_smoteenn)
pred_sgd_smoteenn = sgd_smoteenn.predict(X_test)
accScore_sgd_smoteenn = accuracy_score(y_test, pred_sgd_smoteenn)
precisionScore_sgd_smoteenn = precision_score(y_test, pred_sgd_smoteenn)
recallScore_sgd_smoteenn = recall_score(y_test, pred_sgd_smoteenn)
In [90]:
print 'Over-Sampling (SMOTEENN) Stochastic Gradient Descent: Accuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.'.format(a=accScore_sgd_smoteenn, b=precisionScore_sgd_smoteenn, c=recallScore_sgd_smoteenn)
In [92]:
#from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = SGDClassifier()
for train_index, test_index in kf.split(X):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X_train_smoteenn[train_index], X_train_smoteenn[test_index]
ytrain, ytest = y_train_smoteenn[train_index], y_train_smoteenn[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Over-Sampling (SMOTEENN) Stochastic Gradient Descent - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [93]:
cnf_matrix = confusion_matrix(y_test, pred_sgd_smoteenn)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Stochastic Gradient Descent - Normalized Confusion Matrix')
plt.show()
In [94]:
precision, recall, _ = precision_recall_curve(y_test, pred_sgd_smoteenn)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Stochastic Gradient Descent - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_sgd_smoteenn))
Out[94]:
In [96]:
'''predsSGD_smoteenn = sgd_smoteenn.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predSGD_smoteenn)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Stochastic Gradient Descent - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()'''
Out[96]: