Device Failures - Under-Sampling Tree Based Algorithms
Project - Device Failures (Under Sampling Trees)¶
Purpose/Goal:¶
To use the given data to create a classifcation algorithm that will accurately predict if a device will fail or not.
In this notebook, I will manipulate the data for tree-based algorithms. I will also use undersampling due to the imbalance of device failures.
In [2]:
import numpy as np
from sklearn.cross_validation import train_test_split as tts
from sklearn import linear_model
import pandas as pd
import matplotlib.pyplot as plt
from imblearn.over_sampling import SMOTE
from imblearn.combine import SMOTEENN
from imblearn.under_sampling import RandomUnderSampler
from sklearn.metrics import recall_score, precision_score, accuracy_score
from sklearn.feature_selection import SelectFromModel
import seaborn as sns
from datetime import datetime
%matplotlib inline
In [3]:
df = pd.read_csv('failures.csv')
In [4]:
df.head()
Out[4]:
In [5]:
df.info()
In [6]:
df.corr()
Out[6]:
In [7]:
corr = df.corr()
fig, ax = plt.subplots(figsize=(10,10))
sns.heatmap(corr, xticklabels=corr.columns.values, yticklabels=corr.columns.values, ax=ax)
Out[7]:
In [8]:
df.describe()
Out[8]:
In [9]:
df.nunique()
# Columns could be categorical data
Out[9]:
In [10]:
fig = plt.figure(figsize=(7,10))
ax1 = plt.subplot(331)
ax2 = plt.subplot(332)
ax3 = plt.subplot(333)
ax4 = plt.subplot(334)
ax5 = plt.subplot(335)
ax6 = plt.subplot(336)
ax7 = plt.subplot(337)
ax8 = plt.subplot(338)
ax9 = plt.subplot(339)
df.boxplot(column='attribute1',by='failure',ax=ax1)
df.boxplot(column='attribute2',by='failure',ax=ax2)
df.boxplot(column='attribute3',by='failure',ax=ax3)
df.boxplot(column='attribute4',by='failure',ax=ax4)
df.boxplot(column='attribute5',by='failure',ax=ax5)
df.boxplot(column='attribute6',by='failure',ax=ax6)
df.boxplot(column='attribute7',by='failure',ax=ax7)
df.boxplot(column='attribute8',by='failure',ax=ax8)
df.boxplot(column='attribute9',by='failure',ax=ax9)
plt.suptitle('')
plt.tight_layout()
In [11]:
df.attribute1.value_counts()
# Continuous variables create problems for Tree Based Algorithms. Creates pure tables too quickly
Out[11]:
In [12]:
# Removing Duplicates
print df.duplicated().sum() # No duplicated rows
df.drop('attribute8',axis=1,inplace=True) #attribute 8 is a duplicate of attribute7
df.head()
Out[12]:
In [13]:
# Have an Imbalanced Data Set
numfailure0 = len(df[df.failure==0]) #len(df.failure==0)
numfailure1 = len(df[df.failure==1])
total = numfailure0 + numfailure1
print numfailure0, numfailure1, total
df.failure.value_counts(normalize=True)
Out[13]:
In [14]:
print df.date.nunique(), df.device.nunique()
print df.date.nunique()*df.device.nunique()/float(total)
df.date.value_counts()
# Does not appear to be an obvious pattern between
Out[14]:
In [15]:
df[df.failure==1].head(20)
Out[15]:
In [16]:
# It appears measurement are taken until the device fails, since the last measurement is the failure
print df[df.failure==1].device.duplicated().sum()
print df[df.device=='S1F0P3G2']
print df[df.device=='W1F0P114']
In [17]:
# We must assume all devices are created equal. Thus a combination of the attibutes and maybe 'time' attibutes to failure
# Creating a column to measure time called 'daysActive
df.date = pd.to_datetime(df.date)
df['daysActive'] = (df.date-df.date[0])
df.daysActive = df.daysActive.dt.days
df.head()
Out[17]:
In [18]:
# Creating a feature called 'season'
# Using Northen Hemisphere Seasonality
season = []
for date in df.date:
if date < datetime(2015,3,1):
season.append('winter')
elif date < datetime(2015,6,1):
season.append('spring')
elif date < datetime(2015,9,1):
season.append('summer')
else:
season.append('fall')
df['season'] = season
df.head()
Out[18]:
In [19]:
df.drop(['date','device'],axis=1,inplace=True)
df.info()
Cleaning Data for Tree Algorithms¶
In [21]:
print 'Skew for {a} is: {b}'.format(a='attribute1', b=df.attribute1.skew())
print 'Skew for {a} is: {b}'.format(a='attribute6', b=df.attribute6.skew())
fig = plt.figure(figsize=(10,10))
ax1 = plt.subplot(121)
ax2 = plt.subplot(122)
df.hist(column='attribute1',ax=ax1)
df.hist(column='attribute6',ax=ax2)
Out[21]:
In [22]:
df['attribute1Binned'] = pd.qcut(df.attribute1, 20, labels=False)
df['attribute6Binned'] = pd.qcut(df.attribute6, 20, labels=False)
df['daysActiveBinned'] = pd.qcut(df.daysActive, 20, labels=False)
df.attribute6Binned
Out[22]:
In [23]:
df.drop('attribute1', axis=1, inplace=True)
df.drop('attribute6', axis=1, inplace=True)
df.drop('daysActive', axis=1, inplace=True)
In [24]:
df.info()
In [29]:
df = pd.get_dummies(df,columns=['season'],drop_first=True)
In [30]:
df.head()
Out[30]:
Under-Sampling¶
In [31]:
X = df.drop('failure', axis=1, inplace=False)
y = df.failure
In [32]:
rus = RandomUnderSampler(ratio=1.0,random_state=123)
X, y = rus.fit_sample(X, y)
print X.shape, y.shape
Modeling (Tree Algorithms)¶
In [33]:
from sklearn.cross_validation import train_test_split as tts
X_train, X_test, y_train, y_test = tts(X, y, train_size=0.8,random_state=123, stratify=y) # Stratify Data to Keep Test Balance
print X_train.shape, X_test.shape, y_train.shape, y_test.shape
Random Forests¶
In [34]:
from sklearn.ensemble import RandomForestClassifier as RFC
In [35]:
rf = RFC(random_state=123, n_estimators=100).fit(X_train, y_train)
pred_rf = rf.predict(X_test)
accScore_rf = accuracy_score(y_test, pred_rf)
precisionScore_rf = precision_score(y_test, pred_rf)
recallScore_rf = recall_score(y_test, pred_rf)
print 'Under-Sampling Random Forests: \nAccuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.\n'.format(a=accScore_rf, b=precisionScore_rf, c=recallScore_rf)
In [36]:
from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = RFC(n_estimators=100)
for train_index, test_index in kf.split(X):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X[train_index], X[test_index]
ytrain, ytest = y[train_index], y[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Under-Sampling Random Forests - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [37]:
from sklearn.metrics import confusion_matrix
import itertools
def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion Matrix', cmap=plt.cm.Blues):
# This function prints the confusion matrix
# Normalization can be applied by setting it to true
plt.imshow(cm, interpolation= 'nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=0)
plt.yticks(tick_marks, classes)
if normalize==True:
cm = cm.astype(float)/cm.sum(axis=1)[:, np.newaxis]
print 'Normalized Confusion Matrix'
else:
print 'Confusion Matrix without Normalization'
print cm
thresh = cm.max()/2
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, cm[i,j], horizontalalignment='center', color='white' if cm[i,j] > thresh else 'black')
plt.tight_layout()
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
In [39]:
cnf_matrix = confusion_matrix(y_test, pred_rf)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Random Forests - Normalized Confusion Matrix')
plt.show()
In [40]:
from sklearn.metrics import precision_recall_curve
precision, recall, _ = precision_recall_curve(y_test, pred_rf)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Random Forests - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_rf))
Out[40]:
In [41]:
from sklearn import metrics
predsRF = rf.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predsRF)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Random Forests - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
Gradient Boosting¶
In [42]:
from sklearn.ensemble import GradientBoostingClassifier as GBC
In [43]:
gbc = GBC(random_state=123, n_estimators=100).fit(X_train, y_train)
pred_gbc = gbc.predict(X_test)
accScore_gbc = accuracy_score(y_test, pred_gbc)
precisionScore_gbc = precision_score(y_test, pred_gbc)
recallScore_gbc = recall_score(y_test, pred_gbc)
print 'Under-Sampling Gradient Boosting: Accuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.'.format(a=accScore_gbc, b=precisionScore_gbc, c=recallScore_gbc)
In [44]:
#from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = GBC(n_estimators=100)
for train_index, test_index in kf.split(X):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X[train_index], X[test_index]
ytrain, ytest = y[train_index], y[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Under-Sampling Gradient Boosting - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [46]:
cnf_matrix = confusion_matrix(y_test, pred_gbc)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Gradient Boosting - Normalized Confusion Matrix')
plt.show()
In [47]:
from sklearn.metrics import precision_recall_curve
precision, recall, _ = precision_recall_curve(y_test, pred_gbc)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Gradient Boosting - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_gbc))
Out[47]:
In [48]:
from sklearn import metrics
predsGB = gbc.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predsGB)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Gradient Boosting - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
SVC (Support Vector Classifier)¶
In [49]:
from sklearn.svm import SVC
In [50]:
svc = SVC(random_state=123, probability=True).fit(X_train, y_train)
pred_svc = svc.predict(X_test)
accScore_svc = accuracy_score(y_test, pred_svc)
precisionScore_svc = precision_score(y_test, pred_svc)
recallScore_svc = recall_score(y_test, pred_svc)
In [51]:
print 'Under-Sampling SVC: Accuracy Score: {a}.\nPrecision Score: {b}.\nRecall Score: {c}.'.format(a=accScore_svc, b=precisionScore_svc, c=recallScore_svc)
In [52]:
#from sklearn.model_selection import KFold
kf = KFold(n_splits = 5)
accuracy = []
precision = []
recall = []
model = GBC(n_estimators=100)
for train_index, test_index in kf.split(X):
print 'TRAIN:', train_index, 'TEST:', test_index, '\n'
Xtrain, Xtest = X[train_index], X[test_index]
ytrain, ytest = y[train_index], y[test_index]
pred = model.fit(Xtrain, ytrain).predict(Xtest)
accuracy.append(accuracy_score(ytest, pred))
precision.append(precision_score(ytest, pred))
recall.append(recall_score(ytest, pred))
print 'Under-Sampling Support Vector Machine - {a} Folds Testing: \nAverage Accuracy Score: {b}.\nAverage Precision Score: {c}.\nAverage Recall Score: {d}.\n'.format(a=kf.get_n_splits(X),b=np.array(accuracy).mean(), c=np.array(precision).mean(), d=np.array(recall).mean())
In [54]:
cnf_matrix = confusion_matrix(y_test, pred_svc)
np.set_printoptions(precision=2)
plt.figure()
class_names = ['Not Failure', 'Failure']
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Support Vector Classifier - Normalized Confusion Matrix')
plt.show()
In [55]:
precision, recall, _ = precision_recall_curve(y_test, pred_svc)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Support Vector Classifier - 2-class Precision-Recall curve: AP={0:0.2f}'.format(precisionScore_svc))
Out[55]:
In [56]:
predsSVC = svc.predict_proba(X_test)[:,1]
fpr, tpr, _ = metrics.roc_curve(y_test, predsSVC)
roc_auc = metrics.auc(fpr, tpr)
plt.title('Support Vector Classifier - Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0,1], [0,1], 'r--')
plt.xlim([0,1])
plt.ylim([0,1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
In [ ]: