# Drop some useless columnsto_drop =['state','area_code','phone_number','churned']churn_feat_space = churn_df.drop(to_drop,axis=1)
查看数据的unbalance
看true的比重
# check the propotion of y = 1print(y.sum()/ y.shape *100)
Yes/No转换成True/False
# yes and no have to be converted to boolean valuesyes_no_cols =["intl_plan","voice_mail_plan"]churn_feat_space[yes_no_cols]= churn_feat_space[yes_no_cols]=='yes'
from sklearn import model_selection
# Reserve 20% for testing
X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.2)
print('training data has %d observation with %d features'% X_train.shape)
print('test data has %d observation with %d features'% X_test.shape)
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
# Logistic Regression
classifier_logistic = LogisticRegression()
# K Nearest Neighbors
classifier_KNN = KNeighborsClassifier()
# Random Forest
classifier_RF = RandomForestClassifier()
# SVM
classifier_SVC = SVC()
# Train the model
classifier_logistic.fit(X_train, y_train)
# Prediction of test data
classifier_logistic.predict(X_test)
# Accuracy of test data
classifier_logistic.score(X_test, y_test)
# Use 5-fold Cross Validation to get the accuracy for different models
model_names = ['Logistic Regression','KNN','Random Forest', 'SVM']
model_list = [classifier_logistic, classifier_KNN, classifier_RF, classifier_SVC]
count = 0
for classifier in model_list:
cv_score = model_selection.cross_val_score(classifier, X_train, y_train, cv=5)
# cprint(cv_score)
print('Model accuracy of %s is: %.3f'%(model_names[count],cv_score.mean()))
count += 1
from sklearn.model_selection import GridSearchCV
# helper function for printing out grid search results
def print_grid_search_metrics(gs):
print ("Best score: %0.3f" % gs.best_score_)
print ("Best parameters set:")
best_parameters = gs.best_params_
for param_name in sorted(parameters.keys()):
print("\t%s: %r" % (param_name, best_parameters[param_name]))
# Possible hyperparamter options for Logistic Regression Regularization
# Penalty is choosed from L1 or L2
# C is the lambda value(weight) for L1 and L2
parameters = {
'penalty':('l1', 'l2'),
'C':(1, 5, 10)
}
Grid_LR = GridSearchCV(LogisticRegression(),parameters, cv=5)
Grid_LR.fit(X_train, y_train)
best_LR_model = Grid_LR.best_estimator_
print_grid_search_metrics(Grid_LR)
# Possible hyperparamter options for KNN
# Choose k
parameters = {
'n_neighbors':[3,5,7,10]
'max_depth':[3,5]
}
Grid_KNN = GridSearchCV(KNeighborsClassifier(),parameters, cv=5)
Grid_KNN.fit(X_train, y_train)
print_grid_search_metrics(Grid_KNN)
# Possible hyperparamter options for Random Forest
# Choose the number of trees
parameters = {
'n_estimators' : [40,60,80]
}
Grid_RF = GridSearchCV(RandomForestClassifier(),parameters, cv=5)
Grid_RF.fit(X_train, y_train)
best_RF_model = Grid_RF.best_estimator_
print_grid_search_metrics(Grid_RF)
from sklearn.metrics import roc_curve
from sklearn import metrics
# Use predict_proba to get the probability results of Random Forest
y_pred_rf = best_RF_model.predict_proba(X_test)[:, 1]
fpr_rf, tpr_rf, _ = roc_curve(y_test, y_pred_rf)
# ROC curve of Random Forest result
plt.figure(1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_rf, tpr_rf, label='RF')
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve - RF model')
plt.legend(loc='best')
plt.show()
from sklearn import metrics
# AUC score
metrics.auc(fpr_rf,tpr_rf)
# add L1 regularization to logistic regression
# check the coef for feature selection
scaler = StandardScaler()
X_l1 = scaler.fit_transform(X)
LRmodel_l1 = LogisticRegression(penalty="l1", C = 0.1)
LRmodel_l1.fit(X_l1, y)
LRmodel_l1.coef_[0]
print ("Logistic Regression (L1) Coefficients")
for k,v in sorted(zip(map(lambda x: round(x, 4), LRmodel_l1.coef_[0]), \
churn_feat_space.columns), key=lambda k_v:(-abs(k_v[0]),k_v[1])):
print (v + ": " + str(k))
# add L2 regularization to logistic regression
# check the coef for feature selection
scaler = StandardScaler()
X_l2 = scaler.fit_transform(X)
LRmodel_l2 = LogisticRegression(penalty="l2", C = 5)
LRmodel_l2.fit(X_l2, y)
LRmodel_l2.coef_[0]
print ("Logistic Regression (L2) Coefficients")
for k,v in sorted(zip(map(lambda x: round(x, 4), LRmodel_l2.coef_[0]), \
churn_feat_space.columns), key=lambda k_v:(-abs(k_v[0]),k_v[1])):
print (v + ": " + str(k))
# check feature importance of random forest for feature selection
forest = RandomForestClassifier()
forest.fit(X, y)
importances = forest.feature_importances_
# Print the feature ranking
print("Feature importance ranking by Random Forest Model:")
for k,v in sorted(zip(map(lambda x: round(x, 4), importances), churn_feat_space.columns), reverse=True):
print (v + ": " + str(k))
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression, Lasso, Ridge
from sklearn.metrics import mean_squared_error, r2_score
from math import sqrt
lr_train, lr_validate = train_test_split(model_train, test_size=0.15, random_state = 0)
lr_train_x = lr_train.drop(['video_id', 'cvt_per_day'], axis = 1)
lr_validate_x = lr_validate.drop(['video_id', 'cvt_per_day'], axis = 1)
lr_train_y = lr_train['cvt_per_day']
lr_validate_y = lr_validate['cvt_per_day']
alphas = np.logspace (-0.3, 2.5, num=150)
# alphas= [0.000000001]
scores = np.empty_like(alphas)
opt_a = float('-inf')
max_score = float('-inf')
for i, a in enumerate(alphas):
lasso = Lasso()
lasso.set_params(alpha = a)
lasso.fit(lr_train_x, lr_train_y)
scores[i] = lasso.score(lr_validate_x, lr_validate_y)
if scores[i] > max_score:
max_score = scores[i]
opt_a = a
lasso_save = lasso
plt.plot(alphas, scores, color='b', linestyle='dashed', marker='o',markerfacecolor='blue', markersize=6)
plt.xlabel('alpha')
plt.ylabel('score')
plt.grid(True)
plt.title('score vs. alpha')
plt.show()
print ('The optimaized alpha and score of Lasso linear is: ', opt_a, max_score)
# combine the validate data and training data, use the optimal alpha, re-train the model
lasso_f = Lasso()
lasso_f.set_params(alpha = opt_a)
lasso_f.fit(model_train_x, model_train_y)
# lasso_f is the Lasso model (linear feature), to be tested with final test data.
from sklearn.preprocessing import PolynomialFeatures
poly = PolynomialFeatures(2)
lr_train, lr_validate = train_test_split(model_train, test_size=0.15, random_state = 0)
lr_train_x = lr_train.drop(['video_id', 'cvt_per_day'], axis = 1)
lr_validate_x = lr_validate.drop(['video_id', 'cvt_per_day'], axis = 1)
lr_train_xp = poly.fit_transform(lr_train_x)
lr_validate_xp = poly.fit_transform(lr_validate_x)
lr_train_y = lr_train['cvt_per_day']
lr_validate_y = lr_validate['cvt_per_day']
# lr_train_xp = pd.DataFrame(data=lr_train_xp, index=data[:], columns=data[0,1:])
alphas = np.logspace (-2.6, 2.5, num=80)
# alphas= [1]
scores = np.empty_like(alphas)
opt_a = float('-inf')
max_score = float('-inf')
for i, a in enumerate(alphas):
lasso = Lasso()
lasso.set_params(alpha = a)
lasso.fit(lr_train_xp, lr_train_y)
scores[i] = lasso.score(lr_validate_xp, lr_validate_y)
if scores[i] > max_score:
max_score = scores[i]
opt_a = a
lasso_save = lasso
plt.plot(alphas, scores, color='b', linestyle='dashed', marker='o',markerfacecolor='blue', markersize=6)
plt.xlabel('alpha')
plt.ylabel('score')
plt.grid(True)
plt.title('score vs. alpha')
plt.show()
print ('The optimaized alpha and score of Lasso polynomial is: ', opt_a, max_score)
# combine the validate data and training data, use the optimal alpha, re-train the model
lr_train_xp1 = poly.fit_transform(model_train_x)
lasso_fp = Lasso()
lasso_fp.set_params(alpha = opt_a)
lasso_fp.fit(lr_train_xp1, model_train_y)
# lasso_fp is the Lasso model (polynomial feature), to be tested with test data.
# combine the validate data and training data, use the optimal alpha, re-train the model
lr_train_xp1 = poly.fit_transform(model_train_x)
lasso_fp = Lasso()
lasso_fp.set_params(alpha = opt_a)
lasso_fp.fit(lr_train_xp1, model_train_y)
# lasso_fp is the Lasso model (polynomial feature), to be tested with test data.
# plot the effect of different number of trees and maximum tree-depth druing cross validation
scores = grid_rf.cv_results_['mean_test_score']
n_est = [54, 55, 56, 57, 58, 59, 60, 62]
m_depth=[12, 13, 14, 15, 16, 17]
scores = np.array(scores).reshape(len(m_depth), len(n_est))
fig = plt.figure()
ax = plt.subplot(111)
for ind, i in enumerate(m_depth):
plt.plot(n_est, scores[ind], '-o', label='M Depth' + str(i),)
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.xlabel('N Trees')
plt.ylabel('Mean Scores')
plt.grid(True)
plt.show()
# savefig('rf_1')
# add both training and validation data together as the new training data
rf = RandomForestRegressor(random_state=2, max_features = 'sqrt', max_depth= 14, n_estimators=55)
rf.fit(model_train_x, model_train_y)
# Lasso_f test (with linear features)
lasso_f_score = lasso_f.score(model_test_x, model_test_y)
pred_y = lasso_f.predict(model_test_x)
# The mean squared error and root mean square error
MSE_lasso_f = mean_squared_error(model_test_y, pred_y)
RMSE_lasso_f = sqrt(mean_squared_error(model_test_y, pred_y))
print ('lasso_f score: ', lasso_f_score)
print ('Mean square error of lasso_f: ', MSE_lasso_f)
print ('Root mean squared error of lasso_f:', RMSE_lasso_f)
# print ('Coefficients of lasso_f: ', lasso_f.coef_)
importances = rf.feature_importances_
std = np.std([tree.feature_importances_ for tree in rf.estimators_], axis=0)
indices = np.argsort(importances)[::-1]
feature_name = model_test_x.columns.get_values()
# Print the feature ranking
print("Feature importance ranking:")
for f in range(model_test_x.shape[1]):
print("%d. feature %d %s (%f)" % (f + 1, indices[f], feature_name[f], importances[indices[f]]))
plt.figure(1)
plt.bar(feature_name[:11], importances[indices[:11]])
plt.xticks(rotation=90)
plt.show()
# k-means clustering
from sklearn.cluster import KMeans
# number of clusters
num_clusters = 5
km = KMeans(n_clusters=num_clusters)
km.fit(tfidf_matrix)
clusters = km.labels_.tolist()
# create DataFrame films from all of the input files.
films = { 'title': titles, 'rank': ranks, 'synopsis': synopses, 'cluster': clusters}
frame = pd.DataFrame(films, index = [clusters] , columns = ['rank', 'title', 'cluster'])
print ("Number of films included in each cluster:")
frame['cluster'].value_counts().to_frame()
print ("<Document clustering result by K-means>")
#km.cluster_centers_ denotes the importances of each items in centroid.
#We need to sort it in decreasing-order and get the top k items.
order_centroids = km.cluster_centers_.argsort()[:, ::-1]
Cluster_keywords_summary = {}
for i in range(num_clusters):
print ("Cluster " + str(i) + " words:", end='')
Cluster_keywords_summary[i] = []
for ind in order_centroids[i, :6]: #replace 6 with n words per cluster
Cluster_keywords_summary[i].append(vocab_frame_dict[tf_selected_words[ind]])
print (vocab_frame_dict[tf_selected_words[ind]] + ",", end='')
print ()
#Here ix means index, which is the clusterID of each item.
#Without tolist, the values result from dataframe is <type 'numpy.ndarray'>
cluster_movies = frame.ix[i]['title'].values.tolist()
print ("Cluster " + str(i) + " titles (" + str(len(cluster_movies)) + " movies): ")
print (", ".join(cluster_movies))
print ()
# use pca to reduce dimensions to 2d for visibility, just want to see if there 2d can give us some insights
# this is not an appropriate method, just a guess.
pca = decomposition.PCA(n_components=2)
tfidf_matrix_np=tfidf_matrix.toarray()
pca.fit(tfidf_matrix_np)
X = pca.transform(tfidf_matrix_np)
xs, ys = X[:, 0], X[:, 1]
#set up colors per clusters using a dict
cluster_colors = {0: '#1b9e77', 1: '#d95f02', 2: '#7570b3', 3: '#e7298a', 4: '#66a61e'}
#set up cluster names using a dict
cluster_names = {}
for i in range(num_clusters):
cluster_names[i] = ", ".join(Cluster_keywords_summary[i])
# %matplotlib inline
#create data frame with PCA cluster results
df = pd.DataFrame(dict(x=xs, y=ys, label=clusters, title=titles))
groups = df.groupby(clusters)
# set up plot
fig, ax = plt.subplots(figsize=(16, 9))
#Set color for each cluster/group
for name, group in groups:
ax.plot(group.x, group.y, marker='o', linestyle='', ms=12,
label=cluster_names[name], color=cluster_colors[name],
mec='none')
ax.legend(numpoints=1,loc=4) #show legend with only 1 point, position is right bottom.
plt.show() #show the plot
