originalData.shape

originalData.info()

# kmeans and hierarchical analsysis require that datasets have no null / nan values.  the isnull 
# function shows us that there are no null / nan values

originalData.isnull().sum()

# Sl_N0 appears to be a numerical index for each row of data
# Each row of data also has a bank Customer Key
# these features will not help with clustering so can be dropped

originalData.head(10)

# The range of possible values for the average credit limit is quite a bit larger than the other 
# features so the dataset will need to be scaled prior to performing cluster analysis

originalData.describe()

# Temporarily record if a row of data is duplicated within the dataset
# Display duplicates organized by Customer key

preparedData["Duplicate"] = preparedData.duplicated(subset=['Customer Key'], keep=False)
preparedData[preparedData["Duplicate"] == True].sort_values(by=["Customer Key"])

# scale dataset and visually inspect

preparedData = preparedData.apply(zscore)
preparedData.head(10)

sns.pairplot(preparedData, diag_kind="kde");

correlations = preparedData.corr()
sns.heatmap(correlations, annot = True);

# run the k-means algorithm for 2-10 customer segments
clusters = range(2,10)

meanDistortions = []
kmeansSilhouetteScores = {}

# for each number of segments, group the customers into their respective segment
for k in clusters:
    model = KMeans(n_clusters = k)
    model.fit(preparedData)
    prediction = model.predict(preparedData)
    
    # calculate the silhouette scores to aid in determining which number of customer segments best 
    # organizes the customer base
    
    kmeansSilhouetteScores[k] = silhouette_score(preparedData, prediction)
    
    
    meanDistortions.append(sum(np.min(cdist(preparedData, model.cluster_centers_, 'euclidean'), axis=1)) / preparedData
                           .shape[0])

# plot the average distortion and look the for the elbow of the graph to approximate which value 
# of k most effectively organizes customers into segments such that customers within a segment are similar.

plt.plot(clusters, meanDistortions, 'bx-')
plt.xlabel('k')
plt.ylabel('Average distortion')
plt.title('Selecting k with the Elbow Method');
    

# { k : silhouette score}
kmeansSilhouetteScores

# run the model once more with k = 3
kmeans = KMeans(n_clusters=3, n_init=15, random_state=10)
kmeans.fit(preparedData)

# store the segment assignments to each customer row
# Note that segment 0 has a small number of customers 

unscaledPreparedData = unscaledPreparedData.join(segments)
unscaledPreparedData["kmeans segment"].value_counts()

preparedData.boxplot(by="kmeans segment", layout=(1,5), figsize=(20,10));

%%javascript
IPython.OutputArea.prototype._should_scroll = function(lines) {
    return false;
}

# run hierarchical clustering for each type of linkage method and use the cophenetic correlations to 
# determine which method is best

linkageMethods = ["single", "complete", "average", "weighted", "ward", "centroid"]

cophenetic_correlations = {}

bestC = 0

for method in linkageMethods:
    z = linkage(preparedData, metric="euclidean", method=method)
    c, coph_distances = cophenet(z, pdist(preparedData))
    cophenetic_correlations[method] = c

    # record the best method, z, and cophenetic correlation of the set for use in the subsequent analysis

    if c > bestC:
        bestC = c
        bestMethod = method
        bestZ = z

    # plot a dendrogram for each linkage method to show the clustering
    plt.figure(figsize=(20, 10))
    plt.title(method + " Hierarchical Clustering Dendogram")
    plt.xlabel("Sample Index")
    plt.ylabel("Distance")
    dendrogram(z, leaf_rotation=90.,color_threshold=600,  leaf_font_size=10. )
    plt.tight_layout()
    ax = plt.gca()
    ax.axes.xaxis.set_visible(False)
    plt.show()
    

cophenetic_correlations

# the best linkage method has the highest cophenetic correlation of 0.917
bestC

# the best linkage method is "average"
bestMethod

# plot the dendrogram using the "average" linkage method
# estimate the distance values in preparation for determing its optimal value, choosing distances values
# that segments customers into 2-10 groups
# plot a horizontal line for each distance value to visually confirm the number of customer 
# segments that will result when using this distance to perform clustering

plt.figure(figsize=(20, 10))
plt.title(bestMethod + " Hierarchical Clustering Dendogram")
plt.xlabel("Sample Index")
plt.ylabel("Distance")
dendrogram(bestZ, truncate_mode="lastp", p=10, leaf_rotation=90.,color_threshold=600,  leaf_font_size=10. )
plt.tight_layout()
ax = plt.gca()
ax.axes.xaxis.set_visible(False)
plt.axhline(y=5.8, color='g', linestyle='-')   # 1 customer segment
plt.axhline(y=5, color='b', linestyle='-')     # 2 customer segments
plt.axhline(y=3.3, color='g', linestyle='-')   # 3 customer segments
plt.axhline(y=3, color='b', linestyle='-')     # 4 customer segments
plt.axhline(y=2.7, color='g', linestyle='-')   # 5 customer segments
plt.axhline(y=2.5, color='b', linestyle='-')   # 6 customer segments
plt.axhline(y=2.35, color='g', linestyle='-')  # 7 customer segments
plt.axhline(y=2.25, color='b', linestyle='-')  # 8 customer segments
plt.axhline(y=2.05, color='g', linestyle='-')  # 9 customer segments
plt.axhline(y=1.80, color='b', linestyle='-') # 10 customer segments
plt.show()

hierarchicalSilhouetteScores

# clustering customers into 2 segments may not be that good of an idea since the vast majority of customers 
# would end up in a single segment and therefore be treated the same.

unscaledPreparedData["hierarchical best segment"].value_counts()

# a 3 segment clustering better distributes customers and may make it easier for the bank to tailor its 
# service approach and upselling more effectively

unscaledPreparedData["hierarchical 3 segments"].value_counts()

preparedData.boxplot(by="hierarchical best segment", layout=(1,5), figsize=(20,10));

preparedData.boxplot(by="hierarchical 3 segments", layout=(1,5), figsize=(20,10));