import pandas as pd
from IPython.display import Markdown, display, HTML
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import  scipy.stats as stats
from statsmodels.stats.outliers_influence import variance_inflation_factor
    
# helper class that facilitates completing EDA, particularly producing data summaries and visualizations for numerical and categorical 
# features.  The class is instantiated with (mostly) unprepared data and then produces reports via its method calls for features

class ExploratoryDataAnalysis:
    
    dataframe = None
    
    def __init__(self, dataframe): 
        self.dataframe = dataframe.copy()
    
    def setCategoricalFeatures(self, features):
        self.categoricalFeatures = features
    
    def setNumericalFeatures(self, features):
        self.numericalFeatures = features
        
    def setTargetFeature(self, targetFeature, isCategorical):
        self.targetFeature = targetFeature
        self.isCategoricalTarget = isCategorical
        
    def summary(self):
        markdown = "| Feature | % of nulls | Type| Number of Unique|\n"
        markdown = markdown + "|---|:---:|:---:|:---:|\n"
        
        totalInstances = self.dataframe.shape[0]
        
        for featureName in self.dataframe.columns:
            markdown = markdown + "| " + featureName
            markdown = markdown + " | "+ str(self.dataframe[featureName].isnull().sum()/totalInstances).format(1./1.)
            markdown = markdown + " | " + str(self.dataframe[featureName].dtype)
            markdown = markdown + " | " + str(self.dataframe[featureName].nunique()) + " |\n"
        
        display(Markdown(markdown))

# Numerical Feature Methods --------------------------------------------------------------------------- 

   
    # Calculating VIF -------------------------------------------------------------------------------------------------------------------
    # VIF starts at 1 and has no upper limit
    # VIF = 1, no correlation between the independent variable and the other variables
    # VIF exceeding 5 or 10 indicates high multicollinearity between this independent variable and the others
    
    def VIFSummary(self):

        vif = pd.DataFrame()
        vif["Feature"] = self.dataframe[self.numericalFeatures].columns
        vif["VIF"] = [variance_inflation_factor(self.dataframe[self.numericalFeatures].values, i) for i in range(self.dataframe[self.numericalFeatures].shape[1])]
        return vif

    # ------------------------------------------------------------------------------------------------------------------------------------
    def calculateVIF(self, features):
        vif = pd.DataFrame()
        vif["Feature"] = self.dataframe[features].columns
        vif["VIF"] = [variance_inflation_factor(self.dataframe[features].values, i) for i in range(self.dataframe[features].shape[1])]
        return vif
    
    # ------------------------------------------------------------------------------------------------------------------------------------
    def plotCorrelations(self):
        corr = abs(self.dataframe.corr()) # correlation matrix
        lower_triangle = np.tril(corr, k = -1)  # select only the lower triangle of the correlation matrix
        mask = lower_triangle == 0  # to mask the upper triangle in the following heatmap

        plt.figure(figsize = (15,8))  # setting the figure size
        sns.set_style(style = 'white')  # Setting it to white so that we do not see the grid lines
        
        # cmap = "Blues"
        cmap = sns.diverging_palette(230, 20, as_cmap=True)
        
        sns.heatmap(lower_triangle, center=0.5, cmap=cmap, annot= True, xticklabels = corr.index, yticklabels = corr.columns,
            cbar= False, linewidths= 1, mask = mask)   # Da Heatmap
        plt.xticks(rotation = 90)   # Aesthetic purposes
        plt.yticks(rotation = 0)   # Aesthetic purposes
        plt.show()
    
    # ------------------------------------------------------------------------------------------------------------------------------------
    def generateNumericalBinsTable(self, feature, numberOfBins):
        
        max = self.dataframe[feature].max()
        min = self.dataframe[feature].min()
        
        binSize = (max - min) / numberOfBins
    
        bins = [min]
        
        for i in range(1,numberOfBins):
            bins.append(min + i*binSize)
            
        bins.append(max)
    
        self.dataframe[feature + "-bin"] = pd.cut(x=self.dataframe[feature], bins=bins, labels=bins[:-1])
        
        if self.isCategoricalTarget == True:
            table = self.dataframe[self.dataframe[self.targetFeature] == True][feature + "-bin"].value_counts() / self.dataframe[feature + "-bin"].value_counts()
            
        else:
            table = self.dataframe[feature + "-bin"].value_counts()
        
        self.dataframe.drop(feature + "-bin", axis=1)
        
        return table
    
    # ------------------------------------------------------------------------------------------------------------------------------------
    def numericalFeatureSummary(self):
        
        markdown = "<hr><H2>Numerical Feature Summary</H2><hr>"
        
        for feature in self.numericalFeatures:
            markdown = markdown + "<H3>" + feature + "</H3>"
            markdown = markdown + "<table style=\"width: 100%; background-color:#ffffff\"><tr><td style=\"width:25%; padding-right: 20px;\">"
            markdown = markdown + self.generateDescribeStats(feature) + "</td>"
            markdown = markdown + "<td style=\"width:40%; padding-right: 20px;\">" + self.generateAdditionalStats(feature) + "</td>"
            markdown = markdown + "<td>" + self.generateCorrStats(feature) + "</td></tr></table>" 
        
        
        display(HTML("<style>.output_result { height:auto !important; max-height:500px; } </style>" + markdown))
        
    # -----------------------------------------------------------------------------------------------------------------------------------
    def analyzeNumericalFeature(self, feature):
        self.analyzeNumericalFeature(feature, False)
        
    # provide a report with data tables and plots for a given numerical feature ----------------------------------------------------------
    def analyzeNumericalFeature(self, feature, plot):
        markdown = "<hr><H2>" + feature + " -- Numerical Feature Analysis </H2><hr><H3>Summary</H3>" 
        markdown = markdown + "<table style=\"width: 100%; background-color:#ffffff\"><tr><td style=\"width:25%; padding-right: 20px;\">"
        markdown = markdown + self.generateDescribeStats(feature) + "</td>"
        markdown = markdown + "<td style=\"width:40%; padding-right: 20px;\">" + self.generateAdditionalStats(feature) + "</td>"
        markdown = markdown + "<td>" + self.generateCorrStats(feature) + "</td></tr></table>" 
        
        markdown = markdown + "<H3>Notes</H3>" + self.generateCategoricalNotes(feature)
        markdown = markdown + "<H3>Plots with Numerical Features</H3>"
        
        display(HTML("<style>.output_result { height:auto !important; max-height:500px; } </style>" + markdown))
        
        if plot == True:
            self.displayPlotsForNumericalFeature(feature, True)
     
    # provide the correlations that a feature has with all of the other numerical features in the data set -------------------------------
    def generateCorrStats(self, feature):
        
        html = "<table style=\"width: 100%; border:1px black solid;\"><tr><th>Feature</th><th>Original Correlation</th><th>Adjusted Correlation</th></tr>"
        
        for numericalFeature in self.numericalFeatures:
            if (feature != numericalFeature):
                corr = self.dataframe[feature].corr(self.dataframe[numericalFeature])
                html = html + "<tr><td>" + numericalFeature + "</td><td>" + "{:.2f}".format(corr) + "</td><td>TODO</td></tr>"
        
        html = html + "</table>"
        
        return html
    
    # provide summary statistics to assist with identifying data clean needs -------------------------------------------------------------   
    def generateAdditionalStats(self, feature):
        html = "<table style=\"width: 100%; border:1px black solid;\"><tr><th>Statistic</th><th>Original Value</th><th>Adjusted Value</th>"
        
        html = html + "<tr><td>Datatype</td><td>" + str(self.dataframe[feature].dtype) + "</td><td>" + "TODO" + "</td></tr>"
        html = html + "<tr><td># of Null</td><td>" + str(self.dataframe[feature].isnull().sum()) + "</td><td>" + "TODO" + "</td></tr>"
        
        html = html + "<tr><td># of 0</td><td>" + str(self.dataframe[self.dataframe[feature] ==0].shape[0]) + "</td><td>" + "TODO" + "</td></tr>"
        html = html + "<tr><td># Less Than 0</td><td>" + str(self.dataframe[self.dataframe[feature] < 0].shape[0]) + "</td><td>" + "TODO" + "</td></tr>"
        
        totalInstances = self.dataframe[feature].describe()["count"]
        
        iqr = self.dataframe[feature].describe()["75%"] - self.dataframe[feature].describe()["25%"]
        
        lowOutlierValue = self.dataframe[feature].describe()["25%"] - 1.5*iqr
        numLowOutliers = self.dataframe[self.dataframe[feature] < lowOutlierValue][feature].count()
        percentLowOutliers = 100*numLowOutliers/totalInstances
        html = html + "<tr><td># of Low Outliers</td><td>" + str(numLowOutliers) + " (" + "{:.2f}".format(percentLowOutliers) + "%)</td>" 
        html = html + "<td>" + "TODO" + "</td></tr>"
        
        highOutlierValue = self.dataframe[feature].describe()["75%"] + 1.5*iqr
        numHighOutliers = self.dataframe[self.dataframe[feature] > highOutlierValue][feature].count()
        percentHighOutliers = 100*numHighOutliers / totalInstances
        html = html + "<tr><td># of High Outliers</td><td>" + str(numHighOutliers) + " (" + "{:.2f}".format(percentHighOutliers) + "%)</td>"
        html = html + "<td>" + "TODO" + "</td></tr>"
        
        html = html + "</table>"
        
        return html
    # provide summary statistics like df.describe for a numeric feature -----------------------------------------------------------------
    def generateDescribeStats(self, feature):
        
        describe = self.dataframe[feature].describe()
        
        html = "<table style=\"width: 100%; border:1px black solid;\"><tr><th>Statistic</th><th>Original Value</th><th>Adjusted Value</th></tr>"
            
        for stat in describe.index: 
            html = html + "<tr><td>" + stat + "</td><td>" + "{:.2f}".format(describe[stat]) + "</td><td>TODO</td></tr>"
        
        html = html + "<tr><td>Skewness</td><td>" +"{:.2f}".format(self.dataframe[feature].skew()) + "</td><td>TODO</td></tr>"
        
        corr = self.dataframe[feature].corr(self.dataframe[self.targetFeature])
        html = html + "<tr><td>" + self.targetFeature + " Correlation</td><td>" +"{:.2f}".format(corr) + "</td><td>TODO</td></tr>"
        
        html = html + "</table>"
        
        return html
    # Plot the relationship bettween a numerical feature and other numerical features, as well as plots with the target feature
    def displayPlotsForNumericalFeature(self, featureToAnalyze, plotDensity):
        # assumes the target is categorical, TODO add support for numerical targets
        
        
        if featureToAnalyze in set(self.numericalFeatures):
            adjustment = 1
        else:
            adjustment = 0
        
        rows = int(np.floor((len(self.numericalFeatures)-adjustment)/2))

       
        
        if np.mod(len(self.numericalFeatures)-adjustment, 2) > 0:
            rows = int(rows + 1)

        if plotDensity == True:
            rows = rows + 1
            
        fig, ax = plt.subplots(rows,2, figsize=(20,20))  
        

        if plotDensity == True:
           # fig, ax = plt.subplots(1 ,2, figsize=(15,5))
        
            ax[0][0] = sns.distplot(self.dataframe[featureToAnalyze], ax=ax[0][0])
       
            # NUMERICAL / CATEGORICAL SWITCH TODO
        
            if self.isCategoricalTarget == True:
                ax[0][1] = sns.distplot(self.dataframe[self.dataframe[self.targetFeature]==False][featureToAnalyze],color='r',label=0, ax=ax[0][1])
                ax[0][1] = sns.distplot(self.dataframe[self.dataframe[self.targetFeature]==True][featureToAnalyze],color='g',label=1, ax=ax[0][1]) 
            else:
                 ax[0][1] = sns.regplot(x=featureToAnalyze, y=self.targetFeature, data=self.dataframe, ax=ax[0][1])
                    
            handles, labels = ax[0][1].get_legend_handles_labels()
            ax[0][1].legend(handles, labels)
        
            plt.close()

        i = 0
        c = 0

        for feature in self.numericalFeatures:
            
            if feature != featureToAnalyze:
                r = int(np.floor(i/2)) + 1
                c = int(np.mod(i, 2))
    
                ax[r][c] = sns.regplot(x=feature, y=featureToAnalyze, data=self.dataframe, ax=ax[r][c])
    
                plt.close()
                i = i + 1
    
        for a in fig.axes:
            plt.sca(a)
            plt.xticks(rotation=90)

        fig.subplots_adjust(hspace=0.5, wspace=0.1)

        fig.show()
        
    # ----------------------------------------------------------------------------------------------------------------------------------    
    def displayDensityPlotsForNumericalFeatures(self, featuresToAnalyze):
        # assumes the target is categorical, TODO add support for numerical targets

        for featureToAnalyze in featuresToAnalyze:
            

        
            fig, ax = plt.subplots(1 ,2, figsize=(15,5))
        
            ax[0] = sns.distplot(self.dataframe[featureToAnalyze], ax=ax[0])
           
        
            ax[1] = sns.distplot(self.dataframe[self.dataframe[self.targetFeature]==False][featureToAnalyze],color='r',label="Target 0", ax=ax[1])
        
        
            ax[1] = sns.distplot(self.dataframe[self.dataframe[self.targetFeature]==True][featureToAnalyze],color='g',label="Target 1", ax=ax[1]) 
            
            handles, labels = ax[1].get_legend_handles_labels()
            ax[1].legend(handles, labels)
            
            
            fig.show()
            
            

    # --------------------------------------------------------------------------------------------------------------------------------
    def analyzeSkew(self):
        
        skews = pd.DataFrame()
        skews["OriginalSkew"] = self.dataframe[self.numericalFeatures].skew()

        for feature in self.numericalFeatures:
            minimum = self.dataframe[feature].min()
            if minimum > 0:
                minimum = 0
            skews.at[feature, "LnSkew"] = self.dataframe[feature].apply(lambda x: np.log(x-minimum +1 )).skew()
            skews.at[feature, "SqRtSkew"] = self.dataframe[feature].apply(lambda x: np.sqrt(x-minimum +1 )).skew()
            
        return skews



# Categorical Feature Methods ----------------------------------------------------------------------------------------------------  

    def categoricalFeatureSummary(self):
        
        markdown = "<hr><H2>Categorical Feature Summary</H2><hr>"
            
        for feature in self.categoricalFeatures:
         
            markdown = markdown + "<H3>" + feature + "</H3>" 
            markdown = markdown + "<table style=\"width: 100%; background-color:#ffffff\"><tr><td style=\"width:30%; padding-right: 20px;\">"
            markdown = markdown + self.generateCategoricalSummaryStats(feature) + "</td>"
            markdown = markdown + "<td style=\"width:36%; padding-right: 20px;\">" + self.generateContingencyTable(feature) + "</td>"
            markdown = markdown + "<td>" + self.generateValueCountsTable(feature) + "</td></tr></table>" 
             
        display(HTML("<style>.output_result { height:auto !important; max-height:500px; } </style>" + markdown))
    
    # ----------------------------------------------------------------------------------------------------------------------------------
    def analyzeCategoricalFeature(self, feature):
        self.analyzeCategoricalFeature(feature, False)
    
    # ----------------------------------------------------------------------------------------------------------------------------------
    def analyzeCategoricalFeature(self, feature, plot):
        markdown = "<hr><H2>" + feature + " -- Categorical Feature Analysis </H2><hr><H3>Summary</H3>" 
        markdown = markdown + "<table style=\"width: 100%; background-color:#ffffff\"><tr><td style=\"width:30%; padding-right: 20px;\">"
        markdown = markdown + self.generateCategoricalSummaryStats(feature) + "</td>"
        markdown = markdown + "<td style=\"width:36%; padding-right: 20px;\">" + self.generateContingencyTable(feature) + "</td>"
        markdown = markdown + "<td>" + self.generateValueCountsTable(feature) + "</td></tr></table>" 
        
        markdown = markdown + "<H3>Notes</H3>" + self.generateCategoricalNotes(feature)
        markdown = markdown + "<H3>Plots with Numerical Features</H3>"
        
        display(HTML("<style>.output_result { height:auto !important; max-height:500px; } </style>" + markdown))
        
        if plot == True:
            self.displayPlotsWithNumericalFeatures(feature)
            
    # ---------------------------------------------------------------------------------------------------------------------------------
    def generateCategoricalNotes(self,feature):
        return "TODO"
    
    # ---------------------------------------------------------------------------------------------------------------------------------
    def generateCategoricalSummaryStats(self, feature):
       # markdown = "<table style=\"width: 100%; border:1px black solid;\"><tr><th>Value</th><th>" + tf + " % True </th><th>" + tf + " % False</th></tr>"
        return "TODO"
    
    # provide the contingency table for a categorical feature ------------------------------------------------------------------------
    def generateContingencyTable(self, feature):
        
        crosstab = pd.crosstab(self.dataframe[feature], self.dataframe[self.targetFeature], normalize="index")
        
        tf = self.targetFeature
        
        markdown = "<table style=\"width: 100%; border:1px black solid;\"><tr><th>Value</th><th>" + tf + " % True </th><th>" + tf + " % False</th></tr>"
        for index, row in crosstab.iterrows():
            markdown = markdown + "<tr><td>" + str(index) + "</td><td>" + "{:.1f}".format(100*row[True]) + "</td><td>" + "{:.1f}".format(100*row[False]) + "</td></tr>"
        
        markdown = markdown + "</table>"
        
        return markdown
    # subset the sample size of a numerical feature for the purpose of making plotting faster but still insightful vai random sampling
    def prepareNumericalFeatureSample(self, feature):
        # check if in numerical TODO
        
        sample = self.dataframe.sample(min(1000, int(self.dataframe.shape[0]/10)))
        sample = sample[np.abs(sample[feature] - sample[feature].mean())/sample[feature].std() < 3]

        return sample
    # plot categorical variable relationships with all numerical features ----------------------------------------------------------------
    def displayPlotsWithNumericalFeatures(self, categoricalFeature):
      
        rows = int(np.floor(len(self.numericalFeatures)/2))

        if np.mod(len(self.numericalFeatures), 2) > 0:
            rows = int(rows + 1)

        fig, ax = plt.subplots(rows,2, figsize=(20,20))

        i = 0
        c = 0

        for feature in self.numericalFeatures:
            r = int(np.floor(i/2))
            c = int(np.mod(i, 2))
    
            ax[r][c] = sns.swarmplot(x=categoricalFeature, y=feature, hue="Target", data=self.prepareNumericalFeatureSample(feature), ax=ax[r][c])
    
    
            plt.close()
            i = i + 1
    
        for a in fig.axes:
            plt.sca(a)
            plt.xticks(rotation=90)

        fig.subplots_adjust(hspace=0.5, wspace=0.1)

        fig.show()
        
    # compute the counts for categorical variable values -------------------------------------------------------------------------------
    def generateValueCountsTable(self, feature):
        
        value_counts = self.dataframe[feature].value_counts()
        totalInstances = value_counts.sum()
        
        markdown = "<table style=\"width: 100%; border:1px black solid; \"><tr><th>Value</th><th>Count</th><th>% of Total</th></tr>"
        for itype in value_counts.index:    
            markdown = markdown + "<tr><td>" + str(itype) + "</td><td>" + str(value_counts[itype]) + "</td>"
            markdown = markdown + "<td>" + "{:.1f}".format(100*value_counts[itype]/totalInstances) + "</td></tr>"
        markdown = markdown + "</table>"
        return markdown