import nltk
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
sns.set(style="darkgrid")


import re
import json
import numpy as np
from random import shuffle
from sklearn.svm import SVC
from sklearn import neighbors
from nltk.tokenize import sent_tokenize
from sklearn.metrics import confusion_matrix

from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import train_test_split


from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import NearestNeighbors
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV

from sklearn.pipeline import Pipeline
from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score, log_loss, roc_auc_score

from sklearn.metrics import accuracy_score, recall_score,classification_report 

from imblearn.over_sampling import SMOTE
from imblearn.over_sampling import ADASYN
import warnings
warnings.filterwarnings('ignore')

# Define chunksize and initialize an empty DataFrame to store the results
chunksize = 10000

data_review = pd.DataFrame()
# Iterate over the file in chunks and concatenate the results
for chunk in pd.read_json('/Users/Ethan Vaz Falcao/Downloads/Yelp_data/Github/Yelp Datasets/yelp_academic_dataset_review.json', lines=True, chunksize=chunksize):
    data_review = pd.concat([data_review, chunk], ignore_index=True)

#set the year of the dataset  at 2021 for testing
#data_review_test = data_review[data_review['date'].dt.year >= 2018]
#data_review_test = data_review[data_review['date'].dt.year >= 2020]

data = data_review[['user_id', 'business_id','date','stars', 'text', "label"]]

#change date to date time
data['date'] = pd.to_datetime(data['date'])

#2. Percentage of reviews with positive / negative ratings (2)

#Assuming positive rating is rating >4 and negative rating is rating <4

# if a rating is less than 3 then label it negative, if greater than 3 then label it positive
# else neutral

data['rating_cat']=data['stars'].apply(lambda x:'negative' if x < 3 else 'positive' if x > 3 else 'neutral')

# count number of negative reviews for each user_id
neg_review_count=data.groupby('user_id')['rating_cat'].apply(lambda x: (x=='negative').sum()).reset_index(name='count')['count'].to_list()
# count number of positive reviews for each user_id
pos_review_count=data.groupby('user_id')['rating_cat'].apply(lambda x: (x=='positive').sum()).reset_index(name='count')['count'].to_list()
# count number of neutral reviews for each user_id
neu_review_count=data.groupby('user_id')['rating_cat'].apply(lambda x: (x=='neutral').sum()).reset_index(name='count')['count'].to_list()

review_df=data.groupby('user_id').count().reset_index()[['user_id','text']]

review_df['neg_review_count']=neg_review_count
review_df['pos_review_count']=pos_review_count
review_df['neu_review_count']=neu_review_count

review_df['neg_rating_percentage']=review_df['neg_review_count']/review_df['text']
review_df['pos_rating_percentage']=review_df['pos_review_count']/review_df['text']
review_df['neu_rating_percentage']=review_df['neu_review_count']/review_df['text']

review_df.drop(['neg_review_count','pos_review_count',
          'neu_review_count','text'],axis=1,inplace=True)

# merge review_df
data=pd.merge(data,review_df,on=['user_id'])

#3. Average review length 
data['review_len']=data['text'].apply(lambda x: len(x))

d=data.groupby('user_id')['review_len'].mean().reset_index()
d.rename(columns={'review_len':'avg_review_len'},inplace=True)
data=pd.merge(data,d,on=['user_id'])

#Standard deviation of ratings of the reviewer’s reviews 
d=data.groupby('user_id')['stars'].std().reset_index()
d.rename(columns={'stars':'std_rating'},inplace=True)
data=pd.merge(data,d,on=['user_id'])

data['std_rating'] = data['std_rating'].fillna(0)

def avg_word_length(x):
    sentence = x
    words = sentence.split()
    average = round(sum(len(word) for word in words) / len(words),2)
    return average

data['avg_word_length']=data['text'].apply(lambda x: avg_word_length(x))

def num_sent(x):
    number_of_sentences = sent_tokenize(x)
    return len(number_of_sentences)

data['num_sent']=data['text'].apply(lambda x: num_sent(x))

def percentage_of_numerals(x):
    numbers = sum(c.isdigit() for c in x)
    letter = sum(c.isalpha() for c in x)
    
    try: 
        result = numbers / letter
    except ZeroDivisionError:
        result=-1
    return result

data['percentage_of_numerals']=data['text'].apply(lambda x: percentage_of_numerals(x))

# if numbers/letter returns ZeroDivisionError
# this means there's 0 letter in the review
data[data['percentage_of_numerals']==-1]
#remove reviews that doesn't contain any words
data=data[data['percentage_of_numerals'] != -1 ]

# percentage of capitalized words
def percentage_of_capitalized(x):
    cap = sum(c.isupper() for c in x)
    letter = sum(c.isalpha() for c in x)
    
    try: 
        result = cap / letter
    except ZeroDivisionError:
        result=-1
    return cap/letter

data['percentage_of_capitalized']=data['text'].apply(lambda x: percentage_of_capitalized(x))

import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer

#clean up the review text
from nltk.tokenize import word_tokenize
import string
import re
from nltk.corpus import stopwords

def clean_text(text):
    """
        text: a string
        
        return: modified initial string
    """
    #tokenizer breaks string into a list of words
    text = word_tokenize(text)
    text = " ".join([c for c in text if c not in string.punctuation])
    text = text.lower() # lowercase text
    text = re.compile('"''#&?!:_[/(){}\[\]\|@,;.]').sub(' ', text) # replace symbols by space in text. substitute the matched string with space.
    text = re.sub(r'\d+','', text) # remove symbols and numbers
    text = ' '.join(word for word in text.split() if word not in stopwords.words('english')) # remove stopwors from text
    return text

data=data.reset_index()

data.rename(columns={"index": "review_id"},inplace=True)

data.drop(['user_id', 'business_id', 'date','rating_cat'],axis=1,inplace=True)
data = data.reindex(columns=['label','review_id', 'text', 'stars',
                                          'max_review_on_a_day',
                                          'review_count', 'neg_rating_percentage',
                                          'pos_rating_percentage', 'neu_rating_percentage', 'review_len',
                                          'avg_review_len', 'std_rating', 'avg_word_length', 'num_sent',
                                          'percentage_of_numerals', 'percentage_of_capitalized'])

def make_pipeline():
    "Create a single pipeline that processing the data and then fits the classification."  
    
    rf = RandomForestClassifier(random_state=50)
    gb=GradientBoostingClassifier(random_state=50)
    lr = LogisticRegression(random_state=50)
    classifiers=[rf,gb,lr]

    pipeline = []
    for classifier in classifiers:
        pipe = Pipeline(steps=[('classifier', classifier)])
        pipeline.append(pipe)

    return pipeline

def sort_models(pipelines, X_data, y_data):
    """Sort models based on their f1 score."""
    scores = []
    for pipe in pipelines:
        y_pred = pipe.predict(X_data)
        f1score = f1_score(y_data, y_pred, average='weighted')
        recallscore=recall_score(y_data,y_pred)
        accuracyscore=accuracy_score(y_data,y_pred)

        aucscore=roc_auc_score(y_data, pipe.predict_proba(X_data)[:,1])
        
        classfier_name = pipe.steps[-1][1].__class__.__name__.split('.')[-1]
        scores.append([classfier_name,f1score,recallscore,accuracyscore,aucscore])
        
    scores_sorted = sorted(scores,key=lambda x:x[1],reverse=True)
    
    return scores_sorted

def make_confusion_matrix(cf,
                          group_names=None,
                          categories='auto',
                          count=True,
                          percent=True,
                          cbar=True,
                          xyticks=True,
                          xyplotlabels=True,
                          sum_stats=True,
                          figsize=None,
                          cmap='Blues',
                          title=None):
    # CODE TO GENERATE TEXT INSIDE EACH SQUARE
    blanks = ['' for i in range(cf.size)]

    if group_names and len(group_names)==cf.size:
        group_labels = ["{}\n".format(value) for value in group_names]
    else:
        group_labels = blanks

    if count:
        group_counts = ["{0:0.0f}\n".format(value) for value in cf.flatten()]
    else:
        group_counts = blanks

    if percent:
        group_percentages = ["{0:.2%}".format(value) for value in cf.flatten()/np.sum(cf)]
    else:
        group_percentages = blanks

    box_labels = [f"{v1}{v2}{v3}".strip() for v1, v2, v3 in zip(group_labels,group_counts,group_percentages)]
    box_labels = np.asarray(box_labels).reshape(cf.shape[0],cf.shape[1])


    # CODE TO GENERATE SUMMARY STATISTICS & TEXT FOR SUMMARY STATS
    if sum_stats:
        #Accuracy is sum of diagonal divided by total observations
        accuracy  = np.trace(cf) / float(np.sum(cf))

        #if it is a binary confusion matrix, show some more stats
        if len(cf)==2:
            #Metrics for Binary Confusion Matrices
            precision = cf[1,1] / sum(cf[:,1]) #TP/TP+FP
#             How much were correctly classified as positive out of all positives.
            recall    = cf[1,1] / sum(cf[1,:]) #TP/TP+FN
            specificity = cf[0,0] / sum(cf[0,:]) #TN/TN+FP 
            f1_score  = 2*(precision*recall / (precision + recall))
        
            stats_text = "\n\nAccuracy={:0.3f}\nPrecision={:0.3f}\nRecall={:0.3f}\nSpecificity={:0.3f}\nF1 Score={:0.3f}".format(
                accuracy,precision,recall,specificity,f1_score)
        else:
            stats_text = "\n\nAccuracy={:0.3f}".format(accuracy)
    else:
        stats_text = ""


    # SET FIGURE PARAMETERS ACCORDING TO OTHER ARGUMENTS
    if figsize==None:
        #Get default figure size if not set
        figsize = plt.rcParams.get('figure.figsize')

    if xyticks==False:
        #Do not show categories if xyticks is False
        categories=False


    # MAKE THE HEATMAP VISUALIZATION
    plt.figure(figsize=figsize)
    sns.heatmap(cf,annot=box_labels,fmt="",cmap=cmap,cbar=cbar,xticklabels=categories,yticklabels=categories)

    if xyplotlabels:
        plt.ylabel('True label')
        plt.xlabel('Predicted label' + stats_text)
    else:
        plt.xlabel(stats_text)
    
    if title:
        plt.title(title)

X_up=data.iloc[:,3:]
y_up = data['label']

X_up_train, X_up_test, y_up_train, y_up_test = train_test_split(X_up, y_up,test_size=0.2, random_state=3)


X_smoted_train, y_smoted_train = SMOTE(random_state=42).fit_resample(X_up_train, y_up_train)
X_adasyn_train, y_adasyn_train = ADASYN(random_state=42).fit_resample(X_up_train, y_up_train)

smote_pipelines = make_pipeline()
# Train all the models
for pipe in smote_pipelines:
    pipe.fit(X_smoted_train, y_smoted_train)

# apply sort_models function from above
smoted_sorted_model = sort_models(smote_pipelines, X_up_test, y_up_test)
smoted_sort_model_df = pd.DataFrame(smoted_sorted_model)
smoted_sort_model_df.columns = ['Model','F1 Score','Recall Score','Accuracy Score','AUC Score']
smoted_sort_model_df = pd.melt(smoted_sort_model_df,id_vars=['Model'])

# best smote model performanxce -- gradientboosting
y_pred_smoted=smote_pipelines[1].predict(X_up_test)

adasyn_pipelines = make_pipeline()
# Train all the models
for pipe in adasyn_pipelines:
    pipe.fit(X_adasyn_train, y_adasyn_train)

# apply sort_models function from above
adasyn_sorted_model = sort_models(adasyn_pipelines, X_up_test, y_up_test)
adasyn_sort_model_df = pd.DataFrame(adasyn_sorted_model)
adasyn_sort_model_df.columns = ['Model','F1 Score','Recall Score','Accuracy Score','AUC Score']
adasyn_sort_model_df = pd.melt(adasyn_sort_model_df,id_vars=['Model'])