import json
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score, precision_score, recall_score
from sklearn.linear_model import LogisticRegression
from scipy.sparse import hstack

# IMPORTING ALL THE NECESSARY LIBRARIES AND PACKAGES
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
from nltk.corpus import stopwords
import string
import math
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score, roc_curve
from sklearn.model_selection import GridSearchCV
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')

chunksize = 10000
business_df = pd.DataFrame()

# Iterate over the file in chunks and concatenate the resultss\Yelp_data\Github\Yelp Datasets
for chunk in pd.read_json('/Users/Ethan Vaz Falcao/Downloads/Yelp_data/Github/Yelp Datasets/yelp_academic_dataset_business.json', lines=True, chunksize=chunksize):
    business_df = pd.concat([business_df, chunk], ignore_index=True)

review_df = pd.DataFrame()
# Iterate over the file in chunks and concatenate the results
for chunk in pd.read_json('/Users/Ethan Vaz Falcao/Downloads/Yelp_data/Github/Yelp Datasets/yelp_academic_dataset_review.json', lines=True, chunksize=chunksize):
    review_df = pd.concat([review_df, chunk], ignore_index=True)

tip_df = pd.DataFrame()

# Iterate over the file in chunks and concatenate the results
for chunk in pd.read_json('/Users/Ethan Vaz Falcao/Downloads/Yelp_data/Github/Yelp Datasets/yelp_academic_dataset_tip.json', lines=True, chunksize=chunksize):
    tip_df = pd.concat([tip_df, chunk], ignore_index=True)

# Extract relevant columns
business_attributes = business_df[['business_id', 'attributes']]
review_data = review_df[['user_id', 'text','date','business_id' ]]
tip_data = tip_df[['business_id', 'text', 'user_id','date']]
tip_data = tip_data.rename(columns={'text': 'tip'})

yelp_data = pd.merge(business_attributes, review_data, on='business_id')
yelp_data = pd.merge(yelp_data, tip_data, on=['business_id', 'user_id'])
yelp_data= yelp_data.dropna()

yelp_data = yelp_data.rename(columns={'date_x': 'date'})
yelp_data= yelp_data.drop('date_y', axis=1)

yelp_data = yelp_data.join(yelp_data['attributes'].apply(pd.Series))

# drop the original dictionary column
yelp_data = yelp_data.drop(['attributes','RestaurantsPriceRange2', 'CoatCheck',  'RestaurantsDelivery', 'Caters', 'WiFi', 'BusinessParking', 'WheelchairAccessible', 'HappyHour', 'OutdoorSeating', 'HasTV', 'RestaurantsReservations', 'DogsAllowed', 'ByAppointmentOnly', 'Alcohol', 'RestaurantsAttire', 'NoiseLevel', 'Ambience', 'GoodForKids', 'RestaurantsGoodForGroups', 'RestaurantsTableService', 'DriveThru', 'GoodForMeal', 'BusinessAcceptsBitcoin', 'Smoking', 'Music', 'GoodForDancing', 'BestNights', 'BYOB', 'Corkage', 'AcceptsInsurance', 'BYOBCorkage', 'HairSpecializesIn', 'Open24Hours', 'RestaurantsCounterService', 'AgesAllowed', 'DietaryRestrictions'], axis=1)

yelp_data = yelp_data.fillna(0).replace("True", 1)
yelp_data = yelp_data.replace(['False', 'None'], 0)
yelp_data= yelp_data.dropna()
yelp_data = yelp_data.reset_index(drop=True)

# filter the data by year
yelp_data= yelp_data.dropna()
yelp_data.fillna(value=0, inplace=True)
yelp_data = yelp_data.reset_index(drop=True)

# Define a lambda function to combine the values of the three columns
combine_cols = lambda row: [row['BusinessAcceptsCreditCards'], row['BikeParking'], row['RestaurantsTakeOut']]
# Apply the lambda function to each row of the DataFrame and store the result in a new column
yelp_data['combined_cols'] = yelp_data.apply(combine_cols, axis=1)

yelp_data = yelp_data.drop(['BusinessAcceptsCreditCards','BikeParking','RestaurantsTakeOut'], axis=1)
yelp_data = yelp_data.rename(columns={'combined_cols': 'attributes'})

#3. Average review length 
yelp_data['review_len']=yelp_data['text'].apply(lambda x: len(x))

d=yelp_data.groupby('user_id')['review_len'].mean().reset_index()
d.rename(columns={'review_len':'avg_review_len'},inplace=True)
yelp_data=pd.merge(yelp_data,d,on=['user_id'])

def avg_word_length(x):
    sentence = x
    words = sentence.split()
    average = round(sum(len(word) for word in words) / len(words),2)
    return average

yelp_data['avg_word_length']=yelp_data['text'].apply(lambda x: avg_word_length(x))

import nltk
from nltk.tokenize import sent_tokenize
def num_sent(x):
    number_of_sentences = sent_tokenize(x)
    return len(number_of_sentences)

yelp_data['num_sent']=yelp_data['text'].apply(lambda x: num_sent(x))

def percentage_of_numerals(x):
    numbers = sum(c.isdigit() for c in x)
    letter = sum(c.isalpha() for c in x)
    
    try: 
        result = numbers / letter
    except ZeroDivisionError:
        result=-1
    return result

yelp_data['percentage_of_numerals']=yelp_data['text'].apply(lambda x: percentage_of_numerals(x))

import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer

#clean up the review text
from nltk.tokenize import word_tokenize
import string
import re
from nltk.corpus import stopwords

def clean_text(text):
    """
        text: a string
        
        return: modified initial string
    """
    #tokenizer breaks string into a list of words
    text = word_tokenize(text)
    text = " ".join([c for c in text if c not in string.punctuation])
    text = text.lower() # lowercase text
    text = re.compile('"''#&?!:_[/(){}\[\]\|@,;.]').sub(' ', text) # replace symbols by space in text. substitute the matched string with space.
    text = re.sub(r'\d+','', text) # remove symbols and numbers
    text = ' '.join(word for word in text.split() if word not in stopwords.words('english')) # remove stopwors from text
    return text

yelp_data=yelp_data.reset_index()

# Step 2: Feature Extraction
vectorizer = TfidfVectorizer(stop_words='english')
features = vectorizer.fit_transform(yelp_data['text'])

# Create feature matrix
feature_matrix = pd.DataFrame(features.toarray(), columns=vectorizer.get_feature_names())

# Step 3: Label Encoding
def parse_attributes(x):
    if isinstance(x, str):
        return eval(x)
    else:
        return x
yelp_data['attributes'] = yelp_data['attributes'].apply(parse_attributes)

from sklearn.preprocessing import MultiLabelBinarizer
mlb = MultiLabelBinarizer()
label_matrix = pd.DataFrame(mlb.fit_transform(yelp_data['attributes']), columns=mlb.classes_)

# Step 4: Model Training
X_train, X_test, y_train, y_test = train_test_split(feature_matrix, label_matrix, test_size=0.3, random_state=42)

#randomly chooses a review from the df
import random
def random_review(df):
    """
    Returns a random text from the "text" column of the given DataFrame.
    """
    # Get the "text" column as a Series
    text_series = yelp_data["text"]
    
    # Choose a random index from the Series
    random_index = random.choice(text_series.index)
    
    # Get the text at the chosen index
    random_text = text_series[random_index]
    
    return random_text