#!/usr/bin/env python
# coding: utf-8
"""
for rolling updates:
    - python3 setup.py bdist_wheel sdist
    - twine upload dist/*

"""
from welcome_page import *
from variables import *
from import_funcs import *
import pandas as pd 
import numpy as np
import re
import pandas as pd
import numpy as np
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
from xgboost import XGBClassifier
import statistics
import lightgbm as lgb
import sys
from joblib import Parallel, delayed
import warnings
warnings.filterwarnings("ignore")
warnings.simplefilter(action='ignore', category=FutureWarning)
import multiprocessing

def get_internal_similarities(df):
    df = df.head(2000).copy()
    untouched_df = df.copy()
    print("This is a free version, and supports only upto 2000 rows. Remaining data will be truncated!")
    print("For commercial usage please mail us at hello@linkedlabs.co")
    lis = df.columns
    cat,num,other = cato_numo_other(df)
    for i in cat+num:
        if(df[i].isnull().any()):
            if i in cat:
                df = missing_value_treatment(df,i,cat,num,other,"cat")
            elif i in num:
                df = missing_value_treatment(df,i,cat,num,other,"num")
    df.drop(columns=other,inplace=True)
    features_we_considered = df.columns
    bins = int(np.log(df.shape[0]))
    for _ in num:
        df[_] = pd.qcut(df[_],bins)
    # convert all the catagorical col whose no of cat is greater then 2% then tag it into others 
    # one hot encode the whole df
    df = pd.get_dummies(data=df, columns=df.columns)
    assert df.shape[0] == untouched_df.shape[0]
    df.reset_index(inplace=True,drop=True)
    untouched_df.reset_index(inplace=True,drop=True)
    r = Parallel(n_jobs=multiprocessing.cpu_count()*2, verbose=0)(delayed(get_most_similar_vectors)(df.iloc[_].to_numpy(),df.to_numpy(),_) for _ in tqdm(df.index.values))
    df_scores = pd.concat([pd.DataFrame.from_dict(x, orient='index') for x in tqdm(r)])
    df_scores.reset_index(inplace=True,drop=True)
    df_scores.columns = ["best_similarity_score","best_similar_with_index_number","first_array","similar_array"]
    fin_df = pd.concat([untouched_df,df_scores[["best_similarity_score","best_similar_with_index_number"]]],axis=1)
    # clear()
    # print("your similarity scores are stored in 'best_similarity_score' column and it is most similar with row number stored in 'best_similar_with_index_number' column.(you can use .iloc[index_num] to look at the similar value)")
    return fin_df
def get_similarities(df1,df2):
    df1 = df1.head(1000)
    df2 = df2.head(5000)
    print("This is a free version, and supports only upto 1000 rows and will be compared against first 5000 rows in the 2nd dataframe. Remaining data will be truncated!")
    print("For commercial usage please mail us at hello@linkedlabs.co")
    org_df1 = df1.copy()
    org_df2 = df2.copy()
    if set(df1.columns) != set(df2.columns):
        print("columns of both dataframes are different!")
        exit()
    df1["which_df"]= 'df1'
    df2["which_df"]='df2'
    df =df1.append(df2)
    untouched_df = df1.copy()
    untouched_df_2 = df2.copy()
    cat,num,other = cato_numo_other(df)
    for i in cat+num:
        if(df[i].isnull().any()):
            print(i)
            if i in cat:
                df = missing_value_treatment(df,i,cat,num,other,"cat")
            elif i in num:
                df = missing_value_treatment(df,i,cat,num,other,"num")
    df.drop(columns=other,inplace=True)
    bins = int(np.log(df.shape[0]))
    for _ in num:
        df[_] = pd.qcut(df[_],bins)
    df = pd.get_dummies(data=df, columns=df.columns)
    df1 = df[df['which_df_df1'] ==1]
    df2=df[df['which_df_df2']==1]
    df1.drop(['which_df_df1','which_df_df2'],axis=1,inplace=True)
    df2.drop(['which_df_df1','which_df_df2'],axis=1,inplace=True)
    df1.reset_index(inplace=True,drop=True)
    df2.reset_index(inplace=True,drop=True)
    untouched_df.reset_index(inplace=True,drop=True)
    untouched_df_2.reset_index(inplace=True,drop=True)
    r = Parallel(n_jobs=multiprocessing.cpu_count()*8, verbose=0)(delayed(get_most_similar_vectors)(df1.iloc[_].to_numpy(),df2.to_numpy(),_) for _ in tqdm(df1.index.values))
    df_scores = pd.concat([pd.DataFrame.from_dict(x, orient='index') for x in tqdm(r)])
    df_scores.reset_index(inplace=True,drop=True)
    df_scores.columns = ["best_similarity_score","best_similar_with_index_number","first_array","similar_array"]
    fin_df = pd.concat([untouched_df,df_scores[["best_similarity_score","best_similar_with_index_number"]]],axis=1)
    fin_df.drop(['which_df'],axis=1,inplace=True)
    return fin_df , untouched_df_2
