import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

df = pd.read_csv('land_train.csv') # loading the dataset
df.head()

dfs = df.sample(frac=0.1).reset_index(drop=True) # shuffles the data and takes a fraction of it
dfs.shape

# scatter plot of a part of data
pd.plotting.scatter_matrix(dfs[['X1', 'X2', 'X3', 'X4', 'X5', 'X6', 'I1', 'I2', 'I3', 'I4', 'I5', 'I6']], figsize=(17,10));

import seaborn as sns
sns.set(style="white")

# finding correlation
corr = df[['X1', 'X2', 'X3', 'X4', 'X5', 'X6', 'I1', 'I2', 'I3', 'I4', 'I5', 'I6']].corr()

# Generate a mask for the upper triangle
mask = np.zeros_like(corr, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True

# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(11, 9))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=1, center=0.5, square=True, linewidths=.5, cbar_kws={"shrink": .5});

# preprocessing columns
df['X'] = df[['X1', 'X2', 'X3', 'X4', 'X5', 'X6']].mean(axis=1) # merging all X features in one
df = df.drop(['X1', 'X2', 'X3', 'X4', 'X5', 'X6'], axis=1)      # dropping the rest
df['I14'] = df[['I1', 'I4']].mean(axis=1)                       # averaging I1 and I4
df = df.drop(['I1', 'I4', 'I2', 'I5'], axis=1)                  # dropping the rest
df.head()

df.to_csv('land_train_pruned.csv', index=False) # saving the data

df.describe()

df.target.value_counts().plot('bar');

from sklearn import preprocessing

x = df[['I3', 'I6', 'I14', 'X']].values # returns a numpy array
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(x)
df_n = pd.DataFrame(x_scaled, columns=['I3', 'I6', 'I14', 'X'])
df_norm = df[['clusterID', 'target']].join(df_n)

df_norm.head()

# separating classes
df_1 = df_norm[df_norm.target==1]
df_1 = df_1.sample(frac=1).reset_index(drop=True) # shuffling rows
df_3 = df_norm[df_norm.target==3]

df_1.shape[0], df_3.shape[0]

df_1_split = np.split(df_1, np.arange(47300, 472987, 47300),axis=0) # splits dataframe (of Target=1) in 10 
[i.shape for i in df_1_split]

df_3_dup = pd.concat([df_3]*3, ignore_index=True) # duplicate data (of Target=3) 3 times
df_3_dup.shape

for i in range(len(df_1_split)):
    df_1_split[i] = df_1_split[i].append(df_3_dup, ignore_index=True) # merge Target 1 and 3

[i.shape for i in df_1_split]

df_1_split[9].target.value_counts().plot('bar');

# saving all parts
for i in range(len(df_1_split)):
    df_1_split[i].to_csv(f'land_train_split_{i+1}.csv', index=False)

df_test = pd.read_csv('land_test.csv')
df_test.shape

df_test.head()

# preprocessing columns
df_test['X'] = df_test[['X1', 'X2', 'X3', 'X4', 'X5', 'X6']].mean(axis=1) # merging all X features in one
df_test = df_test.drop(['X1', 'X2', 'X3', 'X4', 'X5', 'X6'], axis=1)      # dropping the rest
df_test['I14'] = df_test[['I1', 'I4']].mean(axis=1)                       # averaging I1 and I4
df_test = df_test.drop(['I1', 'I4', 'I2', 'I5'], axis=1)                  # dropping the rest

x_test = df_test[['I3', 'I6', 'I14', 'X']].values # returns a numpy array
x_test_scaled = min_max_scaler.transform(x_test)  # applying scaler
df_test_n = pd.DataFrame(x_test_scaled, columns=['I3', 'I6', 'I14', 'X'])
df_test_norm = df_test[['clusterID']].join(df_test_n)
df_test_norm.shape

df_test_norm.head()

df_test_norm.to_csv('land_test_preprocessed.csv')

from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
from sklearn.naive_bayes import BernoulliNB
from sklearn.neural_network import MLPClassifier
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.metrics import precision_score, recall_score, f1_score
from sklearn.metrics import confusion_matrix

# different classifiers
num_models = [KNeighborsClassifier(n_neighbors=5), svm.SVC(), LogisticRegression(), GaussianNB(),
              AdaBoostClassifier(), BernoulliNB(), MLPClassifier(),
              RandomForestClassifier(), DecisionTreeClassifier('entropy')]

# training 9 different model on different data splits
# taking average of 9 predictions while testing
# testing set is the 10th data split

models = []
for i in range(9):
    df_train = pd.read_csv('land_train_split_{}.csv'.format(i+1))
    df_train = df_train.sample(frac=1).reset_index(drop=True)

    x = df_train.drop('target', axis=1).values
    y = df_train.target.values
    
    model = num_models[7]
    models.append(model.fit(x, y))
    
    print(f'Model {i+1} trained.', end='\r')

# loading the test set
df_test = pd.read_csv('land_train_split_10.csv')
df_test = df_test.sample(frac=1).reset_index(drop=True)

x_test = df_test.drop('target', axis=1).values
y_test = df_test.target.values

# making predictions
predictions = []
for i in range(9):
    predictions.append(list(models[i].predict(x_test)))
predictions = np.array(predictions)

# getting the mojority Class

pred_maj = []
for i in range(predictions.shape[1]):
    (values, counts) = np.unique(predictions[:,i], return_counts=True)
    pred_maj.append(values[np.argmax(counts)])
pred_maj = np.array(pred_maj)

print('Accuracy: {:.2f}%'.format(np.mean(pred_maj==y_test)*100))

precision, recall, f1 = precision_score(y_test, pred_maj), recall_score(y_test, pred_maj), f1_score(y_test, pred_maj)
print('Precision: {:.2f}\nRecall: {:.2f}\nF1 score: {:.2f}'.format(precision, recall, f1))

cm = confusion_matrix(y_test, pred_maj, labels=[1,3])
sns.heatmap(cm, xticklabels=['False', 'True'], yticklabels=['Negative', 'Positive'])
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.title('Confusion matrix');

df_test_set = pd.read_csv("land_test_preprocessed.csv")
df_test_set.drop('Unnamed: 0', inplace=True, axis=1)
df_test_set.head()

# predictions
predictions = []
for i in range(9):
    predictions.append(list(models[i].predict(df_test_set.values)))
predictions = np.array(predictions)

# getting the mojority Class
pred_maj = []
for i in range(predictions.shape[1]):
    (values, counts) = np.unique(predictions[:,i], return_counts=True)
    pred_maj.append(values[np.argmax(counts)])
pred_maj = np.array(pred_maj)

pred_maj.shape

df_sub = pd.DataFrame(pred_maj, columns=['target'])
df_sub.head()

df_sub.to_csv('submission.csv', index=False)

dfa = pd.read_csv('land_train_split_1.csv')
dfa.groupby(dfa.target).describe().T

estimator = models[0].estimators_[5]

from sklearn.tree import export_graphviz
# Export as dot file
export_graphviz(estimator, out_file='tree.dot', 
                feature_names = ['clusterID', 'I3', 'I6', 'I14', 'X'],
                class_names = ['1', '3'],
                rounded = True, proportion = False, 
                precision = 2, filled = True)

# Convert to png using system command (requires Graphviz)
import pydot
(graph,) = pydot.graph_from_dot_file('tree.dot')
graph.write_png('somefile.png')

# taking sum of all GINI indexes
a = models[0].feature_importances_
for i in range(8):
    a += models[i+1].feature_importances_

a