# Scale dataset to mean=0 variance=1
scaler2 = StandardScaler()
dataset2_x_scaled = scaler2.fit_transform(dataset2_x)

scaler3 = StandardScaler()
dataset3_x_scaled = scaler3.fit_transform(dataset3_x)

scaler4 = StandardScaler()
dataset4_x_scaled = scaler4.fit_transform(dataset4_x)

scaler5 = StandardScaler()
dataset5_x_scaled = scaler5.fit_transform(dataset5_x)

scaler6 = StandardScaler()
dataset6_x_scaled = scaler6.fit_transform(dataset6_x)

scaler7 = StandardScaler()
dataset7_x_scaled = scaler7.fit_transform(dataset7_x)

scaler51 = StandardScaler()
dataset51_x_scaled = scaler51.fit_transform(dataset51_x)

_=plt.scatter(dataset7_x_scaled[:,0], dataset7_x_scaled[:,1], s=100, c=dataset7_y)
_=plt.xlabel("$x_1$")
_=plt.ylabel("$x_2$")
_=plt.title("Dataset scaled with StandartScaler")

# Fit the model
clf = svm.SVC(kernel='rbf', C=10000)
clf.fit(dataset7_x_scaled, dataset7_y)

# Prepare the figure
plt.grid(True)
plt.xlabel("$x_1$")
plt.ylabel("$x_2$")

# Plot the decision function for each datapoint on the grid
xx, yy = np.meshgrid(np.linspace(-3, 3, 500),
                    np.linspace(-3, 3, 500))
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)

# Plot model output for every pixel on the figure
plt.imshow(Z, interpolation='nearest', extent=(xx.min(), xx.max(), yy.min(), yy.max()), aspect='auto', origin='lower', cmap=plt.cm.PuOr_r)

# Plot decision boundary
plt.contour(xx, yy, Z, colors='r', levels=[0], alpha=0.75,
           linestyles=['-'])
plt.contour(xx, yy, Z, colors='b', levels=[-1, 1], alpha=0.5,
           linestyles=['--', '--'])

# Plot support vectors
plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=150,
           linewidth=1, facecolors='none', edgecolors='b', label="Support Vectors")

# Plot dataset
_=plt.scatter(dataset7_x_scaled[:, 0], dataset7_x_scaled[:, 1], s=30, c=dataset7_y, cmap=plt.cm.Paired, edgecolors='k')
_=plt.legend()

def svm_scenario(ax, x, y, zoom, kernel, C, title=None):
    # Fit the model
    clf = svm.SVC(kernel=kernel, C=C)
    clf.fit(x, y)

    # Plot results
    if ax is not None:
        # Prepare the figure
        ax.grid(True)
        ax.set_xlabel("$x_1$")
        ax.set_ylabel("$x_2$")
        if title is not None:
            ax.set_title(title)

        # Plot the decision function for each datapoint on the grid
        xlim = [np.min(x[:,0])-zoom ,np.max(x[:,0])+zoom]
        ylim = [np.min(x[:,1])-zoom ,np.max(x[:,1])+zoom]
        xx, yy = np.meshgrid(np.linspace(xlim[0], xlim[1], 100),
                            np.linspace(ylim[0], ylim[1], 100))
        zz = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
        zz = zz.reshape(xx.shape)

        # Plot model output for every pixel on the figure
        ax.imshow(zz, interpolation='nearest', extent=(xx.min(), xx.max(), yy.min(), yy.max()), aspect='auto', origin='lower', cmap=plt.cm.PuOr_r)

        # Plot decision boundary
        ax.contour(xx, yy, zz, colors='r', levels=[0], alpha=0.75, linestyles=['-'])
        ax.contour(xx, yy, zz, colors='b', levels=[-1, 1], alpha=0.5, linestyles=['--', '--'])

        # Plot support vectors
        ax.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=150, linewidth=1, facecolors='none', edgecolors='b', label="Support Vectors")

        # Plot dataset
        ax.scatter(x[:, 0], x[:, 1], s=30, c=y, cmap=plt.cm.Paired, edgecolors='k')
        ax.legend()


fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12,6))
fig.tight_layout()
svm_scenario(ax=ax1, x=dataset7_x_scaled, y=dataset7_y, zoom=1, kernel='rbf', C=1000, title="RBF Kernel, C=1000")
svm_scenario(ax=ax2, x=dataset7_x_scaled, y=dataset7_y, zoom=1, kernel='rbf', C=1, title="RBF Kernel, C=1")

kernels=["linear", "poly", "rbf", "sigmoid"]
c_values = [0.1, 1, 10, 100] # Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.

fig, axs = plt.subplots(len(c_values), len(kernels), figsize=(24,15))

for i,k in enumerate(kernels):
    for j,c in enumerate(c_values):
        svm_scenario(ax=axs[j,i], x=dataset51_x_scaled, y=dataset51_y, zoom=1, kernel=k, C=c, title=k+" kernel, C="+str(c))

fig.tight_layout()

kernels=["linear", "poly", "rbf", "sigmoid"]
c_values = [0.1, 1, 10, 100] # Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.

fig, axs = plt.subplots(len(c_values), len(kernels), figsize=(24,15))

for i,k in enumerate(kernels):
    for j,c in enumerate(c_values):
        svm_scenario(ax=axs[j,i], x=dataset2_x_scaled, y=dataset2_y, zoom=1, kernel=k, C=c, title=k+" kernel, C="+str(c))

fig.tight_layout()

kernels=["linear", "poly", "rbf", "sigmoid"]
c_values = [0.1, 1, 10, 100] # Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.

fig, axs = plt.subplots(len(c_values), len(kernels), figsize=(24,15))

for i,k in enumerate(kernels):
    for j,c in enumerate(c_values):
        svm_scenario(ax=axs[j,i], x=dataset3_x_scaled, y=dataset3_y, zoom=1, kernel=k, C=c, title=k+" kernel, C="+str(c))

fig.tight_layout()

kernels=["linear", "poly", "rbf", "sigmoid"]
c_values = [0.1, 1, 10, 100] # Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.

fig, axs = plt.subplots(len(c_values), len(kernels), figsize=(24,15))

for i,k in enumerate(kernels):
    for j,c in enumerate(c_values):
        svm_scenario(ax=axs[j,i], x=dataset4_x_scaled, y=dataset4_y, zoom=1, kernel=k, C=c, title=k+" kernel, C="+str(c))

fig.tight_layout()

kernels=["linear", "poly", "rbf", "sigmoid"]
c_values = [0.1, 1, 10, 100] # Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.

fig, axs = plt.subplots(len(c_values), len(kernels), figsize=(24,15))

for i,k in enumerate(kernels):
    for j,c in enumerate(c_values):
        svm_scenario(ax=axs[j,i], x=dataset5_x_scaled, y=dataset5_y, zoom=1, kernel=k, C=c, title=k+" kernel, C="+str(c))

fig.tight_layout()

kernels=["linear", "poly", "rbf", "sigmoid"]
c_values = [0.1, 1, 10, 100] # Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.

fig, axs = plt.subplots(len(c_values), len(kernels), figsize=(24,15))

for i,k in enumerate(kernels):
    for j,c in enumerate(c_values):
        svm_scenario(ax=axs[j,i], x=dataset6_x_scaled, y=dataset6_y, zoom=1, kernel=k, C=c, title=k+" kernel, C="+str(c))

fig.tight_layout()

kernels=["linear", "poly", "rbf", "sigmoid"]
c_values = [0.1, 1, 10, 100] # Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive. The penalty is a squared l2 penalty.

fig, axs = plt.subplots(len(c_values), len(kernels), figsize=(24,15))

for i,k in enumerate(kernels):
    for j,c in enumerate(c_values):
        svm_scenario(ax=axs[j,i], x=dataset7_x_scaled, y=dataset7_y, zoom=1, kernel=k, C=c, title=k+" kernel, C="+str(c))

fig.tight_layout()

datasetTest_x, datasetTest_y = load_dataset("data/PA-E_Test.txt")

_=plt.scatter(datasetTest_x[:,0], datasetTest_x[:,1], s=100, c=datasetTest_y)
_=plt.xlabel("$x_1$")
_=plt.ylabel("$x_2$")

X, Y = load_dataset("data/PA-E_t7.txt")

C_values = 10**np.linspace(-3, 3, 14)
G_values = 10**np.linspace(-3, 3, 7)

tuned_parameters = [{'kernel': ['rbf'],     'C': C_values, 'gamma': G_values},
                    {'kernel': ['linear'],  'C': C_values},
                    {'kernel': ['poly'],    'C': C_values},
                    {'kernel': ['sigmoid'], 'C': C_values}]
scores = ['precision', 'recall']

for score in scores:
    print("# Tuning hyper-parameters for %s" % score)

    clf = GridSearchCV(
        SVC(), tuned_parameters, scoring='%s_macro' % score
    )
    clf.fit(X, Y)

    print("Best parameters set found on development set:")
    print(clf.best_params_)
    #print("Grid scores on development set:")
    #print()
    #means = clf.cv_results_['mean_test_score']
    #stds = clf.cv_results_['std_test_score']
    #for mean, std, params in zip(means, stds, clf.cv_results_['params']):
    #    print("%0.3f (+/-%0.03f) for %r"
    #          % (mean, std * 2, params))
    #print()