Imported Funcion

imp_function.py for grid search, random search, Bayesian optimization

The following piece of code include all the imported functions except for the functions from other libraries:

import os, torch
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
os.environ['PYTHONHASHSEED']=str(1)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import numpy as np
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler

from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.initializers import GlorotUniform

import tensorflow as tf
import random

from ax.service.ax_client import AxClient

class MyThresholdCallback(tf.keras.callbacks.Callback):
    def __init__(self, threshold):
        super(MyThresholdCallback, self).__init__()
        self.threshold = threshold

    def on_epoch_end(self, epoch, logs=None):
        val_loss = logs["loss"]
        if val_loss <= self.threshold:
            self.model.stop_training = True

# Device
device = 'cuda' if torch.cuda.is_available() else 'cpu'

# ----------------------------------------------------------------------------
#                           Normalization Methods
# ----------------------------------------------------------------------------

def std_scalar(data):
    """
        Method for creating std scalar object.
    """

    trans = StandardScaler()
    trans.fit(data)

    return trans

def transform(x, x_transform):
    """
        Method for transforming the input
        based on the std scalar object.
    """

    x = x_transform.transform(x)

    return x

# ----------------------------------------------------------------------------
#                             Analytical Function
# ----------------------------------------------------------------------------

def branin(x):
    """
        Method for calculating branin function.
    """

    x1 = x[:, 0].reshape(-1,1)
    x2 = x[:, 1].reshape(-1,1)

    a = 1
    b = 5.1/(4*np.pi**2)
    c = 5/np.pi
    r = 6
    s = 10
    t = 1/(8*np.pi)

    func = a*(x2 - b*x1**2 + c*x1 - r)**2 + s*(1 - t)*np.cos(x1) + s

    return func

# ----------------------------------------------------------------------------
#                        Training and prediction Methods
# ----------------------------------------------------------------------------

def train(x, y, parameters):
    """
        Method for training the NN for given hyperparameters.
    """

    # Getting the standard scalar object
    x_transform = std_scalar(x)
    y_transform = std_scalar(y)

    # Normalize training and testing data to zero mean and unit variance
    x = transform(x, x_transform)
    y = transform(y, y_transform)

    # Get learning rate and set the optimizer

    lr = 0.001

    opt = Adam(learning_rate=lr)

    activations = {
    1: "relu",
    2: "elu",
    3: "tanh",
    4: "sigmoid"
        }

    activation_get = activations[parameters["activation"]]
    num_hidden_layers = parameters["num_hidden_layers"]
    activation_list = [activation_get] * num_hidden_layers

    layers = [parameters["neurons"]]
    for idx in range(num_hidden_layers):
        if idx != 0:
            layers.append(layers[idx-1])

    # Predefined hyperparameters
    regularizer = None

    epochs = parameters["epochs"]

    initializer = GlorotUniform(seed=10)

    tolerance = 0.0005  # Define your tolerance
    callbacks = MyThresholdCallback(threshold=tolerance)

    # Build the NN structure
    model = Sequential()

    # Input layer - doesn't have any activation function
    model.add(Input(shape=(x.shape[1],)))

    # Hidden layers
    for i in range(len(layers)):
        model.add(Dense(layers[i], activation=activation_list[i], activity_regularizer=regularizer, kernel_initializer=initializer))

    # Output layer
    model.add(Dense(y.shape[1]))

    # Complile the model
    model.compile(optimizer=opt, loss='mean_squared_error')

    model.fit(x, y, epochs=epochs, verbose=0, callbacks=[callbacks], use_multiprocessing=True)

    return model, x_transform, y_transform


def predict(x, x_transform, y_transform, model):
    """
        Common method for prediction from either NN.
        Input model dictates what model will be used.
    """
    # Reshaping x
    dim = x.ndim
    if dim == 1:
        x = x.reshape(1,-1)

    # Scaling, Prediction, and Rescaling
    x = transform(x, x_transform)
    y = model(x, training=False)
    y = y_transform.inverse_transform(y)

    # Reshaping the y
    if dim == 1:
        y = y.reshape(-1,)

    return y


def objective(x, y, x_cv, y_cv, parameters):
    """
        Method for computing the loss for given hyperparameters.
    """
    # Train the model
    model, x_transform, y_transform = train(x, y, parameters)

    # Transform the data
    x = transform(x, x_transform)
    x_cv = transform(x_cv, x_transform)

    # Predict at training data
    y_pred = model(x, training=False)
    y_cv_pred = model(x_cv, training=False)

    # Transform back to original scale
    y_pred = y_transform.inverse_transform(y_pred)
    y_cv_pred = y_transform.inverse_transform(y_cv_pred)

    # Calculate the rmse
    loss = np.sqrt(mean_squared_error(y, y_pred))
    loss_cv = np.sqrt(mean_squared_error(y_cv, y_cv_pred))

    return loss + loss_cv

# ----------------------------------------------------------------------------
#                        Importing Ax for using BO
# ----------------------------------------------------------------------------

def Ax(x, y, x_cv, y_cv):
    """
        Method for tuning the NN using Ax.
    """

    objective_name = "objective"

    # Set up Ax
    ax_client = AxClient(verbose_logging=False, torch_device=device)

    # Defining experiment
    ax_client.create_experiment(
        name='NN_tunning',
        parameters=[
            {
                "name": "epochs",
                "type": "range",
                "bounds": [1000, 10000]
            },
            {
                "name": "neurons",
                "type": "range",
                "bounds": [4, 16]
            },
            {
                "name": "num_hidden_layers",
                "type": "range",
                "bounds": [1, 4]
            },
            {
                "name": "activation",
                "type": "range",
                "bounds": [1, 4],
            }
        ],
        objective_name=objective_name,
        minimize=True
    )

    # Number of trials
    num_trials = 30

    # Perform the experiment
    for _ in range(num_trials):
        parameters, trial_index = ax_client.get_next_trial()
        ax_client.complete_trial(trial_index=trial_index, raw_data=objective(x, y, x_cv, y_cv, parameters))

    # Get the best parameters
    best, res = ax_client.get_best_parameters(use_model_predictions=False)

    return best, res[0][objective_name]

imp_function.py for Hyperband and Bayesian optimization Hyperband

The following piece of code include all the imported functions except for the functions from other libraries:

import os
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

import argparse
import pickle
import numpy as np
from hpbandster.core.worker import Worker
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler
import ConfigSpace as CS
from hpbandster.optimizers import HyperBand
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.initializers import GlorotUniform
import tensorflow as tf
os.environ['PYTHONHASHSEED']=str(1)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

class MyThresholdCallback(tf.keras.callbacks.Callback):
    def __init__(self, threshold):
        super(MyThresholdCallback, self).__init__()
        self.threshold = threshold

    def on_epoch_end(self, epoch, logs=None):
        val_loss = logs["loss"]
        if val_loss <= self.threshold:
            self.model.stop_training = True

def std_scalar(data):
    trans = StandardScaler()
    trans.fit(data)
    return trans

def transform(x, x_transform):
    return x_transform.transform(x)

def branin(x):
    x1 = x[:, 0].reshape(-1, 1)
    x2 = x[:, 1].reshape(-1, 1)
    a = 1
    b = 5.1/(4*np.pi**2)
    c = 5/np.pi
    r = 6
    s = 10
    t = 1/(8*np.pi)
    func = a*(x2 - b*x1**2 + c*x1 - r)**2 + s*(1 - t)*np.cos(x1) + s
    return func

def train(x, y, parameters):
    x_transform = std_scalar(x)
    y_transform = std_scalar(y)
    x = transform(x, x_transform)
    y = transform(y, y_transform)
    lr = 0.001
    opt = Adam(learning_rate=lr)
    activations = {1: "relu", 2: "elu", 3: "tanh", 4: "sigmoid"}
    activation_get = activations[parameters["activation"]]
    num_hidden_layers = parameters["num_hidden_layers"]
    activation_list = [activation_get] * num_hidden_layers
    layers = [parameters["num_neurons"]]
    for idx in range(num_hidden_layers):
        if idx != 0:
            layers.append(layers[idx - 1])
    regularizer = None
    epochs = parameters["num_epochs"]
    initializer = GlorotUniform(seed=10)
    tolerance = 0.0005
    callbacks = MyThresholdCallback(threshold=tolerance)
    model = Sequential()
    model.add(Input(shape=(x.shape[1],)))
    for i in range(len(layers)):
        model.add(Dense(layers[i], activation=activation_list[i], activity_regularizer=regularizer, kernel_initializer=initializer))
    model.add(Dense(y.shape[1]))
    model.compile(optimizer=opt, loss='mean_squared_error')
    model.fit(x, y, epochs=epochs, verbose=0, callbacks=[callbacks])
    return model, x_transform, y_transform

def predict(x, x_transform, y_transform, model):
    dim = x.ndim
    if dim == 1:
        x = x.reshape(1, -1)
    x = transform(x, x_transform)
    y = model(x, training=False)
    y = y_transform.inverse_transform(y)
    if dim == 1:
        y = y.reshape(-1, )
    return y

def objective(x, y, x_cv, y_cv, parameters):
    model, x_transform, y_transform = train(x, y, parameters)
    x = transform(x, x_transform)
    x_cv = transform(x_cv, x_transform)
    y_pred = model(x, training=False)
    y_cv_pred = model(x_cv, training=False)
    y_pred = y_transform.inverse_transform(y_pred)
    y_cv_pred = y_transform.inverse_transform(y_cv_pred)
    loss = np.sqrt(mean_squared_error(y, y_pred))
    loss_cv = np.sqrt(mean_squared_error(y_cv, y_cv_pred))
    return loss + loss_cv

class MyWorker(Worker):
    def __init__(self, x_train, y_train, x_cv, y_cv, **kwargs):
        super().__init__(**kwargs)
        self.x_train = x_train
        self.y_train = y_train
        self.x_cv = x_cv
        self.y_cv = y_cv

    def compute(self, config, budget, **kwargs):
        opt_params = {
            "num_epochs": config["epoch"],
            "activation": config["act"],
            "num_hidden_layers": config["layer"],
            "num_neurons": config["neuron"]
        }
        loss = objective(self.x_train, self.y_train, self.x_cv, self.y_cv, opt_params)
        return {
            'loss': loss,
            'info': {}
        }

def get_configspace():
    cs = CS.ConfigurationSpace()
    cs.add_hyperparameter(CS.UniformIntegerHyperparameter('layer', lower=1, upper=4, log=False))
    cs.add_hyperparameter(CS.UniformIntegerHyperparameter('neuron', lower=4, upper=16, log=True))
    cs.add_hyperparameter(CS.UniformIntegerHyperparameter('act', lower=1, upper=4, log=False))
    cs.add_hyperparameter(CS.UniformIntegerHyperparameter('epoch', lower=1000, upper=10000, log=False))
    return cs

def run_hpbandster(x_train, y_train, x_cv, y_cv, iteration, min_b, max_b):

    cs = get_configspace()

    hb = HyperBand(configspace=cs, run_id='hb', nameserver='127.0.0.1', min_budget=min_b, max_budget=max_b)
    # bohb = BOHB(configspace=cs, run_id='bohb', nameserver='127.0.0.1', min_budget=min_b, max_budget=max_b)

    res = hb.run(n_iterations=iteration)
    # res = bohb.run(n_iterations=iteration)

    hb.shutdown(shutdown_workers=True)
    # bohb.shutdown(shutdown_workers=True)
    return res

if __name__ == "__main__":
    # Argument parsing
    argparser = argparse.ArgumentParser()
    argparser.add_argument('--run_id', type=str, required=True)
    argparser.add_argument('--host', type=str, required=True)
    argparser.add_argument('--worker', type=int, required=True)
    args = argparser.parse_args()

    # Load training data
    with open('train_data.pkl', 'rb') as f:
        data = pickle.load(f)

    x_train = data['x_train']
    y_train = data['y_train']
    x_cv = data['x_cv']
    y_cv = data['y_cv']

    # Start a worker
    w = MyWorker(x_train=x_train, y_train=y_train, x_cv=x_cv, y_cv=y_cv, nameserver=args.host, run_id=args.run_id)
    w.run(background=False)

Note

The difference between imp_function.py for HB and BOHB is under def run_hpbandster. You need to specify which algorithm you want to use.

Run script file

Note

You can use runscript.py to run the code multiple times and save the data. It is recommended to use to plot the box plot afterward