Imported Funcion (Sequential)
The following piece of code include all the imported functions except for the functions from other libraries:
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
import argparse
import pickle
import numpy as np
from hpbandster.core.worker import Worker
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler
import ConfigSpace as CS
import hpbandster.core.nameserver as hpns
from hpbandster.optimizers import HyperBand
from hpbandster.optimizers import BOHB
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.initializers import GlorotUniform
from tensorflow.keras.regularizers import L2
import tensorflow as tf
from scipy.special import erf
from scipy.stats import norm
os.environ['PYTHONHASHSEED']=str(1)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
class MyThresholdCallback(tf.keras.callbacks.Callback):
def __init__(self, threshold):
super(MyThresholdCallback, self).__init__()
self.threshold = threshold
def on_epoch_end(self, epoch, logs=None):
val_loss = logs["loss"]
if val_loss <= self.threshold:
self.model.stop_training = True
def std_scalar(data):
trans = StandardScaler()
trans.fit(data)
return trans
def transform(x, x_transform):
return x_transform.transform(x)
# ----------------------------------------------------------------------------
# Analytical Function
# ----------------------------------------------------------------------------
# Constants used in the function
alpha = np.zeros([4,1])
alpha[0] = 1; alpha[1] =1.2; alpha[2] = 3; alpha[3]=3.2;
A = np.zeros([4,3])
A[0,0] = 3; A[0,1] = 10; A[0,2] = 30;
A[1,0] = 0.1; A[1,1] = 10; A[1,2] = 35;
A[2,0] = 3; A[2,1] = 10; A[2,2] = 30;
A[3,0] = 0.1; A[3,1] = 10; A[3,2] = 35;
P = np.zeros([4,3])
P[0,0] = 0.3689; P[0,1] = 0.1170; P[0,2] = 0.2673;
P[1,0] = 0.4699; P[1,1] = 0.4387; P[1,2] = 0.7470;
P[2,0] = 0.1091; P[2,1] = 0.8732; P[2,2] = 0.5547;
P[3,0] = 0.0381; P[3,1] = 0.5743; P[3,2] = 0.8828;
def HARTMANN3(x):
outer = 0;
for i in range(4):
inner = 0;
for j in range(3):
inner = inner + A[i,j]*(x[j]-P[i,j])**2;
outer = outer + alpha[i] * np.exp(-inner);
f = -outer
return f
# ----------------------------------------------------------------------------
# Training and prediction Methods
# ----------------------------------------------------------------------------
def train(x, y, parameters):
x_transform = std_scalar(x)
y_transform = std_scalar(y)
x = transform(x, x_transform)
y = transform(y, y_transform)
lr = 0.001
opt = Adam(learning_rate=lr)
activations = {1: "relu", 2: "elu", 3: "tanh", 4: "sigmoid"}
activation_get = activations[parameters["activation"]]
num_hidden_layers = parameters["num_hidden_layers"]
activation_list = [activation_get] * num_hidden_layers
layers = [parameters["num_neurons"]]
for idx in range(num_hidden_layers):
if idx != 0:
layers.append(layers[idx - 1])
regularizer = None
epochs = parameters["num_epochs"]
initializer = GlorotUniform(seed=10)
tolerance = 0.0005
callbacks = MyThresholdCallback(threshold=tolerance)
model = Sequential()
model.add(Input(shape=(x.shape[1],)))
for i in range(len(layers)):
model.add(Dense(layers[i], activation=activation_list[i], activity_regularizer=regularizer, kernel_initializer=initializer))
model.add(Dense(y.shape[1]))
model.compile(optimizer=opt, loss='mean_squared_error')
model.fit(x, y, epochs=epochs, verbose=0, callbacks=[callbacks])
return model, x_transform, y_transform
def predict(x, x_transform, y_transform, model):
dim = x.ndim
if dim == 1:
x = x.reshape(1, -1)
x = transform(x, x_transform)
y = model(x, training=False)
y = y_transform.inverse_transform(y)
if dim == 1:
y = y.reshape(-1, )
return y
def objective(x, y, x_cv, y_cv, parameters):
model, x_transform, y_transform = train(x, y, parameters)
x = transform(x, x_transform)
x_cv = transform(x_cv, x_transform)
y_pred = model(x, training=False)
y_cv_pred = model(x_cv, training=False)
y_pred = y_transform.inverse_transform(y_pred)
y_cv_pred = y_transform.inverse_transform(y_cv_pred)
loss = np.sqrt(mean_squared_error(y, y_pred))
loss_cv = np.sqrt(mean_squared_error(y_cv, y_cv_pred))
return loss + loss_cv
# ----------------------------------------------------------------------------
# Methods for BOHB
# ----------------------------------------------------------------------------
class MyWorker(Worker):
def __init__(self, x_train, y_train, x_cv, y_cv, **kwargs):
super().__init__(**kwargs)
self.x_train = x_train
self.y_train = y_train
self.x_cv = x_cv
self.y_cv = y_cv
def compute(self, config, budget, **kwargs):
opt_params = {
"num_epochs": config["epoch"],
"activation": config["act"],
"num_hidden_layers": config["layer"],
"num_neurons": config["neuron"]
}
loss = objective(self.x_train, self.y_train, self.x_cv, self.y_cv, opt_params)
return {
'loss': loss,
'info': {}
}
def get_configspace():
cs = CS.ConfigurationSpace()
cs.add_hyperparameter(CS.UniformIntegerHyperparameter('layer', lower=1, upper=4, log=False))
cs.add_hyperparameter(CS.UniformIntegerHyperparameter('neuron', lower=4, upper=16, log=True))
cs.add_hyperparameter(CS.UniformIntegerHyperparameter('act', lower=1, upper=4, log=False))
cs.add_hyperparameter(CS.UniformIntegerHyperparameter('epoch', lower=1000, upper=10000, log=False))
return cs
def run_hpbandster(x_train, y_train, x_cv, y_cv, iteration, min_b, max_b):
cs = get_configspace()
bohb = BOHB(configspace=cs, run_id='bohb', nameserver='127.0.0.1', min_budget=min_b, max_budget=max_b)
res = bohb.run(n_iterations=iteration)
bohb.shutdown(shutdown_workers=True)
return res
# ----------------------------------------------------------------------------
# Expected Improvement
# ----------------------------------------------------------------------------
def expectedImprovement(x, x_transform, y_transform, x_s2_transform, s2_transform, ymin, mean_model, var_model):
"""
Method to calculate the expected improvement.
Returns negative of EI so that it can be direclty used in minimization.
"""
# mean
yhat = predict(x, x_transform, y_transform, mean_model)
# Standard Deviation
sigma = np.sqrt(predict(x, x_s2_transform, s2_transform, var_model)**2)
# Calculating standard variable
z = (ymin - yhat)/sigma
# Calculating standard normal cdf
cdf = 0.5*(1 + erf(z))
# Calculating standard normal pdf
pdf = norm.pdf(z)
# Calculating expected improvement
ExpImp = np.where(sigma <= 0, 0, sigma * (z * cdf + pdf))
return -ExpImp
if __name__ == "__main__":
# Argument parsing
argparser = argparse.ArgumentParser()
argparser.add_argument('--run_id', type=str, required=True)
argparser.add_argument('--host', type=str, required=True)
argparser.add_argument('--worker', type=int, required=True)
args = argparser.parse_args()
# Load training data
with open('train_data.pkl', 'rb') as f:
data = pickle.load(f)
x_train = data['x_train']
y_train = data['y_train']
x_cv = data['x_cv']
y_cv = data['y_cv']
# Start a worker
w = MyWorker(x_train=x_train, y_train=y_train, x_cv=x_cv, y_cv=y_cv, nameserver=args.host, run_id=args.run_id)
w.run(background=False)