hw1: Tweaks to the python files (comments and such)

This commit is contained in:
Claudio Maggioni 2021-05-07 15:18:31 +02:00
parent 315d736e11
commit eba6899d1e
4 changed files with 63 additions and 38 deletions

View file

@ -1,3 +1,5 @@
""" Define, train and save the linear model and the non-linear model """
from sklearn.linear_model import LinearRegression from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split from sklearn.model_selection import train_test_split
from tensorflow.keras import Input, Model from tensorflow.keras import Input, Model
@ -15,91 +17,114 @@ from keras import backend as K
# #
# FIX THE RANDOM GENERATOR SEEDS # FIX THE RANDOM GENERATOR SEEDS
# #
seed_value = 0
# 1. Set `PYTHONHASHSEED` environment variable at a fixed value # The random generator seed is set to a fixed value for reproducibility
os.environ['PYTHONHASHSEED'] = str(seed_value) # purposes. Since the libraries use different random generators, we set them
# 2. Set `python` built-in pseudo-random generator at a fixed value # all to the fixed value below
random.seed(seed_value) SEED_VALUE = 0
# 3. Set `numpy` pseudo-random generator at a fixed value
np.random.seed(seed_value) os.environ['PYTHONHASHSEED'] = str(SEED_VALUE)
# 4. Set the `tensorflow` pseudo-random generator at a fixed value random.seed(SEED_VALUE)
tf.random.set_seed(seed_value) np.random.seed(SEED_VALUE)
# 5. Configure a new global `tensorflow` session tf.random.set_seed(SEED_VALUE)
session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1) inter_op_parallelism_threads=1)
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(),
config=session_conf) config=session_conf)
tf.compat.v1.keras.backend.set_session(sess) tf.compat.v1.keras.backend.set_session(sess)
d = os.path.dirname(__file__) # Load data
data = np.load("../data/data.npz") data = np.load("../data/data.npz")
xs = data["x"] # 2000x2 xs = data["x"]
y = data["y"] # 2000x1 y = data["y"]
points = np.shape(xs)[0] points = np.shape(xs)[0]
# We manually include in the feature vectors a '1' column corresponding to theta_0, #
# so disable # LINEAR MODEL
#
print("# Linear regression:")
# We manually include in the feature vectors a '1' column corresponding to
# theta_0, so disable the built in intercept in Sci-kit learn
lr = LinearRegression(fit_intercept=False) lr = LinearRegression(fit_intercept=False)
# Build x feature vector with columns for theta_3 and theta_4 # Build X feature matrix with columns for theta_3 and theta_4
X = np.zeros([points, 5]) X = np.zeros([points, 5])
X[:, 0] = 1 X[:, 0] = 1
X[:, 1:3] = xs X[:, 1:3] = xs
X[:, 3] = xs[:, 0] * xs[:, 1] X[:, 3] = xs[:, 0] * xs[:, 1]
X[:, 4] = np.sin(xs[:, 0]) X[:, 4] = np.sin(xs[:, 0])
# Shuffle our data for division in training, and test set # Shuffle and split our data for division in training, and test set
TRAIN_SET_RATIO = 0.1
train_ratio = 0.1 X_t, X_test, y_t, y_test = train_test_split(X, y, test_size=TRAIN_SET_RATIO)
validation_ratio = 0.1
X_t, X_test, y_t, y_test = train_test_split(X, y, test_size=train_ratio)
X_train, X_val, y_train, y_val = train_test_split(X_t, y_t, test_size=validation_ratio)
np.savez('test', x=X_test, y=y_test, allow_pickle=True)
# Fit with train data # Fit with train data
reg = lr.fit(X_t, y_t) reg = lr.fit(X_t, y_t)
print("# Linear regression:")
# Print the resulting parameters # Print the resulting parameters
print("f(x) = %g + %g * x_1 + %g * x_2 + %g * x_1 * x_2 + %g * sin(x_1)" % tuple(reg.coef_)) print("f(x) = %g + %g * x_1 + %g * x_2 + %g * x_1 * x_2 + %g * sin(x_1)" %
tuple(reg.coef_))
# Save the model as .pickle
save_sklearn_model(reg, "../deliverable/linear_regression.pickle") save_sklearn_model(reg, "../deliverable/linear_regression.pickle")
# Non-linear regression: #
# NON-LINEAR MODEL
#
print("\n# Feed-forward NN:") print("\n# Feed-forward NN:")
A = X_val # Divide previously found training set (X_t, y_t) in another training and a
# validation set. This division is used for the FFNN training and architecture
# design/tailoring
VALIDATION_SET_RATIO = 0.1
X_train, X_val, y_train, y_val = \
train_test_split(X_t, y_t, test_size=VALIDATION_SET_RATIO)
np.savez('test', x=X_test, y=y_test, allow_pickle=True)
# Drop additional features added before
X_train = X_train[:, 1:3] X_train = X_train[:, 1:3]
X_val = X_val[:, 1:3] X_val = X_val[:, 1:3]
# Compute mean and std for each feature in the training set
mean = np.mean(X_train, axis=0) mean = np.mean(X_train, axis=0)
std = np.std(X_train, axis=0) std = np.std(X_train, axis=0)
# Normalize training data according to the mean and variance
X_train -= mean X_train -= mean
X_train /= std X_train /= std
# Normalize validation data as well. All further inputs to the NN must be
# normalized using the value `mean` and `std` computed before. Normalization is
# necessary to increase the speed of the learning process
X_val -= mean X_val -= mean
X_val /= std X_val /= std
# Define the network's architecture
network = Sequential() network = Sequential()
network.add(Dense(22, activation='tanh')) network.add(Dense(22, activation='tanh'))
network.add(Dense(15, activation='sigmoid')) network.add(Dense(15, activation='sigmoid'))
network.add(Dense(1, activation='linear')) network.add(Dense(1, activation='linear'))
network.compile(optimizer='adam', loss='mse') network.compile(optimizer='adam', loss='mse')
epochs = 5000 # Define maximum number of iterations and early stopping procedure
EPOCHS = 5000
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=120) callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=120)
network.fit(X_train, y_train, epochs=epochs, verbose=1,
# Fit the model monitoring validation in the learning process
network.fit(X_train, y_train, epochs=EPOCHS, verbose=1,
validation_data=(X_val, y_val), callbacks=[callback]) validation_data=(X_val, y_val), callbacks=[callback])
# Save the fitted model and the normalization parameters as well
network.save("../deliverable/nonlinear_model") network.save("../deliverable/nonlinear_model")
save_sklearn_model({"mean": mean, "std": std}, "../deliverable/nonlinear_model_normalizers.pickle") save_sklearn_model({"mean": mean, "std": std},
"../deliverable/nonlinear_model_normalizers.pickle")
# Print the final validation set MSE, which was used to tailor the NN architecture after # Print the final validation set MSE, which was used to tailor the NN
# several manual trials # architecture after several manual trials
msq = mean_squared_error(network.predict(X_val), y_val) msq = mean_squared_error(network.predict(X_val), y_val)
print(msq) print("Final validation MSE for FFNN: %g" % msq)
# vim: set ts=4 sw=4 et tw=79: