From 2e445c843b1922f53a5f395034a3c855b76b0ab4 Mon Sep 17 00:00:00 2001
From: Claudio Maggioni <maggicl@usi.ch>
Date: Fri, 7 May 2021 15:18:31 +0200
Subject: [PATCH] hw1: Tweaks to the python files (comments and such)

---
 .../nonlinear_model/saved_model.pb            | Bin 87663 -> 87663 bytes
 .../variables/variables.data-00000-of-00001   | Bin 9529 -> 9529 bytes
 .../nonlinear_model/variables/variables.index | Bin 1669 -> 1669 bytes
 assignment_1/src/build_models.py              | 101 +++++++++++-------
 4 files changed, 63 insertions(+), 38 deletions(-)

diff --git a/assignment_1/deliverable/nonlinear_model/saved_model.pb b/assignment_1/deliverable/nonlinear_model/saved_model.pb
index f71850e717786e7f0dc2f0cf27e0f5b896d4fddd..47392a88b360c6b500a5bf4219fc4a8f7ee7a979 100644
GIT binary patch
delta 1149
zcmb`GUr3Wt7{+=1+;%u_i67K9H(Z;|&FS296_sh~=5(8X0wL>aidhJ=3+<*NAqlz)
zq;v5?7m*i2h}fTRur5jjV_kNUQQd?UL=d*>rkkkF`E=`mmw9u(_wXEk&wJkUe1#=L
zVaf0y=~i6C&F;Eu*qxd{sDgM@)8zQYsfn>^N=dD1%#2<dpPHss<bZ0?>vU0e5HH*C
zRAbf(cJ|Z#nuF>D2ygp9kbE-9FOlkZk^(yEnU9mM)V}4U2EC<OmjE@&pr8b5+6fBj
zpj$_Ekb^nBD;yTq+qxRk+iI3+EfOuVlXg(2&9;JFyHPg>E)zA7Ln?&nUS5DG45Unj
zX8uuBXki_GyEf>L*tB!qKP=3ci)s0e`Eu>dP@jceUQTK2XFG}My@@^n+Q6<Yd+=lw
zOxmlFtioniZEW>|1)BzeYO%?Ci%NjRaU%r&a_S&GSg?Y#qE{v9^w?-7lTPQlY_M3#
z4<+<Nd#ITtCDrSPtish)N$rzVwGo~QwZEheNGb`#i}GEGMU|OIEf?=mUAmf)){(gC
zVe=gxbRGd0x(~zV?&OGDOY@+`ISI$aKgSQd5j97pG+XYpHFM#V+IBJ@1L5@lIdoJ?
z55=vv9#B~IZ(IFpg~w=VgL=#!hj^~g2Im34wZjnpX@_A^d2**tz?U+^C#B`1g5P7X
zj+-$^V7?ok=Q_J!AK=3%1o669hQ;zO3V9sqfv<Qo4lnU@672Z9A1v(tqy0FMfHkqR
i;!H2B@C*mp+-U<wQm}vnz5G8crW&J3SjfFe0n)$46;L_=

delta 1187
zcmb`FOK1~O6oxrXr@f8|L#GRqOp-QFC$X(bYkj5FHf_^9np7!B-I%5&i$LNcQ9%kK
zC`b`8aD@XdL>C1q1;r_(n@X{uZbZb53x%NWq(;T16la=5Cnd{l4ww61zW<(c{_-kQ
zUS*y}tn`7!qY;P|&kll-V~LG#%OtO+a(T9k*y%f)#j>9``1Wk-QZBuaXFG_K*X=$?
zRIcbp>n~nMwEVs+$y_=)pGoCefw*e8gDRY;;OLIcY;voCo;|=h`)?V@*8>mLfITW;
z?=JAB!))qP0}s~&v2X$ex@Z@u$@yF+>+8wh8ri2J_bX(Lffy**jiv!L{m35rbw3Ew
zEmr|(w`d^kEJFr)@X`^{K&M3_ofqAD+EyHw*5i*_r5htZb<*riOegu$Up3592fNWi
z&-jDVkGWAouWjn-=0zR$O#>^9XCJS~iwUBj2NO5RlSE=6naLy)tcQfKd;r8^Ko2Wi
zhn9rv#7ISSim;}U#b}*4RT23f*igh+ofxl(#0zg*n~)1ZHc<b&+Kf?CZNSaANQk#l
z-wPX>jzbq(kHU6SvxE<ifthPC9>qHH-}-aY183$M%w2Jvs=8`Vg5uizFq*jrOI4iX
zRp&o`SjSI85XAB@bmH0&xUn=0T=C5?EP-;ojPkLwc<wZOYhk`Xe+g)5%YifFu#NX8
zpck`B8Bxls33!R40SKdUOfI(ra2Myt;59Bz!h5_EgztDNESH@a=;)JY-FPnqTmM3i
a15sGUcnA#gRWB{fGI${Z%f$~-K=v;)_gWwT

diff --git a/assignment_1/deliverable/nonlinear_model/variables/variables.data-00000-of-00001 b/assignment_1/deliverable/nonlinear_model/variables/variables.data-00000-of-00001
index c7dce55dc0162befbfe3de1d7c3d306b9103fd14..aa483178ea6cbce91fb8339488305e7a0d69a2d9 100644
GIT binary patch
delta 317
zcmdn#wbN@uoH(;QhvH;eNx8|JM6~p|Ih2Ihb5l!-GLwtBxHy!B*m4ppQ;UkZ1UXcM
zc){#=n6xy9>LeMd$)ChO0F6)s8=)f#GD02M2n{47G%=0P0vf?A&cQeNzm(zRRg(LF
zMrnhMng%vX2iYiHB%|~&jnW6X=76Z=<bFx%$p+FOqYOYsO%ek+$`IKkBP5fIF-<Z7
knzXq|W+fvBI~Oa5sSu;VWGy+V$-zJzFUPhySMCfO0DSOOZvX%Q

delta 381
zcmdn#wbN@uocQETB3jJS9Ey`;B&GPdIFy9gauO?3i;B7AIg}?Sip$AybEpWh=cbkv
zWhNJM338|k@q!h^Lxd)O5dS2^%fTbWS(aFonV6K5TFfQQp$0Wg9mzBekZF^JBxMyf
zk<HUWHLsXUoP%%je<{Pst0eaU4HiN*SQ~1v4wAvTAcL7DIfN%q6ql0GLpEF=)$qyt
zM5QP9OG-~RkOrA8foiq^)NDf}vyH%x<`<I&27@uO$tI{KZ*GuT&8R5K#md1b#ApDd
gn1mP&ffTb4qY)Pa7dwz?3Y3@}ASb&yQ|<yA0N@T`!~g&Q

diff --git a/assignment_1/deliverable/nonlinear_model/variables/variables.index b/assignment_1/deliverable/nonlinear_model/variables/variables.index
index c5d43a16a552302844b94142dab02aabed6a2c1f..621cc6a4b58b498365c3d87ae5b0ac9f06a9c735 100644
GIT binary patch
delta 24
gcmZqWZRMR{%o6t1^43Q4pR6o}I~{IJHem|{0CoZilmGw#

delta 24
gcmZqWZRMR{%p&z|tMW$kpR6n?cMQ5Fo3I4}0B!#W)&Kwi

diff --git a/assignment_1/src/build_models.py b/assignment_1/src/build_models.py
index 3fa2543..29f8571 100644
--- a/assignment_1/src/build_models.py
+++ b/assignment_1/src/build_models.py
@@ -1,3 +1,5 @@
+""" Define, train and save the linear model and the non-linear model """
+
 from sklearn.linear_model import LinearRegression
 from sklearn.model_selection import train_test_split
 from tensorflow.keras import Input, Model
@@ -15,91 +17,114 @@ from keras import backend as K
 #
 # FIX THE RANDOM GENERATOR SEEDS
 #
-seed_value = 0
-# 1. Set `PYTHONHASHSEED` environment variable at a fixed value
-os.environ['PYTHONHASHSEED'] = str(seed_value)
-# 2. Set `python` built-in pseudo-random generator at a fixed value
-random.seed(seed_value)
-# 3. Set `numpy` pseudo-random generator at a fixed value
-np.random.seed(seed_value)
-# 4. Set the `tensorflow` pseudo-random generator at a fixed value
-tf.random.set_seed(seed_value)
-# 5. Configure a new global `tensorflow` session
+
+# The random generator seed is set to a fixed value for reproducibility
+# purposes. Since the libraries use different random generators, we set them
+# all to the fixed value below
+SEED_VALUE = 0
+
+os.environ['PYTHONHASHSEED'] = str(SEED_VALUE)
+random.seed(SEED_VALUE)
+np.random.seed(SEED_VALUE)
+tf.random.set_seed(SEED_VALUE)
 session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,
-        inter_op_parallelism_threads=1)
+                                        inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(),
-        config=session_conf)
+                            config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess)
 
-d = os.path.dirname(__file__)
-
+# Load data
 data = np.load("../data/data.npz")
-xs = data["x"] # 2000x2
-y = data["y"] # 2000x1
+xs = data["x"]
+y = data["y"]
 points = np.shape(xs)[0]
 
-# We manually include in the feature vectors a '1' column corresponding to theta_0,
-# so disable
+#
+# LINEAR MODEL
+#
+
+print("# Linear regression:")
+
+# We manually include in the feature vectors a '1' column corresponding to
+# theta_0, so disable the built in intercept in Sci-kit learn
 lr = LinearRegression(fit_intercept=False)
 
-# Build x feature vector with columns for theta_3 and theta_4
+# Build X feature matrix with columns for theta_3 and theta_4
 X = np.zeros([points, 5])
 X[:, 0] = 1
 X[:, 1:3] = xs
 X[:, 3] = xs[:, 0] * xs[:, 1]
 X[:, 4] = np.sin(xs[:, 0])
 
-# Shuffle our data for division in training, and test set
-
-train_ratio = 0.1
-validation_ratio = 0.1
-
-X_t, X_test, y_t, y_test = train_test_split(X, y, test_size=train_ratio)
-X_train, X_val, y_train, y_val = train_test_split(X_t, y_t, test_size=validation_ratio)
-np.savez('test', x=X_test, y=y_test, allow_pickle=True)
+# Shuffle and split our data for division in training, and test set
+TRAIN_SET_RATIO = 0.1
+X_t, X_test, y_t, y_test = train_test_split(X, y, test_size=TRAIN_SET_RATIO)
 
 # Fit with train data
 reg = lr.fit(X_t, y_t)
 
-print("# Linear regression:")
-
 # Print the resulting parameters
-print("f(x) = %g + %g * x_1 + %g * x_2 + %g * x_1 * x_2 + %g * sin(x_1)" % tuple(reg.coef_))
+print("f(x) = %g + %g * x_1 + %g * x_2 + %g * x_1 * x_2 + %g * sin(x_1)" %
+      tuple(reg.coef_))
 
+# Save the model as .pickle
 save_sklearn_model(reg, "../deliverable/linear_regression.pickle")
 
-# Non-linear regression:
+#
+# NON-LINEAR MODEL
+#
+
 print("\n# Feed-forward NN:")
 
-A = X_val
+# Divide previously found training set (X_t, y_t) in another training and a
+# validation set.  This division is used for the FFNN training and architecture
+# design/tailoring
+VALIDATION_SET_RATIO = 0.1
+X_train, X_val, y_train, y_val = \
+        train_test_split(X_t, y_t, test_size=VALIDATION_SET_RATIO)
+np.savez('test', x=X_test, y=y_test, allow_pickle=True)
 
+# Drop additional features added before
 X_train = X_train[:, 1:3]
 X_val = X_val[:, 1:3]
 
+# Compute mean and std for each feature in the training set
 mean = np.mean(X_train, axis=0)
 std = np.std(X_train, axis=0)
 
+# Normalize training data according to the mean and variance
 X_train -= mean
 X_train /= std
 
+# Normalize validation data as well. All further inputs to the NN must be
+# normalized using the value `mean` and `std` computed before. Normalization is
+# necessary to increase the speed of the learning process
 X_val -= mean
 X_val /= std
 
+# Define the network's architecture
 network = Sequential()
 network.add(Dense(22, activation='tanh'))
 network.add(Dense(15, activation='sigmoid'))
 network.add(Dense(1, activation='linear'))
 network.compile(optimizer='adam', loss='mse')
 
-epochs = 5000
+# Define maximum number of iterations and early stopping procedure
+EPOCHS = 5000
 callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=120)
-network.fit(X_train, y_train, epochs=epochs, verbose=1,
+
+# Fit the model monitoring validation in the learning process
+network.fit(X_train, y_train, epochs=EPOCHS, verbose=1,
             validation_data=(X_val, y_val), callbacks=[callback])
 
+# Save the fitted model and the normalization parameters as well
 network.save("../deliverable/nonlinear_model")
-save_sklearn_model({"mean": mean, "std": std}, "../deliverable/nonlinear_model_normalizers.pickle")
+save_sklearn_model({"mean": mean, "std": std},
+                   "../deliverable/nonlinear_model_normalizers.pickle")
 
-# Print the final validation set MSE, which was used to tailor the NN architecture after
-# several manual trials
+# Print the final validation set MSE, which was used to tailor the NN
+# architecture after several manual trials
 msq = mean_squared_error(network.predict(X_val), y_val)
-print(msq)
+print("Final validation MSE for FFNN: %g" % msq)
+
+# vim: set ts=4 sw=4 et tw=79: