diff --git a/Project3/src/Figures/DoubleSample/predicted_vs_true_both_samples_sample_1.pdf b/Project3/src/Figures/DoubleSample/predicted_vs_true_both_samples_sample_1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..87187cdd18b429a07c0007d4b6c728dbb2527c68
Binary files /dev/null and b/Project3/src/Figures/DoubleSample/predicted_vs_true_both_samples_sample_1.pdf differ
diff --git a/Project3/src/Figures/predicted_vs_true_sample_1.pdf b/Project3/src/Figures/DoubleSample/predicted_vs_true_both_samples_sample_2.pdf
similarity index 63%
rename from Project3/src/Figures/predicted_vs_true_sample_1.pdf
rename to Project3/src/Figures/DoubleSample/predicted_vs_true_both_samples_sample_2.pdf
index 080806b8481c25c8bec2a7a7eab5ad8b87e73507..7434028d9210c91eac7b93bce147ebcc0617dab0 100644
Binary files a/Project3/src/Figures/predicted_vs_true_sample_1.pdf and b/Project3/src/Figures/DoubleSample/predicted_vs_true_both_samples_sample_2.pdf differ
diff --git a/Project3/src/Figures/InitialCondition.pdf b/Project3/src/Figures/InitialCondition.pdf
deleted file mode 100644
index 916e108fd534bf23a24c7b7c187af03baf9bedd9..0000000000000000000000000000000000000000
Binary files a/Project3/src/Figures/InitialCondition.pdf and /dev/null differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/best_loss_theta.pdf b/Project3/src/Figures/Optimization/SingleSample/best_loss_theta.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..7c98468b0e8af5d1e37b4f1ed80c983b2e89db8e
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/best_loss_theta.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/best_losses.pdf b/Project3/src/Figures/Optimization/SingleSample/best_losses.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..2dbc3ca29e34c74bcff53a91ac59b22fbe0fb6c8
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/best_losses.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/best_predicted_vs_true_single_sample.pdf b/Project3/src/Figures/Optimization/SingleSample/best_predicted_vs_true_single_sample.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..3b509e2bb7969f697673d0423c49bd9db06c4ee1
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/best_predicted_vs_true_single_sample.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/best_prediction_kx_ITERATION.pdf b/Project3/src/Figures/Optimization/SingleSample/best_prediction_kx_ITERATION.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..0e914f083702a2fe5b0ecaeed4899c2b355dfbce
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/best_prediction_kx_ITERATION.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_0.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_0.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..ff5f1b6643a5db2918fd03db1a2c56a46e16248b
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_0.pdf differ
diff --git a/Project3/src/Figures/loss_theta.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_1.pdf
similarity index 59%
rename from Project3/src/Figures/loss_theta.pdf
rename to Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_1.pdf
index 15667db83ac61ba01c5c8fa9dfe8ff96f10ff8cb..a1af6ee52990d634f82a936f223acf0e1bd0916a 100644
Binary files a/Project3/src/Figures/loss_theta.pdf and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_1.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_10.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_10.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..22b4aa635bba157af5333eb7f5113439a86aa233
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_10.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_2.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_2.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..f019371b15094de4ffcdc618e36f961de8a8dce1
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_2.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_3.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_3.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..f7e03c440fa827eff61e351224207541332f02c8
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_3.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_4.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_4.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..aad9824b89ae27c524611b537744d220b1b17798
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_4.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_5.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_5.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..11b229bbb5e93e45d6194b80ec7f1d802e861aec
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_5.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_6.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_6.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..537991cbb8b09d2747bb38a41f5072c9f9a23741
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_6.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_7.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_7.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..affd206dd40978fece6091b733d54d9104dd7635
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_7.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_8.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_8.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..71137ffc4ab871708356faf857a4a220def97920
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_8.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_9.pdf b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_9.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..50df35e3790882bec05d3798a5d19974757b5ce5
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/loss_theta_ITERATION_9.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_0.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_0.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..ca81a0f2d03b1e8c001af10ca7b4da312bd0e0b6
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_0.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_1.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..b2d2165ebc5f618be0b6da3ddce581e323372e60
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_1.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_10.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_10.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..602e8dea69d4b24e457bde84de629e80218efa2b
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_10.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_2.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_2.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..7061e2f656fe93358991b8aaf015591d17e3157c
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_2.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_3.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_3.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..f2c6b5cad88ad65d48e77756bd9a241602e6d44a
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_3.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_4.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_4.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..b30a9f187bd4d7df33436cb8ed0c6fba1e4c9f3a
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_4.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_5.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_5.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..d68684560d456c614947ce7e68bcab7f17327f68
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_5.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_6.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_6.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..8bf467c71c002a4e735cd55687ef51595b6a02b8
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_6.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_7.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_7.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..674d02ccb0e15133eb11fd6a333ba9d0727dac03
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_7.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_8.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_8.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..ddbcdce63b9508862e87aec2e9560b5df95d8edc
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_8.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_9.pdf b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_9.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..b6d773b9de6cdb48ca79afb385512c8db228cef7
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/losses_ITERATION_9.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_0.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_0.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..9515a6c3464d2150137a35204474eb044f71b543
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_0.pdf differ
diff --git a/Project3/src/Figures/predicted_vs_true_both_samples_sample_1.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_1.pdf
similarity index 65%
rename from Project3/src/Figures/predicted_vs_true_both_samples_sample_1.pdf
rename to Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_1.pdf
index 1bdf500fd46d493e27d635d5659c1e86278c68c9..2959796ea2814d288956a011dce8357dac099e66 100644
Binary files a/Project3/src/Figures/predicted_vs_true_both_samples_sample_1.pdf and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_1.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_10.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_10.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..434f81b7fdf899fb41be389d434bc3e09b098af3
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_10.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_2.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_2.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..712807c2dca48fabdbe6cf63b006e79f01ce3e74
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_2.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_3.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_3.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..d039b3fef0a70a1a6cb0f7b9f75ffaa694fabb9d
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_3.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_4.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_4.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..ec03030d364d25511e87cc799418b17adbb04de1
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_4.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_5.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_5.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..5a0b7d7b1db39a7be0440b1e2e660a2303e3597c
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_5.pdf differ
diff --git a/Project3/src/Figures/predicted_vs_true_both_samples_sample_2.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_6.pdf
similarity index 60%
rename from Project3/src/Figures/predicted_vs_true_both_samples_sample_2.pdf
rename to Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_6.pdf
index 8321b28b18799d40bca52a8345fda58572862e23..856a2710e1191a19f64c5e9b1307a8844a11b699 100644
Binary files a/Project3/src/Figures/predicted_vs_true_both_samples_sample_2.pdf and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_6.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_7.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_7.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..7f63152cb94b0a1a6ca542379f0ba6921f982854
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_7.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_8.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_8.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..e312ac188d2dd9b4fbcf0f919328d1338cdfb7e0
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_8.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_9.pdf b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_9.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..12bc391686bb8d2d493d1bd4ffc938ca5abf9916
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/predicted_vs_true_single_sample_ITERATION_9.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_0.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_0.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..06734e94a6a36ad89f744f57909f5b418b4c54bb
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_0.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_1.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_1.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..49f8dcb9c0983ae518c9c7e2ef914c41ae9c4cf0
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_1.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_10.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_10.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..8474b622a92b4132c93531fd4d62b5c8d8600b3b
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_10.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_2.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_2.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..c347397ec031c49cfe0a1557e3a3bad4d6087607
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_2.pdf differ
diff --git a/Project3/src/Figures/losses.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_3.pdf
similarity index 58%
rename from Project3/src/Figures/losses.pdf
rename to Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_3.pdf
index 30dd658aeffbb62590d271fca628f59a571d3777..c4258ad6750d1b8a420aa7c54e848493440d395b 100644
Binary files a/Project3/src/Figures/losses.pdf and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_3.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_4.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_4.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..a8d06b6b7a00dab50e863a82df7dfd596036b8d5
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_4.pdf differ
diff --git a/Project3/src/Figures/prediction_kx.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_5.pdf
similarity index 83%
rename from Project3/src/Figures/prediction_kx.pdf
rename to Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_5.pdf
index 5d5345eb78cee6d80e955fe83eddccf1afd2f6f6..be4a47b6e7b10ca9c3e419754288dcca12a11d05 100644
Binary files a/Project3/src/Figures/prediction_kx.pdf and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_5.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_6.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_6.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..2d83c593ef9d2506cf0223227c10ce344e5a6f8e
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_6.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_7.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_7.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..8099ea23a4075e049a7b3f87192aff76e3407209
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_7.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_8.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_8.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..98f33cf95df29adda34b928e1155d7b956e8f8e3
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_8.pdf differ
diff --git a/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_9.pdf b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_9.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..0072882833aaba801dca86364ca780b86f9a204c
Binary files /dev/null and b/Project3/src/Figures/Optimization/SingleSample/prediction_kx_ITERATION_9.pdf differ
diff --git a/Project3/src/Figures/Optimization/TunerConvergence.pdf b/Project3/src/Figures/Optimization/TunerConvergence.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..df817fc4958cf41872783084064333dd8aafe622
Binary files /dev/null and b/Project3/src/Figures/Optimization/TunerConvergence.pdf differ
diff --git a/Project3/src/Figures/Optimization/TunerConvergence.png b/Project3/src/Figures/Optimization/TunerConvergence.png
new file mode 100644
index 0000000000000000000000000000000000000000..2bd50af546176c906e560aa9c5d9288afbc2ac5f
Binary files /dev/null and b/Project3/src/Figures/Optimization/TunerConvergence.png differ
diff --git a/Project3/src/HyperParameterOptimization.py b/Project3/src/HyperParameterOptimization.py
new file mode 100644
index 0000000000000000000000000000000000000000..79ed06fcb9c3978f5b9c33bc5349b8daf9bca197
--- /dev/null
+++ b/Project3/src/HyperParameterOptimization.py
@@ -0,0 +1,293 @@
+import numpy as np
+import matplotlib.pyplot as plt
+
+import skopt
+from skopt import gp_minimize # Import the skopt library for hyperparameter optimization
+from skopt.plots import plot_convergence, plot_objective
+from skopt.space import Real
+from skopt.utils import use_named_args
+
+from MCMC import *
+
+####################################################################################################
+# Setup
+####################################################################################################
+n_calls = 11 #!25
+
+# MCMC and solver parameters
+MCMC_STEPS = 100#!500
+MCMC_SUBSTEPS = 20
+NUMBER_SAMPLES = 1
+if NUMBER_SAMPLES == 1:
+ FigureFolder = 'Figures/Optimization/SingleSample'
+else:
+ FigureFolder = 'Figures/Optimization/DoubleSample'
+
+x0, x1 = 0, 4
+
+NTime = 500
+T_end = 10
+
+# Upwind viscosity parameter (used in flux computation)
+M = 1
+
+k_i = 9
+k_mu, k_sigma_default = 1., 0.25
+
+beta_default = 0.05
+
+
+# Define hyperparameters for optimization
+dim_beta = Real(low=0.01, high=0.1, name='dim_beta')
+dim_k_sigma = Real(low=0.1, high=0.4, name='dim_k_sigma')
+
+dimensions = [
+ dim_beta,
+ dim_k_sigma,
+]
+
+# start parameter for optimization
+default_parameters = [
+ beta_default,
+ k_sigma_default,
+]
+
+
+####################################################################################################
+# Hyperparameter Optimization
+####################################################################################################
+@use_named_args(dimensions=dimensions)
+def fitness(dim_beta, dim_k_sigma):
+ global ITERATION
+
+ print(f'ITERATION: {ITERATION}')
+ # Print the hyper-parameter configuration
+ print('beta:', dim_beta)
+ print('k_sigma:', dim_k_sigma)
+ print()
+
+ # MCMC
+ Theta, x_Theta, LOSS_THETA, LOSS_PHI = MCMC(
+ MCMC_STEPS=MCMC_STEPS,
+ MCMC_SUBSTEPS=MCMC_SUBSTEPS,
+ tau=tau,
+ beta=dim_beta,
+ gamma=gamma,
+ k_i=k_i,
+ k_mu=k_mu,
+ k_sigma=dim_k_sigma,
+ u_true=u_true,
+ number_samples=NUMBER_SAMPLES
+ )
+
+ # Get the error
+ error = LOSS_THETA[-1]
+
+ print(f'Error: {error}')
+
+ if np.isnan(error):
+ error = 1e10
+
+ # plotting
+ # Combine both loss functions
+ plt.figure()
+ plt.title('Losses')
+ plt.plot(LOSS_THETA[:], '--o', color='red', label='$\Phi(\Theta)$')
+ plt.plot(LOSS_PHI[:], '--o', color='green', label='$\Phi(\phi$)')
+ plt.xlabel('Iteration')
+ plt.ylabel('Loss')
+ plt.legend()
+ plt.grid()
+ plt.savefig(f'{FigureFolder}/losses_ITERATION_{ITERATION}.pdf', bbox_inches='tight')
+ # plt.show()
+ plt.close()
+
+ # Only the phi(theta) loss is plotted
+ plt.figure()
+ plt.title('Evolution of entropy loss')
+ plt.loglog(LOSS_THETA[:], '--o', color='red', label='$\Phi(\Theta)$')
+ plt.xlabel('Iteration')
+ plt.ylabel('$\Phi(\Theta)$')
+ plt.grid()
+ plt.savefig(f'{FigureFolder}/loss_theta_ITERATION_{ITERATION}.pdf', bbox_inches='tight')
+ # plt.show()
+ plt.close()
+
+ # Solve with the final Theta
+ t_vec, h_final = solve_PDE(Theta, T_end, NTime, x, dx, x_Theta, NUMBER_SAMPLES)
+ h_final = [extract_positions(t_vec[i], h_final[i], x) for i in range(NUMBER_SAMPLES)]
+ h_final = np.array([h_final[i].T for i in range(NUMBER_SAMPLES)])
+
+ colors = ['blue', 'red', 'green', 'purple']
+ for sample in range(NUMBER_SAMPLES):
+ plt.figure()
+ plt.title(f'Predicted vs True - sample {sample+1}')
+ for i in range(4):
+ plt.plot([], [], color=colors[i], label=f'x = {x_observed[i]}')
+ plt.plot(h_final[sample, i, :], color=colors[i])
+ plt.plot(u_true[i, :, sample], '--o', color=colors[i])
+ plt.plot([], [], color='black', label='Predicted')
+ plt.plot([], [], '--o', color='black', label='True')
+ plt.xlabel('Time')
+ plt.ylabel('u(x,t)')
+ plt.legend(ncol=4, loc='upper center', bbox_to_anchor=(0.5, -0.1))
+ plt.grid()
+ if NUMBER_SAMPLES == 1:
+ plt.savefig(f'{FigureFolder}/predicted_vs_true_single_sample_ITERATION_{ITERATION}.pdf', bbox_inches='tight')
+ else:
+ plt.savefig(f'{FigureFolder}/predicted_vs_true_both_samples_sample_{sample+1}_ITERATION_{ITERATION}.pdf', bbox_inches='tight')
+ # plt.show()
+ plt.close()
+
+ # Plot the final theta -> prediction for k(x)
+ plt.figure()
+ plt.title('Prediction for k(x)')
+ plt.plot(x_Theta, Theta, '--o', color='blue')
+ plt.xlabel('$x$')
+ plt.ylabel('$k(x)$')
+ plt.grid()
+ plt.savefig(f'{FigureFolder}/prediction_kx_ITERATION_{ITERATION}.pdf', bbox_inches='tight')
+ # plt.show()
+ plt.close()
+
+ # next iteration
+ ITERATION += 1
+
+ return error
+
+
+
+
+####################################################################################################
+# Optimization
+####################################################################################################
+ITERATION = 0
+
+# Run the hyperparameter optimization
+search_result = gp_minimize(
+ func=fitness,
+ dimensions=dimensions,
+ base_estimator='GP',#DUMMY', # ! 'GP', 'RF', 'ET', 'GBRT', 'DUMMY'
+ acq_func='EI', # Expected Improvement.
+ n_calls=n_calls,
+ x0=default_parameters,
+ random_state=42,
+)
+
+# Save results + timings
+with open('data/TunerResults.txt', 'w') as f:
+ f.write('... Tuner Results and Timings ...\n')
+ f.write(f'... Tuner results ...\n')
+ f.write(f'n_calls: {n_calls}\n')
+ f.write(f'Best error: {search_result.fun}\n')
+ f.write(f'Best parameters: {search_result.x}\n')
+ f.write(f'Best beta: {search_result.x[0]}\n')
+ f.write(f'Best k_sigma: {search_result.x[1]}\n')
+
+# Save to another file
+np.savez(
+ 'data/TunerResults.npz',
+ n_calls=n_calls, best_error=search_result.fun,
+ best_parameters=search_result.x,
+ best_beta=search_result.x[0],
+ beta_k_sigma=search_result.x[1],
+)
+
+
+# print result
+print(search_result.x)
+
+# Plot convergence
+plt.figure()
+_ = plot_convergence(search_result)
+plt.savefig('Figures/Optimization/TunerConvergence.pdf')
+plt.savefig('Figures/Optimization/TunerConvergence.png')
+# plt.show()
+plt.close()
+
+# Plot objective
+plt.figure()
+_ = plot_objective(search_result, show_points=True, size=3.8)
+plt.savefig('Figures/Optimization/TunerObjective.pdf')
+plt.savefig('Figures/Optimization/TunerObjective.png')
+# plt.show()
+plt.close()
+
+
+
+# Train best one
+# MCMC
+Theta, x_Theta, LOSS_THETA, LOSS_PHI = MCMC(
+ MCMC_STEPS=MCMC_STEPS,
+ MCMC_SUBSTEPS=MCMC_SUBSTEPS,
+ tau=tau,
+ beta=search_result.x[0],
+ gamma=gamma,
+ k_i=k_i,
+ k_mu=k_mu,
+ k_sigma=search_result.x[1],
+ u_true=u_true,
+ number_samples=NUMBER_SAMPLES
+)
+
+# plotting
+# Combine both loss functions
+plt.figure()
+plt.title('Losses')
+plt.plot(LOSS_THETA[:], '--o', color='red', label='$\Phi(\Theta)$')
+plt.plot(LOSS_PHI[:], '--o', color='green', label='$\Phi(\phi$)')
+plt.xlabel('Iteration')
+plt.ylabel('Loss')
+plt.legend()
+plt.grid()
+plt.savefig(f'{FigureFolder}/best_losses.pdf', bbox_inches='tight')
+# plt.show()
+plt.close()
+
+# Only the phi(theta) loss is plotted
+plt.figure()
+plt.title('Evolution of entropy loss')
+plt.loglog(LOSS_THETA[:], '--o', color='red', label='$\Phi(\Theta)$')
+plt.xlabel('Iteration')
+plt.ylabel('$\Phi(\Theta)$')
+plt.grid()
+plt.savefig(f'{FigureFolder}/best_loss_theta.pdf', bbox_inches='tight')
+# plt.show()
+plt.close()
+
+# Solve with the final Theta
+t_vec, h_final = solve_PDE(Theta, T_end, NTime, x, dx, x_Theta, NUMBER_SAMPLES)
+h_final = [extract_positions(t_vec[i], h_final[i], x) for i in range(NUMBER_SAMPLES)]
+h_final = np.array([h_final[i].T for i in range(NUMBER_SAMPLES)])
+
+colors = ['blue', 'red', 'green', 'purple']
+for sample in range(NUMBER_SAMPLES):
+ plt.figure()
+ plt.title(f'Predicted vs True - sample {sample+1}')
+ for i in range(4):
+ plt.plot([], [], color=colors[i], label=f'x = {x_observed[i]}')
+ plt.plot(h_final[sample, i, :], color=colors[i])
+ plt.plot(u_true[i, :, sample], '--o', color=colors[i])
+ plt.plot([], [], color='black', label='Predicted')
+ plt.plot([], [], '--o', color='black', label='True')
+ plt.xlabel('Time')
+ plt.ylabel('u(x,t)')
+ plt.legend(ncol=4, loc='upper center', bbox_to_anchor=(0.5, -0.1))
+ plt.grid()
+ if NUMBER_SAMPLES == 1:
+ plt.savefig(f'{FigureFolder}/best_predicted_vs_true_single_sample.pdf', bbox_inches='tight')
+ else:
+ plt.savefig(f'{FigureFolder}/best_predicted_vs_true_both_samples_sample_{sample+1}.pdf', bbox_inches='tight')
+ # plt.show()
+ plt.close()
+
+# Plot the final theta -> prediction for k(x)
+plt.figure()
+plt.title('Prediction for k(x)')
+plt.plot(x_Theta, Theta, '--o', color='blue')
+plt.xlabel('$x$')
+plt.ylabel('$k(x)$')
+plt.grid()
+plt.savefig(f'{FigureFolder}/best_prediction_kx_ITERATION.pdf', bbox_inches='tight')
+# plt.show()
+plt.close()
\ No newline at end of file
diff --git a/Project3/src/MCMC.py b/Project3/src/MCMC.py
index d2a57b43165522396ac2df4e096853b71b806544..75667502a74b130c2f607af4e12439d0d2bdfabb 100644
--- a/Project3/src/MCMC.py
+++ b/Project3/src/MCMC.py
@@ -297,8 +297,8 @@ if __name__ == '__main__':
# Combine both loss functions
plt.figure()
plt.title('Losses')
- [plt.plot(LOSS_THETA[:][i], '--o', color='red', label='$\Phi(\Theta)$') for i in range(NUMBER_SAMPLES)]
- [plt.plot(LOSS_PHI[:][i], '--o', color='green', label='$\Phi(\phi$)') for i in range(NUMBER_SAMPLES)]
+ plt.plot(LOSS_THETA[:], '--o', color='red', label='$\Phi(\Theta)$')
+ plt.plot(LOSS_PHI[:], '--o', color='green', label='$\Phi(\phi$)')
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.legend()
@@ -309,7 +309,7 @@ if __name__ == '__main__':
# Only the phi(theta) loss is plotted
plt.figure()
plt.title('Evolution of entropy loss')
- [plt.loglog(LOSS_THETA[:][i], '--o', color='red', label='$\Phi(\Theta)$') for i in range(NUMBER_SAMPLES)]
+ plt.loglog(LOSS_THETA[:], '--o', color='red', label='$\Phi(\Theta)$')
plt.xlabel('Iteration')
plt.ylabel('$\Phi(\Theta)$')
plt.grid()
diff --git a/Project3/src/data/TunerResults.npz b/Project3/src/data/TunerResults.npz
new file mode 100644
index 0000000000000000000000000000000000000000..43e6d2932c9f521ca735f5c6f0bce4af6c675854
Binary files /dev/null and b/Project3/src/data/TunerResults.npz differ
diff --git a/Project3/src/data/TunerResults.txt b/Project3/src/data/TunerResults.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d3f0f66c8ced59b58bd24396bdaaefeb56d41f2d
--- /dev/null
+++ b/Project3/src/data/TunerResults.txt
@@ -0,0 +1,7 @@
+... Tuner Results and Timings ...
+... Tuner results ...
+n_calls: 11
+Best error: 43.32473738706938
+Best parameters: [0.012075618253727419, 0.25743239807751683]
+Best beta: 0.012075618253727419
+Best k_sigma: 0.25743239807751683