fix merging runs with different progress

src/mc.cpp

@@ -16,8 +16,6 @@ void mc::_init() {
 	// simple profiling support: measure the time spent for sweeps/measurements etc
 	measure.register_observable("_ll_checkpoint_read_time", 1);
 	measure.register_observable("_ll_checkpoint_write_time", 1);
-	measure.register_observable("_ll_measurement_time", 1000);
-	measure.register_observable("_ll_sweep_time", 1000);
 
 	if(param.defined("seed")) {
 		rng.reset(new random_number_generator(param.get<uint64_t>("seed")));

src/merger.cpp

@@ -100,6 +100,10 @@ results merge(const std::vector<std::string> &filenames, size_t rebinning_bin_le
 			std::vector<double> samples;
 			obs.name = obs_name;
 
+			if(!g.exists(obs_name)) {
+				continue;
+			}
+
 			g.read(fmt::format("{}/samples", obs_name), samples);
 
 			// rebinning_bin_count*rebinning_bin_length may be smaller than
@@ -145,6 +149,10 @@ results merge(const std::vector<std::string> &filenames, size_t rebinning_bin_le
 
 			size_t vector_length = obs.mean.size();
 
+			if(!g.exists(obs_name)) {
+				continue;
+			}
+
 			g.read(fmt::format("{}/samples", obs_name), samples);
 
 			for(size_t i = sample_skip * vector_length;

test/silly_mc/checkpoint_test.sh

+#!/bin/bash
+
+mc=$1
+testparams=$2
+
+rm -rf silly_job_long.data
+
+mpirun -np 3 $mc $testparams
+if [ $? -ne 0 ]; then
+	exit 1
+fi
+
+mpirun -np 3 $mc $testparams
+if [ $? -ne 0 ]; then
+	exit 1
+fi
+
+echo "removing observable from run0002..."
+# this simulates progress differences in runs of the same task that may or may not yet know about different observables.
+python3 - <<'END'
+import h5py
+
+file = h5py.File('silly_job_long.data/task0001/run0002.meas.h5', 'a')
+del file['/MagicNumber']
+file.close()
+END
+
+echo "trying to merge..."
+$mc merge $testparams
+if [ $? -ne 0 ]; then
+	exit 1
+fi

test/silly_mc/meson.build

@@ -3,6 +3,8 @@ silly_mc = executable('silly_mc_test',
 	dependencies : loadleveller_dep,
 )
 
-testscript = find_program('silly_test.py')
+algebraic_test = find_program('silly_test.py')
+checkpoint_test = find_program('checkpoint_test.sh')
 
-test('algebraic series mc', testscript, args : [silly_mc, files('test_parameters.json')]) 
+test('algebraic series mc', algebraic_test, args : [silly_mc, files('test_parameters.json')]) 
+test('checkpoint test', checkpoint_test, args : [silly_mc, files('test_parameters_long.json')])

test/silly_mc/test_parameters_long.json

+{
+ "jobname": "silly_job_long",
+ "jobconfig": {
+  "num_cores" : 3,
+  "mc_runtime": "00:03",
+  "mc_checkpoint_time": "00:02",
+  "merge_rebin_length": 7
+ },
+ "tasks": {
+  "task0001": {
+   "sweeps": 1000000000,
+   "thermalization": 20,
+   "binsize": 1000
+  }
+ }
+}