Commit 4ea86af9 authored by Manfred Peter Fackeldey's avatar Manfred Peter Fackeldey
Browse files

adapt to new law version, use safe_output, ensure_voms_proxy and localize decorators as default

parent 5b5b77c3
......@@ -29,7 +29,7 @@ class AnalysisTask(law.Task):
)
version = luigi.Parameter(description="task version")
notify = law.NotifyTelegramParameter()
notify = law.telegram.NotifyTelegramParameter()
exclude_params_req = {"notify"}
exclude_params_branch = {"notify"}
......
......@@ -13,54 +13,55 @@ class PileUpReweighting(AnalysisTask):
}
@law.decorator.timeit(publish_message=True)
@law.decorator.safe_output
@law.decorator.localize
def run(self):
import ROOT
ROOT.PyConfig.IgnoreCommandLineOptions = True
ROOT.gROOT.SetBatch()
output = self.output()["pileup"]
with self.output()["pileup"].localize("w") as output:
output_path = output.path
pileup_file = self.config_inst.get_aux("pileup")["pileup_file"]
pileup_mc = self.config_inst.get_aux("pileup")["pileup_mc"]
num_pileup_bins = len(pileup_mc)
output_path = output.path
pileup_file = self.config_inst.get_aux("pileup")["pileup_file"]
pileup_mc = self.config_inst.get_aux("pileup")["pileup_mc"]
num_pileup_bins = len(pileup_mc)
# only nominal for now
min_bias_xsec = self.config_inst.get_aux(
"pileup")["min_bias_xsec"].nominal
# only nominal for now
min_bias_xsec = self.config_inst.get_aux(
"pileup")["min_bias_xsec"].nominal
lumi_file = self.config_inst.get_aux("lumi")["lumi_file"]
cmd = "pileupCalc.py -i {lumi_file} --inputLumiJSON {pileup_file} --calcMode true " \
"--minBiasXsec {min_bias_xsec} --maxPileupBin {num_pileup_bins} " \
"--numPileupBins {num_pileup_bins} {output_path}".format(
**locals())
lumi_file = self.config_inst.get_aux("lumi")["lumi_file"]
cmd = "pileupCalc.py -i {lumi_file} --inputLumiJSON {pileup_file} --calcMode true " \
"--minBiasXsec {min_bias_xsec} --maxPileupBin {num_pileup_bins} " \
"--numPileupBins {num_pileup_bins} {output_path}".format(
**locals())
code = law.util.interruptable_popen(
cmd, shell=True, executable="/bin/bash")[0]
code = law.util.interruptable_popen(
cmd, shell=True, executable="/bin/bash")[0]
# get data and MC pileup
with output.load("UPDATE", formatter="root") as output_tmp:
pileup_hist_data = output_tmp.Get(
"pileup").Clone("pileup_data")
pileup_hist_mc = ROOT.TH1D(
"pileup_mc", "", num_pileup_bins, 0., num_pileup_bins)
for i, prob in enumerate(pileup_mc):
# MC: fill bin and set error to 0 (TH1.Fill returns the bin number)
pileup_hist_mc.SetBinError(
pileup_hist_mc.Fill(i, prob), 0.)
pileup_hist_data.Scale(1. / pileup_hist_data.Integral())
pileup_hist_mc.Scale(1. / pileup_hist_mc.Integral())
pileup_hist_data.Write()
pileup_hist_mc.Write()
# get data and MC pileup
with output.load("UPDATE", formatter="root") as output_tmp:
pileup_hist_data = output_tmp.Get(
"pileup").Clone("pileup_data")
pileup_hist_mc = ROOT.TH1D(
"pileup_mc", "", num_pileup_bins, 0., num_pileup_bins)
for i, prob in enumerate(pileup_mc):
# MC: fill bin and set error to 0 (TH1.Fill returns the bin number)
pileup_hist_mc.SetBinError(
pileup_hist_mc.Fill(i, prob), 0.)
pileup_hist_data.Scale(1. / pileup_hist_data.Integral())
pileup_hist_mc.Scale(1. / pileup_hist_mc.Integral())
pileup_hist_data.Write()
pileup_hist_mc.Write()
# create weight histogram
weight_hist = pileup_hist_data.Clone("pileup_weights")
weight_hist.SetTitle("pileup weights")
weight_hist.Divide(pileup_hist_mc)
weight_hist.Write()
# create weight histogram
weight_hist = pileup_hist_data.Clone("pileup_weights")
weight_hist.SetTitle("pileup weights")
weight_hist.Divide(pileup_hist_mc)
weight_hist.Write()
ROOT.gDirectory.Delete("pileup;*")
output_tmp.Close()
ROOT.gDirectory.Delete("pileup;*")
output_tmp.Close()
# dump output
output.dump()
# dump output
output.dump()
......@@ -22,6 +22,8 @@ class GetDatasetLFNs(DatasetTask, law.TransferLocalFile):
h = law.util.create_hash(self.dataset_inst.keys)
return self.wlcg_target("lfns_{}.json".format(h))
@law.wlcg.ensure_voms_proxy
@law.decorator.safe_output
def run(self):
lfns = []
for key in self.dataset_inst.keys:
......@@ -80,7 +82,10 @@ class NanoProduction(DatasetTask, GridWorkflow, HTCondorWorkflow, law.LocalWorkf
"data": self.wlcg_target("{}.root".format(self.branch)),
}
@law.wlcg.ensure_voms_proxy
@law.decorator.timeit(publish_message=True)
@law.decorator.safe_output
@law.decorator.localize(input=False)
def run(self):
is_data = self.config_inst.get_dataset(self.dataset).is_data
# load the lfn for our branch
......@@ -90,49 +95,49 @@ class NanoProduction(DatasetTask, GridWorkflow, HTCondorWorkflow, law.LocalWorkf
redirector = "xrootd-cms.infn.it"
xrd_url = "root://{}/{}".format(redirector, lfn)
with self.output()["data"].localize("w") as nano:
mini = law.LocalFileTarget(is_tmp="root")
# needs `xrdcp-old` for some reason...
# xrdcp will lead to a symbol lookup error
cmd = "xrdcp-old {} {}".format(xrd_url, mini.path)
self.publish_message(
"Downloading file from {} ...".format(xrd_url))
nano = self.output()["data"]
mini = law.LocalFileTarget(is_tmp="root")
# needs `xrdcp-old` for some reason...
# xrdcp will lead to a symbol lookup error
cmd = "xrdcp-old {} {}".format(xrd_url, mini.path)
self.publish_message(
"Downloading file from {} ...".format(xrd_url))
code = law.util.interruptable_popen(
cmd, shell=True, executable="/bin/bash")[0]
if code != 0:
raise Exception("download failed")
cmsDriver_cmd = " ".join([
"cmsDriver.py",
"HHbbWW",
"-s NANO " + ("--data" if is_data else "--mc"),
"--eventcontent NANOAOD{} --datatier NANOAOD{}".format(
"SIM" if not is_data else "", "SIM" if not is_data else ""),
"--filein file:///{}".format(mini.path),
"--fileout {}".format(nano.path),
"--conditions 102X_mc2017_realistic_v7",
"--era Run2_2017,run2_nanoAOD_94XMiniAODv2",
"-n -1",
"--nThreads={}".format(1 if self.workflow ==
"glite" else os.environ["N_CORES"]),
"--customise_commands=\"from nanoHH.production.nanoHH import addVariables; addVariables(process, {}, '2017', 10000)\\n \
process.add_(cms.Service('InitRootHandlers', EnableIMT = cms.untracked.bool(False)))\"".format(
not is_data),
"--no_exec"
])
with self.publish_step(law.util.colored("Run following cmsDriver command:", color="light_cyan")):
self.publish_message(cmsDriver_cmd)
code = law.util.interruptable_popen(
cmd, shell=True, executable="/bin/bash")[0]
cmsDriver_cmd, shell=True, executable="/bin/bash")[0]
if code != 0:
raise Exception("download failed")
cmsDriver_cmd = " ".join([
"cmsDriver.py",
"HHbbWW",
"-s NANO " + ("--data" if is_data else "--mc"),
"--eventcontent NANOAOD{} --datatier NANOAOD{}".format(
"SIM" if not is_data else "", "SIM" if not is_data else ""),
"--filein file:///{}".format(mini.path),
"--fileout {}".format(nano.path),
"--conditions 102X_mc2017_realistic_v7",
"--era Run2_2017,run2_nanoAOD_94XMiniAODv2",
"-n -1",
"--nThreads={}".format(1 if self.workflow ==
"glite" else os.environ["N_CORES"]),
"--customise_commands=\"from nanoHH.production.nanoHH import addVariables; addVariables(process, {}, '2017', 10000)\\n \
process.add_(cms.Service('InitRootHandlers', EnableIMT = cms.untracked.bool(False)))\"".format(
not is_data),
"--no_exec"
])
with self.publish_step(law.util.colored("Run following cmsDriver command:", color="light_cyan")):
self.publish_message(cmsDriver_cmd)
raise Exception("cmsDriver failed")
with self.publish_step(law.util.colored("Run following cmsRun command:", color="light_cyan")):
cmsRun_cmd = "cmsRun HHbbWW_NANO.py"
self.publish_message(cmsRun_cmd)
code = law.util.interruptable_popen(
cmsDriver_cmd, shell=True, executable="/bin/bash")[0]
cmsRun_cmd, shell=True, executable="/bin/bash")[0]
if code != 0:
raise Exception("cmsDriver failed")
with self.publish_step(law.util.colored("Run following cmsRun command:", color="light_cyan")):
cmsRun_cmd = "cmsRun HHbbWW_NANO.py"
self.publish_message(cmsRun_cmd)
code = law.util.interruptable_popen(
cmsRun_cmd, shell=True, executable="/bin/bash")[0]
if code != 0:
raise Exception("cmsRun failed")
raise Exception("cmsRun failed")
class NanoProductionWrapper(AnalysisTask, law.WrapperTask):
......@@ -238,8 +243,10 @@ class MeasureTreeSizes(AnalysisTask):
def complete(self):
return self.has_run
@law.wlcg.ensure_voms_proxy
@law.decorator.timeit(publish_message=True)
@law.decorator.notify
@law.decorator.safe_output
def run(self):
merged_files = collections.OrderedDict()
......
......@@ -3,7 +3,6 @@
import law
from law.util import make_list
import luigi
import os
from tasks.base import AnalysisTask
from tasks.shapes import MergeHistograms
......@@ -31,33 +30,36 @@ class PrepareDatacardHists(PlotHelper):
def store_parts(self):
return super(PrepareDatacardHists, self).store_parts() + (self.channel,) + (self.category,)
@law.wlcg.ensure_voms_proxy
@law.decorator.timeit(publish_message=True)
@law.decorator.notify
@law.decorator.safe_output
@law.decorator.localize
def run(self):
# store histograms in ROOT file for CombineHarvester
with self.input()["collection"].localize("r") as input_:
with input_.targets[0].load("READ") as input_file:
processes_ = self.config_inst.get_aux(
"process_groups")["default"]
variable = self.config_inst.get_variable(self.variable)
channel = self.config_inst.get_channel(self.channel)
category = channel.get_category(self.category)
with self.output().localize("w") as tmp:
with tmp.dump("RECREATE") as output_file:
output_file.cd()
for process in processes_:
parent_process = self.config_inst.get_process(
process)
child_process_hists = self.prepare_hists(
input_file=input_file,
parent_process=parent_process,
channel=channel,
category=category,
variable=variable
)
proc_hist = self.add_hists(child_process_hists)
proc_hist.SetName(parent_process.name)
proc_hist.Write()
input_ = self.input()["collection"]
with input_.targets[0].load("READ") as input_file:
processes_ = self.config_inst.get_aux(
"process_groups")["default"]
variable = self.config_inst.get_variable(self.variable)
channel = self.config_inst.get_channel(self.channel)
category = channel.get_category(self.category)
tmp = self.output()
with tmp.dump("RECREATE") as output_file:
output_file.cd()
for process in processes_:
parent_process = self.config_inst.get_process(
process)
child_process_hists = self.prepare_hists(
input_file=input_file,
parent_process=parent_process,
channel=channel,
category=category,
variable=variable
)
proc_hist = self.add_hists(child_process_hists)
proc_hist.SetName(parent_process.name)
proc_hist.Write()
class DatacardProducer(AnalysisTask):
......@@ -83,6 +85,7 @@ class DatacardProducer(AnalysisTask):
@law.decorator.timeit(publish_message=True)
@law.decorator.notify
@law.decorator.safe_output
def run(self):
dw = DatacardWriter(analysis=self.analysis_choice,
mass="125", era="2017")
......@@ -157,6 +160,7 @@ class LimitProducer(AnalysisTask):
@law.decorator.timeit(publish_message=True)
@law.decorator.notify
@law.decorator.safe_output
def run(self):
self.output()["limit"].parent.touch()
out_dir = self.output()["limit"].parent.path
......
......@@ -88,102 +88,105 @@ class PlotProducer(PlotHelper):
def store_parts(self):
return super(PlotProducer, self).store_parts() + (self.channel,) + (self.category,)
@law.wlcg.ensure_voms_proxy
@law.decorator.timeit(publish_message=True)
@law.decorator.notify
@law.decorator.safe_output
@law.decorator.localize
def run(self):
SetTDRStyle()
with self.input()["collection"].localize("r") as input_:
with input_.targets[0].load("READ") as input_file:
processes_ = self.config_inst.get_aux(
"process_groups")["default"]
variable = self.config_inst.get_variable(self.variable)
channel = self.config_inst.get_channel(self.channel)
category = channel.get_category(self.category)
legend = ROOT.TLegend(0.3, 0.73, 0.90, 0.88)
legend.SetNColumns(2)
hists = []
for process in processes_:
if "diHiggs" in process:
continue
parent_process = self.config_inst.get_process(process)
child_process_hists = self.prepare_hists(
input_file=input_file,
parent_process=parent_process,
channel=channel,
category=category,
variable=variable
)
hist_to_stack = self.add_hists(child_process_hists)
hist_to_stack.SetLineWidth(0)
hist_to_stack.SetFillColor(
ROOT.TColor.GetColor(
parent_process.color_r,
parent_process.color_g,
parent_process.color_b,
)
)
legend.AddEntry(
hist_to_stack, parent_process.label_root + " [{:.0f}]".format(hist_to_stack.Integral()), "f")
hists.append(hist_to_stack)
hists = sorted(hists, key=lambda x: x.Integral())
c = ROOT.TCanvas("", "", 600, 600)
# draw normal stacked histograms
stack = ROOT.THStack("", "")
for h in hists:
stack.Add(h)
stack.Draw("HIST")
stack.GetXaxis().SetTitle(variable.get_full_x_title())
stack.GetYaxis().SetTitle(variable.get_full_y_title())
# Signal hardcoded for now
dihiggs = self.config_inst.get_process("diHiggs_bbWW_dl")
dihiggs_process_hists = self.prepare_hists(
input_ = self.input()["collection"]
with input_.targets[0].load("READ") as input_file:
processes_ = self.config_inst.get_aux(
"process_groups")["default"]
variable = self.config_inst.get_variable(self.variable)
channel = self.config_inst.get_channel(self.channel)
category = channel.get_category(self.category)
legend = ROOT.TLegend(0.3, 0.73, 0.90, 0.88)
legend.SetNColumns(2)
hists = []
for process in processes_:
if "diHiggs" in process:
continue
parent_process = self.config_inst.get_process(process)
child_process_hists = self.prepare_hists(
input_file=input_file,
parent_process=dihiggs,
parent_process=parent_process,
channel=channel,
category=category,
variable=variable
)
dihiggs_hist = self.add_hists(dihiggs_process_hists)
dihiggs_hist.SetLineWidth(3)
dihiggs_hist.SetLineColor(
hist_to_stack = self.add_hists(child_process_hists)
hist_to_stack.SetLineWidth(0)
hist_to_stack.SetFillColor(
ROOT.TColor.GetColor(
dihiggs.color_r,
dihiggs.color_g,
dihiggs.color_b,
parent_process.color_r,
parent_process.color_g,
parent_process.color_b,
)
)
dihiggs_hist.Scale(100.0)
legend.AddEntry(
dihiggs_hist, dihiggs.label_root + " (x100)", "l")
scale = 1e4 if self.log_scale else 1.4
stack.SetMaximum(stack.GetMaximum() * scale)
stack.SetMinimum(0.)
dihiggs_hist.Draw("HIST SAME")
if self.log_scale:
stack.SetMinimum(1.0)
ROOT.gPad.SetLogy()
legend.SetBorderSize(0)
legend.Draw()
# Add title
latex = ROOT.TLatex()
latex.SetNDC()
latex.SetTextSize(0.04)
latex.SetTextFont(42)
latex.DrawLatex(
0.6, 0.935, "{0:.1f} fb^{{-1}} (2017, 13 TeV)".format(
self.config_inst.get_aux("lumi")[channel] / 1000.))
latex.DrawLatex(0.16, 0.935, "#bf{CMS Private Work}")
ROOT.gPad.RedrawAxis()
# Save
self.output().parent.touch()
c.SaveAs(self.output().path)
hist_to_stack, parent_process.label_root + " [{:.0f}]".format(hist_to_stack.Integral()), "f")
hists.append(hist_to_stack)
hists = sorted(hists, key=lambda x: x.Integral())
c = ROOT.TCanvas("", "", 600, 600)
# draw normal stacked histograms
stack = ROOT.THStack("", "")
for h in hists:
stack.Add(h)
stack.Draw("HIST")
stack.GetXaxis().SetTitle(variable.get_full_x_title())
stack.GetYaxis().SetTitle(variable.get_full_y_title())
# Signal hardcoded for now
dihiggs = self.config_inst.get_process("diHiggs_bbWW_dl")
dihiggs_process_hists = self.prepare_hists(
input_file=input_file,
parent_process=dihiggs,
channel=channel,
category=category,
variable=variable
)
dihiggs_hist = self.add_hists(dihiggs_process_hists)
dihiggs_hist.SetLineWidth(3)
dihiggs_hist.SetLineColor(
ROOT.TColor.GetColor(
dihiggs.color_r,
dihiggs.color_g,
dihiggs.color_b,
)
)
dihiggs_hist.Scale(100.0)
legend.AddEntry(
dihiggs_hist, dihiggs.label_root + " (x100)", "l")
scale = 1e4 if self.log_scale else 1.4
stack.SetMaximum(stack.GetMaximum() * scale)
stack.SetMinimum(0.)
dihiggs_hist.Draw("HIST SAME")
if self.log_scale:
stack.SetMinimum(1.0)
ROOT.gPad.SetLogy()
legend.SetBorderSize(0)
legend.Draw()
# Add title
latex = ROOT.TLatex()
latex.SetNDC()
latex.SetTextSize(0.04)
latex.SetTextFont(42)
latex.DrawLatex(
0.6, 0.935, "{0:.1f} fb^{{-1}} (2017, 13 TeV)".format(
self.config_inst.get_aux("lumi")[channel] / 1000.))
latex.DrawLatex(0.16, 0.935, "#bf{CMS Private Work}")
ROOT.gPad.RedrawAxis()
# Save
self.output().parent.touch()
c.SaveAs(self.output().path)
class ChannelPlots(AnalysisTask, law.WrapperTask):
......@@ -226,94 +229,97 @@ class CompareShapePlot(PlotHelper):
def store_parts(self):
return super(CompareShapePlot, self).store_parts() + (self.channel,) + (self.category,)
@law.wlcg.ensure_voms_proxy
@law.decorator.timeit(publish_message=True)
@law.decorator.notify
@law.decorator.safe_output
@law.decorator.localize
def run(self):
SetTDRStyle()
with self.input()["collection"].localize("r") as input_:
with input_.targets[0].load("READ") as input_file:
processes_ = self.config_inst.get_aux(
"process_groups")["default"]
variable = self.config_inst.get_variable(self.variable)
channel = self.config_inst.get_channel(self.channel)
category = channel.get_category(self.category)
hists = []
for process in processes_:
if "diHiggs" in process:
continue
parent_process = self.config_inst.get_process(process)
child_process_hists = self.prepare_hists(
input_file=input_file,
parent_process=parent_process,
channel=channel,
category=category,
variable=variable
)
hist_to_stack = self.add_hists(*child_process_hists)
hist_to_stack.SetLineWidth(0)
hists.append(hist_to_stack)
c = ROOT.TCanvas("", "", 600, 600)
# draw all mc in grey normalized to unity:
stack = self.add_hists(*hists)
norm = stack.Integral()
stack.Scale(1. / (norm if norm > 0 else 1.))
stack.SetFillColor(ROOT.TColor.GetColor(211, 211, 211))
stack.SetMinimum(0.)
stack.Draw("hist")
legend = ROOT.TLegend(0.4, 0.73, 0.90, 0.88)
legend.AddEntry(stack, "All MC", "f")
stack.GetXaxis().SetTitle(variable.get_full_x_title())
stack.GetYaxis().SetTitle(variable.get_full_y_title())
# Signal hardcoded for now
dihiggs = self.config_inst.get_process("diHiggs_bbWW_dl")
dihiggs_process_hists = self.prepare_hists(
input_ = self.input()["collection"]
with input_.targets[0].load("READ") as input_file:
processes_ = self.config_inst.get_aux(
"process_groups")["default"]
variable = self.config_inst.get_variable(self.variable)
channel = self.config_inst.get_channel(self.channel)
category = channel.get_category(self.category)
hists = []
for process in processes_:
if "diHiggs" in process:
continue
parent_process = self.config_inst.get_process(process)
child_process_hists = self.prepare_hists(
input_file=input_file,
parent_process=dihiggs,
parent_process=parent_process,
channel=channel,
category=category,
variable=variable
)
dihiggs_hist = self.add_hists(dihiggs_process_hists)
dihiggs_hist.SetLineWidth(3)
dihiggs_hist.SetLineColor(
ROOT.TColor.GetColor(
dihiggs.color_r,
dihiggs.color_g,
dihiggs.color_b,
)
hist_to_stack = self.add_hists(*child_process_hists)
hist_to_stack.SetLineWidth(0)
hists.append(hist_to_stack)
c = ROOT.TCanvas("", "", 600, 600)
# draw all mc in grey normalized to unity:
stack = self.add_hists(*hists)
norm = stack.Integral()
stack.Scale(1. / (norm if norm > 0 else 1.))
stack.SetFillColor(ROOT.TColor.GetColor(211, 211, 211))
stack.SetMinimum(0.)
stack.Draw("hist")
legend = ROOT.TLegend(0.4, 0.73, 0.90, 0.88)
legend.AddEntry(stack, "All MC", "f")
stack.GetXaxis().SetTitle(variable.get_full_x_title())
stack.GetYaxis().SetTitle(variable.get_full_y_title())
# Signal hardcoded for now
dihiggs = self.config_inst.get_process("diHiggs_bbWW_dl")
dihiggs_process_hists = self.prepare_hists(
input_file=input_file,
parent_process=dihiggs,
channel=channel,
category=category,
variable=variable
)
dihiggs_hist = self.add_hists(dihiggs_process_hists)
dihiggs_hist.SetLineWidth(3)
dihiggs_hist.SetLineColor(
ROOT.TColor.GetColor(