diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt index b52a705ca..917341962 100644 --- a/Framework/CMakeLists.txt +++ b/Framework/CMakeLists.txt @@ -107,6 +107,7 @@ add_library(O2QualityControl src/PostProcessingInterface.cxx src/PostProcessingDevice.cxx src/TrendingTask.cxx + src/TrendingMachineConfig.cxx src/TrendingTaskConfig.cxx src/DummyDatabase.cxx src/DataProducer.cxx @@ -143,6 +144,7 @@ add_library(O2QualityControl src/LateTaskRunner.cxx src/LateTaskRunnerFactory.cxx src/LateTaskInterface.cxx + src/TrendingMachine.cxx src/Actor.cxx src/ActorHelpers.cxx src/DataProcessorAdapter.cxx @@ -301,6 +303,8 @@ add_executable(o2-qc-test-core test/testVersion.cxx test/testMonitorObjectCollection.cxx test/testTrendingTask.cxx + test/testTrendingMachine.cxx + test/testTrendingMachineConfig.cxx test/testKafkaTests.cxx test/testFlagHelpers.cxx test/testQualitiesToFlagCollectionConverter.cxx diff --git a/Framework/include/QualityControl/TrendingMachine.h b/Framework/include/QualityControl/TrendingMachine.h new file mode 100644 index 000000000..ebfc741e6 --- /dev/null +++ b/Framework/include/QualityControl/TrendingMachine.h @@ -0,0 +1,125 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file TrendingMachine.h +/// \author Piotr Konopka +/// +/// \brief Manager class for trending functionality (TTree management, reductors, plot generation) +/// +/// This class contains code extracted from TrendingTask to eliminate duplication between +/// TrendingTask and LateTrendingTask. +/// +/// Code history can be accessed with: +/// git log --follow Framework/src/TrendingMachine.cxx +/// git blame -C -C -C Framework/src/TrendingMachine.cxx +/// + +#ifndef QUALITYCONTROL_TRENDINGMACHINE_H +#define QUALITYCONTROL_TRENDINGMACHINE_H + +#include "QualityControl/TrendingMachineConfig.h" +#include +#include +#include +#include +#include +#include + +class TTree; +class TCanvas; +class TGraph; +class TAxis; +class TH1; +class TObject; + +namespace o2::quality_control::postprocessing +{ + +class Reductor; + +/// \brief A class to generate trends and trending plots +/// +/// This class runs reductors to extract data points from objects, stores them +/// in a columnar data storage facility and runs plot (trends and correlations) +/// generation. Historically, it was extracted from TrendingTask. +/// +/// \author Piotr Konopka +class TrendingMachine +{ + public: + explicit TrendingMachine(const TrendingMachineConfig& config); + ~TrendingMachine() = default; + + // initializes the trend, optionally with an existing tree if user needs to resume an old trend + /// \brief Prepares the trend (TTree) to be filled. + /// \param treeName the name of the new tree, if one has to be created + /// \param existingTree optionally, a tree to continue (e.g. retrieved from a database). The tree is used only if + /// its structure matches the data sources, otherwise it is destroyed. + /// \return true if the existing tree was taken over and the trend continues, false if a new trend was started + bool initializeTrend(const std::string& treeName, std::unique_ptr existingTree = nullptr); + /// \brief returns a pointer to the trend tree, the ownership is not passed to the caller + // unfortunately non-const, because of ObjectManager and MonitorObject + TTree* getTrend() { return mTrend.get(); } + + /// \brief sets the mandatary metadata for the next trend entry + void setMetadata(UInt_t timestamp, int runNumber); + + /// \brief updates the reductors of all data sources, with the data provided by the caller. + /// \param updater a function which gets a data source and its reductor, and should update the latter, e.g. by + /// retrieving an object from a database. It should return true on success. Failures are logged. + /// \return true if all data sources were updated successfully + bool updateReductors(const std::function& updater); + /// \brief returns a const pointer to the reductor, the ownership is not passed to the caller + const Reductor* getReductor(const std::string& dataSourceName) const; + + /// \brief fills the trend with a new row containing the current state of reductors + void fillTrend(); + + /// \brief generates pre-configured plots out of the latest trend + std::vector> generatePlots(); + + /// \brief resets the trend, clears the plots, but does not delete the trend tree (trending can resume) + void reset(); + + private: + static bool canContinueTrend(TTree* tree, const std::vector& dataSources); + static TCanvas* drawPlot(TTree* trend, const TrendingMachineConfig::Plot& plotConfig); + static void applyStyleToGraph(TGraph* graph, const TrendingMachineConfig::GraphStyle& style); + static std::string deduceGraphLegendOptions(const TrendingMachineConfig::Graph& graphConfig); + static void setUserAxesLabels(TAxis* xAxis, TAxis* yAxis, const std::string& graphAxesLabels); + static void setUserYAxisRange(TH1* hist, const std::string& graphYAxisRange); + static void formatTimeXAxis(TH1* background); + static void formatRunNumberXAxis(TH1* background); + + private: + static constexpr size_t MaxRunNumberStringLength = 6; + struct { + // we store run numbers both as an integer and as a string to allow users to select whether they need + // a trend in integer or label domain (the latter will contain evenly-spaced data points) + Long64_t runNumber = 0; + char runNumberStr[MaxRunNumberStringLength + 1] = { 0 }; + static const char* getBranchLeafList() + { + return "runNumber/L:runNumberStr/C"; + } + } mMetaData; + + std::unique_ptr mTrend; + UInt_t mTime; + std::unordered_map> mReductors; + std::vector mPlotConfigs; + std::vector mDataSources; +}; + +} // namespace o2::quality_control::postprocessing + +#endif // QUALITYCONTROL_TRENDINGMACHINE_H diff --git a/Framework/include/QualityControl/TrendingMachineConfig.h b/Framework/include/QualityControl/TrendingMachineConfig.h new file mode 100644 index 000000000..cdc32072d --- /dev/null +++ b/Framework/include/QualityControl/TrendingMachineConfig.h @@ -0,0 +1,106 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file TrendingMachineConfig.h +/// \author Piotr Konopka +/// + +#ifndef QUALITYCONTROL_TRENDINGMACHINECONFIG_H +#define QUALITYCONTROL_TRENDINGMACHINECONFIG_H + +#include +#include +#include +#include "CustomParameters.h" + +namespace o2::quality_control::postprocessing +{ + +/// \brief Configuration of the TrendingMachine: what to trend (data sources) and what to plot. +/// It does not contain anything related to when and how the machine is triggered or fed - +/// that is up to the callers (e.g. TrendingTaskConfig). +struct TrendingMachineConfig { + TrendingMachineConfig() = default; + /// \param config a node which contains the "plots" and "dataSources" children + explicit TrendingMachineConfig(const boost::property_tree::ptree& config); + ~TrendingMachineConfig() = default; + + // graph style configuration + // colors as defined by ROOT TColor class: + // https://root.cern/doc/master/classTColor.html + // marker colors and styles are as defined by ROOT TAttMarker class + // https://root.cern/doc/master/classTAttMarker.html + // line styles are as defined by ROOT TAttLine class + // https://root.cern/doc/master/classTAttLine.html + // WARNING: Any parameters in this struct will override colliding parameters in option + struct GraphStyle { + int lineColor = -1; + int lineStyle = -1; + int lineWidth = -1; + int markerColor = -1; + int markerStyle = -1; + float markerSize = -1.f; + int fillColor = -1; + int fillStyle = -1; + }; + + // this corresponds to one TTree::Draw() call, i.e. one graph or histogram drawing + struct Graph { + std::string name; + std::string title; + std::string varexp; + std::string selection; + std::string option; // the list of possible options are documented in TGraphPainter and THistPainter + std::string errors; + GraphStyle style; + }; + + // legend configuration + struct LegendConfig { + int nColumns{ 1 }; + float x1{ -1.f }, y1{ -1.f }, x2{ -1.f }, y2{ -1.f }; // NDC coords + }; + + // this corresponds to one canvas which can include multiple graphs + struct Plot { + std::string name; + std::string title; + std::string graphAxisLabel; + std::string graphYRange; + int colorPalette = 0; + LegendConfig legend; + std::vector graphs; + }; + + struct DataSource { + std::string type; + std::string path; + std::string name; + std::string reductorName; + core::CustomParameters reductorParameters; + std::string moduleName; + }; + + std::vector plots; + std::vector dataSources; + + // Static parsing methods, usable also when plots and data sources are not siblings in one tree + static std::vector parsePlots(const boost::property_tree::ptree& plotsTree); + static std::vector parseDataSources(const boost::property_tree::ptree& dataSourcesTree); + static Graph parseGraph(const boost::property_tree::ptree& graphTree, const std::string& fallbackName = ""); + static GraphStyle parseGraphStyle(const boost::property_tree::ptree& styleTree); + static LegendConfig parseLegend(const boost::property_tree::ptree& legendTree); +}; + +} // namespace o2::quality_control::postprocessing + +#endif // QUALITYCONTROL_TRENDINGMACHINECONFIG_H diff --git a/Framework/include/QualityControl/TrendingTask.h b/Framework/include/QualityControl/TrendingTask.h index db1dcb485..94f6bf333 100644 --- a/Framework/include/QualityControl/TrendingTask.h +++ b/Framework/include/QualityControl/TrendingTask.h @@ -18,15 +18,12 @@ #define QUALITYCONTROL_TRENDINGTASK_H #include "QualityControl/PostProcessingInterface.h" -#include "QualityControl/Reductor.h" #include "QualityControl/TrendingTaskConfig.h" #include -#include -#include +#include -class TAxis; -class TCanvas; +class TObject; namespace o2::quality_control::repository { @@ -36,6 +33,8 @@ class DatabaseInterface; namespace o2::quality_control::postprocessing { +class TrendingMachine; + /// \brief A post-processing task which trends values, stores them in a TTree and produces plots. /// /// A post-processing task which trends objects inside QC database (QCDB). It extracts some values of one or multiple @@ -47,8 +46,8 @@ namespace o2::quality_control::postprocessing class TrendingTask : public PostProcessingInterface { public: - TrendingTask() = default; - ~TrendingTask() override = default; + TrendingTask(); + ~TrendingTask() override; void configure(const boost::property_tree::ptree& config) override; void initialize(Trigger, framework::ServiceRegistryRef) override; @@ -56,37 +55,14 @@ class TrendingTask : public PostProcessingInterface void finalize(Trigger, framework::ServiceRegistryRef) override; private: - static constexpr size_t MaxRunNumberStringLength = 6; - struct { - // we store run numbers both as an integer and as a string to allow users to select whether they need - // a trend in integer or label domain (the latter will contain evenly-spaced data points) - Long64_t runNumber = 0; - char runNumberStr[MaxRunNumberStringLength + 1] = { 0 }; // 6 characters + null terminator - static const char* getBranchLeafList() - { - return "runNumber/L:runNumberStr/C"; - } - } mMetaData; - - static void setUserAxesLabels(TAxis* xAxis, TAxis* yAxis, const std::string& graphAxesLabels); - static void setUserYAxisRange(TH1* hist, const std::string& graphYAxisRange); - static void formatTimeXAxis(TH1* background); - static void formatRunNumberXAxis(TH1* background); - static std::string deduceGraphLegendOptions(const TrendingTaskConfig::Graph& graphConfig); - static void applyStyleToGraph(TGraph* graph, const TrendingTaskConfig::GraphStyle& style); - /// returns true only if all datasources were available to update reductor bool trendValues(const Trigger& t, repository::DatabaseInterface&); void generatePlots(); - TCanvas* drawPlot(const TrendingTaskConfig::Plot& plotConfig); void initializeTrend(repository::DatabaseInterface& qcdb); - bool canContinueTrend(TTree* tree); TrendingTaskConfig mConfig; - UInt_t mTime; - std::unique_ptr mTrend; + std::unique_ptr mTrendingMachine; std::map> mPlots; - std::unordered_map> mReductors; }; } // namespace o2::quality_control::postprocessing diff --git a/Framework/include/QualityControl/TrendingTaskConfig.h b/Framework/include/QualityControl/TrendingTaskConfig.h index 2d5b33820..033ee3ca9 100644 --- a/Framework/include/QualityControl/TrendingTaskConfig.h +++ b/Framework/include/QualityControl/TrendingTaskConfig.h @@ -17,10 +17,9 @@ #ifndef QUALITYCONTROL_TRENDINGTASKCONFIG_H #define QUALITYCONTROL_TRENDINGTASKCONFIG_H -#include #include -#include "CustomParameters.h" #include "QualityControl/PostProcessingConfig.h" +#include "QualityControl/TrendingMachineConfig.h" namespace o2::quality_control::postprocessing { @@ -32,68 +31,11 @@ struct TrendingTaskConfig : PostProcessingConfig { TrendingTaskConfig(std::string name, const boost::property_tree::ptree& config); ~TrendingTaskConfig() = default; - // graph style configuration - // colors as defined by ROOT TColor class: - // https://root.cern/doc/master/classTColor.html - // marker colors and styles are as defined by ROOT TAttMarker class - // https://root.cern/doc/master/classTAttMarker.html - // line styles are as defined by ROOT TAttLine class - // https://root.cern/doc/master/classTAttLine.html - // WARNING: Any parameters in this struct will override colliding parameters in option - struct GraphStyle { - int lineColor = -1; - int lineStyle = -1; - int lineWidth = -1; - int markerColor = -1; - int markerStyle = -1; - float markerSize = -1.f; - int fillColor = -1; - int fillStyle = -1; - }; - - // this corresponds to one TTree::Draw() call, i.e. one graph or histogram drawing - struct Graph { - std::string name; - std::string title; - std::string varexp; - std::string selection; - std::string option; // the list of possible options are documented in TGraphPainter and THistPainter - std::string errors; - GraphStyle style; - }; - - // legend configuration - struct LegendConfig { - int nColumns{ 1 }; - float x1{ -1.f }, y1{ -1.f }, x2{ -1.f }, y2{ -1.f }; // NDC coords - }; - - // this corresponds to one canvas which can include multiple graphs - struct Plot { - std::string name; - std::string title; - std::string graphAxisLabel; - std::string graphYRange; - int colorPalette = 0; - LegendConfig legend; - std::vector graphs; - }; - - struct DataSource { - std::string type; - std::string path; - std::string name; - std::string reductorName; - core::CustomParameters reductorParameters; - std::string moduleName; - }; - bool producePlotsOnUpdate{}; bool resumeTrend{}; bool trendIfAllInputs{ false }; std::string trendingTimestamp; - std::vector plots; - std::vector dataSources; + TrendingMachineConfig machine; }; } // namespace o2::quality_control::postprocessing diff --git a/Framework/src/TrendingMachine.cxx b/Framework/src/TrendingMachine.cxx new file mode 100644 index 000000000..9ede66819 --- /dev/null +++ b/Framework/src/TrendingMachine.cxx @@ -0,0 +1,472 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file TrendingMachine.cxx +/// \author Piotr Konopka +/// +/// \brief Implementation of trend generation functionality +/// +/// This class contains code extracted from TrendingTask to eliminate duplication between +/// TrendingTask and LateTrendingTask. +/// +/// Code history can be accessed with: +/// git log --follow Framework/src/TrendingMachine.cxx +/// git blame -C -C -C Framework/src/TrendingMachine.cxx +/// + +#include "QualityControl/TrendingMachine.h" +#include "QualityControl/QcInfoLogger.h" +#include "QualityControl/Reductor.h" +#include "QualityControl/RootClassFactory.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace o2::quality_control; +using namespace o2::quality_control::postprocessing; + +TrendingMachine::TrendingMachine(const TrendingMachineConfig& config) + : mPlotConfigs(config.plots), mDataSources(config.dataSources) +{ + // Create reductors + for (const auto& source : mDataSources) { + auto&& [emplaced, _] = mReductors.emplace(source.name, core::root_class_factory::create(source.moduleName, source.reductorName)); + if (!emplaced->second) { + throw std::runtime_error("Failed to create reductor " + source.reductorName + " from module " + source.moduleName); + } + emplaced->second->setCustomConfig(source.reductorParameters); + } +} + +bool TrendingMachine::initializeTrend(const std::string& treeName, std::unique_ptr existingTree) +{ + mTrend.reset(); + + // If we have an existing tree and can continue it, resume from it + if (existingTree && canContinueTrend(existingTree.get(), mDataSources)) { + ILOG(Info, Support) << "Will continue the trend from the provided tree." << ENDM; + mTrend = std::move(existingTree); + mTrend->SetBranchAddress("meta", &mMetaData); + mTrend->SetBranchAddress("time", &mTime); + for (const auto& [sourceName, reductor] : mReductors) { + mTrend->SetBranchAddress(sourceName.c_str(), reductor->getBranchAddress()); + } + return true; + } + + // did not manage to reuse existingTree, creating a new one + mTrend = std::make_unique(); + mTrend->SetName(treeName.c_str()); + + mTrend->Branch("meta", &mMetaData, mMetaData.getBranchLeafList()); + mTrend->Branch("time", &mTime); + for (const auto& [sourceName, reductor] : mReductors) { + mTrend->Branch(sourceName.c_str(), reductor->getBranchAddress(), reductor->getBranchLeafList()); + } + return false; +} + +void TrendingMachine::setMetadata(UInt_t timestamp, int runNumber) +{ + mTime = timestamp; + mMetaData.runNumber = runNumber; + std::snprintf(mMetaData.runNumberStr, MaxRunNumberStringLength + 1, "%d", runNumber); +} + +const Reductor* TrendingMachine::getReductor(const std::string& dataSourceName) const +{ + auto it = mReductors.find(dataSourceName); + return it != mReductors.end() ? it->second.get() : nullptr; +} + +bool TrendingMachine::updateReductors(const std::function& updater) +{ + bool wereAllSourcesUpdated = true; + for (const auto& dataSource : mDataSources) { + auto it = mReductors.find(dataSource.name); + if (it == mReductors.end()) { + wereAllSourcesUpdated = false; + ILOG(Error, Support) << "Reductor not found for data source '" << dataSource.name << "'." << ENDM; + continue; + } + auto reductor = it->second.get(); + if (!updater(dataSource, *reductor)) { + wereAllSourcesUpdated = false; + ILOG(Error, Support) << "Failed to update reductor for data sources with path '" << dataSource.path + << "', name '" << dataSource.name + << "', type '" << dataSource.type << "'." << ENDM; + } + } + return wereAllSourcesUpdated; +} + +void TrendingMachine::fillTrend() +{ + if (mTrend) { + mTrend->Fill(); + } +} + +std::vector> TrendingMachine::generatePlots() +{ + std::vector> plots; + + if (mTrend == nullptr) { + ILOG(Info, Support) << "The trend object is not there, won't generate any plots." << ENDM; + return plots; + } + + if (mTrend->GetEntries() < 1) { + ILOG(Info, Support) << "No entries in the trend so far, won't generate any plots." << ENDM; + return plots; + } + + ILOG(Info, Support) << "Generating " << mPlotConfigs.size() << " plots." << ENDM; + for (const auto& plotConfig : mPlotConfigs) { + plots.emplace_back(drawPlot(mTrend.get(), plotConfig)); + } + + return plots; +} + +void TrendingMachine::reset() +{ + if (mTrend) { + mTrend->Reset(); + } +} + +TCanvas* TrendingMachine::drawPlot(TTree* trend, const TrendingMachineConfig::Plot& plotConfig) +{ + auto* c = new TCanvas(); + + // Legend + TLegend* legend = nullptr; + if (plotConfig.legend.x1 >= 0 && plotConfig.legend.y1 >= 0 && plotConfig.legend.x2 >= 0 && plotConfig.legend.y2 >= 0) { + legend = new TLegend(plotConfig.legend.x1, plotConfig.legend.y1, + plotConfig.legend.x2, plotConfig.legend.y2, + nullptr, "NDC"); + if (plotConfig.legend.nColumns > 0) { + legend->SetNColumns(plotConfig.legend.nColumns); + } + } else { + legend = new TLegend(0.3, 0.2); + } + legend->SetBorderSize(0); + legend->SetFillStyle(0); + legend->SetTextSize(0.03); + legend->SetMargin(0.15); + + // Keep palette behavior unless user forces explicit colors via per-graph style + if (plotConfig.colorPalette != 0) { + gStyle->SetPalette(plotConfig.colorPalette); + // This makes ROOT store the selected palette for each generated plot. + // TColor::DefinedColors(1); // TODO enable when available + } else { + gStyle->SetPalette(); // default + } + + // regardless whether we draw a graph or a histogram, a histogram is always used by TTree::Draw to draw axes and title + // we attempt to keep it to do some modifications later + TH1* background = nullptr; + bool firstGraphInPlot = true; + // by "graph" we consider anything we can draw, not necessarily TGraph, and we draw all on the same canvas + for (const auto& graphConfig : plotConfig.graphs) { + // we determine the order of the plotConfig, i.e. if it is a histogram (1), graphConfig (2), or any higher dimension. + const size_t plotOrder = std::count(graphConfig.varexp.begin(), graphConfig.varexp.end(), ':') + 1; + + // having "SAME" at the first TTree::Draw() call will not work, we have to add it only in subsequent Draw calls + std::string option = firstGraphInPlot ? graphConfig.option : "SAME " + graphConfig.option; + + // Draw main series + trend->Draw(graphConfig.varexp.c_str(), graphConfig.selection.c_str(), option.c_str()); + + // For graphs, we allow to draw errors if they are specified. + TGraphErrors* graphErrors = nullptr; + if (!graphConfig.errors.empty()) { + if (plotOrder != 2) { + ILOG(Error, Support) << "Non empty graphErrors seen for the plotConfig '" << plotConfig.name << "', which is not a graphConfig, ignoring." << ENDM; + } else { + // We generate some 4-D points, where 2 dimensions represent graph points and 2 others are the error bars + std::string varexpWithErrors(graphConfig.varexp + ":" + graphConfig.errors); + trend->Draw(varexpWithErrors.c_str(), graphConfig.selection.c_str(), "goff"); + graphErrors = new TGraphErrors(trend->GetSelectedRows(), trend->GetVal(1), trend->GetVal(0), + trend->GetVal(2), trend->GetVal(3)); + graphErrors->SetName((graphConfig.name + "_errors").c_str()); + graphErrors->SetTitle((graphConfig.title + " errors").c_str()); + // We draw on the same plotConfig as the main graphConfig, but only error bars + graphErrors->Draw("SAME E"); + } + } + + // Legend entry and styling for graphs + if (auto graph = dynamic_cast(c->FindObject("Graph"))) { + if (plotOrder >= 2) { + // Style objects after Draw so we override palette/auto styling when requested + applyStyleToGraph(graph, graphConfig.style); + // Keep errors visually consistent with the main series + if (graphErrors) { + applyStyleToGraph(graphErrors, graphConfig.style); + } + } + graph->SetName(graphConfig.name.c_str()); + graph->SetTitle(graphConfig.title.c_str()); + legend->AddEntry(graph, graphConfig.title.c_str(), + deduceGraphLegendOptions(graphConfig).c_str()); + } + + // Legend entry and styling for histograms + if (auto htemp = dynamic_cast(c->FindObject("htemp"))) { + if (plotOrder == 1) { + htemp->SetName(graphConfig.name.c_str()); + htemp->SetTitle(graphConfig.title.c_str()); + legend->AddEntry(htemp, graphConfig.title.c_str(), "lpf"); + } else { + htemp->SetName("background"); + htemp->SetTitle("background"); + // htemp was used by TTree::Draw only to draw axes and title, not to plot data, no need to add it to legend + } + // QCG doesn't empty the buffers before visualizing the plotConfig, nor does ROOT when saving the file, + // so we have to do it here. + htemp->BufferEmpty(); + // we keep the pointer to bg histogram for later postprocessing + if (!background) { + background = htemp; + } + } + + firstGraphInPlot = false; + } + + c->SetName(plotConfig.name.c_str()); + c->SetTitle(plotConfig.title.c_str()); + + // Postprocessing the plotConfig - adding specified titles, configuring time-based plots, flushing buffers. + // Notice that axes and title are drawn using a histogram, even in the case of graphs. + if (background) { + // The title of background histogram is printed, not the title of canvas => we set it as well. + background->SetTitle(plotConfig.title.c_str()); + // We have to update the canvas to make the title appear and access it in the next step. + c->Update(); + + // After the update, the title has a different size and it is not in the center anymore. We have to fix that. + if (auto title = dynamic_cast(c->GetPrimitive("title"))) { + title->SetBBoxCenterX(c->GetBBoxCenter().fX); + c->Modified(); + c->Update(); + } else { + ILOG(Error, Devel) << "Could not get the title TPaveText of the plotConfig '" << plotConfig.name << "'." << ENDM; + } + + if (!plotConfig.graphAxisLabel.empty()) { + setUserAxesLabels(background->GetXaxis(), background->GetYaxis(), plotConfig.graphAxisLabel); + } + + if (plotConfig.graphs.back().varexp.find(":time") != std::string::npos) { + // We have to explicitly configure showing time on x axis. + formatTimeXAxis(background); + } else if (plotConfig.graphs.back().varexp.find(":meta.runNumber") != std::string::npos) { + formatRunNumberXAxis(background); + } + + // Set the user-defined range on the y axis if needed. + if (!plotConfig.graphYRange.empty()) { + setUserYAxisRange(background, plotConfig.graphYRange); + c->Modified(); + c->Update(); + } + } else { + ILOG(Error, Devel) << "Could not get the htemp histogram of the plotConfig '" << plotConfig.name << "'." << ENDM; + } + + if (plotConfig.graphs.size() > 1) { + legend->Draw(); + } else { + delete legend; + } + + c->Modified(); + c->Update(); + + return c; +} + +void TrendingMachine::applyStyleToGraph(TGraph* graph, const TrendingMachineConfig::GraphStyle& style) +{ + if (!graph) { + return; + } + + if (style.lineColor >= 0) { + graph->SetLineColor(style.lineColor); + } + if (style.lineStyle >= 0) { + graph->SetLineStyle(style.lineStyle); + } + if (style.lineWidth >= 0) { + graph->SetLineWidth(style.lineWidth); + } + + if (style.markerColor >= 0) { + graph->SetMarkerColor(style.markerColor); + } + if (style.markerStyle >= 0) { + graph->SetMarkerStyle(style.markerStyle); + } + if (style.markerSize >= 0.f) { + graph->SetMarkerSize(style.markerSize); + } + + if (style.fillColor >= 0) { + graph->SetFillColor(style.fillColor); + } + if (style.fillStyle >= 0) { + graph->SetFillStyle(style.fillStyle); + } +} + +std::string TrendingMachine::deduceGraphLegendOptions(const TrendingMachineConfig::Graph& graphConfig) +{ + // Looking at TGraphPainter documentation, the number of TGraph options is rather small, + // so we can try to be smart and deduce the corresponding legend options. + // I am not aware of any ROOT helper which can do this. + std::string options = graphConfig.option; + boost::algorithm::to_lower(options); + // these three options have only an influence on colours but not what is drawn and what not. + for (const auto& toRemove : { "pfc", "plc", "pmc" }) { + if (size_t pos = options.find(toRemove); pos != std::string::npos) { + options.erase(pos, 3); + } + } + auto optionsHave = [&](std::string_view seq) { + return options.find(seq) != std::string::npos; + }; + + std::string out; + if (optionsHave("l") || optionsHave("c")) { + out += "l"; // line + } + if (optionsHave("*") || optionsHave("p")) { + out += "p"; // point + } + if (optionsHave("f") || optionsHave("b")) { + out += "f"; // fill + } + if (!graphConfig.errors.empty()) { + out += "e"; // error bars + } + return out; +} + +void TrendingMachine::setUserAxesLabels(TAxis* xAxis, TAxis* yAxis, const std::string& graphAxesLabels) +{ + if (std::count(graphAxesLabels.begin(), graphAxesLabels.end(), ':') != 1 && graphAxesLabels != "") { + ILOG(Error, Support) << "In setup of graphAxesLabels yLabel:xLabel should be divided by one ':'" << ENDM; + return; + } + const std::size_t posDivider = graphAxesLabels.find(':'); + const std::string yLabel(graphAxesLabels.substr(0, posDivider)); + const std::string xLabel(graphAxesLabels.substr(posDivider + 1)); + + xAxis->SetTitle(xLabel.data()); + yAxis->SetTitle(yLabel.data()); +} + +void TrendingMachine::setUserYAxisRange(TH1* hist, const std::string& graphYAxisRange) +{ + if (std::count(graphYAxisRange.begin(), graphYAxisRange.end(), ':') != 1 && graphYAxisRange != "") { + ILOG(Error, Support) << "In setup of graphYRange yMin:yMax should be divided by one ':'" << ENDM; + return; + } + const std::size_t posDivider = graphYAxisRange.find(':'); + const std::string minString(graphYAxisRange.substr(0, posDivider)); + const std::string maxString(graphYAxisRange.substr(posDivider + 1)); + + float yMin = 0.f; + float yMax = 0.f; + try { + yMin = std::stof(minString); + yMax = std::stof(maxString); + } catch (const std::logic_error& e) { // invalid_argument or out_of_range + ILOG(Error, Support) << "Could not parse graphYRange '" << graphYAxisRange << "' as yMin:yMax numbers (" << e.what() << "), ignoring it." << ENDM; + return; + } + if (!(yMin < yMax)) { + ILOG(Error, Support) << "In setup of graphYRange yMin should be smaller than yMax, got '" << graphYAxisRange << "', ignoring it." << ENDM; + return; + } + hist->GetYaxis()->SetLimits(yMin, yMax); +} + +void TrendingMachine::formatTimeXAxis(TH1* background) +{ + background->GetXaxis()->SetTimeDisplay(1); + // It deals with highly congested dates labels + background->GetXaxis()->SetNdivisions(505); + // Without this it would show dates in order of 2044-12-18 on the day of 2019-12-19. + background->GetXaxis()->SetTimeOffset(0.0); + background->GetXaxis()->SetTimeFormat("%Y-%m-%d %H:%M"); +} + +void TrendingMachine::formatRunNumberXAxis(TH1* background) +{ + background->GetXaxis()->SetNoExponent(true); +} + +bool TrendingMachine::canContinueTrend(TTree* tree, const std::vector& dataSources) +{ + if (tree == nullptr) { + return false; + } + + size_t expectedNBranches = 1 /* meta */ + 1 /* time */ + dataSources.size(); + if (tree->GetNbranches() != expectedNBranches) { + ILOG(Warning, Support) << "The retrieved TTree has different number of branches than expected (" + << tree->GetNbranches() << " vs. " << expectedNBranches << "). " + << "Filling the tree with mismatching branches might produce invalid plots, " + << "thus a new tree will be created" << ENDM; + return false; + } + + std::set expectedBranchNames{ "time", "meta" }; + for (const auto& dataSource : dataSources) { + expectedBranchNames.insert(dataSource.name); + } + + std::set existingBranchNames; + for (const auto& branch : *tree->GetListOfBranches()) { + existingBranchNames.insert(branch->GetName()); + } + + if (expectedBranchNames != existingBranchNames) { + ILOG(Warning, Support) << "The retrieved TTree has the same number of branches," + << " but at least one has a different name." + << " Filling the tree with mismatching branches might produce invalid plots, " + << "thus a new tree will be created" << ENDM; + return false; + } + + return true; +} diff --git a/Framework/src/TrendingMachineConfig.cxx b/Framework/src/TrendingMachineConfig.cxx new file mode 100644 index 000000000..4d02b9b32 --- /dev/null +++ b/Framework/src/TrendingMachineConfig.cxx @@ -0,0 +1,154 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file TrendingMachineConfig.cxx +/// \author Piotr Konopka +/// + +#include "QualityControl/TrendingMachineConfig.h" +#include + +namespace o2::quality_control::postprocessing +{ + +TrendingMachineConfig::TrendingMachineConfig(const boost::property_tree::ptree& config) + : plots(parsePlots(config.get_child("plots"))), + dataSources(parseDataSources(config.get_child("dataSources"))) +{ +} + +TrendingMachineConfig::GraphStyle TrendingMachineConfig::parseGraphStyle(const boost::property_tree::ptree& styleTree) +{ + GraphStyle style; + style.lineColor = styleTree.get("lineColor", -1); + style.lineStyle = styleTree.get("lineStyle", -1); + style.lineWidth = styleTree.get("lineWidth", -1); + style.markerColor = styleTree.get("markerColor", -1); + style.markerStyle = styleTree.get("markerStyle", -1); + style.markerSize = styleTree.get("markerSize", -1.f); + style.fillColor = styleTree.get("fillColor", -1); + style.fillStyle = styleTree.get("fillStyle", -1); + return style; +} + +TrendingMachineConfig::LegendConfig TrendingMachineConfig::parseLegend(const boost::property_tree::ptree& legendTree) +{ + LegendConfig leg; + leg.nColumns = legendTree.get("nColumns", 1); + leg.x1 = legendTree.get("x1", -1.f); + leg.y1 = legendTree.get("y1", -1.f); + leg.x2 = legendTree.get("x2", -1.f); + leg.y2 = legendTree.get("y2", -1.f); + return leg; +} + +TrendingMachineConfig::Graph TrendingMachineConfig::parseGraph(const boost::property_tree::ptree& graphTree, const std::string& fallbackName) +{ + Graph graph; + graph.name = graphTree.get("name", graphTree.get("title", fallbackName)); + graph.title = graphTree.get("title", ""); + graph.varexp = graphTree.get("varexp"); + graph.selection = graphTree.get("selection", ""); + graph.option = graphTree.get("option", ""); + graph.errors = graphTree.get("graphErrors", ""); + + // Parse style if present + if (auto styleTree = graphTree.get_child_optional("style"); styleTree.has_value()) { + graph.style = parseGraphStyle(styleTree.value()); + } + + return graph; +} + +std::vector TrendingMachineConfig::parsePlots(const boost::property_tree::ptree& plotsTree) +{ + std::vector plots; + + for (const auto& [_, plotConfig] : plotsTree) { + std::vector graphs; + + // Support both new format (with graphs array) and old format (direct plot config) + if (const auto& graphsConfig = plotConfig.get_child_optional("graphs"); graphsConfig.has_value()) { + for (const auto& [_, graphConfig] : graphsConfig.value()) { + graphs.push_back(parseGraph(graphConfig, plotConfig.get("name"))); + } + } else { + // Old format: plot config is also graph config + graphs.push_back(parseGraph(plotConfig, plotConfig.get("name", ""))); + } + + Plot plot; + plot.name = plotConfig.get("name"); + plot.title = plotConfig.get("title", ""); + plot.graphAxisLabel = plotConfig.get("graphAxisLabel", ""); + plot.graphYRange = plotConfig.get("graphYRange", ""); + plot.colorPalette = plotConfig.get("colorPalette", 0); + + if (auto legendTree = plotConfig.get_child_optional("legend"); legendTree.has_value()) { + plot.legend = parseLegend(legendTree.value()); + } + + plot.graphs = graphs; + plots.push_back(plot); + } + + return plots; +} + +std::vector TrendingMachineConfig::parseDataSources(const boost::property_tree::ptree& dataSourcesTree) +{ + std::vector dataSources; + + const auto extractReductorParams = [](const boost::property_tree::ptree& dataSourceConfig) -> core::CustomParameters { + core::CustomParameters result; + if (const auto reductorParams = dataSourceConfig.get_child_optional("reductorParameters"); reductorParams.has_value()) { + result.populateCustomParameters(reductorParams.value()); + } + return result; + }; + + for (const auto& dataSourceEntry : dataSourcesTree) { + const auto& dataSourceConfig = dataSourceEntry.second; + + // Support both "names" array and single "name" + if (const auto& sourceNames = dataSourceConfig.get_child_optional("names"); sourceNames.has_value()) { + for (const auto& sourceName : sourceNames.value()) { + DataSource ds; + ds.type = dataSourceConfig.get("type", "repository"); + ds.path = dataSourceConfig.get("path", ""); + ds.name = sourceName.second.data(); + ds.reductorName = dataSourceConfig.get("reductorName"); + ds.reductorParameters = extractReductorParams(dataSourceConfig); + ds.moduleName = dataSourceConfig.get("moduleName"); + dataSources.push_back(ds); + } + } else { + std::string nameValue = dataSourceConfig.get("name", ""); + if (!nameValue.empty()) { + DataSource ds; + ds.type = dataSourceConfig.get("type", "repository"); + ds.path = dataSourceConfig.get("path", ""); + ds.name = nameValue; + ds.reductorName = dataSourceConfig.get("reductorName"); + ds.reductorParameters = extractReductorParams(dataSourceConfig); + ds.moduleName = dataSourceConfig.get("moduleName"); + dataSources.push_back(ds); + } else { + throw std::runtime_error("No 'name' value or a 'names' vector in dataSources configuration"); + } + } + } + + return dataSources; +} + +} // namespace o2::quality_control::postprocessing diff --git a/Framework/src/TrendingTask.cxx b/Framework/src/TrendingTask.cxx index d92e7baba..e6d1a6dca 100644 --- a/Framework/src/TrendingTask.cxx +++ b/Framework/src/TrendingTask.cxx @@ -15,6 +15,7 @@ /// #include "QualityControl/TrendingTask.h" +#include "QualityControl/TrendingMachine.h" #include "QualityControl/QcInfoLogger.h" #include "QualityControl/DatabaseInterface.h" #include "QualityControl/Reductor.h" @@ -30,6 +31,7 @@ #include #include #include +#include #include #include @@ -38,71 +40,25 @@ using namespace o2::quality_control; using namespace o2::quality_control::core; using namespace o2::quality_control::postprocessing; +TrendingTask::TrendingTask() = default; +TrendingTask::~TrendingTask() = default; + void TrendingTask::configure(const boost::property_tree::ptree& config) { // we clear any existing objects, which would be there only in case of reconfiguration // at the time of writing, this not even supported by ECS - mReductors.clear(); - mTrend.reset(); + mTrendingMachine.reset(); // configuration mConfig = TrendingTaskConfig(getID(), config); - for (const auto& source : mConfig.dataSources) { - auto&& [emplaced, _] = mReductors.emplace(source.name, root_class_factory::create(source.moduleName, source.reductorName)); - emplaced->second->setCustomConfig(source.reductorParameters); - } -} - -bool TrendingTask::canContinueTrend(TTree* tree) -{ - if (tree == nullptr) { - return false; - } - - size_t expectedNBranches = 1 /* meta */ + 1 /* time */ + mConfig.dataSources.size(); - if (tree->GetNbranches() != expectedNBranches) { - ILOG(Warning, Support) << "The retrieved TTree has different number of branches than expected (" - << tree->GetNbranches() << " vs. " << expectedNBranches << "). " - << "Filling the tree with mismatching branches might produce invalid plots, " - << "thus a new tree will be created" << ENDM; - return false; - } - - std::set expectedBranchNames{ "time", "meta" }; - for (const auto& dataSource : mConfig.dataSources) { - expectedBranchNames.insert(dataSource.name); - } - - std::set existingBranchNames; - for (const auto& branch : *tree->GetListOfBranches()) { - existingBranchNames.insert(branch->GetName()); - } - - if (expectedBranchNames != existingBranchNames) { - ILOG(Warning, Support) << "The retrieved TTree has the same number of branches," - << " but at least one has a different name." - << " Filling the tree with mismatching branches might produce invalid plots, " - << "thus a new tree will be created" << ENDM; - return false; - } - - return true; + mTrendingMachine = std::make_unique(mConfig.machine); } void TrendingTask::initializeTrend(o2::quality_control::repository::DatabaseInterface& qcdb) { - // tree exists and we can reuse it - if (canContinueTrend(mTrend.get())) { - if (mConfig.resumeTrend == false) { - mTrend->Reset(); - } else { - ILOG(Info, Support) << "Will continue the trend from the previous run." << ENDM; - } - return; - } + std::unique_ptr existingTree; - // tree is not reusable or does not exist => if we want to reuse the latest, we look for it in QCDB - mTrend.reset(); + // Try to retrieve existing tree from QCDB if resuming if (mConfig.resumeTrend) { ILOG(Info, Support) << "Trying to retrieve an existing TTree for this task to continue the trend." << ENDM; auto path = RepoPathUtils::getMoPath(mConfig.detectorName, PostProcessingInterface::getName(), "", "", false); @@ -110,70 +66,15 @@ void TrendingTask::initializeTrend(o2::quality_control::repository::DatabaseInte if (mo && mo->getObject()) { auto tree = dynamic_cast(mo->getObject()); if (tree) { - mTrend = std::unique_ptr(tree); - mo->setIsOwner(false); + mo->setIsOwner(false); // TrendingMachine will take ownership + existingTree.reset(tree); } } else { ILOG(Warning, Support) << "Could not retrieve an existing TTree for this task" << ENDM; } - if (canContinueTrend(mTrend.get())) { - mTrend->SetBranchAddress("meta", &mMetaData); - mTrend->SetBranchAddress("time", &mTime); - for (const auto& [sourceName, reductor] : mReductors) { - mTrend->SetBranchAddress(sourceName.c_str(), reductor->getBranchAddress()); - } - ILOG(Info, Support) << "Will use the latest TTree from QCDB for this task to continue the trend." << ENDM; - return; - } else { - mTrend.reset(); - } } - // we could not reuse the tree or never had one => we create a new one - if (mTrend == nullptr) { - mTrend = std::make_unique(); - mTrend->SetName(PostProcessingInterface::getName().c_str()); - - mTrend->Branch("meta", &mMetaData, mMetaData.getBranchLeafList()); - mTrend->Branch("time", &mTime); - for (const auto& [sourceName, reductor] : mReductors) { - mTrend->Branch(sourceName.c_str(), reductor->getBranchAddress(), reductor->getBranchLeafList()); - } - } -} - -void TrendingTask::applyStyleToGraph(TGraph* graph, const TrendingTaskConfig::GraphStyle& style) -{ - if (!graph) { - return; - } - - if (style.lineColor >= 0) { - graph->SetLineColor(style.lineColor); - } - if (style.lineStyle >= 0) { - graph->SetLineStyle(style.lineStyle); - } - if (style.lineWidth >= 0) { - graph->SetLineWidth(style.lineWidth); - } - - if (style.markerColor >= 0) { - graph->SetMarkerColor(style.markerColor); - } - if (style.markerStyle >= 0) { - graph->SetMarkerStyle(style.markerStyle); - } - if (style.markerSize >= 0.f) { - graph->SetMarkerSize(style.markerSize); - } - - if (style.fillColor >= 0) { - graph->SetFillColor(style.fillColor); - } - if (style.fillStyle >= 0) { - graph->SetFillStyle(style.fillStyle); - } + mTrendingMachine->initializeTrend(PostProcessingInterface::getName(), std::move(existingTree)); } void TrendingTask::initialize(Trigger, framework::ServiceRegistryRef services) @@ -184,7 +85,7 @@ void TrendingTask::initialize(Trigger, framework::ServiceRegistryRef services) initializeTrend(services.get()); if (mConfig.producePlotsOnUpdate) { - getObjectsManager()->startPublishing(mTrend.get(), PublicationPolicy::ThroughStop); + getObjectsManager()->startPublishing(mTrendingMachine->getTrend(), PublicationPolicy::ThroughStop); } } @@ -202,296 +103,47 @@ void TrendingTask::update(Trigger t, framework::ServiceRegistryRef services) void TrendingTask::finalize(Trigger, framework::ServiceRegistryRef) { if (!mConfig.producePlotsOnUpdate) { - getObjectsManager()->startPublishing(mTrend.get()); + getObjectsManager()->startPublishing(mTrendingMachine->getTrend()); } generatePlots(); } bool TrendingTask::trendValues(const Trigger& t, repository::DatabaseInterface& qcdb) { + UInt_t timestamp; if (mConfig.trendingTimestamp == "trigger") { // ROOT expects seconds since epoch. - mTime = t.timestamp / 1000; + timestamp = t.timestamp / 1000; } else if (mConfig.trendingTimestamp == "validFrom") { - mTime = t.activity.mValidity.getMin() / 1000; + timestamp = t.activity.mValidity.getMin() / 1000; } else { // validUntil - mTime = t.activity.mValidity.getMax() / 1000; + timestamp = t.activity.mValidity.getMax() / 1000; } - mMetaData.runNumber = t.activity.mId; - std::snprintf(mMetaData.runNumberStr, MaxRunNumberStringLength + 1, "%d", t.activity.mId); - bool wereAllSourcesInvoked = true; - for (auto& dataSource : mConfig.dataSources) { - if (!reductor_helpers::updateReductor(mReductors[dataSource.name].get(), t, dataSource, qcdb, *this)) { - wereAllSourcesInvoked = false; - ILOG(Error, Support) << "Failed to update reductor for data sources with path '" << dataSource.path - << "', name '" << dataSource.name - << "', type '" << dataSource.type << "'." << ENDM; - } - } + mTrendingMachine->setMetadata(timestamp, t.activity.mId); + + const bool wereAllSourcesInvoked = mTrendingMachine->updateReductors([&](const auto& dataSource, Reductor& reductor) { + return reductor_helpers::updateReductor(&reductor, t, dataSource, qcdb, (core::ConditionAccess&)*this); + }); if (!mConfig.trendIfAllInputs || wereAllSourcesInvoked) { - mTrend->Fill(); + mTrendingMachine->fillTrend(); } return wereAllSourcesInvoked; } -void TrendingTask::setUserAxesLabels(TAxis* xAxis, TAxis* yAxis, const std::string& graphAxesLabels) -{ - // todo if we keep adding this method to pp classes we should move it up somewhere - if (std::count(graphAxesLabels.begin(), graphAxesLabels.end(), ':') != 1 && graphAxesLabels != "") { - ILOG(Error, Support) << "In setup of graphAxesLabels yLabel:xLabel should be divided by one ':'" << ENDM; - return; - } - const std::size_t posDivider = graphAxesLabels.find(':'); - const std::string yLabel(graphAxesLabels.substr(0, posDivider)); - const std::string xLabel(graphAxesLabels.substr(posDivider + 1)); - - xAxis->SetTitle(xLabel.data()); - yAxis->SetTitle(yLabel.data()); -} - -void TrendingTask::setUserYAxisRange(TH1* hist, const std::string& graphYAxisRange) -{ - if (std::count(graphYAxisRange.begin(), graphYAxisRange.end(), ':') != 1 && graphYAxisRange != "") { - ILOG(Error, Support) << "In setup of graphYRange yMin:yMax should be divided by one ':'" << ENDM; - return; - } - const std::size_t posDivider = graphYAxisRange.find(':'); - const std::string minString(graphYAxisRange.substr(0, posDivider)); - const std::string maxString(graphYAxisRange.substr(posDivider + 1)); - - const float yMin = std::stof(minString); - const float yMax = std::stof(maxString); - hist->GetYaxis()->SetLimits(yMin, yMax); -} - -void TrendingTask::formatRunNumberXAxis(TH1* background) -{ - background->GetXaxis()->SetNoExponent(true); -} - -void TrendingTask::formatTimeXAxis(TH1* background) -{ - background->GetXaxis()->SetTimeDisplay(1); - // It deals with highly congested dates labels - background->GetXaxis()->SetNdivisions(505); - // Without this it would show dates in order of 2044-12-18 on the day of 2019-12-19. - background->GetXaxis()->SetTimeOffset(0.0); - background->GetXaxis()->SetTimeFormat("%Y-%m-%d %H:%M"); -} - void TrendingTask::generatePlots() { - if (mTrend == nullptr) { - ILOG(Info, Support) << "The trend object is not there, won't generate any plots." << ENDM; - return; - } - - if (mTrend->GetEntries() < 1) { - ILOG(Info, Support) << "No entries in the trend so far, won't generate any plots." << ENDM; - return; - } - - ILOG(Info, Support) << "Generating " << mConfig.plots.size() << " plots." << ENDM; - for (const auto& plotConfig : mConfig.plots) { - - // Before we generate any new plots, we have to delete existing under the same names. - // It seems that ROOT cannot handle an existence of two canvases with a common name in the same process. - if (mPlots.count(plotConfig.name)) { - mPlots[plotConfig.name].reset(); - } - auto c = drawPlot(plotConfig); - mPlots[plotConfig.name].reset(c); - getObjectsManager()->startPublishing(c, PublicationPolicy::Once); - } -} - -std::string TrendingTask::deduceGraphLegendOptions(const TrendingTaskConfig::Graph& graphConfig) -{ - // Looking at TGraphPainter documentation, the number of TGraph options is rather small, - // so we can try to be smart and deduce the corresponding legend options. - // I am not aware of any ROOT helper which can do this. - std::string options = graphConfig.option; - boost::algorithm::to_lower(options); - // these three options have only an influence on colours but not what is drawn and what not. - for (const auto& toRemove : { "pfc", "plc", "pmc" }) { - if (size_t pos = options.find(toRemove); pos != std::string::npos) { - options.erase(pos, 3); - } - } - auto optionsHave = [&](std::string_view seq) { - return options.find(seq) != std::string::npos; - }; - - std::string out; - if (optionsHave("l") || optionsHave("c")) { - out += "l"; // line - } - if (optionsHave("*") || optionsHave("p")) { - out += "p"; // point - } - if (optionsHave("f") || optionsHave("b")) { - out += "f"; // fill - } - if (!graphConfig.errors.empty()) { - out += "e"; // error bars - } - return out; -} - -TCanvas* TrendingTask::drawPlot(const TrendingTaskConfig::Plot& plotConfig) -{ - auto* c = new TCanvas(); - - // Legend - TLegend* legend = nullptr; - if (plotConfig.legend.x1 >= 0 && plotConfig.legend.y1 >= 0 && plotConfig.legend.x2 >= 0 && plotConfig.legend.y2 >= 0) { - legend = new TLegend(plotConfig.legend.x1, plotConfig.legend.y1, - plotConfig.legend.x2, plotConfig.legend.y2, - nullptr, "NDC"); - if (plotConfig.legend.nColumns > 0) { - legend->SetNColumns(plotConfig.legend.nColumns); - } - } else { - legend = new TLegend(0.3, 0.2); - } - legend->SetBorderSize(0); - legend->SetFillStyle(0); - legend->SetTextSize(0.03); - legend->SetMargin(0.15); - - // Keep palette behavior unless user forces explicit colors via per-graph style - if (plotConfig.colorPalette != 0) { - gStyle->SetPalette(plotConfig.colorPalette); - // This makes ROOT store the selected palette for each generated plot. - // TColor::DefinedColors(1); // TODO enable when available - } else { - gStyle->SetPalette(); // default - } - - // regardless whether we draw a graph or a histogram, a histogram is always used by TTree::Draw to draw axes and title - // we attempt to keep it to do some modifications later - TH1* background = nullptr; - bool firstGraphInPlot = true; - // by "graph" we consider anything we can draw, not necessarily TGraph, and we draw all on the same canvas - for (const auto& graphConfig : plotConfig.graphs) { - // we determine the order of the plotConfig, i.e. if it is a histogram (1), graphConfig (2), or any higher dimension. - const size_t plotOrder = std::count(graphConfig.varexp.begin(), graphConfig.varexp.end(), ':') + 1; - - // having "SAME" at the first TTree::Draw() call will not work, we have to add it only in subsequent Draw calls - std::string option = firstGraphInPlot ? graphConfig.option : "SAME " + graphConfig.option; - - // Draw main series - mTrend->Draw(graphConfig.varexp.c_str(), graphConfig.selection.c_str(), option.c_str()); - - // For graphs, we allow to draw errors if they are specified. - TGraphErrors* graphErrors = nullptr; - if (!graphConfig.errors.empty()) { - if (plotOrder != 2) { - ILOG(Error, Support) << "Non empty graphErrors seen for the plotConfig '" << plotConfig.name << "', which is not a graphConfig, ignoring." << ENDM; - } else { - // We generate some 4-D points, where 2 dimensions represent graph points and 2 others are the error bars - std::string varexpWithErrors(graphConfig.varexp + ":" + graphConfig.errors); - mTrend->Draw(varexpWithErrors.c_str(), graphConfig.selection.c_str(), "goff"); - graphErrors = new TGraphErrors(mTrend->GetSelectedRows(), mTrend->GetVal(1), mTrend->GetVal(0), - mTrend->GetVal(2), mTrend->GetVal(3)); - graphErrors->SetName((graphConfig.name + "_errors").c_str()); - graphErrors->SetTitle((graphConfig.title + " errors").c_str()); - // We draw on the same plotConfig as the main graphConfig, but only error bars - graphErrors->Draw("SAME E"); - } - } - - // Legend entry and styling for graphs - if (auto graph = dynamic_cast(c->FindObject("Graph"))) { - if (plotOrder >= 2) { - // Style objects after Draw so we override palette/auto styling when requested - applyStyleToGraph(graph, graphConfig.style); - // Keep errors visually consistent with the main series - if (graphErrors) { - applyStyleToGraph(graphErrors, graphConfig.style); - } - } - graph->SetName(graphConfig.name.c_str()); - graph->SetTitle(graphConfig.title.c_str()); - legend->AddEntry(graph, graphConfig.title.c_str(), - deduceGraphLegendOptions(graphConfig).c_str()); - } - - // Legend entry and styling for histograms - if (auto htemp = dynamic_cast(c->FindObject("htemp"))) { - if (plotOrder == 1) { - htemp->SetName(graphConfig.name.c_str()); - htemp->SetTitle(graphConfig.title.c_str()); - legend->AddEntry(htemp, graphConfig.title.c_str(), "lpf"); - } else { - htemp->SetName("background"); - htemp->SetTitle("background"); - // htemp was used by TTree::Draw only to draw axes and title, not to plot data, no need to add it to legend - } - // QCG doesn't empty the buffers before visualizing the plotConfig, nor does ROOT when saving the file, - // so we have to do it here. - htemp->BufferEmpty(); - // we keep the pointer to bg histogram for later postprocessing - if (!background) { - background = htemp; - } - } - - firstGraphInPlot = false; - } - - c->SetName(plotConfig.name.c_str()); - c->SetTitle(plotConfig.title.c_str()); - - // Postprocessing the plotConfig - adding specified titles, configuring time-based plots, flushing buffers. - // Notice that axes and title are drawn using a histogram, even in the case of graphs. - if (background) { - // The title of background histogram is printed, not the title of canvas => we set it as well. - background->SetTitle(plotConfig.title.c_str()); - // We have to update the canvas to make the title appear and access it in the next step. - c->Update(); - - // After the update, the title has a different size and it is not in the center anymore. We have to fix that. - if (auto title = dynamic_cast(c->GetPrimitive("title"))) { - title->SetBBoxCenterX(c->GetBBoxCenter().fX); - c->Modified(); - c->Update(); - } else { - ILOG(Error, Devel) << "Could not get the title TPaveText of the plotConfig '" << plotConfig.name << "'." << ENDM; - } - - if (!plotConfig.graphAxisLabel.empty()) { - setUserAxesLabels(background->GetXaxis(), background->GetYaxis(), plotConfig.graphAxisLabel); - } - - if (plotConfig.graphs.back().varexp.find(":time") != std::string::npos) { - // We have to explicitly configure showing time on x axis. - formatTimeXAxis(background); - } else if (plotConfig.graphs.back().varexp.find(":meta.runNumber") != std::string::npos) { - formatRunNumberXAxis(background); - } - - // Set the user-defined range on the y axis if needed. - if (!plotConfig.graphYRange.empty()) { - setUserYAxisRange(background, plotConfig.graphYRange); - c->Modified(); - c->Update(); - } - } else { - ILOG(Error, Devel) << "Could not get the htemp histogram of the plotConfig '" << plotConfig.name << "'." << ENDM; - } + // Before we generate any new plots, we have to delete existing under the same names. + // It seems that ROOT cannot handle an existence of two canvases with a common name in the same process. + mPlots.clear(); - if (plotConfig.graphs.size() > 1) { - legend->Draw(); - } else { - delete legend; + auto plots = mTrendingMachine->generatePlots(); + for (auto& plot : plots) { + // Store plots to keep them alive + auto* published = plot.get(); + mPlots[published->GetName()] = std::move(plot); + getObjectsManager()->startPublishing(published, PublicationPolicy::Once); } - - c->Modified(); - c->Update(); - - return c; } diff --git a/Framework/src/TrendingTaskConfig.cxx b/Framework/src/TrendingTaskConfig.cxx index 8b79ced4b..c0debd473 100644 --- a/Framework/src/TrendingTaskConfig.cxx +++ b/Framework/src/TrendingTaskConfig.cxx @@ -28,96 +28,7 @@ TrendingTaskConfig::TrendingTaskConfig(std::string id, const boost::property_tre trendIfAllInputs = config.get("qc.postprocessing." + id + ".trendIfAllInputs", false); trendingTimestamp = config.get("qc.postprocessing." + id + ".trendingTimestamp", "validUntil"); - for (const auto& [_, plotConfig] : config.get_child("qc.postprocessing." + id + ".plots")) { - // since QC-1155 we allow for more than one graph in a single plot (canvas). we support both the new and old ways - // of configuring the expected plots. - std::vector graphs; - if (const auto& graphsConfig = plotConfig.get_child_optional("graphs"); graphsConfig.has_value()) { - for (const auto& [_, graphConfig] : graphsConfig.value()) { - // first we use name of the graph, if absent, we use graph title, if absent, we use plot (object) name. - const auto& name = graphConfig.get("name", graphConfig.get("title", plotConfig.get("name"))); - GraphStyle style; - style.lineColor = graphConfig.get("style.lineColor", -1); - style.lineStyle = graphConfig.get("style.lineStyle", -1); - style.lineWidth = graphConfig.get("style.lineWidth", -1); - style.markerColor = graphConfig.get("style.markerColor", -1); - style.markerStyle = graphConfig.get("style.markerStyle", -1); - style.markerSize = graphConfig.get("style.markerSize", -1.f); - style.fillColor = graphConfig.get("style.fillColor", -1); - style.fillStyle = graphConfig.get("style.fillStyle", -1); - - graphs.push_back({ name, - graphConfig.get("title", ""), - graphConfig.get("varexp"), - graphConfig.get("selection", ""), - graphConfig.get("option", ""), - graphConfig.get("graphErrors", ""), - style }); - } - } else { - GraphStyle style; - style.lineColor = plotConfig.get("style.lineColor", -1); - style.lineStyle = plotConfig.get("style.lineStyle", -1); - style.lineWidth = plotConfig.get("style.lineWidth", -1); - style.markerColor = plotConfig.get("style.markerColor", -1); - style.markerStyle = plotConfig.get("style.markerStyle", -1); - style.markerSize = plotConfig.get("style.markerSize", -1.f); - style.fillColor = plotConfig.get("style.fillColor", -1); - style.fillStyle = plotConfig.get("style.fillStyle", -1); - graphs.push_back({ plotConfig.get("name", ""), - plotConfig.get("title", ""), - plotConfig.get("varexp"), - plotConfig.get("selection", ""), - plotConfig.get("option", ""), - plotConfig.get("graphErrors", "") }); - } - - LegendConfig leg; - leg.nColumns = plotConfig.get("legend.nColumns", 1); - leg.x1 = plotConfig.get("legend.x1", -1.f); - leg.y1 = plotConfig.get("legend.y1", -1.f); - leg.x2 = plotConfig.get("legend.x2", -1.f); - leg.y2 = plotConfig.get("legend.y2", -1.f); - - plots.push_back({ plotConfig.get("name"), - plotConfig.get("title", ""), - plotConfig.get("graphAxisLabel", ""), - plotConfig.get("graphYRange", ""), - plotConfig.get("colorPalette", 0), - leg, - graphs }); - } - - const auto extractReductorParams = [](const boost::property_tree::ptree& dataSourceConfig) -> core::CustomParameters { - core::CustomParameters result; - if (const auto reductorParams = dataSourceConfig.get_child_optional("reductorParameters"); reductorParams.has_value()) { - result.populateCustomParameters(reductorParams.value()); - } - return result; - }; - - for (const auto& dataSourceConfig : config.get_child("qc.postprocessing." + id + ".dataSources")) { - if (const auto& sourceNames = dataSourceConfig.second.get_child_optional("names"); sourceNames.has_value()) { - for (const auto& sourceName : sourceNames.value()) { - dataSources.push_back({ dataSourceConfig.second.get("type", "repository"), - dataSourceConfig.second.get("path"), - sourceName.second.data(), - dataSourceConfig.second.get("reductorName"), - extractReductorParams(dataSourceConfig.second), - dataSourceConfig.second.get("moduleName") }); - } - } else if (!dataSourceConfig.second.get("name").empty()) { - // "name" : [ "something" ] would return an empty string here - dataSources.push_back({ dataSourceConfig.second.get("type", "repository"), - dataSourceConfig.second.get("path"), - dataSourceConfig.second.get("name"), - dataSourceConfig.second.get("reductorName"), - extractReductorParams(dataSourceConfig.second), - dataSourceConfig.second.get("moduleName") }); - } else { - throw std::runtime_error("No 'name' value or a 'names' vector in the path 'qc.postprocessing." + id + ".dataSources'"); - } - } + machine = TrendingMachineConfig(config.get_child("qc.postprocessing." + id)); } } // namespace o2::quality_control::postprocessing diff --git a/Framework/test/testTrendingMachine.cxx b/Framework/test/testTrendingMachine.cxx new file mode 100644 index 000000000..c8735a9aa --- /dev/null +++ b/Framework/test/testTrendingMachine.cxx @@ -0,0 +1,812 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file testTrendingMachine.cxx +/// \brief Tests of the TrendingMachine: trend filling, plot generation (incl. customization), trend continuation. +/// +/// The tests do not need any database. They use the TH1Reductor from the QcCommon module, so the module has to be +/// loadable at runtime, as in testTrendingTask. +/// + +#include "QualityControl/TrendingMachine.h" +#include "QualityControl/TrendingMachineConfig.h" +#include "QualityControl/Reductor.h" +#include "QualityControl/ReductorTObject.h" +#include "QualityControl/Quality.h" +#include "QualityControl/QualityObject.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +using namespace o2::quality_control::postprocessing; +using Catch::Approx; +using o2::quality_control::core::Quality; +using o2::quality_control::core::QualityObject; + +namespace +{ + +constexpr UInt_t T0 = 1'700'000'000; +constexpr UInt_t TStep = 3600; +constexpr int Run0 = 100; +const std::vector Means = { 10., 20., 30., 40. }; + +TrendingMachineConfig::DataSource makeSource(const std::string& name = "example") +{ + TrendingMachineConfig::DataSource ds; + ds.type = "repository"; + ds.path = "TST/MO/QcTask"; + ds.name = name; + ds.reductorName = "o2::quality_control_modules::common::TH1Reductor"; + ds.moduleName = "QcCommon"; + return ds; +} + +TrendingMachineConfig::Graph makeGraph(const std::string& name, const std::string& varexp, const std::string& option = "*L") +{ + TrendingMachineConfig::Graph g; + g.name = name; + g.title = name + " title"; + g.varexp = varexp; + g.option = option; + return g; +} + +TrendingMachineConfig::Plot makePlot(const std::string& name, std::vector graphs) +{ + TrendingMachineConfig::Plot p; + p.name = name; + p.title = name + " plot title"; + p.graphs = std::move(graphs); + return p; +} + +TrendingMachineConfig makeConfig(std::vector plots = {}, + std::vector sources = { makeSource() }) +{ + TrendingMachineConfig config; + config.plots = std::move(plots); + config.dataSources = std::move(sources); + return config; +} + +// a histogram with a known mean, stddev == 1 and two entries +std::unique_ptr makeHisto(double mean, const std::string& name = "example") +{ + auto h = std::make_unique(name.c_str(), name.c_str(), 200, 0, 200); + h->SetDirectory(nullptr); + h->Fill(mean - 1); + h->Fill(mean + 1); + return h; +} + +// updates the reductors of the data sources with the provided objects, indexed by the data source name +bool updateReductors(TrendingMachine& machine, const std::map& objects) +{ + return machine.updateReductors([&](const TrendingMachineConfig::DataSource& dataSource, Reductor& reductor) { + auto it = objects.find(dataSource.name); + auto* reductorTObject = dynamic_cast(&reductor); + if (it == objects.end() || it->second == nullptr || reductorTObject == nullptr) { + return false; + } + reductorTObject->update(it->second); + return true; + }); +} + +void fillEntry(TrendingMachine& machine, size_t i, const std::string& source = "example") +{ + auto histo = makeHisto(Means[i], source); + machine.setMetadata(T0 + i * TStep, Run0 + i); + REQUIRE(updateReductors(machine, { { source, histo.get() } })); + machine.fillTrend(); +} + +void fillTrend(TrendingMachine& machine, size_t n = Means.size()) +{ + for (size_t i = 0; i < n; i++) { + fillEntry(machine, i); + } +} + +std::vector getColumn(TTree& tree, const std::string& varexp, const std::string& selection = "") +{ + // "goff" skips the graphics, the values are then available via GetV1(), see the TTree::Draw documentation, section + // "How to obtain more info from TTree::Draw": https://root.cern/doc/master/classTTree.html + // V1 in GetV1 is the first part/dimension in varexp (V1:V2), not "Version 1", as I'd thought personally. + const auto n = tree.Draw(varexp.c_str(), selection.c_str(), "goff"); + return std::vector(tree.GetV1(), tree.GetV1() + n); +} + +using CanvasPtr = std::unique_ptr; + +// builds a trending machine, fills the trend and returns the (single) canvas generated for the plot +struct PlotFixture { + explicit PlotFixture(TrendingMachineConfig::Plot plot, size_t nEntries = Means.size()) + : machine(makeConfig({ std::move(plot) })) + { + machine.initializeTrend("trend"); + fillTrend(machine, nEntries); + auto canvases = machine.generatePlots(); + REQUIRE(canvases.size() == 1); + canvas = std::move(canvases[0]); + } + + template + T* find(const std::string& name) const + { + return dynamic_cast(canvas->FindObject(name.c_str())); + } + + TH1* background() const { return find("background"); } + + TLegend* legend() const + { + for (auto* primitive : *canvas->GetListOfPrimitives()) { + if (auto* leg = dynamic_cast(primitive)) { + return leg; + } + } + return nullptr; + } + + TrendingMachine machine; + CanvasPtr canvas; +}; + +// TCanvas objects are created in the tests. In batch mode ROOT does not try to open any graphics window, so the tests +// work also where there is no display (e.g. CI). This has to be set before the first TCanvas is created, hence it is +// done at static initialization. +struct BatchMode { + BatchMode() { gROOT->SetBatch(true); } +}; +const BatchMode batchMode; + +} // namespace + +// --------------------------------------------------------------------------------------------------------------------- +// Construction and trend filling +// --------------------------------------------------------------------------------------------------------------------- + +TEST_CASE("trending_machine_construction") +{ + SECTION("reductors are created for each data source") + { + TrendingMachine machine(makeConfig({}, { makeSource("a"), makeSource("b") })); + REQUIRE(machine.getReductor("a") != nullptr); + REQUIRE(machine.getReductor("b") != nullptr); + REQUIRE(machine.getReductor("a") != machine.getReductor("b")); + REQUIRE(machine.getReductor("c") == nullptr); + } + + SECTION("an unknown reductor results in an exception") + { + auto source = makeSource(); + source.reductorName = "o2::quality_control_modules::common::ThisReductorDoesNotExist"; + REQUIRE_THROWS(TrendingMachine(makeConfig({}, { source }))); + } + + SECTION("an unknown module results in an exception") + { + auto source = makeSource(); + source.moduleName = "ThisModuleDoesNotExist"; + REQUIRE_THROWS(TrendingMachine(makeConfig({}, { source }))); + } +} + +TEST_CASE("trending_machine_trend_structure") +{ + TrendingMachine machine(makeConfig({}, { makeSource("a"), makeSource("b") })); + REQUIRE(machine.getTrend() == nullptr); + + machine.initializeTrend("my_trend"); + auto* trend = machine.getTrend(); + REQUIRE(trend != nullptr); + REQUIRE(std::string(trend->GetName()) == "my_trend"); + REQUIRE(trend->GetEntries() == 0); + // meta + time + one for each data source + REQUIRE(trend->GetNbranches() == 4); + REQUIRE(trend->GetBranch("meta") != nullptr); + REQUIRE(trend->GetBranch("time") != nullptr); + REQUIRE(trend->GetBranch("a") != nullptr); + REQUIRE(trend->GetBranch("b") != nullptr); +} + +TEST_CASE("trending_machine_filling") +{ + TrendingMachine machine(makeConfig()); + machine.initializeTrend("trend"); + + SECTION("values, timestamps and run numbers end up in the trend") + { + fillTrend(machine); + auto* trend = machine.getTrend(); + REQUIRE(trend->GetEntries() == 4); + + REQUIRE(getColumn(*trend, "example.mean") == Means); + REQUIRE(getColumn(*trend, "example.entries") == std::vector(4, 2.)); + for (double stddev : getColumn(*trend, "example.stddev")) { + REQUIRE(stddev == Approx(1.)); + } + + const auto times = getColumn(*trend, "time"); + const auto runs = getColumn(*trend, "meta.runNumber"); + REQUIRE(times.size() == 4); + REQUIRE(runs.size() == 4); + for (size_t i = 0; i < 4; i++) { + REQUIRE(times[i] == T0 + i * TStep); + REQUIRE(runs[i] == Run0 + i); + } + + // the run number is available also as a string, for evenly spaced trends against run numbers + REQUIRE(trend->GetEntries("meta.runNumberStr==\"100\"") == 1); + REQUIRE(trend->GetEntries("meta.runNumberStr==\"103\"") == 1); + } + + SECTION("updateReductors calls the updater for each data source, with the matching reductor") + { + TrendingMachine twoSources(makeConfig({}, { makeSource("a"), makeSource("b") })); + twoSources.initializeTrend("trend"); + std::vector visited; + const bool all = twoSources.updateReductors([&](const TrendingMachineConfig::DataSource& ds, Reductor& reductor) { + visited.push_back(ds.name); + REQUIRE(&reductor == twoSources.getReductor(ds.name)); + return true; + }); + REQUIRE(all); + REQUIRE(visited == std::vector({ "a", "b" })); + } + + SECTION("updateReductors reports a failure of any source, but still tries all of them") + { + TrendingMachine twoSources(makeConfig({}, { makeSource("a"), makeSource("b") })); + twoSources.initializeTrend("trend"); + size_t calls = 0; + const bool all = twoSources.updateReductors([&](const TrendingMachineConfig::DataSource& ds, Reductor&) { + calls++; + return ds.name != "a"; + }); + REQUIRE_FALSE(all); + REQUIRE(calls == 2); + } + + SECTION("updateReductors can fill the trend, as the pull-style callers do") + { + machine.setMetadata(T0, Run0); + auto histo = makeHisto(Means[2]); + REQUIRE(machine.updateReductors([&](const TrendingMachineConfig::DataSource&, Reductor& reductor) { + static_cast(reductor).update(histo.get()); + return true; + })); + machine.fillTrend(); + REQUIRE(getColumn(*machine.getTrend(), "example.mean") == std::vector({ Means[2] })); + } + + SECTION("a source which was not updated keeps its previous values") + { + fillEntry(machine, 1); + // no updateReductors() this time + machine.setMetadata(T0 + TStep, Run0 + 1); + machine.fillTrend(); + REQUIRE(getColumn(*machine.getTrend(), "example.mean") == std::vector({ Means[1], Means[1] })); + } + + SECTION("filling a not initialized machine does not crash") + { + TrendingMachine notInitialized(makeConfig()); + REQUIRE_NOTHROW(notInitialized.fillTrend()); + REQUIRE(notInitialized.getTrend() == nullptr); + } + + SECTION("reset removes the entries, but keeps the structure and allows to fill again") + { + fillTrend(machine); + machine.reset(); + REQUIRE(machine.getTrend()->GetEntries() == 0); + REQUIRE(machine.getTrend()->GetNbranches() == 3); + fillEntry(machine, 2); + REQUIRE(getColumn(*machine.getTrend(), "example.mean") == std::vector({ Means[2] })); + } + + SECTION("reset of a not initialized trending machine does not crash") + { + TrendingMachine notInitialized(makeConfig()); + REQUIRE_NOTHROW(notInitialized.reset()); + } +} + +// --------------------------------------------------------------------------------------------------------------------- +// Trend continuation +// --------------------------------------------------------------------------------------------------------------------- + +TEST_CASE("trending_machine_trend_continuation") +{ + // prepare an existing tree, as if it was retrieved from a database + TrendingMachine original(makeConfig()); + original.initializeTrend("trend"); + fillTrend(original); + std::unique_ptr existing(original.getTrend()->CloneTree()); + REQUIRE(existing->GetEntries() == 4); + + SECTION("an identical structure is continued, existing entries are kept and new ones appended") + { + TrendingMachine machine(makeConfig()); + REQUIRE(machine.initializeTrend("new_trend", std::move(existing))); + REQUIRE(machine.getTrend()->GetEntries() == 4); + REQUIRE(std::string(machine.getTrend()->GetName()) == "trend"); // the name of the existing tree is preserved + + fillEntry(machine, 0); + REQUIRE(machine.getTrend()->GetEntries() == 5); + REQUIRE(getColumn(*machine.getTrend(), "example.mean") == std::vector({ 10., 20., 30., 40., 10. })); + } + + SECTION("an additional data source starts a new trend") + { + TrendingMachine machine(makeConfig({}, { makeSource("example"), makeSource("other") })); + REQUIRE_FALSE(machine.initializeTrend("new_trend", std::move(existing))); + REQUIRE(machine.getTrend() != nullptr); + REQUIRE(machine.getTrend()->GetEntries() == 0); + REQUIRE(std::string(machine.getTrend()->GetName()) == "new_trend"); + REQUIRE(machine.getTrend()->GetNbranches() == 4); + } + + SECTION("a missing data source starts a new trend") + { + TrendingMachine machine(makeConfig({}, {})); + REQUIRE_FALSE(machine.initializeTrend("new_trend", std::move(existing))); + REQUIRE(machine.getTrend()->GetEntries() == 0); + REQUIRE(machine.getTrend()->GetNbranches() == 2); + } + + SECTION("the same number of branches, but with a different name starts a new trend") + { + TrendingMachine machine(makeConfig({}, { makeSource("renamed") })); + REQUIRE_FALSE(machine.initializeTrend("new_trend", std::move(existing))); + REQUIRE(machine.getTrend()->GetEntries() == 0); + REQUIRE(machine.getTrend()->GetBranch("renamed") != nullptr); + REQUIRE(machine.getTrend()->GetBranch("example") == nullptr); + } + + SECTION("no existing tree starts a new trend") + { + TrendingMachine machine(makeConfig()); + REQUIRE_FALSE(machine.initializeTrend("new_trend", nullptr)); + REQUIRE(machine.getTrend() != nullptr); + REQUIRE(machine.getTrend()->GetEntries() == 0); + } +} + +// --------------------------------------------------------------------------------------------------------------------- +// Plot generation +// --------------------------------------------------------------------------------------------------------------------- + +TEST_CASE("trending_machine_plots_generation") +{ + SECTION("nothing is generated without a trend") + { + TrendingMachine machine(makeConfig({ makePlot("p", { makeGraph("g", "example.mean:time") }) })); + REQUIRE(machine.generatePlots().empty()); + } + + SECTION("nothing is generated for an empty trend") + { + TrendingMachine machine(makeConfig({ makePlot("p", { makeGraph("g", "example.mean:time") }) })); + machine.initializeTrend("trend"); + REQUIRE(machine.generatePlots().empty()); + } + + SECTION("one canvas per configured plot, with name and title") + { + TrendingMachine machine(makeConfig({ makePlot("first", { makeGraph("g1", "example.mean:time") }), + makePlot("second", { makeGraph("g2", "example.stddev:time") }), + makePlot("third", { makeGraph("g3", "example.mean") }) })); + machine.initializeTrend("trend"); + fillTrend(machine); + auto canvases = machine.generatePlots(); + REQUIRE(canvases.size() == 3); + const std::vector names = { "first", "second", "third" }; + for (size_t i = 0; i < 3; i++) { + REQUIRE(std::string(canvases[i]->GetName()) == names[i]); + REQUIRE(std::string(canvases[i]->GetTitle()) == names[i] + " plot title"); + } + } + + SECTION("no plots configured means no canvases") + { + TrendingMachine machine(makeConfig()); + machine.initializeTrend("trend"); + fillTrend(machine); + REQUIRE(machine.generatePlots().empty()); + } + + SECTION("plots can be generated repeatedly as the trend grows") + { + TrendingMachine machine(makeConfig({ makePlot("p", { makeGraph("g", "example.mean:time") }) })); + machine.initializeTrend("trend"); + fillEntry(machine, 0); + { + auto canvases = machine.generatePlots(); + REQUIRE(canvases.size() == 1); + REQUIRE(dynamic_cast(canvases[0]->FindObject("g"))->GetN() == 1); + } + fillEntry(machine, 1); + { + auto canvases = machine.generatePlots(); + REQUIRE(canvases.size() == 1); + REQUIRE(dynamic_cast(canvases[0]->FindObject("g"))->GetN() == 2); + } + } +} + +TEST_CASE("trending_machine_plot_graph") +{ + auto graph = makeGraph("g", "example.mean:time"); + PlotFixture fixture(makePlot("p", { graph })); + + SECTION("a graph with the data points from the trend, name and title") + { + auto* tgraph = fixture.find("g"); + REQUIRE(tgraph != nullptr); + REQUIRE(std::string(tgraph->GetTitle()) == "g title"); + REQUIRE(tgraph->GetN() == 4); + for (int i = 0; i < 4; i++) { + REQUIRE(tgraph->GetPointY(i) == Approx(Means[i])); + REQUIRE(tgraph->GetPointX(i) == Approx(T0 + i * TStep)); + } + } + + SECTION("the title of the plot is applied to the axes frame") + { + REQUIRE(fixture.background() != nullptr); + REQUIRE(std::string(fixture.background()->GetTitle()) == "p plot title"); + } + + SECTION("a plot with a single graph does not get a legend") + { + REQUIRE(fixture.legend() == nullptr); + } + + SECTION("a time axis is used for ':time' plots") + { + REQUIRE(fixture.background()->GetXaxis()->GetTimeDisplay()); + } +} + +TEST_CASE("trending_machine_plot_x_axis_types") +{ + SECTION("a run number axis does not use the time display and exponents") + { + PlotFixture fixture(makePlot("p", { makeGraph("g", "example.mean:meta.runNumber") })); + REQUIRE_FALSE(fixture.background()->GetXaxis()->GetTimeDisplay()); + REQUIRE(fixture.background()->GetXaxis()->GetNoExponent()); + } + + SECTION("a correlation plot uses neither") + { + PlotFixture fixture(makePlot("p", { makeGraph("g", "example.mean:example.entries") })); + REQUIRE_FALSE(fixture.background()->GetXaxis()->GetTimeDisplay()); + REQUIRE_FALSE(fixture.background()->GetXaxis()->GetNoExponent()); + } +} + +TEST_CASE("trending_machine_plot_axes_customization") +{ + SECTION("axis labels: yLabel:xLabel") + { + auto plot = makePlot("p", { makeGraph("g", "example.mean:time") }); + plot.graphAxisLabel = "the y label:the x label"; + PlotFixture fixture(plot); + REQUIRE(std::string(fixture.background()->GetYaxis()->GetTitle()) == "the y label"); + REQUIRE(std::string(fixture.background()->GetXaxis()->GetTitle()) == "the x label"); + } + + SECTION("malformed axis labels are ignored") + { + // without a user label, TTree::Draw sets the axis titles to the drawn expressions + PlotFixture reference(makePlot("p", { makeGraph("g", "example.mean:time") })); + auto plot = makePlot("p", { makeGraph("g", "example.mean:time") }); + plot.graphAxisLabel = "too:many:colons"; + PlotFixture fixture(plot); + REQUIRE(std::string(fixture.background()->GetYaxis()->GetTitle()) == reference.background()->GetYaxis()->GetTitle()); + REQUIRE(std::string(fixture.background()->GetXaxis()->GetTitle()) == reference.background()->GetXaxis()->GetTitle()); + } + + SECTION("y axis range") + { + auto plot = makePlot("p", { makeGraph("g", "example.mean:time") }); + plot.graphYRange = "-5:100"; + PlotFixture fixture(plot); + REQUIRE(fixture.background()->GetYaxis()->GetXmin() == Approx(-5.)); + REQUIRE(fixture.background()->GetYaxis()->GetXmax() == Approx(100.)); + } + + SECTION("malformed y axis range is ignored and does not throw") + { + PlotFixture reference(makePlot("p", { makeGraph("g", "example.mean:time") })); + for (const std::string range : { "0:1:2", "abc:def", "0:", ":10", "10", "5:5", "10:0", "1e999:2e999" }) { + CAPTURE(range); + auto plot = makePlot("p", { makeGraph("g", "example.mean:time") }); + plot.graphYRange = range; + std::unique_ptr fixture; + REQUIRE_NOTHROW(fixture = std::make_unique(plot)); + REQUIRE(fixture->background()->GetYaxis()->GetXmin() == Approx(reference.background()->GetYaxis()->GetXmin())); + REQUIRE(fixture->background()->GetYaxis()->GetXmax() == Approx(reference.background()->GetYaxis()->GetXmax())); + } + } +} + +TEST_CASE("trending_machine_plot_graph_selection") +{ + auto graph = makeGraph("g", "example.mean:time"); + graph.selection = "example.mean>15"; + PlotFixture fixture(makePlot("p", { graph })); + auto* tgraph = fixture.find("g"); + REQUIRE(tgraph != nullptr); + REQUIRE(tgraph->GetN() == 3); + REQUIRE(tgraph->GetPointY(0) == Approx(20.)); +} + +TEST_CASE("trending_machine_plot_graph_style") +{ + SECTION("explicit style is applied to the graph") + { + auto styled = makeGraph("styled", "example.mean:time", "LP"); + styled.style.lineColor = kRed; + styled.style.lineStyle = 2; + styled.style.lineWidth = 3; + styled.style.markerColor = kBlue; + styled.style.markerStyle = 21; + styled.style.markerSize = 1.5f; + styled.style.fillColor = kGreen; + styled.style.fillStyle = 3001; + auto plain = makeGraph("plain", "example.mean + 5:time", "LP"); + + PlotFixture fixture(makePlot("p", { styled, plain })); + auto* tgraph = fixture.find("styled"); + REQUIRE(tgraph != nullptr); + REQUIRE(tgraph->GetLineColor() == kRed); + REQUIRE(tgraph->GetLineStyle() == 2); + REQUIRE(tgraph->GetLineWidth() == 3); + REQUIRE(tgraph->GetMarkerColor() == kBlue); + REQUIRE(tgraph->GetMarkerStyle() == 21); + REQUIRE(tgraph->GetMarkerSize() == Approx(1.5)); + REQUIRE(tgraph->GetFillColor() == kGreen); + REQUIRE(tgraph->GetFillStyle() == 3001); + } + + SECTION("style parameters which are not set do not override the defaults") + { + auto graph = makeGraph("styled", "example.mean:time", "LP"); + graph.style.markerStyle = 22; // the only one set + PlotFixture reference(makePlot("p", { makeGraph("g", "example.mean:time", "LP") })); + PlotFixture fixture(makePlot("p", { graph })); + auto* tgraph = fixture.find("styled"); + auto* refGraph = reference.find("g"); + REQUIRE(tgraph != nullptr); + REQUIRE(refGraph != nullptr); + REQUIRE(tgraph->GetMarkerStyle() == 22); + REQUIRE(tgraph->GetMarkerColor() == refGraph->GetMarkerColor()); + REQUIRE(tgraph->GetLineColor() == refGraph->GetLineColor()); + REQUIRE(tgraph->GetLineWidth() == refGraph->GetLineWidth()); + REQUIRE(tgraph->GetMarkerSize() == Approx(refGraph->GetMarkerSize())); + } +} + +TEST_CASE("trending_machine_plot_graph_errors") +{ + SECTION("error bars are drawn as a separate graph with the same style") + { + auto graph = makeGraph("g", "example.mean:time", "P"); + graph.errors = "1:2"; + graph.style.markerColor = kMagenta; + graph.style.lineColor = kMagenta; + PlotFixture fixture(makePlot("p", { graph })); + + auto* main = fixture.find("g"); + auto* errors = fixture.find("g_errors"); + REQUIRE(main != nullptr); + REQUIRE(errors != nullptr); + REQUIRE(errors->GetN() == 4); + for (int i = 0; i < 4; i++) { + REQUIRE(errors->GetPointY(i) == Approx(Means[i])); + REQUIRE(errors->GetErrorX(i) == Approx(1.)); + REQUIRE(errors->GetErrorY(i) == Approx(2.)); + } + REQUIRE(errors->GetMarkerColor() == kMagenta); + REQUIRE(errors->GetLineColor() == kMagenta); + } + + SECTION("errors can be expressions of the trend branches") + { + auto graph = makeGraph("g", "example.mean:time", "P"); + graph.errors = "1:example.mean*0.5"; + PlotFixture fixture(makePlot("p", { graph })); + auto* errors = fixture.find("g_errors"); + REQUIRE(errors != nullptr); + for (int i = 0; i < 4; i++) { + REQUIRE(errors->GetErrorY(i) == Approx(Means[i] * 0.5)); + } + } + + SECTION("errors are ignored for histograms") + { + auto graph = makeGraph("h", "example.mean", ""); + graph.errors = "1:2"; + PlotFixture fixture(makePlot("p", { graph })); + REQUIRE(fixture.find("h_errors") == nullptr); + REQUIRE(fixture.find("h") != nullptr); + } +} + +TEST_CASE("trending_machine_plot_histogram") +{ + PlotFixture fixture(makePlot("p", { makeGraph("h", "example.mean", "") })); + auto* histo = fixture.find("h"); + REQUIRE(histo != nullptr); + // the plot title is what gets displayed on top of the pad, also for plots with histograms + REQUIRE(std::string(histo->GetTitle()) == "p plot title"); + REQUIRE(histo->GetEntries() == 4); + REQUIRE(histo->GetMean() == Approx(25.)); +} + +TEST_CASE("trending_machine_plot_multiple_graphs") +{ + auto first = makeGraph("first", "example.mean:time", "LP"); + auto second = makeGraph("second", "example.mean + 10:time", "LP"); + auto third = makeGraph("third", "example.mean + 20:time", "LP"); + + SECTION("all graphs are drawn on the same canvas and listed in the legend") + { + PlotFixture fixture(makePlot("p", { first, second, third })); + for (const char* name : { "first", "second", "third" }) { + REQUIRE(fixture.find(name) != nullptr); + } + REQUIRE(fixture.find("second")->GetPointY(0) == Approx(Means[0] + 10)); + REQUIRE(fixture.find("third")->GetPointY(0) == Approx(Means[0] + 20)); + + auto* legend = fixture.legend(); + REQUIRE(legend != nullptr); + REQUIRE(legend->GetListOfPrimitives()->GetSize() == 3); + } + + SECTION("the legend placement and number of columns can be customized (NDC)") + { + auto plot = makePlot("p", { first, second, third }); + plot.legend.nColumns = 3; + plot.legend.x1 = 0.11f; + plot.legend.y1 = 0.72f; + plot.legend.x2 = 0.55f; + plot.legend.y2 = 0.88f; + PlotFixture fixture(plot); + auto* legend = fixture.legend(); + REQUIRE(legend != nullptr); + REQUIRE(legend->GetNColumns() == 3); + REQUIRE(legend->GetX1NDC() == Approx(0.11)); + REQUIRE(legend->GetY1NDC() == Approx(0.72)); + REQUIRE(legend->GetX2NDC() == Approx(0.55)); + REQUIRE(legend->GetY2NDC() == Approx(0.88)); + } + + SECTION("the legend is transparent and has no border") + { + PlotFixture fixture(makePlot("p", { first, second })); + auto* legend = fixture.legend(); + REQUIRE(legend != nullptr); + REQUIRE(legend->GetBorderSize() == 0); + REQUIRE(legend->GetFillStyle() == 0); + } +} + +TEST_CASE("trending_machine_plot_color_palette") +{ + auto first = makeGraph("first", "example.mean:time", "LP PLC PMC"); + auto second = makeGraph("second", "example.mean + 10:time", "LP PLC PMC"); + auto third = makeGraph("third", "example.mean + 20:time", "LP PLC PMC"); + + auto plot = makePlot("p", { first, second, third }); + plot.colorPalette = kViridis; + PlotFixture fixture(plot); + + // with PLC/PMC, each graph should get a different color out of the selected palette + const auto c1 = fixture.find("first")->GetLineColor(); + const auto c2 = fixture.find("second")->GetLineColor(); + const auto c3 = fixture.find("third")->GetLineColor(); + REQUIRE(c1 != c2); + REQUIRE(c2 != c3); + REQUIRE(c1 != c3); + REQUIRE(fixture.find("first")->GetMarkerColor() == c1); +} + +// --------------------------------------------------------------------------------------------------------------------- +// Trending of qualities ("repository-quality" data sources) +// --------------------------------------------------------------------------------------------------------------------- + +TEST_CASE("trending_machine_quality_source") +{ + auto qualitySource = makeSource("QcCheck"); + qualitySource.type = "repository-quality"; + qualitySource.path = "TST/QO"; + qualitySource.reductorName = "o2::quality_control_modules::common::QualityReductor"; + + auto graph = makeGraph("quality", "QcCheck.level:time", "*"); + TrendingMachine machine(makeConfig({ makePlot("p", { graph }) }, { makeSource(), qualitySource })); + machine.initializeTrend("trend"); + REQUIRE(machine.getTrend()->GetNbranches() == 4); + REQUIRE(machine.getReductor("QcCheck") != nullptr); + + const std::vector qualities = { Quality::Good, Quality::Medium, Quality::Bad, Quality::Null }; + for (size_t i = 0; i < qualities.size(); i++) { + QualityObject qo(qualities[i], "QcCheck", "TST"); + auto histo = makeHisto(Means[i]); + machine.setMetadata(T0 + i * TStep, Run0 + i); + REQUIRE(updateReductors(machine, { { "example", histo.get() }, { "QcCheck", &qo } })); + machine.fillTrend(); + } + + auto* trend = machine.getTrend(); + REQUIRE(trend->GetEntries() == 4); + + SECTION("quality levels and names are trended next to other sources") + { + const auto levels = getColumn(*trend, "QcCheck.level"); + REQUIRE(levels.size() == 4); + for (size_t i = 0; i < qualities.size(); i++) { + REQUIRE(levels[i] == qualities[i].getLevel()); + } + REQUIRE(trend->GetEntries("QcCheck.name==\"Good\"") == 1); + REQUIRE(trend->GetEntries("QcCheck.name==\"Medium\"") == 1); + REQUIRE(trend->GetEntries("QcCheck.name==\"Bad\"") == 1); + REQUIRE(trend->GetEntries("QcCheck.name==\"Null\"") == 1); + REQUIRE(getColumn(*trend, "example.mean") == Means); + } + + SECTION("a quality trend can be plotted") + { + auto canvases = machine.generatePlots(); + REQUIRE(canvases.size() == 1); + auto* tgraph = dynamic_cast(canvases[0]->FindObject("quality")); + REQUIRE(tgraph != nullptr); + REQUIRE(tgraph->GetN() == 4); + REQUIRE(tgraph->GetPointY(0) == Approx(Quality::Good.getLevel())); + REQUIRE(tgraph->GetPointY(2) == Approx(Quality::Bad.getLevel())); + } + + SECTION("an object of a wrong type does not change the quality values") + { + auto histo = makeHisto(1); + // the reductor is called, but it ignores the object + REQUIRE(updateReductors(machine, { { "example", histo.get() }, { "QcCheck", histo.get() } })); + machine.setMetadata(T0 + 10 * TStep, Run0 + 10); + machine.fillTrend(); + const auto levels = getColumn(*trend, "QcCheck.level"); + REQUIRE(levels.size() == 5); + REQUIRE(levels[4] == levels[3]); + } +} diff --git a/Framework/test/testTrendingMachineConfig.cxx b/Framework/test/testTrendingMachineConfig.cxx new file mode 100644 index 000000000..1bbcac1e0 --- /dev/null +++ b/Framework/test/testTrendingMachineConfig.cxx @@ -0,0 +1,241 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file testTrendingMachineConfig.cxx +/// \brief Tests of the parsing of TrendingMachineConfig and its use in TrendingTaskConfig. +/// + +#include "QualityControl/TrendingMachineConfig.h" + +#include +#include +#include + +#include + +using namespace o2::quality_control::postprocessing; +using Catch::Approx; + +namespace +{ +boost::property_tree::ptree parse(const std::string& json) +{ + boost::property_tree::ptree tree; + std::stringstream ss(json); + boost::property_tree::read_json(ss, tree); + return tree; +} +} // namespace + +TEST_CASE("trending_machine_config_data_sources") +{ + SECTION("a single name, with defaults") + { + auto tree = parse(R"json([{ + "name": "example", + "reductorName": "my::Reductor", + "moduleName": "MyModule" + }])json"); + auto sources = TrendingMachineConfig::parseDataSources(tree); + REQUIRE(sources.size() == 1); + REQUIRE(sources[0].name == "example"); + REQUIRE(sources[0].type == "repository"); + REQUIRE(sources[0].path.empty()); + REQUIRE(sources[0].reductorName == "my::Reductor"); + REQUIRE(sources[0].moduleName == "MyModule"); + REQUIRE_FALSE(sources[0].reductorParameters.atOptional("key").has_value()); + } + + SECTION("a list of names is expanded into separate sources which share the rest of the configuration") + { + auto tree = parse(R"json([{ + "type": "repository-quality", + "path": "TST/QO", + "names": [ "a", "b", "c" ], + "reductorName": "my::Reductor", + "moduleName": "MyModule", + "reductorParameters": { "default": { "default": { "key": "value" } } } + }])json"); + auto sources = TrendingMachineConfig::parseDataSources(tree); + REQUIRE(sources.size() == 3); + const std::vector names = { "a", "b", "c" }; + for (size_t i = 0; i < 3; i++) { + REQUIRE(sources[i].name == names[i]); + REQUIRE(sources[i].type == "repository-quality"); + REQUIRE(sources[i].path == "TST/QO"); + REQUIRE(sources[i].reductorName == "my::Reductor"); + REQUIRE(sources[i].reductorParameters.atOptional("key").value_or("") == "value"); + } + } + + SECTION("neither name nor names") + { + auto tree = parse(R"json([{ "reductorName": "r", "moduleName": "m" }])json"); + REQUIRE_THROWS(TrendingMachineConfig::parseDataSources(tree)); + } + + SECTION("an empty name") + { + auto tree = parse(R"json([{ "name": "", "reductorName": "r", "moduleName": "m" }])json"); + REQUIRE_THROWS(TrendingMachineConfig::parseDataSources(tree)); + } + + SECTION("a missing reductor or module") + { + REQUIRE_THROWS(TrendingMachineConfig::parseDataSources(parse(R"json([{ "name": "a", "moduleName": "m" }])json"))); + REQUIRE_THROWS(TrendingMachineConfig::parseDataSources(parse(R"json([{ "name": "a", "reductorName": "r" }])json"))); + } +} + +TEST_CASE("trending_machine_config_plots") +{ + SECTION("the full format with defaults") + { + auto tree = parse(R"json([{ + "name": "plot", "graphs": [ { "varexp": "a.b:time" } ] + }])json"); + auto plots = TrendingMachineConfig::parsePlots(tree); + REQUIRE(plots.size() == 1); + REQUIRE(plots[0].name == "plot"); + REQUIRE(plots[0].title.empty()); + REQUIRE(plots[0].graphAxisLabel.empty()); + REQUIRE(plots[0].graphYRange.empty()); + REQUIRE(plots[0].colorPalette == 0); + REQUIRE(plots[0].legend.nColumns == 1); + REQUIRE(plots[0].legend.x1 < 0); + REQUIRE(plots[0].legend.y1 < 0); + REQUIRE(plots[0].legend.x2 < 0); + REQUIRE(plots[0].legend.y2 < 0); + REQUIRE(plots[0].graphs.size() == 1); + + const auto& graph = plots[0].graphs[0]; + REQUIRE(graph.varexp == "a.b:time"); + REQUIRE(graph.name == "plot"); // falls back to the plot name + REQUIRE(graph.title.empty()); + REQUIRE(graph.selection.empty()); + REQUIRE(graph.option.empty()); + REQUIRE(graph.errors.empty()); + REQUIRE(graph.style.lineColor == -1); + REQUIRE(graph.style.lineStyle == -1); + REQUIRE(graph.style.lineWidth == -1); + REQUIRE(graph.style.markerColor == -1); + REQUIRE(graph.style.markerStyle == -1); + REQUIRE(graph.style.markerSize == Approx(-1.f)); + REQUIRE(graph.style.fillColor == -1); + REQUIRE(graph.style.fillStyle == -1); + } + + SECTION("everything specified") + { + auto tree = parse(R"json([{ + "name": "plot", "title": "Plot title", "graphAxisLabel": "y:x", "graphYRange": "0:10", "colorPalette": 112, + "legend": { "nColumns": 2, "x1": 0.1, "y1": 0.2, "x2": 0.3, "y2": 0.4 }, + "graphs": [ + { "name": "g1", "title": "G1", "varexp": "a.b:time", "selection": "a.b>0", "option": "*L PLC", "graphErrors": "1:2", + "style": { "lineColor": 2, "lineStyle": 3, "lineWidth": 4, "markerColor": 5, "markerStyle": 6, "markerSize": 1.5, + "fillColor": 7, "fillStyle": 8 } }, + { "title": "second", "varexp": "a.b" } + ] + }])json"); + auto plots = TrendingMachineConfig::parsePlots(tree); + REQUIRE(plots.size() == 1); + const auto& plot = plots[0]; + REQUIRE(plot.title == "Plot title"); + REQUIRE(plot.graphAxisLabel == "y:x"); + REQUIRE(plot.graphYRange == "0:10"); + REQUIRE(plot.colorPalette == 112); + REQUIRE(plot.legend.nColumns == 2); + REQUIRE(plot.legend.x1 == Approx(0.1f)); + REQUIRE(plot.legend.y1 == Approx(0.2f)); + REQUIRE(plot.legend.x2 == Approx(0.3f)); + REQUIRE(plot.legend.y2 == Approx(0.4f)); + + REQUIRE(plot.graphs.size() == 2); + const auto& g1 = plot.graphs[0]; + REQUIRE(g1.name == "g1"); + REQUIRE(g1.title == "G1"); + REQUIRE(g1.varexp == "a.b:time"); + REQUIRE(g1.selection == "a.b>0"); + REQUIRE(g1.option == "*L PLC"); + REQUIRE(g1.errors == "1:2"); + REQUIRE(g1.style.lineColor == 2); + REQUIRE(g1.style.lineStyle == 3); + REQUIRE(g1.style.lineWidth == 4); + REQUIRE(g1.style.markerColor == 5); + REQUIRE(g1.style.markerStyle == 6); + REQUIRE(g1.style.markerSize == Approx(1.5f)); + REQUIRE(g1.style.fillColor == 7); + REQUIRE(g1.style.fillStyle == 8); + + // without a name, the graph is named after its title + REQUIRE(plot.graphs[1].name == "second"); + } + + SECTION("the old format: the plot is also the graph") + { + auto tree = parse(R"json([{ + "name": "old", + "title": "Old title", + "varexp": "a.b:time", + "selection": "a.b>1", + "option": "*", + "graphErrors": "1:2" + }])json"); + auto plots = TrendingMachineConfig::parsePlots(tree); + REQUIRE(plots.size() == 1); + REQUIRE(plots[0].name == "old"); + REQUIRE(plots[0].graphs.size() == 1); + REQUIRE(plots[0].graphs[0].name == "old"); + REQUIRE(plots[0].graphs[0].varexp == "a.b:time"); + REQUIRE(plots[0].graphs[0].selection == "a.b>1"); + REQUIRE(plots[0].graphs[0].option == "*"); + REQUIRE(plots[0].graphs[0].errors == "1:2"); + } + + SECTION("a plot without a name") + { + REQUIRE_THROWS(TrendingMachineConfig::parsePlots(parse(R"json([{ "graphs": [ { "varexp": "a" } ] }])json"))); + } + + SECTION("a graph without a varexp") + { + REQUIRE_THROWS(TrendingMachineConfig::parsePlots(parse(R"json([{ "name": "p", "graphs": [ { "name": "g" } ] }])json"))); + } + + SECTION("an empty plot list") + { + REQUIRE(TrendingMachineConfig::parsePlots(parse(R"json([])json")).empty()); + } +} + +TEST_CASE("trending_machine_config_from_tree") +{ + SECTION("plots and data sources are read from the given node") + { + auto tree = parse(R"json({ + "dataSources": [ { "name": "a", "reductorName": "r", "moduleName": "m" } ], + "plots": [ { "name": "p", "graphs": [ { "varexp": "a.x" } ] } ], + "somethingElse": "ignored" + })json"); + TrendingMachineConfig config(tree); + REQUIRE(config.dataSources.size() == 1); + REQUIRE(config.dataSources[0].name == "a"); + REQUIRE(config.plots.size() == 1); + REQUIRE(config.plots[0].name == "p"); + } + + SECTION("both plots and data sources are required") + { + REQUIRE_THROWS(TrendingMachineConfig(parse(R"json({ "plots": [] })json"))); + REQUIRE_THROWS(TrendingMachineConfig(parse(R"json({ "dataSources": [] })json"))); + } +} \ No newline at end of file diff --git a/Framework/test/testTrendingTask.cxx b/Framework/test/testTrendingTask.cxx index 0c3872483..b4646665c 100644 --- a/Framework/test/testTrendingTask.cxx +++ b/Framework/test/testTrendingTask.cxx @@ -16,6 +16,7 @@ #include "Framework/include/QualityControl/Reductor.h" #include "QualityControl/TrendingTask.h" +#include "QualityControl/TrendingMachine.h" #include "QualityControl/DatabaseFactory.h" #include "QualityControl/MonitorObject.h" #include "QualityControl/Triggers.h" @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -67,9 +69,14 @@ struct DeclareGlobalGet { friend typename Accessor::type get(Accessor) { return Member; } }; -struct TrendingTaskReductorAccessor { - using type = std::unordered_map> TrendingTask::*; - friend type get(TrendingTaskReductorAccessor); +struct TrendingTaskMachineAccessor { + using type = std::unique_ptr TrendingTask::*; + friend type get(TrendingTaskMachineAccessor); +}; + +struct TrendingMachineReductorAccessor { + using type = std::unordered_map> TrendingMachine::*; + friend type get(TrendingMachineReductorAccessor); }; struct ReductorConfigAccessor { @@ -77,7 +84,8 @@ struct ReductorConfigAccessor { friend type get(ReductorConfigAccessor); }; -template struct DeclareGlobalGet; +template struct DeclareGlobalGet; +template struct DeclareGlobalGet; template struct DeclareGlobalGet; TEST_CASE("test_trending_task") @@ -184,7 +192,9 @@ TEST_CASE("test_trending_task") REQUIRE_NOTHROW(task.configure(config)); { - auto& reductors = task.*get(TrendingTaskReductorAccessor()); + auto& machine = task.*get(TrendingTaskMachineAccessor()); + REQUIRE(machine != nullptr); + auto& reductors = *machine.*get(TrendingMachineReductorAccessor()); size_t foundCount{}; for (const auto& reductor : reductors) { auto& config = (*reductor.second.get()).*get(ReductorConfigAccessor());