Extract trend generation out of TrendingTask

Framework/CMakeLists.txt

@@ -107,6 +107,7 @@ add_library(O2QualityControl
   src/PostProcessingInterface.cxx
   src/PostProcessingDevice.cxx
   src/TrendingTask.cxx
+  src/TrendingMachineConfig.cxx
   src/TrendingTaskConfig.cxx
   src/DummyDatabase.cxx
   src/DataProducer.cxx
@@ -143,6 +144,7 @@ add_library(O2QualityControl
   src/LateTaskRunner.cxx
   src/LateTaskRunnerFactory.cxx
   src/LateTaskInterface.cxx
+  src/TrendingMachine.cxx
   src/Actor.cxx
   src/ActorHelpers.cxx
   src/DataProcessorAdapter.cxx
@@ -301,6 +303,8 @@ add_executable(o2-qc-test-core
                test/testVersion.cxx
                test/testMonitorObjectCollection.cxx
                test/testTrendingTask.cxx
+               test/testTrendingMachine.cxx
+               test/testTrendingMachineConfig.cxx
                test/testKafkaTests.cxx
                test/testFlagHelpers.cxx
                test/testQualitiesToFlagCollectionConverter.cxx

Framework/include/QualityControl/TrendingMachine.h

@@ -0,0 +1,125 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file    TrendingMachine.h
+/// \author  Piotr Konopka
+///
+/// \brief   Manager class for trending functionality (TTree management, reductors, plot generation)
+///
+/// This class contains code extracted from TrendingTask to eliminate duplication between
+/// TrendingTask and LateTrendingTask.
+///
+/// Code history can be accessed with:
+///   git log --follow Framework/src/TrendingMachine.cxx
+///   git blame -C -C -C Framework/src/TrendingMachine.cxx
+///
+
+#ifndef QUALITYCONTROL_TRENDINGMACHINE_H
+#define QUALITYCONTROL_TRENDINGMACHINE_H
+
+#include "QualityControl/TrendingMachineConfig.h"
+#include <string>
+#include <vector>
+#include <map>
+#include <memory>
+#include <functional>
+#include <unordered_map>
+
+class TTree;
+class TCanvas;
+class TGraph;
+class TAxis;
+class TH1;
+class TObject;
+
+namespace o2::quality_control::postprocessing
+{
+
+class Reductor;
+
+/// \brief A class to generate trends and trending plots
+///
+/// This class runs reductors to extract data points from objects, stores them
+/// in a columnar data storage facility and runs plot (trends and correlations)
+/// generation. Historically, it was extracted from TrendingTask.
+///
+/// \author Piotr Konopka
+class TrendingMachine
+{
+ public:
+  explicit TrendingMachine(const TrendingMachineConfig& config);
+  ~TrendingMachine() = default;
+
+  // initializes the trend, optionally with an existing tree if user needs to resume an old trend
+  /// \brief Prepares the trend (TTree) to be filled.
+  /// \param treeName the name of the new tree, if one has to be created
+  /// \param existingTree optionally, a tree to continue (e.g. retrieved from a database). The tree is used only if
+  ///        its structure matches the data sources, otherwise it is destroyed.
+  /// \return true if the existing tree was taken over and the trend continues, false if a new trend was started
+  bool initializeTrend(const std::string& treeName, std::unique_ptr<TTree> existingTree = nullptr);
+  /// \brief returns a pointer to the trend tree, the ownership is not passed to the caller
+  // unfortunately non-const, because of ObjectManager and MonitorObject
+  TTree* getTrend() { return mTrend.get(); }
+
+  /// \brief sets the mandatary metadata for the next trend entry
+  void setMetadata(UInt_t timestamp, int runNumber);
+
+  /// \brief updates the reductors of all data sources, with the data provided by the caller.
+  /// \param updater a function which gets a data source and its reductor, and should update the latter, e.g. by
+  ///        retrieving an object from a database. It should return true on success. Failures are logged.
+  /// \return true if all data sources were updated successfully
+  bool updateReductors(const std::function<bool(const TrendingMachineConfig::DataSource&, Reductor&)>& updater);
+  /// \brief returns a const pointer to the reductor, the ownership is not passed to the caller
+  const Reductor* getReductor(const std::string& dataSourceName) const;
+
+  /// \brief fills the trend with a new row containing the current state of reductors
+  void fillTrend();
+
+  /// \brief generates pre-configured plots out of the latest trend
+  std::vector<std::unique_ptr<TCanvas>> generatePlots();
+
+  /// \brief resets the trend, clears the plots, but does not delete the trend tree (trending can resume)
+  void reset();
+
+ private:
+  static bool canContinueTrend(TTree* tree, const std::vector<TrendingMachineConfig::DataSource>& dataSources);
+  static TCanvas* drawPlot(TTree* trend, const TrendingMachineConfig::Plot& plotConfig);
+  static void applyStyleToGraph(TGraph* graph, const TrendingMachineConfig::GraphStyle& style);
+  static std::string deduceGraphLegendOptions(const TrendingMachineConfig::Graph& graphConfig);
+  static void setUserAxesLabels(TAxis* xAxis, TAxis* yAxis, const std::string& graphAxesLabels);
+  static void setUserYAxisRange(TH1* hist, const std::string& graphYAxisRange);
+  static void formatTimeXAxis(TH1* background);
+  static void formatRunNumberXAxis(TH1* background);
+
+ private:
+  static constexpr size_t MaxRunNumberStringLength = 6;
+  struct {
+    // we store run numbers both as an integer and as a string to allow users to select whether they need
+    // a trend in integer or label domain (the latter will contain evenly-spaced data points)
+    Long64_t runNumber = 0;
+    char runNumberStr[MaxRunNumberStringLength + 1] = { 0 };
+    static const char* getBranchLeafList()
+    {
+      return "runNumber/L:runNumberStr/C";
+    }
+  } mMetaData;
+
+  std::unique_ptr<TTree> mTrend;
+  UInt_t mTime;
+  std::unordered_map<std::string, std::unique_ptr<Reductor>> mReductors;
+  std::vector<TrendingMachineConfig::Plot> mPlotConfigs;
+  std::vector<TrendingMachineConfig::DataSource> mDataSources;
+};
+
+} // namespace o2::quality_control::postprocessing
+
+#endif // QUALITYCONTROL_TRENDINGMACHINE_H

Framework/include/QualityControl/TrendingMachineConfig.h

@@ -0,0 +1,106 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file   TrendingMachineConfig.h
+/// \author Piotr Konopka
+///
+
+#ifndef QUALITYCONTROL_TRENDINGMACHINECONFIG_H
+#define QUALITYCONTROL_TRENDINGMACHINECONFIG_H
+
+#include <vector>
+#include <string>
+#include <boost/property_tree/ptree_fwd.hpp>
+#include "CustomParameters.h"
+
+namespace o2::quality_control::postprocessing
+{
+
+/// \brief  Configuration of the TrendingMachine: what to trend (data sources) and what to plot.
+/// It does not contain anything related to when and how the machine is triggered or fed -
+/// that is up to the callers (e.g. TrendingTaskConfig).
+struct TrendingMachineConfig {
+  TrendingMachineConfig() = default;
+  /// \param config a node which contains the "plots" and "dataSources" children
+  explicit TrendingMachineConfig(const boost::property_tree::ptree& config);
+  ~TrendingMachineConfig() = default;
+
+  // graph style configuration
+  // colors as defined by ROOT TColor class:
+  // https://root.cern/doc/master/classTColor.html
+  // marker colors and styles are as defined by ROOT TAttMarker class
+  // https://root.cern/doc/master/classTAttMarker.html
+  // line styles are as defined by ROOT TAttLine class
+  // https://root.cern/doc/master/classTAttLine.html
+  // WARNING: Any parameters in this struct will override colliding parameters in option
+  struct GraphStyle {
+    int lineColor = -1;
+    int lineStyle = -1;
+    int lineWidth = -1;
+    int markerColor = -1;
+    int markerStyle = -1;
+    float markerSize = -1.f;
+    int fillColor = -1;
+    int fillStyle = -1;
+  };
+
+  // this corresponds to one TTree::Draw() call, i.e. one graph or histogram drawing
+  struct Graph {
+    std::string name;
+    std::string title;
+    std::string varexp;
+    std::string selection;
+    std::string option; // the list of possible options are documented in TGraphPainter and THistPainter
+    std::string errors;
+    GraphStyle style;
+  };
+
+  // legend configuration
+  struct LegendConfig {
+    int nColumns{ 1 };
+    float x1{ -1.f }, y1{ -1.f }, x2{ -1.f }, y2{ -1.f }; // NDC coords
+  };
+
+  // this corresponds to one canvas which can include multiple graphs
+  struct Plot {
+    std::string name;
+    std::string title;
+    std::string graphAxisLabel;
+    std::string graphYRange;
+    int colorPalette = 0;
+    LegendConfig legend;
+    std::vector<Graph> graphs;
+  };
+
+  struct DataSource {
+    std::string type;
+    std::string path;
+    std::string name;
+    std::string reductorName;
+    core::CustomParameters reductorParameters;
+    std::string moduleName;
+  };
+
+  std::vector<Plot> plots;
+  std::vector<DataSource> dataSources;
+
+  // Static parsing methods, usable also when plots and data sources are not siblings in one tree
+  static std::vector<Plot> parsePlots(const boost::property_tree::ptree& plotsTree);
+  static std::vector<DataSource> parseDataSources(const boost::property_tree::ptree& dataSourcesTree);
+  static Graph parseGraph(const boost::property_tree::ptree& graphTree, const std::string& fallbackName = "");
+  static GraphStyle parseGraphStyle(const boost::property_tree::ptree& styleTree);
+  static LegendConfig parseLegend(const boost::property_tree::ptree& legendTree);
+};
+
+} // namespace o2::quality_control::postprocessing
+
+#endif // QUALITYCONTROL_TRENDINGMACHINECONFIG_H

Framework/include/QualityControl/TrendingTask.h

@@ -18,15 +18,12 @@
 #define QUALITYCONTROL_TRENDINGTASK_H
 
 #include "QualityControl/PostProcessingInterface.h"
-#include "QualityControl/Reductor.h"
 #include "QualityControl/TrendingTaskConfig.h"
 
 #include <memory>
-#include <unordered_map>
-#include <TTree.h>
+#include <map>
 
-class TAxis;
-class TCanvas;
+class TObject;
 
 namespace o2::quality_control::repository
 {
@@ -36,6 +33,8 @@ class DatabaseInterface;
 namespace o2::quality_control::postprocessing
 {
 
+class TrendingMachine;
+
 /// \brief  A post-processing task which trends values, stores them in a TTree and produces plots.
 ///
 /// A post-processing task which trends objects inside QC database (QCDB). It extracts some values of one or multiple
@@ -47,46 +46,23 @@ namespace o2::quality_control::postprocessing
 class TrendingTask : public PostProcessingInterface
 {
  public:
-  TrendingTask() = default;
-  ~TrendingTask() override = default;
+  TrendingTask();
+  ~TrendingTask() override;
 
   void configure(const boost::property_tree::ptree& config) override;
   void initialize(Trigger, framework::ServiceRegistryRef) override;
   void update(Trigger, framework::ServiceRegistryRef) override;
   void finalize(Trigger, framework::ServiceRegistryRef) override;
 
  private:
-  static constexpr size_t MaxRunNumberStringLength = 6;
-  struct {
-    // we store run numbers both as an integer and as a string to allow users to select whether they need
-    // a trend in integer or label domain (the latter will contain evenly-spaced data points)
-    Long64_t runNumber = 0;
-    char runNumberStr[MaxRunNumberStringLength + 1] = { 0 }; // 6 characters + null terminator
-    static const char* getBranchLeafList()
-    {
-      return "runNumber/L:runNumberStr/C";
-    }
-  } mMetaData;
-
-  static void setUserAxesLabels(TAxis* xAxis, TAxis* yAxis, const std::string& graphAxesLabels);
-  static void setUserYAxisRange(TH1* hist, const std::string& graphYAxisRange);
-  static void formatTimeXAxis(TH1* background);
-  static void formatRunNumberXAxis(TH1* background);
-  static std::string deduceGraphLegendOptions(const TrendingTaskConfig::Graph& graphConfig);
-  static void applyStyleToGraph(TGraph* graph, const TrendingTaskConfig::GraphStyle& style);
-
   /// returns true only if all datasources were available to update reductor
   bool trendValues(const Trigger& t, repository::DatabaseInterface&);
   void generatePlots();
-  TCanvas* drawPlot(const TrendingTaskConfig::Plot& plotConfig);
   void initializeTrend(repository::DatabaseInterface& qcdb);
-  bool canContinueTrend(TTree* tree);
 
   TrendingTaskConfig mConfig;
-  UInt_t mTime;
-  std::unique_ptr<TTree> mTrend;
+  std::unique_ptr<TrendingMachine> mTrendingMachine;
   std::map<std::string, std::unique_ptr<TObject>> mPlots;
-  std::unordered_map<std::string, std::unique_ptr<Reductor>> mReductors;
 };
 
 } // namespace o2::quality_control::postprocessing