OpenShot Library | libopenshot  0.7.0
BeatSync.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2026 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include "BeatSync.h"
14 #include "Exceptions.h"
15 #include "Timeline.h"
16 
17 #include <algorithm>
18 #include <cmath>
19 #include <vector>
20 
21 #include <QImage>
22 
23 #include <AppConfig.h>
24 #include <juce_audio_basics/juce_audio_basics.h>
25 
26 using namespace openshot;
27 
28 namespace {
29  constexpr double PI = 3.14159265358979323846;
30 
31  inline float clampf(float v, float lo, float hi) {
32  return v < lo ? lo : (v > hi ? hi : v);
33  }
34 
35  inline int clampi(int v, int lo, int hi) {
36  return v < lo ? lo : (v > hi ? hi : v);
37  }
38 
39  inline int blend_channel(int low, int high, int inv, int blend) {
40  return (clampi(low, 0, 255) * inv + clampi(high, 0, 255) * blend) >> 8;
41  }
42 
43  // Logarithmic normalized [0,1] → Hz mapping (20–20000 Hz), matching AudioVisualization
44  float normalized_frequency_to_hz(float value) {
45  const float min_hz = 20.0f;
46  const float max_hz = 20000.0f;
47  const float n = clampf(value, 0.0f, 1.0f);
48  return min_hz * std::pow(max_hz / min_hz, n);
49  }
50 
51  float hz_to_normalized_frequency(float hz) {
52  if (hz >= 0.0f && hz <= 1.0f)
53  return hz;
54  const float min_hz = 20.0f;
55  const float max_hz = 20000.0f;
56  hz = clampf(hz, min_hz, max_hz);
57  return clampf(std::log(hz / min_hz) / std::log(max_hz / min_hz), 0.0f, 1.0f);
58  }
59 
60  // Time-domain bandpass energy: difference of two first-order IIR low-pass filters.
61  // Returns linear 0-1 based on RMS + peak of the filtered signal, scaled by gain.
62  // This mirrors the reactive_level() approach in AudioVisualization but adds band filtering.
63  float band_energy(const std::shared_ptr<Frame>& frame, float low_hz, float high_hz, float gain) {
64  const int samples = frame->GetAudioSamplesCount();
65  const int channels = frame->GetAudioChannelsCount();
66  const int sample_rate = std::max(1, frame->SampleRate());
67  if (samples <= 0 || channels <= 0)
68  return 0.0f;
69 
70  const float nyquist = sample_rate * 0.5f;
71  low_hz = clampf(low_hz, 0.0f, nyquist - 1.0f);
72  high_hz = clampf(high_hz, low_hz + 1.0f, nyquist);
73 
74  // First-order IIR coefficients: alpha = dt / (tau + dt), tau = 1/(2*pi*fc)
75  const float dt = 1.0f / sample_rate;
76  const float alpha_hi = dt / (dt + 1.0f / (2.0f * (float)PI * high_hz));
77  const float alpha_lo = low_hz > 1.0f
78  ? dt / (dt + 1.0f / (2.0f * (float)PI * low_hz))
79  : 0.0f; // no high-pass when low_hz is at/near DC
80 
81  auto* buffer = frame->GetAudioSampleBuffer();
82  std::vector<const float*> ch(channels);
83  for (int c = 0; c < channels; ++c)
84  ch[c] = buffer->getReadPointer(c);
85 
86  float lp_hi = 0.0f, lp_lo = 0.0f;
87  double sum_sq = 0.0;
88  float peak = 0.0f;
89  for (int s = 0; s < samples; ++s) {
90  float x = 0.0f;
91  for (int c = 0; c < channels; ++c)
92  x += ch[c][s];
93  x /= channels;
94 
95  lp_hi += alpha_hi * (x - lp_hi); // low-pass at high_hz
96  lp_lo += alpha_lo * (x - lp_lo); // low-pass at low_hz
97  const float filtered = lp_hi - lp_lo; // bandpass = difference
98 
99  const float abs_v = std::fabs(filtered);
100  sum_sq += abs_v * abs_v;
101  peak = std::max(peak, abs_v);
102  }
103 
104  const float rms = std::sqrt((float)(sum_sq / samples));
105  // Combine RMS and peak with the same weighting reactive_level() uses
106  const float combined = std::max(rms * 3.6f, peak * 1.15f);
107  return clampf(combined * gain, 0.0f, 1.0f);
108  }
109 }
110 
111 BeatSync::BeatSync() :
112  low_color((unsigned char)0, (unsigned char)0, (unsigned char)0, (unsigned char)255),
113  high_color((unsigned char)255, (unsigned char)255, (unsigned char)255, (unsigned char)255),
114  intensity(2.0),
115  threshold(0.1),
116  attack_ms(10.0),
117  decay_ms(200.0),
118  frequency_low(0.0),
119  frequency_high(1.0),
120  invert(false),
121  envelope_(0.0f),
122  last_frame_(-1)
123 {
124  init_effect_details();
125 }
126 
127 BeatSync::BeatSync(Color color, Keyframe intensity) :
128  BeatSync()
129 {
130  this->high_color = color;
131  this->intensity = intensity;
132 }
133 
134 void BeatSync::init_effect_details()
135 {
136  InitEffectInfo();
137  info.class_name = "BeatSync";
138  info.name = "Beat Sync";
139  info.description = "Generates an audio-reactive color flash layer, synchronized to the beat.";
140  info.has_audio = false;
141  info.has_video = true;
142 }
143 
144 std::shared_ptr<openshot::Frame> BeatSync::GetFrame(std::shared_ptr<openshot::Frame> frame, int64_t frame_number)
145 {
146  const std::shared_ptr<QImage> frame_image = frame->GetImage();
147  int width = frame_image ? std::max(1, frame_image->width()) : 1;
148  int height = frame_image ? std::max(1, frame_image->height()) : 1;
149  if ((width <= 1 || height <= 1) && ParentTimeline()) {
150  if (Timeline* timeline = dynamic_cast<Timeline*>(ParentTimeline())) {
151  if (timeline->info.width > 1 && timeline->info.height > 1) {
152  width = timeline->info.width;
153  height = timeline->info.height;
154  }
155  }
156  }
157 
158  // Reset envelope on seek (discontinuous frame access)
159  if (last_frame_ >= 0 && std::abs(frame_number - last_frame_) >= 2)
160  envelope_ = 0.0f;
161  last_frame_ = frame_number;
162 
163  // --- Audio energy for the configured frequency band ---
164  const float low_hz = normalized_frequency_to_hz(clampf(frequency_low.GetValue(frame_number), 0.0f, 1.0f));
165  const float high_hz = std::max(low_hz + 1.0f,
166  normalized_frequency_to_hz(clampf(frequency_high.GetValue(frame_number), 0.0f, 1.0f)));
167  const float gain_v = std::max(0.01f, (float)intensity.GetValue(frame_number));
168  const float raw_energy = band_energy(frame, low_hz, high_hz, gain_v);
169 
170  // --- IIR envelope follower ---
171  // Derive per-frame time constant from audio context
172  const int samples = frame->GetAudioSamplesCount();
173  const int sample_rate = std::max(1, frame->SampleRate());
174  const float frame_sec = (samples > 0) ? (float)samples / sample_rate : 1.0f / 24.0f;
175 
176  const float atk = clampf((float)attack_ms.GetValue(frame_number), 1.0f, 5000.0f);
177  const float dec = clampf((float)decay_ms.GetValue(frame_number), 1.0f, 5000.0f);
178  const float atk_coef = std::exp(-frame_sec / (atk * 0.001f));
179  const float dec_coef = std::exp(-frame_sec / (dec * 0.001f));
180 
181  if (raw_energy > envelope_)
182  envelope_ = atk_coef * envelope_ + (1.0f - atk_coef) * raw_energy;
183  else
184  envelope_ = dec_coef * envelope_ + (1.0f - dec_coef) * raw_energy;
185 
186  // --- Apply threshold and response curve ---
187  const float thr = clampf((float)threshold.GetValue(frame_number), 0.0f, 1.0f);
188  float energy = envelope_;
189  if (energy <= thr) {
190  energy = 0.0f;
191  } else {
192  energy = clampf((energy - thr) / std::max(0.001f, 1.0f - thr), 0.0f, 1.0f);
193  }
194  if (invert)
195  energy = 1.0f - energy;
196 
197  const float response = clampf(response_curve.Sample(energy, frame_number), 0.0f, 1.0f);
198  const int blend = clampi(static_cast<int>(response * 256.0f + 0.5f), 0, 256);
199  const int inv = 256 - blend;
200  auto out = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
201  out->fill(QColor(
202  blend_channel(low_color.red.GetInt(frame_number), high_color.red.GetInt(frame_number), inv, blend),
203  blend_channel(low_color.green.GetInt(frame_number), high_color.green.GetInt(frame_number), inv, blend),
204  blend_channel(low_color.blue.GetInt(frame_number), high_color.blue.GetInt(frame_number), inv, blend),
205  blend_channel(low_color.alpha.GetInt(frame_number), high_color.alpha.GetInt(frame_number), inv, blend)));
206 
207  frame->AddImage(out);
208  return frame;
209 }
210 
211 std::string BeatSync::Json() const {
212  return JsonValue().toStyledString();
213 }
214 
215 Json::Value BeatSync::JsonValue() const {
216  Json::Value root = EffectBase::JsonValue();
217  root["type"] = info.class_name;
218  root["low_color"] = low_color.JsonValue();
219  root["high_color"] = high_color.JsonValue();
220  root["intensity"] = intensity.JsonValue();
221  root["threshold"] = threshold.JsonValue();
222  root["attack_ms"] = attack_ms.JsonValue();
223  root["decay_ms"] = decay_ms.JsonValue();
224  root["frequency_low"] = frequency_low.JsonValue();
225  root["frequency_high"] = frequency_high.JsonValue();
226  root["invert"] = invert;
227  root["response_curve"] = response_curve.JsonValue();
228  return root;
229 }
230 
231 void BeatSync::SetJson(const std::string value) {
232  try {
233  const Json::Value root = openshot::stringToJson(value);
234  SetJsonValue(root);
235  } catch (const std::exception& e) {
236  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
237  }
238 }
239 
240 void BeatSync::SetJsonValue(const Json::Value root) {
241  EffectBase::SetJsonValue(root);
242  if (!root["low_color"].isNull()) low_color.SetJsonValue(root["low_color"]);
243  if (!root["high_color"].isNull()) high_color.SetJsonValue(root["high_color"]);
244  if (!root["color"].isNull()) high_color.SetJsonValue(root["color"]);
245  if (!root["intensity"].isNull()) intensity.SetJsonValue(root["intensity"]);
246  if (!root["threshold"].isNull()) threshold.SetJsonValue(root["threshold"]);
247  if (!root["attack_ms"].isNull()) attack_ms.SetJsonValue(root["attack_ms"]);
248  if (!root["decay_ms"].isNull()) decay_ms.SetJsonValue(root["decay_ms"]);
249  if (!root["frequency_low"].isNull()) frequency_low.SetJsonValue(root["frequency_low"]);
250  if (!root["frequency_high"].isNull()) frequency_high.SetJsonValue(root["frequency_high"]);
251  if (!root["invert"].isNull()) invert = root["invert"].asBool();
252  if (!root["response_curve"].isNull()) response_curve.SetJsonValue(root["response_curve"]);
253 }
254 
255 std::string BeatSync::PropertiesJSON(int64_t requested_frame) const {
256  Json::Value root = BasePropertiesJSON(requested_frame);
257 
258  root["low_color"] = add_property_json("Low Color", 0.0, "color", "", &low_color.red, 0, 255, false, requested_frame);
259  root["low_color"]["red"] = add_property_json("Red", low_color.red.GetValue(requested_frame), "float", "", &low_color.red, 0, 255, false, requested_frame);
260  root["low_color"]["green"] = add_property_json("Green", low_color.green.GetValue(requested_frame), "float", "", &low_color.green, 0, 255, false, requested_frame);
261  root["low_color"]["blue"] = add_property_json("Blue", low_color.blue.GetValue(requested_frame), "float", "", &low_color.blue, 0, 255, false, requested_frame);
262  root["low_color"]["alpha"] = add_property_json("Alpha", low_color.alpha.GetValue(requested_frame), "float", "", &low_color.alpha, 0, 255, false, requested_frame);
263 
264  root["high_color"] = add_property_json("High Color", 0.0, "color", "", &high_color.red, 0, 255, false, requested_frame);
265  root["high_color"]["red"] = add_property_json("Red", high_color.red.GetValue(requested_frame), "float", "", &high_color.red, 0, 255, false, requested_frame);
266  root["high_color"]["green"] = add_property_json("Green", high_color.green.GetValue(requested_frame), "float", "", &high_color.green, 0, 255, false, requested_frame);
267  root["high_color"]["blue"] = add_property_json("Blue", high_color.blue.GetValue(requested_frame), "float", "", &high_color.blue, 0, 255, false, requested_frame);
268  root["high_color"]["alpha"] = add_property_json("Alpha", high_color.alpha.GetValue(requested_frame), "float", "", &high_color.alpha, 0, 255, false, requested_frame);
269 
270  root["intensity"] = add_property_json("Intensity", intensity.GetValue(requested_frame), "float", "", &intensity, 0.0, 10.0, false, requested_frame);
271  root["threshold"] = add_property_json("Threshold", threshold.GetValue(requested_frame), "float", "", &threshold, 0.0, 1.0, false, requested_frame);
272  root["attack_ms"] = add_property_json("Attack (ms)", attack_ms.GetValue(requested_frame), "float", "", &attack_ms, 1.0, 500.0, false, requested_frame);
273  root["decay_ms"] = add_property_json("Decay (ms)", decay_ms.GetValue(requested_frame), "float", "", &decay_ms, 1.0, 2000.0, false, requested_frame);
274 
275  root["frequency_low"] = add_property_json("Low Frequency", hz_to_normalized_frequency(frequency_low.GetValue(requested_frame)), "float", "Normalized frequency floor: 0 = 20 Hz, 1 = 20 kHz", &frequency_low, 0.0, 1.0, false, requested_frame);
276  root["frequency_high"] = add_property_json("High Frequency", hz_to_normalized_frequency(frequency_high.GetValue(requested_frame)), "float", "Normalized frequency ceiling: 0 = 20 Hz, 1 = 20 kHz", &frequency_high, 0.0, 1.0, false, requested_frame);
277 
278  root["invert"] = add_property_json("Invert", invert ? 1.0 : 0.0, "int", "", NULL, 0, 1, false, requested_frame);
279  root["invert"]["choices"].append(add_property_choice_json("No", 0, invert ? 1 : 0));
280  root["invert"]["choices"].append(add_property_choice_json("Yes", 1, invert ? 1 : 0));
281 
282  root["response_curve"] = add_property_json("Response Curve", 0.0, "colorgrade_curve", response_curve.Summary(requested_frame), NULL, 0.0, 1.0, false, requested_frame);
283  root["response_curve"]["curve"] = response_curve.JsonValue();
284  root["response_curve"]["channel"] = "all";
285  root["response_curve"]["summary"] = response_curve.Summary(requested_frame);
286 
287  return root.toStyledString();
288 }
openshot::ClipBase::add_property_json
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
Definition: ClipBase.cpp:96
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::ClipBase::timeline
openshot::TimelineBase * timeline
Pointer to the parent timeline instance (if any)
Definition: ClipBase.h:40
openshot::EffectBase::info
EffectInfoStruct info
Information about the current effect.
Definition: EffectBase.h:110
openshot::BeatSync::frequency_high
Keyframe frequency_high
Definition: BeatSync.h:43
openshot::AnimatedCurve::Sample
float Sample(float input, int64_t frame_number) const
Definition: AnimatedCurve.cpp:128
openshot::BeatSync::JsonValue
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: BeatSync.cpp:215
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: AnimatedCurve.h:24
openshot::BeatSync::attack_ms
Keyframe attack_ms
Definition: BeatSync.h:40
openshot::ClipBase::add_property_choice_json
Json::Value add_property_choice_json(std::string name, int value, int selected_value) const
Generate JSON choice for a property (dropdown properties)
Definition: ClipBase.cpp:132
openshot::EffectBase::JsonValue
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: EffectBase.cpp:96
Timeline.h
Header file for Timeline class.
openshot::AnimatedCurve::SetJsonValue
void SetJsonValue(const Json::Value &root)
Definition: AnimatedCurve.cpp:165
openshot::Keyframe::SetJsonValue
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: KeyFrame.cpp:372
openshot::Keyframe::JsonValue
Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: KeyFrame.cpp:339
openshot::BeatSync::response_curve
AnimatedCurve response_curve
Definition: BeatSync.h:45
openshot::Color
This class represents a color (used on the timeline and clips)
Definition: Color.h:27
openshot::EffectBase::BasePropertiesJSON
Json::Value BasePropertiesJSON(int64_t requested_frame) const
Generate JSON object of base properties (recommended to be used by all effects)
Definition: EffectBase.cpp:236
openshot::BeatSync::high_color
Color high_color
Definition: BeatSync.h:37
openshot::Keyframe
A Keyframe is a collection of Point instances, which is used to vary a number or property over time.
Definition: KeyFrame.h:53
openshot::Color::SetJsonValue
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: Color.cpp:117
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:223
openshot::Timeline
This class represents a timeline.
Definition: Timeline.h:153
openshot::BeatSync::BeatSync
BeatSync()
Definition: BeatSync.cpp:111
openshot::EffectBase::InitEffectInfo
void InitEffectInfo()
Definition: EffectBase.cpp:37
openshot::Color::green
openshot::Keyframe green
Curve representing the green value (0 - 255)
Definition: Color.h:31
openshot::EffectInfoStruct::has_audio
bool has_audio
Determines if this effect manipulates the audio of a frame.
Definition: EffectBase.h:44
openshot::BeatSync::threshold
Keyframe threshold
Definition: BeatSync.h:39
openshot::BeatSync
Definition: BeatSync.h:28
openshot::AnimatedCurve::JsonValue
Json::Value JsonValue() const
Definition: AnimatedCurve.cpp:148
openshot::BeatSync::intensity
Keyframe intensity
Definition: BeatSync.h:38
openshot::BeatSync::PropertiesJSON
std::string PropertiesJSON(int64_t requested_frame) const override
Definition: BeatSync.cpp:255
openshot::EffectInfoStruct::class_name
std::string class_name
The class name of the effect.
Definition: EffectBase.h:39
openshot::Color::JsonValue
Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: Color.cpp:86
openshot::Keyframe::GetInt
int GetInt(int64_t index) const
Get the rounded INT value at a specific index.
Definition: KeyFrame.cpp:282
openshot::EffectInfoStruct::description
std::string description
The description of this effect and what it does.
Definition: EffectBase.h:41
openshot::BeatSync::SetJsonValue
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: BeatSync.cpp:240
openshot::EffectInfoStruct::has_video
bool has_video
Determines if this effect manipulates the image of a frame.
Definition: EffectBase.h:43
BeatSync.h
Header file for BeatSync effect class.
openshot::BeatSync::decay_ms
Keyframe decay_ms
Definition: BeatSync.h:41
openshot::BeatSync::frequency_low
Keyframe frequency_low
Definition: BeatSync.h:42
openshot::BeatSync::low_color
Color low_color
Definition: BeatSync.h:36
openshot::BeatSync::Json
std::string Json() const override
Generate JSON string of this object.
Definition: BeatSync.cpp:211
openshot::EffectInfoStruct::name
std::string name
The name of the effect.
Definition: EffectBase.h:40
openshot::Color::alpha
openshot::Keyframe alpha
Curve representing the alpha value (0 - 255)
Definition: Color.h:33
openshot::AnimatedCurve::Summary
std::string Summary(int64_t frame_number) const
Definition: AnimatedCurve.cpp:136
openshot::Color::red
openshot::Keyframe red
Curve representing the red value (0 - 255)
Definition: Color.h:30
openshot::BeatSync::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
This method is required for all derived classes of ClipBase, and returns a new openshot::Frame object...
Definition: BeatSync.h:50
openshot::BeatSync::invert
bool invert
Definition: BeatSync.h:44
openshot::Color::blue
openshot::Keyframe blue
Curve representing the red value (0 - 255)
Definition: Color.h:32
openshot::BeatSync::SetJson
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: BeatSync.cpp:231
Exceptions.h
Header file for all Exception classes.
openshot::EffectBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: EffectBase.cpp:139
openshot::Keyframe::GetValue
double GetValue(int64_t index) const
Get the value at a specific index.
Definition: KeyFrame.cpp:258
Generated by   doxygen 1.8.17