OpenShot Library | libopenshot  0.7.0
FrameScope.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2026 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include "FrameScope.h"
14 
15 #include <algorithm>
16 #include <array>
17 #include <cmath>
18 #include <limits>
19 
20 using namespace openshot;
21 
22 namespace {
23 constexpr float kInv255 = 1.0f / 255.0f;
24 constexpr float kVectorscopeUMax = 0.43600f;
25 constexpr float kVectorscopeVMax = 0.61500f;
26 
27 static int clamp_int(int value, int min_value, int max_value) {
28  return std::max(min_value, std::min(max_value, value));
29 }
30 
31 static int byte_bin(float value) {
32  return clamp_int(static_cast<int>(std::round(value * 255.0f)), 0, 255);
33 }
34 
35 static const std::array<float, 256>& inv_alpha_lut() {
36  static const std::array<float, 256> lut = [] {
37  std::array<float, 256> values{};
38  values[0] = 0.0f;
39  for (int i = 1; i < 256; ++i)
40  values[i] = 255.0f / static_cast<float>(i);
41  return values;
42  }();
43  return lut;
44 }
45 
46 static Json::Value json_array_from_vector(const std::vector<int>& values) {
47  Json::Value array(Json::arrayValue);
48  for (size_t i = 0; i < values.size(); ++i)
49  array.append(values[i]);
50  return array;
51 }
52 
53 static Json::Value json_array_from_vector(const std::vector<uint32_t>& values) {
54  Json::Value array(Json::arrayValue);
55  for (size_t i = 0; i < values.size(); ++i)
56  array.append(Json::Value::UInt(values[i]));
57  return array;
58 }
59 
60 static Json::Value json_array_from_vector(const std::vector<float>& values) {
61  Json::Value array(Json::arrayValue);
62  for (size_t i = 0; i < values.size(); ++i)
63  array.append(values[i]);
64  return array;
65 }
66 }
67 
68 FrameScope::FrameScope()
69  : frame(nullptr),
70  waveform_columns(256),
71  audio_buckets(256),
72  vectorscope_size(256),
73  roi_enabled(false),
74  roi_x(0.0f),
75  roi_y(0.0f),
76  roi_width(1.0f),
77  roi_height(1.0f),
78  waveform_bins(256),
79  waveform_column_map_width(0),
80  waveform_column_map_columns(0),
81  json_dirty(true) {
82  reset();
83 }
84 
85 FrameScope::FrameScope(std::shared_ptr<Frame> new_frame, int new_waveform_columns, int new_audio_buckets, int new_vectorscope_size)
86  : frame(new_frame),
87  waveform_columns(std::max(1, new_waveform_columns)),
88  audio_buckets(std::max(1, new_audio_buckets)),
89  vectorscope_size(std::max(1, new_vectorscope_size)),
90  roi_enabled(false),
91  roi_x(0.0f),
92  roi_y(0.0f),
93  roi_width(1.0f),
94  roi_height(1.0f),
95  waveform_bins(256),
96  waveform_column_map_width(0),
97  waveform_column_map_columns(0),
98  json_dirty(true) {
99  analyze();
100 }
101 
102 void FrameScope::reset() {
103  reset_video();
104  reset_audio();
105  json_dirty = true;
106 }
107 
108 void FrameScope::reset_video() {
109  video_present = false;
110  video_width = 0;
111  video_height = 0;
112  avg_luma = 0.0;
113  clipped_shadows = 0;
114  clipped_highlights = 0;
115  clipped_red = 0;
116  clipped_green = 0;
117  clipped_blue = 0;
118  ensure_video_buffers();
119  std::fill(histogram_luma.begin(), histogram_luma.end(), 0u);
120  std::fill(histogram_red.begin(), histogram_red.end(), 0u);
121  std::fill(histogram_green.begin(), histogram_green.end(), 0u);
122  std::fill(histogram_blue.begin(), histogram_blue.end(), 0u);
123  std::fill(waveform_luma.begin(), waveform_luma.end(), 0u);
124  std::fill(waveform_red.begin(), waveform_red.end(), 0u);
125  std::fill(waveform_green.begin(), waveform_green.end(), 0u);
126  std::fill(waveform_blue.begin(), waveform_blue.end(), 0u);
127  std::fill(vectorscope.begin(), vectorscope.end(), 0u);
128  json_dirty = true;
129 }
130 
131 void FrameScope::reset_audio() {
132  audio_present = false;
133  audio_channels = 0;
134  audio_samples = 0;
135  audio_sample_rate = 0;
136  audio_peak.clear();
137  audio_rms.clear();
138  audio_clipped_samples.clear();
139  audio_waveform_min.clear();
140  audio_waveform_max.clear();
141  json_dirty = true;
142 }
143 
144 void FrameScope::ensure_video_buffers() {
145  histogram_luma.resize(256);
146  histogram_red.resize(256);
147  histogram_green.resize(256);
148  histogram_blue.resize(256);
149  waveform_luma.resize(static_cast<size_t>(waveform_columns) * static_cast<size_t>(waveform_bins));
150  waveform_red.resize(static_cast<size_t>(waveform_columns) * static_cast<size_t>(waveform_bins));
151  waveform_green.resize(static_cast<size_t>(waveform_columns) * static_cast<size_t>(waveform_bins));
152  waveform_blue.resize(static_cast<size_t>(waveform_columns) * static_cast<size_t>(waveform_bins));
153  vectorscope.resize(static_cast<size_t>(vectorscope_size) * static_cast<size_t>(vectorscope_size));
154 }
155 
156 void FrameScope::ensure_audio_buffers() {
157  audio_peak.assign(static_cast<size_t>(audio_channels), 0.0f);
158  audio_rms.assign(static_cast<size_t>(audio_channels), 0.0f);
159  audio_clipped_samples.assign(static_cast<size_t>(audio_channels), 0u);
160  audio_waveform_min.assign(static_cast<size_t>(audio_channels), std::vector<float>(static_cast<size_t>(audio_buckets), 0.0f));
161  audio_waveform_max.assign(static_cast<size_t>(audio_channels), std::vector<float>(static_cast<size_t>(audio_buckets), 0.0f));
162 }
163 
164 void FrameScope::rebuild_waveform_column_map(int width) {
165  if (width == waveform_column_map_width && waveform_columns == waveform_column_map_columns &&
166  static_cast<int>(waveform_column_map.size()) == width)
167  return;
168 
169  waveform_column_map.resize(static_cast<size_t>(width));
170  const int waveform_column_limit = waveform_columns - 1;
171  for (int x = 0; x < width; ++x)
172  waveform_column_map[static_cast<size_t>(x)] = clamp_int((x * waveform_columns) / std::max(1, width), 0, waveform_column_limit);
173 
174  waveform_column_map_width = width;
175  waveform_column_map_columns = waveform_columns;
176 }
177 
178 void FrameScope::SetFrame(std::shared_ptr<Frame> new_frame) {
179  frame = new_frame;
180  analyze();
181 }
182 
183 void FrameScope::SetWaveformColumns(int columns) {
184  waveform_columns = std::max(1, columns);
185  reset_video();
186  if (frame)
187  analyze_video();
188 }
189 
190 void FrameScope::SetAudioBuckets(int buckets) {
191  audio_buckets = std::max(1, buckets);
192  reset_audio();
193  if (frame)
194  analyze_audio();
195 }
196 
197 void FrameScope::SetVectorscopeSize(int size) {
198  vectorscope_size = std::max(1, size);
199  reset_video();
200  if (frame)
201  analyze_video();
202 }
203 
204 void FrameScope::SetVideoRegionNormalized(float x, float y, float width, float height) {
205  roi_x = std::max(0.0f, std::min(1.0f, x));
206  roi_y = std::max(0.0f, std::min(1.0f, y));
207  roi_width = std::max(0.0f, std::min(1.0f - roi_x, width));
208  roi_height = std::max(0.0f, std::min(1.0f - roi_y, height));
209  roi_enabled = roi_width > 0.0f && roi_height > 0.0f;
210  reset_video();
211  if (frame)
212  analyze_video();
213 }
214 
215 void FrameScope::ClearVideoRegion() {
216  roi_enabled = false;
217  roi_x = 0.0f;
218  roi_y = 0.0f;
219  roi_width = 1.0f;
220  roi_height = 1.0f;
221  reset_video();
222  if (frame)
223  analyze_video();
224 }
225 
226 void FrameScope::analyze() {
227  reset();
228  if (!frame)
229  return;
230 
231  analyze_video();
232  analyze_audio();
233  json_dirty = true;
234 }
235 
236 void FrameScope::analyze_video() {
237  // Frame images are always QImage::Format_RGBA8888_Premultiplied (enforced
238  // by Frame::AddImage). Pixel byte order is [R=0, G=1, B=2, A=3].
239  std::shared_ptr<QImage> image = frame->GetImage();
240  if (!image || image->isNull())
241  return;
242 
243  video_present = true;
244  const int width = image->width();
245  const int height = image->height();
246  int start_x = 0;
247  int end_x = width;
248  int start_y = 0;
249  int end_y = height;
250  if (roi_enabled) {
251  start_x = clamp_int(static_cast<int>(std::floor(roi_x * width)), 0, width - 1);
252  start_y = clamp_int(static_cast<int>(std::floor(roi_y * height)), 0, height - 1);
253  end_x = clamp_int(static_cast<int>(std::ceil((roi_x + roi_width) * width)), start_x + 1, width);
254  end_y = clamp_int(static_cast<int>(std::ceil((roi_y + roi_height) * height)), start_y + 1, height);
255  }
256  video_width = std::max(1, end_x - start_x);
257  video_height = std::max(1, end_y - start_y);
258  ensure_video_buffers();
259 
260  double luma_sum = 0.0;
261  double pixel_total = 0.0;
262  clipped_shadows = 0;
263  clipped_highlights = 0;
264  clipped_red = 0;
265  clipped_green = 0;
266  clipped_blue = 0;
267 
268  const int bytes_per_line = image->bytesPerLine();
269  const unsigned char* bits = image->constBits();
270  const auto& inv_alpha = inv_alpha_lut();
271  rebuild_waveform_column_map(video_width);
272  const float vectorscope_center = static_cast<float>(vectorscope_size - 1) * 0.5f;
273  const float vectorscope_scale = vectorscope_center;
274 
275  for (int y = start_y; y < end_y; ++y) {
276  const unsigned char* row = bits + (static_cast<size_t>(y) * bytes_per_line);
277  for (int x = start_x; x < end_x; ++x) {
278  const unsigned char* pixel = row + (static_cast<size_t>(x) * 4);
279  const int red = pixel[0]; // RGBA8888: [R=0, G=1, B=2, A=3]
280  const int green = pixel[1];
281  const int blue = pixel[2];
282  const int alpha = pixel[3]; // premultiplied — divided out below
283  if (alpha <= 0)
284  continue;
285 
286  float redf = 0.0f;
287  float greenf = 0.0f;
288  float bluef = 0.0f;
289  if (alpha == 255) {
290  redf = red * kInv255;
291  greenf = green * kInv255;
292  bluef = blue * kInv255;
293  } else {
294  const float unpremultiply = inv_alpha[alpha];
295  redf = std::min(1.0f, (red * unpremultiply) * kInv255);
296  greenf = std::min(1.0f, (green * unpremultiply) * kInv255);
297  bluef = std::min(1.0f, (blue * unpremultiply) * kInv255);
298  }
299  const float luma = (0.299f * redf) + (0.587f * greenf) + (0.114f * bluef);
300 
301  const int luma_idx = byte_bin(luma);
302  const int red_idx = byte_bin(redf);
303  const int green_idx = byte_bin(greenf);
304  const int blue_idx = byte_bin(bluef);
305  const int roi_column = x - start_x;
306  const size_t waveform_offset = static_cast<size_t>(waveform_column_map[static_cast<size_t>(roi_column)]) * static_cast<size_t>(waveform_bins);
307  const float u = -0.14713f * redf - 0.28886f * greenf + 0.43600f * bluef;
308  const float v = 0.61500f * redf - 0.51499f * greenf - 0.10001f * bluef;
309  const float normalized_u = u / kVectorscopeUMax;
310  const float normalized_v = v / kVectorscopeVMax;
311  const int vector_x = clamp_int(static_cast<int>(std::round(vectorscope_center + (normalized_u * vectorscope_scale))), 0, vectorscope_size - 1);
312  const int vector_y = clamp_int(static_cast<int>(std::round(vectorscope_center - (normalized_v * vectorscope_scale))), 0, vectorscope_size - 1);
313  const size_t vector_offset = (static_cast<size_t>(vector_y) * static_cast<size_t>(vectorscope_size)) + static_cast<size_t>(vector_x);
314 
315  histogram_luma[luma_idx]++;
316  histogram_red[red_idx]++;
317  histogram_green[green_idx]++;
318  histogram_blue[blue_idx]++;
319  waveform_luma[waveform_offset + luma_idx]++;
320  waveform_red[waveform_offset + red_idx]++;
321  waveform_green[waveform_offset + green_idx]++;
322  waveform_blue[waveform_offset + blue_idx]++;
323  vectorscope[vector_offset]++;
324 
325  luma_sum += luma;
326  pixel_total += 1.0;
327  if (luma_idx <= 2)
328  clipped_shadows++;
329  if (luma_idx >= 253)
330  clipped_highlights++;
331  if (red_idx >= 253)
332  clipped_red++;
333  if (green_idx >= 253)
334  clipped_green++;
335  if (blue_idx >= 253)
336  clipped_blue++;
337  }
338  }
339 
340  avg_luma = pixel_total > 0.0 ? (luma_sum / pixel_total) : 0.0;
341 }
342 
343 void FrameScope::analyze_audio() {
344  if (!frame->has_audio_data || !frame->audio)
345  return;
346 
347  const int channels = frame->GetAudioChannelsCount();
348  const int samples = frame->GetAudioSamplesCount();
349  if (channels <= 0 || samples <= 0)
350  return;
351 
352  audio_present = true;
353  audio_channels = channels;
354  audio_samples = samples;
355  audio_sample_rate = frame->SampleRate();
356  ensure_audio_buffers();
357  std::vector<double> rms_sums(static_cast<size_t>(channels), 0.0);
358 
359  for (int channel = 0; channel < channels; ++channel) {
360  float* channel_samples = frame->GetAudioSamples(channel);
361  if (!channel_samples)
362  continue;
363 
364  std::fill(audio_waveform_min[channel].begin(), audio_waveform_min[channel].end(), 1.0f);
365  std::fill(audio_waveform_max[channel].begin(), audio_waveform_max[channel].end(), -1.0f);
366 
367  for (int sample = 0; sample < samples; ++sample) {
368  const float value = channel_samples[sample];
369  const float abs_value = std::abs(value);
370  const int bucket = clamp_int((sample * audio_buckets) / std::max(1, samples), 0, audio_buckets - 1);
371 
372  audio_peak[channel] = std::max(audio_peak[channel], abs_value);
373  rms_sums[channel] += static_cast<double>(value) * static_cast<double>(value);
374  if (abs_value >= 0.999f)
375  audio_clipped_samples[channel]++;
376 
377  audio_waveform_min[channel][bucket] = std::min(audio_waveform_min[channel][bucket], value);
378  audio_waveform_max[channel][bucket] = std::max(audio_waveform_max[channel][bucket], value);
379  }
380 
381  for (int bucket = 0; bucket < audio_buckets; ++bucket) {
382  if (audio_waveform_min[channel][bucket] > audio_waveform_max[channel][bucket]) {
383  audio_waveform_min[channel][bucket] = 0.0f;
384  audio_waveform_max[channel][bucket] = 0.0f;
385  }
386  }
387  }
388 
389  for (int channel = 0; channel < channels; ++channel) {
390  audio_rms[channel] = samples > 0 ? static_cast<float>(std::sqrt(rms_sums[channel] / static_cast<double>(samples))) : 0.0f;
391  }
392 }
393 
394 void FrameScope::rebuild_json() const {
395  scope_data = Json::Value(Json::objectValue);
396  scope_data["version"] = 1;
397 
398  Json::Value video(Json::objectValue);
399  video["present"] = video_present;
400  if (video_present) {
401  video["width"] = video_width;
402  video["height"] = video_height;
403 
404  video["summary"] = Json::Value(Json::objectValue);
405  video["summary"]["avg_luma"] = avg_luma;
406  video["summary"]["clipped_shadows"] = clipped_shadows;
407  video["summary"]["clipped_highlights"] = clipped_highlights;
408  video["summary"]["clipped_red"] = clipped_red;
409  video["summary"]["clipped_green"] = clipped_green;
410  video["summary"]["clipped_blue"] = clipped_blue;
411 
412  video["histogram"] = Json::Value(Json::objectValue);
413  video["histogram"]["luma"] = json_array_from_vector(histogram_luma);
414  video["histogram"]["red"] = json_array_from_vector(histogram_red);
415  video["histogram"]["green"] = json_array_from_vector(histogram_green);
416  video["histogram"]["blue"] = json_array_from_vector(histogram_blue);
417 
418  video["waveform"] = Json::Value(Json::objectValue);
419  video["waveform"]["columns"] = waveform_columns;
420  video["waveform"]["bins"] = waveform_bins;
421  video["waveform"]["luma"] = json_array_from_vector(waveform_luma);
422  video["waveform"]["red"] = json_array_from_vector(waveform_red);
423  video["waveform"]["green"] = json_array_from_vector(waveform_green);
424  video["waveform"]["blue"] = json_array_from_vector(waveform_blue);
425 
426  video["vectorscope"] = Json::Value(Json::objectValue);
427  video["vectorscope"]["size"] = vectorscope_size;
428  video["vectorscope"]["density"] = json_array_from_vector(vectorscope);
429  }
430  scope_data["video"] = video;
431 
432  Json::Value audio(Json::objectValue);
433  audio["present"] = audio_present;
434  if (audio_present) {
435  audio["channels"] = audio_channels;
436  audio["samples"] = audio_samples;
437  audio["sample_rate"] = audio_sample_rate;
438 
439  audio["summary"] = Json::Value(Json::objectValue);
440  audio["summary"]["peak"] = json_array_from_vector(audio_peak);
441  audio["summary"]["rms"] = json_array_from_vector(audio_rms);
442  audio["summary"]["clipped_samples"] = json_array_from_vector(audio_clipped_samples);
443 
444  audio["waveform"] = Json::Value(Json::objectValue);
445  audio["waveform"]["buckets"] = audio_buckets;
446  audio["waveform"]["min"] = Json::Value(Json::arrayValue);
447  audio["waveform"]["max"] = Json::Value(Json::arrayValue);
448  for (int channel = 0; channel < audio_channels; ++channel) {
449  audio["waveform"]["min"].append(json_array_from_vector(audio_waveform_min[static_cast<size_t>(channel)]));
450  audio["waveform"]["max"].append(json_array_from_vector(audio_waveform_max[static_cast<size_t>(channel)]));
451  }
452  }
453  scope_data["audio"] = audio;
454  json_dirty = false;
455 }
456 
457 Json::Value FrameScope::JsonValue() const {
458  if (json_dirty)
459  rebuild_json();
460  return scope_data;
461 }
462 
463 std::string FrameScope::Json() const {
464  if (json_dirty)
465  rebuild_json();
466  return scope_data.toStyledString();
467 }
468 
469 std::vector<int> FrameScope::copy_to_int_vector(const std::vector<uint32_t>& values) {
470  std::vector<int> copy(values.size(), 0);
471  const uint32_t max_int = static_cast<uint32_t>(std::numeric_limits<int>::max());
472  for (size_t i = 0; i < values.size(); ++i)
473  copy[i] = static_cast<int>(std::min(values[i], max_int));
474  return copy;
475 }
476 
477 std::vector<float> FrameScope::GetAudioWaveformMin(int channel) const {
478  if (channel < 0 || channel >= static_cast<int>(audio_waveform_min.size()))
479  return std::vector<float>();
480  return audio_waveform_min[static_cast<size_t>(channel)];
481 }
482 
483 std::vector<float> FrameScope::GetAudioWaveformMax(int channel) const {
484  if (channel < 0 || channel >= static_cast<int>(audio_waveform_max.size()))
485  return std::vector<float>();
486  return audio_waveform_max[static_cast<size_t>(channel)];
487 }
openshot::FrameScope::ClearVideoRegion
void ClearVideoRegion()
Clear any video ROI and re-analyze the full frame.
Definition: FrameScope.cpp:215
openshot::FrameScope::SetFrame
void SetFrame(std::shared_ptr< Frame > new_frame)
Replace the current frame and recompute the scope data.
Definition: FrameScope.cpp:178
openshot::FrameScope::SetVectorscopeSize
void SetVectorscopeSize(int size)
Set the vectorscope plane edge length and re-analyze video.
Definition: FrameScope.cpp:197
openshot::FrameScope::FrameScope
FrameScope()
Create an empty scope analyzer with default bucket sizes.
Definition: FrameScope.cpp:68
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: AnimatedCurve.h:24
openshot::FrameScope::Json
std::string Json() const
Return the current scope payload as a JSON string.
Definition: FrameScope.cpp:463
openshot::FrameScope::SetAudioBuckets
void SetAudioBuckets(int buckets)
Set the number of audio buckets and re-analyze.
Definition: FrameScope.cpp:190
openshot::FrameScope::SetWaveformColumns
void SetWaveformColumns(int columns)
Set the number of horizontal waveform columns and re-analyze.
Definition: FrameScope.cpp:183
openshot::FrameScope::GetAudioWaveformMax
std::vector< float > GetAudioWaveformMax(int channel) const
Return one channel of audio waveform maximum values.
Definition: FrameScope.cpp:483
openshot::FrameScope::SetVideoRegionNormalized
void SetVideoRegionNormalized(float x, float y, float width, float height)
Set a normalized ROI for video analysis and re-analyze video.
Definition: FrameScope.cpp:204
FrameScope.h
Header file for FrameScope class.
openshot::FrameScope::JsonValue
Json::Value JsonValue() const
Return the current scope payload as a Json::Value tree.
Definition: FrameScope.cpp:457
openshot::FrameScope::GetAudioWaveformMin
std::vector< float > GetAudioWaveformMin(int channel) const
Return one channel of audio waveform minimum values.
Definition: FrameScope.cpp:477
Generated by   doxygen 1.8.17