OpenShot Library | libopenshot  0.4.0
Frame.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2019 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include <thread> // for std::this_thread::sleep_for
14 #include <chrono> // for std::chrono::milliseconds
15 #include <iomanip>
16 
17 #include "Frame.h"
18 #include "AudioBufferSource.h"
19 #include "AudioResampler.h"
20 #include "QtUtilities.h"
21 
22 #include <AppConfig.h>
23 #include <juce_audio_basics/juce_audio_basics.h>
24 #include <juce_audio_devices/juce_audio_devices.h>
25 
26 #include <QApplication>
27 #include <QImage>
28 #include <QPixmap>
29 #include <QBitmap>
30 #include <QColor>
31 #include <QString>
32 #include <QVector>
33 #include <QPainter>
34 #include <QHBoxLayout>
35 #include <QWidget>
36 #include <QLabel>
37 #include <QPointF>
38 #include <QWidget>
39 
40 using namespace std;
41 using namespace openshot;
42 
43 // Constructor - image & audio
44 Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
45  : audio(std::make_shared<juce::AudioBuffer<float>>(channels, samples)),
46  number(number), width(width), height(height),
47  pixel_ratio(1,1), color(color),
48  channels(channels), channel_layout(LAYOUT_STEREO),
49  sample_rate(44100),
50  has_audio_data(false), has_image_data(false),
51  max_audio_sample(0)
52 {
53  // zero (fill with silence) the audio buffer
54  audio->clear();
55 }
56 
57 // Delegating Constructor - blank frame
58 Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {}
59 
60 // Delegating Constructor - image only
61 Frame::Frame(int64_t number, int width, int height, std::string color)
62  : Frame::Frame(number, width, height, color, 0, 2) {}
63 
64 // Delegating Constructor - audio only
65 Frame::Frame(int64_t number, int samples, int channels)
66  : Frame::Frame(number, 1, 1, "#000000", samples, channels) {}
67 
68 
69 // Copy constructor
70 Frame::Frame ( const Frame &other )
71 {
72  // copy pointers and data
73  DeepCopy(other);
74 }
75 
76 // Assignment operator
78 {
79  // copy pointers and data
80  DeepCopy(other);
81 
82  return *this;
83 }
84 
85 // Copy data and pointers from another Frame instance
86 void Frame::DeepCopy(const Frame& other)
87 {
88  number = other.number;
89  channels = other.channels;
90  width = other.width;
91  height = other.height;
92  channel_layout = other.channel_layout;
95  sample_rate = other.sample_rate;
96  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
97  color = other.color;
98  max_audio_sample = other.max_audio_sample;
99 
100  if (other.image)
101  image = std::make_shared<QImage>(*(other.image));
102  if (other.audio)
103  audio = std::make_shared<juce::AudioBuffer<float>>(*(other.audio));
104  if (other.wave_image)
105  wave_image = std::make_shared<QImage>(*(other.wave_image));
106 }
107 
108 // Destructor
110  // Clear all pointers
111  image.reset();
112  audio.reset();
113  #ifdef USE_OPENCV
114  imagecv.release();
115  #endif
116 }
117 
118 // Display the frame image to the screen (primarily used for debugging reasons)
120 {
121  if (!QApplication::instance()) {
122  // Only create the QApplication once
123  static int argc = 1;
124  static char* argv[1] = {NULL};
125  previewApp = std::make_shared<QApplication>(argc, argv);
126  }
127 
128  // Get preview image
129  std::shared_ptr<QImage> previewImage = GetImage();
130 
131  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
132  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
133  {
134  // Resize to fix DAR
135  previewImage = std::make_shared<QImage>(previewImage->scaled(
136  previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
137  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
138  }
139 
140  // Create window
141  QWidget previewWindow;
142  previewWindow.setStyleSheet("background-color: #000000;");
143  QHBoxLayout layout;
144 
145  // Create label with current frame's image
146  QLabel previewLabel;
147  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
148  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
149  layout.addWidget(&previewLabel);
150 
151  // Show the window
152  previewWindow.setLayout(&layout);
153  previewWindow.show();
154  previewApp->exec();
155 }
156 
157 // Get an audio waveform image
158 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
159 {
160  // Clear any existing waveform image
161  ClearWaveform();
162 
163  // Init a list of lines
164  QVector<QPointF> lines;
165  QVector<QPointF> labels;
166 
167  // Calculate width of an image based on the # of samples
168  int total_samples = GetAudioSamplesCount();
169  if (total_samples > 0)
170  {
171  // If samples are present...
172  int new_height = 200 * audio->getNumChannels();
173  int height_padding = 20 * (audio->getNumChannels() - 1);
174  int total_height = new_height + height_padding;
175  int total_width = 0;
176  float zero_height = 1.0; // Used to clamp near-zero vales to this value to prevent gaps
177 
178  // Loop through each audio channel
179  float Y = 100.0;
180  for (int channel = 0; channel < audio->getNumChannels(); channel++)
181  {
182  float X = 0.0;
183 
184  // Get audio for this channel
185  const float *samples = audio->getReadPointer(channel);
186 
187  for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
188  {
189  // Sample value (scaled to -100 to 100)
190  float value = samples[sample] * 100.0;
191 
192  // Set threshold near zero (so we don't allow near-zero values)
193  // This prevents empty gaps from appearing in the waveform
194  if (value > -zero_height && value < 0.0) {
195  value = -zero_height;
196  } else if (value > 0.0 && value < zero_height) {
197  value = zero_height;
198  }
199 
200  // Append a line segment for each sample
201  lines.push_back(QPointF(X, Y));
202  lines.push_back(QPointF(X, Y - value));
203  }
204 
205  // Add Channel Label Coordinate
206  labels.push_back(QPointF(5.0, Y - 5.0));
207 
208  // Increment Y
209  Y += (200 + height_padding);
210  total_width = X;
211  }
212 
213  // Create blank image
214  wave_image = std::make_shared<QImage>(
215  total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
216  wave_image->fill(QColor(0,0,0,0));
217 
218  // Load QPainter with wave_image device
219  QPainter painter(wave_image.get());
220 
221  // Set pen color
222  QPen pen;
223  pen.setColor(QColor(Red, Green, Blue, Alpha));
224  pen.setWidthF(1.0);
225  pen.setStyle(Qt::SolidLine);
226  painter.setPen(pen);
227 
228  // Draw the waveform
229  painter.drawLines(lines);
230  painter.end();
231  }
232  else
233  {
234  // No audio samples present
235  wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
236  wave_image->fill(QColor(QString::fromStdString("#000000")));
237  }
238 
239  // Resize Image (if needed)
240  if (wave_image->width() != width || wave_image->height() != height) {
241  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
242  wave_image = std::make_shared<QImage>(scaled_wave_image);
243  }
244 
245  // Return new image
246  return wave_image;
247 }
248 
249 // Clear the waveform image (and deallocate its memory)
251 {
252  if (wave_image)
253  wave_image.reset();
254 }
255 
256 // Get an audio waveform image pixels
257 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
258 {
259  // Get audio wave form image
260  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
261 
262  // Return array of pixel packets
263  return wave_image->constBits();
264 }
265 
266 // Display the wave form
268 {
269  // Get audio wave form image
270  GetWaveform(720, 480, 0, 123, 255, 255);
271 
272  if (!QApplication::instance()) {
273  // Only create the QApplication once
274  static int argc = 1;
275  static char* argv[1] = {NULL};
276  previewApp = std::make_shared<QApplication>(argc, argv);
277  }
278 
279  // Create window
280  QWidget previewWindow;
281  previewWindow.setStyleSheet("background-color: #000000;");
282  QHBoxLayout layout;
283 
284  // Create label with current frame's waveform image
285  QLabel previewLabel;
286  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
287  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
288  layout.addWidget(&previewLabel);
289 
290  // Show the window
291  previewWindow.setLayout(&layout);
292  previewWindow.show();
293  previewApp->exec();
294 
295  // Deallocate waveform image
296  ClearWaveform();
297 }
298 
299 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
300 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
301 {
302  if (channel > 0) {
303  // return average magnitude for a specific channel/sample range
304  return audio->getMagnitude(channel, sample, magnitude_range);
305 
306  } else {
307  // Return average magnitude for all channels
308  return audio->getMagnitude(sample, magnitude_range);
309  }
310 }
311 
312 // Get an array of sample data (and optional reverse the sample values)
313 float* Frame::GetAudioSamples(int channel) {
314 
315  // Copy audio data
316  juce::AudioBuffer<float> *buffer(audio.get());
317 
318  // return JUCE audio data for this channel
319  return buffer->getWritePointer(channel);
320 }
321 
322 // Get an array of sample data (all channels interleaved together), using any sample rate
323 float* Frame::GetInterleavedAudioSamples(int* sample_count)
324 {
325  // Copy audio data
326  juce::AudioBuffer<float> *buffer(audio.get());
327 
328  float *output = NULL;
329  int num_of_channels = audio->getNumChannels();
330  int num_of_samples = GetAudioSamplesCount();
331 
332  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
333  output = new float[num_of_channels * num_of_samples];
334  int position = 0;
335 
336  // Loop through samples in each channel (combining them)
337  for (int sample = 0; sample < num_of_samples; sample++)
338  {
339  for (int channel = 0; channel < num_of_channels; channel++)
340  {
341  // Add sample to output array
342  output[position] = buffer->getReadPointer(channel)[sample];
343 
344  // increment position
345  position++;
346  }
347  }
348 
349  // Update sample count (since it might have changed due to resampling)
350  *sample_count = num_of_samples;
351 
352  // return combined array
353  return output;
354 }
355 
356 // Get number of audio channels
358 {
359  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
360  if (audio)
361  return audio->getNumChannels();
362  else
363  return 0;
364 }
365 
366 // Get number of audio samples
368 {
369  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
370  return max_audio_sample;
371 }
372 
374 {
375  return audio.get();
376 }
377 
378 // Get the size in bytes of this frame (rough estimate)
380 {
381  int64_t total_bytes = 0;
382  if (image) {
383  total_bytes += static_cast<int64_t>(
384  width * height * sizeof(char) * 4);
385  }
386  if (audio) {
387  // approximate audio size (sample rate / 24 fps)
388  total_bytes += (sample_rate / 24.0) * sizeof(float);
389  }
390 
391  // return size of this frame
392  return total_bytes;
393 }
394 
395 // Get pixel data (as packets)
396 const unsigned char* Frame::GetPixels()
397 {
398  // Check for blank image
399  if (!image)
400  // Fill with black
401  AddColor(width, height, color);
402 
403  // Return array of pixel packets
404  return image->constBits();
405 }
406 
407 // Get pixel data (for only a single scan-line)
408 const unsigned char* Frame::GetPixels(int row)
409 {
410  // Check for blank image
411  if (!image)
412  // Fill with black
413  AddColor(width, height, color);
414 
415  // Return array of pixel packets
416  return image->constScanLine(row);
417 }
418 
419 // Check a specific pixel color value (returns True/False)
420 bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
421  int col_pos = col * 4; // Find column array position
422  if (!image || row < 0 || row >= (height - 1) ||
423  col_pos < 0 || col_pos >= (width - 1) ) {
424  // invalid row / col
425  return false;
426  }
427  // Check pixel color
428  const unsigned char* pixels = GetPixels(row);
429  if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
430  pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
431  pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
432  pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
433  // Pixel color matches successfully
434  return true;
435  } else {
436  // Pixel color does not match
437  return false;
438  }
439 }
440 
441 // Set Pixel Aspect Ratio
442 void Frame::SetPixelRatio(int num, int den)
443 {
444  pixel_ratio.num = num;
445  pixel_ratio.den = den;
446 }
447 
448 // Set frame number
449 void Frame::SetFrameNumber(int64_t new_number)
450 {
451  number = new_number;
452 }
453 
454 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
455 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
456 {
457  // Directly return 0 if there are no channels
458  // so that we do not need to deal with NaNs later
459  if (channels == 0) return 0;
460 
461  // Get the total # of samples for the previous frame, and the current frame (rounded)
462  double fps_rate = fps.Reciprocal().ToDouble();
463 
464  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
465  double previous_samples = (sample_rate * fps_rate) * (number - 1);
466  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
467  previous_samples -= previous_samples_remainder;
468 
469  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
470  double total_samples = (sample_rate * fps_rate) * number;
471  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
472  total_samples -= total_samples_remainder;
473 
474  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
475  // be evenly divided into frames, so each frame can have have different # of samples.
476  int samples_per_frame = round(total_samples - previous_samples);
477  if (samples_per_frame < 0)
478  samples_per_frame = 0;
479  return samples_per_frame;
480 }
481 
482 // Calculate the # of samples per video frame (for the current frame number)
483 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
484 {
485  return GetSamplesPerFrame(number, fps, sample_rate, channels);
486 }
487 
488 // Get height of image
490 {
491  return height;
492 }
493 
494 // Get height of image
496 {
497  return width;
498 }
499 
500 // Get the original sample rate of this frame's audio data
502 {
503  return sample_rate;
504 }
505 
506 // Get the original sample rate of this frame's audio data
508 {
509  return channel_layout;
510 }
511 
512 
513 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
514 void Frame::Save(std::string path, float scale, std::string format, int quality)
515 {
516  // Get preview image
517  std::shared_ptr<QImage> previewImage = GetImage();
518 
519  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
520  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
521  {
522  // Resize to fix DAR
523  previewImage = std::make_shared<QImage>(previewImage->scaled(
524  previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
525  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
526  }
527 
528  // scale image if needed
529  if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
530  {
531  // Resize image
532  previewImage = std::make_shared<QImage>(previewImage->scaled(
533  previewImage->size().width() * scale, previewImage->size().height() * scale,
534  Qt::KeepAspectRatio, Qt::SmoothTransformation));
535  }
536 
537  // Save image
538  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
539 }
540 
541 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
542 void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
543  std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
544 
545  // Create blank thumbnail image & fill background color
546  auto thumbnail = std::make_shared<QImage>(
547  new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
548  thumbnail->fill(QColor(QString::fromStdString(background_color)));
549 
550  // Create painter
551  QPainter painter(thumbnail.get());
552  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
553 
554  // Get preview image
555  std::shared_ptr<QImage> previewImage = GetImage();
556 
557  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
558  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
559  {
560  // Calculate correct DAR (display aspect ratio)
561  int aspect_width = previewImage->size().width();
562  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
563 
564  // Resize to fix DAR
565  previewImage = std::make_shared<QImage>(previewImage->scaled(
566  aspect_width, aspect_height,
567  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
568  }
569 
570  // Resize frame image
571  if (ignore_aspect)
572  // Ignore aspect ratio
573  previewImage = std::make_shared<QImage>(previewImage->scaled(
574  new_width, new_height,
575  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
576  else
577  // Maintain aspect ratio
578  previewImage = std::make_shared<QImage>(previewImage->scaled(
579  new_width, new_height,
580  Qt::KeepAspectRatio, Qt::SmoothTransformation));
581 
582  // Composite frame image onto background (centered)
583  int x = (new_width - previewImage->size().width()) / 2.0; // center
584  int y = (new_height - previewImage->size().height()) / 2.0; // center
585  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
586 
587 
588  // Create transform and rotate (if needed)
589  QTransform transform;
590  float origin_x = previewImage->width() / 2.0;
591  float origin_y = previewImage->height() / 2.0;
592  transform.translate(origin_x, origin_y);
593  transform.rotate(rotate);
594  transform.translate(-origin_x,-origin_y);
595  painter.setTransform(transform);
596 
597  // Draw image onto QImage
598  painter.drawImage(x, y, *previewImage);
599 
600 
601  // Overlay Image (if any)
602  if (overlay_path != "") {
603  // Open overlay
604  auto overlay = std::make_shared<QImage>();
605  overlay->load(QString::fromStdString(overlay_path));
606 
607  // Set pixel format
608  overlay = std::make_shared<QImage>(
609  overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
610 
611  // Resize to fit
612  overlay = std::make_shared<QImage>(overlay->scaled(
613  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
614 
615  // Composite onto thumbnail
616  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
617  painter.drawImage(0, 0, *overlay);
618  }
619 
620 
621  // Mask Image (if any)
622  if (mask_path != "") {
623  // Open mask
624  auto mask = std::make_shared<QImage>();
625  mask->load(QString::fromStdString(mask_path));
626 
627  // Set pixel format
628  mask = std::make_shared<QImage>(
629  mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
630 
631  // Resize to fit
632  mask = std::make_shared<QImage>(mask->scaled(
633  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
634 
635  // Negate mask
636  mask->invertPixels();
637 
638  // Get pixels
639  unsigned char *pixels = static_cast<unsigned char *>(thumbnail->bits());
640  const unsigned char *mask_pixels = static_cast<const unsigned char *>(mask->constBits());
641 
642  // Convert the mask image to grayscale
643  // Loop through pixels
644  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
645  {
646  // Get the RGB values from the pixel
647  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
648  int Frame_Alpha = pixels[byte_index + 3];
649  int Mask_Value = constrain(Frame_Alpha - gray_value);
650 
651  // Set all alpha pixels to gray value
652  pixels[byte_index + 3] = Mask_Value;
653  }
654  }
655 
656 
657  // End painter
658  painter.end();
659 
660  // Save image
661  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
662 }
663 
664 // Constrain a color value from 0 to 255
665 int Frame::constrain(int color_value)
666 {
667  // Constrain new color from 0 to 255
668  if (color_value < 0)
669  color_value = 0;
670  else if (color_value > 255)
671  color_value = 255;
672 
673  return color_value;
674 }
675 
676 void Frame::AddColor(int new_width, int new_height, std::string new_color)
677 {
678  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
679  // Update parameters
680  width = new_width;
681  height = new_height;
682  color = new_color;
683  AddColor(QColor(QString::fromStdString(new_color)));
684 }
685 
686 // Add (or replace) pixel data to the frame (based on a solid color)
687 void Frame::AddColor(const QColor& new_color)
688 {
689  // Create new image object, and fill with pixel data
690  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
691  image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
692 
693  // Fill with solid color
694  image->fill(new_color);
695  has_image_data = true;
696 }
697 
698 // Add (or replace) pixel data to the frame
700  int new_width, int new_height, int bytes_per_pixel,
701  QImage::Format type, const unsigned char *pixels_)
702 {
703  if (has_image_data) {
704  // Delete the previous QImage
705  image.reset();
706  }
707 
708  // Create new image object from pixel data
709  auto new_image = std::make_shared<QImage>(
710  pixels_,
711  new_width, new_height,
712  new_width * bytes_per_pixel,
713  type,
714  (QImageCleanupFunction) &openshot::cleanUpBuffer,
715  (void*) pixels_
716  );
717  AddImage(new_image);
718 }
719 
720 // Add (or replace) pixel data to the frame
721 void Frame::AddImage(std::shared_ptr<QImage> new_image)
722 {
723  // Ignore blank images
724  if (!new_image)
725  return;
726 
727  // assign image data
728  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
729  image = new_image;
730 
731  // Always convert to Format_RGBA8888_Premultiplied (if different)
732  if (image->format() != QImage::Format_RGBA8888_Premultiplied)
733  *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
734 
735  // Update height and width
736  width = image->width();
737  height = image->height();
738  has_image_data = true;
739 }
740 
741 // Add (or replace) pixel data to the frame (for only the odd or even lines)
742 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
743 {
744  // Ignore blank new_image
745  if (!new_image)
746  return;
747 
748  // Check for blank source image
749  if (!image) {
750  // Replace the blank source image
751  AddImage(new_image);
752 
753  } else {
754  // Ignore image of different sizes or formats
755  bool ret=false;
756  if (image == new_image || image->size() != new_image->size()) {
757  ret = true;
758  }
759  else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
760  new_image = std::make_shared<QImage>(
761  new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
762  }
763  if (ret) {
764  return;
765  }
766 
767  // Get the frame's image
768  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
769  unsigned char *pixels = image->bits();
770  const unsigned char *new_pixels = new_image->constBits();
771 
772  // Loop through the scanlines of the image (even or odd)
773  int start = 0;
774  if (only_odd_lines)
775  start = 1;
776 
777  for (int row = start; row < image->height(); row += 2) {
778  int offset = row * image->bytesPerLine();
779  memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
780  }
781 
782  // Update height and width
783  height = image->height();
784  width = image->width();
785  has_image_data = true;
786  }
787 }
788 
789 
790 // Resize audio container to hold more (or less) samples and channels
791 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
792 {
793  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
794 
795  // Resize JUCE audio buffer
796  audio->setSize(channels, length, true, true, false);
797  channel_layout = layout;
798  sample_rate = rate;
799 
800  // Calculate max audio sample added
801  max_audio_sample = length;
802 }
803 
804 // Reverse the audio buffer of this frame (will only reverse a single time, regardless of how many times
805 // you invoke this method)
807  if (audio && !audio_reversed) {
808  // Reverse audio buffer
809  audio->reverse(0, audio->getNumSamples());
810  audio_reversed = true;
811  }
812 }
813 
814 // Add audio samples to a specific channel
815 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
816  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
817 
818  // Clamp starting sample to 0
819  int destStartSampleAdjusted = max(destStartSample, 0);
820 
821  // Extend audio container to hold more (or less) samples and channels.. if needed
822  int new_length = destStartSampleAdjusted + numSamples;
823  int new_channel_length = audio->getNumChannels();
824  if (destChannel >= new_channel_length)
825  new_channel_length = destChannel + 1;
826  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
827  audio->setSize(new_channel_length, new_length, true, true, false);
828 
829  // Clear the range of samples first (if needed)
830  if (replaceSamples)
831  audio->clear(destChannel, destStartSampleAdjusted, numSamples);
832 
833  // Add samples to frame's audio buffer
834  audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
835  has_audio_data = true;
836 
837  // Calculate max audio sample added
838  if (new_length > max_audio_sample)
839  max_audio_sample = new_length;
840 
841  // Reset audio reverse flag
842  audio_reversed = false;
843 }
844 
845 // Apply gain ramp (i.e. fading volume)
846 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
847 {
848  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
849 
850  // Apply gain ramp
851  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
852 }
853 
854 // Get pointer to Magick++ image object
855 std::shared_ptr<QImage> Frame::GetImage()
856 {
857  // Check for blank image
858  if (!image)
859  // Fill with black
860  AddColor(width, height, color);
861 
862  return image;
863 }
864 
865 #ifdef USE_OPENCV
866 
867 // Convert Qimage to Mat
868 cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
869 
870  cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
871  cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
872  int from_to[] = { 0,0, 1,1, 2,2 };
873  cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
874  cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
875  return mat2;
876 }
877 
878 // Get pointer to OpenCV image object
880 {
881  // Check for blank image
882  if (!image)
883  // Fill with black
884  AddColor(width, height, color);
885 
886  // if (imagecv.empty())
887  // Convert Qimage to Mat
888  imagecv = Qimage2mat(image);
889 
890  return imagecv;
891 }
892 
893 std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
894  cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
895  QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
896 
897  std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
898 
899  // Always convert to RGBA8888 (if different)
900  if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
901  *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
902 
903  return imgIn;
904 }
905 
906 // Set pointer to OpenCV image object
907 void Frame::SetImageCV(cv::Mat _image)
908 {
909  imagecv = _image;
910  image = Mat2Qimage(_image);
911 }
912 #endif
913 
914 // Play audio samples for this frame
916 {
917  // Check if samples are present
918  if (!GetAudioSamplesCount())
919  return;
920 
921  juce::AudioDeviceManager deviceManager;
922  juce::String error = deviceManager.initialise (
923  0, /* number of input channels */
924  2, /* number of output channels */
925  0, /* no XML settings.. */
926  true /* select default device on failure */);
927 
928  // Output error (if any)
929  if (error.isNotEmpty()) {
930  cout << "Error on initialise(): " << error << endl;
931  }
932 
933  juce::AudioSourcePlayer audioSourcePlayer;
934  deviceManager.addAudioCallback (&audioSourcePlayer);
935 
936  std::unique_ptr<AudioBufferSource> my_source;
937  my_source.reset (new AudioBufferSource (audio.get()));
938 
939  // Create TimeSliceThread for audio buffering
940  juce::TimeSliceThread my_thread("Audio buffer thread");
941 
942  // Start thread
943  my_thread.startThread();
944 
945  juce::AudioTransportSource transport1;
946  transport1.setSource (my_source.get(),
947  5000, // tells it to buffer this many samples ahead
948  &my_thread,
949  (double) sample_rate,
950  audio->getNumChannels()); // sample rate of source
951  transport1.setPosition (0);
952  transport1.setGain(1.0);
953 
954 
955  // Create MIXER
956  juce::MixerAudioSource mixer;
957  mixer.addInputSource(&transport1, false);
958  audioSourcePlayer.setSource (&mixer);
959 
960  // Start transports
961  transport1.start();
962 
963  while (transport1.isPlaying())
964  {
965  cout << "playing" << endl;
966  std::this_thread::sleep_for(std::chrono::seconds(1));
967  }
968 
969  cout << "DONE!!!" << endl;
970 
971  transport1.stop();
972  transport1.setSource (0);
973  audioSourcePlayer.setSource (0);
974  my_thread.stopThread(500);
975  deviceManager.removeAudioCallback (&audioSourcePlayer);
976  deviceManager.closeAudioDevice();
977  deviceManager.removeAllChangeListeners();
978  deviceManager.dispatchPendingMessages();
979 
980  cout << "End of Play()" << endl;
981 
982 
983 }
984 
985 // Add audio silence
986 void Frame::AddAudioSilence(int numSamples)
987 {
988  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
989 
990  // Resize audio container
991  audio->setSize(channels, numSamples, false, true, false);
992  audio->clear();
993  has_audio_data = true;
994 
995  // Calculate max audio sample added
996  max_audio_sample = numSamples;
997 
998  // Reset audio reverse flag
999  audio_reversed = false;
1000 }
openshot::Frame::GetWaveformPixels
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:257
openshot::Frame::SampleRate
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:501
openshot::Frame::operator=
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:77
openshot::Frame::GetAudioSamples
float * GetAudioSamples(int channel)
Get an array of sample data (and optional reverse the sample values)
Definition: Frame.cpp:313
openshot::Frame::SetFrameNumber
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:449
openshot::Frame::has_audio_data
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:117
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
juce::AudioBuffer< float >
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
AudioBufferSource.h
Header file for AudioBufferSource class.
openshot::Frame::Thumbnail
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:542
AudioResampler.h
Header file for AudioResampler class.
openshot::Frame
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:88
QtUtilities.h
Header file for QtUtilities (compatibiity overlay)
openshot::Frame::has_image_data
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:118
openshot::LAYOUT_STEREO
@ LAYOUT_STEREO
Definition: ChannelLayouts.h:31
openshot::Frame::AddAudioSilence
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:986
openshot::Fraction::ToDouble
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
openshot::Frame::GetBytes
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:379
openshot::Frame::ResizeAudio
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:791
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
openshot::Frame::ReverseAudio
void ReverseAudio()
Definition: Frame.cpp:806
openshot::Frame::GetPixels
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:396
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
openshot::Fraction::Reciprocal
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
openshot::Frame::AddImage
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:699
openshot::AudioBufferSource
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
Definition: AudioBufferSource.h:27
juce
Definition: Robotization.h:29
openshot::Frame::ApplyGainRamp
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:846
openshot::Frame::GetAudioChannelsCount
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:357
openshot::Frame::GetHeight
int GetHeight()
Get height of image.
Definition: Frame.cpp:489
path
path
Definition: FFmpegWriter.cpp:1476
Frame.h
Header file for Frame class.
openshot::Frame::GetSamplesPerFrame
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:483
openshot::Frame::GetInterleavedAudioSamples
float * GetInterleavedAudioSamples(int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition: Frame.cpp:323
openshot::Frame::CheckPixel
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:420
openshot::Frame::GetWaveform
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:158
openshot::Frame::AddAudio
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:815
openshot::Frame::GetWidth
int GetWidth()
Get height of image.
Definition: Frame.cpp:495
openshot::Frame::audio
std::shared_ptr< juce::AudioBuffer< float > > audio
Definition: Frame.h:115
openshot::Frame::ClearWaveform
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:250
openshot::Frame::SetPixelRatio
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:442
openshot::Frame::Save
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:514
openshot::Frame::GetImage
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:855
openshot::Frame::GetImageCV
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition: Frame.cpp:879
openshot::Frame::ChannelsLayout
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:507
openshot::Frame::GetAudioSampleBuffer
juce::AudioBuffer< float > * GetAudioSampleBuffer()
Definition: Frame.cpp:373
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
openshot::Frame::Play
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:915
openshot::Frame::DeepCopy
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:86
openshot::Frame::GetAudioSample
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition: Frame.cpp:300
openshot::Frame::number
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:116
openshot::Frame::Mat2Qimage
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition: Frame.cpp:893
openshot::Frame::SetImageCV
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition: Frame.cpp:907
openshot::Frame::Display
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:119
openshot::Frame::~Frame
virtual ~Frame()
Destructor.
Definition: Frame.cpp:109
openshot::Frame::Qimage2mat
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition: Frame.cpp:868
openshot::Frame::Frame
Frame()
Constructor - blank frame.
Definition: Frame.cpp:58
openshot::Frame::GetAudioSamplesCount
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:367
openshot::Frame::AddColor
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:676
openshot::Frame::DisplayWaveform
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:267