OpenShot Library | libopenshot  0.3.3
Frame.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2019 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include <thread> // for std::this_thread::sleep_for
14 #include <chrono> // for std::chrono::milliseconds
15 #include <sstream>
16 #include <iomanip>
17 
18 #include "Frame.h"
19 #include "AudioBufferSource.h"
20 #include "AudioResampler.h"
21 #include "QtUtilities.h"
22 
23 #include <AppConfig.h>
24 #include <juce_audio_basics/juce_audio_basics.h>
25 #include <juce_audio_devices/juce_audio_devices.h>
26 
27 #include <QApplication>
28 #include <QImage>
29 #include <QPixmap>
30 #include <QBitmap>
31 #include <QColor>
32 #include <QString>
33 #include <QVector>
34 #include <QPainter>
35 #include <QHBoxLayout>
36 #include <QWidget>
37 #include <QLabel>
38 #include <QPointF>
39 #include <QWidget>
40 
41 using namespace std;
42 using namespace openshot;
43 
44 // Constructor - image & audio
45 Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels)
46  : audio(std::make_shared<juce::AudioBuffer<float>>(channels, samples)),
47  number(number), width(width), height(height),
48  pixel_ratio(1,1), color(color),
49  channels(channels), channel_layout(LAYOUT_STEREO),
50  sample_rate(44100),
51  has_audio_data(false), has_image_data(false),
52  max_audio_sample(0)
53 {
54  // zero (fill with silence) the audio buffer
55  audio->clear();
56 }
57 
58 // Delegating Constructor - blank frame
59 Frame::Frame() : Frame::Frame(1, 1, 1, "#000000", 0, 2) {}
60 
61 // Delegating Constructor - image only
62 Frame::Frame(int64_t number, int width, int height, std::string color)
63  : Frame::Frame(number, width, height, color, 0, 2) {}
64 
65 // Delegating Constructor - audio only
66 Frame::Frame(int64_t number, int samples, int channels)
67  : Frame::Frame(number, 1, 1, "#000000", samples, channels) {}
68 
69 
70 // Copy constructor
71 Frame::Frame ( const Frame &other )
72 {
73  // copy pointers and data
74  DeepCopy(other);
75 }
76 
77 // Assignment operator
79 {
80  // copy pointers and data
81  DeepCopy(other);
82 
83  return *this;
84 }
85 
86 // Copy data and pointers from another Frame instance
87 void Frame::DeepCopy(const Frame& other)
88 {
89  number = other.number;
90  channels = other.channels;
91  width = other.width;
92  height = other.height;
93  channel_layout = other.channel_layout;
96  sample_rate = other.sample_rate;
97  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
98  color = other.color;
99  max_audio_sample = other.max_audio_sample;
100 
101  if (other.image)
102  image = std::make_shared<QImage>(*(other.image));
103  if (other.audio)
104  audio = std::make_shared<juce::AudioBuffer<float>>(*(other.audio));
105  if (other.wave_image)
106  wave_image = std::make_shared<QImage>(*(other.wave_image));
107 }
108 
109 // Destructor
111  // Clear all pointers
112  image.reset();
113  audio.reset();
114  #ifdef USE_OPENCV
115  imagecv.release();
116  #endif
117 }
118 
119 // Display the frame image to the screen (primarily used for debugging reasons)
121 {
122  if (!QApplication::instance()) {
123  // Only create the QApplication once
124  static int argc = 1;
125  static char* argv[1] = {NULL};
126  previewApp = std::make_shared<QApplication>(argc, argv);
127  }
128 
129  // Get preview image
130  std::shared_ptr<QImage> previewImage = GetImage();
131 
132  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
133  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
134  {
135  // Resize to fix DAR
136  previewImage = std::make_shared<QImage>(previewImage->scaled(
137  previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
138  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
139  }
140 
141  // Create window
142  QWidget previewWindow;
143  previewWindow.setStyleSheet("background-color: #000000;");
144  QHBoxLayout layout;
145 
146  // Create label with current frame's image
147  QLabel previewLabel;
148  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
149  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
150  layout.addWidget(&previewLabel);
151 
152  // Show the window
153  previewWindow.setLayout(&layout);
154  previewWindow.show();
155  previewApp->exec();
156 }
157 
158 // Get an audio waveform image
159 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
160 {
161  // Clear any existing waveform image
162  ClearWaveform();
163 
164  // Init a list of lines
165  QVector<QPointF> lines;
166  QVector<QPointF> labels;
167 
168  // Calculate width of an image based on the # of samples
169  int total_samples = GetAudioSamplesCount();
170  if (total_samples > 0)
171  {
172  // If samples are present...
173  int new_height = 200 * audio->getNumChannels();
174  int height_padding = 20 * (audio->getNumChannels() - 1);
175  int total_height = new_height + height_padding;
176  int total_width = 0;
177  float zero_height = 1.0; // Used to clamp near-zero vales to this value to prevent gaps
178 
179  // Loop through each audio channel
180  float Y = 100.0;
181  for (int channel = 0; channel < audio->getNumChannels(); channel++)
182  {
183  float X = 0.0;
184 
185  // Get audio for this channel
186  const float *samples = audio->getReadPointer(channel);
187 
188  for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
189  {
190  // Sample value (scaled to -100 to 100)
191  float value = samples[sample] * 100.0;
192 
193  // Set threshold near zero (so we don't allow near-zero values)
194  // This prevents empty gaps from appearing in the waveform
195  if (value > -zero_height && value < 0.0) {
196  value = -zero_height;
197  } else if (value > 0.0 && value < zero_height) {
198  value = zero_height;
199  }
200 
201  // Append a line segment for each sample
202  lines.push_back(QPointF(X, Y));
203  lines.push_back(QPointF(X, Y - value));
204  }
205 
206  // Add Channel Label Coordinate
207  labels.push_back(QPointF(5.0, Y - 5.0));
208 
209  // Increment Y
210  Y += (200 + height_padding);
211  total_width = X;
212  }
213 
214  // Create blank image
215  wave_image = std::make_shared<QImage>(
216  total_width, total_height, QImage::Format_RGBA8888_Premultiplied);
217  wave_image->fill(QColor(0,0,0,0));
218 
219  // Load QPainter with wave_image device
220  QPainter painter(wave_image.get());
221 
222  // Set pen color
223  QPen pen;
224  pen.setColor(QColor(Red, Green, Blue, Alpha));
225  pen.setWidthF(1.0);
226  pen.setStyle(Qt::SolidLine);
227  painter.setPen(pen);
228 
229  // Draw the waveform
230  painter.drawLines(lines);
231  painter.end();
232  }
233  else
234  {
235  // No audio samples present
236  wave_image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
237  wave_image->fill(QColor(QString::fromStdString("#000000")));
238  }
239 
240  // Resize Image (if needed)
241  if (wave_image->width() != width || wave_image->height() != height) {
242  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
243  wave_image = std::make_shared<QImage>(scaled_wave_image);
244  }
245 
246  // Return new image
247  return wave_image;
248 }
249 
250 // Clear the waveform image (and deallocate its memory)
252 {
253  if (wave_image)
254  wave_image.reset();
255 }
256 
257 // Get an audio waveform image pixels
258 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
259 {
260  // Get audio wave form image
261  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
262 
263  // Return array of pixel packets
264  return wave_image->constBits();
265 }
266 
267 // Display the wave form
269 {
270  // Get audio wave form image
271  GetWaveform(720, 480, 0, 123, 255, 255);
272 
273  if (!QApplication::instance()) {
274  // Only create the QApplication once
275  static int argc = 1;
276  static char* argv[1] = {NULL};
277  previewApp = std::make_shared<QApplication>(argc, argv);
278  }
279 
280  // Create window
281  QWidget previewWindow;
282  previewWindow.setStyleSheet("background-color: #000000;");
283  QHBoxLayout layout;
284 
285  // Create label with current frame's waveform image
286  QLabel previewLabel;
287  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
288  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
289  layout.addWidget(&previewLabel);
290 
291  // Show the window
292  previewWindow.setLayout(&layout);
293  previewWindow.show();
294  previewApp->exec();
295 
296  // Deallocate waveform image
297  ClearWaveform();
298 }
299 
300 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
301 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
302 {
303  if (channel > 0) {
304  // return average magnitude for a specific channel/sample range
305  return audio->getMagnitude(channel, sample, magnitude_range);
306 
307  } else {
308  // Return average magnitude for all channels
309  return audio->getMagnitude(sample, magnitude_range);
310  }
311 }
312 
313 // Get an array of sample data (and optional reverse the sample values)
314 float* Frame::GetAudioSamples(int channel) {
315 
316  // Copy audio data
317  juce::AudioBuffer<float> *buffer(audio.get());
318 
319  // return JUCE audio data for this channel
320  return buffer->getWritePointer(channel);
321 }
322 
323 // Get an array of sample data (all channels interleaved together), using any sample rate
324 float* Frame::GetInterleavedAudioSamples(int* sample_count)
325 {
326  // Copy audio data
327  juce::AudioBuffer<float> *buffer(audio.get());
328 
329  float *output = NULL;
330  int num_of_channels = audio->getNumChannels();
331  int num_of_samples = GetAudioSamplesCount();
332 
333  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
334  output = new float[num_of_channels * num_of_samples];
335  int position = 0;
336 
337  // Loop through samples in each channel (combining them)
338  for (int sample = 0; sample < num_of_samples; sample++)
339  {
340  for (int channel = 0; channel < num_of_channels; channel++)
341  {
342  // Add sample to output array
343  output[position] = buffer->getReadPointer(channel)[sample];
344 
345  // increment position
346  position++;
347  }
348  }
349 
350  // Update sample count (since it might have changed due to resampling)
351  *sample_count = num_of_samples;
352 
353  // return combined array
354  return output;
355 }
356 
357 // Get number of audio channels
359 {
360  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
361  if (audio)
362  return audio->getNumChannels();
363  else
364  return 0;
365 }
366 
367 // Get number of audio samples
369 {
370  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
371  return max_audio_sample;
372 }
373 
375 {
376  return audio.get();
377 }
378 
379 // Get the size in bytes of this frame (rough estimate)
381 {
382  int64_t total_bytes = 0;
383  if (image) {
384  total_bytes += static_cast<int64_t>(
385  width * height * sizeof(char) * 4);
386  }
387  if (audio) {
388  // approximate audio size (sample rate / 24 fps)
389  total_bytes += (sample_rate / 24.0) * sizeof(float);
390  }
391 
392  // return size of this frame
393  return total_bytes;
394 }
395 
396 // Get pixel data (as packets)
397 const unsigned char* Frame::GetPixels()
398 {
399  // Check for blank image
400  if (!image)
401  // Fill with black
402  AddColor(width, height, color);
403 
404  // Return array of pixel packets
405  return image->constBits();
406 }
407 
408 // Get pixel data (for only a single scan-line)
409 const unsigned char* Frame::GetPixels(int row)
410 {
411  // Check for blank image
412  if (!image)
413  // Fill with black
414  AddColor(width, height, color);
415 
416  // Return array of pixel packets
417  return image->constScanLine(row);
418 }
419 
420 // Check a specific pixel color value (returns True/False)
421 bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) {
422  int col_pos = col * 4; // Find column array position
423  if (!image || row < 0 || row >= (height - 1) ||
424  col_pos < 0 || col_pos >= (width - 1) ) {
425  // invalid row / col
426  return false;
427  }
428  // Check pixel color
429  const unsigned char* pixels = GetPixels(row);
430  if (pixels[col_pos + 0] >= (red - threshold) && pixels[col_pos + 0] <= (red + threshold) &&
431  pixels[col_pos + 1] >= (green - threshold) && pixels[col_pos + 1] <= (green + threshold) &&
432  pixels[col_pos + 2] >= (blue - threshold) && pixels[col_pos + 2] <= (blue + threshold) &&
433  pixels[col_pos + 3] >= (alpha - threshold) && pixels[col_pos + 3] <= (alpha + threshold)) {
434  // Pixel color matches successfully
435  return true;
436  } else {
437  // Pixel color does not match
438  return false;
439  }
440 }
441 
442 // Set Pixel Aspect Ratio
443 void Frame::SetPixelRatio(int num, int den)
444 {
445  pixel_ratio.num = num;
446  pixel_ratio.den = den;
447 }
448 
449 // Set frame number
450 void Frame::SetFrameNumber(int64_t new_number)
451 {
452  number = new_number;
453 }
454 
455 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
456 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
457 {
458  // Directly return 0 if there are no channels
459  // so that we do not need to deal with NaNs later
460  if (channels == 0) return 0;
461 
462  // Get the total # of samples for the previous frame, and the current frame (rounded)
463  double fps_rate = fps.Reciprocal().ToDouble();
464 
465  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
466  double previous_samples = (sample_rate * fps_rate) * (number - 1);
467  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
468  previous_samples -= previous_samples_remainder;
469 
470  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
471  double total_samples = (sample_rate * fps_rate) * number;
472  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
473  total_samples -= total_samples_remainder;
474 
475  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
476  // be evenly divided into frames, so each frame can have have different # of samples.
477  int samples_per_frame = round(total_samples - previous_samples);
478  if (samples_per_frame < 0)
479  samples_per_frame = 0;
480  return samples_per_frame;
481 }
482 
483 // Calculate the # of samples per video frame (for the current frame number)
484 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
485 {
486  return GetSamplesPerFrame(number, fps, sample_rate, channels);
487 }
488 
489 // Get height of image
491 {
492  return height;
493 }
494 
495 // Get height of image
497 {
498  return width;
499 }
500 
501 // Get the original sample rate of this frame's audio data
503 {
504  return sample_rate;
505 }
506 
507 // Get the original sample rate of this frame's audio data
509 {
510  return channel_layout;
511 }
512 
513 
514 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
515 void Frame::Save(std::string path, float scale, std::string format, int quality)
516 {
517  // Get preview image
518  std::shared_ptr<QImage> previewImage = GetImage();
519 
520  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-square pixels)
521  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
522  {
523  // Resize to fix DAR
524  previewImage = std::make_shared<QImage>(previewImage->scaled(
525  previewImage->size().width(), previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble(),
526  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
527  }
528 
529  // scale image if needed
530  if (fabs(scale) > 1.001 || fabs(scale) < 0.999)
531  {
532  // Resize image
533  previewImage = std::make_shared<QImage>(previewImage->scaled(
534  previewImage->size().width() * scale, previewImage->size().height() * scale,
535  Qt::KeepAspectRatio, Qt::SmoothTransformation));
536  }
537 
538  // Save image
539  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
540 }
541 
542 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
543 void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path,
544  std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate) {
545 
546  // Create blank thumbnail image & fill background color
547  auto thumbnail = std::make_shared<QImage>(
548  new_width, new_height, QImage::Format_RGBA8888_Premultiplied);
549  thumbnail->fill(QColor(QString::fromStdString(background_color)));
550 
551  // Create painter
552  QPainter painter(thumbnail.get());
553  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
554 
555  // Get preview image
556  std::shared_ptr<QImage> previewImage = GetImage();
557 
558  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
559  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
560  {
561  // Calculate correct DAR (display aspect ratio)
562  int aspect_width = previewImage->size().width();
563  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
564 
565  // Resize to fix DAR
566  previewImage = std::make_shared<QImage>(previewImage->scaled(
567  aspect_width, aspect_height,
568  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
569  }
570 
571  // Resize frame image
572  if (ignore_aspect)
573  // Ignore aspect ratio
574  previewImage = std::make_shared<QImage>(previewImage->scaled(
575  new_width, new_height,
576  Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
577  else
578  // Maintain aspect ratio
579  previewImage = std::make_shared<QImage>(previewImage->scaled(
580  new_width, new_height,
581  Qt::KeepAspectRatio, Qt::SmoothTransformation));
582 
583  // Composite frame image onto background (centered)
584  int x = (new_width - previewImage->size().width()) / 2.0; // center
585  int y = (new_height - previewImage->size().height()) / 2.0; // center
586  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
587 
588 
589  // Create transform and rotate (if needed)
590  QTransform transform;
591  float origin_x = previewImage->width() / 2.0;
592  float origin_y = previewImage->height() / 2.0;
593  transform.translate(origin_x, origin_y);
594  transform.rotate(rotate);
595  transform.translate(-origin_x,-origin_y);
596  painter.setTransform(transform);
597 
598  // Draw image onto QImage
599  painter.drawImage(x, y, *previewImage);
600 
601 
602  // Overlay Image (if any)
603  if (overlay_path != "") {
604  // Open overlay
605  auto overlay = std::make_shared<QImage>();
606  overlay->load(QString::fromStdString(overlay_path));
607 
608  // Set pixel format
609  overlay = std::make_shared<QImage>(
610  overlay->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
611 
612  // Resize to fit
613  overlay = std::make_shared<QImage>(overlay->scaled(
614  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
615 
616  // Composite onto thumbnail
617  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
618  painter.drawImage(0, 0, *overlay);
619  }
620 
621 
622  // Mask Image (if any)
623  if (mask_path != "") {
624  // Open mask
625  auto mask = std::make_shared<QImage>();
626  mask->load(QString::fromStdString(mask_path));
627 
628  // Set pixel format
629  mask = std::make_shared<QImage>(
630  mask->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
631 
632  // Resize to fit
633  mask = std::make_shared<QImage>(mask->scaled(
634  new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
635 
636  // Negate mask
637  mask->invertPixels();
638 
639  // Get pixels
640  unsigned char *pixels = static_cast<unsigned char *>(thumbnail->bits());
641  const unsigned char *mask_pixels = static_cast<const unsigned char *>(mask->constBits());
642 
643  // Convert the mask image to grayscale
644  // Loop through pixels
645  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
646  {
647  // Get the RGB values from the pixel
648  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
649  int Frame_Alpha = pixels[byte_index + 3];
650  int Mask_Value = constrain(Frame_Alpha - gray_value);
651 
652  // Set all alpha pixels to gray value
653  pixels[byte_index + 3] = Mask_Value;
654  }
655  }
656 
657 
658  // End painter
659  painter.end();
660 
661  // Save image
662  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
663 }
664 
665 // Constrain a color value from 0 to 255
666 int Frame::constrain(int color_value)
667 {
668  // Constrain new color from 0 to 255
669  if (color_value < 0)
670  color_value = 0;
671  else if (color_value > 255)
672  color_value = 255;
673 
674  return color_value;
675 }
676 
677 void Frame::AddColor(int new_width, int new_height, std::string new_color)
678 {
679  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
680  // Update parameters
681  width = new_width;
682  height = new_height;
683  color = new_color;
684  AddColor(QColor(QString::fromStdString(new_color)));
685 }
686 
687 // Add (or replace) pixel data to the frame (based on a solid color)
688 void Frame::AddColor(const QColor& new_color)
689 {
690  // Create new image object, and fill with pixel data
691  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
692  image = std::make_shared<QImage>(width, height, QImage::Format_RGBA8888_Premultiplied);
693 
694  // Fill with solid color
695  image->fill(new_color);
696  has_image_data = true;
697 }
698 
699 // Add (or replace) pixel data to the frame
701  int new_width, int new_height, int bytes_per_pixel,
702  QImage::Format type, const unsigned char *pixels_)
703 {
704  if (has_image_data) {
705  // Delete the previous QImage
706  image.reset();
707  }
708 
709  // Create new image object from pixel data
710  auto new_image = std::make_shared<QImage>(
711  pixels_,
712  new_width, new_height,
713  new_width * bytes_per_pixel,
714  type,
715  (QImageCleanupFunction) &openshot::cleanUpBuffer,
716  (void*) pixels_
717  );
718  AddImage(new_image);
719 }
720 
721 // Add (or replace) pixel data to the frame
722 void Frame::AddImage(std::shared_ptr<QImage> new_image)
723 {
724  // Ignore blank images
725  if (!new_image)
726  return;
727 
728  // assign image data
729  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
730  image = new_image;
731 
732  // Always convert to Format_RGBA8888_Premultiplied (if different)
733  if (image->format() != QImage::Format_RGBA8888_Premultiplied)
734  *image = image->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
735 
736  // Update height and width
737  width = image->width();
738  height = image->height();
739  has_image_data = true;
740 }
741 
742 // Add (or replace) pixel data to the frame (for only the odd or even lines)
743 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
744 {
745  // Ignore blank new_image
746  if (!new_image)
747  return;
748 
749  // Check for blank source image
750  if (!image) {
751  // Replace the blank source image
752  AddImage(new_image);
753 
754  } else {
755  // Ignore image of different sizes or formats
756  bool ret=false;
757  if (image == new_image || image->size() != new_image->size()) {
758  ret = true;
759  }
760  else if (new_image->format() != QImage::Format_RGBA8888_Premultiplied) {
761  new_image = std::make_shared<QImage>(
762  new_image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
763  }
764  if (ret) {
765  return;
766  }
767 
768  // Get the frame's image
769  const std::lock_guard<std::recursive_mutex> lock(addingImageMutex);
770  unsigned char *pixels = image->bits();
771  const unsigned char *new_pixels = new_image->constBits();
772 
773  // Loop through the scanlines of the image (even or odd)
774  int start = 0;
775  if (only_odd_lines)
776  start = 1;
777 
778  for (int row = start; row < image->height(); row += 2) {
779  int offset = row * image->bytesPerLine();
780  memcpy(pixels + offset, new_pixels + offset, image->bytesPerLine());
781  }
782 
783  // Update height and width
784  height = image->height();
785  width = image->width();
786  has_image_data = true;
787  }
788 }
789 
790 
791 // Resize audio container to hold more (or less) samples and channels
792 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
793 {
794  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
795 
796  // Resize JUCE audio buffer
797  audio->setSize(channels, length, true, true, false);
798  channel_layout = layout;
799  sample_rate = rate;
800 
801  // Calculate max audio sample added
802  max_audio_sample = length;
803 }
804 
805 // Reverse the audio buffer of this frame (will only reverse a single time, regardless of how many times
806 // you invoke this method)
808  if (audio && !audio_reversed) {
809  // Reverse audio buffer
810  audio->reverse(0, audio->getNumSamples());
811  audio_reversed = true;
812  }
813 }
814 
815 // Add audio samples to a specific channel
816 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
817  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
818 
819  // Clamp starting sample to 0
820  int destStartSampleAdjusted = max(destStartSample, 0);
821 
822  // Extend audio container to hold more (or less) samples and channels.. if needed
823  int new_length = destStartSampleAdjusted + numSamples;
824  int new_channel_length = audio->getNumChannels();
825  if (destChannel >= new_channel_length)
826  new_channel_length = destChannel + 1;
827  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
828  audio->setSize(new_channel_length, new_length, true, true, false);
829 
830  // Clear the range of samples first (if needed)
831  if (replaceSamples)
832  audio->clear(destChannel, destStartSampleAdjusted, numSamples);
833 
834  // Add samples to frame's audio buffer
835  audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
836  has_audio_data = true;
837 
838  // Calculate max audio sample added
839  if (new_length > max_audio_sample)
840  max_audio_sample = new_length;
841 
842  // Reset audio reverse flag
843  audio_reversed = false;
844 }
845 
846 // Apply gain ramp (i.e. fading volume)
847 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
848 {
849  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
850 
851  // Apply gain ramp
852  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
853 }
854 
855 // Get pointer to Magick++ image object
856 std::shared_ptr<QImage> Frame::GetImage()
857 {
858  // Check for blank image
859  if (!image)
860  // Fill with black
861  AddColor(width, height, color);
862 
863  return image;
864 }
865 
866 #ifdef USE_OPENCV
867 
868 // Convert Qimage to Mat
869 cv::Mat Frame::Qimage2mat( std::shared_ptr<QImage>& qimage) {
870 
871  cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone();
872  cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 );
873  int from_to[] = { 0,0, 1,1, 2,2 };
874  cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 );
875  cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR);
876  return mat2;
877 }
878 
879 // Get pointer to OpenCV image object
881 {
882  // Check for blank image
883  if (!image)
884  // Fill with black
885  AddColor(width, height, color);
886 
887  // if (imagecv.empty())
888  // Convert Qimage to Mat
889  imagecv = Qimage2mat(image);
890 
891  return imagecv;
892 }
893 
894 std::shared_ptr<QImage> Frame::Mat2Qimage(cv::Mat img){
895  cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
896  QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888);
897 
898  std::shared_ptr<QImage> imgIn = std::make_shared<QImage>(qimg.copy());
899 
900  // Always convert to RGBA8888 (if different)
901  if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied)
902  *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied);
903 
904  return imgIn;
905 }
906 
907 // Set pointer to OpenCV image object
908 void Frame::SetImageCV(cv::Mat _image)
909 {
910  imagecv = _image;
911  image = Mat2Qimage(_image);
912 }
913 #endif
914 
915 // Play audio samples for this frame
917 {
918  // Check if samples are present
919  if (!GetAudioSamplesCount())
920  return;
921 
922  juce::AudioDeviceManager deviceManager;
923  juce::String error = deviceManager.initialise (
924  0, /* number of input channels */
925  2, /* number of output channels */
926  0, /* no XML settings.. */
927  true /* select default device on failure */);
928 
929  // Output error (if any)
930  if (error.isNotEmpty()) {
931  cout << "Error on initialise(): " << error << endl;
932  }
933 
934  juce::AudioSourcePlayer audioSourcePlayer;
935  deviceManager.addAudioCallback (&audioSourcePlayer);
936 
937  std::unique_ptr<AudioBufferSource> my_source;
938  my_source.reset (new AudioBufferSource (audio.get()));
939 
940  // Create TimeSliceThread for audio buffering
941  juce::TimeSliceThread my_thread("Audio buffer thread");
942 
943  // Start thread
944  my_thread.startThread();
945 
946  juce::AudioTransportSource transport1;
947  transport1.setSource (my_source.get(),
948  5000, // tells it to buffer this many samples ahead
949  &my_thread,
950  (double) sample_rate,
951  audio->getNumChannels()); // sample rate of source
952  transport1.setPosition (0);
953  transport1.setGain(1.0);
954 
955 
956  // Create MIXER
957  juce::MixerAudioSource mixer;
958  mixer.addInputSource(&transport1, false);
959  audioSourcePlayer.setSource (&mixer);
960 
961  // Start transports
962  transport1.start();
963 
964  while (transport1.isPlaying())
965  {
966  cout << "playing" << endl;
967  std::this_thread::sleep_for(std::chrono::seconds(1));
968  }
969 
970  cout << "DONE!!!" << endl;
971 
972  transport1.stop();
973  transport1.setSource (0);
974  audioSourcePlayer.setSource (0);
975  my_thread.stopThread(500);
976  deviceManager.removeAudioCallback (&audioSourcePlayer);
977  deviceManager.closeAudioDevice();
978  deviceManager.removeAllChangeListeners();
979  deviceManager.dispatchPendingMessages();
980 
981  cout << "End of Play()" << endl;
982 
983 
984 }
985 
986 // Add audio silence
987 void Frame::AddAudioSilence(int numSamples)
988 {
989  const std::lock_guard<std::recursive_mutex> lock(addingAudioMutex);
990 
991  // Resize audio container
992  audio->setSize(channels, numSamples, false, true, false);
993  audio->clear();
994  has_audio_data = true;
995 
996  // Calculate max audio sample added
997  max_audio_sample = numSamples;
998 
999  // Reset audio reverse flag
1000  audio_reversed = false;
1001 }
openshot::Frame::GetWaveformPixels
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:258
openshot::Frame::SampleRate
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:502
openshot::Frame::operator=
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:78
openshot::Frame::GetAudioSamples
float * GetAudioSamples(int channel)
Get an array of sample data (and optional reverse the sample values)
Definition: Frame.cpp:314
openshot::Frame::SetFrameNumber
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:450
openshot::Frame::has_audio_data
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:119
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
juce::AudioBuffer< float >
openshot::Fraction
This class represents a fraction.
Definition: Fraction.h:30
AudioBufferSource.h
Header file for AudioBufferSource class.
openshot::Frame::Thumbnail
void Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:543
AudioResampler.h
Header file for AudioResampler class.
openshot::Frame
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:90
QtUtilities.h
Header file for QtUtilities (compatibiity overlay)
openshot::Frame::has_image_data
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:120
openshot::LAYOUT_STEREO
@ LAYOUT_STEREO
Definition: ChannelLayouts.h:31
openshot::Frame::AddAudioSilence
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:987
openshot::Fraction::ToDouble
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:40
openshot::Frame::GetBytes
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:380
openshot::Frame::ResizeAudio
void ResizeAudio(int channels, int length, int sample_rate, openshot::ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:792
openshot::Fraction::num
int num
Numerator for the fraction.
Definition: Fraction.h:32
openshot::Frame::ReverseAudio
void ReverseAudio()
Definition: Frame.cpp:807
openshot::Frame::GetPixels
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:397
openshot::Fraction::den
int den
Denominator for the fraction.
Definition: Fraction.h:33
openshot::Fraction::Reciprocal
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:78
openshot::Frame::AddImage
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:700
openshot::AudioBufferSource
This class is used to expose an AudioBuffer<float> as an AudioSource in JUCE.
Definition: AudioBufferSource.h:27
juce
Definition: Robotization.h:29
openshot::Frame::ApplyGainRamp
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:847
openshot::Frame::GetAudioChannelsCount
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:358
openshot::Frame::GetHeight
int GetHeight()
Get height of image.
Definition: Frame.cpp:490
path
path
Definition: FFmpegWriter.cpp:1479
Frame.h
Header file for Frame class.
openshot::Frame::GetSamplesPerFrame
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:484
openshot::Frame::GetInterleavedAudioSamples
float * GetInterleavedAudioSamples(int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate.
Definition: Frame.cpp:324
openshot::Frame::CheckPixel
bool CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold)
Check a specific pixel color value (returns True/False)
Definition: Frame.cpp:421
openshot::Frame::GetWaveform
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:159
openshot::Frame::AddAudio
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:816
openshot::Frame::GetWidth
int GetWidth()
Get height of image.
Definition: Frame.cpp:496
openshot::Frame::audio
std::shared_ptr< juce::AudioBuffer< float > > audio
Definition: Frame.h:117
openshot::Frame::ClearWaveform
void ClearWaveform()
Clear the waveform image (and deallocate its memory)
Definition: Frame.cpp:251
openshot::Frame::SetPixelRatio
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:443
openshot::Frame::Save
void Save(std::string path, float scale, std::string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG,...
Definition: Frame.cpp:515
openshot::Frame::GetImage
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:856
openshot::Frame::GetImageCV
cv::Mat GetImageCV()
Get pointer to OpenCV Mat image object.
Definition: Frame.cpp:880
openshot::Frame::ChannelsLayout
openshot::ChannelLayout ChannelsLayout()
Definition: Frame.cpp:508
openshot::Frame::GetAudioSampleBuffer
juce::AudioBuffer< float > * GetAudioSampleBuffer()
Definition: Frame.cpp:374
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
openshot::Frame::Play
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:916
openshot::Frame::DeepCopy
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:87
openshot::Frame::GetAudioSample
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
Definition: Frame.cpp:301
openshot::Frame::number
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:118
openshot::Frame::Mat2Qimage
std::shared_ptr< QImage > Mat2Qimage(cv::Mat img)
Convert OpenCV Mat to QImage.
Definition: Frame.cpp:894
openshot::Frame::SetImageCV
void SetImageCV(cv::Mat _image)
Set pointer to OpenCV image object.
Definition: Frame.cpp:908
openshot::Frame::Display
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:120
openshot::Frame::~Frame
virtual ~Frame()
Destructor.
Definition: Frame.cpp:110
openshot::Frame::Qimage2mat
cv::Mat Qimage2mat(std::shared_ptr< QImage > &qimage)
Convert Qimage to Mat.
Definition: Frame.cpp:869
openshot::Frame::Frame
Frame()
Constructor - blank frame.
Definition: Frame.cpp:59
openshot::Frame::GetAudioSamplesCount
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:368
openshot::Frame::AddColor
void AddColor(int new_width, int new_height, std::string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:677
openshot::Frame::DisplayWaveform
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:268