[Bf-blender-cvs] [ded68fb1027] master: VSE: Fix audaspace not reading ffmpeg files with start offset correctly
Sebastian Parborg
noreply at git.blender.org
Mon Aug 16 15:13:50 CEST 2021
Commit: ded68fb10275c9f9a66e7019171b83cab0e9485d
Author: Sebastian Parborg
Date: Tue Jul 6 19:48:06 2021 +0200
Branches: master
https://developer.blender.org/rBded68fb10275c9f9a66e7019171b83cab0e9485d
VSE: Fix audaspace not reading ffmpeg files with start offset correctly
The duration and start time for audio strips were not correctly read in
audaspace.
Some video files have a "lead in" section of audio that plays before the
video starts playing back. Before this patch, we would play this lead in
audio at the same time as the video started and thus the audio would not
be in sync anymore.
Now the lead in audio is cut off and the duration should be correctly
calculated with this in mind.
If the audio starts after the video, the audio strip is shifted to
account for this, but it will also lead to cut off audio which might not
be wanted. However we don't have a simple way to solve this at this
point.
Differential Revision: http://developer.blender.org/D11917
===================================================================
M extern/audaspace/bindings/C/AUD_Special.cpp
M extern/audaspace/bindings/C/AUD_Types.h
M extern/audaspace/include/IReader.h
M extern/audaspace/include/fx/VolumeReader.h
M extern/audaspace/plugins/ffmpeg/FFMPEGReader.cpp
M extern/audaspace/plugins/ffmpeg/FFMPEGReader.h
M extern/audaspace/src/fx/VolumeReader.cpp
M source/blender/blenkernel/BKE_sound.h
M source/blender/blenkernel/intern/sound.c
M source/blender/editors/space_sequencer/sequencer_add.c
M source/blender/editors/space_sequencer/sequencer_draw.c
M source/blender/imbuf/IMB_imbuf.h
M source/blender/imbuf/intern/IMB_anim.h
M source/blender/imbuf/intern/anim_movie.c
M source/blender/makesdna/DNA_sound_types.h
M source/blender/makesrna/intern/rna_sequencer_api.c
M source/blender/sequencer/SEQ_add.h
M source/blender/sequencer/intern/sound.c
M source/blender/sequencer/intern/strip_add.c
M source/blender/sequencer/intern/strip_time.c
===================================================================
diff --git a/extern/audaspace/bindings/C/AUD_Special.cpp b/extern/audaspace/bindings/C/AUD_Special.cpp
index ac876a01eb3..97e5f5540de 100644
--- a/extern/audaspace/bindings/C/AUD_Special.cpp
+++ b/extern/audaspace/bindings/C/AUD_Special.cpp
@@ -86,6 +86,7 @@ AUD_API AUD_SoundInfo AUD_getInfo(AUD_Sound* sound)
info.specs.channels = AUD_CHANNELS_INVALID;
info.specs.rate = AUD_RATE_INVALID;
info.length = 0.0f;
+ info.start_offset = 0.0f;
try
{
@@ -95,6 +96,7 @@ AUD_API AUD_SoundInfo AUD_getInfo(AUD_Sound* sound)
{
info.specs = convSpecToC(reader->getSpecs());
info.length = reader->getLength() / (float) info.specs.rate;
+ info.start_offset = reader->getStartOffset();
}
}
catch(Exception&)
diff --git a/extern/audaspace/bindings/C/AUD_Types.h b/extern/audaspace/bindings/C/AUD_Types.h
index 75e4ffae18c..c6a96d30d3f 100644
--- a/extern/audaspace/bindings/C/AUD_Types.h
+++ b/extern/audaspace/bindings/C/AUD_Types.h
@@ -176,4 +176,5 @@ typedef struct
{
AUD_Specs specs;
float length;
+ double start_offset;
} AUD_SoundInfo;
diff --git a/extern/audaspace/include/IReader.h b/extern/audaspace/include/IReader.h
index c29900ca579..f6070b0f23b 100644
--- a/extern/audaspace/include/IReader.h
+++ b/extern/audaspace/include/IReader.h
@@ -70,6 +70,12 @@ public:
*/
virtual int getPosition() const=0;
+ /**
+ * Returns the start offset the sound should have to line up with related sources.
+ * \return The required start offset in seconds.
+ */
+ virtual double getStartOffset() const { return 0.0;}
+
/**
* Returns the specification of the reader.
* \return The Specs structure.
diff --git a/extern/audaspace/include/fx/VolumeReader.h b/extern/audaspace/include/fx/VolumeReader.h
index 13b6845e931..f7169f4c78b 100644
--- a/extern/audaspace/include/fx/VolumeReader.h
+++ b/extern/audaspace/include/fx/VolumeReader.h
@@ -67,4 +67,4 @@ public:
virtual void read(int& length, bool& eos, sample_t* buffer);
};
-AUD_NAMESPACE_END
\ No newline at end of file
+AUD_NAMESPACE_END
diff --git a/extern/audaspace/plugins/ffmpeg/FFMPEGReader.cpp b/extern/audaspace/plugins/ffmpeg/FFMPEGReader.cpp
index b46f65eddbf..afdc7fcfcc6 100644
--- a/extern/audaspace/plugins/ffmpeg/FFMPEGReader.cpp
+++ b/extern/audaspace/plugins/ffmpeg/FFMPEGReader.cpp
@@ -68,7 +68,7 @@ int FFMPEGReader::decode(AVPacket& packet, Buffer& buffer)
for(int i = 0; i < m_frame->nb_samples; i++)
{
std::memcpy(((data_t*)buffer.getBuffer()) + buf_pos + ((m_codecCtx->channels * i) + channel) * single_size,
- m_frame->data[channel] + i * single_size, single_size);
+ m_frame->data[channel] + i * single_size, single_size);
}
}
}
@@ -109,7 +109,7 @@ int FFMPEGReader::decode(AVPacket& packet, Buffer& buffer)
for(int i = 0; i < m_frame->nb_samples; i++)
{
std::memcpy(((data_t*)buffer.getBuffer()) + buf_pos + ((m_codecCtx->channels * i) + channel) * single_size,
- m_frame->data[channel] + i * single_size, single_size);
+ m_frame->data[channel] + i * single_size, single_size);
}
}
}
@@ -126,7 +126,10 @@ int FFMPEGReader::decode(AVPacket& packet, Buffer& buffer)
void FFMPEGReader::init()
{
m_position = 0;
+ m_start_offset = 0.0f;
m_pkgbuf_left = 0;
+ m_st_time = 0;
+ m_duration = 0;
if(avformat_find_stream_info(m_formatCtx, nullptr) < 0)
AUD_THROW(FileException, "File couldn't be read, ffmpeg couldn't find the stream info.");
@@ -134,15 +137,41 @@ void FFMPEGReader::init()
// find audio stream and codec
m_stream = -1;
+ double dur_sec = 0;
+
for(unsigned int i = 0; i < m_formatCtx->nb_streams; i++)
{
#ifdef FFMPEG_OLD_CODE
- if((m_formatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
+ if(m_formatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
#else
- if((m_formatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO)
+ if(m_formatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO)
#endif
- && (m_stream < 0))
{
+ AVStream *audio_stream = m_formatCtx->streams[i];
+ double audio_timebase = av_q2d(audio_stream->time_base);
+
+ if (audio_stream->start_time != AV_NOPTS_VALUE)
+ {
+ m_st_time = audio_stream->start_time;
+ }
+
+ int64_t ctx_start_time = 0;
+ if (m_formatCtx->start_time != AV_NOPTS_VALUE) {
+ ctx_start_time = m_formatCtx->start_time;
+ }
+
+ m_start_offset = m_st_time * audio_timebase - (double)ctx_start_time / AV_TIME_BASE;
+
+ if(audio_stream->duration != AV_NOPTS_VALUE)
+ {
+ dur_sec = audio_stream->duration * audio_timebase;
+ }
+ else
+ {
+ /* If the audio starts after the stream start time, subract this from the total duration. */
+ dur_sec = (double)m_formatCtx->duration / AV_TIME_BASE - m_start_offset;
+ }
+
m_stream=i;
break;
}
@@ -213,6 +242,7 @@ void FFMPEGReader::init()
}
m_specs.rate = (SampleRate) m_codecCtx->sample_rate;
+ m_duration = lround(dur_sec * m_codecCtx->sample_rate);
}
FFMPEGReader::FFMPEGReader(std::string filename) :
@@ -338,21 +368,17 @@ void FFMPEGReader::seek(int position)
{
if(position >= 0)
{
- uint64_t st_time = m_formatCtx->start_time;
- uint64_t seek_pos = ((uint64_t)position) * ((uint64_t)AV_TIME_BASE) / ((uint64_t)m_specs.rate);
+ double pts_time_base =
+ av_q2d(m_formatCtx->streams[m_stream]->time_base);
- if(st_time != AV_NOPTS_VALUE) {
- seek_pos += st_time;
- }
+ uint64_t seek_pts = (((uint64_t)position) / ((uint64_t)m_specs.rate)) / pts_time_base;
- double pts_time_base =
- av_q2d(m_formatCtx->streams[m_stream]->time_base);
- uint64_t pts_st_time =
- ((st_time != AV_NOPTS_VALUE) ? st_time : 0)
- / pts_time_base / (uint64_t) AV_TIME_BASE;
+ if(m_st_time != AV_NOPTS_VALUE) {
+ seek_pts += m_st_time;
+ }
// a value < 0 tells us that seeking failed
- if(av_seek_frame(m_formatCtx, -1, seek_pos,
+ if(av_seek_frame(m_formatCtx, m_stream, seek_pts,
AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_ANY) >= 0)
{
avcodec_flush_buffers(m_codecCtx);
@@ -374,7 +400,7 @@ void FFMPEGReader::seek(int position)
if(packet.pts != AV_NOPTS_VALUE)
{
// calculate real position, and read to frame!
- m_position = (packet.pts - pts_st_time) * pts_time_base * m_specs.rate;
+ m_position = (packet.pts - m_st_time) * pts_time_base * m_specs.rate;
if(m_position < position)
{
@@ -405,8 +431,7 @@ void FFMPEGReader::seek(int position)
int FFMPEGReader::getLength() const
{
// return approximated remaning size
- return (int)((m_formatCtx->duration * m_codecCtx->sample_rate)
- / AV_TIME_BASE)-m_position;
+ return m_duration - m_position;
}
int FFMPEGReader::getPosition() const
@@ -414,6 +439,11 @@ int FFMPEGReader::getPosition() const
return m_position;
}
+double FFMPEGReader::getStartOffset() const
+{
+ return m_start_offset;
+}
+
Specs FFMPEGReader::getSpecs() const
{
return m_specs.specs;
@@ -450,11 +480,13 @@ void FFMPEGReader::read(int& length, bool& eos, sample_t* buffer)
// decode the package
pkgbuf_pos = decode(packet, m_pkgbuf);
- // copy to output buffer
- data_size = std::min(pkgbuf_pos, left * sample_size);
- m_convert((data_t*) buf, (data_t*) m_pkgbuf.getBuffer(), data_size / AUD_FORMAT_SIZE(m_specs.format));
- buf += data_size / AUD_FORMAT_SIZE(m_specs.format);
- left -= data_size / sample_size;
+ if (packet.pts >= m_st_time) {
+ // copy to output buffer
+ data_size = std::min(pkgbuf_pos, left * sample_size);
+ m_convert((data_t*) buf, (data_t*) m_pkgbuf.getBuffer(), data_size / AUD_FORMAT_SIZE(m_specs.format));
+ buf += data_size / AUD_FORMAT_SIZE(m_specs.format);
+ left -= data_size / sample_size;
+ }
}
av_packet_unref(&packet);
}
diff --git a/extern/audaspace/plugins/ffmpeg/FFMPEGReader.h b/extern/audaspace/plugins/ffmpeg/FFMPEGReader.h
index f62436ac83b..d613457c220 100644
--- a/extern/audaspace/plugins/ffmpeg/FFMPEGReader.h
+++ b/extern/audaspace/plugins/ffmpeg/FFMPEGReader.h
@@ -54,6 +54,22 @@ private:
*/
int m_position;
+ /**
+ * The start offset in seconds relative to the media container start time.
+ * IE how much the sound should be delayed to be kept in sync with the rest of the containter streams.
+ */
+ double m_start_offset;
+
+ /**
+ * The start time pts of the stream. All packets before this timestamp shouldn't be played back (only decoded).
+ */
+ int64_t m_st_time;
+
+ /**
+ * The duration of the audio stream in samples.
+ */
+ int64_t m_duration;
+
/**
* The specification of the audio data.
*/
@@ -182,6 +198,7 @@ public:
virtual void seek(int position);
virtual int getLength() const;
virtual int getPosition() const;
+ virtual double getStartOffset() const;
virtual Specs getSpecs() const;
virtual void read(int& length, bool& eos, sample_t* buffer);
};
diff --git a/extern/audaspace/src/fx/VolumeReader.cpp b/extern/audaspace/src/fx/VolumeReader.cpp
index ac1d4882a87..627acbac9ef 100644
--- a/extern/audaspace/src/fx/VolumeReader.cpp
+++ b/extern/audaspace/src/fx/VolumeReader.cpp
@@ -57,4 +57,4 @@ void VolumeReader::read(int& length, bool& eos, sample_t* buffer)
buffer[i] = buffer[i] * m_volumeStorage->getVolume();
}
-AUD_NAMESPACE_END
\ No newline at end of file
+AUD_NAMESPACE_END
diff --git a/source/blender/blenkernel/BKE_sound.h b/source/blender/blenkernel/BKE_sound.h
index 57ce33a239f..4b257b3b8ab 100644
--- a/source/blender/blenkernel/BKE_sound.h
+++ b/source/blender/blenkernel/BKE_sound.h
@@ -97,6 +97,7 @@ typedef struct SoundInfo {
eSoundChannels channels;
} specs;
float length;
+ double start_offset;
} SoundInfo;
/* Get information about given sound. Returns truth on success., false if sound can not be loaded
@@ -139,8 +140,12 @@ void BKE_sound_remove_scene_sound(struct Scene *scene, void *handle);
void BKE_sound_mute_scene_sound(void *handle, char mute);
-void BKE_sound_move_scene_sound(
- struct Scene *scene, void *handle, int startframe, int endframe, int frameskip);
+void BKE_sound_move_scene_sound(struct Scene *scene,
+ void *handle,
+ int startframe,
+ int endframe,
+ i
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list