avcodec/avcodec: Check for more invalid channel layouts

In particular, check the provided channel layout for encoders
without AVCodec.ch_layouts set. This fixes an infinite loop
in the WavPack encoder (and maybe other issues in other encoders
as well) in case the channel count is zero.

Reviewed-by: James Almer <jamrial@gmail.com>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
This commit is contained in:
Andreas Rheinhardt
2022-09-16 02:27:25 +02:00
parent 859d9d70f9
commit a06a2d8943
3 changed files with 14 additions and 10 deletions

View File

@@ -246,6 +246,20 @@ FF_DISABLE_DEPRECATION_WARNINGS
FF_ENABLE_DEPRECATION_WARNINGS
#endif
/* AV_CODEC_CAP_CHANNEL_CONF is a decoder-only flag; so the code below
* in particular checks that nb_channels is set for all audio encoders. */
if (avctx->codec_type == AVMEDIA_TYPE_AUDIO && !avctx->ch_layout.nb_channels
&& !(codec->capabilities & AV_CODEC_CAP_CHANNEL_CONF)) {
av_log(avctx, AV_LOG_ERROR, "%s requires channel layout to be set\n",
av_codec_is_decoder(codec) ? "Decoder" : "Encoder");
ret = AVERROR(EINVAL);
goto free_and_end;
}
if (avctx->ch_layout.nb_channels && !av_channel_layout_check(&avctx->ch_layout)) {
av_log(avctx, AV_LOG_ERROR, "Invalid channel layout\n");
ret = AVERROR(EINVAL);
goto free_and_end;
}
if (avctx->ch_layout.nb_channels > FF_SANE_NB_CHANNELS) {
av_log(avctx, AV_LOG_ERROR, "Too many channels: %d\n", avctx->ch_layout.nb_channels);
ret = AVERROR(EINVAL);