avcodec/mpegvideo_enc: Check for integer overflow in ff_mpv_reallocate_putbitbuffer()

Fixes assertion failure
Fixes: 6568d187979ce17878b6fe5fbbb89142/signal_sigabrt_7ffff6ae7cb7_7176_564bbc6741bdcf907f5c4e685c9a77a2.mpg

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit b65efbc0

)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/mpegvideo_enc.c

@@ -2740,6 +2740,11 @@ int ff_mpv_reallocate_putbitbuffer(MpegEncContext *s, size_t threshold, size_t s
         uint8_t *new_buffer = NULL;
         int new_buffer_size = 0;
 
+        if ((s->avctx->internal->byte_buffer_size + size_increase) >= INT_MAX/8) {
+            av_log(s->avctx, AV_LOG_ERROR, "Cannot reallocate putbit buffer\n");
+            return AVERROR(ENOMEM);
+        }
+
         av_fast_padded_malloc(&new_buffer, &new_buffer_size,
                               s->avctx->internal->byte_buffer_size + size_increase);
         if (!new_buffer)