I'm trying to create a video file from a remote stream, to store it on the harddrive. I'm using Humble Video for the encoding. Recording and creating the video file works fine. But now I want to add audio to the video.
I am receiving JPG images and audio data as byte arrays from the remote source. Audio data has a sample rate of 11025 and is 1 channel 16 bit values.
I think I am still missing something regarding the timebase and timestamp of the audio data I try to insert into the video file. Most of the time all recorded audio is very scrambled and only hearable in the first few seconds of the video.
Here is my code:
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import javax.imageio.ImageIO;
import io.humble.video.AudioChannel.Layout;
import io.humble.video.AudioFormat.Type;
import io.humble.video.Codec;
import io.humble.video.Encoder;
import io.humble.video.MediaAudio;
import io.humble.video.MediaPacket;
import io.humble.video.MediaPicture;
import io.humble.video.Muxer;
import io.humble.video.MuxerFormat;
import io.humble.video.PixelFormat;
import io.humble.video.Rational;
import io.humble.video.awt.MediaPictureConverter;
import io.humble.video.awt.MediaPictureConverterFactory;
public class VideoStream {
private static final int FPS = 30;
private static final int AUDIO_SAMPLE_RATE = 11025;
private static final int AUDIO_SAMPLE_SIZE = 11025;
private final String filename;
private final long startTimestampVideo;
private final long startTimestampAudio;
private final Muxer muxer;
private final Encoder videoEncoder;
private final Encoder audioEncoder;
private final Rational timebase;
private MediaPicture picture;
private MediaPictureConverter converter;
private MediaPacket packet;
private MediaAudio sound;
private boolean finished = false;
private static class EncoderTask implements Runnable {
private final byte[] data;
private final long timestamp;
private final Consumer<EncoderTask> encoder;
public EncoderTask(byte[] data, long timestamp, Consumer<EncoderTask> encoder) {
if (encoder == null) {
throw new IllegalArgumentException("Encoder must not be null.");
}
this.data = data;
this.timestamp = timestamp;
this.encoder = encoder;
}
@Override
public void run() {
encoder.accept(this);
}
public byte[] getData() {
return data;
}
public long getTimestamp() {
return timestamp;
}
public Consumer<EncoderTask> getEncoder() {
return encoder;
}
}
private ExecutorService threadPool = Executors.newSingleThreadExecutor();
public VideoStream(String filename, int width, int height, long startTimestampVideo, long startTimestampAudio)
throws IOException, InterruptedException {
this.filename = filename;
this.startTimestampVideo = startTimestampVideo;
this.startTimestampAudio = startTimestampAudio;
this.timebase = Rational.make(1, FPS);
this.muxer = Muxer.make(filename, null, null);
PixelFormat.Type pixelFormat = PixelFormat.Type.PIX_FMT_YUV420P;
Codec videoCodec = Codec.findEncodingCodec(muxer.getFormat().getDefaultVideoCodecId());
Codec audioCodec = Codec.findEncodingCodec(muxer.getFormat().getDefaultAudioCodecId());
this.videoEncoder = createVideoEncoder(videoCodec, width, height, pixelFormat);
this.audioEncoder = createAudioEncoder(audioCodec);
videoEncoder.open(null, null);
audioEncoder.open(null, null);
muxer.addNewStream(videoEncoder);
muxer.addNewStream(audioEncoder);
muxer.open(null, null);
picture = MediaPicture.make(videoEncoder.getWidth(), videoEncoder.getHeight(), pixelFormat);
picture.setTimeBase(timebase);
sound = MediaAudio.make(AUDIO_SAMPLE_SIZE, AUDIO_SAMPLE_RATE, 1, Layout.CH_LAYOUT_MONO, Type.SAMPLE_FMT_S16);
sound.setTimeBase(timebase);
packet = MediaPacket.make();
}
private Encoder createVideoEncoder(Codec codec, int width, int height, PixelFormat.Type pixelFormat) {
Encoder encoder = Encoder.make(codec);
encoder.setWidth(width);
encoder.setHeight(height);
encoder.setPixelFormat(pixelFormat);
encoder.setTimeBase(timebase);
if (muxer.getFormat().getFlag(MuxerFormat.Flag.GLOBAL_HEADER)) {
encoder.setFlag(Encoder.Flag.FLAG_GLOBAL_HEADER, true);
}
return encoder;
}
private Encoder createAudioEncoder(Codec codec) {
Encoder encoder = Encoder.make(codec);
encoder.setSampleRate(AUDIO_SAMPLE_RATE);
encoder.setChannels(1);
encoder.setChannelLayout(Layout.CH_LAYOUT_MONO);
encoder.setSampleFormat(Type.SAMPLE_FMT_S16);
if (muxer.getFormat().getFlag(MuxerFormat.Flag.GLOBAL_HEADER)) {
encoder.setFlag(Encoder.Flag.FLAG_GLOBAL_HEADER, true);
}
return encoder;
}
public String getFilename() {
return filename;
}
public synchronized void addImage(byte[] imageData, long timestamp) {
if (!finished) {
System.out.println("Adding image: " + (timestamp - startTimestampVideo));
threadPool.execute(new EncoderTask(imageData, timestamp, this::encodeImage));
}
}
public synchronized void addAudio(byte[] audioData, long timestamp) {
if (!finished) {
System.out.println("Adding audio: " + (timestamp - startTimestampAudio));
threadPool.execute(new EncoderTask(audioData, timestamp, this::encodeAudio));
}
}
public synchronized void finish() {
if (!finished) {
threadPool.execute(new EncoderTask(null, 0, this::finish));
}
}
private synchronized void encodeImage(EncoderTask task) {
BufferedImage jpegImage = convertImageDataToBufferedImage(task.getData());
if (jpegImage == null) {
return;
}
BufferedImage image = convertToType(jpegImage, BufferedImage.TYPE_3BYTE_BGR);
if (converter == null) {
converter = MediaPictureConverterFactory.createConverter(image, picture);
}
long t = Math.round((task.getTimestamp() - startTimestampVideo) * timebase.getDouble());
converter.toPicture(picture, image, t);
System.out.println("Encoding video: " + t);
do {
videoEncoder.encode(packet, picture);
if (packet.isComplete()) {
muxer.write(packet, false);
}
} while (packet.isComplete());
}
private synchronized void encodeAudio(EncoderTask task) {
System.out.println(
"Audio delta: " + (task.getTimestamp() - startTimestampAudio) + " timebase: " + timebase.getDouble());
long t = Math.round((task.getTimestamp() - startTimestampAudio) * timebase.getDouble() * AUDIO_SAMPLE_RATE);
sound.getData(0).put(task.data, 0, 0, task.data.length);
sound.setNumSamples(task.data.length);
sound.setTimeStamp(t);
sound.setComplete(true);
System.out.println("Encoding audio: " + t);
do {
audioEncoder.encode(packet, sound);
if (packet.isComplete()) {
muxer.write(packet, false);
}
} while (packet.isComplete());
}
private synchronized void finish(EncoderTask task) {
do {
videoEncoder.encode(packet, null);
if (packet.isComplete()) {
muxer.write(packet, false);
}
} while (packet.isComplete());
do {
audioEncoder.encode(packet, null);
if (packet.isComplete()) {
muxer.write(packet, false);
}
} while (packet.isComplete());
muxer.close();
finished = true;
threadPool.shutdown();
}
private static BufferedImage convertImageDataToBufferedImage(byte[] imageData) {
try (InputStream in = new ByteArrayInputStream(imageData)) {
return ImageIO.read(in);
} catch (IOException e) {
return null;
}
}
private static BufferedImage convertToType(BufferedImage sourceImage, int targetType) {
BufferedImage image;
if (sourceImage.getType() == targetType) {
// if the source image is already the target type, return the source image
image = sourceImage;
} else {
// otherwise create a new image of the target type and draw the new
// image
image = new BufferedImage(sourceImage.getWidth(), sourceImage.getHeight(), targetType);
image.getGraphics().drawImage(sourceImage, 0, 0, null);
}
return image;
}
}
It would be great if anyone has a sample on how to encode audio and video in the same file. The demos in the Humble Video git are only for encoding video data.