`BatchedInferencePipeline` cannot be used with multithreading?

[Crissium]

How do you use multiple GPUs to do batched transcription in parallel? Strangely with the following code snippet, only one GPU is used at one time, the other's usage staying at 0%, then moments later, the latter spins up while the former falls back to zero.

python transcribe.py ../../Data/Original/LibriSpeech/test-clean ../../Data/Generated/LibriSpeech/test-clean/whisper-large-v3.jsonl 0 1

transcribe.py

import json
import os
import sys
from concurrent.futures import ThreadPoolExecutor
from faster_whisper import WhisperModel, BatchedInferencePipeline
from typing import Generator


BATCH_SIZE = 50


def transcribe_file(
	pipeline: BatchedInferencePipeline,
	input_filename: str
) -> Generator[tuple[str, int, int, str], None, None]:
	segments, _ = pipeline.transcribe(input_filename, language='en', batch_size=BATCH_SIZE)
	for segment in segments:
		yield (
			os.path.basename(input_filename),
			int(segment.start * 1000),
			int(segment.end * 1000),
			segment.text.strip()
		)


def transcribe_directory(
	pipeline: BatchedInferencePipeline,
	root_dir: str,
	output_file: str,
	num_workers: int
) -> None:
	with ThreadPoolExecutor(max_workers=num_workers) as executor:
		futures = []
		for root, _, files in os.walk(root_dir):
			for file in files:
				if file.endswith('.flac'):
					input_filename = os.path.join(root, file)
					futures.append(executor.submit(transcribe_file, pipeline, input_filename))

		with open(output_file, 'w') as f:
			for future in futures:
				for segment in future.result():
					f.write(json.dumps({
						'filename': segment[0],
						'start_time': segment[1],
						'end_time': segment[2],
						'text': segment[3]
					}) + '\n')


if __name__ == '__main__':
	root_dir = sys.argv[1]
	output_file = sys.argv[2]
	device_indexes = list(map(int, sys.argv[3:]))
	model = WhisperModel(
		'large-v3',
		device='cuda',
		device_index=device_indexes,
		num_workers=len(device_indexes)
	)
	pipeline = BatchedInferencePipeline(model)
	transcribe_directory(
		pipeline,
		root_dir,
		output_file,
		num_workers=len(device_indexes)
	)

And I saw this (I was using GPU 0 & 1):

2025-07-28.16-09-43.mp4

Is there something I am missing?

[MahmoudAshraf97]

You are not doing anything wrong, in order to get multigpu efficient utilization you'll have to do one of three things:

1. use a model instance on each gpu and handle the load balancing yourself, the limitation with this approach is you can't use all the gpus to serve a single request
2. Rewrite the inference part in the model transcribe function to divide the input batch over the gpus, this is more efficient but harder to implement, and if you are not using BatchedInferencePipeline you will also hit the same limitation as approach 1
3. Use TRT-LLM which is much faster and more suitable for production, you have to decide if the development overhead is worth it or not

[Crissium]

Thank you for your detailed explanation, Mahmoud! But I am sorry that I don't quite follow. Let me rephrase my question:

• I instantiated a WhisperModel with the arguments device_index=[0,1] and num_workers=2 and wrapped it with BatchedInferencePipeline
• I could verify that each GPU had one Whisper replica loaded by checking the memory usage
• I submitted transcription tasks to a thread pool with two workers
• In theory, when one transcription task is in progress, it releases the GIL, according to this line

https://github.com/OpenNMT/CTranslate2/blob/617405f4b050e994e829d527da6caa0e0030afe7/python/cpp/whisper.cc#L262C33-L262C51

Thus allowing another thread to launch another transcription task in parallel to it, and both GPUs should be spinning now.

Yet in reality, only one thread at most seemed to be running. What's wrong?

[MahmoudAshraf97]

Since CT2 seems unmaintained, even if we know what's wrong we won't be able to solve it, so I may not have a direct answer to your question but I have a solution for the general problem which is one of the above 3 points

[Crissium]

Strangely I tried the unbatched transcribe() method of WhisperModel and observed the same behaviour: the threads cannot run at the same time.

Here's my code:

import json
import os
import sys
from concurrent.futures import ThreadPoolExecutor
from faster_whisper import WhisperModel
from typing import Generator


def transcribe_file(
	model: WhisperModel,
	input_filename: str
) -> Generator[tuple[str, int, int, str], None, None]:
	segments, _ = model.transcribe(input_filename, language='en')
	for segment in segments:
		yield (
			os.path.basename(input_filename),
			int(segment.start * 1000),
			int(segment.end * 1000),
			segment.text.strip()
		)


def transcribe_directory(
	model: WhisperModel,
	root_dir: str,
	output_file: str,
	num_workers: int
) -> None:
	with ThreadPoolExecutor(max_workers=num_workers) as executor:
		futures = []
		for root, _, files in os.walk(root_dir):
			for file in files:
				if file.endswith('.flac'):
					input_filename = os.path.join(root, file)
					futures.append(executor.submit(transcribe_file, model, input_filename))

		with open(output_file, 'w') as f:
			for future in futures:
				for segment in future.result():
					f.write(json.dumps({
						'filename': segment[0],
						'start_time': segment[1],
						'end_time': segment[2],
						'text': segment[3]
					}) + '\n')


if __name__ == '__main__':
	root_dir = sys.argv[1]
	output_file = sys.argv[2]
	device_indexes = list(map(int, sys.argv[3:]))
	model = WhisperModel(
		'large-v3',
		device='cuda',
		device_index=device_indexes,
		num_workers=len(device_indexes)
	)
	transcribe_directory(
		model,
		root_dir,
		output_file,
		num_workers=len(device_indexes)
	)

Since faster-whisper is a pure-Python wrapper around CT2, could you please confirm that the issue is not caused by faster-whisper? If so, I'll open an issue at CT2.

By the way, multiprocessing with ProcessPoolExecutor works perfectly, for anyone interested yet having no need for fine-grained load-balancing.

---

Edit: I tried with 4 GPUs this time and I could observe up to two threads running at the same time. So there shouldn't be a GIL bug. Besides, I tried directly using CT2 with the following notebook:

# %%
import ctranslate2
import numpy as np
from concurrent.futures import ThreadPoolExecutor
from ctranslate2.models import Whisper

# %%
model = Whisper(
	'/mnt/users_home/cpii.local/yxing/.cache/huggingface/hub/models--Systran--faster-whisper-large-v3/snapshots/edaa852ec7e145841d8ffdb056a99866b5f0a478',
	device='cuda',
	device_index=[4,5,6,7],
	compute_type='float16',
	inter_threads=4
)

# %%
def get_ctranslate2_storage(segment: np.ndarray) -> ctranslate2.StorageView:
    segment = np.ascontiguousarray(segment)
    segment = ctranslate2.StorageView.from_array(segment)
    return segment

# %%
def generate_random_features() -> np.ndarray:
	return get_ctranslate2_storage(np.random.uniform(-1., 1., (1, 128, 3000)).astype(np.float32))

# %%
def encode_and_generate(model: Whisper, features: ctranslate2.StorageView):
	encoder_output = model.encode(features, to_cpu=True)
	return model.generate(
		encoder_output,
		[[50258, 50259, 50360]],
		length_penalty=1,
		repetition_penalty=1,
		no_repeat_ngram_size=0,
		max_length=448,
		return_scores=True,
		return_no_speech_prob=True,
		suppress_blank=True,
		suppress_tokens=(1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50359, 50360, 50361, 50362),
		max_initial_timestamp_index=50,
		beam_size=5,
		patience=1
	)[0]

# %%
with ThreadPoolExecutor(max_workers=4) as executor:
	futures = [
		executor.submit(encode_and_generate, model, generate_random_features())
		for _ in range(1000)
	]
	for future in futures:
		assert len(future.result().sequences) > 0

And I could see all 4 GPUs being used. So perhaps there is something I am not doing right here?