Skip to content

Commit 3286ccc

Browse files
author
Rehan Durrani
committed
Update sys mem calc
Signed-off-by: Rehan Durrani <rehan@ponder.io>
1 parent 487f563 commit 3286ccc

File tree

1 file changed

+10
-3
lines changed

1 file changed

+10
-3
lines changed

modin/core/execution/ray/common/utils.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -221,12 +221,19 @@ def initialize_ray(
221221
GPU_MANAGERS.append(GPUManager.remote(i))
222222
else:
223223
ray_obj_store_mem = ray.available_resources()["object_store_memory"]
224-
virtual_memory = psutil.virtual_memory().total
225-
if (ray_obj_store_mem // 1e9) < (0.6 * virtual_memory) // 1e9:
224+
system_memory = psutil.virtual_memory().total
225+
if sys.platform.startswith("linux"):
226+
shm_fd = os.open("/dev/shm", os.O_RDONLY)
227+
try:
228+
shm_stats = os.fstatvfs(shm_fd)
229+
system_memory = shm_stats.f_bsize * shm_stats.f_bavail
230+
finally:
231+
os.close(shm_fd)
232+
if (ray_obj_store_mem // 1e9) < (0.6 * system_memory) // 1e9:
226233
warnings.warn(
227234
"Modin has detected that it is running on a pre-initialized Ray cluster. "
228235
+ f"This cluster has currently allocated {ray_obj_store_mem // 1e9} GB for its "
229-
+ f"object store, but the device has {virtual_memory // 1e9} GB of RAM available. "
236+
+ f"object store, but the device has {system_memory // 1e9} GB of RAM available. "
230237
+ "Modin recommends initializing Ray with at least 60% of available RAM to prevent "
231238
+ "Out Of Memory errors."
232239
)

0 commit comments

Comments
 (0)