`os.path.getsize()` slow on Network Drive (Python, Windows)

I don't know why getsize() is as slow as it is over the network, however to speed it up you could try calling it concurrently:

import os
from multiprocessing.pool import ThreadPool

def get_total_filesize_concurrently(paths):
    total = 0

    with ThreadPool(10) as pool:
        for size in pool.imap_unordered(lambda path: os.path.getsize(path), paths):
            total += size

    return total


print(get_total_filesize_concurrently([
    "E:\Path\To\File.txt",
    "E:\Path\To\File2.txt"
    "E:\Path\To\File3.txt"
    ...
]))

You can also play around with the number of threads defined in ThreadPool(10) to potentially increase performance even further.

If you are looking for performance it is better to try more native API. win32file is from pywin

import os
import win32file
import time
import ctypes
import ctypes.wintypes as types
from random import shuffle
from multiprocessing.dummy import Pool as ThreadPool

# Your directory
SRC_DIR = r'\\pc\share'

POOL_SIZE = 16

GetFileAttributesEx = ctypes.windll.kernel32.GetFileAttributesExW
GetFileAttributesEx.restype = ctypes.c_int
GetFileExInfoStandard = 0

def GetSizePool(src_dir:str, pool_size)->int:

    def dirsize_add(fullpath:str):
        nonlocal dirsize
        dirsize.append(GetFileSize(fullpath))

    dirsize = []
    pool = ThreadPool(pool_size)
    pool.map(dirsize_add, dir_files(src_dir) )
    pool.close()
    pool.join()
    return sum(dirsize)

def dir_files(src_dir):
    for dirpath, _, filenames in os.walk(src_dir):
        for f in filenames:
            yield os.path.join(dirpath, f)

def GetFileSize(fullpath:str)->int:
    fhandle = win32file.CreateFile(
        fullpath
        , win32file.GENERIC_READ
        , 0
        , None
        , win32file.OPEN_EXISTING
        , 0
        , None
    )
    size = win32file.GetFileSize(fhandle)
    fhandle = None
    return size

class WIN32_FILE_ATTRIBUTE_DATA(ctypes.Structure):
_fields_ = [
        ("dwFileAttributes", types.DWORD)
        , ("ftCreationTime", types.FILETIME)
        , ("ftLastAccessTime", types.FILETIME)
        , ("ftLastWriteTime", types.FILETIME)
        , ("nFileSizeHigh", types.DWORD)
        , ("nFileSizeLow", types.DWORD)
    ]

def GetAttCtypes(fullpath:str)->int:
    wfad = WIN32_FILE_ATTRIBUTE_DATA()
    GetFileAttributesEx(
        types.LPWSTR(fullpath)
        , GetFileExInfoStandard
        , ctypes.byref(wfad)
    )
    size = wfad.nFileSizeLow
    wfad = None
    return size

def GetAttPywin(fullpath:str)->int:
    return win32file.GetFileAttributesEx(fullpath)[-1]

tstart = time.process_time()
print(
    f'GetSizePool-{POOL_SIZE}:\t'
    + str(GetSizePool(SRC_DIR, pool_size=POOL_SIZE))
    + '\t'
    + str(time.process_time() - tstart)
)
func_lst = [GetAttCtypes, os.path.getsize, GetFileSize, GetAttPywin]
# Randomize function list for every run:
shuffle(func_lst)
for func in func_lst:
    filesizes = 0
    tstart = time.process_time()
    for dirpath, dirs, filenames in os.walk(SRC_DIR):
        for fname in filenames:
            filesizes += func(os.path.join(dirpath, fname))
    print(f'{func.__name__}:\t{filesizes}\t{time.process_time() - tstart}')

Note that GetAttCtypes will not work for a large files.

In my case, I get best result with GetFileSize. It looks like it takes advantage of the file cache.

GetSizePool-16: 7273209090      0.2808018
GetAttCtypes:   7273209090      0.5304034
GetAttPywin:    7273209090      0.6708042
GetFileSize:    7273209090      0.2496016
getsize:        7273209090      0.4680030

The use of threadpool is, well, very questionable, but shows good results sometimes.

Using GetFileSizeEx. You cannot have code with less syscalls.

This is a trimmed down code from this gist: https://gist.github.com/Pagliacii/774ed5d3ea78a36cdb0754be6a25408d

from ctypes import windll, wintypes
from ctypes import Structure, c_longlong, byref, POINTER, WinError

class LARGE_INTEGER(Structure):
    _fields_ = [('QuadPart', c_longlong)]

CreateFileW = windll.kernel32.CreateFileW
CreateFileW.argtypes = (
    wintypes.LPCWSTR,
    wintypes.DWORD,
    wintypes.DWORD,
    wintypes.LPVOID,
    wintypes.DWORD,
    wintypes.DWORD,
    wintypes.HANDLE)
CreateFileW.restype = wintypes.HANDLE

CloseHandle = windll.kernel32.CloseHandle
CloseHandle.argtypes = (wintypes.HANDLE,)
CloseHandle.restype = wintypes.BOOL

GetFileSizeEx = windll.kernel32.GetFileSizeEx
GetFileSizeEx.argtypes = (wintypes.HANDLE, POINTER(LARGE_INTEGER))
GetFileSizeEx.restype = wintypes.BOOL

GENERIC_READ = 31
OPEN_EXISTING = 3
FILE_SHARE_DISABLE = 0
FILE_ATTRIBUTE_NORMAL = 0x80


def get_filesize_win32(path):
    size = LARGE_INTEGER()
    handle = CreateFileW(path,
        GENERIC_READ,
        FILE_SHARE_DISABLE,
        None, OPEN_EXISTING,
        FILE_ATTRIBUTE_NORMAL,
        None)

    try:
        result = GetFileSizeEx(handle, byref(size))
        if not result:
            raise WinError()
        return size.QuadPart
    finally:
        CloseHandle(handle)

if __name__ == '__main__':
    print(get_filesize_win32(__file__))

You Can Try Getting Path From pathlib

from pathlib import Path

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent

If This Did Not Help, Learn More About This Python Module [PathLib]