I'm working on an photo management project recently,one function is to scan JPG files,and I used boost::filesystem::recursive_directory_iterator with std::unordered_multimap but the performance is very poor.
with indexing:
[2022-08-10 14:11:48.556] [debug] 16.837325 seconds,969178 files,57561 files per second
without indexing:
[2022-08-10 14:13:47.389] [debug] 66.298863 seconds,732820 files,11053 files per second
My code is here:
uint64_t count = 0;
auto start = std::chrono::steady_clock::now();
std::unordered_multimap<std::wstring, std::wstring> type_map{};
boost::filesystem::recursive_directory_iterator end_iter;
for (boost::filesystem::recursive_directory_iterator iter(L"\\\\.\\F:\\", boost::filesystem::directory_options::skip_permission_denied); iter != end_iter; iter++)
{
type_map.emplace(iter->path().extension().c_str(), iter->path().c_str());
++count;
}
auto end = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
auto time = (double)duration.count() * std::chrono::microseconds::period::num / std::chrono::microseconds::period::den;
spdlog::debug("{} seconds,{} files,{} files per second", time, count, (uint64_t)(count / time));
auto range = type_map.equal_range(L".jpg");
for (auto iter = range.first; iter != range.second; ++iter)
{
spdlog::info(L"found {}", iter->second);
}
msbuild parameters are:
/permissive- /Yu"stdafx.h" /ifcOutput "x64\Release\" /GS /GL /W3 /Gy /Zc:wchar_t /Zi /Gm- /O2 /sdl /Fd"x64\Release\vc143.pdb" /FI"stdafx.h" /Zc:inline /fp:precise /D "NDEBUG" /D "_CONSOLE" /D "_UNICODE" /D "UNICODE" /errorReport:prompt /WX- /Zc:forScope /std:c17 /Gd /Oi /MT /std:c++latest /FC /Fa"x64\Release\" /EHsc /nologo /Fo"x64\Release\" /Fp"x64\Release\jpg_searcher.pch" /diagnostics:column