From 2501b868567ed7695b3aa5b2f030b430e736324b Mon Sep 17 00:00:00 2001 From: Garux Date: Thu, 18 Sep 2025 21:07:03 +0500 Subject: [PATCH] * refactor to work without -maxshaderinfo limit, improve shader search performance --- ...mplete_list_of_command_line_parameters.htm | 1 - docs/changelog-custom.txt | 1 + libs/container/container.h | 15 +- tools/quake3/common/unsortedset.h | 232 ++++++++++++++++++ tools/quake3/q3map2/help.cpp | 1 - tools/quake3/q3map2/light.cpp | 2 +- tools/quake3/q3map2/main.cpp | 7 - tools/quake3/q3map2/map.cpp | 2 +- tools/quake3/q3map2/q3map2.h | 30 ++- tools/quake3/q3map2/shaders.cpp | 79 ++---- 10 files changed, 291 insertions(+), 79 deletions(-) create mode 100644 tools/quake3/common/unsortedset.h diff --git a/docs/Complete_list_of_command_line_parameters.htm b/docs/Complete_list_of_command_line_parameters.htm index c29aea55..775d2240 100644 --- a/docs/Complete_list_of_command_line_parameters.htm +++ b/docs/Complete_list_of_command_line_parameters.htm @@ -115,7 +115,6 @@ td.formatted_questions ol { margin-top: 0px; margin-bottom: 0px; }
  • -fs_pakpath path: Specify a package directory (can be used more than once to look in multiple paths)
  • -game gamename: Load settings for the given game (default: quake3), -help -game lists available games
  • -maxmapdrawsurfs N: Sets max amount of mapDrawSurfs, used during .map compilation (-bsp, -convert), default = 131072
    • -
  • -maxshaderinfo N: Sets max amount of shaderInfo, default = 8192
  • -subdivisions F: multiplier for patch subdivisions quality
  • -threads N: number of threads to use
  • -v: Verbose mode
    • diff --git a/docs/changelog-custom.txt b/docs/changelog-custom.txt index 4615ccff..0e7c0239 100644 --- a/docs/changelog-custom.txt +++ b/docs/changelog-custom.txt @@ -182,6 +182,7 @@ Q3map2: * tighten FilterPatchIntoTree() regarding to even (pink) points * fix tiny structural brushes breaking bsp tree * handle patch lodbounds in -shift and -scale + * refactor to work without -maxshaderinfo limit, improve shader search performance diff --git a/libs/container/container.h b/libs/container/container.h index e13d5ab9..24832a46 100644 --- a/libs/container/container.h +++ b/libs/container/container.h @@ -221,27 +221,22 @@ public: } iterator push_back( const Value& value ){ - typename decltype( m_set )::iterator it; + std::tuple tuple = m_set.emplace( value ); if constexpr ( UniqueValues ){ - bool inserted; - std::tie( it, inserted ) = m_set.emplace( value ); - ASSERT_MESSAGE( inserted, "UnsortedSet::insert: already added" ); + ASSERT_MESSAGE( std::get<1>( tuple ), "UnsortedSet::insert: already added" ); } - else{ - it = m_set.emplace( value ); - } - Node *newNode = &const_cast( *it ); + Node *newNode = &const_cast( *std::get<0>( tuple ) ); Node::link( m_end.m_prev, newNode ); Node::link( newNode, m_end.asNode() ); return iterator( newNode ); } void erase( const Value& value ){ - const auto it = m_set.find( value ); + const auto it = m_set.find( value ); // note: multiset finds w/e value from equals ASSERT_MESSAGE( it != m_set.cend(), "UnsortedSet::erase: not found" ); Node::link( it->m_prev, it->m_next ); m_set.erase( it ); } - const_iterator find( const Value& value ) const { + const_iterator find( const Value& value ) const { // note: multiset finds w/e value from equals const auto it = m_set.find( value ); return ( it == m_set.cend() )? end() : const_iterator( &( *it ) ); } diff --git a/tools/quake3/common/unsortedset.h b/tools/quake3/common/unsortedset.h new file mode 100644 index 00000000..7281e62d --- /dev/null +++ b/tools/quake3/common/unsortedset.h @@ -0,0 +1,232 @@ +#pragma once + +#include + +//variant of container from container/container.h for more heavy objects +//provides CompareValue param, efficient construction and search + +/// \brief An adaptor to make std::set or std::multiset into a SequenceContainer. +/// It's illegal to modify inserted values directly! +/// \param Value Uniquely identifies itself. Must provide a copy-constructor and an equality operator. +template> +class UnsortedSet +{ + struct Node + { + Node *m_prev; + Node *m_next; + Value m_value; + Node( const Value& value ) : m_value( value ){ + } + Node( Value&& value ) : m_value( std::move( value ) ){ + } + template + Node( Args&&...args ) : m_value( std::forward( args )... ){ + } + static void link( Node *prev, Node *next ){ + prev->m_next = next; + next->m_prev = prev; + } + }; + /// special thin sentinel node to avoid DefaultConstructible \param Value requirement + struct SentinelNode + { + Node *m_prev; + Node *m_next; + SentinelNode(){ + selfLink(); + } + void selfLink(){ + m_prev = m_next = asNode(); + } + Node* asNode(){ + return reinterpret_cast( this ); + } + const Node* asNode() const { + return reinterpret_cast( this ); + } + }; + static_assert( offsetof( SentinelNode, m_next ) == offsetof( Node, m_next ) && + offsetof( SentinelNode, m_prev ) == offsetof( Node, m_prev ), + "Node layouts must be compatible for reinterpret_cast" ); + SentinelNode m_end; + + template + class Iterator + { + public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = Value; + using difference_type = std::ptrdiff_t; + using pointer = std::conditional_t; + using reference = std::conditional_t; + using node_ptr = std::conditional_t; + private: + node_ptr m_node; + public: + Iterator( node_ptr node = nullptr ) : m_node( node ){ + } + reference operator*() const { + return m_node->m_value; + } + pointer operator->() const { + return &m_node->m_value; + } + Iterator& operator++(){ + if constexpr ( IsReverse ) + m_node = m_node->m_prev; + else + m_node = m_node->m_next; + return *this; + } + Iterator& operator--(){ + if constexpr ( IsReverse ) + m_node = m_node->m_next; + else + m_node = m_node->m_prev; + return *this; + } + Iterator operator++( int ){ + auto ret = *this; + ++( *this ); + return ret; + } + Iterator operator--( int ){ + auto ret = *this; + --( *this ); + return ret; + } + + friend bool operator==( const Iterator& lhs, const Iterator& rhs ) { + return lhs.m_node == rhs.m_node; + } + + // Conversion from non-const to const iterator + template + requires ( OtherIsConst && !IsConst && ( OtherIsReverse == IsReverse ) ) + operator Iterator() const { + return Iterator( m_node ); + } + }; + +public: + using iterator = Iterator; + using const_iterator = Iterator; + using reverse_iterator = Iterator; + using const_reverse_iterator = Iterator; + + iterator begin() { return m_end.m_next; } + const_iterator begin() const { return m_end.m_next; } + iterator end() { return m_end.asNode(); } + const_iterator end() const { return m_end.asNode(); } + reverse_iterator rbegin() { return m_end.m_prev; } + const_reverse_iterator rbegin() const { return m_end.m_prev; } + reverse_iterator rend() { return m_end.asNode(); } + const_reverse_iterator rend() const { return m_end.asNode(); } +private: + struct Compare{ + using is_transparent = void; + + bool operator()( const Node& one, const Node& other ) const { + return CompareValue()( one.m_value, other.m_value ); + } + template // Transparent comparison: Value with key type + bool operator()( const Node& node, const K& key ) const { + return CompareValue()( node.m_value, key ); + } + template // Transparent comparison: key type with Value + bool operator()( const K& key, const Node& node ) const { + return CompareValue()( key, node.m_value ); + } + }; + std::conditional_t, std::multiset> m_set; +public: + UnsortedSet() = default; + UnsortedSet( const UnsortedSet& other ) = delete; + UnsortedSet( UnsortedSet&& ) noexcept = delete; + UnsortedSet& operator=( const UnsortedSet& other ){ + clear(); + for( const auto& value : other ) + push_back( value ); + return *this; + }; + UnsortedSet& operator=( UnsortedSet&& ) noexcept = delete; + + bool empty() const { + return m_set.empty(); + } + std::size_t size() const { + return m_set.size(); + } + void clear(){ + m_end.selfLink(); + m_set.clear(); + } + + void swap( UnsortedSet& other ){ + std::swap( m_set, other.m_set ); + std::swap( m_end.m_next, other.m_end.m_next ); + std::swap( m_end.m_prev, other.m_end.m_prev ); + for( auto *set : { this, &other } ){ // note: would be trivial swap with allocated end node; unused function + if( set->empty() ) + set->m_end.selfLink(); + else + set->m_end.m_prev->m_next = set->m_end.m_next->m_prev = set->m_end.asNode(); + } + } +private: + iterator link_back( auto tuple ){ + if constexpr ( UniqueValues ){ + if( !std::get<1>( tuple ) ) // not inserted //!assert or do not link + return iterator( &const_cast( *std::get<0>( tuple ) ) ); + } + Node *newNode = &const_cast( *std::get<0>( tuple ) ); + Node::link( m_end.m_prev, newNode ); + Node::link( newNode, m_end.asNode() ); + return iterator( newNode ); + } +public: + iterator push_back( const Value& value ){ + return link_back( std::tuple( m_set.emplace( value ) ) ); + } + iterator push_back( Value&& value ){ + return link_back( std::tuple( m_set.emplace( std::move( value ) ) ) ); + } + template + iterator emplace_back( Args&&...args ) { + return link_back( std::tuple( m_set.emplace( std::forward( args )... ) ) ); + } + void erase( const Value& value ){ + const auto it = m_set.find( value ); // note: multiset finds w/e value from equals + if( it != m_set.cend() ){ //!assert or do not erase + Node::link( it->m_prev, it->m_next ); + m_set.erase( it ); + } + } + template + const_iterator find( const K& key ) const { // note: multiset finds w/e value from equals + const auto it = m_set.find( key ); + return ( it == m_set.cend() )? end() : const_iterator( &( *it ) ); + } + template + iterator find( const K& key ) { // note: multiset finds w/e value from equals + const auto it = m_set.find( key ); + return ( it == m_set.cend() )? end() : iterator( &const_cast( *it ) ); + } + template + requires ( !UniqueValues ) + iterator find_first( const K& key ) { + auto it = m_set.find( key ); // multiset finds w/e value from equals, find 1st one + if( it == m_set.cend() ) + return end(); + while( it != m_set.cbegin() && !Compare()( *std::prev( it ), *it ) ) + --it; + return iterator( &const_cast( *it ) ); + } + Value& back(){ + return m_end.m_prev->m_value; + } + const Value& back() const { + return m_end.m_prev->m_value; + } +}; diff --git a/tools/quake3/q3map2/help.cpp b/tools/quake3/q3map2/help.cpp index b86592b9..dc045f2b 100644 --- a/tools/quake3/q3map2/help.cpp +++ b/tools/quake3/q3map2/help.cpp @@ -460,7 +460,6 @@ static void HelpCommon() { "-fs_pakpath ", "Specify a package directory (can be used more than once to look in multiple paths)" }, { "-game ", "Load settings for the given game (default: quake3), -help -game lists available games" }, { "-maxmapdrawsurfs ", "Sets max amount of mapDrawSurfs, used during .map compilation (-bsp, -convert), default = 131072" }, - { "-maxshaderinfo ", "Sets max amount of shaderInfo, default = 8192" }, { "-subdivisions ", "multiplier for patch subdivisions quality" }, { "-threads ", "number of threads to use" }, { "-v", "Verbose mode" }, diff --git a/tools/quake3/q3map2/light.cpp b/tools/quake3/q3map2/light.cpp index 54198bd1..10e9ea27 100644 --- a/tools/quake3/q3map2/light.cpp +++ b/tools/quake3/q3map2/light.cpp @@ -1792,7 +1792,7 @@ static void WriteBSPFileAfterLight( const char *bspFileName ){ // stupidly search in shader text, ( numExtLightmaps > 0 ) check wont work when e.g. deluxemaps // also would excessively include lightstyles using $lightmap reference if( !externalLightmaps ){ // unless native ext lms: e.g. in ET with lightstyles hack preloading lm imgs breaks r_mapOverbrightBits - for ( const shaderInfo_t& si : Span( shaderInfo, numShaderInfo ) ) + for ( const shaderInfo_t& si : shaderInfo ) { if ( si.custom && !strEmptyOrNull( si.shaderText ) ) { const char *txt = si.shaderText; diff --git a/tools/quake3/q3map2/main.cpp b/tools/quake3/q3map2/main.cpp index d56c6d73..4872cb0c 100644 --- a/tools/quake3/q3map2/main.cpp +++ b/tools/quake3/q3map2/main.cpp @@ -120,13 +120,6 @@ int main( int argc, char **argv ){ Sys_Printf( "max_map_draw_surfs = %d, mapDrawSurfs size = %.2f MBytes \n", max_map_draw_surfs, sizeof( mapDrawSurface_t ) * max_map_draw_surfs / ( 1024.f * 1024.f ) ); } - - /* max_shader_info */ - while ( args.takeArg( "-maxshaderinfo" ) ) { - max_shader_info = abs( atoi( args.takeNext() ) ); - Sys_Printf( "max_shader_info = %d, shaderInfo size = %.2f MBytes \n", - max_shader_info, sizeof( shaderInfo_t ) * max_shader_info / ( 1024.f * 1024.f ) ); - } } /* init model library */ diff --git a/tools/quake3/q3map2/map.cpp b/tools/quake3/q3map2/map.cpp index fed3530a..e9bacd16 100644 --- a/tools/quake3/q3map2/map.cpp +++ b/tools/quake3/q3map2/map.cpp @@ -1003,7 +1003,7 @@ static void ParseRawBrush( bool onlyLights ){ const String64 shader( "textures/", token ); /* set default flags and values */ - shaderInfo_t *si = onlyLights? &shaderInfo[ 0 ] + shaderInfo_t *si = onlyLights? shaderInfo.begin().operator->() : ShaderInfoForShader( shader ); side.shaderInfo = si; side.surfaceFlags = si->surfaceFlags; diff --git a/tools/quake3/q3map2/q3map2.h b/tools/quake3/q3map2/q3map2.h index 856a1188..38ffdb39 100644 --- a/tools/quake3/q3map2/q3map2.h +++ b/tools/quake3/q3map2/q3map2.h @@ -69,6 +69,7 @@ #include #include #include "qmath.h" +#include "unsortedset.h" #include #include @@ -489,7 +490,7 @@ enum class EImplicitMap }; -struct shaderInfo_t +struct shaderInfo_t_data { String64 shader; int surfaceFlags; @@ -601,6 +602,16 @@ struct shaderInfo_t bool finished; }; +struct shaderInfo_t : public shaderInfo_t_data +{ + const String64 shader; + shaderInfo_t( const char *shaderName ) : shaderInfo_t_data{}, shader( shaderName ){ // zero-initialze shaderInfo_t_data + } + void copyData( const shaderInfo_t& other ) noexcept { + static_cast( *this ) = static_cast( other ); + } +}; + /* ------------------------------------------------------------------------------- @@ -1720,10 +1731,21 @@ void InjectCommandLine( const char *stage, const std::vec ------------------------------------------------------------------------------- */ +struct shaderInfo_t_compare +{ + bool operator()( const shaderInfo_t& si, const shaderInfo_t& si2 ) const { + return RawStringLessNoCase()( si.shader, si2.shader ); + } + bool operator()( const shaderInfo_t& si, const char *si2 ) const { + return RawStringLessNoCase()( si.shader, si2 ); + } + bool operator()( const char *si, const shaderInfo_t& si2 ) const { + return RawStringLessNoCase()( si, si2.shader ); + } +}; + /* general */ -inline shaderInfo_t *shaderInfo; -inline int numShaderInfo; -inline int max_shader_info = 8192; +inline UnsortedSet shaderInfo; inline String64 mapName; /* ydnar: per-map custom shaders for larger lightmaps */ inline CopiedString mapShaderFile; diff --git a/tools/quake3/q3map2/shaders.cpp b/tools/quake3/q3map2/shaders.cpp index edb8942c..bce9a59f 100644 --- a/tools/quake3/q3map2/shaders.cpp +++ b/tools/quake3/q3map2/shaders.cpp @@ -292,7 +292,7 @@ void WriteMapShaderFile(){ } /* are there any custom shaders? */ - if( std::ranges::none_of( Span( shaderInfo, numShaderInfo ), &shaderInfo_t::custom ) ) + if( std::ranges::none_of( shaderInfo, &shaderInfo_t::custom ) ) return; /* note it */ @@ -315,7 +315,7 @@ void WriteMapShaderFile(){ /* walk the shader list */ int num = 0; - for ( const shaderInfo_t& si : Span( shaderInfo, numShaderInfo ) ) + for ( const shaderInfo_t& si : shaderInfo ) { if ( si.custom && !strEmptyOrNull( si.shaderText ) ) { num++; @@ -485,9 +485,8 @@ const shaderInfo_t *CustomShader( const shaderInfo_t *si, const char *find, char return csi; } - /* clone the existing shader and rename */ - *csi = *si; - csi->shader = shader; + /* clone the existing shader data */ + csi->copyData( *si ); csi->custom = true; /* store new shader text */ @@ -537,27 +536,11 @@ void EmitVertexRemapShader( char *from, char *to ){ allocates and initializes a new shader */ -static shaderInfo_t *AllocShaderInfo(){ - shaderInfo_t *si; - - - /* allocate? */ - if ( shaderInfo == NULL ) { - shaderInfo = safe_malloc( sizeof( shaderInfo_t ) * max_shader_info ); - numShaderInfo = 0; - } - - /* bounds check */ - if ( numShaderInfo == max_shader_info ) { - Error( "max_shader_info (%d) exceeded. Remove some PK3 files or shader scripts from shaderlist.txt and try again." - " Or consider -maxshaderinfo to increase.", max_shader_info ); - } - si = &shaderInfo[ numShaderInfo ]; - numShaderInfo++; +static shaderInfo_t *AllocShaderInfo( const char *shaderName ){ + shaderInfo_t *si = shaderInfo.emplace_back( shaderName ).operator->(); /* ydnar: clear to 0 first */ -// memset( si, 0, sizeof( shaderInfo_t ) ); - new( si ) shaderInfo_t{}; // placement new + // new( si ) shaderInfo_t{}; // placement new /* set defaults */ ApplySurfaceParm( "default", &si->contentFlags, &si->surfaceFlags, &si->compileFlags ); @@ -750,10 +733,6 @@ shaderInfo_t *ShaderInfoForShaderNull( const char *shaderName ){ } shaderInfo_t *ShaderInfoForShader( const char *shaderName ){ - int i; - int deprecationDepth; - shaderInfo_t *si; - /* dummy check */ if ( strEmptyOrNull( shaderName ) ) { Sys_Warning( "Null or empty shader name\n" ); @@ -764,39 +743,37 @@ shaderInfo_t *ShaderInfoForShader( const char *shaderName ){ String64 shader( PathExtensionless( shaderName ) ); /* search for it */ - deprecationDepth = 0; - for ( i = 0; i < numShaderInfo; i++ ) + int deprecationDepth = 0; + + while( true ) { - si = &shaderInfo[ i ]; - if ( striEqual( shader, si->shader ) ) { + auto si = shaderInfo.find_first( shader ); + if ( si != shaderInfo.end() ) { /* check if shader is deprecated */ if ( deprecationDepth < MAX_SHADER_DEPRECATION_DEPTH && !strEmptyOrNull( si->deprecateShader ) ) { /* override name */ shader( PathExtensionless( si->deprecateShader ) ); - /* increase deprecation depth */ - deprecationDepth++; - if ( deprecationDepth == MAX_SHADER_DEPRECATION_DEPTH ) { + if ( ++deprecationDepth == MAX_SHADER_DEPRECATION_DEPTH ) { Sys_Warning( "Max deprecation depth of %i is reached on shader '%s'\n", MAX_SHADER_DEPRECATION_DEPTH, shader.c_str() ); } /* search again from beginning */ - i = -1; continue; } /* load image if necessary */ if ( !si->finished ) { - LoadShaderImages( si ); - FinishShader( si ); + LoadShaderImages( si.operator->() ); + FinishShader( si.operator->() ); } /* return it */ - return si; + return si.operator->(); } + break; } /* allocate a default shader */ - si = AllocShaderInfo(); - si->shader = shader; + shaderInfo_t *si = AllocShaderInfo( shader ); LoadShaderImages( si ); FinishShader( si ); @@ -840,14 +817,12 @@ static void ParseShaderFile( const char *filename ){ while ( GetToken( true ) ) /* test for end of file */ { /* shader name is initial token */ - shaderInfo_t *si = AllocShaderInfo(); - /* ignore ":q3map" suffix */ const bool isQ3mapOnlyShader = striEqualSuffix( token, ":q3map" ); if( isQ3mapOnlyShader ) - si->shader << StringRange( token, strlen( token ) - strlen( ":q3map" ) ); - else - si->shader << token; + strClear( token + strlen( token ) - strlen( ":q3map" ) ); + + shaderInfo_t *si = AllocShaderInfo( token ); /* handle { } section */ if ( !( text.GetToken( true ) && strEqual( token, "{" ) ) ) { @@ -1147,14 +1122,10 @@ static void ParseShaderFile( const char *filename ){ /* subclass it */ if ( si2 != NULL ) { - /* preserve name */ - const String64 temp = si->shader; + /* copy shader data */ + si->copyData( *si2 ); - /* copy shader */ - *si = *si2; - - /* restore name and set to unfinished */ - si->shader = temp; + /* set to unfinished */ si->shaderWidth = 0; si->shaderHeight = 0; si->finished = false; @@ -1901,5 +1872,5 @@ void LoadShaderInfo(){ } /* emit some statistics */ - Sys_FPrintf( SYS_VRB, "%9d shaderInfo\n", numShaderInfo ); + Sys_FPrintf( SYS_VRB, "%9zu shaderInfo\n", shaderInfo.size() ); }