From b2f9f2caab1f691e2dbbd98db900c685bed6746a Mon Sep 17 00:00:00 2001 From: Garux Date: Sat, 9 Jul 2022 18:35:11 +0300 Subject: [PATCH] repack skies mod: pack all found sky shaders WARNING444 on texture reuse MAX_SHADER_INFO bumped load all uniquely named shader files, only from folders, not packs VFS mod to not keep pk3s opened (avoid hitting OS half opened files limit)(risky and slow for normal use) -packDups: vfsPackSkyImage: pack only new images, referring to main and dups DBs; pack alt image versions to separate pk3s vfsFindSkyFiles: pack sky images, not referenced by any shader and not present in main DB and dups DB (crc check) normal vfsPackFile: split output pk3s to approximately 2 GB chunks -packQer: also pack qer_editorimage -validateShaders: useful comments about referenced textures state right in out shader, do not save entirely invalid shaders (no packaging) -shaderDups: write similar shaders to files for latter evaluation by diff tool (no packaging) WARNING666 when equally named shaders reference different sky boxes --- tools/quake3/common/scriplib.cpp | 2 +- tools/quake3/common/scriplib.h | 2 +- tools/quake3/common/vfs.cpp | 254 ++++++++++++++++++++++---- tools/quake3/common/vfs.h | 2 + tools/quake3/q3map2/autopk3.cpp | 299 ++++++++++++++++++++++++++++++- tools/quake3/q3map2/q3map2.h | 2 +- tools/quake3/q3map2/shaders.cpp | 2 +- 7 files changed, 522 insertions(+), 41 deletions(-) diff --git a/tools/quake3/common/scriplib.cpp b/tools/quake3/common/scriplib.cpp index 7747ce55..c39ec9e6 100644 --- a/tools/quake3/common/scriplib.cpp +++ b/tools/quake3/common/scriplib.cpp @@ -105,7 +105,7 @@ bool LoadScriptFile( const char *filename, int index /* = 0 */, bool verbose /* ParseFromMemory ============== */ -void ParseFromMemory( char *buffer, size_t size ){ +void ParseFromMemory( const char *buffer, size_t size ){ scriptstack.clear(); tokenready = false; MemBuffer bu( size ); diff --git a/tools/quake3/common/scriplib.h b/tools/quake3/common/scriplib.h index 39cf8db8..06b9b9cb 100644 --- a/tools/quake3/common/scriplib.h +++ b/tools/quake3/common/scriplib.h @@ -32,7 +32,7 @@ extern int scriptline; /// \param[in] index >= 0: \p filename is relative path in VSF, Nth occurrence of file /// \return true on success bool LoadScriptFile( const char *filename, int index = 0, bool verbose = true ); -void ParseFromMemory( char *buffer, size_t size ); +void ParseFromMemory( const char *buffer, size_t size ); /// \param[in] crossline true: write next token to \c token or return false on EOF /// \param[in] crossline false: find next token on the current line or emit \c Error diff --git a/tools/quake3/common/vfs.cpp b/tools/quake3/common/vfs.cpp index d1e7f948..36304759 100644 --- a/tools/quake3/common/vfs.cpp +++ b/tools/quake3/common/vfs.cpp @@ -73,12 +73,12 @@ struct VFS_PAK { - unzFile zipfile; + // unzFile zipfile; const CopiedString unzFilePath; - VFS_PAK( unzFile zipfile, const char *unzFilePath ) : zipfile( zipfile ), unzFilePath( unzFilePath ) {}; + VFS_PAK( const char *unzFilePath ) : unzFilePath( unzFilePath ) {}; VFS_PAK( VFS_PAK&& ) noexcept = delete; ~VFS_PAK(){ - unzClose( zipfile ); + // unzClose( zipfile ); } }; @@ -90,6 +90,27 @@ struct VFS_PAKFILE const guint32 size; }; + +static MemBuffer read_file_from_pak( const VFS_PAKFILE& file ){ + MemBuffer buffer; + unzFile zipfile = unzOpen( file.pak.unzFilePath.c_str() ); + if( zipfile != nullptr ){ + *(unz_s*)zipfile = file.zipinfo; + + if ( unzOpenCurrentFile( zipfile ) == UNZ_OK ) { + buffer = MemBuffer( file.size ); + + if ( unzReadCurrentFile( zipfile, buffer.data(), file.size ) < 0 ) { + buffer = MemBuffer(); + } + unzCloseCurrentFile( zipfile ); + } + unzClose( zipfile ); + } + return buffer; +} + + // ============================================================================= // Global variables @@ -106,7 +127,7 @@ char g_strLoadedFileLocation[1024]; static void vfsInitPakFile( const char *filename ){ unzFile uf = unzOpen( filename ); if ( uf != NULL ) { - VFS_PAK& pak = g_paks.emplace_front( uf, filename ); + VFS_PAK& pak = g_paks.emplace_front( filename ); if ( unzGoToFirstFile( uf ) == UNZ_OK ) { do { @@ -129,6 +150,7 @@ static void vfsInitPakFile( const char *filename ){ } ); } } while( unzGoToNextFile( uf ) == UNZ_OK ); + unzClose( uf ); } } } @@ -195,6 +217,70 @@ void vfsInitDirectory( const char *path ){ } +#include + +void vfsFindSkyFiles( const std::vector& present_list ){ + for ( const VFS_PAKFILE& file : g_pakFiles ) + { + const char *name = file.name.c_str(); + if ( path_extension_is( name, "tga" ) || path_extension_is( name, "jpg" ) || path_extension_is( name, "png" ) ) { + auto n = StringOutputStream( 64 )( PathExtensionless( name ) ); + if( string_equal_suffix_nocase( n, "_bk" ) + || string_equal_suffix_nocase( n, "_rt" ) + || string_equal_suffix_nocase( n, "_ft" ) + || string_equal_suffix_nocase( n, "_lf" ) + || string_equal_suffix_nocase( n, "_up" ) + || string_equal_suffix_nocase( n, "_dn" ) + ){ + bool found = false; + for( const auto& pre : present_list ) + if( striEqual( pre.c_str(), n ) ) + found = true; + if( !found ){ + auto add_crc = [crcs = std::set()]( const MemBuffer& boo )mutable->bool{ + return crcs.insert( crc32( 0, boo.data(), boo.size() ) ).second; + }; + + struct FTYPE + { + const char *ext; + int idx; + }; + FTYPE ftypes[3]{ { ".png", 1 }, { ".tga", 1 }, { ".jpg", 1 } }; + + for( auto& ftype : ftypes ) + { + const StringOutputStream fullnames[] = { StringOutputStream( 256 )( "c:/qsky/__/skpk/", n, ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", n, ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", n, " (2)", ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", n, " (3)", ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", n, " (4)", ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", n, " (5)", ftype.ext ) }; + for( const auto& fullname : fullnames ) + { + if( FileExists( fullname ) ){ + ++ftype.idx; + add_crc( LoadFile( fullname ) ); + } + } + } + + + if( MemBuffer buffer = read_file_from_pak( file ) ){ + if( add_crc( buffer ) ){ + Sys_Printf( "WARNING777: %s :: %s\n", name, file.pak.unzFilePath.c_str() ); + if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( "c:/qsky/test_bigbox_repacked_noShader.pk3", name, buffer.data(), buffer.size(), 0, 0, 9, file.zipinfo.cur_file_info.dosDate ) ){ + Error( "Failed creating zip archive \"%s\"!\n", "c:/qsky/test_bigbox_repacked_noShader.pk3" ); + } + } + } + } + } + } + } +} + + // lists all unique .shader files with extension and w/o path std::vector vfsListShaderFiles( const char *shaderPath ){ std::vector list; @@ -220,6 +306,7 @@ std::vector vfsListShaderFiles( const char *shaderPath ){ } } /* search in packs */ +#if 0 for ( const VFS_PAKFILE& file : g_pakFiles ) { const char *name = file.name.c_str(); @@ -228,7 +315,7 @@ std::vector vfsListShaderFiles( const char *shaderPath ){ insert( path_get_filename_start( name ) ); } } - +#endif return list; } @@ -302,36 +389,143 @@ MemBuffer vfsLoadFile( const char *filename, int index /* = 0 */ ){ auto fixed = StringOutputStream( 64 )( PathCleaned( filename ) ); strLower( fixed.c_str() ); - MemBuffer buffer; - for ( const VFS_PAKFILE& file : g_pakFiles ) { if ( strEqual( file.name.c_str(), fixed ) && 0 == index-- ) { snprintf( g_strLoadedFileLocation, sizeof( g_strLoadedFileLocation ), "%s :: %s", file.pak.unzFilePath.c_str(), filename ); - unzFile zipfile = file.pak.zipfile; - *(unz_s*)zipfile = file.zipinfo; - - if ( unzOpenCurrentFile( zipfile ) == UNZ_OK ) { - buffer = MemBuffer( file.size ); - - if ( unzReadCurrentFile( zipfile, buffer.data(), file.size ) < 0 ) { - buffer = MemBuffer(); - } - unzCloseCurrentFile( zipfile ); - } - return buffer; + // unzFile zipfile = file.pak.zipfile; + return read_file_from_pak( file ); } } - return buffer; + return {}; } -bool vfsPackFile( const char *filename, const char *packname, const int compLevel ){ +#include + +bool vfsPackSkyImage( const char *filename, const char *packname, const int compLevel ){ + bool packed = false; + bool resInMain = false; + + auto add_crc = [crcs = std::set()]( const MemBuffer& boo )mutable->bool{ + return crcs.insert( crc32( 0, boo.data(), boo.size() ) ).second; + }; + + struct FTYPE + { + const char *ext; + int idx; + }; + FTYPE ftypes[3]{ { ".png", 1 }, { ".tga", 1 }, { ".jpg", 1 } }; + + for( auto& ftype : ftypes ) + { + const StringOutputStream fullnames[] = { StringOutputStream( 256 )( "c:/qsky/__/skpk/", filename, ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, " (2)", ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, " (3)", ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, " (4)", ftype.ext ), + StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, " (5)", ftype.ext ) }; + for( const auto& fullname : fullnames ) + { + if( FileExists( fullname ) ){ + resInMain = packed = true; + ++ftype.idx; + add_crc( LoadFile( fullname ) ); + } + } + } + + for( auto& ftype : ftypes ) + { + for ( const auto& dir : g_strDirs ) + { + const auto fullname = StringOutputStream( 256 )( dir, filename, ftype.ext ); + if( FileExists( fullname ) && add_crc( LoadFile( fullname ) ) ){ + if( !packed ){ + packed = vfsPackFile_Absolute_Path( fullname, StringOutputStream( 256 )( filename, ftype.ext ), packname, compLevel ); + } + else{ + #if 0 + auto toname = StringOutputStream( 256 )( "c:/qsky/m_dups/", filename ); + if( idx > 1 ) + toname << " (" << idx << ")"; + toname << ext; + std::filesystem::create_directories( std::filesystem::path( toname.c_str() ).remove_filename() ); + std::filesystem::copy_file( fullname.c_str(), toname.c_str() ); + #else + auto toname = StringOutputStream( 256 )( filename ); + if( ftype.idx > 1 ) // original name for files in alt format, (2) etc suffix for alt versions in the same format + toname << " (" << ftype.idx << ")"; + toname << ftype.ext; + if( resInMain ) + vfsPackFile_Absolute_Path( fullname, toname, StringOutputStream( 256 )( PathExtensionless( packname ), "_altMain.", path_get_extension( packname ) ), compLevel ); + else + vfsPackFile_Absolute_Path( fullname, toname, StringOutputStream( 256 )( PathExtensionless( packname ), "_altNew.", path_get_extension( packname ) ), compLevel ); + #endif + } + ++ftype.idx; + } + } + } + + auto fixed = StringOutputStream( 64 )( PathCleaned( filename ) ); + strLower( fixed.c_str() ); + + for( auto& ftype : ftypes ) + { + const auto fullfixed = StringOutputStream( 256 )( fixed, ftype.ext ); + for ( const VFS_PAKFILE& file : g_pakFiles ) + { + if ( strEqual( file.name.c_str(), fullfixed ) ) + { + // unzFile zipfile = file.pak.zipfile; + if ( MemBuffer buffer = read_file_from_pak( file ) ) { + if ( add_crc( buffer ) ) { + if( !packed ){ + packed = true; + if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( packname, StringOutputStream( 256 )( filename, ftype.ext ), buffer.data(), buffer.size(), 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){ + Error( "Failed creating zip archive \"%s\"!\n", packname ); + } + } + else{ + auto toname = StringOutputStream( 256 )( filename ); + if( ftype.idx > 1 ) // original name for files in alt format, (2) etc suffix for alt versions in the same format + toname << " (" << ftype.idx << ")"; + toname << ftype.ext; + if( resInMain ){ + if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( StringOutputStream( 256 )( PathExtensionless( packname ), "_altMain.", path_get_extension( packname ) ), toname, buffer.data(), buffer.size(), 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){ + Error( "Failed creating zip archive \"%s\"!\n", packname ); + } + } + else{ + if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( StringOutputStream( 256 )( PathExtensionless( packname ), "_altNew.", path_get_extension( packname ) ), toname, buffer.data(), buffer.size(), 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){ + Error( "Failed creating zip archive \"%s\"!\n", packname ); + } + } + } + ++ftype.idx; + } + } + } + } + } + return packed; +} + +bool vfsPackFile( const char *filename, const char *packname_in, const int compLevel ){ + static int packNum = 0; + auto packname = StringOutputStream( 256 )( PathExtensionless( packname_in ), packNum, '.', path_get_extension( packname_in ) ); + if( std::filesystem::exists( packname.c_str() ) && std::filesystem::file_size( packname.c_str() ) > 2147483648 - 10485760 ){ + ++packNum; + packname( PathExtensionless( packname_in ), packNum, '.', path_get_extension( packname_in ) ); + } + for ( const auto& dir : g_strDirs ) { if( vfsPackFile_Absolute_Path( StringOutputStream( 256 )( dir, filename ), filename, packname, compLevel ) ) @@ -345,20 +539,12 @@ bool vfsPackFile( const char *filename, const char *packname, const int compLeve { if ( strEqual( file.name.c_str(), fixed ) ) { - unzFile zipfile = file.pak.zipfile; - *(unz_s*)zipfile = file.zipinfo; - - if ( unzOpenCurrentFile( zipfile ) == UNZ_OK ) { - MemBuffer buffer( file.size ); - - const int size = unzReadCurrentFile( zipfile, buffer.data(), file.size ); - unzCloseCurrentFile( zipfile ); - if ( size >= 0 ) { - if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( packname, filename, buffer.data(), size, 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){ - Error( "Failed creating zip archive \"%s\"!\n", packname ); - } - return true; + // unzFile zipfile = file.pak.zipfile; + if( MemBuffer buffer = read_file_from_pak( file ) ){ + if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( packname, filename, buffer.data(), buffer.size(), 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){ + Error( "Failed creating zip archive \"%s\"!\n", packname.c_str() ); } + return true; } return false; } diff --git a/tools/quake3/common/vfs.h b/tools/quake3/common/vfs.h index 6bdd9fa6..c128392d 100644 --- a/tools/quake3/common/vfs.h +++ b/tools/quake3/common/vfs.h @@ -40,6 +40,8 @@ int vfsGetFileCount( const char *filename ); MemBuffer vfsLoadFile( const char *filename, int index = 0 ); std::vector vfsListShaderFiles( const char *shaderPath ); bool vfsPackFile( const char *filename, const char *packname, const int compLevel ); +bool vfsPackSkyImage( const char *filename, const char *packname, const int compLevel ); +void vfsFindSkyFiles( const std::vector& present_list ); bool vfsPackFile_Absolute_Path( const char *filepath, const char *filename, const char *packname, const int compLevel ); extern std::vector g_strForbiddenDirs; diff --git a/tools/quake3/q3map2/autopk3.cpp b/tools/quake3/q3map2/autopk3.cpp index de3007d2..4b731bae 100644 --- a/tools/quake3/q3map2/autopk3.cpp +++ b/tools/quake3/q3map2/autopk3.cpp @@ -668,6 +668,256 @@ int pk3BSPMain( Args& args ){ } +static void findShaderDups(){ + LoadShaderInfo(); + struct ShaderMaps + { + String64 box; + StrList maps; + const shaderInfo_t *si; + + bool equal_box( const ShaderMaps& other ) const { + return striEqual( box, other.box ); + } + bool equal_maps( const ShaderMaps& other ) const { + if( maps.size() != other.maps.size() ) + return false; + for( size_t i = 0; i < maps.size(); ++i ){ + if( !striEqual( maps[i], other.maps[i] ) ){ + return false; + } + } + return true; + } + }; + + std::vector shaderMapsList; + + for( const auto& si : Span( shaderInfo, shaderInfo + numShaderInfo ) ){ + /* load the shader */ + ParseFromMemory( si.shaderText, strlen( si.shaderText ) + 1 ); + + /* tokenize it */ + { + /* handle { } section */ + if ( !( GetToken( true ) && strEqual( token, "{" ) ) ) { + Error( "ParseShader: %s, line %d: { not found!\nFound instead: %s\nFile location be: %s", + si.shader.c_str(), scriptline, token, g_strLoadedFileLocation ); + } + + bool hasmap = false; + ShaderMaps& shaderMaps = shaderMapsList.emplace_back(); + shaderMaps.si = &si; + + while ( GetToken( true ) && !strEqual( token, "}" ) ) + { + /* parse stage directives */ + if ( strEqual( token, "{" ) ) { + while ( GetToken( true ) && !strEqual( token, "}" ) ) + { + /* digest any images */ + if ( striEqual( token, "map" ) || + striEqual( token, "clampMap" ) ) { + hasmap = true; + + /* get an image */ + GetToken( false ); + if ( token[ 0 ] != '*' && token[ 0 ] != '$' ) { + shaderMaps.maps.emplace_back( token ); + } + } + else if ( striEqual( token, "animMap" ) || + striEqual( token, "clampAnimMap" ) ) { + hasmap = true; + + GetToken( false );// skip num + while ( TokenAvailable() ){ + shaderMaps.maps.emplace_back( token ); + GetToken( false ); // append token, modified by tex2list() + } + } + else if ( striEqual( token, "videoMap" ) ){ + hasmap = true; + GetToken( false ); + FixDOSName( token ); + shaderMaps.maps.emplace_back( token ); + } + else if ( striEqual( token, "mapComp" ) || striEqual( token, "mapNoComp" ) || striEqual( token, "animmapcomp" ) || striEqual( token, "animmapnocomp" ) ){ + Sys_FPrintf( SYS_WRN, "WARNING7: %s : %s shader\n", si.shader.c_str(), token ); + hasmap = true; + } + } + } + + /* skyparms */ + else if ( striEqual( token, "skyParms" ) ) { + hasmap = true; + /* get image base */ + GetToken( false ); + + /* ignore bogus paths */ + if ( !strEqual( token, "-" ) && !striEqual( token, "full" ) ) { + shaderMaps.box = token; + } + /* skip rest of line */ + GetToken( false ); + GetToken( false ); + } + } + } + } + + StringOutputStream outa, outb; + + const auto save = [&outa, &outb]( const char *suffix ){ + StringOutputStream filename( 256 ); + filename( g_enginePath, "shadersDiff", suffix, "A.shader" ); + SaveFile( filename, outa.c_str(), outa.end() - outa.begin() ); + Sys_Printf( "\nSaved to %s\n", filename.c_str() ); + filename( g_enginePath, "shadersDiff", suffix, "B.shader" ); + SaveFile( filename, outb.c_str(), outb.end() - outb.begin() ); + Sys_Printf( "\nSaved to %s\n", filename.c_str() ); + outa.clear(); + outb.clear(); + }; + + for( auto a = shaderMapsList.cbegin(); a != shaderMapsList.cend(); ++a ) + { + for( auto b = a + 1; b != shaderMapsList.cend(); ++b ) + { + if( a->equal_box( *b ) && a->equal_maps( *b ) ){ + outa << "\n//========================================\n"; + outb << "\n//========================================\n"; + outa << a->si->shader << a->si->shaderText << '\n'; + outb << b->si->shader << b->si->shaderText << '\n'; + } + } + } + save( "" ); +#if 1 + for( auto a = shaderMapsList.cbegin(); a != shaderMapsList.cend(); ++a ) + { + for( auto b = a + 1; b != shaderMapsList.cend(); ++b ) + { + if( !a->maps.empty() && a->equal_maps( *b ) ){ + outa << "\n//========================================@@@@@@@@===EQ_CLOUDS\n"; + outb << "\n//========================================@@@@@@@@===EQ_CLOUDS\n"; + outa << a->si->shader << a->si->shaderText << '\n'; + outb << b->si->shader << b->si->shaderText << '\n'; + } + } + } + save( "_EQ_CLOUDS_" ); + + std::sort( shaderMapsList.begin(), shaderMapsList.end(), + []( const ShaderMaps& a, const ShaderMaps& b ){ + return ( string_compare_nocase( a.box, b.box ) < 0 ); + } ); + for( auto a = shaderMapsList.cbegin(); a != shaderMapsList.cend(); ++a ) + { + for( auto b = a + 1; b != shaderMapsList.cend(); ++b ) + { + if( !a->box.empty() && a->equal_box( *b ) ){ + outa << "\n//========================================######===EQ_BOX\n"; + outb << "\n//========================================######===EQ_BOX\n"; + outa << a->si->shader << a->si->shaderText << '\n'; + outb << b->si->shader << b->si->shaderText << '\n'; + } + } + } + save( "_EQ_BOX_" ); + + for( auto a = shaderMapsList.cbegin(); a != shaderMapsList.cend(); ++a ) + { + for( auto b = a + 1; b != shaderMapsList.cend(); ++b ) + { + if( !a->equal_maps( *b ) ){ + for( const auto& amap : a->maps ) + for( const auto& bmap : b->maps ) + if( striEqual( amap, bmap ) ){ + outa << "\n//===========@@@@@@@@===EQ_MAP: " << amap << '\n'; + outb << "\n//===========@@@@@@@@===EQ_MAP: " << bmap << '\n'; + outa << a->si->shader << a->si->shaderText << '\n'; + outb << b->si->shader << b->si->shaderText << '\n'; + } + } + } + } + save( "_EQ_MAP_" ); +#endif + std::sort( shaderInfo, shaderInfo + numShaderInfo, + []( const shaderInfo_t& a, const shaderInfo_t& b ){ + return ( string_compare_nocase( a.shader, b.shader ) < 0 ); + } ); + for( auto si = shaderInfo; si != shaderInfo + numShaderInfo - 1; ++si ) + if( striEqual( si->shader, (si + 1)->shader ) ){ + outa << "\n//========================================$$$===EQ_NAME\n"; + outb << "\n//========================================$$$===EQ_NAME\n"; + outa << si->shader << si->shaderText << '\n'; + outb << (si + 1)->shader << (si + 1)->shaderText << '\n'; + } + save( "_EQ_NAME_" ); +} + +// checks shader resources presence +class ShaderValidator +{ +private: + const bool m_doThings; + const StrList& m_excludedTexs; + bool m_usesBase = false; + bool m_usesNotBase = false; + bool m_isPartiallyValid = false; //has some present res + bool m_isInvalid = false; //has some missing res + StrList m_missing; +public: + ShaderValidator( bool doThings, const StrList& excludedTexs ) : m_doThings( doThings ), m_excludedTexs( excludedTexs ){ + } + void test( const char *intex ){ + if( m_doThings ){ + const auto tex = String64()( PathExtensionless( intex ) ); + if( StrList_find( m_excludedTexs, tex ) ){ + m_usesBase = true; + m_isPartiallyValid = true; + } + else{ + m_usesNotBase = true; + bool found = false; + for( const auto ext : { ".png", ".tga", ".jpg" } ) + { + if( vfsGetFileCount( String64()( tex, ext ) ) ){ + found = true; + break; + } + } + m_isPartiallyValid |= found; + m_isInvalid |= !found; + if( !found ) + m_missing.emplace_back( intex ); + } + } + } + bool valid() const { + return !m_doThings || ( m_isPartiallyValid ); + } + const StringOutputStream getComment() const { + StringOutputStream str; + if( m_doThings ){ + if( m_isInvalid ){ + for( const auto& miss : m_missing ) + str << "\n//! m_isInvalid " << miss; + } + if( m_usesBase ) + if( !m_usesNotBase ) + str << "\n//! m_isPureBase"; + else + str << "\n//! m_usesBase"; + } + return str; + } +}; + + /* repackBSPMain() repack multiple maps, strip out only required shaders @@ -695,6 +945,14 @@ int repackBSPMain( Args& args ){ Sys_Printf( "Compression level set to %i\n", compLevel ); } } + const bool validateShaders = args.takeArg( "-validateShaders" ); + const bool packDups = args.takeArg( "-packDups" ); + const bool packQer = args.takeArg( "-packQer" ); + + if( args.takeArg( "-shaderDups" ) ){ + findShaderDups(); + return 0; + } /* extract pack name */ const CopiedString nameOFpack( PathFilename( fileName ) ); @@ -757,6 +1015,22 @@ int repackBSPMain( Args& args ){ } } + std::sort( shaderInfo, shaderInfo + numShaderInfo, + []( const shaderInfo_t& a, const shaderInfo_t& b ){ + if( string_compare_nocase( a.shader, b.shader ) < 0 ) + return true; + else if( string_compare_nocase( a.shader, b.shader ) > 0 ) + return false; + else if( string_compare_nocase( a.skyParmsImageBase, b.skyParmsImageBase ) < 0 ) + return true; + else + return false; + } ); + for( auto si = shaderInfo; si != shaderInfo + numShaderInfo - 1; ++si ) + if( striEqual( si->shader, (si + 1)->shader ) && !striEqual( si->skyParmsImageBase, (si + 1)->skyParmsImageBase ) ) + Sys_Printf( "WARNING666: %s : %s | %s\n", si->shader.c_str(), si->skyParmsImageBase.c_str(), (si + 1)->skyParmsImageBase.c_str() ); + + pk3Shaderfiles = vfsListShaderFiles( g_game->shaderPath ); if( dbg ){ @@ -803,6 +1077,8 @@ int repackBSPMain( Args& args ){ /* do wanna le shader? */ String64 *wantShader = StrList_find( pk3Shaders, token ); + ShaderValidator shaderValidator( validateShaders, ex.textures ); + /* handle { } section */ if ( !( text.GetToken( true ) && strEqual( token, "{" ) ) ) { Error( "ParseShaderFile: %s, line %d: { not found!\nFound instead: %s\nFile location be: %s", @@ -830,6 +1106,7 @@ int repackBSPMain( Args& args ){ GetToken( false ); if ( token[ 0 ] != '*' && token[ 0 ] != '$' ) { tex2list( pk3Textures, ex.textures, &rex.textures ); + shaderValidator.test( token ); } text.tokenAppend(); // append token, modified by tex2list() } @@ -840,6 +1117,7 @@ int repackBSPMain( Args& args ){ text.GetToken( false );// skip num while ( TokenAvailable() ){ tex2list( pk3Textures, ex.textures, &rex.textures ); + shaderValidator.test( token ); text.GetToken( false ); // append token, modified by tex2list() } } @@ -877,12 +1155,20 @@ int repackBSPMain( Args& args ){ for( const auto side : { "up", "dn", "lf", "rt", "bk", "ft" } ){ memcpy( skysidestring, side, 2 ); tex2list( pk3Textures, ex.textures, &rex.textures ); + shaderValidator.test( token ); } } /* skip rest of line */ text.GetToken( false ); text.GetToken( false ); } + else if ( packQer && striEqual( token, "qer_editorimage" ) ) { + /* get an image */ + GetToken( false ); + tex2list( pk3Textures, ex.textures, &rex.textures ); + shaderValidator.test( token ); + text.tokenAppend(); // append token, modified by tex2list() + } else if ( striEqualPrefix( token, "implicit" ) ){ Sys_FPrintf( SYS_WRN, "WARNING5: %s : %s shader\n", wantShader->c_str(), token ); hasmap = true; @@ -905,7 +1191,8 @@ int repackBSPMain( Args& args ){ wantShader = nullptr; } if ( wantShader ){ - allShaders << text.text << '\n'; + if( shaderValidator.valid() ) + allShaders << shaderValidator.getComment() << text.text << '\n'; wantShader->clear(); } } @@ -953,6 +1240,9 @@ int repackBSPMain( Args& args ){ SaveFile( stream, allShaders.c_str(), allShaders.end() - allShaders.begin() ); Sys_Printf( "Shaders saved to %s\n", stream.c_str() ); + if( validateShaders ) + return 0; + /* make a pack */ stream( g_enginePath, nameOFpack, "_repacked.pk3" ); remove( stream ); @@ -962,7 +1252,7 @@ int repackBSPMain( Args& args ){ Sys_Printf( "\n\tShader referenced textures....\n" ); for ( const auto& s : pk3Textures ){ - if( !packTexture( s, stream, compLevel, png ) ){ + if( !( ( packDups && vfsPackSkyImage( s, stream, compLevel ) ) || ( !packDups && packTexture( s, stream, compLevel, png ) ) ) ){ Sys_FPrintf( SYS_WRN, " !FAIL! %s\n", s.c_str() ); } } @@ -971,7 +1261,7 @@ int repackBSPMain( Args& args ){ for ( const auto& s : pk3Shaders ){ if ( !s.empty() ){ - if( !packTexture( s, stream, compLevel, png ) ){ + if( !( ( packDups && vfsPackSkyImage( s, stream, compLevel ) ) || ( !packDups && packTexture( s, stream, compLevel, png ) ) ) ){ Sys_FPrintf( SYS_WRN, " !FAIL! %s\n", s.c_str() ); } } @@ -997,6 +1287,9 @@ int repackBSPMain( Args& args ){ Sys_Printf( "\nSaved to %s\n", stream.c_str() ); + if( packDups ) + vfsFindSkyFiles( std::vector( pk3Textures.cbegin(), pk3Textures.cend() ) ); + /* return to sender */ return 0; } diff --git a/tools/quake3/q3map2/q3map2.h b/tools/quake3/q3map2/q3map2.h index 78ca120b..04137954 100644 --- a/tools/quake3/q3map2/q3map2.h +++ b/tools/quake3/q3map2/q3map2.h @@ -100,7 +100,7 @@ #define DEF_RADIOSITY_BOUNCE 1.0f /* ydnar: default to 100% re-emitted light */ -#define MAX_SHADER_INFO 8192 * 16 +#define MAX_SHADER_INFO 8192 * 32 /* epair parsing (note case-sensitivity directive) */ diff --git a/tools/quake3/q3map2/shaders.cpp b/tools/quake3/q3map2/shaders.cpp index 7e7adc24..6ed20b43 100644 --- a/tools/quake3/q3map2/shaders.cpp +++ b/tools/quake3/q3map2/shaders.cpp @@ -1858,7 +1858,7 @@ void LoadShaderInfo(){ /* we can pile up several shader files, the one in baseq3 and ones in the mod dir or other spots */ const auto filename = StringOutputStream( 64 )( g_game->shaderPath, "/shaderlist.txt" ); - const int count = vfsGetFileCount( filename ); + const int count = 0; // vfsGetFileCount( filename ); /* load them all */ for ( int i = 0; i < count; i++ )