repack skies mod:

pack all found sky shaders
WARNING444 on texture reuse
MAX_SHADER_INFO bumped
load all uniquely named shader files, only from folders, not packs
VFS mod to not keep pk3s opened (avoid hitting OS half opened files limit)(risky and slow for normal use)
-packDups:
	vfsPackSkyImage: pack only new images, referring to main and dups DBs; pack alt image versions to separate pk3s
	vfsFindSkyFiles: pack sky images, not referenced by any shader and not present in main DB and dups DB (crc check)
normal vfsPackFile: split output pk3s to approximately 2 GB chunks
-packQer: also pack qer_editorimage
-validateShaders: useful comments about referenced textures state right in out shader, do not save entirely invalid shaders (no packaging)
-shaderDups: write similar shaders to files for latter evaluation by diff tool (no packaging)
WARNING666 when equally named shaders reference different sky boxes

tools/quake3/common/scriplib.cpp

@@ -105,7 +105,7 @@ bool LoadScriptFile( const char *filename, int index /* = 0 */, bool verbose /*
    ParseFromMemory
    ==============
  */
-void ParseFromMemory( char *buffer, size_t size ){
+void ParseFromMemory( const char *buffer, size_t size ){
 	scriptstack.clear();
 	tokenready = false;
 	MemBuffer bu( size );

tools/quake3/common/scriplib.h

@@ -32,7 +32,7 @@ extern int scriptline;
 /// \param[in] index >= 0: \p filename is relative path in VSF, Nth occurrence of file
 /// \return true on success
 bool LoadScriptFile( const char *filename, int index = 0, bool verbose = true );
-void ParseFromMemory( char *buffer, size_t size );
+void ParseFromMemory( const char *buffer, size_t size );
 
 /// \param[in] crossline true: write next token to \c token or return false on EOF
 /// \param[in] crossline false: find next token on the current line or emit \c Error

tools/quake3/common/vfs.cpp

@@ -73,12 +73,12 @@
 
 struct VFS_PAK
 {
-	unzFile zipfile;
+	// unzFile zipfile;
 	const CopiedString unzFilePath;
-	VFS_PAK( unzFile zipfile, const char *unzFilePath ) : zipfile( zipfile ), unzFilePath( unzFilePath ) {};
+	VFS_PAK( const char *unzFilePath ) : unzFilePath( unzFilePath ) {};
 	VFS_PAK( VFS_PAK&& ) noexcept = delete;
 	~VFS_PAK(){
-		unzClose( zipfile );
+		// unzClose( zipfile );
 	}
 };
 
@@ -90,6 +90,27 @@ struct VFS_PAKFILE
 	const guint32 size;
 };
 
+
+static MemBuffer read_file_from_pak( const VFS_PAKFILE& file ){
+	MemBuffer buffer;
+	unzFile zipfile = unzOpen( file.pak.unzFilePath.c_str() );
+	if( zipfile != nullptr ){
+		*(unz_s*)zipfile = file.zipinfo;
+
+		if ( unzOpenCurrentFile( zipfile ) == UNZ_OK ) {
+			buffer = MemBuffer( file.size );
+
+			if ( unzReadCurrentFile( zipfile, buffer.data(), file.size ) < 0 ) {
+				buffer = MemBuffer();
+			}
+			unzCloseCurrentFile( zipfile );
+		}
+		unzClose( zipfile );
+	}
+	return buffer;
+}
+
+
 // =============================================================================
 // Global variables
 
@@ -106,7 +127,7 @@ char g_strLoadedFileLocation[1024];
 static void vfsInitPakFile( const char *filename ){
 	unzFile uf = unzOpen( filename );
 	if ( uf != NULL ) {
-		VFS_PAK& pak = g_paks.emplace_front( uf, filename );
+		VFS_PAK& pak = g_paks.emplace_front( filename );
 
 		if ( unzGoToFirstFile( uf ) == UNZ_OK ) {
 			do {
@@ -129,6 +150,7 @@ static void vfsInitPakFile( const char *filename ){
 					} );
 				}
 			} while( unzGoToNextFile( uf ) == UNZ_OK );
+			unzClose( uf );
 		}
 	}
 }
@@ -195,6 +217,70 @@ void vfsInitDirectory( const char *path ){
 }
 
 
+#include <set>
+
+void vfsFindSkyFiles( const std::vector<CopiedString>& present_list ){
+	for ( const VFS_PAKFILE& file : g_pakFiles )
+	{
+		const char *name = file.name.c_str();
+		if ( path_extension_is( name, "tga" ) || path_extension_is( name, "jpg" ) || path_extension_is( name, "png" ) ) {
+			auto n = StringOutputStream( 64 )( PathExtensionless( name ) );
+			if( string_equal_suffix_nocase( n, "_bk" )
+			 || string_equal_suffix_nocase( n, "_rt" )
+			 || string_equal_suffix_nocase( n, "_ft" )
+			 || string_equal_suffix_nocase( n, "_lf" )
+			 || string_equal_suffix_nocase( n, "_up" )
+			 || string_equal_suffix_nocase( n, "_dn" )
+			 ){
+				bool found = false;
+				for( const auto& pre : present_list )
+					if( striEqual( pre.c_str(), n ) )
+						found = true;
+				if( !found ){
+					auto add_crc = [crcs = std::set<unsigned long>()]( const MemBuffer& boo )mutable->bool{
+						return crcs.insert( crc32( 0, boo.data(), boo.size() ) ).second;
+					};
+
+					struct FTYPE
+					{
+						const char *ext;
+						int idx;
+					};
+					FTYPE ftypes[3]{ { ".png", 1 }, { ".tga", 1 }, { ".jpg", 1 } };
+
+					for( auto& ftype : ftypes )
+					{
+						const StringOutputStream fullnames[] = { StringOutputStream( 256 )( "c:/qsky/__/skpk/", n, ftype.ext ),
+						StringOutputStream( 256 )( "c:/qsky/__/dups/", n, ftype.ext ),
+						StringOutputStream( 256 )( "c:/qsky/__/dups/", n, " (2)", ftype.ext ),
+						StringOutputStream( 256 )( "c:/qsky/__/dups/", n, " (3)", ftype.ext ),
+						StringOutputStream( 256 )( "c:/qsky/__/dups/", n, " (4)", ftype.ext ),
+						StringOutputStream( 256 )( "c:/qsky/__/dups/", n, " (5)", ftype.ext ) };
+						for( const auto& fullname : fullnames )
+						{
+							if( FileExists( fullname ) ){
+								++ftype.idx;
+								add_crc( LoadFile( fullname ) );
+							}
+						}
+					}
+
+
+					if( MemBuffer buffer = read_file_from_pak( file ) ){
+						if( add_crc( buffer ) ){
+							Sys_Printf( "WARNING777: %s :: %s\n", name, file.pak.unzFilePath.c_str() );
+							if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( "c:/qsky/test_bigbox_repacked_noShader.pk3", name, buffer.data(), buffer.size(), 0, 0, 9, file.zipinfo.cur_file_info.dosDate ) ){
+								Error( "Failed creating zip archive \"%s\"!\n", "c:/qsky/test_bigbox_repacked_noShader.pk3" );
+							}
+						}
+					}
+				}
+			 }
+		}
+	}
+}
+
+
 // lists all unique .shader files with extension and w/o path
 std::vector<CopiedString> vfsListShaderFiles( const char *shaderPath ){
 	std::vector<CopiedString> list;
@@ -220,6 +306,7 @@ std::vector<CopiedString> vfsListShaderFiles( const char *shaderPath ){
 		}
 	}
 	/* search in packs */
+#if 0
 	for ( const VFS_PAKFILE& file : g_pakFiles )
 	{
 		const char *name = file.name.c_str();
@@ -228,7 +315,7 @@ std::vector<CopiedString> vfsListShaderFiles( const char *shaderPath ){
 			insert( path_get_filename_start( name ) );
 		}
 	}
-
+#endif
 	return list;
 }
 
@@ -302,36 +389,143 @@ MemBuffer vfsLoadFile( const char *filename, int index /* = 0 */ ){
 	auto fixed = StringOutputStream( 64 )( PathCleaned( filename ) );
 	strLower( fixed.c_str() );
 
-	MemBuffer buffer;
-
 	for ( const VFS_PAKFILE& file : g_pakFiles )
 	{
 		if ( strEqual( file.name.c_str(), fixed ) && 0 == index-- )
 		{
 			snprintf( g_strLoadedFileLocation, sizeof( g_strLoadedFileLocation ), "%s :: %s", file.pak.unzFilePath.c_str(), filename );
 
-			unzFile zipfile = file.pak.zipfile;
-			*(unz_s*)zipfile = file.zipinfo;
-
-			if ( unzOpenCurrentFile( zipfile ) == UNZ_OK ) {
-				buffer = MemBuffer( file.size );
-
-				if ( unzReadCurrentFile( zipfile, buffer.data(), file.size ) < 0 ) {
-					buffer = MemBuffer();
-				}
-				unzCloseCurrentFile( zipfile );
-			}
-			return buffer;
+			// unzFile zipfile = file.pak.zipfile;
+			return read_file_from_pak( file );
 		}
 	}
 
-	return buffer;
+	return {};
 }
 
 
 
 
-bool vfsPackFile( const char *filename, const char *packname, const int compLevel ){
+#include <filesystem>
+
+bool vfsPackSkyImage( const char *filename, const char *packname, const int compLevel ){
+	bool packed = false;
+	bool resInMain = false;
+
+	auto add_crc = [crcs = std::set<unsigned long>()]( const MemBuffer& boo )mutable->bool{
+		return crcs.insert( crc32( 0, boo.data(), boo.size() ) ).second;
+	};
+
+	struct FTYPE
+	{
+		const char *ext;
+		int idx;
+	};
+	FTYPE ftypes[3]{ { ".png", 1 }, { ".tga", 1 }, { ".jpg", 1 } };
+
+	for( auto& ftype : ftypes )
+	{
+		const StringOutputStream fullnames[] = { StringOutputStream( 256 )( "c:/qsky/__/skpk/", filename, ftype.ext ),
+		StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, ftype.ext ),
+		StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, " (2)", ftype.ext ),
+		StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, " (3)", ftype.ext ),
+		StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, " (4)", ftype.ext ),
+		StringOutputStream( 256 )( "c:/qsky/__/dups/", filename, " (5)", ftype.ext ) };
+		for( const auto& fullname : fullnames )
+		{
+			if( FileExists( fullname ) ){
+				resInMain = packed = true;
+				++ftype.idx;
+				add_crc( LoadFile( fullname ) );
+			}
+		}
+	}
+
+	for( auto& ftype : ftypes )
+	{
+		for ( const auto& dir : g_strDirs )
+		{
+			const auto fullname = StringOutputStream( 256 )( dir, filename, ftype.ext );
+			if( FileExists( fullname ) && add_crc( LoadFile( fullname ) ) ){
+				if( !packed ){
+					packed = vfsPackFile_Absolute_Path( fullname, StringOutputStream( 256 )( filename, ftype.ext ), packname, compLevel );
+				}
+				else{
+					#if 0
+					auto toname = StringOutputStream( 256 )( "c:/qsky/m_dups/", filename );
+					if( idx > 1 )
+						toname << " (" << idx << ")";
+					toname << ext;
+					std::filesystem::create_directories( std::filesystem::path( toname.c_str() ).remove_filename() );
+					std::filesystem::copy_file( fullname.c_str(), toname.c_str() );
+					#else
+					auto toname = StringOutputStream( 256 )( filename );
+					if( ftype.idx > 1 ) // original name for files in alt format, (2) etc suffix for alt versions in the same format
+						toname << " (" << ftype.idx << ")";
+					toname << ftype.ext;
+					if( resInMain )
+						vfsPackFile_Absolute_Path( fullname, toname, StringOutputStream( 256 )( PathExtensionless( packname ), "_altMain.", path_get_extension( packname ) ), compLevel );
+					else
+						vfsPackFile_Absolute_Path( fullname, toname, StringOutputStream( 256 )( PathExtensionless( packname ), "_altNew.", path_get_extension( packname ) ), compLevel );
+					#endif
+				}
+				++ftype.idx;
+			}
+		}
+	}
+
+	auto fixed = StringOutputStream( 64 )( PathCleaned( filename ) );
+	strLower( fixed.c_str() );
+
+	for( auto& ftype : ftypes )
+	{
+		const auto fullfixed = StringOutputStream( 256 )( fixed, ftype.ext );
+		for ( const VFS_PAKFILE& file : g_pakFiles )
+		{
+			if ( strEqual( file.name.c_str(), fullfixed ) )
+			{
+				// unzFile zipfile = file.pak.zipfile;
+				if ( MemBuffer buffer = read_file_from_pak( file ) ) {
+					if ( add_crc( buffer ) ) {
+						if( !packed ){
+							packed = true;
+							if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( packname, StringOutputStream( 256 )( filename, ftype.ext ), buffer.data(), buffer.size(), 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){
+								Error( "Failed creating zip archive \"%s\"!\n", packname );
+							}
+						}
+						else{
+							auto toname = StringOutputStream( 256 )( filename );
+							if( ftype.idx > 1 ) // original name for files in alt format, (2) etc suffix for alt versions in the same format
+								toname << " (" << ftype.idx << ")";
+							toname << ftype.ext;
+							if( resInMain ){
+								if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( StringOutputStream( 256 )( PathExtensionless( packname ), "_altMain.", path_get_extension( packname ) ), toname, buffer.data(), buffer.size(), 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){
+									Error( "Failed creating zip archive \"%s\"!\n", packname );
+								}
+							}
+							else{
+								if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( StringOutputStream( 256 )( PathExtensionless( packname ), "_altNew.", path_get_extension( packname ) ), toname, buffer.data(), buffer.size(), 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){
+									Error( "Failed creating zip archive \"%s\"!\n", packname );
+								}
+							}
+						}
+						++ftype.idx;
+					}
+				}
+			}
+		}
+	}
+	return packed;
+}
+
+bool vfsPackFile( const char *filename, const char *packname_in, const int compLevel ){
+	static int packNum = 0;
+	auto packname = StringOutputStream( 256 )( PathExtensionless( packname_in ), packNum, '.', path_get_extension( packname_in ) );
+	if( std::filesystem::exists( packname.c_str() ) && std::filesystem::file_size( packname.c_str() ) > 2147483648 - 10485760 ){
+		++packNum;
+		packname( PathExtensionless( packname_in ), packNum, '.', path_get_extension( packname_in ) );
+	}
+
 	for ( const auto& dir : g_strDirs )
 	{
 		if( vfsPackFile_Absolute_Path( StringOutputStream( 256 )( dir, filename ), filename, packname, compLevel ) )
@@ -345,20 +539,12 @@ bool vfsPackFile( const char *filename, const char *packname, const int compLeve
 	{
 		if ( strEqual( file.name.c_str(), fixed ) )
 		{
-			unzFile zipfile = file.pak.zipfile;
-			*(unz_s*)zipfile = file.zipinfo;
-
-			if ( unzOpenCurrentFile( zipfile ) == UNZ_OK ) {
-				MemBuffer buffer( file.size );
-
-				const int size = unzReadCurrentFile( zipfile, buffer.data(), file.size );
-				unzCloseCurrentFile( zipfile );
-				if ( size >= 0 ) {
-					if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( packname, filename, buffer.data(), size, 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){
-						Error( "Failed creating zip archive \"%s\"!\n", packname );
-					}
-					return true;
+			// unzFile zipfile = file.pak.zipfile;
+			if( MemBuffer buffer = read_file_from_pak( file ) ){
+				if ( !mz_zip_add_mem_to_archive_file_in_place_with_time( packname, filename, buffer.data(), buffer.size(), 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate ) ){
+					Error( "Failed creating zip archive \"%s\"!\n", packname.c_str() );
 				}
+				return true;
 			}
 			return false;
 		}

tools/quake3/common/vfs.h

@@ -40,6 +40,8 @@ int vfsGetFileCount( const char *filename );
 MemBuffer vfsLoadFile( const char *filename, int index = 0 );
 std::vector<CopiedString> vfsListShaderFiles( const char *shaderPath );
 bool vfsPackFile( const char *filename, const char *packname, const int compLevel );
+bool vfsPackSkyImage( const char *filename, const char *packname, const int compLevel );
+void vfsFindSkyFiles( const std::vector<CopiedString>& present_list );
 bool vfsPackFile_Absolute_Path( const char *filepath, const char *filename, const char *packname, const int compLevel );
 
 extern std::vector<CopiedString> g_strForbiddenDirs;

tools/quake3/q3map2/autopk3.cpp

@@ -668,6 +668,256 @@ int pk3BSPMain( Args& args ){
 }
 
 
+static void findShaderDups(){
+	LoadShaderInfo();
+	struct ShaderMaps
+	{
+		String64 box;
+		StrList maps;
+		const shaderInfo_t *si;
+
+		bool equal_box( const ShaderMaps& other ) const {
+			return striEqual( box, other.box );
+		}
+		bool equal_maps( const ShaderMaps& other ) const {
+			if( maps.size() != other.maps.size() )
+				return false;
+			for( size_t i = 0; i < maps.size(); ++i ){
+				if( !striEqual( maps[i], other.maps[i] ) ){
+					return false;
+				}
+			}
+			return true;
+		}
+	};
+
+	std::vector<ShaderMaps> shaderMapsList;
+
+	for( const auto& si : Span( shaderInfo, shaderInfo + numShaderInfo ) ){
+		/* load the shader */
+		ParseFromMemory( si.shaderText, strlen( si.shaderText ) + 1 );
+
+		/* tokenize it */
+		{
+			/* handle { } section */
+			if ( !( GetToken( true ) && strEqual( token, "{" ) ) ) {
+				Error( "ParseShader: %s, line %d: { not found!\nFound instead: %s\nFile location be: %s",
+				       si.shader.c_str(), scriptline, token, g_strLoadedFileLocation );
+			}
+
+			bool hasmap = false;
+			ShaderMaps& shaderMaps = shaderMapsList.emplace_back();
+			shaderMaps.si = &si;
+
+			while ( GetToken( true ) && !strEqual( token, "}" ) )
+			{
+				/* parse stage directives */
+				if ( strEqual( token, "{" ) ) {
+					while ( GetToken( true ) && !strEqual( token, "}" ) )
+					{
+						/* digest any images */
+						if ( striEqual( token, "map" ) ||
+						     striEqual( token, "clampMap" ) ) {
+							hasmap = true;
+
+							/* get an image */
+							GetToken( false );
+							if ( token[ 0 ] != '*' && token[ 0 ] != '$' ) {
+								shaderMaps.maps.emplace_back( token );
+							}
+						}
+						else if ( striEqual( token, "animMap" ) ||
+						          striEqual( token, "clampAnimMap" ) ) {
+							hasmap = true;
+
+							GetToken( false );// skip num
+							while ( TokenAvailable() ){
+								shaderMaps.maps.emplace_back( token );
+								GetToken( false ); // append token, modified by tex2list()
+							}
+						}
+						else if ( striEqual( token, "videoMap" ) ){
+							hasmap = true;
+							GetToken( false );
+							FixDOSName( token );
+							shaderMaps.maps.emplace_back( token );
+						}
+						else if ( striEqual( token, "mapComp" ) || striEqual( token, "mapNoComp" ) || striEqual( token, "animmapcomp" ) || striEqual( token, "animmapnocomp" ) ){
+							Sys_FPrintf( SYS_WRN, "WARNING7: %s : %s shader\n", si.shader.c_str(), token );
+							hasmap = true;
+						}
+					}
+				}
+
+				/* skyparms <outer image> <cloud height> <inner image> */
+				else if ( striEqual( token, "skyParms" ) ) {
+					hasmap = true;
+					/* get image base */
+					GetToken( false );
+
+					/* ignore bogus paths */
+					if ( !strEqual( token, "-" ) && !striEqual( token, "full" ) ) {
+						shaderMaps.box = token;
+					}
+					/* skip rest of line */
+					GetToken( false );
+					GetToken( false );
+				}
+			}
+		}
+	}
+
+	StringOutputStream outa, outb;
+
+	const auto save = [&outa, &outb]( const char *suffix ){
+		StringOutputStream filename( 256 );
+		filename( g_enginePath, "shadersDiff", suffix, "A.shader" );
+		SaveFile( filename, outa.c_str(), outa.end() - outa.begin() );
+		Sys_Printf( "\nSaved to %s\n", filename.c_str() );
+		filename( g_enginePath, "shadersDiff", suffix, "B.shader" );
+		SaveFile( filename, outb.c_str(), outb.end() - outb.begin() );
+		Sys_Printf( "\nSaved to %s\n", filename.c_str() );
+		outa.clear();
+		outb.clear();
+	};
+
+	for( auto a = shaderMapsList.cbegin(); a != shaderMapsList.cend(); ++a )
+	{
+		for( auto b = a + 1; b != shaderMapsList.cend(); ++b )
+		{
+			if( a->equal_box( *b ) && a->equal_maps( *b ) ){
+				outa << "\n//========================================\n";
+				outb << "\n//========================================\n";
+				outa << a->si->shader << a->si->shaderText << '\n';
+				outb << b->si->shader << b->si->shaderText << '\n';
+			}
+		}
+	}
+	save( "" );
+#if 1
+	for( auto a = shaderMapsList.cbegin(); a != shaderMapsList.cend(); ++a )
+	{
+		for( auto b = a + 1; b != shaderMapsList.cend(); ++b )
+		{
+			if( !a->maps.empty() && a->equal_maps( *b ) ){
+				outa << "\n//========================================@@@@@@@@===EQ_CLOUDS\n";
+				outb << "\n//========================================@@@@@@@@===EQ_CLOUDS\n";
+				outa << a->si->shader << a->si->shaderText << '\n';
+				outb << b->si->shader << b->si->shaderText << '\n';
+			}
+		}
+	}
+	save( "_EQ_CLOUDS_" );
+
+	std::sort( shaderMapsList.begin(), shaderMapsList.end(),
+	[]( const ShaderMaps& a, const ShaderMaps& b ){
+			return ( string_compare_nocase( a.box, b.box ) < 0 );
+		} );
+	for( auto a = shaderMapsList.cbegin(); a != shaderMapsList.cend(); ++a )
+	{
+		for( auto b = a + 1; b != shaderMapsList.cend(); ++b )
+		{
+			if( !a->box.empty() && a->equal_box( *b ) ){
+				outa << "\n//========================================######===EQ_BOX\n";
+				outb << "\n//========================================######===EQ_BOX\n";
+				outa << a->si->shader << a->si->shaderText << '\n';
+				outb << b->si->shader << b->si->shaderText << '\n';
+			}
+		}
+	}
+	save( "_EQ_BOX_" );
+
+	for( auto a = shaderMapsList.cbegin(); a != shaderMapsList.cend(); ++a )
+	{
+		for( auto b = a + 1; b != shaderMapsList.cend(); ++b )
+		{
+			if( !a->equal_maps( *b ) ){
+				for( const auto& amap : a->maps )
+					for( const auto& bmap : b->maps )
+						if( striEqual( amap, bmap ) ){
+							outa << "\n//===========@@@@@@@@===EQ_MAP: " << amap << '\n';
+							outb << "\n//===========@@@@@@@@===EQ_MAP: " << bmap << '\n';
+							outa << a->si->shader << a->si->shaderText << '\n';
+							outb << b->si->shader << b->si->shaderText << '\n';
+						}
+			}
+		}
+	}
+	save( "_EQ_MAP_" );
+#endif
+	std::sort( shaderInfo, shaderInfo + numShaderInfo,
+	[]( const shaderInfo_t& a, const shaderInfo_t& b ){
+			return ( string_compare_nocase( a.shader, b.shader ) < 0 );
+		} );
+	for( auto si = shaderInfo; si != shaderInfo + numShaderInfo - 1; ++si )
+		if( striEqual( si->shader, (si + 1)->shader ) ){
+			outa << "\n//========================================$$$===EQ_NAME\n";
+			outb << "\n//========================================$$$===EQ_NAME\n";
+			outa << si->shader << si->shaderText << '\n';
+			outb << (si + 1)->shader << (si + 1)->shaderText << '\n';
+		}
+	save( "_EQ_NAME_" );
+}
+
+// checks shader resources presence
+class ShaderValidator
+{
+private:
+	const bool m_doThings;
+	const StrList& m_excludedTexs;
+	bool m_usesBase = false;
+	bool m_usesNotBase = false;
+	bool m_isPartiallyValid = false; //has some present res
+	bool m_isInvalid = false; //has some missing res
+	StrList m_missing;
+public:
+	ShaderValidator( bool doThings, const StrList& excludedTexs ) : m_doThings( doThings ), m_excludedTexs( excludedTexs ){
+	}
+	void test( const char *intex ){
+		if( m_doThings ){
+			const auto tex = String64()( PathExtensionless( intex ) );
+			if( StrList_find( m_excludedTexs, tex ) ){
+				m_usesBase = true;
+				m_isPartiallyValid = true;
+			}
+			else{
+				m_usesNotBase = true;
+				bool found = false;
+				for( const auto ext : { ".png", ".tga", ".jpg" } )
+				{
+					if( vfsGetFileCount( String64()( tex, ext ) ) ){
+						found = true;
+						break;
+					}
+				}
+				m_isPartiallyValid |= found;
+				m_isInvalid |= !found;
+				if( !found )
+					m_missing.emplace_back( intex );
+			}
+		}
+	}
+	bool valid() const {
+		return !m_doThings || ( m_isPartiallyValid );
+	}
+	const StringOutputStream getComment() const {
+		StringOutputStream str;
+		if( m_doThings ){
+			if( m_isInvalid ){
+				for( const auto& miss : m_missing )
+					str << "\n//! m_isInvalid " << miss;
+			}
+			if( m_usesBase )
+				if( !m_usesNotBase )
+					str << "\n//! m_isPureBase";
+				else
+					str << "\n//! m_usesBase";
+		}
+		return str;
+	}
+};
+
+
 /*
    repackBSPMain()
    repack multiple maps, strip out only required shaders
@@ -695,6 +945,14 @@ int repackBSPMain( Args& args ){
 			Sys_Printf( "Compression level set to %i\n", compLevel );
 		}
 	}
+	const bool validateShaders = args.takeArg( "-validateShaders" );
+	const bool packDups = args.takeArg( "-packDups" );
+	const bool packQer = args.takeArg( "-packQer" );
+
+	if( args.takeArg( "-shaderDups" ) ){
+		findShaderDups();
+		return 0;
+	}
 
 	/* extract pack name */
 	const CopiedString nameOFpack( PathFilename( fileName ) );
@@ -757,6 +1015,22 @@ int repackBSPMain( Args& args ){
 		}
 	}
 
+	std::sort( shaderInfo, shaderInfo + numShaderInfo,
+	[]( const shaderInfo_t& a, const shaderInfo_t& b ){
+			if( string_compare_nocase( a.shader, b.shader ) < 0 )
+				return true;
+			else if( string_compare_nocase( a.shader, b.shader ) > 0 )
+				return false;
+			else if( string_compare_nocase( a.skyParmsImageBase, b.skyParmsImageBase ) < 0 )
+				return true;
+			else
+				return false;
+		} );
+	for( auto si = shaderInfo; si != shaderInfo + numShaderInfo - 1; ++si )
+		if( striEqual( si->shader, (si + 1)->shader ) && !striEqual( si->skyParmsImageBase, (si + 1)->skyParmsImageBase ) )
+			Sys_Printf( "WARNING666: %s : %s | %s\n", si->shader.c_str(), si->skyParmsImageBase.c_str(), (si + 1)->skyParmsImageBase.c_str() );
+
+
 	pk3Shaderfiles = vfsListShaderFiles( g_game->shaderPath );
 
 	if( dbg ){
@@ -803,6 +1077,8 @@ int repackBSPMain( Args& args ){
 			/* do wanna le shader? */
 			String64 *wantShader = StrList_find( pk3Shaders, token );
 
+			ShaderValidator shaderValidator( validateShaders, ex.textures );
+
 			/* handle { } section */
 			if ( !( text.GetToken( true ) && strEqual( token, "{" ) ) ) {
 				Error( "ParseShaderFile: %s, line %d: { not found!\nFound instead: %s\nFile location be: %s",
@@ -830,6 +1106,7 @@ int repackBSPMain( Args& args ){
 							GetToken( false );
 							if ( token[ 0 ] != '*' && token[ 0 ] != '$' ) {
 								tex2list( pk3Textures, ex.textures, &rex.textures );
+								shaderValidator.test( token );
 							}
 							text.tokenAppend(); // append token, modified by tex2list()
 						}
@@ -840,6 +1117,7 @@ int repackBSPMain( Args& args ){
 							text.GetToken( false );// skip num
 							while ( TokenAvailable() ){
 								tex2list( pk3Textures, ex.textures, &rex.textures );
+								shaderValidator.test( token );
 								text.GetToken( false ); // append token, modified by tex2list()
 							}
 						}
@@ -877,12 +1155,20 @@ int repackBSPMain( Args& args ){
 						for( const auto side : { "up", "dn", "lf", "rt", "bk", "ft" } ){
 							memcpy( skysidestring, side, 2 );
 							tex2list( pk3Textures, ex.textures, &rex.textures );
+							shaderValidator.test( token );
 						}
 					}
 					/* skip rest of line */
 					text.GetToken( false );
 					text.GetToken( false );
 				}
+				else if ( packQer && striEqual( token, "qer_editorimage" ) ) {
+					/* get an image */
+					GetToken( false );
+					tex2list( pk3Textures, ex.textures, &rex.textures );
+					shaderValidator.test( token );
+					text.tokenAppend(); // append token, modified by tex2list()
+				}
 				else if ( striEqualPrefix( token, "implicit" ) ){
 					Sys_FPrintf( SYS_WRN, "WARNING5: %s : %s shader\n", wantShader->c_str(), token );
 					hasmap = true;
@@ -905,7 +1191,8 @@ int repackBSPMain( Args& args ){
 					wantShader = nullptr;
 				}
 				if ( wantShader ){
-					allShaders << text.text << '\n';
+					if( shaderValidator.valid() )
+						allShaders << shaderValidator.getComment() << text.text << '\n';
 					wantShader->clear();
 				}
 			}
@@ -953,6 +1240,9 @@ int repackBSPMain( Args& args ){
 	SaveFile( stream, allShaders.c_str(), allShaders.end() - allShaders.begin() );
 	Sys_Printf( "Shaders saved to %s\n", stream.c_str() );
 
+	if( validateShaders )
+		return 0;
+
 	/* make a pack */
 	stream( g_enginePath, nameOFpack, "_repacked.pk3" );
 	remove( stream );
@@ -962,7 +1252,7 @@ int repackBSPMain( Args& args ){
 	Sys_Printf( "\n\tShader referenced textures....\n" );
 
 	for ( const auto& s : pk3Textures ){
-		if( !packTexture( s, stream, compLevel, png ) ){
+		if( !( ( packDups && vfsPackSkyImage( s, stream, compLevel ) ) || ( !packDups && packTexture( s, stream, compLevel, png ) ) ) ){
 			Sys_FPrintf( SYS_WRN, "  !FAIL! %s\n", s.c_str() );
 		}
 	}
@@ -971,7 +1261,7 @@ int repackBSPMain( Args& args ){
 
 	for ( const auto& s : pk3Shaders ){
 		if ( !s.empty() ){
-			if( !packTexture( s, stream, compLevel, png ) ){
+			if( !( ( packDups && vfsPackSkyImage( s, stream, compLevel ) ) || ( !packDups && packTexture( s, stream, compLevel, png ) ) ) ){
 				Sys_FPrintf( SYS_WRN, "  !FAIL! %s\n", s.c_str() );
 			}
 		}
@@ -997,6 +1287,9 @@ int repackBSPMain( Args& args ){
 
 	Sys_Printf( "\nSaved to %s\n", stream.c_str() );
 
+	if( packDups )
+		vfsFindSkyFiles( std::vector<CopiedString>( pk3Textures.cbegin(), pk3Textures.cend() ) );
+
 	/* return to sender */
 	return 0;
 }

tools/quake3/q3map2/q3map2.h

@@ -100,7 +100,7 @@
 
 #define DEF_RADIOSITY_BOUNCE    1.0f    /* ydnar: default to 100% re-emitted light */
 
-#define MAX_SHADER_INFO         8192 * 16
+#define MAX_SHADER_INFO         8192 * 32
 
 
 /* epair parsing (note case-sensitivity directive) */

tools/quake3/q3map2/shaders.cpp

@@ -1858,7 +1858,7 @@ void LoadShaderInfo(){
 
 	/* we can pile up several shader files, the one in baseq3 and ones in the mod dir or other spots */
 	const auto filename = StringOutputStream( 64 )( g_game->shaderPath, "/shaderlist.txt" );
-	const int count = vfsGetFileCount( filename );
+	const int count = 0; // vfsGetFileCount( filename );
 
 	/* load them all */
 	for ( int i = 0; i < count; i++ )