* fix and simplify FloodAreas

was bugged since original q3map
bug behavior was made consistent in c379aae1d4 (was determined by 1st brush face being areaportal before)

docs/changelog-custom.txt

@@ -183,6 +183,7 @@ Q3map2:
 	* fix tiny structural brushes breaking bsp tree
 	* handle patch lodbounds in -shift and -scale
 	* refactor to work without -maxshaderinfo limit, improve shader search performance
+	* fix and simplify FloodAreas
 
 
 

tools/quake3/common/polylib.cpp

@@ -805,3 +805,40 @@ void    AddWindingToConvexHull( const winding_t& w, winding_t& hull, const Vecto
 
 	hull = winding_t( hullPoints, hullPoints + numHullPoints );
 }
+
+
+// Project polygon onto an axis
+static std::pair<double, double> project_winding( const winding_t& w, const DoubleVector3& axis ) {
+	double min_proj = std::numeric_limits<double>::infinity();
+	double max_proj = -std::numeric_limits<double>::infinity();
+
+	for ( const auto& vertex : w ) {
+		const double projection = vector3_dot( vertex, axis );
+		value_minimize( min_proj, projection );
+		value_maximize( max_proj, projection );
+	}
+
+	return { min_proj, max_proj };
+}
+
+// Check intersection of two coplanar convex polygons in 3D (Separating Axis Theorem (SAT))
+bool windings_intersect_coplanar( const winding_t& w1, const winding_t& w2, const Plane3& plane ) {
+	// Collect in-plane normals for both polygons
+	std::vector<DoubleVector3> normals;
+	normals.reserve( w1.size() + w2.size() );
+
+	for( auto *w : { &w1, &w2 } )
+		for( auto prev = std::prev( w->cend() ), next = w->cbegin(); next != w->cend(); prev = next, ++next )
+			normals.push_back( VectorNormalized( vector3_cross( DoubleVector3( *next - *prev ), plane.normal() ) ) );
+
+	// Test each axis
+	for ( const auto& axis : normals ) {
+		const auto [ min1, max1 ] = project_winding( w1, axis );
+		const auto [ min2, max2 ] = project_winding( w2, axis );
+		if ( max1 < min2 + 1 || max2 < min1 + 1 ) { // epsilon to filter false positive intersections
+			return false; // Separating axis found
+		}
+	}
+
+	return true; // No separating axis, polygons intersect
+}

tools/quake3/common/polylib.h

@@ -70,6 +70,7 @@ void    ChopWindingInPlace( winding_t& w, const Plane3f& plane, float epsilon );
 
 void pw( const winding_t& w );
 
+bool windings_intersect_coplanar( const winding_t& w1, const winding_t& w2, const Plane3& plane );
 
 ///////////////////////////////////////////////////////////////////////////////////////
 // Below is double-precision stuff.  This was initially needed by the base winding code

tools/quake3/q3map2/brush.cpp

@@ -448,18 +448,23 @@ static int FilterBrushIntoTree_r( brush_t&& b, node_t *node ){
 
 	/* add it to the leaf list */
 	if ( node->planenum == PLANENUM_LEAF ) {
-		/* something somewhere is hammering brushlist */
 		node->brushlist.push_front( std::move( b ) );
 
 		/* classify the leaf by the structural brush */
 		if ( !b.detail ) {
 			if ( b.opaque ) {
 				node->opaque = true;
-				node->areaportal = false;
 			}
-			else if ( b.compileFlags & C_AREAPORTAL ) {
-				if ( !node->opaque ) {
-					node->areaportal = true;
+			else if ( b.compileFlags & C_AREAPORTAL ) { // find and flag C_AREAPORTAL portals, this is not always passed through node->compileFlags
+				const auto side = std::ranges::find_if( b.original->sides, []( const side_t& side ){ return side.compileFlags & C_AREAPORTAL; } );
+				int s;
+				for ( portal_t *p = node->portals; p; p = p->next[ s ] )
+				{
+					s = ( p->nodes[1] == node );
+					if( p->onnode != nullptr && ( p->onnode->planenum | 1 ) == ( side->planenum | 1 ) )
+						if( windings_intersect_coplanar( ( p->onnode->planenum == side->planenum )
+						                                 ? p->winding : ReverseWinding( p->winding ), side->winding, side->plane ) )
+							p->compileFlags |= C_AREAPORTAL;
 				}
 			}
 		}

tools/quake3/q3map2/map.cpp

@@ -542,12 +542,7 @@ static void SetBrushContents( brush_t& b ){
 	}
 
 	/* opaque? */
-	if ( compileFlags & C_TRANSLUCENT ) {
-		b.opaque = false;
-	}
-	else{
-		b.opaque = true;
-	}
+	b.opaque = !( compileFlags & C_TRANSLUCENT );
 
 	/* areaportal? */
 	if ( compileFlags & C_AREAPORTAL ) {
@@ -769,6 +764,28 @@ static void MergeOrigin( entity_t& ent, const Vector3& origin ){
 	ent.setKeyValue( "origin", string );
 }
 
+static void FixAreaportalBrush( brush_t& brush, const entity_t& mapEnt ){
+	if( std::ranges::count_if( brush.sides, []( const side_t& side ){ return side.compileFlags & C_AREAPORTAL; } ) > 1 ){
+		Sys_FPrintf( SYS_WRN, "Entity %i, Brush %i: areaportal brush with > 1 areaportal faces\nLeaving one biggest face\n",
+		             mapEnt.mapEntityNum, brush.brushNum );
+
+		side_t *bestSide{};
+		float bestArea{};
+		for( auto& side : brush.sides ){
+			if( side.compileFlags & C_AREAPORTAL ){
+				const float area = WindingArea( side.winding );
+				if( bestArea < area ){
+					bestArea = area;
+					bestSide = &side;
+				}
+			}
+		}
+		for( auto& side : brush.sides )
+			if( bestSide != &side )
+				side.compileFlags &= ~C_AREAPORTAL;
+	}
+}
+
 /*
    FinishBrush()
    produces a final brush based on the buildBrush->sides array
@@ -805,6 +822,7 @@ static void FinishBrush( bool noCollapseGroups, entity_t& mapEnt ){
 			xml_Select( "areaportals only allowed in world", mapEnt.mapEntityNum, buildBrush.brushNum, false );
 			return;
 		}
+		FixAreaportalBrush( buildBrush, mapEnt );
 	}
 
 	/* add bevel planes */
@@ -1166,9 +1184,6 @@ static void ParseBrush( bool onlyLights, bool noCollapseGroups, entity_t& mapEnt
 		return;
 	}
 
-	/* set some defaults */
-	buildBrush.portalareas[ 0 ] = -1;
-	buildBrush.portalareas[ 1 ] = -1;
 	/* set map entity and brush numbering */
 	buildBrush.entityNum = mapEnt.mapEntityNum;
 	buildBrush.brushNum = mapPrimitiveNum;

tools/quake3/q3map2/portals.cpp

@@ -639,38 +639,6 @@ static int c_areas;
  */
 
 static void FloodAreas_r( node_t *node ){
-	int s;
-	portal_t    *p;
-
-
-	if ( node->areaportal ) {
-		if ( node->area == -1 ) {
-			node->area = c_areas;
-		}
-
-		/* this node is part of an area portal brush */
-		brush_t *b = node->brushlist.front().original;
-
-		/* if the current area has already touched this portal, we are done */
-		if ( b->portalareas[ 0 ] == c_areas || b->portalareas[ 1 ] == c_areas ) {
-			return;
-		}
-
-		// note the current area as bounding the portal
-		if ( b->portalareas[ 1 ] != -1 ) {
-			Sys_Warning( "areaportal brush %i touches > 2 areas\n", b->brushNum );
-			return;
-		}
-		if ( b->portalareas[ 0 ] != -1 ) {
-			b->portalareas[ 1 ] = c_areas;
-		}
-		else{
-			b->portalareas[ 0 ] = c_areas;
-		}
-
-		return;
-	}
-
 	if ( node->area != -1 ) {
 		return;
 	}
@@ -685,11 +653,16 @@ static void FloodAreas_r( node_t *node ){
 		skyboxArea = c_areas;
 	}
 
-	for ( p = node->portals; p; p = p->next[ s ] )
+	int s;
+	for ( portal_t *p = node->portals; p; p = p->next[ s ] )
 	{
 		s = ( p->nodes[1] == node );
 
 		/* ydnar: allow areaportal portals to block area flow */
+		/* this check alone w/o node->areaportal path seems sufficient
+		   besides when node->compileFlags are overriden by hint or struct split flush with areportal
+		   we make it persistent in FilterBrushIntoTree_r()
+		   note: node->areaportal way fails for leafs with only opaque and areaportal portals */
 		if ( p->compileFlags & C_AREAPORTAL ) {
 			continue;
 		}
@@ -717,7 +690,7 @@ static void FindAreas_r( node_t *node ){
 		return;
 	}
 
-	if ( node->opaque || node->areaportal || node->area != -1 ) {
+	if ( node->opaque || node->area != -1 ) {
 		return;
 	}
 
@@ -746,14 +719,6 @@ static void CheckAreas_r( const node_t *node ){
 			Sys_Warning( "cluster %d has area set to -1\n", node->cluster );
 		}
 	}
-	if ( node->areaportal ) {
-		const brush_t *b = node->brushlist.front().original;
-
-		// check if the areaportal touches two areas
-		if ( b->portalareas[0] == -1 || b->portalareas[1] == -1 ) {
-			Sys_Warning( "areaportal brush %i doesn't touch two areas\n", b->brushNum );
-		}
-	}
 }
 
 

tools/quake3/q3map2/q3map2.h

@@ -720,8 +720,6 @@ struct brush_t
 	bool detail;
 	bool opaque;
 
-	int portalareas[ 2 ];
-
 	MinMax minmax;
 
 	std::vector<side_t> sides;
@@ -970,7 +968,6 @@ struct node_t
 
 	/* leafs only */
 	bool opaque;                        /* view can never be inside */
-	bool areaportal;
 	bool skybox;                        /* ydnar: a skybox leaf */
 	bool sky;                           /* ydnar: a sky leaf */
 	int cluster;                        /* for portalfile writing */