Wip/spacer gizmo selection

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "lib/rapidjson"]
 	path = lib/rapidjson
 	url = https://github.com/Tencent/rapidjson.git
+[submodule "lib/im3d"]
+	path = lib/im3d
+	url = https://github.com/john-chapman/im3d.git

CMakeLists.txt

@@ -84,6 +84,27 @@ target_sources(${PROJECT_NAME} PRIVATE ${OPENGOTHIC_SOURCES} ${ObjCSOURCES})
 add_subdirectory(shader)
 target_link_libraries(${PROJECT_NAME} GothicShaders)
 
+# Spacer-only editor shaders. add_shader() is defined by shader/CMakeLists.txt;
+# point its output back at the GothicShaders binary directory while registering
+# the additional generated SPIR-V files.
+set(OPENGOTHIC_ROOT_PROJECT_BINARY_DIR ${PROJECT_BINARY_DIR})
+set(PROJECT_BINARY_DIR "${CMAKE_BINARY_DIR}/shader")
+set(OPENGOTHIC_SHADERS_SPRV "")
+add_shader(im3d_points.vert          editor/im3d.vert -DPOINTS)
+add_shader(im3d_points.frag          editor/im3d.frag -DPOINTS)
+add_shader(im3d_lines.vert           editor/im3d.vert -DLINES)
+add_shader(im3d_lines.geom           editor/im3d.geom)
+add_shader(im3d_lines.frag           editor/im3d.frag -DLINES)
+add_shader(im3d_triangles.vert       editor/im3d.vert -DTRIANGLES)
+add_shader(im3d_triangles.frag       editor/im3d.frag -DTRIANGLES)
+add_shader(selection_outline.vert    editor/selection_outline.vert)
+add_shader(selection_outline.frag    editor/selection_outline.frag)
+add_shader(selection_outline_composite.frag editor/selection_outline_composite.frag)
+set(SPACER_EDITOR_SHADERS ${OPENGOTHIC_SHADERS_SPRV})
+set(PROJECT_BINARY_DIR ${OPENGOTHIC_ROOT_PROJECT_BINARY_DIR})
+add_custom_target(spacer_editor_shaders DEPENDS ${SPACER_EDITOR_SHADERS})
+add_dependencies(shader spacer_editor_shaders)
+
 # editor icons and such
 add_subdirectory(spacer/assets)
 
@@ -144,7 +165,7 @@ target_link_libraries(${PROJECT_NAME} miniz)
 
 # bullet physics
 set(BULLET2_MULTITHREADING       ON  CACHE INTERNAL "")
-set(USE_MSVC_RUNTIME_LIBRARY_DLL ON  CACHE INTERNAL "")
+set(USE_MSVC_RUNTIME_LIBRARY_DLL ON CACHE INTERNAL "")
 set(USE_GRAPHICAL_BENCHMARK      OFF CACHE INTERNAL "")
 set(BUILD_BULLET2_DEMOS          OFF CACHE INTERNAL "") # No samples
 set(BUILD_OPENGL3_DEMOS          OFF CACHE INTERNAL "") # No samples
@@ -159,15 +180,15 @@ add_subdirectory(lib/bullet3)
 if(NOT MSVC)
   target_compile_options(BulletDynamics         PRIVATE -Wno-argument-outside-range)
   target_compile_options(BulletInverseDynamics  PRIVATE -Wno-argument-outside-range)
-  target_compile_options(BulletSoftBody         PRIVATE -Wno-argument-outside-range)
+  target_compile_options(BulletSoftBody          PRIVATE -Wno-argument-outside-range)
   target_compile_options(Bullet3Common          PRIVATE -Wno-argument-outside-range)
   target_compile_options(BulletCollision        PRIVATE -Wno-argument-outside-range)
   target_compile_options(LinearMath             PRIVATE -Wno-argument-outside-range)
 endif()
 if(NOT MSVC AND NOT APPLE AND NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
   target_compile_options(BulletDynamics         PRIVATE -Wno-stringop-overflow)
   target_compile_options(BulletInverseDynamics  PRIVATE -Wno-stringop-overflow)
-  target_compile_options(BulletSoftBody         PRIVATE -Wno-stringop-overflow)
+  target_compile_options(BulletSoftBody          PRIVATE -Wno-stringop-overflow)
   target_compile_options(Bullet3Common          PRIVATE -Wno-stringop-overflow)
   target_compile_options(BulletCollision        PRIVATE -Wno-stringop-overflow)
   target_compile_options(LinearMath             PRIVATE -Wno-stringop-overflow)
@@ -198,8 +219,8 @@ target_sources(Gothic2Notr PRIVATE game/main.cpp icon.rc)
 target_link_libraries(Gothic2Notr PRIVATE OpenGothic)
 
 # editor
-target_sources(Spacer PRIVATE ${SPACER_SOURCES} spacer/main.cpp)
-target_include_directories(Spacer PRIVATE "spacer")
+target_sources(Spacer PRIVATE ${SPACER_SOURCES} spacer/main.cpp lib/im3d/im3d.cpp)
+target_include_directories(Spacer PRIVATE "spacer" "lib/im3d")
 target_link_libraries(Spacer PRIVATE OpenGothic SpacerAssets)
 
 # installation

common/graphics/renderer.h

@@ -32,6 +32,9 @@ class Renderer final {
 
     Tempest::Attachment screenshoot(uint8_t frameId);
 
+    // Editor overlays may reuse the completed scene depth without owning another renderer.
+    Tempest::ZBuffer& sceneDepthBuffer() { return zbuffer; }
+
   private:
     enum Quality : uint8_t {
       None,

shader/editor/im3d.frag

@@ -0,0 +1,36 @@
+#version 450
+
+#extension GL_ARB_separate_shader_objects : enable
+
+#if !defined(POINTS) && !defined(LINES) && !defined(TRIANGLES)
+#error im3d primitive type is not defined
+#endif
+
+#if defined(LINES)
+layout(location = 0) in noperspective float inEdgeDistance;
+layout(location = 1) in noperspective float inSize;
+layout(location = 2) in smooth vec4 inColor;
+#elif defined(POINTS)
+layout(location = 0) in float inSize;
+layout(location = 1) in vec4  inColor;
+#else
+layout(location = 0) in vec4 inColor;
+#endif
+
+layout(location = 0) out vec4 outColor;
+
+const float antialiasing = 2.0;
+
+void main() {
+  outColor = inColor;
+
+#if defined(LINES)
+  float coverage = abs(inEdgeDistance)/inSize;
+  coverage = smoothstep(1.0,1.0-antialiasing/inSize,coverage);
+  outColor.a *= coverage;
+#elif defined(POINTS)
+  float coverage = length(gl_PointCoord.xy-vec2(0.5));
+  coverage = smoothstep(0.5,0.5-antialiasing/inSize,coverage);
+  outColor.a *= coverage;
+#endif
+  }

shader/editor/im3d.geom

@@ -0,0 +1,44 @@
+#version 450
+
+#extension GL_ARB_separate_shader_objects : enable
+
+layout(lines) in;
+layout(triangle_strip, max_vertices = 4) out;
+
+layout(push_constant, std430) uniform Push {
+  mat4 viewProject;
+  vec2 viewport;
+  } push;
+
+layout(location = 0) in float inSize[];
+layout(location = 1) in vec4  inColor[];
+
+layout(location = 0) out noperspective float outEdgeDistance;
+layout(location = 1) out noperspective float outSize;
+layout(location = 2) out smooth vec4 outColor;
+
+void emitLineVertex(vec2 position, int endpoint, float edgeDistance) {
+  gl_Position     = vec4(position*gl_in[endpoint].gl_Position.w,
+                        gl_in[endpoint].gl_Position.zw);
+  outEdgeDistance = edgeDistance;
+  outSize         = inSize[endpoint];
+  outColor        = inColor[endpoint];
+  EmitVertex();
+  }
+
+void main() {
+  vec2 pos0 = gl_in[0].gl_Position.xy/gl_in[0].gl_Position.w;
+  vec2 pos1 = gl_in[1].gl_Position.xy/gl_in[1].gl_Position.w;
+
+  vec2 dir = pos0-pos1;
+  dir = normalize(vec2(dir.x,dir.y*push.viewport.y/push.viewport.x));
+  vec2 tangent = vec2(-dir.y,dir.x);
+  vec2 tangent0 = tangent*inSize[0]/push.viewport;
+  vec2 tangent1 = tangent*inSize[1]/push.viewport;
+
+  emitLineVertex(pos0-tangent0,0,-inSize[0]);
+  emitLineVertex(pos0+tangent0,0, inSize[0]);
+  emitLineVertex(pos1-tangent1,1,-inSize[1]);
+  emitLineVertex(pos1+tangent1,1, inSize[1]);
+  EndPrimitive();
+  }

shader/editor/im3d.vert

@@ -0,0 +1,47 @@
+#version 450
+
+#extension GL_ARB_separate_shader_objects : enable
+
+#if !defined(POINTS) && !defined(LINES) && !defined(TRIANGLES)
+#error im3d primitive type is not defined
+#endif
+
+layout(location = 0) in vec4 inPositionSize;
+layout(location = 1) in vec4 inColor;
+
+layout(push_constant, std430) uniform Push {
+  mat4 viewProject;
+  vec2 viewport;
+  } push;
+
+#if defined(LINES)
+layout(location = 0) out float outSize;
+layout(location = 1) out vec4  outColor;
+#elif defined(POINTS)
+layout(location = 0) out float outSize;
+layout(location = 1) out vec4  outColor;
+#else
+layout(location = 0) out vec4 outColor;
+#endif
+
+out gl_PerVertex {
+  vec4  gl_Position;
+  float gl_PointSize;
+  };
+
+const float antialiasing = 2.0;
+
+void main() {
+  gl_Position = push.viewProject * vec4(inPositionSize.xyz,1.0);
+
+#if defined(POINTS) || defined(LINES)
+  outSize  = max(inPositionSize.w,antialiasing);
+  outColor = inColor;
+  outColor.a *= smoothstep(0.0,1.0,inPositionSize.w/antialiasing);
+#if defined(POINTS)
+  gl_PointSize = outSize;
+#endif
+#else
+  outColor = inColor;
+#endif
+  }

shader/editor/selection_outline.frag

@@ -0,0 +1,9 @@
+#version 450
+
+#extension GL_ARB_separate_shader_objects : enable
+
+layout(location = 0) out vec4 outColor;
+
+void main() {
+  outColor = vec4(1.0);
+  }

shader/editor/selection_outline.vert

@@ -0,0 +1,28 @@
+#version 450
+
+#extension GL_ARB_separate_shader_objects : enable
+
+layout(location = 0) in vec3 inPosition;
+layout(location = 1) in vec3 inNormal;
+layout(location = 2) in vec2 inUv;
+layout(location = 3) in uint inColor;
+
+layout(push_constant, std430) uniform Push {
+  mat4 model;
+  mat4 viewProject;
+  vec4 params;
+  } push;
+
+out gl_PerVertex {
+  vec4 gl_Position;
+  };
+
+void main() {
+  vec4 clip = push.viewProject * push.model * vec4(inPosition,1.0);
+
+  // A minute bias toward the camera makes an equal-depth replay of the
+  // selected mesh robust against floating-point differences. Occluding scene
+  // geometry still wins the regular LEqual depth test.
+  clip.z -= push.params.x * clip.w;
+  gl_Position = clip;
+  }

shader/editor/selection_outline_composite.frag

@@ -0,0 +1,40 @@
+#version 450
+
+#extension GL_ARB_separate_shader_objects : enable
+
+layout(binding = 0) uniform sampler2D selectionMask;
+
+layout(push_constant, std430) uniform Push {
+  vec4  color;
+  vec2  viewport;
+  float radius;
+  float padding;
+  } push;
+
+layout(location = 0) out vec4 outColor;
+
+void main() {
+  const vec2 uv        = gl_FragCoord.xy / push.viewport;
+  // Offsets are measured in final editor pixels even when the renderer uses a
+  // reduced internal resolution.
+  const vec2 texelSize = 1.0 / push.viewport;
+  const float center   = texture(selectionMask,uv).r;
+
+  float expanded = 0.0;
+  const int maxRadius = 4;
+  for(int y=-maxRadius; y<=maxRadius; ++y) {
+    for(int x=-maxRadius; x<=maxRadius; ++x) {
+      if(abs(x)>int(push.radius) || abs(y)>int(push.radius))
+        continue;
+      if(x*x+y*y>int(push.radius*push.radius))
+        continue;
+      expanded = max(expanded,
+                     texture(selectionMask,uv+vec2(x,y)*texelSize).r);
+      }
+    }
+
+  const float outline = expanded * (1.0-center);
+  if(outline<=0.0)
+    discard;
+  outColor = vec4(push.color.rgb,push.color.a*outline);
+  }