# HG changeset patch
# User Teemu Piippo <teemu.s.piippo@gmail.com>
# Date 1656426956 -10800
# Node ID 0fd926ebb03b3266f2d445108974f7c518242ba3
# Parent  a0ddbc9a4e77e7eadcd3d7bd99c97605773e3e8d
Fix picking and rendering of selected colors

diff -r a0ddbc9a4e77 -r 0fd926ebb03b src/gl/common.h
--- a/src/gl/common.h	Tue Jun 28 14:53:22 2022 +0300
+++ b/src/gl/common.h	Tue Jun 28 17:35:56 2022 +0300
@@ -147,6 +147,18 @@
 	return stream >> enum_value_cast(style);
 }
 
+constexpr glm::vec3 idToColor(std::int32_t id)
+{
+	const int r = (id & 0x0000ff) >> 0;
+	const int g = (id & 0x00ff00) >> 8;
+	const int b = (id & 0xff0000) >> 16;
+	return {
+		static_cast<float>(r) / 255.0f,
+		static_cast<float>(g) / 255.0f,
+		static_cast<float>(b) / 255.0f,
+	};
+}
+
 #include <QPainter>
 #include <QPainterPath>
 inline void drawBorderedText(QPainter* painter, const QPointF& point, const QFont& font, const QString& text)
diff -r a0ddbc9a4e77 -r 0fd926ebb03b src/gl/compiler.cpp
--- a/src/gl/compiler.cpp	Tue Jun 28 14:53:22 2022 +0300
+++ b/src/gl/compiler.cpp	Tue Jun 28 17:35:56 2022 +0300
@@ -28,8 +28,8 @@
 layout(location=0) in vec3 position;
 layout(location=1) in vec4 color;
 layout(location=2) in vec3 normal;
-layout(location=3) in int id;
-layout(location=4) in int selected;
+layout(location=3) in vec3 pickcolor;
+layout(location=4) in float selected;
 out vec4 vColor;
 out vec3 vFragPos;
 out vec3 vNormal;
@@ -37,7 +37,7 @@
 uniform mat4 viewMatrix;
 uniform mat4 projectionMatrix;
 uniform int fragmentStyle;
-uniform vec3 selectedColor;
+uniform vec4 selectedColor;
 uniform int highlighted;
 
 const int FRAGSTYLE_Normal = 0;
@@ -53,17 +53,7 @@
 	vNormal = normalize(normalMatrix * normal);
 	if (fragmentStyle == FRAGSTYLE_Id)
 	{
-		/* Calculate a color based from this index. This method caters for
-		 * 16777216 objects. I don't think that will be exceeded anytime soon.
-		 */
-		int r = (id / 0x10000) % 0x100;
-		int g = (id / 0x100) % 0x100;
-		int b = id % 0x100;
-		vColor = vec4(r / 255.0, g / 255.0, b / 255.0, 1.0);
-	}
-	else if (selected == 1)
-	{
-		vColor = vec4(selectedColor, 1.0);
+		vColor = vec4(pickcolor, 1.0);
 	}
 	else
 	{
@@ -83,12 +73,14 @@
 		{
 			vColor = color;
 		}
+/*
 		if (highlighted == id)
 		{
 			vColor = (vColor + vec4(selectedColor, 1.0) * 0.6) / 1.6;
 		}
+*/
+		vColor = (1 - selected) * vColor + selected * selectedColor;
 	}
-
 	vFragPos = vec3(modelMatrix * vec4(position, 1.0));
 	gl_Position = projectionMatrix * viewMatrix * vec4(vFragPos, 1.0);
 }
@@ -206,14 +198,12 @@
 				shader.program->enableAttributeArray(k);
 			}
 			using Vertex = ModelShaders::Vertex;
-			constexpr int stride = sizeof(Vertex);
+			constexpr std::size_t stride = sizeof(Vertex);
 			shader.program->setAttributeBuffer(0, GL_FLOAT, offsetof(Vertex, position), 3, stride);
 			shader.program->setAttributeBuffer(1, GL_FLOAT, offsetof(Vertex, color), 4, stride);
 			shader.program->setAttributeBuffer(2, GL_FLOAT, offsetof(Vertex, normal), 3, stride);
-			const void* vertexptr = reinterpret_cast<const void*>(offsetof(Vertex, id));
-			glfunc().glVertexAttribPointer(3, 1, GL_INT, GL_FALSE, stride, vertexptr);
-			const void* selectedptr = reinterpret_cast<const void*>(offsetof(Vertex, selected));
-			glfunc().glVertexAttribPointer(4, 1, GL_INT, GL_FALSE, stride, selectedptr);
+			shader.program->setAttributeBuffer(3, GL_FLOAT, offsetof(Vertex, pickcolor), 3, stride);
+			shader.program->setAttributeBuffer(4, GL_FLOAT, offsetof(Vertex, selected), 1, stride);
 			shader.vertexArray.release();
 			shader.buffer.release();
 			shader.program->release();
@@ -307,6 +297,7 @@
 			vertex.normal = normal;
 			vertex.color = glm::vec4{color.redF(), color.greenF(), color.blueF(), color.alphaF()};
 			vertex.id = polygon.id.value;
+			vertex.pickcolor = idToColor(polygon.id.value);
 		});
 	});
 	for (gl::ModelShaders::ShaderObject& shader : shaders->shaderObjects)
@@ -319,9 +310,13 @@
 	}
 }
 
-ModelId gl::idFromColor(const std::array<GLubyte, 3>& data)
+ModelId gl::idFromUcharColor(const std::array<GLubyte, 3>& data)
 {
-	return {data[0] * std::int32_t{0x10000} + data[1] * std::int32_t{0x100} + data[2]};
+	return {
+		static_cast<std::int32_t>(data[0]) |
+		static_cast<std::int32_t>(data[1]) << 8 |
+		static_cast<std::int32_t>(data[2]) << 16
+	};
 }
 
 void gl::bindModelShaderVertexArray(gl::ModelShaders* shaders, gl::ArrayClass arrayClass)
diff -r a0ddbc9a4e77 -r 0fd926ebb03b src/gl/compiler.h
--- a/src/gl/compiler.h	Tue Jun 28 14:53:22 2022 +0300
+++ b/src/gl/compiler.h	Tue Jun 28 17:35:56 2022 +0300
@@ -55,8 +55,9 @@
 			glm::vec3 position;
 			glm::vec4 color;
 			glm::vec3 normal;
+			glm::vec3 pickcolor;
 			glm::int32 id;
-			glm::int32 selected = 0;
+			float selected = 0.0f;
 		};
 		bool initialized = false;
 		struct ShaderObject
@@ -80,31 +81,39 @@
 	void releaseModelShaderVertexArray(gl::ModelShaders* shaders, gl::ArrayClass arrayClass);
 	void setModelShaderSelectedObjects(gl::ModelShaders* shaders, const QSet<ModelId>& ids);
 	std::size_t vertexCount(const ModelShaders *shaders, gl::ArrayClass arrayClass);
-	ModelId idFromColor(const std::array<GLubyte, 3>& data);
+	ModelId idFromUcharColor(const std::array<GLubyte, 3>& data);
 
-	template<typename T>
-	void setShaderUniform(gl::ModelShaders* shaders, const char* uniformName, T&& value)
+	template<typename... Ts>
+	void setShaderUniform(gl::ModelShaders* shaders, const char* uniformName, Ts&&... args)
 	{
 		for (gl::ModelShaders::ShaderObject& shader : shaders->shaderObjects)
 		{
 			shader.program->bind();
 			const int location = glfunc().glGetUniformLocation(shader.program->programId(), uniformName);
-			Q_ASSERT(location != -1);
-			shader.program->setUniformValue(location, std::forward<T>(value));
+			if (location != -1) {
+				shader.program->setUniformValue(location, std::forward<Ts>(args)...);
+			}
 			shader.program->release();
 		}
 	}
 
-	template<typename Float, glm::qualifier Prec>
-	void setShaderUniformMatrix(
+	inline void setShaderUniformMatrix(
 		gl::ModelShaders* shaders,
 		const char* uniformName,
-		const glm::mat<4, 4, Float, Prec>& value)
+		const glm::mat<4, 4, float>& value)
 	{
 		const float (*array)[4][4] = reinterpret_cast<const float(*)[4][4]>(glm::value_ptr(value));
 		setShaderUniform(shaders, uniformName, *array);
 	}
 
+	inline void setShaderUniformVector(
+		gl::ModelShaders* shaders,
+		const char* uniformName,
+		const glm::vec4& value)
+	{
+		setShaderUniform(shaders, uniformName, value.x, value.y, value.z, value.w);
+	}
+
 	BoundingBox boundingBoxForModel(Model* model, DocumentManager* context);
 }
 
diff -r a0ddbc9a4e77 -r 0fd926ebb03b src/gl/partrenderer.cpp
--- a/src/gl/partrenderer.cpp	Tue Jun 28 14:53:22 2022 +0300
+++ b/src/gl/partrenderer.cpp	Tue Jun 28 17:35:56 2022 +0300
@@ -187,12 +187,13 @@
 	}
 	else
 	{
-		glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
+		glClearColor(1.0f, 1.0f, 1.0f, 1.0f);
 		gl::setShaderUniform(&this->shaders, "useLighting", GL_FALSE);
 	}
 	this->checkForGLErrors();
-	const QVector3D color = calcQVector3DFromQColor(this->renderPreferences.selectedColor);
-	gl::setShaderUniform(&this->shaders, "selectedColor", color);
+	const QColor qs = this->renderPreferences.selectedColor;
+	const glm::vec4 selectedColor{qs.redF(), qs.greenF(), qs.blueF(), 1.0f};
+	gl::setShaderUniformVector(&this->shaders, "selectedColor", selectedColor);
 	gl::setShaderUniform(&this->shaders, "highlighted", this->highlighted.value);
 	this->checkForGLErrors();
 	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
@@ -473,7 +474,7 @@
 	this->checkForGLErrors();
 	fbo.release();
 	this->renderPreferences.style = oldRenderStyle;
-	return gl::idFromColor(data);
+	return gl::idFromUcharColor(data);
 }
 
 /**