There are different options, to achieve what you want. On of them is to create a function, which intersects a ray and sphere.
The following algorithm is taken from Peter Shirley's book Ray Tracing in One Weekend.
The ray is defined by an origin and a direction. The sphere is defined by a center point and a radius:
float hit_sphere(vec3 origin, vec3 direction, vec3 center, float radius)
{
vec3 oc = origin - center;
float a = dot(direction, direction);
float b = 2.0 * dot(oc, direction);
float c = dot(oc, oc) - radius*radius;
float discriminant = b*b - 4*a*c;
if (discriminant > 0.0)
{
float temp = (-b - sqrt(discriminant)) / (2*a);
if (temp > 0.0)
return 1.0;
temp = (-b + sqrt(discriminant)) / (2*a);
if (temp > 0.0)
return 1.0;
}
return 0.0;
}
Compute 2 points on the ray. Each point in normalized device space with the xy coordinates xy = gl_FragCoord.xy / u_resolution.xy * 2.0 - 1.0 is on the same ray.
vec4(xy, -1.0, 1.0) is on the near plane and vec4(xy, 1.0, 1.0) is on the far plane. Transform this points by the inverse model view projection matrix (u_mvp) and divide the xyz components by the w component to get a ray in model space in cartesian coordinates.
Size the algorithm computes 2 points on the ray, this works for perspective projection and orthographic projection.
vec4 pn = inverse(u_mvp) * vec4(xy, -1.0, 1.0);
vec4 pf = inverse(u_mvp) * vec4(xy, 1.0, 1.0);
vec3 orig = pn.xyz/pn.w;
vec3 dir = pf.xyz/pf.w - orig;
Use the ray to intersect the sphere in model space:
vec3 center = vec3(0.0);
float radius = 1.0;
float r = hit_sphere(orig, dir, center, radius);
I've tested the algorithm with the view matrix glm::translate(glm::mat4(1), glm::vec3(0, 0, -3)) and the projection matrix glm::perspective(glm::radians(90), aspect, 0.1, 10.0)
Fragment shader:
#version 330 core
out vec4 frag_color;
uniform mat4 u_mvp;
uniform vec2 u_resolution;
float hit_sphere(vec3 origin, vec3 direction, vec3 center, float radius)
{
vec3 oc = origin - center;
float a = dot(direction, direction);
float b = 2.0 * dot(oc, direction);
float c = dot(oc, oc) - radius*radius;
float discriminant = b*b - 4*a*c;
if (discriminant > 0.0)
{
float temp = (-b - sqrt(discriminant)) / (2*a);
if (temp > 0.0)
return 1.0;
temp = (-b + sqrt(discriminant)) / (2*a);
if (temp > 0.0)
return 1.0;
}
return 0.0;
}
void main()
{
vec2 xy = gl_FragCoord.xy / u_resolution.xy * 2.0 - 1.0;
vec4 pn = inverse(u_mvp) * vec4(xy, -1.0, 1.0);
vec4 pf = inverse(u_mvp) * vec4(xy, 1.0, 1.0);
vec3 orig = pn.xyz/pn.w;
vec3 dir = pf.xyz/pf.w - orig;
vec3 center = vec3(0.0);
float radius = 1.0;
float r = hit_sphere(orig, dir, center, radius);
vec4 backcolor = vec4(0.2, 0.3, 0.3, 1.0);
vec4 color = vec4(0.7, 0.8, 0.1, 1.0);
frag_color = mix(backcolor, color, r);
}