Triangles are not selected from the obj model?

Question

Java

The user selects an area on the OpenGL scene , a rectangle appears (as when selecting folders and files) and then you need to select the triangles from the obj model (eagle) that fell into the frustum area.
Here , near is the front rectangle, far is the back rectangle. planeA,planeB...planeF - planes, prevx,prevy - start coordinates for selection, curx,cury - end coordinates for selection.

///
{
  Vector3f[] near = new Vector3f[4];
  near[0]=new Vector3f(Rectangle.getPrevX(),Rectangle.getPrevY(),1);
  near[1]=new Vector3f(Rectangle.getPrevX(),Rectangle.getCurY(),1);
  near[2]=new Vector3f(Rectangle.getCurX(),Rectangle.getCurY(),1);
  near[3]=new Vector3f(Rectangle.getCurX(),Rectangle.getPrevY(),1);
  Vector3f[] far = new Vector3f[4];
  far[0]=new Vector3f(Rectangle.getPrevX(),Rectangle.getPrevY(),-100);
  far[1]=new Vector3f(Rectangle.getPrevX(),Rectangle.getCurY(),-100);
  far[2]=new Vector3f(Rectangle.getCurX(),Rectangle.getCurY(),-100);
  far[3]=new Vector3f(Rectangle.getCurX(),Rectangle.getPrevY(),-100);
  Vector3f center = getCenter(near,far);// I get the center
  Vector4f planeA = Util.getABCD(new Vector3f(Rectangle.getPrevX(), Rectangle.getCurY(), 1),
  new Vector3f(Rectangle.getPrevX(), Rectangle.getPrevY(), 1),
  new Vector3f(Rectangle.getCurX(), Rectangle.getCurY(), 1),center);
  Vector4f planeE = Util.getABCD(new Vector3f(Rectangle.getPrevX(), Rectangle.getCurY(), 1),
  new Vector3f(Rectangle.getPrevX(), Rectangle.getPrevY(), 1),
  new Vector3f(Rectangle.getPrevX(), Rectangle.getPrevY(), -100),center);
  Vector4f planeB = Util.getABCD(new Vector3f(Rectangle.getCurX(), Rectangle.getPrevY(), 1),
  new Vector3f(Rectangle.getCurX(), Rectangle.getCurY(), 1),
  new Vector3f(Rectangle.getCurX(), Rectangle.getCurY(), -100),center);
  Vector4f planeD = Util.getABCD(new Vector3f(Rectangle.getPrevX(), Rectangle.getPrevY(), 1),
  new Vector3f(Rectangle.getPrevX(), Rectangle.getPrevY(), -100),
  new Vector3f(Rectangle.getCurX(), Rectangle.getPrevY(), -100),center);
  Vector4f planeC = Util.getABCD(new Vector3f(Rectangle.getPrevX(), Rectangle.getCurY(), 1),
  new Vector3f(Rectangle.getPrevX(), Rectangle.getCurY(), -100),
  new Vector3f(Rectangle.getCurX(), Rectangle.getCurY(), -100),center);
  Vector4f planeF = Util.getABCD(new Vector3f(Rectangle.getPrevX(), Rectangle.getPrevY(), -100),
  new Vector3f(Rectangle.getPrevX(), Rectangle.getCurY(), -100),
  new Vector3f(Rectangle.getCurX(), Rectangle.getCurY(), -100),center);
  FloatBuffer fbuf= BufferUtils.createFloatBuffer(3 * 8 * Float.BYTES).put(new float[]{
      planeA.x,planeA.y,planeA.z,planeA.w,
      planeB.x,planeB.y,planeB.z,planeB.w,
      planeC.x,planeC.y,planeC.z,planeC.w,
      planeD.x,planeD.y,planeD.z,planeD.w,
      planeE.x,planeE.y,planeE.z,planeE.w,
      planeF.x,planeF.y,planeF.z,planeF.w
     }).rewind();
  glUniform4fv(glGetUniformLocation(shaderProgram, "planes"), fbuf);
}
for (ArrayIdTriangle aVbao : sceneManyTexture.getVbao()) {
Map<String, Material> materials = sceneManyTexture.getMaterials();
String nameTexture = aVbao.getObjTexture().getNameTexture();
Optional<Texture> texture = Optional.ofNullable(materials.get(nameTexture).getTexture());
if (texture.isPresent()) {
   glActiveTexture(GL_TEXTURE0);
   glBindTexture(GL_TEXTURE_2D, texture.get().getId());
}
glBindVertexArray(aVbao.getId());
glEnableVertexAttribArray(vertexLocation);
glEnableVertexAttribArray(textureLocation);
glEnableVertexAttribArray(normalLocation);
glDrawElements(GL_TRIANGLES, aVbao.getSize(), GL_UNSIGNED_INT, 0);

Here I am getting the components a. bcd from the plane equation:

public static Vector4f getABCD(Vector3f p0, Vector3f p1, Vector3f p2,Vector3f center) {
        Vector3f n = (p1.sub(p0)).cross(p2.sub(p1));
        float d=-n.dot(p0);
        if(n.dot(center)+d<0.0f){
            n=n.mul(-1);
           d=-d;
        }
        return new Vector4f(n,d);
    }

Vertex shader:

#version 330
 
layout(location = 0) in vec3 vertexPos;
layout(location = 1) in vec3 normal;
layout (location = 2) in vec2 texCoord;
 
out float[6] dis;
out vec3 normal_modelspace;
out vec3 vertex_modelspace;
out vec2 TexCoord;
out vec4 vertexColor;
uniform mat4 P;
uniform mat4 V;
uniform mat4 M;
 
uniform vec4[6] planes;
 
 
void main() {
    TexCoord = texCoord;
    vertex_modelspace = (M * vec4(vertexPos.xyz, 1.0)).xyz;
    vertexColor = vec4(0.5f, 0.0f, 0.0f, 1.0f);
    gl_Position = P * V * vec4(vertex_modelspace.xyz, 1.0);
    normal_modelspace = (M * vec4(normal.xyz, 1.0)).xyz;
    vec3 EyeDirection_cameraspace = vec3(0,0,0) - (V * M * vec4(vertexPos,1)).xyz;
 
     for(int i = 0;i<6;i++){
       float denom = sqrt(planes[0].x * planes[0].x + planes[0].y * planes[0].y + planes[0].z * planes[0].z);
       dis[i] = float((planes[0].x * vertex_modelspace.x + planes[0].y * vertex_modelspace.y + planes[0].z * vertex_modelspace.z + planes[0].w) / denom);
     }
}

Fragment shader:

#version 330 core
 
in vec3 normal_modelspace;
in vec3 vertex_modelspace;
in vec2 TexCoord;
in vec4 vertexColor;
in float[6] dis;
out vec4 color;
 
uniform vec3 light_worldspace;
 
uniform sampler2D ourTexture;
 
void main() {
  vec3 n = normalize(normal_modelspace);
  vec3 l = normalize(light_worldspace - vertex_modelspace);
  float cosTheta = clamp( dot( n, l), 0,1 );
  float ambient = 0.05;
    int i=0;
    while(i<6 && dis[i]<=0){
      i++;
    }
    if(i==6){
      color = texture(ourTexture, TexCoord);
    }
    else
      color = vertexColor;
}

Video link below:
https://youtu.be/OgZzKX9Qwzg

Answer 1

When you right click you have the coordinates of the click, then you build a rectangle around the click - so you can find the coordinates of the vertices of the rectangle - all coordinates in the window's coordinate system. Here is the link in the mouse selection section https://ahbejarano.gitbook.io/lwjglgamedev/3d-obje... There is an easy way to find a direction vector from window coordinates, view and perspective matrices (the direction where the mouse points - in your case, not the mouse, but the vertices of the rectangle). When you find the vector guides in joml that you use, there is an intersectionof class and it has methods for finding the intersection of a ray and a triangle, so you need it. (model obj is triangles just in case). you run through this method all the triangles from the model and the rays that you got and voila you have triangles that intersect with the direction vectors from the vertices of the rectangle. according to the idea, to find all the intersections, you need to break the entire rectangle into squares (50-60) and start a directing vector from the middle of each and find the intersections of the triangles with them from the model. This is a crooked and naive way, but a working one.