Check if mouse is in triangle with three x,y coordinates

Say I have a triangle made with 3 points.

makeTriangle(x1, y1, x2, y2, x3, y3);

How do I check if said triangle contains the a certain set of points?

I’m trying to make an interactive UI with P5.js that includes an arrow that allows you to resize the object. The wireframe code is:

let Size, x, y, moving;

//runs once at the start of the program
function setup() {
  
  createCanvas(400, 400);
  
  Size = 100;
  x = 10;
  y = 10;
  moving = false;
  
}
//runs once every frame
function draw() {
  
  background(220);
  
  handleMouse();
  
  fill("grey");
  noStroke();
  
  square(x, y, Size, 5);
  
  fill("black");
  
  triangle( x + Size * 0.9, y + Size * 0.9,
  x + Size * 0.7, y + Size * 0.9,
  x + Size * 0.9, y + Size * 0.7 );
  
}

function handleMouse(){
  
  if(mouseInTriangle(/* x1, y1, x2, y2, x3, y3 */) && mouseIsPressed || mouseIsPressed && moving){
    
    moving = true;
    
  } else {
    
    moving = false;
    
  }
  
  if(moving){
    
    Size = max((mouseX + mouseY)/2 + x + y, 50);
    
  }
  
}

function mouseInTriangle(x1, y1, x2, y2, x3, y3){
  
  //Is mouse in triangle?
  
  return true;
  
}

<script src="https://cdn.jsdelivr.net/npm/p5@1.4.1/lib/p5.js"></script>

Is there a dynamic way to tell if a point is within a triangle?

Answer

I suggest to use an algorithm that compares the areas of triangles. See Check whether a given point lies inside a triangle or not.If the point is in a triangle, that point divides the triangle into 3 smaller triangles. Calculate the sum of the areas of these 3 triangles and compare it to the area of the originally triangle:

function getArea(a, b, c) {
    return abs((a[0]*(b[1]-c[1]) + b[0]*(c[1]-a[1])+ c[0]*(a[1]-b[1]))/2);
}

function mouseInTriangle(x1, y1, x2, y2, x3, y3){
    let point = [mouseX, mouseY];
    let area = getArea([x1, y1], [x2, y2], [x3, y3]);
    let areaA = getArea([x1, y1], [x2, y2], point);
    let areaB = getArea(point, [x2, y2], [x3, y3]);
    let areaC = getArea([x1, y1], point, [x3, y3]);
    return abs(areaA + areaB + areaC - area) < 0.001;
}

let x1, y1, x2, y2, x3, y3;    
let Size, x, y, moving;

//runs once at the start of the program
function setup() {
  
  createCanvas(400, 400);
  
  Size = 100;
  x = 10;
  y = 10;
  moving = false;
  
}
//runs once every frame
function draw() {
  
  background(220);
  handleMouse();
  
  fill("grey");
  noStroke();
  square(x, y, Size, 5);
  fill("black");
  
  x1 = x + Size * 0.9;
  y1 = y + Size * 0.9;
  x2 = x + Size * 0.7;
  y2 = y + Size * 0.9;
  x3 = x + Size * 0.9;
  y3 = y + Size * 0.7;
  triangle(x1, y1, x2, y2, x3, y3);
  
}

function handleMouse(){
    if(mouseIsPressed && (moving || mouseInTriangle(x1, y1, x2, y2, x3, y3))) {
        moving = true;
        Size = max((mouseX + mouseY)/2 + x + y, 50);
    } else {
        moving = false;
    }
}

function getArea(a, b, c) {
    return abs((a[0]*(b[1]-c[1]) + b[0]*(c[1]-a[1])+ c[0]*(a[1]-b[1]))/2);
}

function mouseInTriangle(x1, y1, x2, y2, x3, y3){
    let point = [mouseX, mouseY];
    let area = getArea([x1, y1], [x2, y2], [x3, y3]);
    let areaA = getArea([x1, y1], [x2, y2], point);
    let areaB = getArea(point, [x2, y2], [x3, y3]);
    let areaC = getArea([x1, y1], point, [x3, y3]);
    return abs(areaA + areaB + areaC - area) < 0.001;
}

<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.1/p5.min.js"></script>