private static int GetPolyMergeValue(List<Polygon> polys, int polyA, int polyB, List<PolyVertex> verts, out int edgeA, out int edgeB)
{
int numVertsA = polys[polyA].VertexCount;
int numVertsB = polys[polyB].VertexCount;
//check if polygons share an edge
edgeA = -1;
edgeB = -1;
//don't merge if result is too big
if (numVertsA + numVertsB - 2 > polys[polyA].Vertices.Length)
return -1;
//iterate through all the vertices of polygonA
for (int i = 0; i < numVertsA; i++)
{
//take two nearby vertices
int va0 = polys[polyA].Vertices[i];
int va1 = polys[polyA].Vertices[(i + 1) % numVertsA];
//make sure va0 < va1
if (va0 > va1)
{
int temp = va0;
va0 = va1;
va1 = temp;
}
//iterate through all the vertices of polygon B
for (int j = 0; j < numVertsB; j++)
{
//take two nearby vertices
int vb0 = polys[polyB].Vertices[j];
int vb1 = polys[polyB].Vertices[(j + 1) % numVertsB];
//make sure vb0 < vb1
if (vb0 > vb1)
{
int temp = vb0;
vb0 = vb1;
vb1 = temp;
}
//edge shared, since vertices are equal
if (va0 == vb0 && va1 == vb1)
{
edgeA = i;
edgeB = j;
break;
}
}
}
//no common edge
if (edgeA == -1 || edgeB == -1)
return -1;
//check if merged polygon would be convex
int vertA, vertB, vertC;
vertA = polys[polyA].Vertices[(edgeA + numVertsA - 1) % numVertsA];
vertB = polys[polyA].Vertices[edgeA];
vertC = polys[polyB].Vertices[(edgeB + 2) % numVertsB];
if (!ULeft(verts[vertA], verts[vertB], verts[vertC]))
return -1;
vertA = polys[polyB].Vertices[(edgeB + numVertsB - 1) % numVertsB];
vertB = polys[polyB].Vertices[edgeB];
vertC = polys[polyA].Vertices[(edgeA + 2) % numVertsA];
if (!ULeft(verts[vertA], verts[vertB], verts[vertC]))
return -1;
vertA = polys[polyA].Vertices[edgeA];
vertB = polys[polyA].Vertices[(edgeA + 1) % numVertsA];
int dx = (int)(verts[vertA].X - verts[vertB].X);
int dy = (int)(verts[vertA].Z - verts[vertB].Z);
return dx * dx + dy * dy;
}