SharpNav.PolyMesh.GetPolyMergeValue C# (CSharp) Method

GetPolyMergeValue() private static method

Try to merge two polygons. If possible, return the distance squared between two vertices.
private static GetPolyMergeValue ( List polys, int polyA, int polyB, List verts, int &edgeA, int &edgeB ) : int
polys List Polygon list
polyA int Polygon A
polyB int Polygon B
verts List Vertex list
edgeA int Shared edge's endpoint A
edgeB int Shared edge's endpoint B
return int
        private static int GetPolyMergeValue(List<Polygon> polys, int polyA, int polyB, List<PolyVertex> verts, out int edgeA, out int edgeB)
        {
            int numVertsA = polys[polyA].VertexCount;
            int numVertsB = polys[polyB].VertexCount;

            //check if polygons share an edge
            edgeA = -1;
            edgeB = -1;

            //don't merge if result is too big
            if (numVertsA + numVertsB - 2 > polys[polyA].Vertices.Length)
                return -1;

            //iterate through all the vertices of polygonA
            for (int i = 0; i < numVertsA; i++)
            {
                //take two nearby vertices
                int va0 = polys[polyA].Vertices[i];
                int va1 = polys[polyA].Vertices[(i + 1) % numVertsA];

                //make sure va0 < va1
                if (va0 > va1)
                {
                    int temp = va0;
                    va0 = va1;
                    va1 = temp;
                }

                //iterate through all the vertices of polygon B
                for (int j = 0; j < numVertsB; j++)
                {
                    //take two nearby vertices
                    int vb0 = polys[polyB].Vertices[j];
                    int vb1 = polys[polyB].Vertices[(j + 1) % numVertsB];

                    //make sure vb0 < vb1
                    if (vb0 > vb1)
                    {
                        int temp = vb0;
                        vb0 = vb1;
                        vb1 = temp;
                    }

                    //edge shared, since vertices are equal
                    if (va0 == vb0 && va1 == vb1)
                    {
                        edgeA = i;
                        edgeB = j;
                        break;
                    }
                }
            }

            //no common edge
            if (edgeA == -1 || edgeB == -1)
                return -1;

            //check if merged polygon would be convex
            int vertA, vertB, vertC;

            vertA = polys[polyA].Vertices[(edgeA + numVertsA - 1) % numVertsA];
            vertB = polys[polyA].Vertices[edgeA];
            vertC = polys[polyB].Vertices[(edgeB + 2) % numVertsB];
            if (!ULeft(verts[vertA], verts[vertB], verts[vertC]))
                return -1;

            vertA = polys[polyB].Vertices[(edgeB + numVertsB - 1) % numVertsB];
            vertB = polys[polyB].Vertices[edgeB];
            vertC = polys[polyA].Vertices[(edgeA + 2) % numVertsA];
            if (!ULeft(verts[vertA], verts[vertB], verts[vertC]))
                return -1;

            vertA = polys[polyA].Vertices[edgeA];
            vertB = polys[polyA].Vertices[(edgeA + 1) % numVertsA];

            int dx = (int)(verts[vertA].X - verts[vertB].X);
            int dy = (int)(verts[vertA].Z - verts[vertB].Z);
            return dx * dx + dy * dy;
        }