[WIN32SS][NTGDI] Fix wrong IN/OUT (#1539)
[reactos.git] / win32ss / gdi / ntgdi / xformobj.c
index 6d7ca2d..e1d2241 100644 (file)
 #define NDEBUG
 #include <debug.h>
 
-C_ASSERT(sizeof(FIX) == sizeof(LONG));
-#define FIX2LONG(x) ((x) >> 4)
-#define LONG2FIX(x) ((x) << 4)
-
-#define FLOATOBJ_Equal _FLOATOBJ_Equal
-#define FLOATOBJ_GetLong _FLOATOBJ_GetLong
-#define FLOATOBJ_GetFix _FLOATOBJ_GetFix
-#define FLOATOBJ_IsLong _FLOATOBJ_IsLong
-#define FLOATOBJ_Equal0 _FLOATOBJ_Equal0
-#define FLOATOBJ_Equal1 _FLOATOBJ_Equal1
+#define DOES_VALUE_OVERFLOW_LONG(x) \
+    (((__int64)((long)(x))) != (x))
 
 /** Inline helper functions ***************************************************/
 
@@ -113,7 +105,7 @@ HintFromAccel(ULONG flAccel)
 ULONG
 NTAPI
 XFORMOBJ_UpdateAccel(
-    IN XFORMOBJ *pxo)
+    IN OUT XFORMOBJ *pxo)
 {
     PMATRIX pmx = XFORMOBJ_pmx(pxo);
 
@@ -154,16 +146,25 @@ XFORMOBJ_UpdateAccel(
 ULONG
 NTAPI
 XFORMOBJ_iSetXform(
-    OUT XFORMOBJ *pxo,
+    IN OUT XFORMOBJ *pxo,
     IN const XFORML *pxform)
 {
     PMATRIX pmx = XFORMOBJ_pmx(pxo);
+    FLOATOBJ ef1, ef2, efTemp;
 
     /* Check parameters */
     if (!pxo || !pxform) return DDI_ERROR;
 
     /* Check if the xform is valid */
-    if ((pxform->eM11 == 0) || (pxform->eM22 == 0)) return DDI_ERROR;
+    /* M11 * M22 - M12 * M21 != 0 */
+    FLOATOBJ_SetFloat(&ef1, pxform->eM11);
+    FLOATOBJ_SetFloat(&efTemp, pxform->eM22);
+    FLOATOBJ_Mul(&ef1, &efTemp);
+    FLOATOBJ_SetFloat(&ef2, pxform->eM12);
+    FLOATOBJ_SetFloat(&efTemp, pxform->eM21);
+    FLOATOBJ_Mul(&ef2, &efTemp);
+    if (FLOATOBJ_Equal(&ef1, &ef2))
+        return DDI_ERROR;
 
     /* Copy members */
     FLOATOBJ_SetFloat(&pmx->efM11, pxform->eM11);
@@ -188,7 +189,7 @@ XFORMOBJ_iSetXform(
 ULONG
 NTAPI
 XFORMOBJ_iCombine(
-    IN XFORMOBJ *pxo,
+    IN OUT XFORMOBJ *pxo,
     IN XFORMOBJ *pxo1,
     IN XFORMOBJ *pxo2)
 {
@@ -221,7 +222,7 @@ XFORMOBJ_iCombine(
 ULONG
 NTAPI
 XFORMOBJ_iCombineXform(
-    IN XFORMOBJ *pxo,
+    IN OUT XFORMOBJ *pxo,
     IN XFORMOBJ *pxo1,
     IN XFORML *pxform,
     IN BOOL bLeftMultiply)
@@ -298,75 +299,153 @@ XFORMOBJ_iInverse(
 }
 
 
+/*!
+ * \brief Transforms fix-point coordinates in an array of POINTL structures using
+ *        the transformation matrix from the XFORMOBJ.
+ *
+ * \param pxo - Pointer to the XFORMOBJ
+ *
+ * \param cPoints - Number of coordinates to transform
+ *
+ * \param pptIn - Pointer to an array of POINTL structures containing the
+ *        source coordinates.
+ *
+ * \param pptOut - Pointer to an array of POINTL structures, receiving the
+ *        transformed coordinates. Can be the same as pptIn.
+ *
+ * \return TRUE if the operation was successful, FALSE if any of the calculations
+ *         caused an integer overflow.
+ *
+ * \note If the function returns FALSE, it might still have written to the
+ *       output buffer. If pptIn and pptOut are equal, the source coordinates
+ *       might have been partly overwritten!
+ */
+static
 BOOL
 NTAPI
 XFORMOBJ_bXformFixPoints(
-    IN XFORMOBJ  *pxo,
-    IN ULONG  cPoints,
-    IN PPOINTL  pptIn,
-    OUT PPOINTL  pptOut)
+    _In_ XFORMOBJ *pxo,
+    _In_ ULONG cPoints,
+    _In_reads_(cPoints) PPOINTL pptIn,
+    _Out_writes_(cPoints) PPOINTL pptOut)
 {
     PMATRIX pmx;
     INT i;
     FLOATOBJ fo1, fo2;
     FLONG flAccel;
+    LONG lM11, lM12, lM21, lM22, lTemp;
+    register LONGLONG llx, lly;
 
     pmx = XFORMOBJ_pmx(pxo);
     flAccel = pmx->flAccel;
 
     if ((flAccel & (XFORM_SCALE|XFORM_UNITY)) == (XFORM_SCALE|XFORM_UNITY))
     {
-        /* Identity transformation, nothing todo */
+        /* Identity transformation */
+        RtlCopyMemory(pptOut, pptIn, cPoints * sizeof(POINTL));
     }
     else if (flAccel & XFORM_INTEGER)
     {
         if (flAccel & XFORM_UNITY)
         {
-            /* 1-scale integer transform */
-            LONG lM12 = FLOATOBJ_GetLong(&pmx->efM12);
-            LONG lM21 = FLOATOBJ_GetLong(&pmx->efM21);
+            /* 1-scale integer transform, get the off-diagonal elements */
+            if (!FLOATOBJ_bConvertToLong(&pmx->efM12, &lM12) ||
+                !FLOATOBJ_bConvertToLong(&pmx->efM21, &lM21))
+            {
+                NT_ASSERT(FALSE);
+                return FALSE;
+            }
 
             i = cPoints - 1;
             do
             {
-                LONG x = pptIn[i].x + pptIn[i].y * lM21;
-                LONG y = pptIn[i].y + pptIn[i].x * lM12;
-                pptOut[i].y = y;
-                pptOut[i].x = x;
+                /* Calculate x in 64 bit and check for overflow */
+                llx = Int32x32To64(pptIn[i].y, lM21) + pptIn[i].x;
+                if (DOES_VALUE_OVERFLOW_LONG(llx))
+                {
+                    return FALSE;
+                }
+
+                /* Calculate y in 64 bit and check for overflow */
+                lly = Int32x32To64(pptIn[i].x, lM12) + pptIn[i].y;
+                if (DOES_VALUE_OVERFLOW_LONG(lly))
+                {
+                    return FALSE;
+                }
+
+                /* Write back the results */
+                pptOut[i].x = (LONG)llx;
+                pptOut[i].y = (LONG)lly;
             }
             while (--i >= 0);
         }
         else if (flAccel & XFORM_SCALE)
         {
-            /* Diagonal integer transform */
-            LONG lM11 = FLOATOBJ_GetLong(&pmx->efM11);
-            LONG lM22 = FLOATOBJ_GetLong(&pmx->efM22);
+            /* Diagonal integer transform, get the diagonal elements */
+            if (!FLOATOBJ_bConvertToLong(&pmx->efM11, &lM11) ||
+                !FLOATOBJ_bConvertToLong(&pmx->efM22, &lM22))
+            {
+                NT_ASSERT(FALSE);
+                return FALSE;
+            }
 
             i = cPoints - 1;
             do
             {
-                pptOut[i].x = pptIn[i].x * lM11;
-                pptOut[i].y = pptIn[i].y * lM22;
+                /* Calculate x in 64 bit and check for overflow */
+                llx = Int32x32To64(pptIn[i].x, lM11);
+                if (DOES_VALUE_OVERFLOW_LONG(llx))
+                {
+                    return FALSE;
+                }
+
+                /* Calculate y in 64 bit and check for overflow */
+                lly = Int32x32To64(pptIn[i].y, lM22);
+                if (DOES_VALUE_OVERFLOW_LONG(lly))
+                {
+                    return FALSE;
+                }
+
+                /* Write back the results */
+                pptOut[i].x = (LONG)llx;
+                pptOut[i].y = (LONG)lly;
             }
             while (--i >= 0);
         }
         else
         {
             /* Full integer transform */
-            LONG lM11 = FLOATOBJ_GetLong(&pmx->efM11);
-            LONG lM12 = FLOATOBJ_GetLong(&pmx->efM12);
-            LONG lM21 = FLOATOBJ_GetLong(&pmx->efM21);
-            LONG lM22 = FLOATOBJ_GetLong(&pmx->efM22);
+            if (!FLOATOBJ_bConvertToLong(&pmx->efM11, &lM11) ||
+                !FLOATOBJ_bConvertToLong(&pmx->efM12, &lM12) ||
+                !FLOATOBJ_bConvertToLong(&pmx->efM21, &lM21) ||
+                !FLOATOBJ_bConvertToLong(&pmx->efM22, &lM22))
+            {
+                NT_ASSERT(FALSE);
+                return FALSE;
+            }
 
             i = cPoints - 1;
             do
             {
-                LONG x;
-                x  = pptIn[i].x * lM11;
-                x += pptIn[i].y * lM21;
-                pptOut[i].y  = pptIn[i].y * lM22;
-                pptOut[i].y += pptIn[i].x * lM12;
-                pptOut[i].x = x;
+                /* Calculate x in 64 bit and check for overflow */
+                llx  = Int32x32To64(pptIn[i].x, lM11);
+                llx += Int32x32To64(pptIn[i].y, lM21);
+                if (DOES_VALUE_OVERFLOW_LONG(llx))
+                {
+                    return FALSE;
+                }
+
+                /* Calculate y in 64 bit and check for overflow */
+                lly  = Int32x32To64(pptIn[i].y, lM22);
+                lly += Int32x32To64(pptIn[i].x, lM12);
+                if (DOES_VALUE_OVERFLOW_LONG(lly))
+                {
+                    return FALSE;
+                }
+
+                /* Write back the results */
+                pptOut[i].x = (LONG)llx;
+                pptOut[i].y = (LONG)lly;
             }
             while (--i >= 0);
         }
@@ -377,12 +456,35 @@ XFORMOBJ_bXformFixPoints(
         i = cPoints - 1;
         do
         {
+            /* Calculate x in 64 bit and check for overflow */
             fo1 = pmx->efM21;
             FLOATOBJ_MulLong(&fo1, pptIn[i].y);
+            if (!FLOATOBJ_bConvertToLong(&fo1, &lTemp))
+            {
+                return FALSE;
+            }
+            llx = (LONGLONG)pptIn[i].x + lTemp;
+            if (DOES_VALUE_OVERFLOW_LONG(llx))
+            {
+                return FALSE;
+            }
+
+            /* Calculate y in 64 bit and check for overflow */
             fo2 = pmx->efM12;
             FLOATOBJ_MulLong(&fo2, pptIn[i].x);
-            pptOut[i].x = pptIn[i].x + FLOATOBJ_GetLong(&fo1);
-            pptOut[i].y = pptIn[i].y + FLOATOBJ_GetLong(&fo2);
+            if (!FLOATOBJ_bConvertToLong(&fo2, &lTemp))
+            {
+                return FALSE;
+            }
+            lly = (LONGLONG)pptIn[i].y + lTemp;
+            if (DOES_VALUE_OVERFLOW_LONG(lly))
+            {
+                return FALSE;
+            }
+
+            /* Write back the results */
+            pptOut[i].x = (LONG)llx;
+            pptOut[i].y = (LONG)lly;
         }
         while (--i >= 0);
     }
@@ -394,10 +496,17 @@ XFORMOBJ_bXformFixPoints(
         {
             fo1 = pmx->efM11;
             FLOATOBJ_MulLong(&fo1, pptIn[i].x);
-            pptOut[i].x = FLOATOBJ_GetLong(&fo1);
+            if (!FLOATOBJ_bConvertToLong(&fo1, &pptOut[i].x))
+            {
+                return FALSE;
+            }
+
             fo2 = pmx->efM22;
             FLOATOBJ_MulLong(&fo2, pptIn[i].y);
-            pptOut[i].y = FLOATOBJ_GetLong(&fo2);
+            if (!FLOATOBJ_bConvertToLong(&fo2, &pptOut[i].y))
+            {
+                return FALSE;
+            }
         }
         while (--i >= 0);
     }
@@ -407,10 +516,21 @@ XFORMOBJ_bXformFixPoints(
         i = cPoints - 1;
         do
         {
+            /* Calculate x as FLOATOBJ */
             MulAddLong(&fo1, &pmx->efM11, pptIn[i].x, &pmx->efM21, pptIn[i].y);
+
+            /* Calculate y as FLOATOBJ */
             MulAddLong(&fo2, &pmx->efM12, pptIn[i].x, &pmx->efM22, pptIn[i].y);
-            pptOut[i].x = FLOATOBJ_GetLong(&fo1);
-            pptOut[i].y = FLOATOBJ_GetLong(&fo2);
+
+            if (!FLOATOBJ_bConvertToLong(&fo1, &pptOut[i].x))
+            {
+                return FALSE;
+            }
+
+            if (!FLOATOBJ_bConvertToLong(&fo2, &pptOut[i].y))
+            {
+                return FALSE;
+            }
         }
         while (--i >= 0);
     }
@@ -421,8 +541,19 @@ XFORMOBJ_bXformFixPoints(
         i = cPoints - 1;
         do
         {
-            pptOut[i].x += pmx->fxDx;
-            pptOut[i].y += pmx->fxDy;
+            llx = (LONGLONG)pptOut[i].x + pmx->fxDx;
+            if (DOES_VALUE_OVERFLOW_LONG(llx))
+            {
+                return FALSE;
+            }
+            pptOut[i].x = (LONG)llx;
+
+            lly = (LONGLONG)pptOut[i].y + pmx->fxDy;
+            if (DOES_VALUE_OVERFLOW_LONG(lly))
+            {
+                return FALSE;
+            }
+            pptOut[i].y = (LONG)lly;
         }
         while (--i >= 0);
     }
@@ -500,7 +631,7 @@ XFORMOBJ_bApplyXform(
 {
     MATRIX mx;
     XFORMOBJ xoInv;
-    POINTL *pptl;
+    PPOINTL pptlIn, pptlOut;
     INT i;
 
     /* Check parameters */
@@ -523,12 +654,16 @@ XFORMOBJ_bApplyXform(
     /* Convert POINTL to POINTFIX? */
     if (iMode == XF_LTOFX || iMode == XF_LTOL || iMode == XF_INV_LTOL)
     {
-        pptl = pvIn;
+        pptlIn = pvIn;
+        pptlOut = pvOut;
         for (i = cPoints - 1; i >= 0; i--)
         {
-            pptl[i].x = LONG2FIX(pptl[i].x);
-            pptl[i].y = LONG2FIX(pptl[i].y);
+            pptlOut[i].x = LONG2FIX(pptlIn[i].x);
+            pptlOut[i].y = LONG2FIX(pptlIn[i].y);
         }
+
+        /* The input is in the out buffer now! */
+        pvIn = pvOut;
     }
 
     /* Do the actual fixpoint transformation */
@@ -540,11 +675,11 @@ XFORMOBJ_bApplyXform(
     /* Convert POINTFIX to POINTL? */
     if (iMode == XF_INV_FXTOL || iMode == XF_INV_LTOL || iMode == XF_LTOL)
     {
-        pptl = pvOut;
+        pptlOut = pvOut;
         for (i = cPoints - 1; i >= 0; i--)
         {
-            pptl[i].x = FIX2LONG(pptl[i].x);
-            pptl[i].y = FIX2LONG(pptl[i].y);
+            pptlOut[i].x = FIX2LONG(pptlOut[i].x);
+            pptlOut[i].y = FIX2LONG(pptlOut[i].y);
         }
     }