Move surface data checking to a separate function to make it less "totally nuts"

2 files changed, 47 insertions, 12 deletions

diff --git a/dom/base/nsContentUtils.cpp b/dom/base/nsContentUtils.cpp
index 8612e76df4..2f85c1b7e1 100644
--- a/dom/base/nsContentUtils.cpp
+++ b/dom/base/nsContentUtils.cpp
@@ -7597,6 +7597,23 @@ nsContentUtils::IsFileImage(nsIFile* aFile, nsACString& aType)
 }
 
 nsresult
+nsContentUtils::CalculateBufferSizeForImage(const uint32_t& aStride,
+                                            const IntSize& aImageSize,
+                                            const SurfaceFormat& aFormat,
+                                            size_t* aMaxBufferSize,
+                                            size_t* aUsedBufferSize)
+{
+  CheckedInt32 requiredBytes =
+    CheckedInt32(aStride) * CheckedInt32(aImageSize.height);
+  if (!requiredBytes.isValid()) {
+    return NS_ERROR_FAILURE;
+  }
+  *aMaxBufferSize = requiredBytes.value();
+  *aUsedBufferSize = *aMaxBufferSize - aStride + (aImageSize.width * BytesPerPixel(aFormat));
+  return NS_OK;
+}
+
+nsresult
 nsContentUtils::DataTransferItemToImage(const IPCDataTransferItem& aItem,
                                         imgIContainer** aContainer)
 {
@@ -7611,6 +7628,21 @@ nsContentUtils::DataTransferItemToImage(const IPCDataTransferItem& aItem,
 
   Shmem data = aItem.data().get_Shmem();
 
+  // Validate shared memory buffer size
+  size_t imageBufLen = 0;
+  size_t maxBufLen = 0;
+  nsresult rv = CalculateBufferSizeForImage(imageDetails.stride(),
+                                            size,
+                                            imageDetails.format(),
+                                            &maxBufLen,
+                                            &imageBufLen);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  if (imageBufLen > data.Size<uint8_t>()) {
+    return NS_ERROR_FAILURE;
+  }
+
   RefPtr<DataSourceSurface> image =
       CreateDataSourceSurfaceFromData(size,
                                       static_cast<SurfaceFormat>(imageDetails.format()),
@@ -7950,20 +7982,17 @@ GetSurfaceDataImpl(mozilla::gfx::DataSourceSurface* aSurface,
     return GetSurfaceDataContext::NullValue();
   }
 
-  mozilla::gfx::IntSize size = aSurface->GetSize();
-  mozilla::CheckedInt32 requiredBytes =
-    mozilla::CheckedInt32(map.mStride) * mozilla::CheckedInt32(size.height);
-  if (!requiredBytes.isValid()) {
+  size_t bufLen = 0;
+  size_t maxBufLen = 0;
+  nsresult rv = nsContentUtils::CalculateBufferSizeForImage(map.mStride,
+                                                            aSurface->GetSize(),
+                                                            aSurface->GetFormat(),
+                                                            &maxBufLen,
+                                                            &bufLen);
+  if (NS_FAILED(rv)) {
     return GetSurfaceDataContext::NullValue();
   }
 
-  size_t maxBufLen = requiredBytes.value();
-  mozilla::gfx::SurfaceFormat format = aSurface->GetFormat();
-
-  // Surface data handling is totally nuts. This is the magic one needs to
-  // know to access the data.
-  size_t bufLen = maxBufLen - map.mStride + (size.width * BytesPerPixel(format));
-
   // nsDependentCString wants null-terminated string.
   typename GetSurfaceDataContext::ReturnType surfaceData = aContext.Allocate(maxBufLen + 1);
   if (GetSurfaceDataContext::GetBuffer(surfaceData)) {
diff --git a/dom/base/nsContentUtils.h b/dom/base/nsContentUtils.h
index c255f813af..98df92efb0 100644
--- a/dom/base/nsContentUtils.h
+++ b/dom/base/nsContentUtils.h
@@ -975,11 +975,17 @@ public:
 
   static bool PrefetchEnabled(nsIDocShell* aDocShell);
 
+  static nsresult CalculateBufferSizeForImage(const uint32_t& aStride,
+                                              const mozilla::gfx::IntSize& aImageSize,
+                                              const mozilla::gfx::SurfaceFormat& aFormat,
+                                              size_t* aMaxBufferSize,
+                                              size_t* aUsedBufferSize);
+
+private:
   /**
    * Fill (with the parameters given) the localized string named |aKey| in
    * properties file |aFile|.
    */
-private:
   static nsresult FormatLocalizedString(PropertiesFile aFile,
                                         const char* aKey,
                                         const char16_t** aParams,