Issue #1667 - Part 3: Fix OpenGL load and runtime issues on Big Sur

This fix is included in NSPR 4.27 and Mozilla bug 1652330.
Also put a main thread check in the cocoa draw callback.

1 files changed, 33 insertions, 6 deletions

diff --git a/nsprpub/pr/src/linking/prlink.c b/nsprpub/pr/src/linking/prlink.c
index 1f3430714e..4e73d9df1c 100644
--- a/nsprpub/pr/src/linking/prlink.c
+++ b/nsprpub/pr/src/linking/prlink.c
@@ -776,6 +776,9 @@ pr_LoadLibraryByPathname(const char *name, PRIntn flags)
         int dl_flags = 0;
 #endif
         void *h = NULL;
+#if defined(DARWIN)
+        PRBool okToLoad = PR_FALSE;
+#endif
 
         if (flags & PR_LD_LAZY) {
             dl_flags |= RTLD_LAZY;
@@ -790,12 +793,36 @@ pr_LoadLibraryByPathname(const char *name, PRIntn flags)
             dl_flags |= RTLD_LOCAL;
         }
 #if defined(DARWIN)
-        /* ensure the file exists if it contains a slash character i.e. path */
-        /* DARWIN's dlopen ignores the provided path and checks for the */
-        /* plain filename in DYLD_LIBRARY_PATH */
-        if (strchr(name, PR_DIRECTORY_SEPARATOR) == NULL ||
-            PR_Access(name, PR_ACCESS_EXISTS) == PR_SUCCESS) {
-            h = dlopen(name, dl_flags);
+        /* If the file contains an absolute or relative path (slash)
+         * and the path doesn't look like a System path, then require
+         * the file exists.
+         * The reason is that DARWIN's dlopen ignores the provided path
+         * and checks for the plain filename in DYLD_LIBRARY_PATH,
+         * which could load an unexpected version of a library. */
+        if (strchr(name, PR_DIRECTORY_SEPARATOR) == NULL) {
+          /* no slash, allow to load from any location */
+          okToLoad = PR_TRUE;
+        } else {
+          const char systemPrefix1[] = "/System/";
+          const size_t systemPrefixLen1 = strlen(systemPrefix1);
+          const char systemPrefix2[] = "/usr/lib/";
+          const size_t systemPrefixLen2 = strlen(systemPrefix2);
+          const name_len = strlen(name);
+          if (((name_len > systemPrefixLen1) &&
+               (strncmp(name, systemPrefix1, systemPrefixLen1) == 0)) ||
+             ((name_len > systemPrefixLen2) &&
+               (strncmp(name, systemPrefix2, systemPrefixLen2) == 0))) {
+            /* found at beginning, it's a system library.
+             * Skip filesystem check (required for macOS 11),
+             * allow loading from any location */
+            okToLoad = PR_TRUE;
+          } else if (PR_Access(name, PR_ACCESS_EXISTS) == PR_SUCCESS) {
+            /* file exists, allow to load */
+            okToLoad = PR_TRUE;
+          }
+        }
+        if (okToLoad) {
+          h = dlopen(name, dl_flags);
         }
 #else
         h = dlopen(name, dl_flags);