linux-menubar: check for dbus-service (AppMenu) instead of desktop env filters

fixed PY-32237 GUI tests Unable to register window, error: GDBus.Error:org.freedesktop.DBus.Error.ServiceUnknown: The name com.canonical.AppMenu.Registrar was not provided by any .service files
fixed IDEA-200273 Exceptions: GlobalMenuLinux - Unable to register window and AppMenu-service can't register xid

native/LinuxGlobalMenu/DbusMenuWrapper.c

@@ -20,6 +20,9 @@
 
 static GMainLoop *_ourMainLoop = NULL;
 static jlogger _ourLogger = NULL;
+static jrunnable _ourOnServiceAppearedCallback = NULL;
+static jrunnable _ourOnServiceVanishedCallback = NULL;
+static guint _ourServiceNameWatcher = 0;
 
 typedef struct _WndInfo {
   guint32 xid;
@@ -28,6 +31,7 @@ typedef struct _WndInfo {
   DbusmenuServer *server;
   DbusmenuMenuitem *menuroot;
   jeventcallback jhandler;
+  GList* linkedXids;
 } WndInfo;
 
 static void _error(const char *msg) {
@@ -65,16 +69,30 @@ static void _printWndInfo(const WndInfo *wi, char *out, int outLen) {
            wi->registrar, wi->server, wi->menuroot);
 }
 
-void runDbusServer(jlogger jlog) {
+static void _onNameAppeared(GDBusConnection *connection, const gchar *name, const gchar *name_owner, gpointer user_data) {
+  if (_ourOnServiceAppearedCallback != NULL)
+    (*((jrunnable) _ourOnServiceAppearedCallback))();
+}
+
+static void _onNameVanished(GDBusConnection *connection, const gchar *name, gpointer user_data) {
+  if (_ourOnServiceVanishedCallback != NULL)
+    (*((jrunnable) _ourOnServiceVanishedCallback))();
+}
+
+void runDbusServer(jlogger jlog, jrunnable onAppmenuServiceAppeared, jrunnable onAppmenuServiceVanished) {
   // NOTE: main-loop is necessary for communication with dbus (via glib and it's signals)
   _ourLogger = jlog;
   _ourMainLoop = g_main_loop_new(NULL/*will be used g_main_context_default()*/, FALSE);
+  _ourOnServiceAppearedCallback = onAppmenuServiceAppeared;
+  _ourOnServiceVanishedCallback = onAppmenuServiceVanished;
+  _ourServiceNameWatcher = g_bus_watch_name(G_BUS_TYPE_SESSION, DBUS_NAME, G_BUS_NAME_WATCHER_FLAGS_NONE, _onNameAppeared, _onNameVanished, NULL, NULL);
   _info("glib main loop is running");
   g_main_loop_run(_ourMainLoop);
 }
 
 void stopDbusServer() {
   g_main_loop_quit(_ourMainLoop);
+  g_bus_unwatch_name(_ourServiceNameWatcher);
   // _info("glib main loop is stopped");
 }
 
@@ -120,8 +138,8 @@ static void _onDbusOwnerChange(GObject *gobject, GParamSpec *pspec, gpointer use
   GError *error = NULL;
   g_dbus_proxy_call_sync(wi->registrar, "RegisterWindow",
                          g_variant_new("(uo)",
-                                       wi->xid,
-                                       wi->menuPath),
+                         wi->xid,
+                         wi->menuPath),
                          G_DBUS_CALL_FLAGS_NONE, -1, NULL, &error);
   if (error != NULL) {
     _logmsg(LOG_LEVEL_ERROR, "Unable to re-register window, error: %s", error->message);
@@ -141,6 +159,19 @@ static void _releaseMenuItem(gpointer data) {
   }
 }
 
+static void _unregisterWindow(long xid, GDBusProxy * registrar) {
+  // NOTE: sync call g_dbus_proxy_call_sync(wi->registrar, "UnregisterWindow", g_variant_new("(u)", wi->xid), G_DBUS_CALL_FLAGS_NONE, -1, NULL, &error)
+  // under ubuntu18 (with GlobalMenu plugin) executes several minutes.
+  // We make async call and don't care about results and errors (i.e. NULL callbacks)
+  g_dbus_proxy_call(registrar,
+                    "UnregisterWindow",
+                    g_variant_new("(u)", xid),
+                    G_DBUS_CALL_FLAGS_NONE, -1,
+                    NULL,
+                    NULL,
+                    NULL);
+}
+
 static void _releaseWindow(WndInfo *wi) {
   if (wi == NULL) return;
   if (wi->menuPath == NULL) {
@@ -162,10 +193,21 @@ static void _releaseWindow(WndInfo *wi) {
   }
 
   if (wi->registrar != NULL) {
+    _unregisterWindow(wi->xid, wi->registrar);
+    if (wi->linkedXids != NULL) {
+      for (GList* l = wi->linkedXids; l != NULL; l = l->next)
+        _unregisterWindow(l->data, wi->registrar);
+    }
+
     g_object_unref(wi->registrar);
     wi->registrar = NULL;
   }
 
+  if (wi->linkedXids != NULL) {
+    g_list_free(wi->linkedXids);
+    wi->linkedXids = NULL;
+  }
+
   free(wi);
 }
 
@@ -255,6 +297,43 @@ WndInfo *registerWindow(long windowXid, jeventcallback handler) {
   return wi;
 }
 
+void bindNewWindow(WndInfo * wi, long windowXid) {
+  if (wi == NULL || wi->server == NULL || wi->menuPath == NULL)
+    return;
+
+  GError *error = NULL;
+  g_dbus_proxy_call_sync(
+    wi->registrar,
+    "RegisterWindow",
+    g_variant_new("(uo)", windowXid, wi->menuPath),
+    G_DBUS_CALL_FLAGS_NONE,
+    -1,
+    NULL,
+    &error);
+
+  if (error != NULL) {
+    _logmsg(LOG_LEVEL_ERROR, "Unable to bind new window, menu-server '%s', error: %s", wi->menuPath, error->message);
+    g_error_free(error);
+    return;
+  }
+
+  if (wi->linkedXids == NULL) {
+    wi->linkedXids = g_list_alloc();
+    wi->linkedXids->data = windowXid;
+  } else
+    g_list_append(wi->linkedXids, windowXid);
+}
+
+void unbindWindow(WndInfo * wi, long windowXid) {
+  if (wi == NULL || wi->server == NULL || wi->menuPath == NULL)
+    return;
+
+  _unregisterWindow(windowXid, wi->registrar);
+
+  if (wi->linkedXids != NULL)
+    g_list_remove(wi->linkedXids, windowXid);
+}
+
 static gboolean _execReleaseWindow(gpointer user_data) {
   _releaseWindow(user_data);
   return FALSE;

native/LinuxGlobalMenu/DbusMenuWrapper.h

@@ -37,13 +37,16 @@ extern "C"{
 
 // runs main loop of glib (which is needed to communicate with dbus)
 // must be called from java thread (to avoid detach, so jna-callbacks will be invoked from same thread)
-void runDbusServer(jlogger jlogger);
+void runDbusServer(jlogger jlogger, jrunnable onAppmenuServiceAppeared, jrunnable onAppmenuServiceVanished);
 void stopDbusServer();
 void execOnMainLoop(jrunnable run);
 
 WndInfo* registerWindow(long windowXid, jeventcallback handler); // creates menu-server and binds to xid
 void releaseWindowOnMainLoop(WndInfo* wi);
 
+void bindNewWindow(WndInfo * wi, long windowXid);
+void unbindWindow(WndInfo * wi, long windowXid);
+
 void createMenuRootForWnd(WndInfo *wi);
 void clearRootMenu(WndInfo* wi);
 void clearMenu(DbusmenuMenuitem* menu);

platform/platform-impl/src/com/intellij/openapi/wm/impl/GlobalMenuLinux.java

@@ -29,18 +29,26 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.text.SimpleDateFormat;
 import java.util.List;
-import java.util.*;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.Date;
 
 interface GlobalMenuLib extends Library {
-  void runDbusServer(JLogger jlogger);
+  void runDbusServer(JLogger jlogger, JRunnable onAppmenuServiceAppeared, JRunnable onAppmenuServiceVanished);
   void stopDbusServer();
   void execOnMainLoop(JRunnable run);
 
   Pointer registerWindow(long windowXid, EventHandler handler);
   void releaseWindowOnMainLoop(Pointer wi);
 
+  void bindNewWindow(Pointer wi, long windowXid);
+  void unbindWindow(Pointer wi, long windowXid);
+
   void clearRootMenu(Pointer wi);
   void clearMenu(Pointer dbmi);
 
@@ -100,11 +108,14 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
   private static final GlobalMenuLib ourLib;
   private static final GlobalMenuLib.JLogger ourGLogger;
   private static final Thread ourGlibMainLoopThread;
+  private static final Map<Long, GlobalMenuLinux> ourInstances = new ConcurrentHashMap<>();
+  private static boolean ourIsServiceAvailable = false;
 
   private final long myXid;
+  private final @NotNull JFrame myFrame;
   private List<MenuItemInternal> myRoots;
   private Pointer myWindowHandle;
-  private GlobalMenuLib.JRunnable myGlibLoopRunnable; // only to hold runnable object until it executed
+  private boolean myIsProcessed = false;
 
   private final EventFilter myEventFilter = new EventFilter();
 
@@ -120,7 +131,7 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
           LOG.error(msg);
         }
       };
-      ourGlibMainLoopThread = new Thread(()->ourLib.runDbusServer(ourGLogger), "Glib-main-loop");
+      ourGlibMainLoopThread = new Thread(()->ourLib.runDbusServer(ourGLogger, GlobalMenuLinux::_onAppmenuServiceAppeared, GlobalMenuLinux::_onAppmenuServiceVanished), "Glib-main-loop");
       ourGlibMainLoopThread.start();
     } else {
       ourGLogger = null;
@@ -130,12 +141,14 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
 
   public static GlobalMenuLinux create(@NotNull JFrame frame) {
     final long xid = _getX11WindowXid(frame);
-    return xid == 0 ? null : new GlobalMenuLinux(xid);
+    return xid == 0 ? null : new GlobalMenuLinux(xid, frame);
   }
 
-  private GlobalMenuLinux(long xid) {
+  private GlobalMenuLinux(long xid, @NotNull JFrame frame) {
     LOG.info("created instance of GlobalMenuLinux for xid=0x" + Long.toHexString(xid));
     myXid = xid;
+    myFrame = frame;
+    ourInstances.put(myXid, this);
   }
 
   @Override
@@ -149,6 +162,20 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
     }
   }
 
+  public void bindNewWindow(@NotNull Window frame) {
+    final long xid = _getX11WindowXid(frame);
+    if (xid == 0)
+      return;
+    ourLib.bindNewWindow(myWindowHandle, xid);
+  }
+
+  public void unbindNewWindow(@NotNull Window frame) {
+    final long xid = _getX11WindowXid(frame);
+    if (xid == 0)
+      return;
+    ourLib.unbindWindow(myWindowHandle, xid);
+  }
+
   public void setRoots(List<ActionMenu> roots) {
     if (ourLib == null)
       return;
@@ -168,29 +195,36 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
 
     myRoots = newRoots;
     _trace("set new menu roots, count=%d", size);
+    myIsProcessed = false;
+    ourLib.execOnMainLoop(ourProcessQueue);
+  }
 
-    myGlibLoopRunnable = () -> {
-      // Glib-loop
+  private void _processRoots() {
+    // exec at glib-thread
+    if (myIsProcessed)
+      return;
+
+    myIsProcessed = true;
+
+    if (myWindowHandle == null) {
+      myWindowHandle = ourLib.registerWindow(myXid, this);
       if (myWindowHandle == null) {
-        myWindowHandle = ourLib.registerWindow(myXid, this);
-        if (myWindowHandle == null) {
-          LOG.error("AppMenu-service can't register xid " + myXid);
-          return;
-        }
-      }
-
-      ourLib.clearRootMenu(myWindowHandle);
-
-      final List<MenuItemInternal> croots = myRoots;
-      if (croots == null || croots.isEmpty())
+        LOG.error("AppMenu-service can't register xid " + myXid);
         return;
-
-      for (MenuItemInternal mi: croots) {
-        mi.nativePeer = ourLib.addRootMenu(myWindowHandle, mi.uid, mi.txt);
       }
-    };
+    }
 
-    ourLib.execOnMainLoop(myGlibLoopRunnable); // TODO: clean ref myGlibLoopRunnable
+    ourLib.clearRootMenu(myWindowHandle);
+
+    final List<MenuItemInternal> croots = myRoots;
+    if (croots == null || croots.isEmpty())
+      return;
+
+    for (MenuItemInternal mi: croots)
+      mi.nativePeer = ourLib.addRootMenu(myWindowHandle, mi.uid, mi.txt);
+
+    if (!Registry.is("linux.native.menu.debug.show.frame.menu", false))
+      ApplicationManager.getApplication().invokeLater(()->myFrame.getJMenuBar().setVisible(false));
   }
 
   private MenuItemInternal _findMenuItem(int uid) {
@@ -448,30 +482,10 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
 
   public static boolean isAvailable() { return ourLib != null; }
 
-  private static boolean _isLinuxEnvSupportsGlobalMenu() {
-    if (!SystemInfo.isLinux || !Registry.is("linux.native.menu"))
-      return false;
-
-    if (!Registry.is("linux.native.menu.debug.check.desktop"))
-      return true;
-
-    String desktop = System.getenv("XDG_CURRENT_DESKTOP");
-    if (desktop == null)
-      return false;
-
-    desktop = desktop.toLowerCase();
-    return desktop.startsWith("unity") || desktop.startsWith("ubuntu") || desktop.equals("kde");
-  }
-
   private static GlobalMenuLib _loadLibrary() {
     if (!SystemInfo.isLinux)
       return null;
 
-    if (!_isLinuxEnvSupportsGlobalMenu()) {
-      LOG.info("skip loading of dbusmenu wrapper because not-supported desktop used: " + String.valueOf(System.getenv("XDG_CURRENT_DESKTOP")));
-      return null;
-    }
-
     UrlClassLoader.loadPlatformLibrary("dbm");
 
     // Set JNA to convert java.lang.String to char* using UTF-8, and match that with
@@ -629,6 +643,7 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
   private class EventFilter {
     private Timer myTimer;
     private long myLastFirstRootEventMs = 0;
+    private GlobalMenuLib.JRunnable myGlibLoopRunnable;
 
     boolean check(int uid, int eventType, @NotNull MenuItemInternal mi) {
       final long timeMs = System.currentTimeMillis();
@@ -717,7 +732,7 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
   }
 
   @SuppressWarnings("deprecation")
-  private static long _getX11WindowXid(@NotNull JFrame frame) {
+  private static long _getX11WindowXid(@NotNull Window frame) {
     final ComponentPeer wndPeer = frame.getPeer();
     if (wndPeer == null) {
       // wait a little for X11-peer to be connected
@@ -806,4 +821,28 @@ public class GlobalMenuLinux implements GlobalMenuLib.EventHandler, Disposable {
     else
       LOG.info(msg);
   }
+
+  private static final GlobalMenuLib.JRunnable ourProcessQueue = () -> {
+    // exec at glib-thread
+    if (!ourIsServiceAvailable)
+      return;
+
+    for (GlobalMenuLinux gml: ourInstances.values())
+      gml._processRoots();
+  };
+
+  private static void _onAppmenuServiceAppeared() {
+    // exec at glib-thread
+    ourIsServiceAvailable = true;
+    ourProcessQueue.run();
+  }
+
+  private static void _onAppmenuServiceVanished() {
+    // exec at glib-thread
+    ourIsServiceAvailable = false;
+    for (GlobalMenuLinux gml: ourInstances.values()) {
+      gml.myWindowHandle = null;
+      ApplicationManager.getApplication().invokeLater(()->gml.myFrame.getJMenuBar().setVisible(true));
+    }
+  }
 }

platform/platform-impl/src/com/intellij/openapi/wm/impl/IdeMenuBar.java

@@ -16,9 +16,11 @@ import com.intellij.openapi.actionSystem.impl.WeakTimerListener;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.application.ModalityState;
 import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.ui.FrameWrapper;
 import com.intellij.openapi.util.Disposer;
 import com.intellij.openapi.util.SystemInfo;
 import com.intellij.openapi.util.registry.Registry;
+import com.intellij.openapi.wm.IdeFrame;
 import com.intellij.openapi.wm.ex.IdeFrameEx;
 import com.intellij.openapi.wm.impl.status.ClockPanel;
 import com.intellij.ui.ColorUtil;
@@ -35,10 +37,7 @@ import org.jetbrains.annotations.Nullable;
 import javax.swing.*;
 import javax.swing.border.Border;
 import java.awt.*;
-import java.awt.event.ActionEvent;
-import java.awt.event.ActionListener;
-import java.awt.event.MouseAdapter;
-import java.awt.event.MouseEvent;
+import java.awt.event.*;
 import java.awt.geom.AffineTransform;
 import java.awt.geom.GeneralPath;
 import java.awt.geom.RoundRectangle2D;
@@ -558,13 +557,34 @@ public class IdeMenuBar extends JMenuBar implements IdeEventQueue.EventDispatche
         frameMenuBar.myGlobalMenuLinux = gml;
         Disposer.register(frameMenuBar.myDisposable, gml);
         frameMenuBar.updateMenuActions(true);
-        if (!Registry.is("linux.native.menu.debug.show.frame.menu"))
-          frameMenuBar.setVisible(false);
       }
     } else if (frame.getJMenuBar() != null)
       LOG.info("The menubar '" + frame.getJMenuBar() + "' of frame '" + frame + "' isn't instance of IdeMenuBar");
   }
 
+  public static void bindAppMenuOfParent(@NotNull final Window frame, IdeFrame parent) {
+    if (!GlobalMenuLinux.isAvailable())
+      return;
+    if (!(parent instanceof JFrame))
+      return;
+
+    final JFrame fr = (JFrame)parent;
+    if (fr.getJMenuBar() instanceof IdeMenuBar) {
+      final IdeMenuBar frameMenuBar = (IdeMenuBar)fr.getJMenuBar();
+      if (frameMenuBar.myGlobalMenuLinux != null) {
+        frame.addWindowListener(new WindowAdapter() {
+          @Override
+          public void windowClosing(WindowEvent e) {
+            frameMenuBar.myGlobalMenuLinux.unbindNewWindow(frame);
+          }
+          public void windowOpened(WindowEvent e) {
+            frameMenuBar.myGlobalMenuLinux.bindNewWindow(frame);
+          }
+        });
+      }
+    }
+  }
+
   private static class MyExitFullScreenButton extends JButton {
     private MyExitFullScreenButton() {
       setFocusable(false);

platform/platform-impl/src/com/intellij/openapi/wm/impl/ToolWindowManagerImpl.java

@@ -1990,6 +1990,9 @@ public class ToolWindowManagerImpl extends ToolWindowManagerEx implements Persis
           hideToolWindow(Objects.requireNonNull(info.getId()), false);
         }
       });
+
+      final IdeFrame parent = WindowManager.getInstance().getIdeFrame(myProject);
+      IdeMenuBar.bindAppMenuOfParent(window, parent);
     }
 
     @Override

platform/util/resources/misc/registry.properties

@@ -858,8 +858,6 @@ ide.mac.message.sheets.java.emulation.dialogs=true
 ide.mac.message.sheets.java.emulation.dialogs.description=Use Java message sheets based on awt dialogs instead of native sheets
 linux.native.menu=true
 linux.native.menu.description=Enables native menu on Ubuntu
-linux.native.menu.debug.check.desktop=true
-linux.native.menu.debug.show.frame.menu=false
 windows.jumplist=false
 windows.jumplist.description=Enables JumpLists on Windows