How to make an OpenGL rendering context with transparent background?
After spending some reputation on a unsuccessful bounty to get some help on this issue, I finally realized how complex was the problem I was interested in.
The few individuals that have accomplished this task don't share much. During my research I found different ways to achieve what I was looking for. One of the most interesting ones is AeroGL, and it shows snippets of code using a technique that was not mentioned so far, which is rendering the graphics to a device-independent bitmap (DIB).
To close this thread permanently, the source code below implements that technique. The code itself is a slight modification of an application presented here (big thanks to Andrei Sapronov Y.).
The end result can be seen in the image below:
The code has been tested on Windows XP (32-bits) and Windows 8.1 (32-bits).Enjoy!
#define _WIN32_WINNT 0x0500#include <windows.h>#include <windowsx.h>#include <GL/gl.h>#include <GL/glu.h>#pragma comment (lib, "opengl32.lib")#pragma comment (lib, "glu32.lib")#include <assert.h>#include <tchar.h>#ifdef assert#define verify(expr) if(!expr) assert(0)#else verify(expr) expr#endifconst TCHAR szAppName[]=_T("TransparentGL");const TCHAR wcWndName[]=_T("WS_EX_LAYERED OpenGL");HDC hDC; HGLRC m_hrc; int w(240);int h(240); HDC pdcDIB; HBITMAP hbmpDIB; void *bmp_cnt(NULL); int cxDIB(0); int cyDIB(0); BITMAPINFOHEADER BIH; BOOL initSC(){ glEnable(GL_ALPHA_TEST); glEnable(GL_DEPTH_TEST); glEnable(GL_COLOR_MATERIAL); glEnable(GL_LIGHTING); glEnable(GL_LIGHT0); glEnable(GL_BLEND); glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); glClearColor(0, 0, 0, 0); return 0;}void resizeSC(int width,int height){ glViewport(0,0,width,height); glMatrixMode(GL_PROJECTION); glLoadIdentity(); glMatrixMode(GL_MODELVIEW ); glLoadIdentity();}BOOL renderSC(){ glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); glPushMatrix(); glColor3f(0, 1, 1); glBegin(GL_TRIANGLES); // Drawing Using Triangles glColor3f(1.0f,0.0f,0.0f); // Set The Color To Red glVertex3f( 0.0f, 1.0f, 0.0f); // Top glColor3f(0.0f,1.0f,0.0f); // Set The Color To Green glVertex3f(-1.0f,-1.0f, 0.0f); // Bottom Left glColor3f(0.0f,0.0f,1.0f); // Set The Color To Blue glVertex3f( 1.0f,-1.0f, 0.0f); // Bottom Right glEnd(); glPopMatrix(); glFlush(); return 0;}// DIB -> hDCvoid draw(HDC pdcDest){ assert(pdcDIB); verify(BitBlt(pdcDest, 0, 0, w, h, pdcDIB, 0, 0, SRCCOPY));}void CreateDIB(int cx, int cy){ assert(cx > 0); assert(cy > 0); cxDIB = cx ; cyDIB = cy ; int iSize = sizeof(BITMAPINFOHEADER); memset(&BIH, 0, iSize); BIH.biSize = iSize; BIH.biWidth = cx; BIH.biHeight = cy; BIH.biPlanes = 1; BIH.biBitCount = 24; BIH.biCompression = BI_RGB; if(pdcDIB) verify(DeleteDC(pdcDIB)); pdcDIB = CreateCompatibleDC(NULL); assert(pdcDIB); if(hbmpDIB) verify(DeleteObject(hbmpDIB)); hbmpDIB = CreateDIBSection( pdcDIB, (BITMAPINFO*)&BIH, DIB_RGB_COLORS, &bmp_cnt, NULL, 0); assert(hbmpDIB); assert(bmp_cnt); if(hbmpDIB) SelectObject(pdcDIB, hbmpDIB);}BOOL CreateHGLRC(){ DWORD dwFlags = PFD_SUPPORT_OPENGL | PFD_DRAW_TO_BITMAP; PIXELFORMATDESCRIPTOR pfd ; memset(&pfd,0, sizeof(PIXELFORMATDESCRIPTOR)) ; pfd.nSize = sizeof(PIXELFORMATDESCRIPTOR); pfd.nVersion = 1; pfd.dwFlags = dwFlags ; pfd.iPixelType = PFD_TYPE_RGBA ; pfd.cColorBits = 24 ; pfd.cDepthBits = 32 ; pfd.iLayerType = PFD_MAIN_PLANE ; int PixelFormat = ChoosePixelFormat(pdcDIB, &pfd); if (PixelFormat == 0){ assert(0); return FALSE ; } BOOL bResult = SetPixelFormat(pdcDIB, PixelFormat, &pfd); if (bResult==FALSE){ assert(0); return FALSE ; } m_hrc = wglCreateContext(pdcDIB); if (!m_hrc){ assert(0); return FALSE; } return TRUE;}LRESULT CALLBACK WindowFunc(HWND hWnd,UINT msg, WPARAM wParam, LPARAM lParam){ PAINTSTRUCT ps; switch(msg) { case WM_ERASEBKGND: return 0; break; case WM_CREATE: break; case WM_DESTROY: if(m_hrc) { wglMakeCurrent(NULL, NULL); wglDeleteContext(m_hrc) ; } PostQuitMessage(0) ; break; case WM_PAINT: hDC = BeginPaint(hWnd, &ps); renderSC(); // OpenGL -> DIB draw(hDC); // DIB -> hDC EndPaint(hWnd, &ps); break; case WM_SIZE: w = LOWORD(lParam); h = HIWORD(lParam); wglMakeCurrent(NULL, NULL); wglDeleteContext(m_hrc); CreateDIB(w, h); CreateHGLRC(); verify(wglMakeCurrent(pdcDIB, m_hrc)); initSC(); resizeSC(w, h); renderSC(); break; default: return DefWindowProc(hWnd,msg,wParam,lParam); } return 0;}int WINAPI _tWinMain(HINSTANCE hThisInst, HINSTANCE hPrevInst, LPSTR str,int nWinMode){ WNDCLASSEX wc; memset(&wc, 0, sizeof(wc)); wc.cbSize = sizeof(WNDCLASSEX); wc.hIconSm = LoadIcon(NULL, IDI_APPLICATION); wc.style = CS_HREDRAW | CS_VREDRAW; wc.lpfnWndProc = (WNDPROC)WindowFunc; wc.cbClsExtra = 0; wc.cbWndExtra = 0; wc.hInstance = hThisInst; wc.hIcon = LoadIcon(NULL, IDI_APPLICATION); wc.hCursor = LoadCursor(NULL, IDC_ARROW); wc.hbrBackground = (HBRUSH) (COLOR_WINDOW); wc.lpszClassName = szAppName; if(!RegisterClassEx(&wc)) { MessageBox(NULL, _T("RegisterClassEx - failed"), _T("Error"), MB_OK | MB_ICONERROR); return FALSE; } HWND hWnd = CreateWindowEx(WS_EX_LAYERED, szAppName, wcWndName, WS_VISIBLE | WS_POPUP, 200, 150, w, h, NULL, NULL, hThisInst, NULL); if(!hWnd){ MessageBox(NULL, _T("CreateWindowEx - failed"), _T("Error"), MB_OK | MB_ICONERROR); return FALSE; } verify(SetLayeredWindowAttributes(hWnd, 0x0, 0, LWA_COLORKEY)); MSG msg; while(1) { while (PeekMessage(&msg,NULL,0,0,PM_NOREMOVE)){ if (GetMessage(&msg, NULL, 0, 0)) { TranslateMessage(&msg); DispatchMessage(&msg); } else return 0; } } return (FALSE); }
Since all the answers given so far target Windows only, but there's surely also a demand doing this on X11 with a composited window manager, for reference I post my example code here (also to be found at https://github.com/datenwolf/codesamples/blob/master/samples/OpenGL/x11argb_opengl/x11argb_opengl.c
/*------------------------------------------------------------------------ * A demonstration of OpenGL in a ARGB window * => support for composited window transparency * * (c) 2011 by Wolfgang 'datenwolf' Draxinger * See me at comp.graphics.api.opengl and StackOverflow.com * License agreement: This source code is provided "as is". You * can use this source code however you want for your own personal * use. If you give this source code to anybody else then you must * leave this message in it. * * This program is based on the simplest possible * Linux OpenGL program by FTB (see info below) The simplest possible Linux OpenGL program? Maybe... (c) 2002 by FTB. See me in comp.graphics.api.opengl -- <\___/> / O O \ \_____/ FTB.------------------------------------------------------------------------*/#include <stdlib.h>#include <stdio.h>#include <string.h>#include <math.h>#include <GL/gl.h>#include <GL/glx.h>#include <GL/glxext.h>#include <X11/Xatom.h>#include <X11/extensions/Xrender.h>#include <X11/Xutil.h>#define USE_CHOOSE_FBCONFIGstatic void fatalError(const char *why){ fprintf(stderr, "%s", why); exit(0x666);}static int Xscreen;static Atom del_atom;static Colormap cmap;static Display *Xdisplay;static XVisualInfo *visual;static XRenderPictFormat *pict_format;static GLXFBConfig *fbconfigs, fbconfig;static int numfbconfigs;static GLXContext render_context;static Window Xroot, window_handle;static GLXWindow glX_window_handle;static int width, height;static int VisData[] = {GLX_RENDER_TYPE, GLX_RGBA_BIT,GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT,GLX_DOUBLEBUFFER, True,GLX_RED_SIZE, 8,GLX_GREEN_SIZE, 8,GLX_BLUE_SIZE, 8,GLX_ALPHA_SIZE, 8,GLX_DEPTH_SIZE, 16,None};static int isExtensionSupported(const char *extList, const char *extension){ const char *start; const char *where, *terminator; /* Extension names should not have spaces. */ where = strchr(extension, ' '); if ( where || *extension == '\0' ) return 0; /* It takes a bit of care to be fool-proof about parsing the OpenGL extensions string. Don't be fooled by sub-strings, etc. */ for ( start = extList; ; ) { where = strstr( start, extension ); if ( !where ) break; terminator = where + strlen( extension ); if ( where == start || *(where - 1) == ' ' ) if ( *terminator == ' ' || *terminator == '\0' ) return 1; start = terminator; } return 0;}static Bool WaitForMapNotify(Display *d, XEvent *e, char *arg){ return d && e && arg && (e->type == MapNotify) && (e->xmap.window == *(Window*)arg);}static void describe_fbconfig(GLXFBConfig fbconfig){ int doublebuffer; int red_bits, green_bits, blue_bits, alpha_bits, depth_bits; glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_DOUBLEBUFFER, &doublebuffer); glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_RED_SIZE, &red_bits); glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_GREEN_SIZE, &green_bits); glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_BLUE_SIZE, &blue_bits); glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_ALPHA_SIZE, &alpha_bits); glXGetFBConfigAttrib(Xdisplay, fbconfig, GLX_DEPTH_SIZE, &depth_bits); fprintf(stderr, "FBConfig selected:\n" "Doublebuffer: %s\n" "Red Bits: %d, Green Bits: %d, Blue Bits: %d, Alpha Bits: %d, Depth Bits: %d\n", doublebuffer == True ? "Yes" : "No", red_bits, green_bits, blue_bits, alpha_bits, depth_bits);}static void createTheWindow(){ XEvent event; int x,y, attr_mask; XSizeHints hints; XWMHints *startup_state; XTextProperty textprop; XSetWindowAttributes attr = {0,}; static char *title = "FTB's little OpenGL example - ARGB extension by WXD"; Xdisplay = XOpenDisplay(NULL); if (!Xdisplay) { fatalError("Couldn't connect to X server\n"); } Xscreen = DefaultScreen(Xdisplay); Xroot = RootWindow(Xdisplay, Xscreen); fbconfigs = glXChooseFBConfig(Xdisplay, Xscreen, VisData, &numfbconfigs); fbconfig = 0; for(int i = 0; i<numfbconfigs; i++) { visual = (XVisualInfo*) glXGetVisualFromFBConfig(Xdisplay, fbconfigs[i]); if(!visual) continue; pict_format = XRenderFindVisualFormat(Xdisplay, visual->visual); XFree(visual); if(!pict_format) continue; fbconfig = fbconfigs[i]; if(pict_format->direct.alphaMask > 0) { break; } } if(!fbconfig) { fatalError("No matching FB config found"); } describe_fbconfig(fbconfig); /* Create a colormap - only needed on some X clients, eg. IRIX */ cmap = XCreateColormap(Xdisplay, Xroot, visual->visual, AllocNone); attr.colormap = cmap; attr.background_pixmap = None; attr.border_pixmap = None; attr.border_pixel = 0; attr.event_mask = StructureNotifyMask | EnterWindowMask | LeaveWindowMask | ExposureMask | ButtonPressMask | ButtonReleaseMask | OwnerGrabButtonMask | KeyPressMask | KeyReleaseMask; attr_mask = CWBackPixmap| CWColormap| CWBorderPixel| CWEventMask; width = DisplayWidth(Xdisplay, DefaultScreen(Xdisplay))/2; height = DisplayHeight(Xdisplay, DefaultScreen(Xdisplay))/2; x=width/2, y=height/2; window_handle = XCreateWindow( Xdisplay, Xroot, x, y, width, height, 0, visual->depth, InputOutput, visual->visual, attr_mask, &attr); if( !window_handle ) { fatalError("Couldn't create the window\n"); }#if USE_GLX_CREATE_WINDOW int glXattr[] = { None }; glX_window_handle = glXCreateWindow(Xdisplay, fbconfig, window_handle, glXattr); if( !glX_window_handle ) { fatalError("Couldn't create the GLX window\n"); }#else glX_window_handle = window_handle;#endif textprop.value = (unsigned char*)title; textprop.encoding = XA_STRING; textprop.format = 8; textprop.nitems = strlen(title); hints.x = x; hints.y = y; hints.width = width; hints.height = height; hints.flags = USPosition|USSize; startup_state = XAllocWMHints(); startup_state->initial_state = NormalState; startup_state->flags = StateHint; XSetWMProperties(Xdisplay, window_handle,&textprop, &textprop, NULL, 0, &hints, startup_state, NULL); XFree(startup_state); XMapWindow(Xdisplay, window_handle); XIfEvent(Xdisplay, &event, WaitForMapNotify, (char*)&window_handle); if ((del_atom = XInternAtom(Xdisplay, "WM_DELETE_WINDOW", 0)) != None) { XSetWMProtocols(Xdisplay, window_handle, &del_atom, 1); }}static int ctxErrorHandler( Display *dpy, XErrorEvent *ev ){ fputs("Error at context creation", stderr); return 0;}static void createTheRenderContext(){ int dummy; if (!glXQueryExtension(Xdisplay, &dummy, &dummy)) { fatalError("OpenGL not supported by X server\n"); }#if USE_GLX_CREATE_CONTEXT_ATTRIB #define GLX_CONTEXT_MAJOR_VERSION_ARB 0x2091 #define GLX_CONTEXT_MINOR_VERSION_ARB 0x2092 render_context = NULL; if( isExtensionSupported( glXQueryExtensionsString(Xdisplay, DefaultScreen(Xdisplay)), "GLX_ARB_create_context" ) ) { typedef GLXContext (*glXCreateContextAttribsARBProc)(Display*, GLXFBConfig, GLXContext, Bool, const int*); glXCreateContextAttribsARBProc glXCreateContextAttribsARB = (glXCreateContextAttribsARBProc)glXGetProcAddressARB( (const GLubyte *) "glXCreateContextAttribsARB" ); if( glXCreateContextAttribsARB ) { int context_attribs[] = { GLX_CONTEXT_MAJOR_VERSION_ARB, 3, GLX_CONTEXT_MINOR_VERSION_ARB, 0, //GLX_CONTEXT_FLAGS_ARB , GLX_CONTEXT_FORWARD_COMPATIBLE_BIT_ARB, None }; int (*oldHandler)(Display*, XErrorEvent*) = XSetErrorHandler(&ctxErrorHandler); render_context = glXCreateContextAttribsARB( Xdisplay, fbconfig, 0, True, context_attribs ); XSync( Xdisplay, False ); XSetErrorHandler( oldHandler ); fputs("glXCreateContextAttribsARB failed", stderr); } else { fputs("glXCreateContextAttribsARB could not be retrieved", stderr); } } else { fputs("glXCreateContextAttribsARB not supported", stderr); } if(!render_context) {#else {#endif render_context = glXCreateNewContext(Xdisplay, fbconfig, GLX_RGBA_TYPE, 0, True); if (!render_context) { fatalError("Failed to create a GL context\n"); } } if (!glXMakeContextCurrent(Xdisplay, glX_window_handle, glX_window_handle, render_context)) { fatalError("glXMakeCurrent failed for window\n"); }}static int updateTheMessageQueue(){ XEvent event; XConfigureEvent *xc; while (XPending(Xdisplay)) { XNextEvent(Xdisplay, &event); switch (event.type) { case ClientMessage: if (event.xclient.data.l[0] == del_atom) { return 0; } break; case ConfigureNotify: xc = &(event.xconfigure); width = xc->width; height = xc->height; break; } } return 1;}/* 6----7 /| /| 3----2 | | 5--|-4 |/ |/ 0----1*/GLfloat cube_vertices[][8] = { /* X Y Z Nx Ny Nz S T */ {-1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0}, // 0 { 1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0}, // 1 { 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0}, // 2 {-1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0}, // 3 { 1.0, -1.0, -1.0, 0.0, 0.0, -1.0, 0.0, 0.0}, // 4 {-1.0, -1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 0.0}, // 5 {-1.0, 1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 1.0}, // 6 { 1.0, 1.0, -1.0, 0.0, 0.0, -1.0, 0.0, 1.0}, // 7 {-1.0, -1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0}, // 5 {-1.0, -1.0, 1.0, -1.0, 0.0, 0.0, 1.0, 0.0}, // 0 {-1.0, 1.0, 1.0, -1.0, 0.0, 0.0, 1.0, 1.0}, // 3 {-1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 1.0}, // 6 { 1.0, -1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0}, // 1 { 1.0, -1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 0.0}, // 4 { 1.0, 1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 1.0}, // 7 { 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0}, // 2 {-1.0, -1.0, -1.0, 0.0, -1.0, 0.0, 0.0, 0.0}, // 5 { 1.0, -1.0, -1.0, 0.0, -1.0, 0.0, 1.0, 0.0}, // 4 { 1.0, -1.0, 1.0, 0.0, -1.0, 0.0, 1.0, 1.0}, // 1 {-1.0, -1.0, 1.0, 0.0, -1.0, 0.0, 0.0, 1.0}, // 0 {-1.0, 1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0}, // 3 { 1.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0}, // 2 { 1.0, 1.0, -1.0, 0.0, 1.0, 0.0, 1.0, 1.0}, // 7 {-1.0, 1.0, -1.0, 0.0, 1.0, 0.0, 0.0, 1.0}, // 6};static void draw_cube(void){ glEnableClientState(GL_VERTEX_ARRAY); glEnableClientState(GL_NORMAL_ARRAY); glEnableClientState(GL_TEXTURE_COORD_ARRAY); glVertexPointer(3, GL_FLOAT, sizeof(GLfloat) * 8, &cube_vertices[0][0]); glNormalPointer(GL_FLOAT, sizeof(GLfloat) * 8, &cube_vertices[0][3]); glTexCoordPointer(2, GL_FLOAT, sizeof(GLfloat) * 8, &cube_vertices[0][6]); glDrawArrays(GL_QUADS, 0, 24);}float const light0_dir[]={0,1,0,0};float const light0_color[]={78./255., 80./255., 184./255.,1};float const light1_dir[]={-1,1,1,0};float const light1_color[]={255./255., 220./255., 97./255.,1};float const light2_dir[]={0,-1,0,0};float const light2_color[]={31./255., 75./255., 16./255.,1};static void redrawTheWindow(){ float const aspect = (float)width / (float)height; static float a=0; static float b=0; static float c=0; glDrawBuffer(GL_BACK); glViewport(0, 0, width, height); // Clear with alpha = 0.0, i.e. full transparency glClearColor(0.0, 0.0, 0.0, 0.0); glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT); glMatrixMode(GL_PROJECTION); glLoadIdentity(); glFrustum(-aspect, aspect, -1, 1, 2.5, 10); glMatrixMode(GL_MODELVIEW); glLoadIdentity(); glEnable(GL_DEPTH_TEST); glEnable(GL_CULL_FACE); glEnable(GL_BLEND); glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); glLightfv(GL_LIGHT0, GL_POSITION, light0_dir); glLightfv(GL_LIGHT0, GL_DIFFUSE, light0_color); glLightfv(GL_LIGHT1, GL_POSITION, light1_dir); glLightfv(GL_LIGHT1, GL_DIFFUSE, light1_color); glLightfv(GL_LIGHT2, GL_POSITION, light2_dir); glLightfv(GL_LIGHT2, GL_DIFFUSE, light2_color); glTranslatef(0., 0., -5.); glRotatef(a, 1, 0, 0); glRotatef(b, 0, 1, 0); glRotatef(c, 0, 0, 1); glEnable(GL_LIGHT0); glEnable(GL_LIGHT1); glEnable(GL_LIGHTING); glEnable(GL_COLOR_MATERIAL); glColorMaterial(GL_FRONT_AND_BACK, GL_AMBIENT_AND_DIFFUSE); glColor4f(1., 1., 1., 0.5); glCullFace(GL_FRONT); draw_cube(); glCullFace(GL_BACK); draw_cube(); a = fmod(a+0.1, 360.); b = fmod(b+0.5, 360.); c = fmod(c+0.25, 360.); glXSwapBuffers(Xdisplay, glX_window_handle);}int main(int argc, char *argv[]){ createTheWindow(); createTheRenderContext(); while (updateTheMessageQueue()) { redrawTheWindow(); } return 0;}The main trick is getting the right FBConfig. You need to ask for a alpha channel and test the associated XRenderPictFormat for the presence of an alpha mask.
I know this is possible with Windows 7, not sure about earlier versions.
To get rid of the window border you need to remove the WS_OVERLAPPEDWINDOW style from the window and add the WS_POPUP style:
DWORD style = ::GetWindowLong(hWnd, GWL_STYLE);style &= ~WS_OVERLAPPEDWINDOW;style |= WS_POPUP;::SetWindowLong(hWnd, GWL_STYLE, style);To make the background of the OpenGL window transparent, you will need to use the DwmEnableBlurBehindWindow function:
DWM_BLURBEHIND bb = {0};bb.dwFlags = DWM_BB_ENABLE;bb.fEnable = true;bb.hRgnBlur = NULL;DwmEnableBlurBehindWindow(hWnd, &bb);You will also need to specify 0 for the alpha value when calling glClearColor.
glClearColor(0.0f,0.0f,0.0f,0.0f);Also, when creating your OpenGL context, make sure you allocate an alpha channel.
Now your background should be fully transparent. If you keep the window decorations, then the background will use the Aero blur look and you can adjust the level of transparency using the alpha value in glClearColor.