Testing mat-dialogs can be tricky. I tend to use a spy object for the return from a dialog open (dialogRefSpyObj below) so I can more easily track and control tests. In your case it might look something like the following:

describe('ModalService', () => {    let modalService: ModalService;    let dialogSpy: jasmine.Spy;    let dialogRefSpyObj = jasmine.createSpyObj({ afterClosed : of({}), close: null });    dialogRefSpyObj.componentInstance = { body: '' }; // attach componentInstance to the spy object...    beforeEach(() => {        TestBed.configureTestingModule({            imports: [MatDialogModule],            providers: [ModalService]        });        modalService = TestBed.get(ModalService);    });    beforeEach(() => {        dialogSpy = spyOn(TestBed.get(MatDialog), 'open').and.returnValue(dialogRefSpyObj);    });    it('open modal ', () => {        modalService.open(TestComponent, '300px');        expect(dialogSpy).toHaveBeenCalled();        // You can also do things with this like:        expect(dialogSpy).toHaveBeenCalledWith(TestComponent, { maxWidth: '100vw' });        // and ...        expect(dialogRefSpyObj.afterClosed).toHaveBeenCalled();    });});

I have a better solution that still works on 2019

header.component.ts

import { BeforeLogOutComponent } from '@app-global/components/before-log-out/before-log-out.component';  /**   * The method triggers before the logout.   * Opens the dialog and warns the user before log Out.   */  public beforeLogOut(): void {    this._dialog.open(BeforeLogOutComponent, { width: '400px', disableClose: false, panelClass: 'dialog_before_log_out' })    .afterClosed()    .subscribe((res) => {      if (res && res.action === true) { this.loggedOut(); }    }, err => {      console.error(err);    });  }

header.component.spec.ts

import { async, ComponentFixture, TestBed } from '@angular/core/testing';import { MatDialog } from '@angular/material';import { Observable, of } from 'rxjs';<<-- Create a MatDialog mock class -->>export class MatDialogMock {  // When the component calls this.dialog.open(...) we'll return an object  // with an afterClosed method that allows to subscribe to the dialog result observable.  open() {    return {      afterClosed: () => of({action: true})    };  }}describe('HeaderComponent', () => {  let component: HeaderComponent;  let fixture: ComponentFixture<HeaderComponent>;  beforeEach(async(() => {    TestBed.configureTestingModule({      imports: [        MaterialModule, RouterTestingModule, HttpModule, BrowserAnimationsModule,        HttpClientModule, FlexLayoutModule,      ],      declarations: [        HeaderComponent,      ],      providers: [        { provide: MatDialog, useClass: MatDialogMock } <<-- look this      ]    })    .compileComponents();  }));  beforeEach(async() => {    fixture = TestBed.createComponent(HeaderComponent);    component = fixture.componentInstance;    component.ngOnInit();    component.ngAfterViewInit();    await fixture.whenStable();    fixture.detectChanges();  });  fit('should create', () => {    expect(component).toBeTruthy();  });  // I test the dialog here.  fit('should open the dialog', () => {    component.beforeLogOut();  });}

I do not have the exact answer for your case but I also did some tests on MatDialog. I can show you what I did. Maybe look at the inject() part:

(I deleted some things for clarity and confidentiality)

describe('MyDialogComponent', () => {  let dialog: MatDialog;  let overlayContainer: OverlayContainer;  let component: MyDialogComponent;  let fixture: ComponentFixture<MyDialogComponent>;  const mockDialogRef = {    close: jasmine.createSpy('close')  };  beforeEach(async(() => {    TestBed.configureTestingModule({      imports: [        BrowserAnimationsModule,        ReactiveFormsModule,        AngularMaterialModule,      ],      providers: [        { provide: MatDialogRef, useValue: mockDialogRef },        {          provide: MAT_DIALOG_DATA,          useValue: {            title: 'myTitle',          }        }      ],      declarations: [MyDialogComponent],    });    TestBed.overrideModule(BrowserDynamicTestingModule, {      set: {        entryComponents: [MyDialogComponent]      }    });    TestBed.compileComponents();  }));  beforeEach(inject([MatDialog, OverlayContainer],    (d: MatDialog, oc: OverlayContainer) => {      dialog = d;      overlayContainer = oc;    })  );  afterEach(() => {    overlayContainer.ngOnDestroy();  });  beforeEach(() => {    fixture = TestBed.createComponent(MyDialogComponent);    component = fixture.componentInstance;    fixture.detectChanges();  });  it('should create', () => {    expect(component).toBeTruthy();  });  it('onCancel should close the dialog', () => {    component.onCancel();    expect(mockDialogRef.close).toHaveBeenCalled();  });});