Videos of other participants are not always displayed

I started your agora project with deepar but ran into a problem. That is, when I connect two people to streaming, the video of the other participant is not always displayed. Let’s say I launch a stream of two people three times and everything works, I launch it for the fourth time and one of them stops displaying the video of another participant.
How to solve this problem?
The example (GitHub - DeepARSDK/videocall-ios-swift: Video call using Agora.io SDK with face masks provided by DeepAR SDK) has the same behavior.
DeepAR SDK version 4.0.3
Agora version 4.1.0

import UIKit
import DeepAR
import AgoraRtcKit

class ViewController: UIViewController {

    @IBOutlet weak var localVideo: UIView!
    @IBOutlet weak var remoteVideo: UIView!
    @IBOutlet weak var hangUpButton: UIButton!
    @IBOutlet weak var micButton: UIButton!
    @IBOutlet weak var cameraButton: UIButton!
    
    // Defines localView
//    var localView: UIView!

    var agoraKit: AgoraRtcEngineKit!
    var userRole: AgoraClientRole = .broadcaster
    
    private var deepAr: DeepAR!
    private var arView: ARView!
    private var cameraController: CameraController!
    private var maskPaths = [Bundle.main.path(forResource: "mask-1.deepar", ofType: nil),
                             Bundle.main.path(forResource: "mask-2.deepar", ofType: nil),
                             Bundle.main.path(forResource: "mask-3.deepar", ofType: nil)]

    let AppID: String = "dff2d9630bb0426386af15e5796e0eb0"
    let token = "007eJxTYHDZdTGn5sjlrRG7LzPeyuMNVbm2bU2Q9oxHJy5dqgtZJnhDgSElLc0oxdLM2CApycDEyMzYwiwxzdA01dTc0izVIDXJwP/y7uSGQEaG9oWRjIwMEAjiszOUZaak5ju6MDAAAG+CIsQ="
    var channelID = "videoAD"
    var deepArID = "c7bd70481e36df05b8af9eb3cee77785a7d5248ad56fc35603bc786a0576edc8c248ab8e98351543"

    override func viewDidLoad() {
        super.viewDidLoad()
        initializeAgoraEngine()
        setupVideo()
        setupDeepAR()
        setupARCamera()
        joinChannel()
//        setupLocalVideo()
        switchMode()
    }

    @IBAction func didClickHangUpButton(_ sender: UIButton) {
        sender.isSelected.toggle()
        micButton.isHidden.toggle()
        cameraButton.isHidden.toggle()
        if sender.isSelected {
            agoraKit.stopPreview()
            agoraKit.leaveChannel(nil)
            UIApplication.shared.isIdleTimerDisabled = false
        } else {
            initializeAgoraEngine()
            setupVideo()
//            setupLocalVideo()
            setupDeepAR()
            setupARCamera()
            joinChannel()
            switchMode()
        }
    }

    @IBAction func didClickMuteButton(_ sender: UIButton) {
        sender.isSelected.toggle()
        agoraKit.muteLocalAudioStream(sender.isSelected)
    }

    @IBAction func didClickSwitchCameraButton(_ sender: UIButton) {
        sender.isSelected.toggle()
        agoraKit.switchCamera()
    }

    func initializeAgoraEngine() {
        agoraKit = AgoraRtcEngineKit.sharedEngine(withAppId: AppID, delegate: self)
    }
    
    func setupDeepAR() {
        deepAr = DeepAR()
        deepAr.setLicenseKey(deepArID)
        deepAr.delegate = self
    }

    func setupVideo() {
        agoraKit.enableVideo()
        agoraKit.disableAudio()
        agoraKit.setExternalVideoSource(true, useTexture: true, sourceType: .videoFrame)
        agoraKit.setVideoEncoderConfiguration(
            AgoraVideoEncoderConfiguration(size: AgoraVideoDimension640x360,
                                           frameRate: .fps15,
                                           bitrate: AgoraVideoBitrateStandard,
                                           orientationMode: .adaptative,
                                           mirrorMode: .auto))
        
        agoraKit.setClientRole(.broadcaster)
        agoraKit.setChannelProfile(.liveBroadcasting)
    }
    
    private func setupARCamera() {
        arView = deepAr.createARView(withFrame: localVideo.frame) as! ARView
        arView.translatesAutoresizingMaskIntoConstraints = false
        localVideo.addSubview(arView)
        arView.leftAnchor.constraint(equalTo: localVideo.leftAnchor, constant: 0).isActive = true
        arView.rightAnchor.constraint(equalTo: localVideo.rightAnchor, constant: 0).isActive = true
        arView.topAnchor.constraint(equalTo: localVideo.topAnchor, constant: 0).isActive = true
        arView.bottomAnchor.constraint(equalTo: localVideo.bottomAnchor, constant: 0).isActive = true

        localVideo.isHidden = true

        cameraController = CameraController()
        cameraController.deepAR = deepAr
        cameraController.startCamera()
    }


    func joinChannel() {
        
        let option = AgoraRtcChannelMediaOptions()
        option.clientRoleType = .broadcaster
        option.channelProfile = .communication
        
        
        agoraKit.joinChannel(byToken: token, channelId: channelID, uid: 0, mediaOptions: option) { (channel, uid, elapsed) in
            UIApplication.shared.isIdleTimerDisabled = true
            self.deepAr.startCapture(withOutputWidth: 360, outputHeight: 640, subframe: CGRect(x: 0.0, y: 0.0, width: 1.0, height: 1.0))
            
            let videoCanvas = AgoraRtcVideoCanvas()
            videoCanvas.uid = 0
            videoCanvas.renderMode = .hidden
            videoCanvas.view = self.localVideo
            // Set the local video view
            self.agoraKit.setupLocalVideo(videoCanvas)
            
            self.localVideo.isHidden = false
            self.remoteVideo.isHidden = false
        }
        
    }
    
    func switchMode() {
        deepAr.switchEffect(withSlot: "effect", path: maskPaths[0])
    }
}

extension ViewController: AgoraRtcEngineDelegate {
    
    func rtcEngine(_ engine: AgoraRtcEngineKit, didJoinedOfUid uid: UInt, elapsed: Int) {
        let videoCanvas = AgoraRtcVideoCanvas()
        videoCanvas.uid = uid
        videoCanvas.renderMode = .hidden
        videoCanvas.view = remoteVideo
        agoraKit.setupRemoteVideo(videoCanvas)
    }
    

    func rtcEngine(_ engine: AgoraRtcEngineKit, didOfflineOfUid uid:UInt, reason:AgoraUserOfflineReason) {
        remoteVideo.isHidden = true
    }

    func rtcEngine(_ engine: AgoraRtcEngineKit, didVideoMuted muted:Bool, byUid:UInt) {
        remoteVideo.isHidden = muted
    }

    func rtcEngine(_ engine: AgoraRtcEngineKit, didOccurWarning warningCode: AgoraWarningCode) {
        print("did occur ***WARNING***, code: \(warningCode.rawValue)")
    }

    func rtcEngine(_ engine: AgoraRtcEngineKit, didOccurError errorCode: AgoraErrorCode) {
        print("did occur ***ERROR***, code: \(errorCode.rawValue)")
    }
}

extension ViewController: DeepARDelegate {
    func didFinishPreparingForVideoRecording() {}
    
    func didStartVideoRecording() {}
    
    func frameAvailable(_ sampleBuffer: CMSampleBuffer!) {
        
        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
            print("*** NO BUFFER ERROR")
            return
        }

        let time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)

        let videoFrame = AgoraVideoFrame()
        videoFrame.format = 12
        videoFrame.time = time
        videoFrame.textureBuf = pixelBuffer
        videoFrame.rotation = 0
        
        agoraKit?.pushExternalVideoFrame(videoFrame)
    }
    
    func didFinishVideoRecording(_ videoFilePath: String!) {}
    
    func recordingFailedWithError(_ error: Error!) {}
    
    func didTakeScreenshot(_ screenshot: UIImage!) {}
    
    func didInitialize() {}
    
    func faceVisiblityDidChange(_ faceVisible: Bool) {
        print("faceVisible: \(faceVisible)")
    }
}

Hi, are you maybe getting any console logs when this issue occurs?

Whether I get this problem or not, I get a multithreading related warning. If you remove the deepar and leave only the agora, then this problem does not arise.

2023-01-12 18:55:44.548713+0300 videoAD[958:94677] Metal API Validation Enabled
Thread Performance Checker: Thread running at QOS_CLASS_USER_INTERACTIVE waiting on a lower QoS thread running at QOS_CLASS_DEFAULT. Investigate ways to avoid priority inversions
PID: 958, TID: 94308
Backtrace
=================================================================
3   DeepAR                              0x000000010329bcb4 _ZN2bx9Semaphore4waitEi + 64
4   DeepAR                              0x000000010329ef48 _ZN2bx6Thread4initEPFiPS0_PvES2_jPKc + 124
5   DeepAR                              0x00000001030d1224 _ZN2ar10Controller10initializeENS_20DeepARInitParametersE + 448
6   DeepAR                              0x00000001030de5bc _ZN2ar10DeepARImpl10initializeENS_20DeepARInitParametersE + 116
7   DeepAR                              0x0000000103094744 -[DeepAR initializeWithWidth:height:window:] + 456
8   DeepAR                              0x00000001030900f8 -[ARView initialize] + 568
9   DeepAR                              0x0000000103094a50 -[DeepAR createARViewWithFrame:] + 92
10  videoAD                             0x00000001022d4e08 $s7videoAD14ViewControllerC13setupARCamera33_3435F638B8686A15CC1F87CED0121BC0LLyyF + 492
11  videoAD                             0x00000001022d3be8 $s7videoAD14ViewControllerC11viewDidLoadyyF + 188
12  videoAD                             0x00000001022d3c74 $s7videoAD14ViewControllerC11viewDidLoadyyFTo + 36
13  UIKitCore                           0x00000001ab9499b4 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 3504564
14  UIKitCore                           0x00000001ab60aef4 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 102132
15  UIKitCore                           0x00000001ab60ac20 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 101408
16  UIKitCore                           0x00000001ab7dfff0 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 2023408
17  UIKitCore                           0x00000001ab7dfef8 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 2023160
18  UIKitCore                           0x00000001ab7df7e0 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 2021344
19  UIKitCore                           0x00000001ab923920 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 3348768
20  UIKitCore                           0x00000001ac3b1db0 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 14417328
21  UIKitCore                           0x00000001ab7585cc AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 1467852
22  UIKitCore                           0x00000001ab7e7194 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 2052500
23  UIKitCore                           0x00000001ab7e6c98 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 2051224
24  UIKitCore                           0x00000001ab7e6abc AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 2050748
25  FrontBoardServices                  0x00000001bdc5f5d4 BDFDA47E-9B3E-3944-AAFF-A8D9F8410A3E + 9684
26  FrontBoardServices                  0x00000001bdc98d50 BDFDA47E-9B3E-3944-AAFF-A8D9F8410A3E + 245072
27  FrontBoardServices                  0x00000001bdc62f70 BDFDA47E-9B3E-3944-AAFF-A8D9F8410A3E + 24432
28  FrontBoardServices                  0x00000001bdc989d8 BDFDA47E-9B3E-3944-AAFF-A8D9F8410A3E + 244184
29  libdispatch.dylib                   0x000000010416d8c4 _dispatch_client_callout + 16
30  libdispatch.dylib                   0x0000000104170dac _dispatch_block_invoke_direct + 228
31  FrontBoardServices                  0x00000001bdc6c474 BDFDA47E-9B3E-3944-AAFF-A8D9F8410A3E + 62580
32  FrontBoardServices                  0x00000001bdc6c0b0 BDFDA47E-9B3E-3944-AAFF-A8D9F8410A3E + 61616
33  FrontBoardServices                  0x00000001bdc6e574 BDFDA47E-9B3E-3944-AAFF-A8D9F8410A3E + 71028
34  CoreFoundation                      0x00000001a474e2b8 A68FCE15-6C1A-38C6-9ED3-7063272D237E + 836280
35  CoreFoundation                      0x00000001a4759c18 A68FCE15-6C1A-38C6-9ED3-7063272D237E + 883736
36  CoreFoundation                      0x00000001a46e4000 A68FCE15-6C1A-38C6-9ED3-7063272D237E + 401408
37  CoreFoundation                      0x00000001a46f8fb0 A68FCE15-6C1A-38C6-9ED3-7063272D237E + 487344
38  CoreFoundation                      0x00000001a46fdb7c CFRunLoopRunSpecific + 584
39  GraphicsServices                    0x00000001dea85984 GSEventRunModal + 160
40  UIKitCore                           0x00000001ab9673c8 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 3625928
41  UIKitCore                           0x00000001ab967040 UIApplicationMain + 312
42  libswiftUIKit.dylib                 0x00000001b1847240 $s5UIKit17UIApplicationMainys5Int32VAD_SpySpys4Int8VGGSgSSSgAJtF + 100
43  videoAD                             0x00000001022d980c $sSo21UIApplicationDelegateP5UIKitE4mainyyFZ + 104
44  videoAD                             0x00000001022d9794 $s7videoAD11AppDelegateC5$mainyyFZ + 44
45  videoAD                             0x00000001022d9890 main + 28
46  dyld                                0x00000001c6114df0 A2AA3A68-07F7-3D52-A49E-D29D136813F3 + 81392
Thread Performance Checker: Thread running at QOS_CLASS_USER_INTERACTIVE waiting on a lower QoS thread running at QOS_CLASS_DEFAULT. Investigate ways to avoid priority inversions
PID: 958, TID: 94308
Backtrace
=================================================================
3   DeepAR                              0x000000010329bcb4 _ZN2bx9Semaphore4waitEi + 64
4   DeepAR                              0x00000001033f8d14 _ZN4bgfx7Context11renderFrameEi + 292
5   DeepAR                              0x00000001033f8adc _ZN4bgfx11renderFrameEi + 52
6   DeepAR                              0x00000001030d17ac _ZThn8_N2ar10Controller6renderEv + 24
7   QuartzCore                          0x00000001aab334b0 564FD4F3-2F8C-351F-904F-0E562D7DFC8C + 160944
8   QuartzCore                          0x00000001aab444c0 564FD4F3-2F8C-351F-904F-0E562D7DFC8C + 230592
9   CoreFoundation                      0x00000001a46f8c60 A68FCE15-6C1A-38C6-9ED3-7063272D237E + 486496
10  CoreFoundation                      0x00000001a4713f24 A68FCE15-6C1A-38C6-9ED3-7063272D237E + 597796
11  CoreFoundation                      0x00000001a4715a50 A68FCE15-6C1A-38C6-9ED3-7063272D237E + 604752
12  CoreFoundation                      0x00000001a46f95b8 A68FCE15-6C1A-38C6-9ED3-7063272D237E + 488888
13  CoreFoundation                      0x00000001a46fdb7c CFRunLoopRunSpecific + 584
14  GraphicsServices                    0x00000001dea85984 GSEventRunModal + 160
15  UIKitCore                           0x00000001ab9673c8 AA0A89BD-E48F-31DB-AD5C-5C724CE77D12 + 3625928
16  UIKitCore                           0x00000001ab967040 UIApplicationMain + 312
17  libswiftUIKit.dylib                 0x00000001b1847240 $s5UIKit17UIApplicationMainys5Int32VAD_SpySpys4Int8VGGSgSSSgAJtF + 100
18  videoAD                             0x00000001022d980c $sSo21UIApplicationDelegateP5UIKitE4mainyyFZ + 104
19  videoAD                             0x00000001022d9794 $s7videoAD11AppDelegateC5$mainyyFZ + 44
20  videoAD                             0x00000001022d9890 main + 28
21  dyld                                0x00000001c6114df0 A2AA3A68-07F7-3D52-A49E-D29D136813F3 + 81392
2023-01-12 18:55:46.229951+0300 videoAD[958:94308] Incorrect NSStringEncoding value 0x8000100 detected. Assuming NSASCIIStringEncoding. Will stop this compatibility mapping behavior in the near future.
2023-01-12 18:55:46.231595+0300 videoAD[958:94677] Incorrect NSStringEncoding value 0x8000100 detected. Assuming NSASCIIStringEncoding. Will stop this compatibility mapping behavior in the near future.
==> Face Tracking mode: XMG_NON_RIGID_FACE_REG2D_3D is activated
2023-01-12 18:55:46.720242+0300 videoAD[958:94694] NSBundle </private/var/containers/Bundle/Application/9C6B8981-2754-4B0C-8865-EE40F4D57A6C/videoAD.app> (loaded)
2023-01-12 18:55:46.720871+0300 videoAD[958:94694] /
2023-01-12 18:55:46.721022+0300 videoAD[958:94694] NSBundle </private/var/containers/Bundle/Application/9C6B8981-2754-4B0C-8865-EE40F4D57A6C/videoAD.app> (loaded)
2023-01-12 18:55:46.721113+0300 videoAD[958:94694] /
2023-01-12 18:55:48.303990+0300 videoAD[958:94308] Incorrect NSStringEncoding value 0x8000100 detected. Assuming NSASCIIStringEncoding. Will stop this compatibility mapping behavior in the near future.
2023-01-12 18:55:48.306653+0300 videoAD[958:94677] Incorrect NSStringEncoding value 0x8000100 detected. Assuming NSASCIIStringEncoding. Will stop this compatibility mapping behavior in the near future.
2023-01-12 18:55:48.312227+0300 videoAD[958:94644] [assertion] Error acquiring assertion: <Error Domain=RBSServiceErrorDomain Code=1 "target is not running or doesn't have entitlement com.apple.runningboard.assertions.webkit" UserInfo={NSLocalizedFailureReason=target is not running or doesn't have entitlement com.apple.runningboard.assertions.webkit}>

these lines can be removed.

Looks like you need to add this entitlement to your project. Could you try that?

com.apple.runningboard.assertions.webkit is the key or value? How should it be spelled out in the Entitlements file? Have you launched your “videocall-ios-swift” project? Is there such a bug there?

I don’t recall seeing this log when testing our example, we don’t use an Entitlements file, if you see this log in our sample let us know.

The black input stream should not be due to DeepAR. Remote view comes over the network. Can you try skipping DeepAR and just sending raw stream and see if it works.

Your example has not been updated since 2019. Some methods on both the deepAr and agora sides have been updated. Could you see the relevance of your example? Since if I disconnect from the deepar project everything works correctly.

We will check the example, thanks for reporting.

1 Like

After check, please tell us the result.

Have you checked the example?

It’s in the pipeline, I will let you know when I have updates.