fix: multi push/render bug fixed

tamworth · tamworth · commit a2c717e8a668 · 2024-10-24T17:11:47.000+08:00
diff --git a/iOS/APIExample/APIExample/Common/ExternalVideo/AgoraSampleBufferRender.m b/iOS/APIExample/APIExample/Common/ExternalVideo/AgoraSampleBufferRender.m
@@ -64,14 +64,13 @@ - (void)layoutDisplayLayer {
     CGSize videoSize;
     if (videoRatio >= viewRatio) {
         videoSize.height = viewHeight;
-        videoSize.width = videoSize.height * videoRatio;
+        videoSize.width = viewHeight * videoRatio;
     }else {
         videoSize.width = viewWidth;
-        videoSize.height = videoSize.width / videoRatio;
+        videoSize.height = viewWidth / videoRatio;
     }
     
-    CGRect renderRect = CGRectMake(0.5 * (viewWidth - videoSize.width), 0.5 * (viewHeight - videoSize.height), videoSize.width, videoSize.height);
-    
+    CGRect renderRect = CGRectMake(0, 0, viewWidth, viewHeight);
     if (!CGRectEqualToRect(renderRect, self.displayLayer.frame)) {
         self.displayLayer.frame = renderRect;
     }
diff --git a/iOS/APIExample/APIExample/Common/Utils/KFMP4Demuxer.m b/iOS/APIExample/APIExample/Common/Utils/KFMP4Demuxer.m
@@ -52,6 +52,7 @@ @interface KFMP4Demuxer () {
 @property (nonatomic, assign, readwrite) BOOL audioEOF; // Indicates whether audio has ended.
 @property (nonatomic, assign, readwrite) BOOL videoEOF; // Indicates whether video has ended.
 @property (nonatomic, assign, readwrite) CGAffineTransform preferredTransform; // Transformation information for the image, e.g., video image rotation.
+@property (nonatomic, assign) CGFloat frameRate; // Frame rate of the video.
 @end
 
 @implementation KFMP4Demuxer
@@ -233,6 +234,9 @@ - (void)fetchAndSaveDemuxedData {
                 [self saveSampleBuffer:videoBuffer];
                 CFRelease(videoBuffer);
             }
+            
+            NSTimeInterval deley = 1000 / self.frameRate;
+            usleep(deley * 1000);
         }
         if (self.demuxerStatus == KFMP4DemuxerStatusCompleted) {
             NSLog(@"KFMP4Demuxer complete");
@@ -391,6 +395,9 @@ - (void)_setupDemuxReader:(NSError**)error {
         AVAssetTrack *videoTrack = [[self.config.asset tracksWithMediaType:AVMediaTypeVideo] firstObject];
         _hasVideoTrack = videoTrack ? YES : NO;
         if (_hasVideoTrack) {
+            // Get frame rate
+            self.frameRate = videoTrack.nominalFrameRate > 0 ? videoTrack.nominalFrameRate : 30;
+            
             // Get the image transformation information.
             _preferredTransform = videoTrack.preferredTransform;
             
diff --git a/iOS/APIExample/APIExample/Examples/Advanced/CustomVideoSourcePushMulti/CustomVideoSourcePushMulti.swift b/iOS/APIExample/APIExample/Examples/Advanced/CustomVideoSourcePushMulti/CustomVideoSourcePushMulti.swift
@@ -14,10 +14,27 @@ class UserModel {
     var uid: UInt = 0
     var canvasView: SampleBufferDisplayView?
     var trackId: UInt32 = 0
-    var isJoin: Bool = false
-    var customSource: AgoraYUVImageSourcePush?
+    var isJoin: Bool = false {
+        didSet {
+            self.canvasView?.videoView.isHidden = !isJoin
+            self.canvasView?.videoView.reset()
+        }
+    }
     var isEncode: Bool = false
+    var customSource: AgoraYUVImageSourcePush?
     var customEncodeSource: KFMP4Demuxer?
+    
+    func reset() {
+        self.customSource?.stopSource()
+        self.customSource = nil
+        self.customEncodeSource?.cancelReading()
+        self.customEncodeSource = nil
+        
+        self.uid = UInt(Int.random(in: 10000...99999))
+        self.trackId = 0
+        self.isJoin = false
+        self.isEncode = false
+    }
 }
 
 class CustomVideoSourcePushMultiEntry: UIViewController {
@@ -46,16 +63,14 @@ class CustomVideoSourcePushMultiEntry: UIViewController {
 }
 
 class CustomVideoSourcePushMultiMain: BaseViewController {
-    var localVideo = Bundle.loadView(fromNib: "VideoViewSampleBufferDisplayView",
-                                     withType: SampleBufferDisplayView.self)
-    lazy var remoteVideos: [UserModel] = (0..<3).map({ _ in
+    private let localUid: UInt = UInt(Int.random(in: 10000...99999))
+    lazy var remoteVideos: [UserModel] = (0..<4).map({ _ in
         let model = UserModel()
-        model.uid = UInt(Int.random(in: 10000...99999))
         model.canvasView = Bundle.loadView(fromNib: "VideoViewSampleBufferDisplayView",
                                            withType: SampleBufferDisplayView.self)
+        model.reset()
         return model
     })
-    var customCamera: AgoraYUVImageSourcePush?
     
     @IBOutlet weak var container: AGEVideoContainer!
     var agoraKit: AgoraRtcEngineKit!
@@ -77,26 +92,14 @@ class CustomVideoSourcePushMultiMain: BaseViewController {
         Util.configPrivatization(agoraKit: agoraKit)
         agoraKit.setLogFile(LogUtils.sdkLogPath())
         
-        container.layoutStream2x2(views: [localVideo] + remoteVideos.compactMap({ $0.canvasView }))
+        container.layoutStream2x2(views: remoteVideos.compactMap({ $0.canvasView }))
         // make myself a broadcaster
         // agoraKit.setClientRole(.broadcaster)
         
         // enable video module and set up video encoding configs
         agoraKit.enableVideo()
         agoraKit.enableAudio()
         
-        // setup my own camera as custom video source
-        // note setupLocalVideo is not working when using pushExternalVideoFrame
-        // so you will have to prepare the preview yourself
-        customCamera = AgoraYUVImageSourcePush(size: CGSize(width: 320, height: 180),
-                                               fileName: "sample",
-                                               frameRate: 15,
-                                               isHDR: false)
-        customCamera?.trackId = agoraKit.createCustomVideoTrack()
-        customCamera?.delegate = self
-        customCamera?.startSource()
-        agoraKit.setExternalVideoSource(true, useTexture: true, sourceType: .videoFrame)
-        
         let resolution = (GlobalSettings.shared.getSetting(key: "resolution")?.selectedOption().value as? CGSize) ?? .zero
         let fps = (GlobalSettings.shared.getSetting(key: "fps")?.selectedOption().value as? AgoraVideoFrameRate) ?? .fps15
         let orientation = (GlobalSettings.shared.getSetting(key: "orientation")?
@@ -116,29 +119,39 @@ class CustomVideoSourcePushMultiMain: BaseViewController {
         // 2. If app certificate is turned on at dashboard, token is needed
         // when joining channel. The channel name and uid used to calculate
         // the token has to match the ones used for channel join
-        joinChannel(uid: 999,
-                    trackId: customCamera?.trackId ?? 0,
+        joinChannel(uid: localUid,
+                    trackId: nil,
                     publishEncodedVideoTrack: false)
     }
-    private func joinChannel(uid: UInt, trackId: UInt32, publishEncodedVideoTrack: Bool) {
+    private func joinChannel(uid: UInt, trackId: UInt32?, publishEncodedVideoTrack: Bool) {
         guard let channelName = configs["channelName"] as? String else {return}
         let option = AgoraRtcChannelMediaOptions()
-        option.publishCustomVideoTrack = !publishEncodedVideoTrack
         option.publishMicrophoneTrack = false
-        option.autoSubscribeAudio = true
-        option.autoSubscribeVideo = true
-        option.customVideoTrackId = Int(trackId)
-        option.publishEncodedVideoTrack = publishEncodedVideoTrack
-        option.clientRoleType = GlobalSettings.shared.getUserRole()
-        let connection = AgoraRtcConnection()
-        connection.localUid = uid
-        connection.channelId = channelName
+        option.publishCameraTrack = false
+        
+        let delegate: AgoraRtcEngineDelegate? = self
+        if let trackId = trackId {
+            option.publishCustomVideoTrack = !publishEncodedVideoTrack
+            option.customVideoTrackId = Int(trackId)
+            option.publishEncodedVideoTrack = publishEncodedVideoTrack
+            option.clientRoleType = .broadcaster
+            option.autoSubscribeAudio = false
+            option.autoSubscribeVideo = false
+//            delegate = nil
+        } else {
+            option.clientRoleType = .audience
+            option.autoSubscribeAudio = true
+            option.autoSubscribeVideo = true
+        }
         NetworkManager.shared.generateToken(channelName: channelName, uid: uid) { token in
+            let connection = AgoraRtcConnection()
+            connection.localUid = uid
+            connection.channelId = channelName
             let result = self.agoraKit.joinChannelEx(byToken: token,
-                                                connection: connection,
-                                                delegate: self,
-                                                mediaOptions: option,
-                                                joinSuccess: nil)
+                                                     connection: connection,
+                                                     delegate: delegate,
+                                                     mediaOptions: option,
+                                                     joinSuccess: nil)
             if result != 0 {
                 // Usually happens with invalid parameters
                 // Error code description can be found at:
@@ -211,51 +224,61 @@ class CustomVideoSourcePushMultiMain: BaseViewController {
     
     @IBAction func onDestoryVideoTrack(_ sender: Any) {
         guard let channelName = configs["channelName"] as? String else {return}
-        let userModel = remoteVideos.filter({ $0.isJoin == true }).last
-        userModel?.isJoin = false
-        userModel?.customSource?.stopSource()
-        userModel?.customEncodeSource?.cancelReading()
-        userModel?.customEncodeSource = nil
-        userModel?.canvasView?.videoView.reset()
-        userModel?.customSource = nil
+        guard let userModel = remoteVideos.filter({ $0.trackId != 0 }).last else { return }
+        let trackId = UInt(userModel.trackId)
         let connection = AgoraRtcConnection()
-        connection.localUid = userModel?.uid ?? 0
+        connection.localUid = userModel.uid
         connection.channelId = channelName
-        agoraKit.destroyCustomVideoTrack(UInt(userModel?.trackId ?? 0))
-        agoraKit.destroyCustomEncodedVideoTrack(UInt(userModel?.trackId ?? 0))
-        userModel?.trackId = 0
-        userModel?.isEncode = false
-        userModel?.uid = UInt(Int.random(in: 10000...99999))
+        agoraKit.destroyCustomVideoTrack(trackId)
+        agoraKit.destroyCustomEncodedVideoTrack(trackId)
         agoraKit.leaveChannelEx(connection) { state in
-            LogUtils.log(message: "warning: \(state.description)", level: .info)
+            LogUtils.log(message: "leaveChannelEx: \(state.description)", level: .info)
         }
+        
+        cleanCanvas(uid: userModel.uid)
     }
     
     override func willMove(toParent parent: UIViewController?) {
         if parent == nil {
-            // stop capture
-            customCamera?.stopSource()
             // leave channel when exiting the view
             if isJoined, let channelName = configs["channelName"] as? String {
                 remoteVideos.forEach({
                     let connection = AgoraRtcConnection()
                     connection.localUid = $0.uid
                     connection.channelId = channelName
-                    $0.customSource?.stopSource()
                     agoraKit.leaveChannelEx(connection) { state in
-                        LogUtils.log(message: "warning: \(state.description)", level: .info)
+                        LogUtils.log(message: "leaveChannelEx: \(state.description)", level: .info)
                     }
+                    $0.reset()
                 })
                 let connection = AgoraRtcConnection()
-                connection.localUid = 999
+                connection.localUid = localUid
                 connection.channelId = channelName
                 agoraKit.leaveChannelEx(connection) { state in
-                    LogUtils.log(message: "warning: \(state.description)", level: .info)
+                    LogUtils.log(message: "leaveChannelEx: \(state.description)", level: .info)
                 }
-                AgoraRtcEngineKit.destroy()
             }
+            AgoraRtcEngineKit.destroy()
         }
     }
+    
+    private func cleanCanvas(uid: UInt) {
+        guard let channelName = configs["channelName"] as? String,
+              let userModel = remoteVideos.first(where: { $0.uid == uid }) else { return }
+        LogUtils.log(message: "cleanCanvas: \(uid)", level: .info)
+        
+        // to unlink your view from sdk, so that your view reference will be released
+        // note the video will stay at its last frame, to completely remove it
+        // you will need to remove the EAGL sublayer from your binded view
+        userModel.reset()
+        
+        let videoCanvas = AgoraRtcVideoCanvas()
+        videoCanvas.uid = uid
+        let connect = AgoraRtcConnection()
+        connect.localUid = localUid
+        connect.channelId = channelName
+        agoraKit.setupRemoteVideoEx(videoCanvas, connection: connect)
+    }
 }
 
 /// agora rtc engine delegate events
@@ -267,7 +290,7 @@ extension CustomVideoSourcePushMultiMain: AgoraRtcEngineDelegate {
     /// cn: https://docs.agora.io/cn/Voice/API%20Reference/oc/Constants/AgoraWarningCode.html
     /// @param warningCode warning code of the problem
     func rtcEngine(_ engine: AgoraRtcEngineKit, didOccurWarning warningCode: AgoraWarningCode) {
-        LogUtils.log(message: "warning: \(warningCode.description)", level: .warning)
+        LogUtils.log(message: "didOccurWarning: \(warningCode.description)", level: .warning)
     }
     
     /// callback when error occured for agora sdk, you are recommended to display the error descriptions on demand
@@ -277,8 +300,8 @@ extension CustomVideoSourcePushMultiMain: AgoraRtcEngineDelegate {
     /// cn: https://doc.shengwang.cn/api-ref/rtc/ios/error-code
     /// @param errorCode error code of the problem
     func rtcEngine(_ engine: AgoraRtcEngineKit, didOccurError errorCode: AgoraErrorCode) {
-        LogUtils.log(message: "error: \(errorCode)", level: .error)
-        self.showAlert(title: "Error", message: "Error \(errorCode.description) occur")
+        LogUtils.log(message: "didOccurError: \(errorCode)", level: .error)
+        self.showAlert(title: "Error", message: "didOccurError \(errorCode.description) occur")
     }
     
     func rtcEngine(_ engine: AgoraRtcEngineKit, didJoinChannel channel: String, withUid uid: UInt, elapsed: Int) {
@@ -290,58 +313,42 @@ extension CustomVideoSourcePushMultiMain: AgoraRtcEngineDelegate {
     /// @param uid uid of remote joined user
     /// @param elapsed time elapse since current sdk instance join the channel in ms
     func rtcEngine(_ engine: AgoraRtcEngineKit, didJoinedOfUid uid: UInt, elapsed: Int) {
+        guard let channelName = configs["channelName"] as? String else {return}
+        if let _ = remoteVideos.first(where: { $0.uid == uid }) { return }
+        guard let userModel = remoteVideos.first(where: { $0.isJoin == false && $0.trackId == 0 }) else { return }
         LogUtils.log(message: "remote user join: \(uid) \(elapsed)ms", level: .info)
+        userModel.uid = uid
+        userModel.isJoin = true
         
-        // Only one remote video view is available for this
-        // tutorial. Here we check if there exists a surface
-        // view tagged as this uid.
-        if uid == 999 { return }
-        for model in remoteVideos where model.uid == uid {
-            return
-        }
+        // the view to be binded
         let videoCanvas = AgoraRtcVideoCanvas()
         videoCanvas.uid = uid
-        // the view to be binded
-        guard let userModel = remoteVideos.first(where: { $0.isJoin == false }) else { return }
         videoCanvas.view = userModel.canvasView?.videoView
-        videoCanvas.renderMode = .hidden
-        userModel.uid = uid
-        userModel.isJoin = true
-        agoraKit.setupRemoteVideo(videoCanvas)
+        videoCanvas.renderMode = .fit
+        
+        let connect = AgoraRtcConnection()
+        connect.localUid = localUid
+        connect.channelId = channelName
+        agoraKit.setupRemoteVideoEx(videoCanvas, connection: connect)
     }
     
     /// callback when a remote user is leaving the channel, note audience in live broadcast mode will NOT trigger this event
     /// @param uid uid of remote joined user
     /// @param reason reason why this user left, note this event may be triggered when the remote user
     /// become an audience in live broadcasting profile
     func rtcEngine(_ engine: AgoraRtcEngineKit, didOfflineOfUid uid: UInt, reason: AgoraUserOfflineReason) {
+        guard let _ = remoteVideos.first(where: { $0.uid == uid }) else { return }
         LogUtils.log(message: "remote user left: \(uid) reason \(reason)", level: .info)
         
-        // to unlink your view from sdk, so that your view reference will be released
-        // note the video will stay at its last frame, to completely remove it
-        // you will need to remove the EAGL sublayer from your binded view
-        let userModel = remoteVideos.first(where: { $0.uid == uid })
-        userModel?.isJoin = false
-        userModel?.uid = UInt(Int.random(in: 10000...99999))
-        userModel?.canvasView?.videoView.reset()
+        cleanCanvas(uid: uid)
     }
 }
 
 /// agora camera video source, the delegate will get frame data from camera
 extension CustomVideoSourcePushMultiMain: AgoraYUVImageSourcePushDelegate {
     func onVideoFrame(_ buffer: CVPixelBuffer, size: CGSize, trackId: UInt, rotation: Int32) {
         let videoFrame = AgoraVideoFrame()
-        /** Video format:
-         * - 1: I420
-         * - 2: BGRA
-         * - 3: NV21
-         * - 4: RGBA
-         * - 5: IMC2
-         * - 7: ARGB
-         * - 8: NV12
-         * - 12: iOS texture (CVPixelBufferRef)
-         */
-        videoFrame.format = 12
+        videoFrame.format = AgoraVideoFormat.cvPixelNV12.rawValue
         videoFrame.textureBuf = buffer
         videoFrame.rotation = Int32(rotation)
         
@@ -350,12 +357,8 @@ extension CustomVideoSourcePushMultiMain: AgoraYUVImageSourcePushDelegate {
         outputVideoFrame.height = Int32(size.height)
         outputVideoFrame.pixelBuffer = buffer
         outputVideoFrame.rotation = rotation
-        if customCamera?.trackId ?? 0 == trackId {
-            localVideo.videoView.renderVideoPixelBuffer(outputVideoFrame)
-        } else {
-            let userModel = remoteVideos.first(where: { $0.trackId == trackId })
-            userModel?.canvasView?.videoView.renderVideoPixelBuffer(outputVideoFrame)
-        }
+        let userModel = remoteVideos.first(where: { $0.trackId == trackId })
+        userModel?.canvasView?.videoView.renderVideoPixelBuffer(outputVideoFrame)
         // once we have the video frame, we can push to agora sdk
         agoraKit?.pushExternalVideoFrame(videoFrame, videoTrackId: trackId)
     }
diff --git a/macOS/APIExample/Common/Utils/KFMP4Demuxer.m b/macOS/APIExample/Common/Utils/KFMP4Demuxer.m
@@ -52,6 +52,7 @@ @interface KFMP4Demuxer () {
 @property (nonatomic, assign, readwrite) BOOL audioEOF; // Indicates whether audio has ended.
 @property (nonatomic, assign, readwrite) BOOL videoEOF; // Indicates whether video has ended.
 @property (nonatomic, assign, readwrite) CGAffineTransform preferredTransform; // Transformation information for the image, e.g., video image rotation.
+@property (nonatomic, assign) CGFloat frameRate; // Frame rate of the video.
 @end
 
 @implementation KFMP4Demuxer
@@ -233,6 +234,9 @@ - (void)fetchAndSaveDemuxedData {
                 [self saveSampleBuffer:videoBuffer];
                 CFRelease(videoBuffer);
             }
+            
+            NSTimeInterval deley = 1000 / self.frameRate;
+            usleep(deley * 1000);
         }
         if (self.demuxerStatus == KFMP4DemuxerStatusCompleted) {
             NSLog(@"KFMP4Demuxer complete");
@@ -391,6 +395,9 @@ - (void)_setupDemuxReader:(NSError**)error {
         AVAssetTrack *videoTrack = [[self.config.asset tracksWithMediaType:AVMediaTypeVideo] firstObject];
         _hasVideoTrack = videoTrack ? YES : NO;
         if (_hasVideoTrack) {
+            // Get frame rate
+            self.frameRate = videoTrack.nominalFrameRate > 0 ? videoTrack.nominalFrameRate : 30;
+            
             // Get the image transformation information.
             _preferredTransform = videoTrack.preferredTransform;
             
diff --git a/macOS/APIExample/Examples/Advanced/CustomVideoSourcePushMulti/CustomVideoSourcePushMulti.swift b/macOS/APIExample/Examples/Advanced/CustomVideoSourcePushMulti/CustomVideoSourcePushMulti.swift
