1.实现采集yuv本地视频, 2.实现渲染本地piexlBuffer 3.添加多路视频自采集

Author: zhaoyongqiang

iOS/APIExample/APIExample.xcodeproj/project.pbxproj

@@ -142,6 +142,10 @@
 		E728B84C28B6015800674A4A /* AgoraPictureInPictureController.m in Sources */ = {isa = PBXBuildFile; fileRef = E728B84B28B6015800674A4A /* AgoraPictureInPictureController.m */; };
 		E728B84F28B601A300674A4A /* AgoraSampleBufferRender.m in Sources */ = {isa = PBXBuildFile; fileRef = E728B84E28B601A300674A4A /* AgoraSampleBufferRender.m */; };
 		E728B85128B60D5B00674A4A /* VideoViewSampleBufferDisplayView.xib in Resources */ = {isa = PBXBuildFile; fileRef = E728B85028B60D5B00674A4A /* VideoViewSampleBufferDisplayView.xib */; };
+		E728B85828B86B0700674A4A /* CustomVideoSourcePushMulti.strings in Resources */ = {isa = PBXBuildFile; fileRef = E728B85328B86B0700674A4A /* CustomVideoSourcePushMulti.strings */; };
+		E728B85928B86B0700674A4A /* CustomVideoSourcePushMulti.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = E728B85528B86B0700674A4A /* CustomVideoSourcePushMulti.storyboard */; };
+		E728B85A28B86B0700674A4A /* CustomVideoSourcePushMulti.swift in Sources */ = {isa = PBXBuildFile; fileRef = E728B85728B86B0700674A4A /* CustomVideoSourcePushMulti.swift */; };
+		E728B85C28B8971200674A4A /* sample.yuv in Resources */ = {isa = PBXBuildFile; fileRef = E728B85B28B8971200674A4A /* sample.yuv */; };
 		E74877B328A23B2F00CA2F58 /* SimpleFilter.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 8B10BE0D26AFFFA6002E1373 /* SimpleFilter.framework */; };
 		E74877B728A23B8B00CA2F58 /* NetworkManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = E74877B628A23B8B00CA2F58 /* NetworkManager.swift */; };
 		E74877BA28A23C1400CA2F58 /* JSONObject.swift in Sources */ = {isa = PBXBuildFile; fileRef = E74877B928A23C1400CA2F58 /* JSONObject.swift */; };
@@ -385,6 +389,10 @@
 		E728B84D28B601A300674A4A /* AgoraSampleBufferRender.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AgoraSampleBufferRender.h; sourceTree = "<group>"; };
 		E728B84E28B601A300674A4A /* AgoraSampleBufferRender.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AgoraSampleBufferRender.m; sourceTree = "<group>"; };
 		E728B85028B60D5B00674A4A /* VideoViewSampleBufferDisplayView.xib */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.xib; path = VideoViewSampleBufferDisplayView.xib; sourceTree = "<group>"; };
+		E728B85428B86B0700674A4A /* zh-Hans */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "zh-Hans"; path = "zh-Hans.lproj/CustomVideoSourcePushMulti.strings"; sourceTree = "<group>"; };
+		E728B85628B86B0700674A4A /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/CustomVideoSourcePushMulti.storyboard; sourceTree = "<group>"; };
+		E728B85728B86B0700674A4A /* CustomVideoSourcePushMulti.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CustomVideoSourcePushMulti.swift; sourceTree = "<group>"; };
+		E728B85B28B8971200674A4A /* sample.yuv */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = sample.yuv; sourceTree = "<group>"; };
 		E74877B628A23B8B00CA2F58 /* NetworkManager.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NetworkManager.swift; sourceTree = "<group>"; };
 		E74877B928A23C1400CA2F58 /* JSONObject.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = JSONObject.swift; sourceTree = "<group>"; };
 		E74877C928A2611C00CA2F58 /* ToastView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ToastView.swift; sourceTree = "<group>"; };
@@ -670,6 +678,7 @@
 				8B349FE22681E2CE007247F2 /* agora-logo.png */,
 				576CA80925A9CC3A0091520B /* output.raw */,
 				03414B5425546DEC00AB114D /* frames0.yuv */,
+				E728B85B28B8971200674A4A /* sample.yuv */,
 				03BEED0C251CAB9C005E78F4 /* audioeffect.mp3 */,
 				03BEED0A251C4446005E78F4 /* audiomixing.mp3 */,
 			);
@@ -933,6 +942,7 @@
 				671BD6B227E1DB2D0076D5E1 /* CustomAudioRender */,
 				033A9EF1252D61E200BC26E1 /* CustomVideoRender */,
 				033A9EEF252D61E200BC26E1 /* CustomVideoSourcePush */,
+				E728B85228B86B0700674A4A /* CustomVideoSourcePushMulti */,
 				033A9EE3252D5C6900BC26E1 /* VideoMetadata */,
 				034C625C2524A06800296ECF /* VoiceChanger */,
 				038576772521E568003C369A /* MediaChannelRelay */,
@@ -970,6 +980,17 @@
 			path = PictureInPicture;
 			sourceTree = "<group>";
 		};
+		E728B85228B86B0700674A4A /* CustomVideoSourcePushMulti */ = {
+			isa = PBXGroup;
+			children = (
+				E728B85328B86B0700674A4A /* CustomVideoSourcePushMulti.strings */,
+				E728B85528B86B0700674A4A /* CustomVideoSourcePushMulti.storyboard */,
+				E728B85728B86B0700674A4A /* CustomVideoSourcePushMulti.swift */,
+			);
+			name = CustomVideoSourcePushMulti;
+			path = APIExample/Examples/Advanced/CustomVideoSourcePushMulti;
+			sourceTree = SOURCE_ROOT;
+		};
 		E74877B528A23B8B00CA2F58 /* NetworkManager */ = {
 			isa = PBXGroup;
 			children = (
@@ -1158,6 +1179,7 @@
 				033A9F2A252D737900BC26E1 /* Localizable.strings in Resources */,
 				8BE7ABC2279E065000DFBCEF /* FusionCDN.strings in Resources */,
 				576EA54825AC3523000B3D79 /* CustomPcmAudioSource.storyboard in Resources */,
+				E728B85828B86B0700674A4A /* CustomVideoSourcePushMulti.strings in Resources */,
 				6709B23C2806BB4A000BCC58 /* RawAudioData.storyboard in Resources */,
 				033A9F7F252D8B5900BC26E1 /* AudioMixing.storyboard in Resources */,
 				E728B84728B5FFCB00674A4A /* PictureInPicture.strings in Resources */,
@@ -1168,8 +1190,10 @@
 				03BEED0D251CAB9C005E78F4 /* audioeffect.mp3 in Resources */,
 				A7CA48C424553CF700507435 /* Popover.storyboard in Resources */,
 				03D13BDC2448758B00B599B3 /* LaunchScreen.storyboard in Resources */,
+				E728B85C28B8971200674A4A /* sample.yuv in Resources */,
 				8BC751D5273E502700552265 /* LiveStreaming.strings in Resources */,
 				8B349FE32681E2CE007247F2 /* agora-logo.png in Resources */,
+				E728B85928B86B0700674A4A /* CustomVideoSourcePushMulti.storyboard in Resources */,
 				033A9F23252D70E400BC26E1 /* JoinChannelVideo.storyboard in Resources */,
 				03414B5525546DEC00AB114D /* frames0.yuv in Resources */,
 				8B5E5B53274CBF760040E97D /* VideoProcess.storyboard in Resources */,
@@ -1327,6 +1351,7 @@
 				6709B23B2806B0EA000BCC58 /* RawAudioData.swift in Sources */,
 				033A9EEA252D5F5E00BC26E1 /* JoinMultiChannel.swift in Sources */,
 				0339BE64251DCA3B007D4FDD /* GlobalSettings.swift in Sources */,
+				E728B85A28B86B0700674A4A /* CustomVideoSourcePushMulti.swift in Sources */,
 				E728B84C28B6015800674A4A /* AgoraPictureInPictureController.m in Sources */,
 				8407E0942472320800AC5DE8 /* (null) in Sources */,
 				8B5E5B50274CB68E0040E97D /* RhythmPlayer.swift in Sources */,
@@ -1731,6 +1756,22 @@
 			name = PictureInPicture.storyboard;
 			sourceTree = "<group>";
 		};
+		E728B85328B86B0700674A4A /* CustomVideoSourcePushMulti.strings */ = {
+			isa = PBXVariantGroup;
+			children = (
+				E728B85428B86B0700674A4A /* zh-Hans */,
+			);
+			name = CustomVideoSourcePushMulti.strings;
+			sourceTree = "<group>";
+		};
+		E728B85528B86B0700674A4A /* CustomVideoSourcePushMulti.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				E728B85628B86B0700674A4A /* Base */,
+			);
+			name = CustomVideoSourcePushMulti.storyboard;
+			sourceTree = "<group>";
+		};
 		E7899BD72861673600851463 /* CreateDataStream.strings */ = {
 			isa = PBXVariantGroup;
 			children = (

iOS/APIExample/APIExample/Common/ExternalVideo/AgoraSampleBufferRender.h

@@ -20,6 +20,8 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (void)renderVideoData:(AgoraOutputVideoFrame *_Nonnull)videoData;
 
+- (void)renderVideoPixelBuffer:(AgoraOutputVideoFrame *_Nonnull)videoData;
+
 @end
 
 NS_ASSUME_NONNULL_END

iOS/APIExample/APIExample/Common/ExternalVideo/AgoraSampleBufferRender.m

@@ -152,4 +152,36 @@ - (void)renderVideoData:(AgoraOutputVideoFrame *_Nonnull)videoData {
     }
 }
 
+- (void)renderVideoPixelBuffer:(AgoraOutputVideoFrame *_Nonnull)videoData {
+    if (!videoData) {
+        return;
+    }
+    
+    dispatch_async(dispatch_get_main_queue(), ^{
+        self->_videoWidth = videoData.width;
+        self->_videoHeight = videoData.height;
+        
+        [self layoutDisplayLayer];
+    });
+    
+    @autoreleasepool {
+        CVPixelBufferRef pixelBuffer = videoData.pixelBuffer;
+       
+        CMVideoFormatDescriptionRef videoInfo;
+        CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &videoInfo);
+        
+        CMSampleTimingInfo timingInfo;
+        timingInfo.duration = kCMTimeZero;
+        timingInfo.decodeTimeStamp = kCMTimeInvalid;
+        timingInfo.presentationTimeStamp = CMTimeMake(CACurrentMediaTime()*1000, 1000);
+        
+        CMSampleBufferRef sampleBuffer;
+        CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBuffer, videoInfo, &timingInfo, &sampleBuffer);
+
+        [self.displayLayer enqueueSampleBuffer:sampleBuffer];
+        CMSampleBufferInvalidate(sampleBuffer);
+        CFRelease(sampleBuffer);
+    }
+}
+
 @end

iOS/APIExample/APIExample/Common/ExternalVideo/AgoraYUVImageSourcePush.h

@@ -13,15 +13,21 @@ NS_ASSUME_NONNULL_BEGIN
 
 @protocol AgoraYUVImageSourcePushDelegate <NSObject>
 
--(void)onVideoFrame:(NSData*)buffer size:(CGSize)size rotation:(int)rotation;
+-(void)onVideoFrame:(CVPixelBufferRef)buffer
+               size:(CGSize)size
+          trackId: (NSUInteger)trackId
+           rotation:(int)rotation;
 
 @end
 
 @interface AgoraYUVImageSourcePush : NSObject
+- (instancetype)initWithSize: (CGSize)size fileName: (NSString *)fileName frameRate: (int)frameRate;
+@property(nonatomic, assign)UInt32 trackId;
 @property(nonatomic, weak)id<AgoraYUVImageSourcePushDelegate> delegate;
 
 -(void)startSource;
 -(void)stopSource;
 @end
 
+
 NS_ASSUME_NONNULL_END

iOS/APIExample/APIExample/Common/ExternalVideo/AgoraYUVImageSourcePush.m

@@ -7,33 +7,77 @@
 //
 
 #import "AgoraYUVImageSourcePush.h"
+#import <UIKit/UIKit.h>
+#import "MediaUtils.h"
 
 @interface AgoraYUVImageSourcePush ()
 @property(nonatomic, strong) NSData* yuvData;
 @property(nonatomic, strong) NSTimer* timer;
+@property(nonatomic, assign) int videoW;
+@property(nonatomic, assign) int videoH;
+@property(nonatomic, strong)NSFileHandle *fileHandle;
+@property(nonatomic, assign) int frameRate;
+@property(nonatomic, copy) NSString *fileName;
 @end
 
 @implementation AgoraYUVImageSourcePush
 
--(id)init
-{
+- (instancetype)initWithSize: (CGSize)size fileName: (NSString *)fileName frameRate: (int)frameRate {
     if(self = [super init]) {
-        // initialize yuv data
-        NSString *yuvPath = [[NSBundle mainBundle] pathForResource:@"frames0" ofType:@"yuv"];
-        self.yuvData = [NSData dataWithContentsOfFile:yuvPath options:NSDataReadingUncached error:NULL];
+        self.fileName = fileName;
+        self.videoW = size.width;
+        self.videoH = size.height;
+        self.frameRate = frameRate;
+        [self loadData];
     }
     return self;
 }
 
+- (void)loadData {
+    NSString *yuvPath = [[NSBundle mainBundle] pathForResource:self.fileName ofType:@"yuv"];
+    self.fileHandle = [NSFileHandle fileHandleForReadingAtPath:yuvPath];
+}
+
 -(void)startSource
 {
     if(self.timer) {
         return;
     }
+    NSUInteger frameSizeY = self.videoW*self.videoH;
+    NSUInteger frameSizeU = self.videoW*self.videoH/4;
+    NSUInteger frameSizeV = self.videoW*self.videoH/4;
+    
+    __block NSUInteger i = 0;
+    NSTimeInterval sT = NSDate.date.timeIntervalSince1970;
+    
     __weak AgoraYUVImageSourcePush * weakSelf = self;
-    self.timer = [NSTimer scheduledTimerWithTimeInterval:0.5 repeats:YES block:^(NSTimer * _Nonnull timer) {
+    self.timer = [NSTimer scheduledTimerWithTimeInterval:self.frameRate / 100.0 repeats:YES block:^(NSTimer * _Nonnull timer) {
         if(weakSelf.delegate) {
-            [weakSelf.delegate onVideoFrame:weakSelf.yuvData size:CGSizeMake(176, 144) rotation:0];
+            NSData *dataY = [weakSelf.fileHandle readDataOfLength:frameSizeY];
+            NSData *dataU = [weakSelf.fileHandle readDataOfLength:frameSizeU];
+            NSData *dataV = [weakSelf.fileHandle readDataOfLength:frameSizeV];
+            
+            if (dataY.length==0 || dataV.length==0 || dataU.length==0) {
+                i = 0;
+                [weakSelf loadData];
+                dataY = [weakSelf.fileHandle readDataOfLength:frameSizeY];
+                dataU = [weakSelf.fileHandle readDataOfLength:frameSizeU];
+                dataV = [weakSelf.fileHandle readDataOfLength:frameSizeV];
+            }
+            CVPixelBufferRef pixelBuffer = [MediaUtils i420ToPixelBuffer:(void *)dataY.bytes
+                                                                    srcU:(void *)dataU.bytes
+                                                                    srcV:(void *)dataV.bytes
+                                                                   width:weakSelf.videoW
+                                                                  height:weakSelf.videoH];
+            [weakSelf.delegate onVideoFrame:pixelBuffer size:CGSizeMake(weakSelf.videoW, weakSelf.videoH) trackId: self.trackId rotation:0];
+            CVPixelBufferRelease(pixelBuffer);
+            NSTimeInterval cT = NSDate.date.timeIntervalSince1970;
+            NSTimeInterval dT = cT-sT;
+            NSTimeInterval tT = i/25.0;
+            if (dT<tT) {
+                [NSThread sleepForTimeInterval:tT-dT];
+            }
+            i++;
         }
     }];
 }

iOS/APIExample/APIExample/Common/Utils/MediaUtils.h

@@ -13,9 +13,12 @@ NS_ASSUME_NONNULL_BEGIN
 
 @interface MediaUtils : NSObject
 
++ (CVPixelBufferRef)i420ToPixelBuffer:(void *)srcY srcU:(void *)srcU srcV:(void *)srcV width:(int)width height:(int)height;
 + (nullable UIImage *)i420ToImage:(nullable void *)srcY srcU:(nullable void *)srcU srcV:(nullable void *)srcV width:(int)width height:(int)height;
 + (nullable UIImage *)pixelBufferToImage:(CVPixelBufferRef)pixelBuffer;
 
++ (CVPixelBufferRef)CVPixelBufferRefFromUiImage:(UIImage *)img;
+
 @end
 
 NS_ASSUME_NONNULL_END