Skip to content

Commit 06ea3f2

Browse files
author
lizhengyong
committed
add simple filter demo on iOS
1 parent bb983fb commit 06ea3f2

10 files changed

Lines changed: 367 additions & 91 deletions

‎iOS/Podfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ end
2020

2121
target 'SimpleFilter' do
2222
pod 'AgoraNGSDK', :path => 'sdk'
23+
pod 'OpenCV', '~> 4.3.0'
2324
use_frameworks!
2425

2526
end

‎iOS/SimpleFilter/AudioProcessor.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
// Created by DYF on 2020/7/13.
33
//
44

5-
#ifndef AGORAWITHBYTEDANCE_AUDIOPROCESSOR_H
6-
#define AGORAWITHBYTEDANCE_AUDIOPROCESSOR_H
5+
#ifndef AGORA_AUDIOPROCESSOR_H
6+
#define AGORA_AUDIOPROCESSOR_H
77

88
#include <thread>
99
#include <string>
@@ -40,4 +40,4 @@ namespace agora {
4040
}
4141

4242

43-
#endif //AGORAWITHBYTEDANCE_AUDIOPROCESSOR_H
43+
#endif //AGORA_AUDIOPROCESSOR_H

‎iOS/SimpleFilter/ExtensionAudioFilter.hpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
// Created by 张涛 on 2020/4/26.
33
//
44

5-
#ifndef AGORAWITHBYTEDANCE_EXTENSIONAUDIOFILTER_H
6-
#define AGORAWITHBYTEDANCE_EXTENSIONAUDIOFILTER_H
5+
#ifndef AGORA_EXTENSIONAUDIOFILTER_H
6+
#define AGORA_EXTENSIONAUDIOFILTER_H
77

88
#include "AgoraRtcKit/NGIAgoraMediaNode.h"
99
#include <AgoraRtcKit/AgoraRefCountedObject.h>
@@ -14,7 +14,7 @@ namespace agora {
1414
namespace extension {
1515
class ExtensionAudioFilter : public agora::rtc::IAudioFilter {
1616
public:
17-
ExtensionAudioFilter(const char* name, agora_refptr<AdjustVolumeAudioProcessor> byteDanceProcessor);
17+
ExtensionAudioFilter(const char* name, agora_refptr<AdjustVolumeAudioProcessor> processor);
1818
~ExtensionAudioFilter();
1919
bool adaptAudioFrame(const media::base::AudioPcmFrame& inAudioPcmFrame,
2020
media::base::AudioPcmFrame& adaptedPcmFrame) override;
@@ -34,4 +34,4 @@ namespace agora {
3434
}
3535

3636

37-
#endif //AGORAWITHBYTEDANCE_EXTENSIONAUDIOFILTER_H
37+
#endif //AGORA_EXTENSIONAUDIOFILTER_H

‎iOS/SimpleFilter/ExtensionProvider.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
// Created by 张涛 on 2020/4/26.
33
//
44

5-
#ifndef AGORAWITHBYTEDANCE_EXTENSION_PROVIDER_H
6-
#define AGORAWITHBYTEDANCE_EXTENSION_PROVIDER_H
5+
#ifndef AGORA_EXTENSION_PROVIDER_H
6+
#define AGORA_EXTENSION_PROVIDER_H
77

88
#include "AgoraRtcKit/NGIAgoraExtensionProvider.h"
99
#include "ExtensionAudioFilter.hpp"
@@ -31,4 +31,4 @@ namespace agora {
3131
};
3232
}
3333
}
34-
#endif //AGORAWITHBYTEDANCE_EXTENSION_PROVIDER_H
34+
#endif //AGORA_EXTENSION_PROVIDER_H

‎iOS/SimpleFilter/ExtensionVideoFilter.cpp‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
namespace agora {
99
namespace extension {
1010

11-
ExtensionVideoFilter::ExtensionVideoFilter(agora_refptr<WatermarkProcessor> processer) {
12-
waterMarkProcessor_ = processer;
11+
ExtensionVideoFilter::ExtensionVideoFilter(agora_refptr<WatermarkProcessor> processor):threadPool_(1) {
12+
waterMarkProcessor_ = processor;
1313
}
1414

1515
ExtensionVideoFilter::~ExtensionVideoFilter() {
@@ -35,26 +35,28 @@ namespace agora {
3535
}
3636

3737
int ExtensionVideoFilter::start(agora::agora_refptr<Control> control) {
38+
printf("ExtensionVideoFilter::start\n");
3839
if (!waterMarkProcessor_) {
3940
return -1;
4041
}
4142
if (control) {
4243
control_ = control;
4344
waterMarkProcessor_->setExtensionControl(control);
44-
// }
45-
// if (mode_ == ProcessMode::kAsync){
46-
// invoker_id = threadPool_.RegisterInvoker("thread_videofilter");
47-
// auto res = threadPool_.PostTaskWithRes(invoker_id, [waterMarkProcessor=waterMarkProcessor_] {
48-
// return waterMarkProcessor->initOpenGL();
49-
// });
50-
// isInitOpenGL = res.get();
45+
}
46+
if (mode_ == ProcessMode::kAsync){
47+
invoker_id = threadPool_.RegisterInvoker("thread_videofilter");
48+
auto res = threadPool_.PostTaskWithRes(invoker_id, [waterMarkProcessor=waterMarkProcessor_] {
49+
return waterMarkProcessor->initOpenGL();
50+
});
51+
isInitOpenGL = res.get();
5152
} else {
5253
isInitOpenGL = waterMarkProcessor_->initOpenGL();
5354
}
5455
return 0;
5556
}
5657

5758
int ExtensionVideoFilter::stop() {
59+
printf("ExtensionVideoFilter::stop\n");
5860
if (waterMarkProcessor_) {
5961
waterMarkProcessor_->releaseOpenGL();
6062
isInitOpenGL = false;
@@ -69,14 +71,14 @@ namespace agora {
6971

7072
bool isAsyncMode = (mode_ == ProcessMode::kAsync);
7173
if (isAsyncMode && waterMarkProcessor_ && control_ && invoker_id >= 0) {
72-
// threadPool_.PostTask(invoker_id, [videoFrame=frame, byteDanceProcessor=waterMarkProcessor_, control=control_] {
73-
// rtc::VideoFrameData srcData;
74-
// videoFrame->getVideoFrameData(srcData);
75-
// byteDanceProcessor->processFrame(srcData);
76-
// // In asynchronous mode (mode is set to Async),
77-
// // the plug-in needs to call this method to return the processed video frame to the SDK.
78-
// control->deliverVideoFrame(videoFrame);
79-
// });
74+
threadPool_.PostTask(invoker_id, [videoFrame=frame, processor=waterMarkProcessor_, control=control_] {
75+
rtc::VideoFrameData srcData;
76+
videoFrame->getVideoFrameData(srcData);
77+
processor->processFrame(srcData);
78+
// In asynchronous mode (mode is set to Async),
79+
// the plug-in needs to call this method to return the processed video frame to the SDK.
80+
control->deliverVideoFrame(videoFrame);
81+
});
8082
return kSuccess;
8183
}
8284
return kBypass;
@@ -102,6 +104,7 @@ namespace agora {
102104
// Agora SDK will call this method to set video plug-in properties
103105
int ExtensionVideoFilter::setProperty(const char *key, const void *buf,
104106
size_t buf_size) {
107+
printf("setProperty %s %s\n", key, buf);
105108
std::string stringParameter((char*)buf);
106109
waterMarkProcessor_->setParameters(stringParameter);
107110
return 0;

‎iOS/SimpleFilter/ExtensionVideoFilter.hpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,13 @@
99
#include <AgoraRtcKit/AgoraRefCountedObject.h>
1010
#include "AgoraRtcKit/AgoraRefPtr.h"
1111
#include "VideoProcessor.hpp"
12+
#include "external_thread_pool.h"
1213

1314
namespace agora {
1415
namespace extension {
1516
class ExtensionVideoFilter : public agora::rtc::IExtensionVideoFilter {
1617
public:
17-
ExtensionVideoFilter(agora_refptr<WatermarkProcessor> byteDanceProcessor);
18+
ExtensionVideoFilter(agora_refptr<WatermarkProcessor> processor);
1819

1920
~ExtensionVideoFilter();
2021

@@ -37,10 +38,10 @@ namespace agora {
3738
agora::agora_refptr<WatermarkProcessor> waterMarkProcessor_;
3839
bool isInitOpenGL = false;
3940
ProcessMode mode_;
41+
agora::extension::ThreadPool threadPool_;
4042
int invoker_id = -1;
4143
protected:
42-
ExtensionVideoFilter() = default;
43-
44+
ExtensionVideoFilter() = delete;
4445
};
4546
}
4647
}

‎iOS/SimpleFilter/VideoProcessor.cpp‎

Lines changed: 11 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,7 @@
66

77
#include <chrono>
88

9-
10-
#define CHECK_BEF_AI_RET_SUCCESS(ret, ...) \
11-
if(ret != 0){\
12-
PRINTF_ERROR(__VA_ARGS__);\
13-
}
9+
#include <opencv2/opencv.hpp>
1410

1511
namespace agora {
1612
namespace extension {
@@ -21,67 +17,31 @@ namespace agora {
2117

2218
bool WatermarkProcessor::releaseOpenGL() {
2319
const std::lock_guard<std::mutex> lock(mutex_);
24-
25-
#if defined(__ANDROID__) || defined(TARGET_OS_ANDROID)
26-
if (eglCore_) {
27-
if (offscreenSurface_) {
28-
eglCore_->releaseSurface(offscreenSurface_);
29-
}
30-
delete eglCore_;
31-
eglCore_ = nullptr;
32-
}
33-
#endif
3420
return true;
3521
}
3622

3723
int WatermarkProcessor::processFrame(agora::rtc::VideoFrameData &capturedFrame) {
38-
// PRINTF_INFO("processFrame: w: %d, h: %d, r: %d, enable: %d", capturedFrame.width, capturedFrame.height, capturedFrame.rotation, wmEffectEnabled_);
3924
if (wmEffectEnabled_) {
4025
addWatermark(capturedFrame);
4126
}
4227
return 0;
4328
}
4429

4530
void WatermarkProcessor::addWatermark(const agora::rtc::VideoFrameData &capturedFrame) {
46-
// cv::Mat image(capturedFrame.height, capturedFrame.width, CV_8U,
47-
// (void*)capturedFrame.pixels.data);
48-
// double fontSize = image.cols / 800;
49-
// if (fontSize == 0.0) {
50-
// fontSize = 2.0;
51-
// }
52-
// cv::Point point(image.rows/2, image.cols/2);
53-
// cv::Scalar scalar(255, 0, 0);
54-
// cv::Mat textImg = cv::Mat::zeros(image.rows, image.cols, image.type());
55-
// cv::putText(textImg, wmStr_, point, cv::FONT_HERSHEY_DUPLEX, fontSize, scalar,3);
56-
// cv::flip(textImg, textImg, 0);
57-
// image = image + textImg;
31+
cv::Mat image(capturedFrame.height, capturedFrame.width, CV_8U,
32+
(void*)capturedFrame.pixels.data);
33+
double fontSize = image.cols / 400.0;
34+
cv::Point point(image.cols/2, image.rows/2);
35+
cv::Scalar scalar(255, 0, 0);
36+
cv::Mat textImg = cv::Mat::zeros(image.rows, image.cols, image.type());
37+
cv::putText(textImg, wmStr_, point, cv::FONT_HERSHEY_DUPLEX, fontSize, scalar, 3);
38+
cv::flip(textImg, textImg, 1);
39+
image = image + textImg;
5840
}
5941

6042
int WatermarkProcessor::setParameters(std::string parameter) {
6143
const std::lock_guard<std::mutex> lock(mutex_);
62-
// Document d;
63-
// d.Parse(parameter.c_str());
64-
// if (d.HasParseError()) {
65-
// return -ERROR_INVALID_JSON;
66-
// }
67-
//
68-
//
69-
// if (d.HasMember("plugin.watermark.wmEffectEnabled")) {
70-
// Value& enabled = d["plugin.watermark.wmEffectEnabled"];
71-
// if (!enabled.IsBool()) {
72-
// return -ERROR_INVALID_JSON_TYPE;
73-
// }
74-
// wmEffectEnabled_ = enabled.GetBool();
75-
// }
76-
//
77-
// if (d.HasMember("plugin.watermask.wmStr")) {
78-
// Value& wmStr = d["plugin.watermask.wmStr"];
79-
// if (!wmStr.IsString()) {
80-
// return -ERROR_INVALID_JSON_TYPE;
81-
// }
82-
// wmStr_ = wmStr.GetString();
83-
// }
84-
44+
8545
return 0;
8646
}
8747

‎iOS/SimpleFilter/VideoProcessor.hpp‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
// Created by DYF on 2020/7/13.
33
//
44

5-
#ifndef AGORAWITHBYTEDANCE_VIDEOPROCESSOR_H
6-
#define AGORAWITHBYTEDANCE_VIDEOPROCESSOR_H
5+
#ifndef AGORA_VIDEOPROCESSOR_H
6+
#define AGORA_VIDEOPROCESSOR_H
77

88
#include <thread>
99
#include <string>
@@ -38,18 +38,14 @@ namespace agora {
3838
private:
3939
void addWatermark(const agora::rtc::VideoFrameData &capturedFrame);
4040
void dataCallback(const char* data);
41-
42-
#if defined(__ANDROID__) || defined(TARGET_OS_ANDROID)
43-
EglCore *eglCore_ = nullptr;
44-
EGLSurface offscreenSurface_ = nullptr;
45-
#endif
41+
4642
std::mutex mutex_;
4743
agora::agora_refptr<rtc::IExtensionVideoFilter::Control> control_;
48-
bool wmEffectEnabled_ = false;
49-
std::string wmStr_= "";
44+
bool wmEffectEnabled_ = true;
45+
std::string wmStr_= "Agora";
5046
};
5147
}
5248
}
5349

5450

55-
#endif //AGORAWITHBYTEDANCE_VIDEOPROCESSOR_H
51+
#endif //AGORA_VIDEOPROCESSOR_H

0 commit comments

Comments
 (0)