添加SimpleFilter文件

Author: zhaoyongqiang

macOS/APIExample.xcodeproj/project.pbxproj

@@ -0,0 +1,43 @@
+//
+// Created by DYF on 2020/7/13.
+//
+
+#ifndef AGORA_AUDIOPROCESSOR_H
+#define AGORA_AUDIOPROCESSOR_H
+
+#include <thread>
+#include <string>
+#include <mutex>
+#include <vector>
+#include <AgoraRtcKit/AgoraRefPtr.h>
+#include "AgoraRtcKit/NGIAgoraMediaNode.h"
+#include "AgoraRtcKit/AgoraMediaBase.h"
+
+namespace agora {
+    namespace extension {
+        class AdjustVolumeAudioProcessor  : public RefCountInterface {
+        public:
+            int processFrame(const agora::media::base::AudioPcmFrame &audioPcmFrame,
+                             media::base::AudioPcmFrame& adaptedPcmFrame);
+
+            void dataCallback(const char* data);
+
+            void setVolume(int volume) { volume_ = volume / 100.0f; }
+
+            int setExtensionControl(agora::agora_refptr<rtc::IExtensionVideoFilter::Control> control){
+                control_ = control;
+                return 0;
+            };
+        protected:
+            ~AdjustVolumeAudioProcessor() {}
+        private:
+            static int16_t FloatS16ToS16(float v);
+        private:
+            std::atomic<float> volume_ = {1.0f};
+            agora::agora_refptr<rtc::IExtensionVideoFilter::Control> control_;
+        };
+    }
+}
+
+
+#endif //AGORA_AUDIOPROCESSOR_H

macOS/SimpleFilter/AudioProcessor.hpp

@@ -0,0 +1,37 @@
+//
+// Created by DYF on 2020/7/13.
+//
+
+#include "AudioProcessor.hpp"
+#include <chrono>
+
+
+namespace agora {
+    namespace extension {
+        using limits_int16 = std::numeric_limits<int16_t>;
+        int16_t AdjustVolumeAudioProcessor::FloatS16ToS16(float v) {
+            static const float kMaxRound = (limits_int16::max)() - 0.5f;
+            static const float kMinRound = (limits_int16::min)() + 0.5f;
+            if (v > 0) {
+                return v >= kMaxRound ? (limits_int16::max)() : static_cast<int16_t>(v + 0.5f);
+            }
+            return v <= kMinRound ? (limits_int16::min)() : static_cast<int16_t>(v - 0.5f);
+        }
+
+        int AdjustVolumeAudioProcessor::processFrame(const media::base::AudioPcmFrame& inAudioPcmFrame,
+                                                      media::base::AudioPcmFrame& adaptedPcmFrame) {
+//            PRINTF_ERROR("adaptAudioFrame %f", volume_.load());
+            size_t length = inAudioPcmFrame.samples_per_channel_ * inAudioPcmFrame.num_channels_;
+            for (int idx = 0; idx < length; idx++) {
+                adaptedPcmFrame.data_[idx] = FloatS16ToS16(inAudioPcmFrame.data_[idx] * volume_);
+            }
+            return 0;
+        }
+
+        void AdjustVolumeAudioProcessor::dataCallback(const char* data){
+            if (control_) {
+                control_->postEvent("volume", data);
+            }
+        }
+    }
+}

macOS/SimpleFilter/AudioProcessor.mm

@@ -0,0 +1,35 @@
+//
+// Created by 张涛 on 2020/4/26.
+//
+
+#include "ExtensionAudioFilter.hpp"
+#include <sstream>
+
+namespace agora {
+    namespace extension {
+        ExtensionAudioFilter::ExtensionAudioFilter(const char* name, agora_refptr<AdjustVolumeAudioProcessor> audioProcessor) {
+            filterName_ = name;
+            audioProcessor_ = audioProcessor;
+        }
+
+        ExtensionAudioFilter::~ExtensionAudioFilter() {
+
+        }
+
+        bool ExtensionAudioFilter::adaptAudioFrame(const media::base::AudioPcmFrame& inAudioPcmFrame,
+                                                      media::base::AudioPcmFrame& adaptedPcmFrame) {
+            return audioProcessor_->processFrame(inAudioPcmFrame, adaptedPcmFrame) == 0;
+        }
+
+        int ExtensionAudioFilter::setProperty(const char* key, const void* buf, int buf_size) {
+            std::string str_volume = "100";
+            if (std::string(key) == "volume") {
+                str_volume = std::string(static_cast<const char*>(buf), buf_size);
+            }
+
+            int int_volume_ = atoi(str_volume.c_str());
+            audioProcessor_->setVolume(int_volume_);
+            return ERR_OK;
+        }
+    }
+}

macOS/SimpleFilter/ExtensionAudioFilter.cpp

@@ -0,0 +1,37 @@
+//
+// Created by 张涛 on 2020/4/26.
+//
+
+#ifndef AGORA_EXTENSIONAUDIOFILTER_H
+#define AGORA_EXTENSIONAUDIOFILTER_H
+
+#include "AgoraRtcKit/NGIAgoraMediaNode.h"
+#include <AgoraRtcKit/AgoraRefCountedObject.h>
+#include "AgoraRtcKit/AgoraRefPtr.h"
+#include "AudioProcessor.hpp"
+
+namespace agora {
+    namespace extension {
+        class ExtensionAudioFilter : public agora::rtc::IAudioFilter {
+        public:
+            ExtensionAudioFilter(const char* name, agora_refptr<AdjustVolumeAudioProcessor> processor);
+            ~ExtensionAudioFilter();
+            bool adaptAudioFrame(const media::base::AudioPcmFrame& inAudioPcmFrame,
+                                 media::base::AudioPcmFrame& adaptedPcmFrame) override;
+            void setEnabled(bool enable) override { enabled_ = enable; }
+            bool isEnabled() const override { return enabled_; }
+            int setProperty(const char* key, const void* buf, int buf_size) override;
+            int getProperty(const char* key, void* buf, int buf_size) const override { return ERR_OK; }
+            const char* getName() const override { return filterName_.c_str(); }
+        private:
+            std::atomic_bool enabled_ = {true};
+            std::string filterName_;
+            agora_refptr<AdjustVolumeAudioProcessor> audioProcessor_;
+        protected:
+            ExtensionAudioFilter() = default;
+        };
+    }
+}
+
+
+#endif //AGORA_EXTENSIONAUDIOFILTER_H

macOS/SimpleFilter/ExtensionAudioFilter.hpp

@@ -0,0 +1,57 @@
+//
+// Created by 张涛 on 2020/4/26.
+//
+
+#include "ExtensionProvider.hpp"
+
+namespace agora {
+    namespace extension {
+        ExtensionProvider::ExtensionProvider() {
+            audioProcessor_ = new agora::RefCountedObject<AdjustVolumeAudioProcessor>();
+            YUVProcessor_ = new agora::RefCountedObject<YUVImageProcessor>();
+        }
+
+        ExtensionProvider::~ExtensionProvider() {
+            audioProcessor_.reset();
+            YUVProcessor_.reset();
+        }
+
+        // Provide information about all plug-ins that support packaging.
+        // When the SDK loads the plug-in, it will call this method to send a callback to the plug-in.
+        // You need to provide information about all plug-ins that support encapsulation.
+        void ExtensionProvider::enumerateExtensions(ExtensionMetaInfo* extension_list,
+                                                           int& extension_count) {
+            extension_count = 2;
+            //Declare a Video Filter, and IExtensionProvider::createVideoFilter will be called
+            ExtensionMetaInfo i;
+            i.type = EXTENSION_TYPE::VIDEO_PRE_PROCESSING_FILTER;
+            i.extension_name = agora::extension::VIDEO_FILTER_NAME;
+            extension_list[0] = i;
+
+            //Declare an Audio Filter, and IExtensionProvider::createAudioFilter will be called
+            ExtensionMetaInfo j;
+            j.type = EXTENSION_TYPE::AUDIO_FILTER;
+            j.extension_name = agora::extension::AUDIO_FILTER_NAME;
+            extension_list[1] = j;
+        }
+
+        // Create a video plug-in. After the SDK calls this method, you need to return the IExtensionVideoFilter instance
+        agora_refptr<agora::rtc::IExtensionVideoFilter> ExtensionProvider::createVideoFilter(const char* name) {
+            auto videoFilter = new agora::RefCountedObject<agora::extension::ExtensionVideoFilter>(YUVProcessor_);
+            return videoFilter;
+        }
+
+        // Create a video plug-in. After the SDK calls this method, you need to return the IAudioFilter instance
+        agora_refptr<agora::rtc::IAudioFilter> ExtensionProvider::createAudioFilter(const char* name) {
+            auto audioFilter = new agora::RefCountedObject<agora::extension::ExtensionAudioFilter>(name, audioProcessor_);
+            return audioFilter;
+        }
+
+        agora_refptr<agora::rtc::IVideoSinkBase> ExtensionProvider::createVideoSink(const char* name) {
+            return nullptr;
+        }
+
+        void ExtensionProvider::setExtensionControl(rtc::IExtensionControl* control){
+        }
+    }
+}

macOS/SimpleFilter/ExtensionProvider.cpp

@@ -0,0 +1,34 @@
+//
+// Created by 张涛 on 2020/4/26.
+//
+
+#ifndef AGORA_EXTENSION_PROVIDER_H
+#define AGORA_EXTENSION_PROVIDER_H
+
+#include "AgoraRtcKit/NGIAgoraExtensionProvider.h"
+#include "ExtensionAudioFilter.hpp"
+#include "ExtensionVideoFilter.hpp"
+
+namespace agora {
+    namespace extension {
+        static const char* AUDIO_FILTER_NAME = "VolumeChange";
+        static const char* VIDEO_FILTER_NAME = "Grey";
+
+        class ExtensionProvider : public agora::rtc::IExtensionProvider {
+        private:
+            agora_refptr<AdjustVolumeAudioProcessor> audioProcessor_;
+            agora_refptr<YUVImageProcessor> YUVProcessor_;
+        public:
+            ExtensionProvider();
+
+            ~ExtensionProvider();
+
+            void setExtensionControl(rtc::IExtensionControl* control) override;
+            void enumerateExtensions(ExtensionMetaInfo* extension_list, int& extension_count) override;
+            agora_refptr<rtc::IAudioFilter> createAudioFilter(const char* name) override;
+            agora_refptr<rtc::IExtensionVideoFilter> createVideoFilter(const char* name) override;
+            agora_refptr<rtc::IVideoSinkBase> createVideoSink(const char* name) override;
+        };
+    }
+}
+#endif //AGORA_EXTENSION_PROVIDER_H

macOS/SimpleFilter/ExtensionProvider.hpp

@@ -0,0 +1,126 @@
+//
+// Created by 张涛 on 2020/4/26.
+//
+
+#include "ExtensionVideoFilter.hpp"
+#include <sstream>
+
+namespace agora {
+    namespace extension {
+    
+    ExtensionVideoFilter::ExtensionVideoFilter(agora_refptr<YUVImageProcessor> processor):threadPool_(1) {
+        YUVProcessor = processor;
+    }
+
+        ExtensionVideoFilter::~ExtensionVideoFilter() {
+            YUVProcessor->releaseOpenGL();
+        }
+
+        //Set the process mode between the SDK and the video plug-in
+        //If set ProcessMode to Sync mode, Agora SDK and video extension will pass data through adaptVideoFrame
+        //If set ProcessMode to Async mode, Agora SDK and video extension will pass data through pendVideoFrame and deliverVideoFrame
+        //If set independent_thread to false, all callbacks sent by Agora SDK are performed on the internal video processing thread
+        //If set independent_thread to true, all callbacks sent by Agora SDK are performed on a separate thread
+        void ExtensionVideoFilter::getProcessMode(ProcessMode& mode, bool& independent_thread) {
+            mode = ProcessMode::kSync;
+            independent_thread = false;
+            mode_ = mode;
+        }
+
+        //Set the type and format of the video data to be processed
+        void ExtensionVideoFilter::getVideoFormatWanted(rtc::VideoFrameData::Type& type,
+                                                        rtc::RawPixelBuffer::Format& format) {
+            type = rtc::VideoFrameData::Type::kRawPixels;
+            format = rtc::RawPixelBuffer::Format::kI420;
+        }
+
+        int ExtensionVideoFilter::start(agora::agora_refptr<Control> control) {
+            printf("ExtensionVideoFilter::start\n");
+            if (!YUVProcessor) {
+                return -1;
+            }
+            if (control) {
+                control_ = control;
+                YUVProcessor->setExtensionControl(control);
+            }
+            if (mode_ == ProcessMode::kAsync){
+                invoker_id = threadPool_.RegisterInvoker("thread_videofilter");
+                auto res = threadPool_.PostTaskWithRes(invoker_id, [yuvProcessor=YUVProcessor] {
+                     return yuvProcessor->initOpenGL();
+                });
+                isInitOpenGL = res.get();
+            } else {
+                isInitOpenGL = YUVProcessor->initOpenGL();
+            }
+            return 0;
+        }
+
+        int ExtensionVideoFilter::stop() {
+            printf("ExtensionVideoFilter::stop\n");
+            if (YUVProcessor) {
+                YUVProcessor->releaseOpenGL();
+                isInitOpenGL = false;
+            }
+            return 0;
+        }
+
+        rtc::IExtensionVideoFilter::ProcessResult ExtensionVideoFilter::pendVideoFrame(agora::agora_refptr<rtc::IVideoFrame> frame) {
+            if (!frame || !isInitOpenGL) {
+                return kBypass;
+            }
+
+            bool isAsyncMode = (mode_ == ProcessMode::kAsync);
+            if (isAsyncMode && YUVProcessor && control_ && invoker_id >= 0) {
+                threadPool_.PostTask(invoker_id, [videoFrame=frame, processor=YUVProcessor, control=control_] {
+                    rtc::VideoFrameData srcData;
+                    videoFrame->getVideoFrameData(srcData);
+                    processor->processFrame(srcData);
+                    // In asynchronous mode (mode is set to Async),
+                    // the plug-in needs to call this method to return the processed video frame to the SDK.
+                    control->deliverVideoFrame(videoFrame);
+                });
+                return kSuccess;
+            }
+            return kBypass;
+        }
+
+        rtc::IExtensionVideoFilter::ProcessResult ExtensionVideoFilter::adaptVideoFrame(agora::agora_refptr<rtc::IVideoFrame> src,
+                                                                               agora::agora_refptr<rtc::IVideoFrame>& dst) {
+            if (!isInitOpenGL) {
+                return kBypass;
+            }
+            bool isSyncMode = (mode_ == ProcessMode::kSync);
+            if (isSyncMode && YUVProcessor) {
+                rtc::VideoFrameData srcData;
+                src->getVideoFrameData(srcData);
+                YUVProcessor->processFrame(srcData);
+                dst = src;
+                return kSuccess;
+            }
+            return kBypass;
+        }
+
+        // When the app developer calls RtcEngine.setExtensionProperty,
+        // Agora SDK will call this method to set video plug-in properties
+        int ExtensionVideoFilter::setProperty(const char *key, const void *buf,
+                                                 size_t buf_size) {
+            printf("setProperty  %s  %s\n", key, buf);
+            std::string stringParameter((char*)buf);
+            YUVProcessor->setParameters(stringParameter);
+            return 0;
+        }
+
+        // When the app developer calls getExtensionProperty,
+        // Agora SDK will call this method to get the properties of the video plug-in
+        int ExtensionVideoFilter::getProperty(const char *key, void *buf, size_t buf_size) {
+            return -1;
+        }
+
+        void ExtensionVideoFilter::setEnabled(bool enable) {
+        }
+
+        bool ExtensionVideoFilter::isEnabled() {
+            return true;
+        }
+    }
+}