探索iOS之多摄像头预览架构
在iOS13.0开始支持多摄像头预览AVCaptureMultiCamSession，然后iOS15.0增加支持摄像头画中画预览。在使用之前，我们通过isMultiCamSupported()判断是否支持多Camera同时预览。让我们先看下效果：
一、Camera架构
1、Camera流水线
Camera由AVCaptureDeviceInput、AVCaptureSession、AVCaptureOutput构成。如下图所示：
2、单Camera架构
单个Camera架构意味只有一个AVCaptureDeviceInput，同步输出VideoData和DepthData，支持预览和输出文件。如下图所示：
3、多Camera架构
与单Camera架构相比，多Camera架构包括多个输入源AVCaptureDeviceInput，多个摄像头同时预览，如下图所示：
二、Camera类图结构
Camera类图包括AVCaptureDeviceInput、AVCaptureMultiCamSession、AVCaptureVideoDataOutput、AVCaptureVideoPreviewLayer、AVAssetWriter。如下图所示：
三、Camera输入输出
Camera的输入包括：前置Camera、后置Camera、麦克风，输出包括：预览数据、图片、文件、Metadata，由AVCaptureMultiCamSession进行管理。如下图所示：
四、MultiCamera流同步
多个Camera同时预览，它们共享分辨率和帧率。也需要进行流同步，包括如下：
- 曝光
- 对焦
- 白平衡
五、Camera画中画预览
1、初始化
初始化阶段，主要设置预览图层、配置capture session，示例代码如下：
override func viewDidLoad() {
super.viewDidLoad()
// 设置前置、后置camera预览图层
backCameraVideoPreviewView.videoPreviewLayer.setSessionWithNoConnection(session)
frontCameraVideoPreviewView.videoPreviewLayer.setSessionWithNoConnection(session)
// 配置 capture session
sessionQueue.async {
self.configureSession()
}
}2、配置session
配置capture session的示例代码如下：
private func configureSession() {
guard setupResult == .success else { return }
guard AVCaptureMultiCamSession.isMultiCamSupported else {
print("MultiCam not supported on this device")
setupResult = .multiCamNotSupported
return
}
session.beginConfiguration()
defer {
session.commitConfiguration()
if setupResult == .success {
checkSystemCost()
}
}
guard configureBackCamera() else {
setupResult = .configurationFailed
return
}
guard configureFrontCamera() else {
setupResult = .configurationFailed
return
}
}3、配置后置Camera
配置流程包括：查找后置Camera、添加到session、连接输入设备到输出数据、连接输入设备到预览图层等，示例代码如下：
private func configureBackCamera() -> Bool {
session.beginConfiguration()
defer {
session.commitConfiguration()
}
// 查找后置camera
guard let backCamera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back) else {
print("Could not find the back camera")
return false
}
// 添加后置camera到session
do {
backCameraDeviceInput = try AVCaptureDeviceInput(device: backCamera)
guard let backCameraDeviceInput = backCameraDeviceInput,
session.canAddInput(backCameraDeviceInput) else {
return false
}
session.addInputWithNoConnections(backCameraDeviceInput)
} catch {
print("Could not create back camera device input: \(error)")
return false
}
// 查找后置camera输入视频端口
guard let backCameraDeviceInput = backCameraDeviceInput,
let backCameraVideoPort = backCameraDeviceInput.ports(for: .video,
sourceDeviceType: backCamera.deviceType,
sourceDevicePosition: backCamera.position).first else {
return false
}
// 添加后置camera输出视频数据
guard session.canAddOutput(backCameraVideoDataOutput) else {
print("Could not add the back camera video data output")
return false
}
session.addOutputWithNoConnections(backCameraVideoDataOutput)
backCameraVideoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)]
backCameraVideoDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)
// 连接后置camera输入到数据输出
let backCameraVideoDataOutputConnection = AVCaptureConnection(inputPorts: [backCameraVideoPort],
output: backCameraVideoDataOutput)
guard session.canAddConnection(backCameraVideoDataOutputConnection) else {
print("Could not add a connection to the back camera video data output")
return false
}
session.addConnection(backCameraVideoDataOutputConnection)
backCameraVideoDataOutputConnection.videoOrientation = .portrait
// 连接后置camera输入到预览图层
guard let backCameraVideoPreviewLayer = backCameraVideoPreviewLayer else {
return false
}
let backCameraVideoPreviewLayerConnection = AVCaptureConnection(inputPort: backCameraVideoPort, videoPreviewLayer: backCameraVideoPreviewLayer)
guard session.canAddConnection(backCameraVideoPreviewLayerConnection) else {
print("Could not add a connection to the back camera video preview layer")
return false
}
session.addConnection(backCameraVideoPreviewLayerConnection)
return true
}4、配置前置Camera
前置Camera与后置的配置流程类似，只是把back换成front。另外，前置Camera开启镜像。示例代码如下：
private func configureFrontCamera() -> Bool {
session.beginConfiguration()
defer {
session.commitConfiguration()
}
// 查找前置camera
guard let frontCamera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front) else {
print("Could not find the front camera")
return false
}
// 添加前置camera到session
do {
frontCameraDeviceInput = try AVCaptureDeviceInput(device: frontCamera)
guard let frontCameraDeviceInput = frontCameraDeviceInput,
session.canAddInput(frontCameraDeviceInput) else {
return false
}
session.addInputWithNoConnections(frontCameraDeviceInput)
} catch {
print("Could not create front camera device input: \(error)")
return false
}
// 查找前置camera输入视频端口
guard let frontCameraDeviceInput = frontCameraDeviceInput,
let frontCameraVideoPort = frontCameraDeviceInput.ports(for: .video,
sourceDeviceType: frontCamera.deviceType,
sourceDevicePosition: frontCamera.position).first else {
return false
}
// 添加前置camera输出视频数据
guard session.canAddOutput(frontCameraVideoDataOutput) else {
print("Could not add the front camera video data output")
return false
}
session.addOutputWithNoConnections(frontCameraVideoDataOutput)
frontCameraVideoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)]
frontCameraVideoDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)
// 连接前置camera输入到数据输出
let frontCameraVideoDataOutputConnection = AVCaptureConnection(inputPorts: [frontCameraVideoPort],
output: frontCameraVideoDataOutput)
guard session.canAddConnection(frontCameraVideoDataOutputConnection) else {
print("Could not add a connection to the front camera video data output")
return false
}
session.addConnection(frontCameraVideoDataOutputConnection)
frontCameraVideoDataOutputConnection.videoOrientation = .portrait
// 连接前置camera输入到预览图层
guard let frontCameraVideoPreviewLayer = frontCameraVideoPreviewLayer else {
return false
}
let frontCameraVideoPreviewLayerConnection = AVCaptureConnection(inputPort: frontCameraVideoPort, videoPreviewLayer: frontCameraVideoPreviewLayer)
guard session.canAddConnection(frontCameraVideoPreviewLayerConnection) else {
print("Could not add a connection to the front camera video preview layer")
return false
}
session.addConnection(frontCameraVideoPreviewLayerConnection)
// 前置camera开启镜像
frontCameraVideoPreviewLayerConnection.isVideoMirrored = true
frontCameraVideoPreviewLayerConnection.automaticallyAdjustsVideoMirroring = false
return true
}5、配置双向麦克风
除了提供画中画Camera，还提供前后双向麦克风。示例代码如下：
private func configureMicrophone() -> Bool {
session.beginConfiguration()
defer {
session.commitConfiguration()
}
// 查找麦克风
guard let microphone = AVCaptureDevice.default(for: .audio) else {
print("Could not find the microphone")
return false
}
// 添加麦克风到session
do {
microphoneDeviceInput = try AVCaptureDeviceInput(device: microphone)
guard let microphoneDeviceInput = microphoneDeviceInput,
session.canAddInput(microphoneDeviceInput) else {
return false
}
session.addInputWithNoConnections(microphoneDeviceInput)
} catch {
print("Could not create microphone input: \(error)")
return false
}
// 查找输入设备的后置音频端口
guard let microphoneDeviceInput = microphoneDeviceInput,
let backMicrophonePort = microphoneDeviceInput.ports(for: .audio,
sourceDeviceType: microphone.deviceType,
sourceDevicePosition: .back).first else {
return false
}
// 查找输入设备的前置音频端口
guard let frontMicrophonePort = microphoneDeviceInput.ports(for: .audio,
sourceDeviceType: microphone.deviceType,
sourceDevicePosition: .front).first else {
return false
}
// 添加后置麦克风到输出数据
guard session.canAddOutput(backMicrophoneAudioDataOutput) else {
print("Could not add the back microphone audio data output")
return false
}
session.addOutputWithNoConnections(backMicrophoneAudioDataOutput)
backMicrophoneAudioDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)
// 添加前置麦克风到输出数据
guard session.canAddOutput(frontMicrophoneAudioDataOutput) else {
print("Could not add the front microphone audio data output")
return false
}
session.addOutputWithNoConnections(frontMicrophoneAudioDataOutput)
frontMicrophoneAudioDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue)
// 连接后置麦克风到输出数据
let backMicrophoneAudioDataOutputConnection = AVCaptureConnection(inputPorts: [backMicrophonePort],
output: backMicrophoneAudioDataOutput)
guard session.canAddConnection(backMicrophoneAudioDataOutputConnection) else {
print("Could not add a connection to the back microphone audio data output")
return false
}
session.addConnection(backMicrophoneAudioDataOutputConnection)
// 连接前置麦克风到输出数据
let frontMicrophoneAudioDataOutputConnection = AVCaptureConnection(inputPorts: [frontMicrophonePort],
output: frontMicrophoneAudioDataOutput)
guard session.canAddConnection(frontMicrophoneAudioDataOutputConnection) else {
print("Could not add a connection to the front microphone audio data output")
return false
}
session.addConnection(frontMicrophoneAudioDataOutputConnection)
return true
}六、降低功耗
iOS提供API获取硬件功耗：
var hardwareCost: Float { get } // 取值[0.0, 1.0]同时提供API获取系统压力功耗：
var systemPressureCost: Float { get } // 取值[0.0, 1.0]关于降低功耗的可行方案如下：
- 设置最大帧率
- 降低Camera分辨率
- 选择低精度像素格式