Unreal EngineとiPhoneで始めるリアルタイムAR配信

NDIWrapper.m
- (void)send:(CMSampleBufferRef)sampleBuffer
width:(int)width height:(int)height
metadata:(const char *)metadata {
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
NDIlib_video_frame_v2_t video_frame;
video_frame.frame_rate_N = 30000;
video_frame.frame_rate_D = 1000;
video_frame.xres = width;
video_frame.yres = height;
video_frame.FourCC = NDIlib_FourCC_type_BGRA;
video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
video_frame.picture_aspect_ratio = (float)width / (float)height;
video_frame.line_stride_in_bytes = video_frame.xres * 4;
video_frame.p_metadata = metadata;
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
}

144.

[beta]

NDIWrapper.m
- (void)send:(CMSampleBufferRef)sampleBuffer
width:(int)width height:(int)height
metadata:(const char *)metadata {
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
NDIlib_video_frame_v2_t video_frame;
video_frame.frame_rate_N = 30000;
video_frame.frame_rate_D = 1000;
video_frame.xres = width;
video_frame.yres = height;
video_frame.FourCC = NDIlib_FourCC_type_BGRA;
video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
video_frame.picture_aspect_ratio = (float)width / (float)height;
video_frame.line_stride_in_bytes = video_frame.xres * 4;
video_frame.p_metadata = metadata;
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
}

145.

[beta]

NDIWrapper.m
- (void)send:(CMSampleBufferRef)sampleBuffer
width:(int)width height:(int)height
metadata:(const char *)metadata {
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
NDIlib_video_frame_v2_t video_frame;
video_frame.frame_rate_N = 30000;
video_frame.frame_rate_D = 1000;
video_frame.xres = width;
video_frame.yres = height;
video_frame.FourCC = NDIlib_FourCC_type_BGRA;
video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
video_frame.picture_aspect_ratio = (float)width / (float)height;
video_frame.line_stride_in_bytes = video_frame.xres * 4;
video_frame.p_metadata = metadata;
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
}

146.

[beta]

NDIWrapper.m
- (void)send:(CMSampleBufferRef)sampleBuffer
width:(int)width height:(int)height
metadata:(const char *)metadata {
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
NDIlib_video_frame_v2_t video_frame;
video_frame.frame_rate_N = 30000;
video_frame.frame_rate_D = 1000;
video_frame.xres = width;
video_frame.yres = height;
video_frame.FourCC = NDIlib_FourCC_type_BGRA;
video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
video_frame.picture_aspect_ratio = (float)width / (float)height;
video_frame.line_stride_in_bytes = video_frame.xres * 4;
video_frame.p_metadata = metadata;
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
}

147.

[beta]

NDIWrapper.m
- (void)send:(CMSampleBufferRef)sampleBuffer
width:(int)width height:(int)height
metadata:(const char *)metadata {
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
NDIlib_video_frame_v2_t video_frame;
video_frame.frame_rate_N = 30000;
video_frame.frame_rate_D = 1000;
video_frame.xres = width;
video_frame.yres = height;
video_frame.FourCC = NDIlib_FourCC_type_BGRA;
video_frame.frame_format_type = NDIlib_frame_format_type_progressive;
video_frame.picture_aspect_ratio = (float)width / (float)height;
video_frame.line_stride_in_bytes = video_frame.xres * 4;
video_frame.p_metadata = metadata;
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
video_frame.p_data = CVPixelBufferGetBaseAddress(imageBuffer);
CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
NDIlib_send_send_video_async_v2(my_ndi_send, &video_frame);
}

148.

149.

ViewController.swift import CoreMotion private let motionManager = CMMotionManager() private var baseAttitude: CMAttitude? override func viewDidLoad() { super.viewDidLoad() ... motionManager.deviceMotionUpdateInterval = 1.0 / 60.0 motionManager.startDeviceMotionUpdates(using: .xArbitraryCorrectedZVertical) }

150.

ViewController.swift import CoreMotion private let motionManager = CMMotionManager() private var baseAttitude: CMAttitude? override func viewDidLoad() { super.viewDidLoad() ... motionManager.deviceMotionUpdateInterval = 1.0 / 60.0 motionManager.startDeviceMotionUpdates(using: .xArbitraryCorrectedZVertical) }

151.

[beta]

ViewController.swift

private func makeMetaData() -> String {
guard let deviceMotion = motionManager.deviceMotion else { return "" }
let attitude = deviceMotion.attitude
if let baseAttitude = baseAttitude {
attitude.multiply(byInverseOf: baseAttitude)
}
let quat = attitude.quaternion
let arguments = [Float(quat.x), Float(quat.y), Float(quat.z), Float(quat.w)]
return arguments.map { String(format: "%.6f", $0) }.joined(separator: ",")
}

152.

[beta]

ViewController.swift

private func makeMetaData() -> String {
guard let deviceMotion = motionManager.deviceMotion else { return "" }
let attitude = deviceMotion.attitude
if let baseAttitude = baseAttitude {
attitude.multiply(byInverseOf: baseAttitude)
}
let quat = attitude.quaternion
let arguments = [Float(quat.x), Float(quat.y), Float(quat.z), Float(quat.w)]
return arguments.map { String(format: "%.6f", $0) }.joined(separator: ",")
}

153.

なぜオイラー角ではなくクォータニオンを送るのか？

154.

→ ジンバルロックを回避するため

155.

当初はFreeDでオイラー角を送っていたしかしジンバルロック発生のためクォータニオンを送るよう変更

156.

ジンバルロック "3軸の全てに自由な運動がある場合は、機体の回転によって3つのジンバルリングのうち2つの軸が同一平面上にそろってしまうジンバルロックという現象が発生しうる。発生すると、本来3あるはずの自由度が2になってしまう。" https://ja.wikipedia.org/wiki/%E3%82%B8%E3%83%B3%E3%83%90%E3%83%AB

https://ja.wikipedia.org/wiki/ジンバル

157.

Euler (gimbal lock) Explained https://www.youtube.com/watch?v=zc8b2Jo7mno&t=58s

https://www.youtube.com/watch?v=zc8b2Jo7mno&t=58s

158.

159.

クォータニオン4：クォータニオンの演算より https://www.sports-sensing.com/brands/labss/motionmeasurement/motion̲biomechanics/quaternion04.html

160.

ViewController.swift extension ViewController : AVCaptureVideoDataOutputSampleBufferDelegate { func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let ndiWrapper = self.ndiWrapper, isSending else { return } ndiWrapper.send(sampleBuffer, width: 1920, height: 1080, metadata: makeMetaData()) } }

161.

ViewController.swift extension ViewController : AVCaptureVideoDataOutputSampleBufferDelegate { func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { guard let ndiWrapper = self.ndiWrapper, isSending else { return } ndiWrapper.send(sampleBuffer, width: 1920, height: 1080, metadata: makeMetaData()) } }

162.

163.

NDI Plugin インストール NDI Plugin 有効化 NDI Media Receiver作成 BluePrint作成 iPhoneアプリ実装カメラへ映像を表示カメラに動作を反映

164.

NDI Plugin インストール NDI Plugin 有効化 NDI Media Receiver作成 BluePrint作成 iPhoneアプリ実装カメラへ映像を表示カメラに動作を反映

165.

現実世界のカメラ情報をCineCameraActorに設定

166.

iPhone 13 Proのカメラ情報をCineCameraActorに設定

167.

iPhone 13 Pro 広角カメラのカメラ情報は？

168.

注: 正直、試行錯誤中です

169.

知りたいのは Sensor Width Focal Length

170.

Sensor Width Apple公式情報なし調査すると「Sony IMX703」が使用されているようだサイズは 1/1.66 型横幅情報が見つからないため 1/1.7 型のサイズ 7.5 5.6 mm を参照

171.

Focal Length https://www.youtube.com/watch?v=EvGOlAkLSLw "26mm focal length" を35mm換算すると5.7

https://www.youtube.com/watch?v=EvGOlAkLSLw

172.

iPhone 13 Pro のField of view func session(_ session: ARSession, didUpdate frame: ARFrame) { let a = Float(frame.camera.imageResolution.width) let b = 2 * frame.camera.intrinsics[0,0] let horizontalFov = 2 * atan(a / b) * 180 / Float.pi print(horizontalFov) } ... 70.362946 70.36179 70.359474 70.35788 70.35992 70.36179 70.36122 ...

173.

動作させながら微調整した暫定値

174.

175.

176.

一旦、平面に映像を表示してみる

177.

Planeを追加し位置とサイズ調整

178.

179.

180.

181.