智慧课堂app（二）Flutter+springboot+netty 实现WebRtc1V1音视频实时通信

先看效果图

实现思路参考WEBRTC 的工作原理图

需要的工具和步骤

springboot
netty
coturn服务器
flutter
flutter-webrtc插件

1. 先在linux下搭建好coturn服务器

此步骤可以查找网上webrtc的coturn服务器搭建方法

2.更具原理图，编写netty的方法

后端关键代码

protected void channelRead0(ChannelHandlerContext channelHandlerContext, TextWebSocketFrame textWebSocketFrame) throws Exception {
System.out.println("收到消息: " + textWebSocketFrame.text());
Channel channel = channelHandlerContext.channel();
//////////////self////////
Client client_self = null;
for (Client client : clients) {
    if (client.getChannel().id() == channel.id()) {
        client_self = client;
    }
}
//////////////
JSONObject object = JSONObject.parseObject(textWebSocketFrame.text());
String type = object.getString("type");
System.out.println(type);
switch (type) {
    case "new": {//新成员加入
        String id = object.getString("id");
        String name = object.getString("name");
        String user_agent = object.getString("user_agent");
        Client client = new Client(id, name, user_agent, channel);
        if (!clients.contains(clients.add(client))) {
            clients.add(client);
        }
        Map msg = new HashMap<>();
        msg.put("type", "peers");
        msg.put("data", clients);
        sendMessageToAll(JSONObject.toJSONString(msg));
        //向客户端发送有成员进入房间
        break;
    }
    case "bye": {//离开房间
        Client del = null;
        for (Client ch : clients) {
            if (ch.getChannel().id() == channel.id()) {
                del = ch;
                Map data = ImmutableMap.of("from", ch.getId());
                Map msg = ImmutableMap.of("type", "bye", "data", data);
                sendMessageToAll(JSONObject.toJSONString(msg));
            }
        }
        if (del != null) {
            clients.remove(del);
        }
        break;
    }
    case "offer": {//转发offer
        Client peer = null;
        String peerId = object.getString("to");//toid对方的client
        for (Client client : clients) {
            if (peerId != null && client.getId().equals(peerId)) {
                peer = client;
            }
        }
        if (peer != null) {
            String peer_sessionId = object.getString("session_id");
            Map data = ImmutableMap.of("to",peerId,"from", client_self.getId(), "session_id", peer_sessionId,
                    "description", object.get("description"));
            Map msg = ImmutableMap.of("type", "offer", "data", data);
            sendMessageToUser(JSONObject.toJSONString(msg), peer.getChannel());
            //组织sessionId
            peer.setSessionId(peer_sessionId);
            client_self.setSessionId(peer_sessionId);
            Map sessions = ImmutableMap.of("id", peer_sessionId, "from", client_self.getId(), "to", peerId);
            session.add(sessions);
        }
        break;
    }
    case "answer": {//转发answer
        String to = object.getString("to");
        Object description = object.get("description");
        String session_id = object.getString("session_id");
        Map data = ImmutableMap.of("to",to,"from",client_self.getId(),"description",description);
        Map msg = ImmutableMap.of("type", "answer", "data", data);
        clients.forEach((client) -> {
            if (client.getId().equals(to) && client.getSessionId().equals(session_id)) {//
                sendMessageToUser(JSONObject.toJSONString(msg), client.getChannel());
            }
        });
        break;
    }
    case "candidate": {//收到候选者转发 candidate
        String to = object.getString("to");
        String clientId = client_self.getId();
        String session_id = object.getString("session_id");
        Object candidate = object.get("candidate");
        Map data = ImmutableMap.of("from", clientId, "to", to, "candidate", candidate);
        Map msg = ImmutableMap.of("type", "candidate", "data", data);
        clients.forEach((ch -> {//
            if (ch.getId().equals(to) && ch.getSessionId().equals(session_id)) {//
                sendMessageToUser(JSONObject.toJSONString(msg), ch.getChannel());
            }
        }));
        break;
    }
    case "keeplive": {//keeplive
        Map msg = ImmutableMap.of("type", "candidate", "data", "");
        sendMessageToUser(JSONObject.toJSONString(msg), channel);
        break;
    }

}

}


private void sendMessageToAll(String msg) {
    clients.forEach((ch) -> {
        ch.getChannel().writeAndFlush(new TextWebSocketFrame(msg));
    });
}

private void sendMessageToUser(String msg, Channel user) {
    user.writeAndFlush(new TextWebSocketFrame(msg));
}


//保存会话的session容器
private static Set clients = new HashSet<>();

Client实体类

@Data
@AllArgsConstructor
@NoArgsConstructor
public class Client implements Serializable {
    private String id;
    private String name;
    private String user_agent;
    private Channel channel;
    private String sessionId;

    public Client(String id, String name, String user_agent, Channel channel) {
        this.id = id;
        this.name = name;
        this.user_agent = user_agent;
        this.channel = channel;
    }
}

Flutter代码实现

 1.引入flutter-webrtc插件

import 'dart:convert';
import 'dart:io';

import 'package:flutter_webrtc/webrtc.dart';
import 'package:random_string/random_string.dart';
import 'package:web_socket_channel/io.dart';

import 'data_content.dart';

//信令状态的回调
typedef void SignalingStateCallback(SignalingState state);
//媒体流的状态回调
typedef void StreamStateCallback(MediaStream stream);
//对方进入房间的回调
typedef void OtherEventCallback(dynamic event);

//信令状态
enum SignalingState {
  CallStateNew, //新进入房间
  CallStateRinging,
  CallStateInvite,
  CallStateConnected, //连接
  CallStateBye, //离开
  CallStateOpen,
  CallStateClosed,
  CallStateError,
}

class RTCSignaling {
  final String _selfId = randomNumeric(6);
  IOWebSocketChannel _channel;
  String _sessionId; //会话id
  String url; //websocket url
  String display; //展示名称
  Map _peerConnections =
      Map();

  MediaStream _localStream;
  List _remoteStreams;

  SignalingStateCallback onStateChange;

  StreamStateCallback onLocalStream;
  StreamStateCallback onAddRemoteStream;
  StreamStateCallback onRemoveRemoteStream;

  OtherEventCallback onPeerUpdate;

  JsonDecoder _decoder = JsonDecoder();
  JsonEncoder _encoder = JsonEncoder();

  /**
   * turn stun 服务器的地址
   */
  Map _iceServers = {
    'iceServers': [
      {
        'urls': 'turn:ip:port',
        'username': 'coturn配置的账号',
        'credential': 'coturn配置的密码',
      },
      {
        'urls': 'stun:ip:port',
      },
    ]
  };

  /**
   * DTLS 是否开启， 一个传输安全的协议
   */
  final Map _config = {
    'mandatory': {},
    'optional': [
      {'DtlsSrtpKeyAgreement': true},
    ]
  };

  /**
   * 音视频约束 ，固定的
   */

  final Map _constraints = {
    'mandatory': {'OfferToReceiveAudio': true, 'OfferToReceiveVideo': true},
    'optional': []
  };

  RTCSignaling({this.url, this.display});

//socket 连接
  void connect() async {
    _channel = IOWebSocketChannel.connect(url);
    print('listen...');
    _channel.stream.listen((msg) {
      // print('收到的内容: $msg');
      onMessage(msg);
    }, onDone: () {
      print('closed by server');
    });

/*
  向信令服务发送注册消息
 */
    send('new', {
      'name': display,
      'id': _selfId,
      'user_agent': 'flutter-webrtc+${Platform.operatingSystem}'
    });
  }

//创建本地媒体流
  Future creasteStream() async {
    final Map mediaConstraints = {
      'audio': true,
      'video': {
        'mandatory': {
          'minWidth': '640',
          'minHeight': '480',
          'minFrameRate': '30',
        },
        'facingMode': 'user',
        'optional': [],
      },
    };
    //获取媒体流
    MediaStream stream = await navigator.getUserMedia(mediaConstraints);
    if (this.onLocalStream != null) {
      this.onLocalStream(stream);
    }
    return stream;
  }

//关闭本地媒体，断开socket
  void close() {
    if (_localStream != null) {
      _localStream.dispose();
      _localStream = null;
    }
    _peerConnections.forEach((key, pc) {
      pc.close();
    });

    if (_channel != null) {
      _channel.sink.close();
    }
  }

//切换前后摄像头
  void switchCamara() {
    _localStream?.getVideoTracks()[0].switchCamera();
  }

//邀请对方进行会话
  void invite(String peer_id) {
    this._sessionId = '$_selfId-$peer_id';
    this.onStateChange(SignalingState.CallStateNew);
    //创建一个peerconnection
    print('peer_id  :${peer_id}');
    _createPeerConnection(peer_id).then((pc) {
      _peerConnections[peer_id] = pc;
      _createOffer(peer_id, pc);
    });
  }

/*
收到消息处理逻辑
 */
  void onMessage(message) async {
    Map mapData = _decoder.convert(message);
    var data = mapData['data'];
    print(mapData['type']);
    switch (mapData['type']) {
      case 'peers':
        {
          //新成员加入刷新界面
          List peers = data;
          if (this.onPeerUpdate != null) {
            Map event = Map();
            event['self'] = _selfId;
            event['peers'] = peers;
            this.onPeerUpdate(event);
          }
          break;
        }
      case 'offer':
        {
          String id = data['from'];
          var description = data['description'];
          var sessionId = data['session_id'];
          _sessionId = sessionId;
          if (this.onStateChange != null)
            this.onStateChange(SignalingState.CallStateNew);
          /*
      收到offer后，创建本地的peerconnection
      之后设置远端的媒体信息，并向对端发送answer 进行应答
       */
          _createPeerConnection(id).then((pc) {
            _peerConnections[id] = pc;
            pc.setRemoteDescription(
                RTCSessionDescription(description['sdp'], description['type']));
            _createAnswer(id, pc);
          });
          break;
        }
      case 'answer':
        {
          //收到对端的answer
          String id = data['from'];
          Map description = data['description'];
          RTCPeerConnection pc = _peerConnections[id];
          print('crea ${description}');
          pc?.setRemoteDescription(
              RTCSessionDescription(description['sdp'], description['type']));
          break;
        }
      case 'candidate':
        {
          //收到对端后选者，并添加候选者
          String id = data['from'];
          Map candidateMap = data['candidate'];
          RTCPeerConnection pc = _peerConnections[id];
          RTCIceCandidate candidate = RTCIceCandidate(candidateMap['candidate'],
              candidateMap['sdpMid'], candidateMap['sdpMLineIndex']);
          pc?.addCandidate(candidate);
          break;
        }
      case 'bye':
        {
          //离开房间
          String id = data['from'];
          _localStream?.dispose();
          _localStream = null;

          RTCPeerConnection pc = _peerConnections[id];
          pc?.close();
          _peerConnections.remove(pc);
          _sessionId = null;
          if (this.onStateChange != null) {
            this.onStateChange(SignalingState.CallStateBye);
          }
          break;
        }
      case 'keepalive':
        {
          {
            print('收到心跳检查');
          }
          break;
        }
    }
  }

//结束会话
  void bye() {
    send('bye', {'session_id': _sessionId, 'from': _selfId});
  }

//创建PeerConnection
  Future _createPeerConnection(id) async {
    //获取本地媒体
    _localStream = await creasteStream();
    RTCPeerConnection pc = await createPeerConnection(_iceServers, _config);
    //本地媒体流赋值给peerconnection
    pc.addStream(_localStream);
    //获取候选者
    pc.onIceCandidate = (candidate) {
      send('candidate', {
        'to': id,
        'candidate': {
          'sdpMLineIndex': candidate.sdpMlineIndex,
          'sdpMid': candidate.sdpMid,
          'candidate': candidate.candidate,
        },
        'session_id': _sessionId
      });
    };
    //获取远端媒体流
    pc.onAddStream = (stream) {
      if (this.onAddRemoteStream != null) this.onAddRemoteStream(stream);
    };

    /**
     * 移除媒体流
     */
    pc.onRemoveStream = (stream) {
      if (this.onRemoveRemoteStream != null) this.onRemoveRemoteStream(stream);
      _remoteStreams.removeWhere((it) {
        return (it.id == stream.id);
      });
    };
    return pc;
  }

/*
   创建offer
  */
  void _createOffer(String id, RTCPeerConnection pc) async {
    RTCSessionDescription sdp = await pc.createOffer(_constraints);
    pc.setLocalDescription(sdp);
    //向对端发送自己的媒体信息（1V1) ，如果是1VN的话是向服务器发送，SFU
    send('offer', {
      'to': id,
      'description': {'sdp': sdp.sdp, 'type': sdp.type},
      'session_id': _sessionId
    });
  }

/*
  创建answer
 */
  void _createAnswer(String id, RTCPeerConnection pc) async {
    RTCSessionDescription sdp = await pc.createAnswer(_constraints);
    pc.setLocalDescription(sdp);
    /*
    发送answer
   */
    send('answer', {
      'to': id,
      'description': {'sdp': sdp.sdp, 'type': sdp.type},
      'session_id': _sessionId
    });
  }

/*
  消息发送
 */

  void send(event, data) {
    data['type'] = event;
    DataContent dataContent = DataContent(
        action: ActionEnum.VIDEO_1V1.index,
        subAction: ChatActionEnum.CONNECT.index,
        data: data);
    _channel?.sink.add(_encoder.convert(dataContent));
//    _channel?.sink.add(_encoder.convert(data));
//    print('${_encoder.convert(data)}');
  }
}

UI界面代码

import 'dart:io';

import 'package:course_app/utils/rtc_signaling.dart';
import 'package:flutter/material.dart';
import 'package:flutter_webrtc/webrtc.dart';

class P2PDemo extends StatefulWidget {
  final String url;

  P2PDemo({Key key, @required this.url}) : super(key: key);

  @override
  _P2PDemoState createState() => _P2PDemoState(serverUrl: url);
}

class _P2PDemoState extends State {
  final String serverUrl;

  _P2PDemoState({Key key, @required this.serverUrl});

  //信令对象
  RTCSignaling _signaling;

  //本地设备名称
  String _displayName =
      '${Platform.localeName.substring(0, 2)}+(${Platform.operatingSystem} )';

  //房间内的peer对象
  List _peers;

  var _selfId;

  //本地媒体窗口
  RTCVideoRenderer _localRenderer = RTCVideoRenderer();

  //对端媒体渲染对象
  RTCVideoRenderer _remoteRendered = RTCVideoRenderer();

  //是否处于通话状态
  bool _inCalling = false;

  @override
  void initState() {
    // TODO: implement initState
    super.initState();
    _initRenderers();
    _connect();
  }

  @override
  void dispose() {
    // TODO: implement dispose
    _hangup();
    super.dispose();
  }

  //懒加载本地和对端渲染窗口
  void _initRenderers() async {
    await _localRenderer.initialize();
    await _remoteRendered.initialize();
  }

  //连接socket
  void _connect() async {
    if (_signaling == null) {
      _signaling = RTCSignaling(url: serverUrl, display: _displayName);
    }
    //信令状态回调
    _signaling.onStateChange = (SignalingState state) {
      switch (state) {
        case SignalingState.CallStateNew:
          {
            setState(() {
              _inCalling = true;
            });
          }
          break;
        case SignalingState.CallStateRinging:
          // TODO: Handle this case.
          break;
        case SignalingState.CallStateInvite:
          // TODO: Handle this case.
          break;
        case SignalingState.CallStateConnected:
          // TODO: Handle this case.
          break;
        case SignalingState.CallStateBye:
          {
            setState(() {
              _localRenderer.srcObject = null;
              _remoteRendered.srcObject = null;
              _inCalling = false;
            });
          }
          break;
        case SignalingState.CallStateOpen:
          // TODO: Handle this case.
          break;
        case SignalingState.CallStateClosed:
          // TODO: Handle this case.
          break;
        case SignalingState.CallStateError:
          // TODO: Handle this case.
          break;
      }
    };
    //更新房间成员列表
    _signaling.onPeerUpdate = ((event) {
      setState(() {
        print(event);
        _selfId = event['self'];
        _peers = event['peers'];
      });
    });
    //设置本地媒体
    _signaling.onLocalStream = ((stream) {
      _localRenderer.srcObject = stream;
    });
    //设置远端媒体
    _signaling.onAddRemoteStream = ((stream) {
      _remoteRendered.srcObject = stream;
    });

    //socket进行连接
    _signaling.connect();
  }

  //邀请对方
  void _invitePeer(peerId) async {
    print('peerId: ${peerId}');
_signaling?.invite(peerId);
  }

  //挂断
  void _hangup() {
    _signaling?.bye();
  }

  //切换q前后摄像头
  void _switchCamera() {
    _signaling.switchCamara();
    _localRenderer.mirror = true;
  }

  Widget _buildRow(context, peer) {
    bool isSelf = (peer['id'] == _selfId);
    print(isSelf);
    return ListBody(
      children: [
        ListTile(
          title: Text(isSelf
              ? peer['name'] + 'self'
              : '${peer['name']} ${peer['user_agent']}'),
          trailing: SizedBox(
            width: 100,
            child: IconButton(
              icon: Icon(Icons.videocam),
              onPressed: () => _invitePeer(peer['id']),
            ),
          ),
        )
      ],
    );
  }

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      appBar: AppBar(
        title: Text('P2P demo'),
      ),
      floatingActionButton: _inCalling
          ? SizedBox(
              width: 200,
              child: Row(
                mainAxisAlignment: MainAxisAlignment.spaceBetween,
                children: [
                  FloatingActionButton(
                    onPressed: _switchCamera,
                    child: Icon(Icons.autorenew),
                  ),
                  FloatingActionButton(
                    onPressed: _hangup,
                    backgroundColor: Colors.deepOrange,
                    child: Icon(
                      Icons.call_end,
                      color: Colors.white,
                    ),

                  ),
                ],
              ),
            )
          : null,
      floatingActionButtonLocation: FloatingActionButtonLocation.centerFloat,
  body: _inCalling
      ? OrientationBuilder(builder: (context, orientation) {
          return Container(
            child: Stack(
              children: [
                Positioned(
                    left: 0,
                    right: 0,
                    top: 0,
                    bottom: 0,
                    child: Container(
                      margin: EdgeInsets.all(0),
                      width: MediaQuery.of(context).size.width,
                      height: MediaQuery.of(context).size.height,
                      child: RTCVideoView(_remoteRendered),
                      decoration: BoxDecoration(color: Colors.grey),
                    )),
                Positioned(
                  child: Container(
                    width:
                        orientation == Orientation.portrait ? 90.0 : 120.0,
                    height:
                        orientation == Orientation.portrait ? 120.0 : 90.0,
                    child: RTCVideoView(_localRenderer),
                    decoration: BoxDecoration(color: Colors.black54.withOpacity(0.5)),
                  ),
                  right: 20.0,
                  top: 20.0,
                ),
              ],
            ),
          );
        })
      : ListView.builder(
          shrinkWrap: true,
          padding: EdgeInsets.all(1),
          itemBuilder: (context, index) {
            return _buildRow(context, _peers[index]);
          },
          itemCount: (_peers != null) ? _peers.length : 0,
        ),
);
  }
}


最后代码改进的对方交给大家