android可见即可说实现方案
- 依赖于科大讯飞的asr识别能力，使用Android无障碍服务获取页面文本作为热词，注册到讯飞api，注册过后语音识别到热词的asr返回,利用WindowManager和无障碍的点击实现可见即可说功能
## 无障碍服务获取需要注册的热词
```
package com..model;
import android.accessibilityservice.AccessibilityService;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.ServiceConnection;
import android.graphics.Rect;
import android.os.IBinder;
import android.util.Log;
import android.view.accessibility.AccessibilityEvent;
import android.view.accessibility.AccessibilityNodeInfo;
import androidx.lifecycle.Observer;
import .HotWordsBean;
import .GsonUtil;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class MyAccessibilityService extends AccessibilityService implements IAccessibilityHotWord {
private String TAG = MyAccessibilityService.class.getSimpleName();
private StringBuilder hotWords = new StringBuilder();
private VrSpService vrSpeechService;
private HotWordsBean hotWordsBean = new HotWordsBean();
private AccessibilityNodeInfo rootInActiveWindow;
private Set accessibilityNodeInfoSet = new HashSet<>();
HotWordsBean.UserDataBean userDataBean = new HotWordsBean.UserDataBean();
HotWordsBean.UserDataBean.CMD cmd = new HotWordsBean.UserDataBean.CMD();
private String hotWordJsonString = "";
private Context context;
@Override
public void onCreate() {
super.onCreate();
Log.i(TAG, "------------ super.onCreate --------------------: ");
bindVrSpeechService();
context = this;
HotWordReceiver.hotWordLiveData.observeForever( new Observer() {
@Override
public void onChanged(String hotWord) {
Log.d(TAG, "onChanged: ytf ------------ hotWord change :" + hotWord);
if (accessibilityNodeInfoSet.size() > 0) {
for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {
if (nodeInfo.getText() != null && nodeInfo.getText().toString().equalsIgnoreCase(hotWord)) {
Log.d(TAG, "ytf, hotWord Shot:" + hotWord);
handlePerformAction(hotWord);
}
}
// 通过无障碍服务设置seekbar 进度值
if ("android.widget.SeekBar".equals(nodeInfo.getClassName())) {
// AccessibilityAction: ACTION_SET_PROGRESS - null
// mStateDescription 76%
Bundle arguments = new Bundle();
arguments.putFloat(AccessibilityNodeInfo.ACTION_ARGUMENT_PROGRESS_VALUE, 50.0f);
nodeInfo.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_PROGRESS.getId(), arguments);
Log.d(TAG, "onChanged: ytf,seekBar " + nodeInfo);
// 反射获取seekbar当前进度progress
// readAttributeValue(nodeInfo);
try {
@SuppressLint("BlockedPrivateApi") Field field = nodeInfo.getClass().getDeclaredField("mStateDescription");
//设置对象的访问权限，保证对private的属性的访问
field.setAccessible(true);
Log.d(TAG, "onChanged: ytf,stateDescription = " + field.get(nodeInfo));
} catch (Exception e) {
Log.e(TAG, "onChanged: ytf,========== " + e.toString());
}
}
} else {
Log.d(TAG, "ytf hotWord Shot: accessibilityNodeInfoSet size = " + accessibilityNodeInfoSet.size());
}
}
});
}
@Override
public void onAccessibilityEvent(AccessibilityEvent accessibilityEvent) {
String packageName = accessibilityEvent.getPackageName() == null ? "" : accessibilityEvent.getPackageName().toString();
if (!"com.saicmotor.settings".equals(packageName)) {
return;
}
int eventType = accessibilityEvent.getEventType();
// Log.d(TAG, "ytf,onAccessibilityEvent [eventType: " + eventType + "], [ packageName: " + packageName + "]");
switch (eventType) {
// case AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED:
case AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED:
case AccessibilityEvent.TYPE_VIEW_CLICKED:
accessibilityNodeInfoSet.clear();
hotWords.setLength(0);
rootInActiveWindow = getRootInActiveWindow();
if (null == rootInActiveWindow) {
Log.d(TAG, "ytf, onAccessibilityEvent: rootInActiveWindow == null");
return;
} else {
recycle(rootInActiveWindow);
}
String[] splitHotWords = hotWords.toString().split(",");
//vrSpeechService.notifyHotWordLoad(GsonUtil.HOT_WORD_TEST_1);
hotWordsBean.setHotWords(splitHotWords);
cmd.setActiveStatus("");
userDataBean.setCmd(cmd);
hotWordsBean.setUserData(userDataBean);
try {
hotWordJsonString = GsonUtil.getInstance().getGson().toJson(hotWordsBean);
if (hotWordJsonString != null) {
vrSpeechService.notifyHotWordLoad(hotWordJsonString);
Log.d(TAG, "onAccessibilityEvent: ytf, hotWordJsonString = " + hotWordJsonString);
}
} catch (Exception e) {
Log.e(TAG, "onAccessibilityEvent: ytf, e: " + e.toString());
}
break;
default:
break;
}
}
private void recycle(AccessibilityNodeInfo info) {
if (info.getChildCount() == 0) {
if ("android.widget.SeekBar".equals(info.getClassName())) {
// Class aClass = info.getClass();
// Log.d(TAG, "recycle: ytf aClass = " + aClass.getSimpleName());
// SeekBar seekBar = (SeekBar) info.getClassName();
// Log.d(TAG, "recycle: ytf, SeekBar 调节前：" + seekBar.getProgress() + ", getViewIdResourceName:" + info.getViewIdResourceName());
// Bundle bundle = new Bundle();
// bundle.putFloat(AccessibilityNodeInfo.ACTION_ARGUMENT_PROGRESS_VALUE, 50.0f);
// seekBar.performAccessibilityAction(R.id.accessibilityActionSetProgress, bundle);
// Log.d(TAG, "recycle: ytf, SeekBar 调节后：" + seekBar.getProgress() + ", getViewIdResourceName:" + info.getViewIdResourceName());
} else if ("android.widget.ScrollView".contentEquals(info.getClassName())) {
// Log.d(TAG, "recycle: ytf, android.widget.ScrollView ACTION_SCROLL_FORWARD");
// info.performAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD);
}
// Log.i(TAG, "recycle ytf, [ClassName: " + info.getClassName() + "], [Text: " + info.getText() + "], [resId: " + info.getViewIdResourceName() + "]");
if (null != info.getText()) {
String text = info.getText().toString();
hotWords.append(text + ",");
accessibilityNodeInfoSet.add(info);
}
} else {
for (int i = 0; i < info.getChildCount(); i++) {
if (info.getChild(i) != null) {
// Log.d(TAG, "ytf 容器: [" + info.getClassName() + "], [resId:" + info.getViewIdResourceName() + "]");
recycle(info.getChild(i));
}
}
}
}
private void handlePerformAction(String targetHotWord) {
Log.d(TAG, "handlePerformAction: ytf, accessibilityNodeInfoSet.size = " + accessibilityNodeInfoSet.size());
for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {
// Log.d(TAG, "ytf handlePerformAction: nodeInfo.getText().toString() = " + nodeInfo.getText().toString() + ", targetHotWord = " + targetHotWord);
if (nodeInfo.getText().toString().equalsIgnoreCase(targetHotWord)) {
Log.d(TAG, "ytf, 命中可见即可说 handlePerformAction: " + nodeInfo.getText().toString());
forceClick(nodeInfo);
// nodeInfo.performAction(AccessibilityNodeInfo.ACTION_CLICK);
}
}
}
private void forceClick(AccessibilityNodeInfo nodeInfo) {
Log.d(TAG, "forceClick: ytf,------------");
try {
Rect rect = new Rect();
nodeInfo.getBoundsInScreen(rect);
Log.d(TAG, "ytf, forceClick: " + rect.left + " " + rect.top + " " + rect.right + " " + rect.bottom);
int x = (rect.left + rect.right) / 2;
int y = (rect.top + rect.bottom) / 2;
String cmd = "input tap " + String.valueOf(x) + " " + String.valueOf(y);
ProcessBuilder builder = new ProcessBuilder();
String[] order = {
"input",
"tap",
String.valueOf(x),
String.valueOf(y)
};
try {
builder.command(order).start();
Log.d(TAG, "ytf, forceClick: [ " + x + ", " + y + "]");
} catch (IOException e) {
Log.d(TAG, "ytf, forceClick: error: " + e.toString());
}
} catch (Exception e) {
Log.e(TAG, "ytf,error forceClick: " + e.toString());
}
}
@Override
public void onInterrupt() {
Log.i(TAG, "ytf, onInterrupt");
}
private void bindVrSpeechService() {
Intent intent = new Intent(getApplicationContext(), VrSpeechService.class);
boolean result = bindService(intent, serviceConnection, Context.BIND_AUTO_CREATE);
Log.d(TAG, "vrSpeechService: ytf bind result:" + result);
}
private ServiceConnection serviceConnection = new ServiceConnection() {
@Override
public void onServiceConnected(ComponentName componentName, IBinder iBinder) {
try {
vrSpeechService = ((VrSpeechService.LocalBinder) iBinder).getService();
Log.d(TAG, "onServiceConnected: ytf, vrSpeechService bind ");
vrSpeechService.setAccessibilityHotWord((IAccessibilityHotWord) context);
} catch (Exception e) {
e.printStackTrace();
Log.e(TAG, "ytf, onServiceConnected: " + e.toString() );
}
}
@Override
public void onServiceDisconnected(ComponentName componentName) {
}
};
@Override
public void hotWordShot(String hotWord) {
// Log.d(TAG, "ytf, hotWordShot: " + hotWord);
if (accessibilityNodeInfoSet.size() > 0) {
for (AccessibilityNodeInfo nodeInfo : accessibilityNodeInfoSet) {
if (nodeInfo.getText().toString().equalsIgnoreCase(hotWord)) {
Log.d(TAG, "ytf, hotWordShot:" + hotWord);
handlePerformAction(hotWord);
}
}
} else {
Log.d(TAG, "ytf hotWordShot: accessibilityNodeInfoSet size = " + accessibilityNodeInfoSet.size());
}
}
}
```
-
清单文件：
-
@xml/accessibility
-
模拟asr热词命中
/**
* @Author yangtianfu
* @Date 2023/9/15 13:05
* @Describe 监听热词回传执行可见可说
* adb shell am broadcast -a com.saicmotor.voiceservice.hotword -n com.saicmotor.voiceservice/.model.HotWordReceiver --es hotWord “sound”
*/
public class HotWordReceiver extends BroadcastReceiver {
private static final String TAG = “HotWordReceiver”;
private final String ACTION_HOT_WORD_RECEIVER = “com.saicmotor.voiceservice.hotword”;
// private IAccessibilityHotWord iAccessibilityHotWord;
public static MutableLiveData hotWordLiveData = new MutableLiveData<>();@Override public void onReceive(Context context, Intent intent) { String action = intent.getAction(); Log.d(TAG, "ytf, onReceive: intent,action = " + action); if (ACTION_HOT_WORD_RECEIVER.equals(action)) { String hotWord = intent.getStringExtra("hotWord"); Log.d(TAG, "ytf, onReceive: com.saicmotor.voiceservice.hotword :" + hotWord); hotWordLiveData.postValue(hotWord); } }}
## 科大讯飞注册热词
热词格式：
public static final String HOT_WORD_TEST_1 = "{\n" +
" \"HotWords\":[\"High\"],\n" +
" \"UserData\":{\n" +
" \"cmd\":{\n" +
" \"activeStatus\":\"bg\",\n" +
" \"data\":{\n" +
"\n" +
" },\n" +
" \"sceneStatus\":\"default\"\n" +
" }\n" +
" }\n" +
"}";
int result = libisssr.uploadData(hotWords, 2);
## VuiService.java语音Vui服务（带有语音形象的app）
透明activity无法跨应用实现点击穿透效果，会导致可见即可说点击无效果，需要在service中使用WindowManager的addview方法，把语音的app作为view添加到WindowManager中,这样就可以实现语音app全透明状态下识别到asr之后可以利用Android无障碍服务去点击指定位置或者指定控件。
public class VoiceVuiService extends Service {
private WindowManager.LayoutParams mParams;
private WindowManager mWindowManager;
private ImageView voiceImage;
private TextView textView;
@Override
public void onCreate() {
mParams = new WindowManager.LayoutParams();
//设置type.系统提示型窗口，一般都在应用程序窗口之上.
mParams.type = WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY;
// mParams.type = WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY;
//设置效果为背景透明.
// mParams.format = PixelFormat.RGBA_8888;
mParams.format = PixelFormat.TRANSLUCENT;
//设置flags.不可聚焦及不可使用按钮对悬浮窗进行操控.
// mParams.flags = WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE | WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE |
// WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL | WindowManager.LayoutParams.FIRST_SYSTEM_WINDOW;
mParams.flags = WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE | WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE;
mParams.alpha = 0.8f;
mWindowManager = (WindowManager) getSystemService(Context.WINDOW_SERVICE);
DisplayMetrics dm = new DisplayMetrics();
mWindowManager.getDefaultDisplay().getMetrics(dm);
mParams.width = 136;
mParams.height = 136;
// mParams.x = 100;
// mParams.y = 100;
voiceImage = new ImageView(this);
voiceImage.setBackground(getResources().getDrawable(R.mipmap.assistant100025));
textView = new TextView(this);
textView.setTextSize(36);
textView.setTextColor(Color.RED);
textView.setText("语音形象");
// vrView = new HSPortraitVrViewForA11V(getApplicationContext());
// vrView.startFlipping();
Log.d(TAG, "initView: ytf, dm.widthPixels = " + dm.widthPixels + ",mParams.height = " + mParams.height);
mParams.gravity = Gravity.TOP;
LogUtils.i(TAG, "onCreate.....");
super.onCreate();
-----------------------
@Override
public int onStartCommand(Intent intent, int flags, int startId) {
LogUtils.i(TAG, "onStartCommand,LteService onStartCommand");
// Android 8以上特殊处理
setNotificationChannel();
// mWindowManager.addView(voiceImage, mParams);
// mWindowManager.addView(textView, mParams);
// if(CommonUtils.isUseHalfServiceMode()){
// if(offlineAgentService == null){
// bindHalfEngineService();
// }
// }
return START_STICKY;
}
-------------------------------
@Override
public void onDestroy() {
// if (voiceImage.getParent() != null) {
// mWindowManager.removeView(voiceImage);
// }
// if (textView.getParent() != null) {
// mWindowManager.removeView(textView);
// }
super.onDestroy();
}
private static void readAttributeValue(Object obj) {
String nameVlues = "";
//得到class
Class cls = obj.getClass();
//得到所有属性
Field[] fields = cls.getDeclaredFields();
for (int i = 0; i < fields.length; i++) {//遍历
try {
//得到属性
Field field = fields[i];
//打开私有访问
field.setAccessible(true);
//获取属性
String name = field.getName();
//获取属性值
Object value = field.get(obj);
//一个个赋值
nameVlues += field.getName() + ":" + value + ",";
} catch (IllegalAccessException e) {
e.printStackTrace();
}
}
//获取最后一个逗号的位置
int lastIndex = nameVlues.lastIndexOf(",");
//不要最后一个逗号","
String result = nameVlues.substring(0, lastIndex);
System.out.println("ytf, 反射获取：" + result);
}