互动油画——Processing和Kinect联合实现

项目总结

作为大二上学期的合作项目，总结一下制作过程和代码
我们完成了一个模拟笔刷画油画的效果，并最终用投影的形式表现出来。其中笔刷的频率会根据人的位置（Kinect捕捉）和音乐的频率变化。
我主要参与了创意策划和负责代码中笔刷的优化。
主要过程是，先找了一些现成的代码，按照代码分析功能，学习Processing，然后不断修改得到了我们想要的效果。
在此过程中，遗憾的是也有一些效果无法实现，部分是由于Processing的自身限制。所以想要达到更好的效果，还需要配合其他软件，如TouchDesigner。

主要参考网站：
有很多开源的项目 https://www.openprocessing.org/
p5.js的官方指南 https://p5js.org/reference/

效果图

代码部分

说明：

使用processing3.5.3直接打开，需要安装minim和Kinect库

结构描述：
读取图片（放在同一个文件夹下，需要名字和String[] imgNames里面对应）-> 启动Kinect/播放音乐 -> Kinect获取脸部坐标数据/获取音乐音量 -> 根据脸部坐标数据切换底层图片/根据音量大小确定笔触大小 -> 根据笔触算法（paintStroke函数）随机在屏幕上生成笔触 -> 等待画面逐渐完成

代码如下：

String[] imgNames = {"portrait_1.png", "portrait_2.png", "portrait_3.png"};
PImage img;
int imgIndex = 1;

int headX;
int headY;
int headZ;
int headXdis;

int headXmax = 900;
int headXmin = 900;
int headYmax = 500;
int headYmin = 500;
int time = 0;
int preTime = 0;

import ddf.minim.*;
import ddf.minim.effects.*;

Minim minim;
AudioPlayer air;

import KinectPV2.KJoint;
import KinectPV2.*;
KinectPV2 kinect;

void nextImage() {
  loop();
  frameCount = 0;
  
  img = loadImage(imgNames[imgIndex]);
  //img.resize(width / 2 / 4 * 3, height / 4 * 3);
  //img.resize(width, height);
  img.loadPixels();
  
  imgIndex += 1;
  if (imgIndex >= imgNames.length) {
    imgIndex = 0;
  }
}


void paintStroke(float strokeLength, color strokeColor, int strokeThickness) {
  float stepLength = strokeLength/4.0;
  
  // Determines if the stroke is curved. A straight line is 0.
  float tangent1 = 0;
  float tangent2 = 0;
  
  float odds = random(1.0);
  
  if (odds < 0.9) {
    //tangent1 = random(-hue(strokeColor), hue(strokeColor));
    //tangent2 = random(-hue(strokeColor), hue(strokeColor));
    tangent1 = random(-1, 1);
    tangent2 = random(-1, 1);
  } 
  
  // Draw a big stroke
  noFill();
  stroke(strokeColor);
  //strokeWeight(strokeThickness);
  //bezier(tangent1, -stepLength*2, 0, -stepLength, 0, stepLength, tangent2, stepLength*2);
  
  int z = 1;
  
  // Draw stroke's details
  for (int num = strokeThickness; num > 0; num --) {
    float offset = random(-10, 10);
    color newColor = color(hue(strokeColor)+offset, saturation(strokeColor)+offset, brightness(strokeColor)+offset, random(50, 150));
    
    stroke(newColor);
    strokeWeight((int)random(0, 10));
    float curveT=random(-2, 2);
    curveTightness(curveT);
    curve(tangent1, -stepLength*2, z-strokeThickness/2, -stepLength*random(0.9, 1.1), z-strokeThickness/2, stepLength*random(0.9, 1.1), tangent2, stepLength*2);
    //bezier(tangent1, -stepLength*2, z-strokeThickness/2, -stepLength*random(0.9, 1.1), z-strokeThickness/2, stepLength*random(0.9, 1.1), tangent2, stepLength*2);
    z += 1;
  }
}


void setup() {
  //size(1280, 720);
  fullScreen();
  background(0);
  frameRate(30);
  noCursor();
  nextImage();
  
  colorMode(HSB);
  
  minim = new Minim(this);
  air = minim.loadFile("air.mp3", 2048);
  air.loop();
  
  kinect = new KinectPV2(this);

  kinect.enableSkeletonColorMap(true);
  kinect.enableColorImg(true);

  kinect.init();
}


void draw() {
  //translate(width/4, height/2);
  //background(img);
  translate(width / 2, height / 2);
  
  ArrayList<KSkeleton> skeletonArray =  kinect.getSkeletonColorMap();
  for (int i = 0; i < skeletonArray.size(); i++) {
    KSkeleton skeleton = (KSkeleton) skeletonArray.get(i);
    if (skeleton.isTracked()) {
      KJoint[] joints = skeleton.getJoints();

      color col  = skeleton.getIndexColor();
      fill(col);
      stroke(col);
     
      //draw different color for each hand state
      drawHandState(joints[KinectPV2.JointType_Head]);
       
    } 
  }
  
  int index = 0;
  for (int y = 0; y < height; y+=1) {
    for (int x = 0; x < width; x+=1) {
      int odds = (int)random(80000);
      //int odds = 0 ;
      if (odds < 1) {
        //println(odds);
        color pixelColor = img.pixels[index];
        headXdis = Math.abs(headX-1075);
        pixelColor = color(hue(pixelColor), saturation(pixelColor), brightness(pixelColor), 10);
        
        pushMatrix();
        translate(x-img.width/2, y-img.height/2);
        println("headX:" + headX);
        println("headY:" + headY);
        println("headXdis:" + headXdis);
        rotate(radians(hue(pixelColor) + headX));
        
        // Paint by layers from rough strokes to finer details
        int sound = (int)(500 * air.left.get((int)random(2048)));
        if(sound < 0){
          sound = -sound;
        }
        
        paintStroke((int)random(sound + 1, sound + 5), pixelColor, (int)random(sound / 10 + 1, sound / 10 + 5));
        //paintStroke((int)random(headXdis / 5 + 2, headXdis / 5 + 10), pixelColor, (int)random(headXdis / 50 + 1, headXdis / 50 + 5));
        popMatrix();
      }
      
      index += 1;
    }
  }
  
  if(frameCount % 60 == 0){
    frameCount = 0;
    if (headXdis > 150) {
      if (headX < 1075){
        time = 0;
        if(time != preTime){
          img = loadImage(imgNames[0]);
          //img.resize(width / 2 / 4 * 3, height / 4 * 3);
          //img.resize(width, height);
          img.loadPixels();
          time = 0;
          preTime = time;
        }
      
      }
      else if(headX > 1075){
        time = 2;
        if(time != preTime){
          img = loadImage(imgNames[2]);
          //img.resize(width / 2 / 4 * 3, height / 4 * 3);
          //img.resize(width, height);
          img.loadPixels();
          time = 2;
          preTime = time;
        }
      
      }
    }
    else{
        time = 1;
        if(time != preTime){
          img = loadImage(imgNames[1]);
          //img.resize(width / 2 / 4 * 3, height / 4 * 3);
          //img.resize(width, height);
          img.loadPixels();
          time = 1;
          preTime = time;
        }
      
    }
  }
  
}


void mousePressed() {
  nextImage();
}

void drawHandState(KJoint joint) {
  headX = int(joint.getX());
  headY = int(joint.getY());
}