Audio Software Mirror Redux

The audio software mirror has evolved yet again, becoming a …

The Conversation Cloud Generator is a wearable device that generates on-screen word clouds by listening to conversation and surveying syntax and volume. The interface is concealed within wrappings of wireform mesh (a thinner, more pliable version of the material that covers standard microphones) and sewn to a waist belt. The innocuous wearable contains two microphones and an Arduino. On microphone takes volume readings via the Arduino while the second sends four second audio clips (wav files) directly to the computer. Via Processing, the audio data is sent to Google Speech and the returned result is saved as text to a data file, accompanied by its corresponding volume readings (by four second delay). As the user’s leisure, the word cloud is generated via mouse click. Word size is determined by merging the volume values with the frequency of word use (the WordCram library is used to determine word frequency); the largest words are the louder, more frequently used words. This serves as a visual translation of a conversation experience.

DATA FILE EXAMPLE

Click here to see a recent cloud’s data file.

CUSTOMIZING WORDCRAM

THE BUILD

CONNECTIONS

PROCESSING CODE

import com.getflourish.stt.*;
import processing.serial.*;
import wordcram.*;

Serial myPort;  
STT stt;
WordCram wordCram;
Volumecircle[] volumeCircles = new Volumecircle[7];

PFont myFont;
String result;
String dataFile = "/Users/mcortese_/Documents/
  Processing/STT_cloud/data/data.txt";
int typeSize;
int lastRecordingStartTime;
int wordReload;
boolean firstContact = false;

void setup() {
  size(1200, 720);
  background(0);
  stt = new STT(this);
  stt.enableDebug();
  stt.setLanguage("en");
  myFont = createFont("GothamHTF-Bold-48.vlw", 48);
  result = "";
  println(Serial.list());
  String portName = Serial.list()[0];
  myPort = new Serial(this, portName, 9600);
  for (int i = 0; i < volumeCircles.length; i ++ ) {
    volumeCircles[i] = new Volumecircle(130, 50,
    typeSize*volumeCircles.length);
  }
  fill(255, 40, 80);
  text("L I S T E N I N G", 285, 25);
  wordCram = new WordCram(this)
    .withFont(myFont)
      .fromTextFile("/data/data.txt")
        .withColor(#ededed)
          .sizedByWeight(15, 200);
}

void draw () {
  if (millis() - lastRecordingStartTime > 4000) {
    stt.end();
    stt.begin();
    lastRecordingStartTime = millis();
  }
  if (mousePressed == true) {
    wordCram.drawAll();
  }
  saveData(dataFile, result, typeSize, true);
  for (int i = 0; i < volumeCircles.length; i ++ ) {
    volumeCircles[i].display();
  }
  fill(255, 40, 80, 30);
  ellipse(130, 50, typeSize*2, typeSize*2);
  fill(0, 30);
  ellipse(130, 50, typeSize*2, typeSize*2);
  fill(255, 40, 80, 30);
}

void serialEvent(Serial myPort) {
  int inByte = myPort.read();
  if (firstContact == false) {
    if (inByte == 'A') {
      myPort.clear();
      firstContact = true;
      myPort.write('A');
    }
  }
  else {
    typeSize= inByte;
    typeSize=abs(typeSize-128);
    println(typeSize);
    myPort.write('A');
  }
}

void transcribe (String utterance, float confidence) {
  println(utterance);
  result = utterance;
}

void saveData(String fileName, String newData,
  int volume, boolean appendData) {
  BufferedWriter bw = null;
  try {  
    FileWriter fw = new FileWriter(fileName, appendData);
    bw = new BufferedWriter(fw);
    bw.write(newData + " " + volume + System.getProperty
    ("line.separator"));
  }
  catch (IOException e) {
  }
  finally {
    if (bw != null) {
      try {
        bw.close();
      }
      catch (IOException e) {
      }
    }
  }
}

void mouseReleased(){
  fill(0, 255);
  rect(285, 15, 100, 30);
}

class Volumecircle {

  float xplace;
  float yplace;
  float diameter;

  Volumecircle(float xplace_, float yplace_, float diameter_) {
    xplace = xplace_;
    yplace = yplace_;
    diameter = diameter_;
  }

  void display() {
    fill(255, 40, 80, 30);
    ellipse(xplace,yplace, diameter, diameter);
    fill(0, 30);
    ellipse(xplace,yplace, diameter, diameter);
  }
}

ARDUINO CODE

int analogCell1 = 0;
int inByte = 0;

void setup()
{
  Serial.begin(9600);
  while (!Serial) {
    ;
  }
  establishContact();
}

void loop()
{
  if (Serial.available() > 0) {
    inByte = Serial.read();
    int typeSize=0;
    typeSize = analogRead(analogCell1)/4;
    Serial.write(typeSize);
  }
}

void establishContact() {
  while (Serial.available() <= 0) {
    Serial.print('A');
    delay(4000);
  }
}

And finally, I leave you with a text cloud created via a discussion on The Oatmeal’s “5 Very Good Reasons to Punch a Dolphin in the Mouth”