import java.io.StreamTokenizer; import java.io.BufferedReader; import java.net.URL; import java.awt.Image; import java.awt.RenderingHints; import java.util.GregorianCalendar; /* * zazaface.java * New Zaza face applet/application * * Copyright (C) 2001-2002 Brian Rudy (brudyNO@SPAMpraecogito.com) * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * Created on January 16, 2002, 1:17 PM * Brian Rudy (brudyNO@SPAM.praecogito.com) * * 1.05 6/14/2002 * Fixed bug related to speaking while downloading next cue data, * performance improvements. * * 1.04 6/2/2002 * Fixed dual-instance problem. * * 1.03 5/21/2002 * Added support for cue stack in voiceServer 0.3. Added * debugging option to applet mode. Added support for back-end * emotion change (emotion not supported in applet yet). * * 1.02 5/3/2002 * Fixed interruptions caused by cue request sent to voiceServer * too quickly. * * 1.01 3/8/2002 * Added sanity check in manual playback button code. Updated * imageicon creation to correctly scale visemes to applet size. * * 1.00 2/14/2002 * Initial support for voiceServer/faceClient. * * 0.04 1/27/2001 * Updated timing to use more accurate GregorianCalendar * functions for ensuring timing consistancy between platforms. * Script progress monitoring increased to every 1ms, from * every 10ms. * * 0.03 1/22/2001 * Replaced deprecated show() methods in init() and main(). * * 0.02 1/21/2001 * Added viseme Look Up Table. Added auto-loading/parsing of * Festival phoneme data at startup. * * 0.01 1/18/2001 * Test to find if it is possible to synchronize 'Disney 13' * visemes with phonetic output from Festival. Answer=yes ;) * Auto-loads viseme images and WAV audio file at startup. * * * Bugs * */ //package home.brudy.nbuser32.development.zazaface; /** * * @author brudy * * Disney 13 viseme mapping table * * MS SAPI uses 22, Festival uses 49. * * * Array index * 0, // SP_VISEME_0 = 0, // Silence 11, // SP_VISEME_1, // AE, AX, AH 11, // SP_VISEME_2, // AA 11, // SP_VISEME_3, // AO 10, // SP_VISEME_4, // EY, EH, UH 11, // SP_VISEME_5, // ER 9, // SP_VISEME_6, // y, IY, IH, IX 2, // SP_VISEME_7, // w, UW 13, // SP_VISEME_8, // OW 9, // SP_VISEME_9, // AW 12, // SP_VISEME_10, // OY 11, // SP_VISEME_11, // AY 9, // SP_VISEME_12, // h 3, // SP_VISEME_13, // r 6, // SP_VISEME_14, // l 7, // SP_VISEME_15, // s, z 8, // SP_VISEME_16, // SH, CH, JH, ZH 5, // SP_VISEME_17, // TH, DH 4, // SP_VISEME_18, // f, v 7, // SP_VISEME_19, // d, t, n 9, // SP_VISEME_20, // k, g, NG 1 // SP_VISEME_21, // p, b, m * * */ public class zazaface extends javax.swing.JApplet { public void init() { // Get images for (int i = 1; i <= 13; i++) { faceImg[i-1] = getFaceImage( "http://" + hostname + visemepath + "viseme_"+i+".gif"); } // initialize GUI //new zazaface().setVisible(true); if (getParameter("DEBUG").equals("0")) { this.jPanel1.setVisible(false); } setFace(0, faceImg); // do stuff doneLoading = true; //getCues(); } public void loadWaveform(String cuefinger) { // Get wav/au file System.out.println("Getting waveform for " + cuefinger); String waveformURL = "http://" + hostname + datapath + cuefinger + ".wav"; //System.out.println("Getting " + waveformURL); try { soundclip = zazaface.newAudioClip(new URL(waveformURL)); } catch (java.net.MalformedURLException er) { } catch (java.io.IOException er) { } gotWaveform = true; } // Connect, and render new cues when they arrive public void getCues() { //System.out.println("Connecting to faceClient CGI..."); loadingCues = true; String cueURL= "http://" + hostname + "/cgi-bin/faceClient?give=" + currentTimestamp; //System.out.println("Getting " + cueURL); String thistimestamp = "0"; String linefinger = "0"; try { URL url = new URL(cueURL); java.net.URLConnection connection = url.openConnection(); connection.setUseCaches(false); BufferedReader in = new BufferedReader( new java.io.InputStreamReader(connection.getInputStream())); StreamTokenizer datain = new StreamTokenizer(in); datain.resetSyntax(); datain.wordChars(33, 255); datain.whitespaceChars(0, ' '); datain.parseNumbers(); datain.eolIsSignificant(true); int token = datain.nextToken(); thistimestamp = String.valueOf((long)datain.nval); //System.out.println("timestamp: " + thistimestamp); if (thistimestamp.equals("0")) { //System.out.println("We are current"); } else { token = datain.nextToken(); linefinger = String.valueOf((long)datain.nval); //System.out.println("linefinger: " + linefinger); token = datain.nextToken(); emotion = datain.sval; //System.out.println("emotion: " + emotion); } in.close(); //System.out.println("Got " + thistimestamp + ", " + linefinger + ", " + emotion); if (thistimestamp.equals(lastact) || (!donePlaying) || linefinger.equals("0") || thistimestamp.equals("0")) { // Don't do anything, we have already performed this act if (linefinger.equals("0") && !thistimestamp.equals("0")) { System.out.println("Emotion only change."); lastact = thistimestamp; currentTimestamp = thistimestamp; } System.out.println("Skipping act."); } else { lastact = thistimestamp; //System.out.println("Got: " + linefinger + " from faceClient"); loadWaveform(linefinger); loadPhones(linefinger); currentTimestamp = thistimestamp; //done loading, start playing // Reset the timer //timeIndex = 0; gCal = new GregorianCalendar(); startTime = gCal.getTime().getTime(); // Set the script index to the beginning scriptIndex = 0; donePlaying = false; soundclip.play(); } //System.out.println("faceClient Connection closed!"); gotCue = true; } catch (java.net.MalformedURLException e) { System.out.println("Oops, MalformedURLException"); } catch (java.io.IOException e) { System.out.println("Oops, IOException"); } loadingCues = false; } public Image getFaceImage(String name) { int compWidth = getWidth(); int compHeight = getHeight(); int imageWidth, imageHeight; java.awt.image.BufferedImage bi; Image img = getImage(getCodeBase(), name); try { java.awt.MediaTracker tracker = new java.awt.MediaTracker(this); tracker.addImage(img, 0); tracker.waitForID(0); } catch (Exception e) {} if ((compWidth > 0) && (compHeight > 0)) { int width = img.getWidth(this); int height = img.getHeight(this); bi = new java.awt.image.BufferedImage(compWidth, compHeight, java.awt.image.BufferedImage.TYPE_INT_RGB); java.awt.Graphics2D biContext = bi.createGraphics(); biContext.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); biContext.setRenderingHint(RenderingHints.KEY_RENDERING, RenderingHints.VALUE_RENDER_QUALITY); biContext.drawImage(img, 0, 0, compWidth,compHeight, null); //img.flush(); //bi.flush(); return bi; } else { System.out.println("Oops! Image is invalid!"); return img; } } /** Creates new form zazaface */ public zazaface() { initComponents(); } /** This method is called from within the constructor to * initialize the form. * WARNING: Do NOT modify this code. The content of this method is * always regenerated by the Form Editor. */ private void initComponents() {//GEN-BEGIN:initComponents timer1 = new org.netbeans.examples.lib.timerbean.Timer(); jPanel1 = new javax.swing.JPanel(); jButton1 = new javax.swing.JButton(); jLabel1 = new javax.swing.JLabel(); jLabel2 = new javax.swing.JLabel(); jLabel3 = new javax.swing.JLabel(); timer1.setDelay(1L); timer1.addTimerListener(new org.netbeans.examples.lib.timerbean.TimerListener() { public void onTime(java.awt.event.ActionEvent evt) { timer1OnTime(evt); } }); setBackground(new java.awt.Color(204, 204, 204)); jPanel1.setLayout(new java.awt.GridLayout(1, 3)); jButton1.setToolTipText("Repeat last cue"); jButton1.setText("Speak"); jButton1.setBorder(new javax.swing.border.SoftBevelBorder(javax.swing.border.BevelBorder.RAISED)); jButton1.addMouseListener(new java.awt.event.MouseAdapter() { public void mouseReleased(java.awt.event.MouseEvent evt) { jButton1MouseReleased(evt); } }); jPanel1.add(jButton1); jLabel1.setText("Time Index"); jLabel1.setBackground(java.awt.Color.white); jLabel1.setHorizontalAlignment(javax.swing.SwingConstants.CENTER); jPanel1.add(jLabel1); jLabel2.setText("0"); jLabel2.setBackground(java.awt.Color.white); jLabel2.setHorizontalAlignment(javax.swing.SwingConstants.CENTER); jPanel1.add(jLabel2); getContentPane().add(jPanel1, java.awt.BorderLayout.NORTH); jLabel3.setBackground(java.awt.Color.white); jLabel3.setHorizontalAlignment(javax.swing.SwingConstants.CENTER); getContentPane().add(jLabel3, java.awt.BorderLayout.CENTER); }//GEN-END:initComponents private void jButton1MouseReleased(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_jButton1MouseReleased // Reset the timer //timeIndex = 0; if (doneLoading && gotCue && gotWaveform) { gCal = new GregorianCalendar(); startTime = gCal.getTime().getTime(); // Set the script index to the beginning scriptIndex = 0; donePlaying = false; soundclip.play(); } }//GEN-LAST:event_jButton1MouseReleased // This gets called every 1ms private void timer1OnTime(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timer1OnTime int currentTime = 0; if (doneLoading) { GregorianCalendar myCal = new GregorianCalendar(); long thisTime = myCal.getTime().getTime(); long difference = thisTime - startTime; //System.out.println("Difference=" + difference); checkIndex((int) difference); if ((timeIndex >= 100) && (donePlaying) && (!loadingCues)) { timeIndex = 0; getCues(); } } timeIndex++; }//GEN-LAST:event_timer1OnTime // Check current time, and make sure we are showing // the right viseme, update if not. public void checkIndex(int currentTime) { if ((scriptLines > 1 ) && (!donePlaying)) { if (speakScript[scriptLines-1][0] > currentTime) { // Script is playing if (currentTime > speakScript[scriptIndex][0]) { for (int index = 0; index < scriptLines; index++) { if (currentTime >= speakScript[index][0]) { scriptIndex = index; } } scriptIndex++; } if (currentViseme != speakScript[scriptIndex][1]) { // We need to update //System.out.println("Setting the face to viseme " // + speakScript[scriptIndex][1] + " timecode " // + speakScript[scriptIndex][0] + " current time " // + currentTime); setFace(speakScript[scriptIndex][1], faceImg); } jLabel2.setText(String.valueOf(currentTime)); } else { // This will only get run if the last phoneme is not a closed mouth if (currentViseme != 0) { System.out.println("Done speaking."); setFace(0, faceImg); } donePlaying = true; } } } public void loadPhones(String cuefinger) { // Download 'raw' phoneme output from Festival System.out.println("Getting phonemes for " + cuefinger); String phoneURL= "http://" + hostname + datapath + cuefinger + ".script"; //System.out.println("Getting " + phoneURL); try { URL url = new URL(phoneURL); java.net.URLConnection connection = url.openConnection(); BufferedReader in = new BufferedReader( new java.io.InputStreamReader(connection.getInputStream())); StreamTokenizer datain = new StreamTokenizer(in); // Read until there is no more datain.resetSyntax(); datain.wordChars(33, 255); datain.whitespaceChars(0, ' '); datain.parseNumbers(); datain.eolIsSignificant(true); int token = datain.nextToken(); int scriptline = 0; while (token != StreamTokenizer.TT_EOF) { // Load scripty goodness // script format: (uncorrected time) (phoneme)(EOL) speakScript[scriptline] = new int[2]; speakScript[scriptline][0] = Double2Int(datain.nval*1000); // time //token = datain.nextToken(); //stress (unused) token = datain.nextToken(); speakScript[scriptline][1] = matchPhone(datain.sval); // phoneme token = datain.nextToken(); // EOL token = datain.nextToken(); scriptline++; } scriptLines = scriptline-1; System.out.println("End of File, " + scriptLines + " lines read."); in.close(); } catch (java.net.MalformedURLException e) { } catch (java.io.IOException e) { } } public int Double2Int(double d_number) { int p1=0; int p2=0; int t=0; String str1, str2; str1 = " " + d_number + ".0"; p1 = str1.indexOf(" "); p2 = str1.indexOf("."); if (p2 > p1) {str2 = str1.substring(1, p2);} else {str2 = str1;} if (p2 > p1) {str2 = str1.substring(1, p2);} else {str2 = str1;} try { t = Integer.parseInt(str2);} catch (NumberFormatException e) { t = 0;} return t; } // Compare given string with 'Disney 13' viseme Look Up Table // Return appropriate viseme frame number public int matchPhone(java.lang.String phone) { for (int a = 0; a <= 13; a++) { for (int innerindex = 0; innerindex < visemeLUT[a].length; innerindex++) { if (phone.equals(visemeLUT[a][innerindex])) { //System.out.println("Phoneme " + phone + "=" + a); return a; } } } // no match System.out.println("I cannot find a viseme in the LUT for " + phone + "!"); return 0; } public void setFace(int myframe, Image[] frameImg) { //jLabel3.setIcon(new javax.swing.ImageIcon(frameImg[0])); //int myframe = zazaface.timeIndex%13; try { //System.out.println("Now showing image " + myframe); if (frameImg[12] != null) { if (myframe == 0) { jLabel3.setIcon(new javax.swing.ImageIcon(frameImg[myframe])); } else { jLabel3.setIcon(new javax.swing.ImageIcon(frameImg[myframe - 1])); } //jLabel3.setIcon(new javax.swing.ImageIcon(frameImg[myframe])); } else { System.out.println("Frame " + myframe + " is invalid!"); } currentViseme = myframe; } catch (ArrayIndexOutOfBoundsException e) { //On rare occasions, this method can be called //when frameNumber is still -1. Do nothing. return; } } // Only used for running outside of a browser public void main(java.lang.String args[]) { // Get images for (int i = 1; i <= 13; i++) { faceImg[i-1] = java.awt.Toolkit.getDefaultToolkit().getImage( "http://" + hostname + visemepath + "images/viseme_"+i+".gif"); } javax.swing.JFrame f = new javax.swing.JFrame("zazaface"); final zazaface controller = new zazaface(); // initialize GUI controller.setVisible(true); setFace(0, faceImg); // do stuff doneLoading = true; //getCues(); } // Variables declaration - do not modify//GEN-BEGIN:variables private org.netbeans.examples.lib.timerbean.Timer timer1; private javax.swing.JButton jButton1; private javax.swing.JLabel jLabel3; private javax.swing.JLabel jLabel2; private javax.swing.JLabel jLabel1; private javax.swing.JPanel jPanel1; // End of variables declaration//GEN-END:variables Image[] faceImg = new Image[13]; static int timeIndex = 0; java.applet.AudioClip soundclip; int scriptIndex = 0; int currentViseme = 0; boolean doneLoading = false; boolean donePlaying = true; boolean gotWaveform = false; boolean gotCue = false; boolean loadingCues = false; int scriptLines = 0; GregorianCalendar gCal; long startTime; String lastact; String currentTimestamp = "0"; String emotion = "happy"; // faceServer hostname String hostname = "zazaconsole.exhibits.thetech.org"; //String hostname = "localhost"; // path to waveform/script files String datapath = "/~brudy/zaza/speechdata/"; // path to visemes String visemepath = "/~brudy/zaza/java/images/"; // 400 phonemes per sentence max. Longest should be <200, but why not? int[][] speakScript = new int[400][]; // First try, started from VC++ sample, then added unmatched phonemes // from faceapplet ph2lip CGI. This will only work with American English phones String visemeLUT[][] = { {"pau", "sil"}, // 0, mouth closed, smile {"p", "b", "m", "em"}, // 1, mouth closed {"w", "uw"}, // 2 {"r"}, // 3 {"f", "v"}, // 4 {"th", "dh", "dx"}, // 5 {"l", "el"}, // 6 {"s", "z", "d", "t", "n", "en", "nx"}, // 7 {"sh", "ch", "jh", "zh"}, // 8 {"h", "y", "iy", "ih", "ix", "aw", "k", "g", "ng", "hh", "hv"}, // 9 {"ey", "eh", "uh"}, // 10 {"ae", "ax", "ah", "aa", "ao", "er", "ay", "axr"}, // 11 {"oy"}, // 12 {"ow"} // 13 }; }