/* * zazaface.java * New Zaza Face applet/application * * Created on January 16, 2002, 1:17 PM * Brian Rudy (brudyNO@SPAM.praecogito.com) * * * 0.02 1/21/2001 * Added viseme Look Up Table. Added auto-loading/parsing of * Festival phoneme data at startup. * * 0.01 1/18/2001 * Test to find if it is possible to synchronize 'Disney 13' * visemes with phonetic output from Festival. Answer=yes ;) * Auto-loads viseme images and WAV audio file at startup. * * * Bugs * */ package praecogito.development.zazaface; /** * * @author brudy * * Disney 13 viseme mapping table * * MS SAPI uses 22, Festival uses 49. * * * Array index * 0, // SP_VISEME_0 = 0, // Silence 11, // SP_VISEME_1, // AE, AX, AH 11, // SP_VISEME_2, // AA 11, // SP_VISEME_3, // AO 10, // SP_VISEME_4, // EY, EH, UH 11, // SP_VISEME_5, // ER 9, // SP_VISEME_6, // y, IY, IH, IX 2, // SP_VISEME_7, // w, UW 13, // SP_VISEME_8, // OW 9, // SP_VISEME_9, // AW 12, // SP_VISEME_10, // OY 11, // SP_VISEME_11, // AY 9, // SP_VISEME_12, // h 3, // SP_VISEME_13, // r 6, // SP_VISEME_14, // l 7, // SP_VISEME_15, // s, z 8, // SP_VISEME_16, // SH, CH, JH, ZH 5, // SP_VISEME_17, // TH, DH 4, // SP_VISEME_18, // f, v 7, // SP_VISEME_19, // d, t, n 9, // SP_VISEME_20, // k, g, NG 1 // SP_VISEME_21, // p, b, m * * */ public class zazaface extends javax.swing.JApplet { public void init() { // Get images for (int i = 1; i <= 13; i++) { faceImg[i-1] = getFaceImage( "http://www.praecogito.com/~brudy/zaza/java/images/viseme_"+i+".gif"); } // Get wav/au file try { soundclip = zazaface.newAudioClip(new java.net.URL( "http://www.praecogito.com/~brudy/zaza/java/faceapplet/festival.wav")); } catch (java.net.MalformedURLException er) { } catch (java.io.IOException er) { } loadPhones(); // initialize GUI new zazaface().show(); setFace(0, faceImg); // do stuff doneLoading = true; } public java.awt.Image getFaceImage(String name) { java.awt.Image img = getImage(getCodeBase(), name); try { java.awt.MediaTracker tracker = new java.awt.MediaTracker(this); tracker.addImage(img, 0); tracker.waitForID(0); } catch (Exception e) {} return img; } /** Creates new form zazaface */ public zazaface() { initComponents(); } /** This method is called from within the constructor to * initialize the form. * WARNING: Do NOT modify this code. The content of this method is * always regenerated by the Form Editor. */ private void initComponents() {//GEN-BEGIN:initComponents timer1 = new org.netbeans.examples.lib.timerbean.Timer(); jPanel1 = new javax.swing.JPanel(); jLabel1 = new javax.swing.JLabel(); jLabel2 = new javax.swing.JLabel(); jButton1 = new javax.swing.JButton(); jLabel3 = new javax.swing.JLabel(); jPanel3 = new javax.swing.JPanel(); jTextField2 = new javax.swing.JTextField(); timer1.setDelay(10L); timer1.addTimerListener(new org.netbeans.examples.lib.timerbean.TimerListener() { public void onTime(java.awt.event.ActionEvent evt) { timer1OnTime(evt); } }); jPanel1.setLayout(new java.awt.GridLayout(1, 3)); jPanel1.setBackground(java.awt.Color.white); jLabel1.setText("Time Index"); jLabel1.setForeground(java.awt.Color.black); jLabel1.setBackground(java.awt.Color.white); jLabel1.setHorizontalAlignment(javax.swing.SwingConstants.CENTER); jPanel1.add(jLabel1); jLabel2.setText("000,000"); jLabel2.setBackground(java.awt.Color.white); jLabel2.setHorizontalAlignment(javax.swing.SwingConstants.CENTER); jPanel1.add(jLabel2); jButton1.setToolTipText("Speak the text"); jButton1.setText("Speak"); jButton1.addMouseListener(new java.awt.event.MouseAdapter() { public void mouseReleased(java.awt.event.MouseEvent evt) { jButton1MouseReleased(evt); } }); jPanel1.add(jButton1); getContentPane().add(jPanel1, java.awt.BorderLayout.NORTH); jLabel3.setBackground(java.awt.Color.white); jLabel3.setHorizontalAlignment(javax.swing.SwingConstants.CENTER); jLabel3.setBorder(new javax.swing.border.SoftBevelBorder(javax.swing.border.BevelBorder.RAISED)); getContentPane().add(jLabel3, java.awt.BorderLayout.CENTER); jPanel3.setLayout(new java.awt.GridLayout(1, 0)); jTextField2.setText("Speak this!"); jTextField2.setBorder(null); jPanel3.add(jTextField2); getContentPane().add(jPanel3, java.awt.BorderLayout.SOUTH); }//GEN-END:initComponents private void jButton1MouseReleased(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_jButton1MouseReleased // Reset the timer timeIndex = 0; // Set the script index to the beginning scriptIndex = 0; donePlaying = false; soundclip.play(); }//GEN-LAST:event_jButton1MouseReleased // This gets called every 10ms private void timer1OnTime(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timer1OnTime int currentTime = 0; if (doneLoading) { if (timeIndex == 0) { // Just got restarted, should do something here } else { // Time in milliseconds currentTime = timeIndex * 10; } checkIndex(currentTime); } timeIndex++; }//GEN-LAST:event_timer1OnTime // Check current time, and make sure we are showing // the right viseme, update if not. public void checkIndex(int currentTime) { if ((scriptLines > 1 ) && (!donePlaying)) { if (speakScript[scriptLines-1][0] > currentTime) { // Script is playing if (currentTime > speakScript[scriptIndex][0]) { for (int index = 0; index < scriptLines; index++) { if (currentTime >= speakScript[index][0]) { scriptIndex = index; } } scriptIndex++; } if (currentViseme != speakScript[scriptIndex][1]) { // We need to update //System.out.println("Setting the face to viseme " // + speakScript[scriptIndex][1] + " timecode " // + speakScript[scriptIndex][0] + " current time " // + currentTime); setFace(speakScript[scriptIndex][1], faceImg); } jLabel2.setText(String.valueOf(currentTime)); } else { // This will only get run if the last phoneme is not a closed mouth if (currentViseme != 0) { System.out.println("Done speaking."); setFace(0, faceImg); } donePlaying = true; } } } public void loadPhones() { // Download 'raw' phoneme output from Festival System.out.println("Getting phonemes..."); String phoneStr= "http://www.praecogito.com/~brudy/zaza/java/faceapplet/phone.data"; try { java.net.URL url = new java.net.URL(phoneStr); java.net.URLConnection connection = url.openConnection(); java.io.BufferedReader in = new java.io.BufferedReader( new java.io.InputStreamReader(connection.getInputStream())); java.io.StreamTokenizer datain = new java.io.StreamTokenizer(in); // Read until there is no more datain.resetSyntax(); datain.wordChars(33, 255); datain.whitespaceChars(0, ' '); datain.parseNumbers(); datain.eolIsSignificant(true); int token = datain.nextToken(); int scriptline = 0; while (token != java.io.StreamTokenizer.TT_EOF) { if (scriptline == 0) { //System.out.println("Ignoring the '#' char at start of script."); token = datain.nextToken(); // # token = datain.nextToken(); // EOL } else { //while (token != java.io.StreamTokenizer.TT_EOL) { // if (datain.ttype == java.io.StreamTokenizer.TT_NUMBER) { // System.out.println("Token: " + datain.nval + "."); // } // else { // System.out.println("Token: " + datain.sval + "."); // } // token = datain.nextToken(); //} //System.out.println("End of Line"); // Load scripty goodness // script format: (uncorrected time) (stress) (phoneme)(EOL) speakScript[scriptline-1] = new int[2]; speakScript[scriptline-1][0] = Double2Int(datain.nval*1000); // time token = datain.nextToken(); //stress (unused) token = datain.nextToken(); speakScript[scriptline-1][1] = matchPhone(datain.sval); // phoneme token = datain.nextToken(); // EOL token = datain.nextToken(); } scriptline++; } scriptLines = scriptline-1; System.out.println("End of File, " + scriptLines + " lines read."); // Print speakScript for verification //for (int i = 0; i < scriptLines; i++) { // System.out.println("Timecode=" + speakScript[i][0] // + ", viseme=" + speakScript[i][1]); //} } catch (java.net.MalformedURLException e) { } catch (java.io.IOException e) { } } public int Double2Int(double d_number) { int p1=0; int p2=0; int t=0; String str1, str2; str1 = " " + d_number + ".0"; p1 = str1.indexOf(" "); p2 = str1.indexOf("."); if (p2 > p1) {str2 = str1.substring(1, p2);} else {str2 = str1;} if (p2 > p1) {str2 = str1.substring(1, p2);} else {str2 = str1;} try { t = Integer.parseInt(str2);} catch (NumberFormatException e) { t = 0;} return t; } // Compare given string with 'Disney 13' viseme Look Up Table // Return appropriate viseme frame number public int matchPhone(java.lang.String phone) { for (int a = 0; a <= 13; a++) { for (int innerindex = 0; innerindex < visemeLUT[a].length; innerindex++) { if (phone.equals(visemeLUT[a][innerindex])) { //System.out.println("Phoneme " + phone + "=" + a); return a; } } } // no match System.out.println("I cannot find a viseme in the LUT for " + phone + "!"); return 0; } public void setFace(int myframe, java.awt.Image frameImg[]) { //jLabel3.setIcon(new javax.swing.ImageIcon(frameImg[0])); //int myframe = zazaface.timeIndex%13; try { //System.out.println("Now showing image " + myframe); if (frameImg[12] != null) { if (myframe == 0) { jLabel3.setIcon(new javax.swing.ImageIcon(frameImg[myframe])); } else { jLabel3.setIcon(new javax.swing.ImageIcon(frameImg[myframe - 1])); } //jLabel3.setIcon(new javax.swing.ImageIcon(frameImg[myframe])); } else { System.out.println("Frame " + myframe + " is invalid!"); } currentViseme = myframe; } catch (ArrayIndexOutOfBoundsException e) { //On rare occasions, this method can be called //when frameNumber is still -1. Do nothing. return; } } // Only used for running outside of a browser public void main(java.lang.String args[]) { // Get images for (int i = 1; i <= 13; i++) { faceImg[i-1] = java.awt.Toolkit.getDefaultToolkit().getImage( "images/viseme_"+i+".gif"); } // Get wav/au file try { soundclip = zazaface.newAudioClip(new java.net.URL( "http://www.praecogito.com/~brudy/zaza/java/faceapplet/festival.wav")); } catch (java.net.MalformedURLException er) { } catch (java.io.IOException er) { } loadPhones(); // initialize GUI new zazaface().show(); setFace(0, faceImg); // do stuff doneLoading = true; } // Variables declaration - do not modify//GEN-BEGIN:variables private org.netbeans.examples.lib.timerbean.Timer timer1; private javax.swing.JPanel jPanel1; private javax.swing.JLabel jLabel1; private javax.swing.JLabel jLabel2; private javax.swing.JButton jButton1; private javax.swing.JLabel jLabel3; private javax.swing.JPanel jPanel3; private javax.swing.JTextField jTextField2; // End of variables declaration//GEN-END:variables java.awt.Image[] faceImg = new java.awt.Image[13]; static int timeIndex = 0; java.applet.AudioClip soundclip; int scriptIndex = 0; int currentViseme = 0; boolean doneLoading = false; boolean donePlaying = true; int scriptLines = 0; // 400 phonemes per sentance max. Longest should be <200, but why not? int[][] speakScript = new int[400][]; // First try, started from VC++ sample, then added unmatched phonemes // from faceapplet ph2lip CGI String visemeLUT[][] = { {"pau", "sil"}, // 0, mouth closed, smile {"p", "b", "m", "em"}, // 1, mouth closed {"w", "uw"}, // 2 {"r"}, // 3 {"f", "v"}, // 4 {"th", "dh", "dx"}, // 5 {"l", "el"}, // 6 {"s", "z", "d", "t", "n", "en", "nx"}, // 7 {"sh", "ch", "jh", "zh"}, // 8 {"h", "y", "iy", "ih", "ix", "aw", "k", "g", "ng", "hh", "hv"}, // 9 {"ey", "eh", "uh"}, // 10 {"ae", "ax", "ah", "aa", "ao", "er", "ay", "axr"}, // 11 {"oy"}, // 12 {"ow"} // 13 }; // 'My name is zaw zaw the robot, whats yours? /* int speakScript[][] = { {220, 0}, // pau {289, 1}, // m {425, 11}, // ay {509, 7}, // n {648, 10}, // ey {711, 1}, // m {778, 9}, // ih {845, 7}, // z {919, 7}, // z {1043, 11}, // ao {1129, 7}, // z {1253, 11}, // ao {1286, 5}, // dh {1326, 11}, // ax {1408, 3}, // r {1555, 13}, // ow {1630, 1}, // b {1780, 11}, // aa {1873, 7}, // t {2093, 0}, // pau {2146, 2}, // w {2176, 11}, // ax {2236, 7}, // t {2327, 7}, // s {2376, 9}, // y {2506, 10}, // uh {2602, 3}, // r {2775, 7}, // z {3224, 0} // pau }; */ }