/* -*- mode: java; c-basic-offset: 2; indent-tabs-mode: nil -*- */

/*
  PdePreprocessor - wrapper for default ANTLR-generated parser
  Part of the Processing project - http://processing.org

  Copyright (c) 2004-15 Ben Fry and Casey Reas
  Copyright (c) 2001-04 Massachusetts Institute of Technology

  ANTLR-generated parser and several supporting classes written
  by Dan Mosedale via funding from the Interaction Institute IVREA.

  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or
  (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software Foundation,
  Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

package processing.mode.java.preproc;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import processing.app.Messages;
import processing.app.Preferences;
import processing.app.SketchException;
import processing.core.PApplet;
import processing.data.StringList;

import antlr.*;
import antlr.collections.AST;


/**
 * Class that orchestrates preprocessing p5 syntax into straight Java.
 * <P/>
 * <B>Current Preprocessor Subsitutions:</B>
 * <UL>
 * <LI>any function not specified as being protected or private will
 * be made 'public'. this means that <TT>void setup()</TT> becomes
 * <TT>public void setup()</TT>. This is important to note when
 * coding with core.jar outside of the PDE.
 * <LI><TT>compiler.substitute_floats</TT> (currently "substitute_f")
 * treat doubles as floats, i.e. 12.3 becomes 12.3f so that people
 * don't have to add f after their numbers all the time since it's
 * confusing for beginners.
 * <LI><TT>compiler.enhanced_casting</TT> byte(), char(), int(), float()
 * works for casting. this is basic in the current implementation, but
 * should be expanded as described above. color() works similarly to int(),
 * however there is also a *function* called color(r, g, b) in p5.
 * <LI><TT>compiler.color_datatype</TT> 'color' is aliased to 'int'
 * as a datatype to represent ARGB packed into a single int, commonly
 * used in p5 for pixels[] and other color operations. this is just a
 * search/replace type thing, and it can be used interchangeably with int.
 * <LI><TT>compiler.web_colors</TT> (currently "inline_web_colors")
 * color c = #cc0080; should unpack to 0xffcc0080 (the ff at the top is
 * so that the color is opaque), which is just an int.
 * </UL>
 * <B>Other preprocessor functionality</B>
 * <UL>
 * <LI>detects what 'mode' the program is in: static (no function
 * brackets at all, just assumes everything is in draw), active
 * (setup plus draw or loop), and java mode (full java support).
 * http://processing.org/reference/environment/
 * </UL>
 * <P/>
 * The PDE Preprocessor is based on the Java Grammar that comes with
 * ANTLR 2.7.2.  Moving it forward to a new version of the grammar
 * shouldn't be too difficult.
 * <P/>
 * Here's some info about the various files in this directory:
 * <P/>
 * <TT>java.g:</TT> this is the ANTLR grammar for Java 1.3/1.4 from the
 * ANTLR distribution.  It is in the public domain.  The only change to
 * this file from the original this file is the uncommenting of the
 * clauses required to support assert().
 * <P/>
 * <TT>java.tree.g:</TT> this describes the Abstract Syntax Tree (AST)
 * generated by java.g.  It is only here as a reference for coders hacking
 * on the preprocessor, it is not built or used at all.  Note that pde.g
 * overrides some of the java.g rules so that in PDE ASTs, there are a
 * few minor differences.  Also in the public domain.
 * <P/>
 * <TT>pde.g:</TT> this is the grammar and lexer for the PDE language
 * itself. It subclasses the java.g grammar and lexer.  There are a couple
 * of overrides to java.g that I hope to convince the ANTLR folks to fold
 * back into their grammar, but most of this file is highly specific to
 * PDE itself.
 * <TT>PdeEmitter.java:</TT> this class traverses the AST generated by
 * the PDE Recognizer, and emits it as Java code, doing any necessary
 * transformations along the way.  It is based on JavaEmitter.java,
 * available from antlr.org, written by Andy Tripp <atripp@comcast.net>,
 * who has given permission for it to be distributed under the GPL.
 * <P/>
 * <TT>ExtendedCommonASTWithHiddenTokens.java:</TT> this adds a necessary
 * initialize() method, as well as a number of methods to allow for XML
 * serialization of the parse tree in a such a way that the hidden tokens
 * are visible.  Much of the code is taken from the original
 * CommonASTWithHiddenTokens class.  I hope to convince the ANTLR folks
 * to fold these changes back into that class so that this file will be
 * unnecessary.
 * <P/>
 * <TT>TokenStreamCopyingHiddenTokenFilter.java:</TT> this class provides
 * TokenStreamHiddenTokenFilters with the concept of tokens which can be
 * copied so that they are seen by both the hidden token stream as well
 * as the parser itself.  This is useful when one wants to use an
 * existing parser (like the Java parser included with ANTLR) that throws
 * away some tokens to create a parse tree which can be used to spit out
 * a copy of the code with only minor modifications.  Partially derived
 * from ANTLR code.  I hope to convince the ANTLR folks to fold this
 * functionality back into ANTLR proper as well.
 * <P/>
 * <TT>whitespace_test.pde:</TT> a torture test to ensure that the
 * preprocessor is correctly preserving whitespace, comments, and other
 * hidden tokens correctly.  See the comments in the code for details about
 * how to run the test.
 * <P/>
 * All other files in this directory are generated at build time by ANTLR
 * itself.  The ANTLR manual goes into a fair amount of detail about the
 * what each type of file is for.
 * <P/>
 */
public class PdePreprocessor {
  protected static final String UNICODE_ESCAPES = "0123456789abcdefABCDEF";

  // used for calling the ASTFactory to get the root node
  private static final int ROOT_ID = 0;

  protected final String indent;
  private final String name;

  public static enum Mode {
    STATIC, ACTIVE, JAVA
  }

  private TokenStreamCopyingHiddenTokenFilter filter;

  private String advClassName = "";
  protected Mode mode;
  Set<String> foundMethods;

  SurfaceInfo sizeInfo;


  /**
   * Regular expression for parsing the size() method. This should match
   * against any uses of the size() function, whether numbers or variables
   * or whatever. This way, no warning is shown if size() isn't actually used
   * in the sketch, which is the case especially for anyone who is cutting
   * and pasting from the reference.
   */
//  public static final String SIZE_REGEX =
//    "(?:^|\\s|;)size\\s*\\(\\s*([^\\s,]+)\\s*,\\s*([^\\s,\\)]+)\\s*,?\\s*([^\\)]*)\\s*\\)\\s*\\;";
//  static private final String SIZE_CONTENTS_REGEX =
//    "(?:^|\\s|;)size\\s*\\(([^\\)]+)\\)\\s*\\;";
//  static private final String FULL_SCREEN_CONTENTS_REGEX =
//    "(?:^|\\s|;)fullScreen\\s*\\(([^\\)]+)\\)\\s*\\;";
//  /** Test whether there's a void somewhere (the program has functions). */
//  static private final String VOID_REGEX =
//    "(?:^|\\s|;)void\\s";
  /** Used to grab the start of setup() so we can mine it for size() */
  static private final String VOID_SETUP_REGEX =
    "(?:^|\\s|;)void\\s+setup\\s*\\(";


  // Can't only match any 'public class', needs to be a PApplet
  // http://code.google.com/p/processing/issues/detail?id=551
  static private final Pattern PUBLIC_CLASS =
    Pattern.compile("(^|;)\\s*public\\s+class\\s+\\S+\\s+extends\\s+PApplet", Pattern.MULTILINE);


//  private static final Pattern FUNCTION_DECL =
//    Pattern.compile("(^|;)\\s*((public|private|protected|final|static)\\s+)*" +
//                    "(void|int|float|double|String|char|byte)" +
//                    "(\\s*\\[\\s*\\])?\\s+[a-zA-Z0-9]+\\s*\\(",
//                    Pattern.MULTILINE);

  /**
   * Matches setup or draw function declaration. We search for all those
   * modifiers and return types in order to have proper error message
   * when people use incompatible modifiers or non-void return type
   */
  private static final Pattern SETUP_OR_DRAW_FUNCTION_DECL =
      Pattern.compile("(^|;)\\s*((public|private|protected|final|static)\\s+)*" +
                      "(void|int|float|double|String|char|byte|boolean)" +
                      "(\\s*\\[\\s*\\])?\\s+(setup|draw)\\s*\\(",
                      Pattern.MULTILINE);


  public PdePreprocessor(final String sketchName) {
    this(sketchName, Preferences.getInteger("editor.tabs.size"));
  }


  public PdePreprocessor(final String sketchName, final int tabSize) {
    this.name = sketchName;
    final char[] indentChars = new char[tabSize];
    Arrays.fill(indentChars, ' ');
    indent = new String(indentChars);
  }


  public SurfaceInfo initSketchSize(String code,
                                 boolean sizeWarning) throws SketchException {
    sizeInfo = parseSketchSize(code, sizeWarning);
    return sizeInfo;
  }


  /**
   * Break on commas, except those inside quotes,
   * e.g.: size(300, 200, PDF, "output,weirdname.pdf");
   * No special handling implemented for escaped (\") quotes.
   */
  static private StringList breakCommas(String contents) {
    StringList outgoing = new StringList();

    boolean insideQuote = false;
    // The current word being read
    StringBuilder current = new StringBuilder();
    char[] chars = contents.toCharArray();
    for (int i = 0; i < chars.length; i++) {
      char c = chars[i];
      if (insideQuote) {
        current.append(c);
        if (c == '\"') {
          insideQuote = false;
        }
      } else {
        if (c == ',') {
          if (current.length() != 0) {
            outgoing.append(current.toString());
            current.setLength(0);
          }
        } else {
          current.append(c);
          if (c == '\"') {
            insideQuote = true;
          }
        }
      }
    }
    if (current.length() != 0) {
      outgoing.append(current.toString());
    }
    return outgoing;
  }


  /**
   * Parse a chunk of code and extract the size() command and its contents.
   * Also goes after fullScreen(), smooth(), and noSmooth().
   * @param code The code from the main tab in the sketch
   * @param fussy true if it should show an error message if bad size()
   * @return null if there was an error, otherwise an array (might contain some/all nulls)
   */
  static public SurfaceInfo parseSketchSize(String code,
                                         boolean fussy) throws SketchException {
    // This matches against any uses of the size() function, whether numbers
    // or variables or whatever. This way, no warning is shown if size() isn't
    // actually used in the applet, which is the case especially for anyone
    // who is cutting/pasting from the reference.

//    String scrubbed = scrubComments(sketch.getCode(0).getProgram());
//    String[] matches = PApplet.match(scrubbed, SIZE_REGEX);
//    String[] matches = PApplet.match(scrubComments(code), SIZE_REGEX);

      /*
   1. no size() or fullScreen() method at all
      will use the non-overridden settings() method in PApplet
   2. size() or fullScreen() found inside setup() (static mode sketch or otherwise)
      make sure that it uses numbers (or displayWidth/Height), copy into settings
   3. size() or fullScreen() already in settings()
      don't mess with the sketch, don't insert any defaults

   really only need to deal with situation #2.. nothing to be done for 1 and 3
   */
    // if static mode sketch, all we need is regex
    // easy proxy for static in this case is whether [^\s]void\s is present

    String searchArea = scrubComments(code);
    String[] setupMatch = PApplet.match(searchArea, VOID_SETUP_REGEX);
    if (setupMatch != null) {
      String found = setupMatch[0];
      int start = searchArea.indexOf(found) + found.length();
      int openBrace = searchArea.indexOf("{", start);
      char[] c = searchArea.toCharArray();
      int depth = 0;
      int closeBrace = -1;
      StringBuilder sb = new StringBuilder();
      for (int i = openBrace; i < c.length; i++) {
        if (c[i] == '{') {
          depth++;
        } else if (c[i] == '\'') {
          String quoted = readSingleQuote(c, i);
          sb.append(quoted);
          i += quoted.length() - 1;

        } else if (c[i] == '\"') {
          String quoted = readDoubleQuote(c, i);
          sb.append(quoted);
          i += quoted.length() - 1;

        } else if (c[i] == '}') {
          depth--;
          if (depth == 0) {
            closeBrace = ++i;
            break;
          }
        } else {
          sb.append(c[i]);
        }
      }
      if (closeBrace == -1) {
        throw new SketchException("Found a { that's missing a matching }", false);
//        return null;
      }
      searchArea = sb.toString();
    }

    StringList extraStatements = new StringList();

    // First look for noSmooth() or smooth(N) so we can hoist it into settings.
    String[] smoothContents = matchMethod("smooth", searchArea);
    if (smoothContents != null) {
      extraStatements.append(smoothContents[0]);
    }
    String[] noContents = matchMethod("noSmooth", searchArea);
    if (noContents != null) {
      if (extraStatements.size() != 0) {
        throw new SketchException("smooth() and noSmooth() cannot be used in the same sketch");
      } else {
        extraStatements.append(noContents[0]);
      }
    }
    String[] pixelDensityContents = matchMethod("pixelDensity", searchArea);
    if (pixelDensityContents != null) {
      extraStatements.append(pixelDensityContents[0]);
    } else {
      pixelDensityContents = matchDensityMess(searchArea);
      if (pixelDensityContents != null) {
        extraStatements.append(pixelDensityContents[0]);
      }
    }

    String[] sizeContents = matchMethod("size", searchArea);
    String[] fullContents = matchMethod("fullScreen", searchArea);
    // First check and make sure they aren't both being used, otherwise it'll
    // throw a confusing state exception error that one "can't be used here".
    if (sizeContents != null && fullContents != null) {
      throw new SketchException("size() and fullScreen() cannot be used in the same sketch", false);
    }

    // Get everything inside the parens for the size() method
    //String[] contents = PApplet.match(searchArea, SIZE_CONTENTS_REGEX);
    if (sizeContents != null) {
      StringList args = breakCommas(sizeContents[1]);
      SurfaceInfo info = new SurfaceInfo();
//      info.statement = sizeContents[0];
      info.addStatement(sizeContents[0]);
      info.width = args.get(0).trim();
      info.height = args.get(1).trim();
      info.renderer = (args.size() >= 3) ? args.get(2).trim() : null;
      info.path = (args.size() >= 4) ? args.get(3).trim() : null;

      // Trying to remember why we wanted to allow people to use displayWidth
      // as the height or displayHeight as the width, but maybe it's for
      // making a square sketch window? Not going to

      if (info.hasOldSyntax()) {
//        return null;
        throw new SketchException("Please update your code to continue.", false);
      }

      if (info.hasBadSize() && fussy) {
        // found a reference to size, but it didn't seem to contain numbers
        final String message =
          "The size of this sketch could not be determined from your code.\n" +
          "Use only numbers (not variables) for the size() command.\n" +
          "Read the size() reference for more details.";
        Messages.showWarning("Could not find sketch size", message, null);
//        new Exception().printStackTrace(System.out);
//        return null;
        throw new SketchException("Please fix the size() line to continue.", false);
      }

      info.addStatements(extraStatements);
      info.checkEmpty();
      return info;
      //return new String[] { contents[0], width, height, renderer, path };
    }
    // if no size() found, check for fullScreen()
    //contents = PApplet.match(searchArea, FULL_SCREEN_CONTENTS_REGEX);
    if (fullContents != null) {
      SurfaceInfo info = new SurfaceInfo();
//      info.statement = fullContents[0];
      info.addStatement(fullContents[0]);
      StringList args = breakCommas(fullContents[1]);
      if (args.size() > 0) {  // might have no args
        String args0 = args.get(0).trim();
        if (args.size() == 1) {
          // could be either fullScreen(1) or fullScreen(P2D), figure out which
          if (args0.equals("SPAN") || PApplet.parseInt(args0, -1) != -1) {
            // it's the display parameter, not the renderer
            info.display = args0;
          } else {
            info.renderer = args0;
          }
        } else if (args.size() == 2) {
          info.renderer = args0;
          info.display = args.get(1).trim();
        } else {
          throw new SketchException("That's too many parameters for fullScreen()");
        }
      }
      info.width = "displayWidth";
      info.height = "displayHeight";
//      if (extraStatements.size() != 0) {
//        info.statement += extraStatements.join(" ");
//      }
      info.addStatements(extraStatements);
      info.checkEmpty();
      return info;
    }

    // Made it this far, but no size() or fullScreen(), and still
    // need to pull out the noSmooth() and smooth(N) methods.
    if (extraStatements.size() != 0) {
      SurfaceInfo info = new SurfaceInfo();
//      info.statement = extraStatements.join(" ");
      info.addStatements(extraStatements);
      return info;
    }

    // not an error, just no size() specified
    //return new String[] { null, null, null, null, null };
    return new SurfaceInfo();
  }


  static String readSingleQuote(char[] c, int i) {
    StringBuilder sb = new StringBuilder();
    try {
      sb.append(c[i++]);  // add the quote
      if (c[i] == '\\') {
        sb.append(c[i++]);  // add the escape
        if (c[i] == 'u') {
          // grabs uNNN and the fourth N will be added below
          for (int j = 0; j < 4; j++) {
            sb.append(c[i++]);
          }
        }
      }
      sb.append(c[i++]);  // get the char, escapee, or last unicode digit
      sb.append(c[i++]);  // get the closing quote

    } catch (ArrayIndexOutOfBoundsException ignored) {
      // this means they have bigger problems with their code
    }
    return sb.toString();
  }


  static String readDoubleQuote(char[] c, int i) {
    StringBuilder sb = new StringBuilder();
    try {
      sb.append(c[i++]);  // add the quote
      while (i < c.length) {
        if (c[i] == '\\') {
          sb.append(c[i++]);  // add the escape
          sb.append(c[i++]);  // add whatever was escaped
        } else if (c[i] == '\"') {
          sb.append(c[i++]);
          break;
        } else {
          sb.append(c[i++]);
        }
      }
    } catch (ArrayIndexOutOfBoundsException ignored) {
      // this means they have bigger problems with their code
    }
    return sb.toString();
  }


  static protected String[] matchMethod(String methodName, String searchArea) {
    final String left = "(?:^|\\s|;)";
    // doesn't match empty pairs of parens
    //final String right = "\\s*\\(([^\\)]+)\\)\\s*\\;";
    final String right = "\\s*\\(([^\\)]*)\\)\\s*\\;";
    return PApplet.match(searchArea, left + methodName + right);
  }


  static protected String[] matchDensityMess(String searchArea) {
    final String regexp =
      "(?:^|\\s|;)pixelDensity\\s*\\(\\s*displayDensity\\s*\\([^\\)]*\\)\\s*\\)\\s*\\;";
    return PApplet.match(searchArea, regexp);
  }


  /**
   * Replace all commented portions of a given String as spaces.
   * Utility function used here and in the preprocessor.
   */
  static public String scrubComments(String what) {
    char p[] = what.toCharArray();
    // Track quotes to avoid problems with code like: String t = "*/*";
    // http://code.google.com/p/processing/issues/detail?id=1435
    boolean insideQuote = false;

    int index = 0;
    while (index < p.length) {
      // for any double slash comments, ignore until the end of the line
      if (!insideQuote &&
          (p[index] == '/') &&
          (index < p.length - 1) &&
          (p[index+1] == '/')) {
        p[index++] = ' ';
        p[index++] = ' ';
        while ((index < p.length) &&
               (p[index] != '\n')) {
          p[index++] = ' ';
        }

        // check to see if this is the start of a new multiline comment.
        // if it is, then make sure it's actually terminated somewhere.
      } else if (!insideQuote &&
                 (p[index] == '/') &&
                 (index < p.length - 1) &&
                 (p[index+1] == '*')) {
        p[index++] = ' ';
        p[index++] = ' ';
        boolean endOfRainbow = false;
        while (index < p.length - 1) {
          if ((p[index] == '*') && (p[index+1] == '/')) {
            p[index++] = ' ';
            p[index++] = ' ';
            endOfRainbow = true;
            break;

          } else {
            // continue blanking this area
            p[index++] = ' ';
          }
        }
        if (!endOfRainbow) {
          throw new RuntimeException("Missing the */ from the end of a " +
                                     "/* comment */");
        }
      } else if (p[index] == '"' && index > 0 && p[index-1] != '\\') {
        insideQuote = !insideQuote;
        index++;

      } else {  // any old character, move along
        index++;
      }
    }
    return new String(p);
  }


  public void addMethod(String methodName) {
    foundMethods.add(methodName);
  }


  public boolean hasMethod(String methodName) {
    return foundMethods.contains(methodName);
  }


//  public void setFoundMain(boolean foundMain) {
//    this.foundMain = foundMain;
//  }


//  public boolean getFoundMain() {
//    return foundMain;
//  }


  public void setAdvClassName(final String advClassName) {
    this.advClassName = advClassName;
  }


  public void setMode(final Mode mode) {
    //System.err.println("Setting mode to " + mode);
    this.mode = mode;
  }


  CommonHiddenStreamToken getHiddenAfter(final CommonHiddenStreamToken t) {
    return filter.getHiddenAfter(t);
  }


  CommonHiddenStreamToken getInitialHiddenToken() {
    return filter.getInitialHiddenToken();
  }


  private static int countNewlines(final String s) {
    int count = 0;
    for (int pos = s.indexOf('\n', 0); pos >= 0; pos = s.indexOf('\n', pos + 1))
      count++;
    return count;
  }


  private static void checkForUnterminatedMultilineComment(final String program)
      throws SketchException {
    final int length = program.length();
    for (int i = 0; i < length; i++) {
      // for any double slash comments, ignore until the end of the line
      if ((program.charAt(i) == '/') && (i < length - 1)
          && (program.charAt(i + 1) == '/')) {
        i += 2;
        while ((i < length) && (program.charAt(i) != '\n')) {
          i++;
        }
        // check to see if this is the start of a new multiline comment.
        // if it is, then make sure it's actually terminated somewhere.
      } else if ((program.charAt(i) == '/') && (i < length - 1)
          && (program.charAt(i + 1) == '*')) {
        final int startOfComment = i;
        i += 2;
        boolean terminated = false;
        while (i < length - 1) {
          if ((program.charAt(i) == '*') && (program.charAt(i + 1) == '/')) {
            i += 2;
            terminated = true;
            break;
          } else {
            i++;
          }
        }
        if (!terminated) {
          throw new SketchException("Unclosed /* comment */", 0,
                                    countNewlines(program.substring(0,
                                      startOfComment)));
        }
      } else if (program.charAt(i) == '"') {
        final int stringStart = i;
        boolean terminated = false;
        for (i++; i < length; i++) {
          final char c = program.charAt(i);
          if (c == '"') {
            terminated = true;
            break;
          } else if (c == '\\') {
            if (i == length - 1) {
              break;
            }
            i++;
          } else if (c == '\n') {
            break;
          }
        }
        if (!terminated) {
          throw new SketchException("Unterminated string constant", 0,
                                    countNewlines(program.substring(0,
                                      stringStart)));
        }
      } else if (program.charAt(i) == '\'') {
        i++;  // step over the initial quote
        if (i >= length) {
          throw new SketchException("Unterminated character constant (after initial quote)", 0,
                                    countNewlines(program.substring(0, i)));
        }
        boolean escaped = false;
        if (program.charAt(i) == '\\') {
          i++;  // step over the backslash
          escaped = true;
        }
        if (i >= length) {
          throw new SketchException("Unterminated character constant (after backslash)", 0,
                                    countNewlines(program.substring(0, i)));
        }
        if (escaped && program.charAt(i) == 'u') {  // unicode escape sequence?
          i++;  // step over the u
          //i += 4;  // and the four digit unicode constant
          for (int j = 0; j < 4; j++) {
            if (UNICODE_ESCAPES.indexOf(program.charAt(i)) == -1) {
              throw new SketchException("Bad or unfinished \\uXXXX sequence " +
              		                      "(malformed Unicode character constant)", 0,
                                        countNewlines(program.substring(0, i)));
            }
            i++;
          }
        } else {
          i++;  // step over a single character
        }
        if (i >= length) {
          throw new SketchException("Unterminated character constant", 0,
                                    countNewlines(program.substring(0, i)));
        }
        if (program.charAt(i) != '\'') {
          throw new SketchException("Badly formed character constant " +
          		                      "(expecting quote, got " + program.charAt(i) + ")", 0,
                                    countNewlines(program.substring(0, i)));
        }
      }
    }
  }


  public PreprocessorResult write(final Writer out, String program)
      throws SketchException, RecognitionException, TokenStreamException {
    return write(out, program, null);
  }


  public PreprocessorResult write(Writer out, String program,
                                  StringList codeFolderPackages)
      throws SketchException, RecognitionException, TokenStreamException {

    // these ones have the .* at the end, since a class name might be at the end
    // instead of .* which would make trouble other classes using this can lop
    // off the . and anything after it to produce a package name consistently.
    final ArrayList<String> programImports = new ArrayList<String>();

    // imports just from the code folder, treated differently
    // than the others, since the imports are auto-generated.
    final ArrayList<String> codeFolderImports = new ArrayList<String>();

    // need to reset whether or not this has a main()
//    foundMain = false;
    foundMethods = new HashSet<String>();

    // http://processing.org/bugs/bugzilla/5.html
    if (!program.endsWith("\n")) {
      program += "\n";
    }

    checkForUnterminatedMultilineComment(program);

    if (Preferences.getBoolean("preproc.substitute_unicode")) {
      program = substituteUnicode(program);
    }

    // For 0215, adding } as a legitimate prefix to the import (along with
    // newline and semicolon) for cases where a tab ends with } and an import
    // statement starts the next tab.
    final String importRegexp =
      "((?:^|;|\\})\\s*)(import\\s+)((?:static\\s+)?\\S+)(\\s*;)";
    final Pattern importPattern = Pattern.compile(importRegexp);
    String scrubbed = scrubComments(program);
    Matcher m = null;
    int offset = 0;
    boolean found = false;
    do {
      m = importPattern.matcher(scrubbed);
      found = m.find(offset);
      if (found) {
//        System.out.println("found " + m.groupCount() + " groups");
        String before = m.group(1);
        String piece = m.group(2) + m.group(3) + m.group(4);
//        int len = piece.length(); // how much to trim out

        if (!ignoreImport(m.group(3))) {
          programImports.add(m.group(3)); // the package name
        }

        // find index of this import in the program
        int start = m.start() + before.length();
        int stop = start + piece.length();
//        System.out.println(start + " " + stop + " " + piece);
        //System.out.println("found " + m.group(3));
//        System.out.println("removing '" + program.substring(start, stop) + "'");

        // Remove the import from the main program
        program = program.substring(0, start) + program.substring(stop);
        scrubbed = scrubbed.substring(0, start) + scrubbed.substring(stop);
        // Set the offset to start, because everything between
        // start and stop has been deleted.
        offset = m.start();
      }
    } while (found);
//    System.out.println("program now:");
//    System.out.println(program);

    if (codeFolderPackages != null) {
      for (String item : codeFolderPackages) {
        codeFolderImports.add(item + ".*");
      }
    }

    final PrintWriter stream = new PrintWriter(out);
    final int headerOffset =
      writeImports(stream, programImports, codeFolderImports);
    return new PreprocessorResult(mode, headerOffset + 2,
                                  write(program, stream), programImports);
  }


  static String substituteUnicode(String program) {
    // check for non-ascii chars (these will be/must be in unicode format)
    char p[] = program.toCharArray();
    int unicodeCount = 0;
    for (int i = 0; i < p.length; i++) {
      if (p[i] > 127)
        unicodeCount++;
    }
    if (unicodeCount == 0)
      return program;
    // if non-ascii chars are in there, convert to unicode escapes
    // add unicodeCount * 5.. replacing each unicode char
    // with six digit uXXXX sequence (xxxx is in hex)
    // (except for nbsp chars which will be a replaced with a space)
    int index = 0;
    char p2[] = new char[p.length + unicodeCount * 5];
    for (int i = 0; i < p.length; i++) {
      if (p[i] < 128) {
        p2[index++] = p[i];
      } else if (p[i] == 160) { // unicode for non-breaking space
        p2[index++] = ' ';
      } else {
        int c = p[i];
        p2[index++] = '\\';
        p2[index++] = 'u';
        char str[] = Integer.toHexString(c).toCharArray();
        // add leading zeros, so that the length is 4
        //for (int i = 0; i < 4 - str.length; i++) p2[index++] = '0';
        for (int m = 0; m < 4 - str.length; m++)
          p2[index++] = '0';
        System.arraycopy(str, 0, p2, index, str.length);
        index += str.length;
      }
    }
    return new String(p2, 0, index);
  }


  /**
   * preprocesses a pde file and writes out a java file
   * @return the class name of the exported Java
   */
  private String write(final String program, final PrintWriter stream)
      throws SketchException, RecognitionException, TokenStreamException {

    // Match on the uncommented version, otherwise code inside comments used
    // http://code.google.com/p/processing/issues/detail?id=1404
    String uncomment = scrubComments(program);
    PdeRecognizer parser = createParser(program);
    if (PUBLIC_CLASS.matcher(uncomment).find()) {
      try {
        final PrintStream saved = System.err;
        try {
          // throw away stderr for this tentative parse
          System.setErr(new PrintStream(new ByteArrayOutputStream()));
          parser.javaProgram();
        } finally {
          System.setErr(saved);
        }
        setMode(Mode.JAVA);
      } catch (Exception e) {
        // I can't figure out any other way of resetting the parser.
        parser = createParser(program);
        parser.pdeProgram();
      }
    } else if (SETUP_OR_DRAW_FUNCTION_DECL.matcher(uncomment).find()) {
      setMode(Mode.ACTIVE);
      parser.activeProgram();
    } else {
      parser.pdeProgram();
    }

    // set up the AST for traversal by PdeEmitter
    //
    ASTFactory factory = new ASTFactory();
    AST parserAST = parser.getAST();
    AST rootNode = factory.create(ROOT_ID, "AST ROOT");
    rootNode.setFirstChild(parserAST);

    makeSimpleMethodsPublic(rootNode);

    // unclear if this actually works, but it's worth a shot
    //
    //((CommonAST)parserAST).setVerboseStringConversion(
    //  true, parser.getTokenNames());
    // (made to use the static version because of jikes 1.22 warning)
    BaseAST.setVerboseStringConversion(true, parser.getTokenNames());

    final String className;
    if (mode == Mode.JAVA) {
      // if this is an advanced program, the classname is already defined.
      className = getFirstClassName(parserAST);
    } else {
      className = this.name;
    }

    // if 'null' was passed in for the name, but this isn't
    // a 'java' mode class, then there's a problem, so punt.
    //
    if (className == null)
      return null;

    // debug
    if (false) {
      final StringWriter buf = new StringWriter();
      final PrintWriter bufout = new PrintWriter(buf);
      writeDeclaration(bufout, className);
      new PdeEmitter(this, bufout).print(rootNode);
      writeFooter(bufout, className);
      debugAST(rootNode, true);
      System.err.println(buf.toString());
    }

    writeDeclaration(stream, className);
    new PdeEmitter(this, stream).print(rootNode);
    writeFooter(stream, className);

    // if desired, serialize the parse tree to an XML file.  can
    // be viewed usefully with Mozilla or IE
    if (Preferences.getBoolean("preproc.output_parse_tree")) {
      writeParseTree("parseTree.xml", parserAST);
    }

    return className;
  }


  private PdeRecognizer createParser(final String program) {
    // create a lexer with the stream reader, and tell it to handle
    // hidden tokens (eg whitespace, comments) since we want to pass these
    // through so that the line numbers when the compiler reports errors
    // match those that will be highlighted in the PDE IDE
    //
    PdeLexer lexer = new PdeLexer(new StringReader(program));
    lexer.setTokenObjectClass("antlr.CommonHiddenStreamToken");

    // create the filter for hidden tokens and specify which tokens to
    // hide and which to copy to the hidden text
    //
    filter = new TokenStreamCopyingHiddenTokenFilter(lexer);
    filter.hide(PdePartialTokenTypes.SL_COMMENT);
    filter.hide(PdePartialTokenTypes.ML_COMMENT);
    filter.hide(PdePartialTokenTypes.WS);
    filter.copy(PdePartialTokenTypes.SEMI);
    filter.copy(PdePartialTokenTypes.LPAREN);
    filter.copy(PdePartialTokenTypes.RPAREN);
    filter.copy(PdePartialTokenTypes.LCURLY);
    filter.copy(PdePartialTokenTypes.RCURLY);
    filter.copy(PdePartialTokenTypes.COMMA);
    filter.copy(PdePartialTokenTypes.RBRACK);
    filter.copy(PdePartialTokenTypes.LBRACK);
    filter.copy(PdePartialTokenTypes.COLON);
    filter.copy(PdePartialTokenTypes.TRIPLE_DOT);

    // Because the meanings of < and > are overloaded to support
    // type arguments and type parameters, we have to treat them
    // as copyable to hidden text (or else the following syntax,
    // such as (); and what not gets lost under certain circumstances)
    // -- jdf
    filter.copy(PdePartialTokenTypes.LT);
    filter.copy(PdePartialTokenTypes.GT);
    filter.copy(PdePartialTokenTypes.SR);
    filter.copy(PdePartialTokenTypes.BSR);

    // create a parser and set what sort of AST should be generated
    //
    final PdeRecognizer parser = new PdeRecognizer(this, filter);

    // use our extended AST class
    //
    parser.setASTNodeClass("antlr.ExtendedCommonASTWithHiddenTokens");
    return parser;
  }

  /**
   * Walk the tree looking for METHOD_DEFs. Any simple METHOD_DEF (one
   * without TYPE_PARAMETERS) lacking an
   * access specifier is given public access.
   * @param node
   */
  private void makeSimpleMethodsPublic(final AST node) {
    if (node.getType() == PdeTokenTypes.METHOD_DEF) {
      final AST mods = node.getFirstChild();
      final AST oldFirstMod = mods.getFirstChild();
      for (AST mod = oldFirstMod; mod != null; mod = mod.getNextSibling()) {
        final int t = mod.getType();
        if (t == PdeTokenTypes.LITERAL_private ||
            t == PdeTokenTypes.LITERAL_protected ||
            t == PdeTokenTypes.LITERAL_public) {
          return;
        }
      }
      if (mods.getNextSibling().getType() == PdeTokenTypes.TYPE_PARAMETERS) {
        return;
      }
      final CommonHiddenStreamToken publicToken =
        new CommonHiddenStreamToken(PdeTokenTypes.LITERAL_public, "public") {
        {
          setHiddenAfter(new CommonHiddenStreamToken(PdeTokenTypes.WS, " "));
        }
      };
      final AST publicNode = new CommonASTWithHiddenTokens(publicToken);
      publicNode.setNextSibling(oldFirstMod);
      mods.setFirstChild(publicNode);
    } else {
      for (AST kid = node.getFirstChild(); kid != null; kid = kid
          .getNextSibling())
        makeSimpleMethodsPublic(kid);
    }
  }

  protected void writeParseTree(String filename, AST ast) {
    try {
      PrintStream stream = new PrintStream(new FileOutputStream(filename));
      stream.println("<?xml version=\"1.0\"?>");
      stream.println("<document>");
      OutputStreamWriter writer = new OutputStreamWriter(stream);
      if (ast != null) {
        ((CommonAST) ast).xmlSerialize(writer);
      }
      writer.flush();
      stream.println("</document>");
      writer.close();
    } catch (IOException e) {

    }
  }

  /**
   *
   * @param out
   * @param programImports
   * @param codeFolderImports
   * @return the header offset
   */
  protected int writeImports(final PrintWriter out,
                             final List<String> programImports,
                             final List<String> codeFolderImports) {
    int count = writeImportList(out, getCoreImports());
    count += writeImportList(out, programImports);
    count += writeImportList(out, codeFolderImports);
    count += writeImportList(out, getDefaultImports());
    return count;
  }

  protected int writeImportList(PrintWriter out, List<String> imports) {
    return writeImportList(out, imports.toArray(new String[0]));
  }

  protected int writeImportList(PrintWriter out, String[] imports) {
    int count = 0;
    if (imports != null && imports.length != 0) {
      for (String item : imports) {
        out.println("import " + item + "; ");
        count++;
      }
      out.println();
      count++;
    }
    return count;
  }

  /**
   * Write any required header material (eg imports, class decl stuff)
   *
   * @param out                 PrintStream to write it to.
   * @param exporting           Is this being exported from PDE?
   * @param className           Name of the class being created.
   */
  protected void writeDeclaration(PrintWriter out, String className) {
    if (mode == Mode.JAVA) {
      // Print two blank lines so that the offset doesn't change
      out.println();
      out.println();

    } else if (mode == Mode.ACTIVE) {
      // Print an extra blank line so the offset is identical to the others
      out.println("public class " + className + " extends PApplet {");
      out.println();

    } else if (mode == Mode.STATIC) {
      out.println("public class " + className + " extends PApplet {");
      out.println(indent + "public void setup() {");
    }
  }

  /**
   * Write any necessary closing text.
   *
   * @param out PrintStream to write it to.
   */
  protected void writeFooter(PrintWriter out, String className) {
    if (mode == Mode.STATIC) {
      // close off setup() definition
      out.println(indent + indent + "noLoop();");
      out.println(indent + "}");
      out.println();
    }

    if ((mode == Mode.STATIC) || (mode == Mode.ACTIVE)) {
      // doesn't remove the original size() method, but calling size()
      // again in setup() is harmless.
      if (!hasMethod("settings") && sizeInfo.hasSettings()) {
        out.println(indent + "public void settings() { " + sizeInfo.getSettings() + " }");
//        out.println(indent + "public void settings() {");
//        out.println(indent + indent + sizeStatement);
//        out.println(indent + "}");
      }
      /*
      if (sketchWidth != null && !hasMethod("sketchWidth")) {
        // Only include if it's a number (a variable will be a problem)
        if (PApplet.parseInt(sketchWidth, -1) != -1 || sketchWidth.equals("displayWidth")) {
          out.println(indent + "public int sketchWidth() { return " + sketchWidth + "; }");
        }
      }
      if (sketchHeight != null && !hasMethod("sketchHeight")) {
        // Only include if it's a number
        if (PApplet.parseInt(sketchHeight, -1) != -1 || sketchHeight.equals("displayHeight")) {
          out.println(indent + "public int sketchHeight() { return " + sketchHeight + "; }");
        }
      }
      if (sketchRenderer != null && !hasMethod("sketchRenderer")) {
        // Only include if it's a known renderer (otherwise it might be a variable)
        //if (PConstants.rendererList.hasValue(sketchRenderer)) {
        out.println(indent + "public String sketchRenderer() { return " + sketchRenderer + "; }");
        //}
      }
      if (sketchOutputPath != null && !hasMethod("sketchOutputPath")) {
        out.println(indent + "public String sketchOutputPath() { return " + sketchOutputPath + "; }");
      }
      */

      if (!hasMethod("main")) {
        out.println(indent + "static public void main(String[] passedArgs) {");
        //out.print(indent + indent + "PApplet.main(new String[] { ");
        out.print(indent + indent + "String[] appletArgs = new String[] { ");

        if (Preferences.getBoolean("export.application.present")) {
          out.print("\"" + PApplet.ARGS_PRESENT + "\", ");

          String farbe = Preferences.get("run.present.bgcolor");
          out.print("\"" + PApplet.ARGS_WINDOW_COLOR + "=" + farbe + "\", ");

          if (Preferences.getBoolean("export.application.stop")) {
            farbe = Preferences.get("run.present.stop.color");
            out.print("\"" + PApplet.ARGS_STOP_COLOR + "=" + farbe + "\", ");
          } else {
            out.print("\"" + PApplet.ARGS_HIDE_STOP + "\", ");
          }
//        } else {
//          // This is set initially based on the system control color, just
//          // sets the color for what goes behind the sketch before it's added.
//          String farbe = Preferences.get("run.window.bgcolor");
//          out.print("\"" + PApplet.ARGS_BGCOLOR + "=" + farbe + "\", ");
        }
        out.println("\"" + className + "\" };");

        out.println(indent + indent + "if (passedArgs != null) {");
        out.println(indent + indent + "  PApplet.main(concat(appletArgs, passedArgs));");
        out.println(indent + indent + "} else {");
        out.println(indent + indent + "  PApplet.main(appletArgs);");
        out.println(indent + indent + "}");

        out.println(indent + "}");
      }

      // close off the class definition
      out.println("}");
    }
  }

  public String[] getCoreImports() {
    return new String[] {
      "processing.core.*",
      "processing.data.*",
      "processing.event.*",
      "processing.opengl.*"
    };
  }

  public String[] getDefaultImports() {
    // These may change in-between (if the prefs panel adds this option)
    //String prefsLine = Preferences.get("preproc.imports");
    //return PApplet.splitTokens(prefsLine, ", ");
    return new String[] {
      "java.util.HashMap",
      "java.util.ArrayList",
      "java.io.File",
      "java.io.BufferedReader",
      "java.io.PrintWriter",
      "java.io.InputStream",
      "java.io.OutputStream",
      "java.io.IOException"
    };
  }

  /**
   * Return true if this import should be removed from the code. This is used
   * for packages like processing.xml which no longer exist.
   * @param pkg something like processing.xml.XMLElement or processing.xml.*
   * @return true if this shouldn't be added to the final code
   */
  public boolean ignoreImport(String pkg) {
    return false;
//    return pkg.startsWith("processing.xml.");
  }


  // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .


  /**
   * Find the first CLASS_DEF node in the tree, and return the name of the
   * class in question.
   *
   * TODO [dmose] right now, we're using a little hack to the grammar to get
   * this info.  In fact, we should be descending the AST passed in.
   */
  String getFirstClassName(AST ast) {
    String t = advClassName;
    advClassName = "";
    return t;
  }


  public void debugAST(final AST ast, final boolean includeHidden) {
    System.err.println("------------------");
    debugAST(ast, includeHidden, 0);
  }


  private void debugAST(final AST ast, final boolean includeHidden,
                        final int indent) {
    for (int i = 0; i < indent; i++)
      System.err.print("    ");
    if (includeHidden) {
      System.err.print(debugHiddenBefore(ast));
    }
    if (ast.getType() > 0 && !ast.getText().equals(TokenUtil.nameOf(ast))) {
      System.err.print(TokenUtil.nameOf(ast) + "/");
    }
    System.err.print(ast.getText().replace("\n", "\\n"));
    if (includeHidden) {
      System.err.print(debugHiddenAfter(ast));
    }
    System.err.println();
    for (AST kid = ast.getFirstChild(); kid != null; kid = kid.getNextSibling())
      debugAST(kid, includeHidden, indent + 1);
  }


  private String debugHiddenAfter(AST ast) {
    return (ast instanceof antlr.CommonASTWithHiddenTokens) ?
      debugHiddenTokens(((antlr.CommonASTWithHiddenTokens) ast).getHiddenAfter()) : "";
  }

  private String debugHiddenBefore(AST ast) {
    if (!(ast instanceof antlr.CommonASTWithHiddenTokens)) {
      return "";
    }
    antlr.CommonHiddenStreamToken parent =
      ((antlr.CommonASTWithHiddenTokens) ast).getHiddenBefore();

    if (parent == null) {
      return "";
    }

    antlr.CommonHiddenStreamToken child = null;
    do {
      child = parent;
      parent = child.getHiddenBefore();
    } while (parent != null);

    return debugHiddenTokens(child);
  }


  private String debugHiddenTokens(antlr.CommonHiddenStreamToken t) {
    final StringBuilder sb = new StringBuilder();
    for (; t != null; t = filter.getHiddenAfter(t)) {
      if (sb.length() == 0) {
        sb.append("[");
      }
      sb.append(t.getText().replace("\n", "\\n"));
    }
    if (sb.length() > 0) {
      sb.append("]");
    }
    return sb.toString();
  }
}