ParseTree.pm   [plain text]

#! /usr/bin/perl -w
#
# Class name: 	ParseTree
# Synopsis: 	Used by headerdoc2html to hold parse trees
# Last Updated: $Date: 2012/04/05 16:29:36 $
# 
# Copyright (c) 1999-2004 Apple Computer, Inc.  All rights reserved.
#
# @APPLE_LICENSE_HEADER_START@
#
# This file contains Original Code and/or Modifications of Original Code
# as defined in and that are subject to the Apple Public Source License
# Version 2.0 (the 'License'). You may not use this file except in
# compliance with the License. Please obtain a copy of the License at
# http://www.opensource.apple.com/apsl/ and read it before using this
# file.
# 
# The Original Code and all software distributed under the License are
# distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
# EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
# INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
# Please see the License for the specific language governing rights and
# limitations under the License.
#
# @APPLE_LICENSE_HEADER_END@
#
######################################################################

# /*!
#     @header
#     @abstract
#         <code>ParseTree</code> class package file.
#     @discussion
#         This module contains the <code>ParseTree</code> class, a data
#         structure for holding the parse tree as generated by
#  {@link //apple_ref/perl/instm/HeaderDoc::BlockParse/blockParse//() blockParse}.
#
#         See the class documentation below for more details.
#     @indexgroup HeaderDoc Parser Pieces
#  */

# /*!
#     @abstract
#         Parse tree (or token therein) generated by parser.
#     @discussion
#         The <code>ParseTree</code> class is a data structure
#         for holding the parse tree as generated by
#  {@link //apple_ref/perl/instm/HeaderDoc::BlockParse/blockParse//() blockParse}.
#
#         In addition, this class provides a series of
#         utility functions for manipulating the
#         parse tree, generating output from the
#         parse tree in HTML, XML, and plain text
#         formats, and extracting Objective-C
#         method parameter lists.
#
#  @vargroup Node data and tree pointers
#     @var TOKEN
#         The token itself.
#     @var NEXT
#         The next (sibling) node in the parse tree.
#     @var FIRSTCHILD
#         The first child (descendant) node of this node.
#     @var PARENT
#         The parent node in the parse tree.
#
#  @vargroup File information
#     @var FILENAME
#         The filename component of the header or source file that contains
#         the API content that this parse tree describes.  Does not include
#         leading path parts.
#     @var FULLPATH
#         The full (possibly relative) path to the header or source file that
#         contains the the API content that this parse tree describes.
#     @var INPUTCOUNTER
#         The line number where this tree object appeared relative to the
#         start of the {@link //apple_ref/perl/cl/HeaderDoc::LineRange LineRange}
#         object.
#     @var BLOCKOFFSET
#         The line number of the first line of the
#         {@link //apple_ref/perl/cl/HeaderDoc::LineRange LineRange} object.
#     @var LINENUM
#         The actual line number in the header (<code>$self->{INPUTCOUNTER} +
#         $self->{BLOCKOFFSET}</code>).
#
#  @vargroup Administrative information
#     @var REFCOUNT
#         The number of API objects that reference this parse tree.  The
#         parse tree cannot be purged until this count goes to zero.
#     @var APIOWNERS
#         An array of API objects that reference the parse tree rooted in
#         this node.
#     @var RAWPARSEDPARAMETERS
#         An array of parsed parameters associated with this node.
#         Each entry in this array is a string containing the
#         text of a parameter.  This is currently unused.
#     @var PARSEDPARAMS
#         An array of parsed parameter objects associated with this node.
#         Each entry in this array should be a MinorAPIElement object,
#         but right now, it's a mixture of objects and strings.
#         This is not currrently used for anything, but may be used in
#         the future once support is fully integrated in the parser.
#     @var PETDONE
#         Set to 1 by {@link processEmbeddedTags} to indicate that this
#         parse tree has already been processed.  This prevents duplicate
#         insertion of symbols nested within classes.
#
#  @vargroup Token attributes
#     @var HIDEMACROLASTTOKEN
#         Set to 1 on the space or 2 on the curly brace after a
#         macro name.  Used to hide the contents of a macro for
#         output purposes, depending on flags.
#     @var ACCESSCONTROLSTATE
#         The access control state in force at the time this node was created.
#     @var PARSERSTATE
#         A parser state object stored at this node in the parse tree.
#         used for reparsing class contents.
#     @var HIDDEN
#         Indicates that this node should not be emitted in HTML output.
#     @var ISAVAILABILITYMACRO
#         Indicates that this node is an availability macro.  This
#         is used for wrapping content in both HTML and XML.
#     @var RE_STATE
#         Indicates that this node is part of a regular expression.
#         This value contains one of the following values:
#
#         <code>RE_PREFIX</code> (the optional "command" letters before
#         the expression)
#
#         <code>RE_START</code> (the opening slash or other delimiter)
#
#         <code>RE_PARTSEP</code> (the intermediate delimiter or delimiters)
#
#         <code>RE_END</code> (the trailing slash or other delimiter).
#
#         Note that the trailing flags are not tagged in any special way
#         because you can trivially distinguish the parse tree node that
#         contains them by inspecting the node after <code>RE_END</code>.
#
#  @vargroup Caches
#
#     @var HTMLTREE
#         A cached copy of the HTML tree for the parse tree rooted in this node,
#         as generated by {@link htmlTree}.
#     @var XMLTREE
#         A cached copy of the XML tree for the parse tree rooted in this node,
#         as generated by {@link xmlTree}.
#     @var CPNC
#         A cached copy of the results of {@link childpeeknc} for this node.
#     @var NTNC
#         A cached copy of the results of {@link nextTokenNoComments} for this
#         node.
# */
package HeaderDoc::ParseTree;

use strict;
use vars qw($VERSION @ISA);
use HeaderDoc::Utilities qw(isKeyword parseTokens stringToFields casecmp emptyHDok complexAvailabilityToArray printHash validTag);
use HeaderDoc::BlockParse qw(blockParse nspaces);
use Carp qw(cluck);

# use WeakRef;

# /*!
#     @abstract
#         The revision control revision number for this module.
#     @discussion
#         In the git repository, contains the number of seconds since
#         January 1, 1970.
#  */
$HeaderDoc::ParseTree::VERSION = '$Revision: 1333668576 $';
################ General Constants ###################################
my $debugging = 0;

my $apioDebug = 0;

# 0 = normal; 1 = debug; 2 = debug + backtraces
my $treeDebug = 0;

my %defaults = (
	# TOKEN => undef,
	# NEXT => undef,
	# FIRSTCHILD => undef,
	# APIOWNERS => undef,
	# PARSEDPARAMS => undef,
	PETDONE => 0,
	REFCOUNT => 0,
	# XMLTREE => undef,
	# HTMLTREE => undef,
	# CPNC => undef,
	# NTNC => undef,
	# RAWPARSEDPARAMETERS => undef,
	# PARSERSTATE => undef,
	# ACCESSCONTROLSTATE => undef,
	# FILENAME => undef,
	# LINENUM => undef,
	# HIDDEN => undef
    );

# /*!
#     @abstract
#         Creates a new <code>ParseTree</code> object.
#     @param param
#         A reference to the relevant package object (e.g.
#         <code>HeaderDoc::ParseTree->new()</code> to allocate
#         a new instance of this class).
#  */
sub new {
    my($param) = shift;
    my($class) = ref($param) || $param;
    my %selfhash = %defaults;
    my $self = \%selfhash;

    # cluck("Parse tree $self created\n");
    
    bless($self, $class);
    $self->_initialize();
    # Now grab any key => value pairs passed in
    my (%attributeHash) = @_;
    foreach my $key (keys(%attributeHash)) {
        $self->{$key} = $attributeHash{$key};
    }  
    return ($self);
}

# /*!
#     @abstract
#         Initializes an instance of a <code>ParseTree</code> object.
#     @param self
#         The object to initialize.
#  */
sub _initialize {
    my($self) = shift;

    $self->{ACCESSCONTROLSTATE} = $HeaderDoc::AccessControlState;
    $self->{FILENAME} = $HeaderDoc::headerObject->filename();
    $self->{FULLPATH} = $HeaderDoc::headerObject->fullpath();
    $self->{LINENUM} = $HeaderDoc::CurLine;
    $self->{HIDDEN} = $HeaderDoc::hidetokens;
    $self->{APIOWNERS} = ();
    $self->{PARSEDPARAMS} = ();
    $self->{RAWPARSEDPARAMETERS} = ();

    return;

    my($self) = shift;
    # $self->{TOKEN} = undef;
    # $self->{NEXT} = undef;
    # $self->{FIRSTCHILD} = undef;
    $self->{APIOWNERS} = ();
    $self->{ACCESSCONTROLSTATE} = $HeaderDoc::AccessControlState;
    $self->{PARSEDPARAMS} = ();
    $self->{FILENAME} = $HeaderDoc::headerObject->filename();
    $self->{FULLPATH} = $HeaderDoc::headerObject->fullpath();
    $self->{LINENUM} = $HeaderDoc::CurLine; # $HeaderDoc::headerObject->linenum();
    # $self->{PETDONE} = 0;
    $self->{HIDDEN} = $HeaderDoc::hidetokens;
    $self->{REFCOUNT} = 0;
    # $self->{XMLTREE} = undef;
    # $self->{HTMLTREE} = undef;
    # $self->{CPNC} = undef;
    # # $self->{NPCACHE} = undef;
    # $self->{NTNC} = undef;
    # # $self->{CTSUB} = undef;
    # # $self->{CTSTRING} = undef;
    $self->{RAWPARSEDPARAMETERS} = ();
    $self->{PARSERSTATE} = undef; # $HeaderDoc::curParserState;
}

my $colorDebug = 0;

# /*!
#     @abstract
#         Duplicates this <code>ParseTree</code> object into another one.
#     @param self
#         The object to clone.
#     @param clone
#         The victim object.
#  */
sub clone {
    my $self = shift;
    my $clone = undef;
    if (@_) {
        $clone = shift;
    } else {
        $clone = HeaderDoc::ParseTree->new();
    }

    # $self->SUPER::clone($clone);

    # now clone stuff specific to ParseTree

    $clone->{TOKEN} = $self->{TOKEN};

    # Note: apiOwner is no longer recursive, so there is no need
    # to recursively clone parse trees.  Only the top node will
    # ever be modified legitimately (except when pruning headerdoc
    # comments, but that needs to occur for all instances).

    $clone->{FIRSTCHILD} = $self->{FIRSTCHILD};
    $clone->{NEXT} = $self->{NEXT};

    # $clone->{FIRSTCHILD} = undef;
    # if ($self->{FIRSTCHILD}) {
	# my $firstchild = $self->{FIRSTCHILD};
	# $clone->{FIRSTCHILD} = $firstchild->clone();
    # }
    # $clone->{NEXT} = undef;
    # if ($self->{NEXT}) {
	# my $next = $self->{NEXT};
	# $clone->{NEXT} = $next->clone();
    # }

    $clone->{APIOWNERS} = $self->{APIOWNERS};
    $clone->{PARSEDPARAMS} = $self->{PARSEDPARAMS};
    $clone->{PETDONE} = 0;

    return $clone;
}

# /*!
#     @abstract
#         Adds a sibling to this node in the parse tree.
#     @param self
#         The current parse tree node.
#     @param name
#         The token value for the new node.
#     @param hide
#         Pass 1 if this node (and its children) should be
#         hidden when emitting the tree as HTML, else 0.
#     @result
#         Returns the new sibling node.
#  */
sub addSibling
{
    my $self = shift;
    my $name = shift;
    my $hide = shift;
    my $newnode = HeaderDoc::ParseTree->new();
    my $localDebug = 0;

# print STDERR "addSibling $self\n";
    print STDERR "addSibling $self \"$name\" HIDDEN: $hide\n" if ($treeDebug || $localDebug);
    if ($treeDebug == 2) { cluck("backtrace"); }

    # Always hide siblings of descendants of elements that the parser
    # tells us to hide.  Also spaces if they are the immediate successor
    # to the element right after a hidden element, regardless.
    if ($self->hidden() == 2) {
	$hide = 2;
    } elsif ($name =~ /^\s+$/ && $self->hidden()) {
	$hide = 1;
    }

    print STDERR "HIDE NOW $hide\n" if ($treeDebug || $localDebug);

    my $parent = $self->parent;

    my $pos = $self;
    # if ($parent) {
	# $pos = $parent->lastchild();
	# bless($pos, "HeaderDoc::ParseTree");
    # } else {
	# warn "NOPARENTA!\nNOPARENTB!\nNOPARENTC!\n";
	bless($pos, "HeaderDoc::ParseTree");

	# print STDERR "POS: $pos\n";
	# while ($pos && $pos->next()) {
		# $pos = $pos->next();
		# bless($pos, "HeaderDoc::ParseTree");
		# # print STDERR "POS: $pos: ".$pos->token()."\n";
	# }
    	# bless($pos, "HeaderDoc::ParseTree");
    # }
    $newnode->token($name);
    if ($hide) { $newnode->hidden($hide); }
    $newnode->parent($parent);
    # if ($parent) { $parent->lastchild($newnode); }

    my $noderef = $newnode;

    my $retval = $pos->next($noderef);
    print STDERR "New sibling: ".$retval if ($treeDebug);
    return $retval;
	# print STDERR "$self (".$self->token().") RET $ret (".$ret->token().")\n";
    # return $ret;
}

# /*!
#     @abstract
#         Adds a child to this node in the parse tree.
#     @param self
#         The current parse tree node.
#     @param name
#         The token value for the new node.
#     @param hide
#         Pass 1 if this node (and its children) should be
#         hidden when emitting the tree as HTML, else 0.
#     @result
#         Returns the new child node.
#  */
sub addChild
{
    my $self = shift;
    my $name = shift;
    my $hide = shift;

# print STDERR "addChild\n";
    print STDERR "addChild to $self \"$name\"\n" if ($treeDebug);
    if ($treeDebug == 2) { cluck("backtrace"); }

    # If the parser wants a node hidden, any children of such a node
    # should be hidden, as should any siblings of those children or their
    # descendants.  Siblings of the original node should not, however.
    #
    # The block parser passes in 3 for $hide to hide the node and its
    # descendants.  Its descendants have their hidden value set to 2 so
    # that their siblings will also be hidden, but the top level node
    # still retains the original hidden value of 3.
    if ($self->hidden() == 2 || $self->hidden() == 3) { $hide = 2; }

    if (!$self->firstchild()) {
	my $newnode = HeaderDoc::ParseTree->new();
	if ($hide) { $newnode->hidden($hide); }
	$newnode->token($name);
	my $noderef = $newnode;
	$newnode->parent($self);
	# $self->lastchild($noderef);
	my $retval = $self->firstchild($noderef);
	print STDERR "New child: ".$retval if ($treeDebug);
	return $retval; # $self->firstchild($noderef);
    } else {
	warn "addChild called when firstchild exists.  Dropping.\n";
	# my $node = $self->firstchild();
	# bless($node, "HeaderDoc::ParseTree");
	# return $node->addSibling($name, $hide);
    }
}

# /*! @abstract
#         Checks to see if this node is at the same level of the parse tree
#         and occurs after the node specified (or is the node specified).
#     @param self
#         The current node.
#     @param node
#         The specified node for comparison.
#  */
sub isAfter
{
	my $self = shift;
	my $node = shift;

	my $ptr = $node;
	while ($ptr) {
		if ($ptr == $self) {
			return 1;
		}
		$ptr = $ptr->next();
	}
	return 0;
}

# /*! @abstract
#         Adds an additional API owner for a tree.
#     @param self
#         This node.
#     @param newapio
#         The API owner to add.
#  */
sub addAPIOwner {
    my $self = shift;
    my $newapio = shift;

    print STDERR "addAPIOwner: SELF WAS $self\n" if ($apioDebug);
    print STDERR "addAPIOwner: APIO WAS $newapio\n" if ($apioDebug);
    if (!$newapio) {
	warn("apiOwner called with empty APIO!\n");
	return undef;
    } else {
	$self->{REFCOUNT}++;
	cluck("REFCOUNT $self -> ".$self->{REFCOUNT}." because of $newapio (add)\n") if ($HeaderDoc::debugAllocations);
	# weaken($newapio);
	push(@{$self->{APIOWNERS}}, $newapio);
    }

    return $newapio;
}

# /*!
#     @abstract
#         Replaces an API owner for the tree.
#     @param self
#         This tree node.
#     @param old
#         The previous API owner to delete.
#     @param new
#         The new API owner to insert.
#  */
sub apiOwnerSub
 {
    my $self = shift;
    my $old = shift;
    my $new = shift;
    my $quiet = shift;

    my $localDebug = 0;

    cluck("apiOwnerSub called with SELF=$self OLD=$old NEW=$new\n") if ($localDebug);

    my @arr = ();

    my $found = 0;
    foreach my $possowner (@{$self->{APIOWNERS}}) {
	print "IN apiOwnerSub: $possowner\n" if ($HeaderDoc::debugAllocations);
	if ($possowner == $new) {
		$new = undef; # lower refcount later.
	}
	if ($possowner != $old) {
		push(@arr, $possowner);
	} else {
		$found++;
	}
    }
    if (!$found && !$quiet) {
	warn("OLD API OWNER NOT FOUND IN apiOwnerSub().  Please file a bug.\n");
    }
    if ($new) {
	push(@arr, $new);
    } elsif ($found) {
	$self->{REFCOUNT} -= $found;
	print STDERR "REFCOUNT $self -> ".$self->{REFCOUNT}." because of $old going away\n" if ($HeaderDoc::debugAllocations);
	if (!$self->{REFCOUNT}) {
		cluck("Ref count went to zero in apiOwnerSub($self)\n") if ($HeaderDoc::debugAllocations);
	}
    }

    $self->{APIOWNERS} = \@arr; 
}

# /*!
#     @abstract
#         Creates a new state object for use by colorTreeSub.
#     @param withref
#         
#     @param morevalues
#         Additional values to add.
#
#     @discussion
#         This greatly reduces the number of arguments to
#         {@link colorTreeSub}.  It has no other purpose beyond making
#         the code more readable by treating the current
#         state as an object.
#
#         Keys and legal values are described as local variables for
#         {@link colorTreeSub}.
#  */
sub newCTState
{
	my %statedata = ();

	if (@_) {
		my $withref = shift;
		if ($withref) {
			my %withvalues = %{$withref};
			foreach my $key (keys %withvalues) {
				$statedata{$key} = $withvalues{$key};
			}
		}
		if (@_) {
			my (%morevalues) = @_;
			foreach my $key (keys(%morevalues)) {
				$statedata{$key} = $morevalues{$key};
			}
		}  
	}

	return \%statedata;
}

# /*!
#     @abstract
#         Gests/sets the API owner for the tree.
#     @param self
#         This tree node.
#     @param newapio
#         The new API owner value. (Optional.)
#  */
sub apiOwner {
    my $self = shift;

    my $localDebug = 0;

    if (@_) {
	my $newapio = shift;

	if (!$newapio) {
		warn("apiOwner called with empty APIO!\n");
	}

	print STDERR "apiOwner: SETTING TO $newapio (".$newapio->rawname().")\n" if ($apioDebug || $localDebug);

	$self->{APIOWNERS} = ();
        push(@{$self->{APIOWNERS}}, $newapio);
	$self->{REFCOUNT} = 1;
	print STDERR "REFCOUNT $self -> 1 because of $newapio (set)\n" if ($HeaderDoc::debugAllocations);
    }

    my $apio = undef;
    foreach my $possowner (@{$self->{APIOWNERS}}) {
	print STDERR "TESTING $possowner\n" if ($localDebug);
	if ($possowner !~ /HeaderDoc::HeaderElement/) {
		if ($possowner !~ /HeaderDoc::APIOwner/) {
			if ($possowner) {
				$apio = $possowner;
				print STDERR "CHOSE $apio\n" if ($localDebug);
			}
		}
	}
    }
    if (!$apio) {
	$apio = pop(@{$self->{APIOWNERS}});
	push(@{$self->{APIOWNERS}}, $apio);
	print STDERR "GUESSING $apio\n" if ($localDebug);
    }

    # Try this if you run into trouble, but....
    # if (!$apio && $self->{PARENT}) {
	# return $self->{PARENT}->apiOwner();
    # }

    return $apio;
}

# /*!
#     @abstract
#         Returns all of the API owners for the tree.
#     @param self
#         This tree node.
#  */
sub apiOwners
{
    my $self = shift;

    # foreach my $apio (@{$self->{APIOWNERS}} ) {
	# print STDERR "APIOWNERS LIST INCLUDES $apio\n";
    # }

    return $self->{APIOWNERS};
}

# /*!
#     @abstract
#         Returns the rightmost sibling of this tree node.
#     @param self
#         This tree node.
#  */
sub lastSibling {
    my $self = shift;

    while ($self && $self->next()) { $self = $self->next(); }

    return $self;
}

# /*!
#     @abstract
#         Gets/sets the access control state (public, private,
#         etc.) for this tree node.
#     @param self
#         This tree node.
#     @param ACCESSCONTROLSTATE
#         The new access control value to set. (Optional.)
#  */
sub acs {
    my $self = shift;

    if (@_) {
        $self->{ACCESSCONTROLSTATE} = shift;
    }
    return $self->{ACCESSCONTROLSTATE};
}

# /*!
#     @abstract
#         Gets/sets the token for this tree node.
#     @param self
#         This tree node.
#     @param TOKEN
#         The new token value to set. (Optional.)
#  */
sub token {
    my $self = shift;

    if (@_) {
        $self->{TOKEN} = shift;
    }
    return $self->{TOKEN};
}

# /*!
#     @abstract
#         Gets/sets whether this node and its children
#         should be hidden when emitting HTML.
#     @discussion
#         If a new value is specified, this function
#         recursively modifies its descendants.
#     @param self
#         This tree node.
#     @param value
#         The new token value to set. (Optional.)
#  */
sub hidden {
    my $self = shift;

    if (@_) {
	my $value = shift;
        $self->{HIDDEN} = $value;
	my $fc = $self->firstchild();
	if ($fc) { $fc->hiddenrec($value); }
    }
    return $self->{HIDDEN};
}

# /*!
#     @abstract
#         Sets whether this node, its children, and its
#         siblings should be hidden when emitting HTML.
#     @discussion
#         This function recursively modifies its
#         descendants and siblings.
#     @param self
#         This tree node.
#     @param value
#         The new token value to set.
#  */
sub hiddenrec
{
    my $self = shift;
    my $value = shift;

    # print STDERR "SETTING HIDDEN VALUE OF TOKEN ".$self->token()." to $value\n";
    # $self->hidden($value);
    $self->{HIDDEN} = $value;

    my $fc = $self->firstchild();
    if ($fc) { $fc->hiddenrec($value); }
    my $nx = $self->next();
    if ($nx) { $nx->hiddenrec($value); }
}

# /*!
#     @abstract
#         Scrapes the parameters for an Objective-C method from
#         the parse tree and returns an array.
#     @param self
#         The parse tree for the method.
#  */
sub objCparsedParams()
{
    my $self = shift;
    my $lang = shift;
    my $sublang = shift;

    my @parsedParams = ();
    my $objCParmDebug = 0;

    my $inType = 0;
    my $inName = 0;
    my $position = 0;
    my $curType = "";
    my $curName = "";
    my $cur = $self;
    my @stack = ();

    my $eoDec = 0;
    my $lastTag = "";
    my $tagName = "";

    my $noParse = 1;
    my $noTag = 1;
    while ($cur || scalar(@stack)) {
	while (!$cur && !$eoDec) {
	    if (!($cur = pop(@stack))) {
		$eoDec = 1;
	    } else {
		$cur = $cur->next();
	    }
	}

	if ($eoDec) { last; }

	# process this element
	my $token = $cur->token();
	if ($token eq ":") {
	    if (!$noTag) {
		$tagName = $lastTag;
	    }
	    $noParse = 0;
	} elsif ($noParse) {
	    # drop token on the floor.  It's part of the name.
	} elsif ($token eq "(") {
	    $inType++;
	    $curType .= $token;
	} elsif ($token eq ")") {
	    if (!(--$inType)) {
		$inName = 1;
		$noTag = 0;
	    }
	    $curType .= $token;
	} elsif ($token =~ /^[\s\W]/o && !$inType) {
	    # drop white space and symbols on the floor (except
	    # for pointer types)

	    if ($inName && ($curName ne "")) {
		$inName = 0;
		my $param = HeaderDoc::MinorAPIElement->new("LANG" => $lang, "SUBLANG" => $sublang);
		$param->linenuminblock($self->apiOwner()->linenuminblock());
		$param->blockoffset($self->apiOwner()->blockoffset());
		# $param->linenum($self->apiOwner()->linenum());
		$param->outputformat($self->apiOwner()->outputformat());
		$param->tagname($tagName);
		$param->name($curName);
		$param->type($curType);
		$param->position($position++);
		print STDERR "ADDED $curType $curName [$tagName]\n" if ($objCParmDebug);
		$curName = "";
		$curType = "";
		push(@parsedParams, $param);
		$noParse = 1;
	    }

	} elsif ($inType) {
	    $curType .= $token;
	} elsif ($inName) {
	    $curName .= $token;
	}

	my $fc = $cur->firstchild();
	if ($fc) {
	    push(@stack, $cur);
	    $cur = $fc;
	} else { 
	    $cur = $cur->next();
	}
	if ($token =~ /\w/) {
		$lastTag = $token;
	}
    }

    if ($objCParmDebug) {
	foreach my $parm (@parsedParams) {
	    print STDERR "OCCPARSEDPARM: ".$parm->type()." ".$parm->name()."\n";
	}
    }

    return @parsedParams;
}

# /*! 
#     @abstract
#         Sets the parsed parameter list for this node or
#         searches the parse tree for parsed parameter lists and
#         reurns the result.
#     @param self
#         This object.
#     @param pplref
#         A reference to a parsed parameter list. (Optional.)
#     @discussion
#         This subroutine is for future transition.  The end goal is to
#         move the parsed parameter support from the HeaderElement level
#         entirely into the parse tree.
#  */
sub parsedParams($)
{
    my $self = shift;
    my $lang = shift;
    my $sublang = shift;

    my @array = ();

# print STDERR "parsedParams called\n";

    if (@_) {
	if ($self->apiOwner() eq "HeaderDoc::Method") {
	    @{$self->{PARSEDPARAMS}} = $self->objCparsedParams($lang, $sublang);
	} else {
	    my $pplref = shift;
	    # @{$self->{PARSEDPARAMS}} = @_;
	    @{$self->{PARSEDPARAMS}} = @{$pplref};
	    # for my $parm (@{$pplref}) {
		# print STDERR "ADDING PARM $parm\n";
	    # }
	}
    }

    if (!($self->{PARSEDPARAMS})) {
	# print STDERR "PARSEDPARAMS PROBLEM: TOKEN WAS ".$self->token()."\n";
	# print STDERR "PRINTTREE:\n";
	# $self->printTree();
	# print STDERR "ENDOFTREE\n";
	my $next = $self->next();
	if ($next) { return $next->parsedParams($lang, $sublang); }
	else { return undef; }
	# else { die("Can't find parsed params\n"); }
    }
# print STDERR "HERE: $self : ". $self->token." : ".$self->{PARSEDPARAMS}."\n";
    # foreach my $parm (@{$self->{PARSEDPARAMS}}) {
	# print STDERR "FOUND PARM $parm\n";
    # }
    return @{$self->{PARSEDPARAMS}};
}

# /*!
#     @abstract
#         Finds the previous node by laborious tree walking of
#         the parent node's children.
#     @discussion
#         This function should be avoided if at all possible.
#         It  is currently unused, though it was used in the
#         distant past.  It is retained for completeness.
#     @param self
#         This tree node.
#  */
sub slowprev()
{
    my $self = shift;

    my $parent = $self->parent;
    if (!$parent) { return undef; }
    my $fc = $parent->firstchild;
    if ($self == $fc) { return undef; }
    while ($fc && $fc->next && ($fc->next != $self)) { $fc = $fc->next; }
    return $fc;
}

# /*!
#     @abstract
#         Copies a parsed parameter list and sets the parsed parameter
#         list for this tree node to be a reference to the resulting
#         array.
#     @param self
#         This tree node.
#     @param pplref
#         A reference to the parsed parameter list array to copy.
#  */
sub parsedParamCopy
{
    my $self = shift;
    my $pplref = shift;
    my $lang = shift;
    my $sublang = shift;

    my $localDebug = 0;

    my @parms = @{$pplref};
    my @newparms = ();

    foreach my $parm (@parms) {
	push(@newparms, $parm);
    }

    $self->parsedParams($lang, $sublang, \@newparms);
print STDERR "PARSEDPARAMCOPY -> $self\n" if ($localDebug);
print STDERR "TOKEN WAS ".$self->token()."\n" if ($localDebug);
}

# /*!
#     @abstract
#         Handles embedded HeaderDoc markup, returning a list
#         of parameters, constants, etc.
#     @param self
#         The top of the parse tree/subtree to scan.
#     @param xmlmode
#         Set to 1 if output mode is XML.
#     @param apiOwner
#         If this parse tree is owned by an API owner, this
#         variable indicates the primary API owner to which
#         the resulting objects should be added.  If this
#         parse tree is owned by a non-API-oowner object (e.g.
#         a Struct object), this value is unused.
#  */
sub processEmbeddedTags
{
    my $self = shift;
    my $xmlmode = shift;
    my $apiOwner = shift; # Always subclass of APIOwner, used as "true" apio value for calling APIOprocessEmbeddedTagsRec

    my $apiolist = $self->apiOwners();
    my $apio = $self->apiOwner();

    # $self->printTree();
    # $self->dbprint();
    my $localDebug = 0;
    # if ($apio->isAPIOwner()) { $localDebug = 1; }

    print STDERR "PET: $apio\n" if ($localDebug);
    print STDERR $apio->name()."\n" if ($localDebug);
    print STDERR "APIOLIST IS $apiolist\n" if ($localDebug);;
    # for my $tempapio (@{$apiolist}) {
	# print STDERR "RETURNED APIOLIST INCLUDES $tempapio\n";
    # }


    ### Make the APIO owner list unique. ###

    # print STDERR "APIOLIST:\n";
    my %newapiohash = ();
    foreach my $tempapio (@{$apiolist}) {
	# print $tempapio."\n";
	$newapiohash{$tempapio} = $tempapio;
    }
    # print STDERR "END APIOLIST\n";

    my @newapiolist = ();
    foreach my $tempapio (keys %newapiohash) {
	push(@newapiolist, $newapiohash{$tempapio});
    }
    $apiolist = \@newapiolist;

    if ($self->{PETDONE}) {
	print STDERR "SHORTCUT\n" if ($localDebug);
	return;
    }
    $self->{PETDONE} = 1;

    if (!$apio) { return; }

    my $apioclass = ref($apio) || $apio;

    my $old_enable_cpp = $HeaderDoc::enable_cpp;
    if ($apioclass =~ /HeaderDoc::PDefine/ && $apio->parseOnly()) {
	if ($HeaderDoc::enable_cpp) {
		print STDERR "CPP Enabled.  Not processing comments embedded in #define macros marked as 'parse only'.\n" if ($localDebug);
		return;
	}
    } elsif ($apioclass =~ /HeaderDoc::PDefine/) {
	if ($HeaderDoc::enable_cpp) {
		print STDERR "Temporarily disabling CPP.\n" if ($localDebug);
		$HeaderDoc::enable_cpp = 0;
	}
    }

    # my ($sotemplate, $eotemplate, $operator, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $sofunction,
        # $parseTokens{soprocedure}, $parseTokens{sopreproc}, $parseTokens{lbrace}, $parseTokens{rbrace}, $unionname, $structname,
	# $enumname,
        # $typedefname, $varname, $constname, $structisbrace, $macronamesref,
	# $classregexp, $classbraceregexp, $classclosebraceregexp, $accessregexp,
        # $requiredregexp, $propname, $objcdynamicname, $objcsynthesizename, $moduleregexp, $definename,
	# $functionisbrace, $classisbrace, $parseTokens{lbraceconditionalre}, $parseTokens{lbraceunconditionalre}, $assignmentwithcolon,
	# $labelregexp, $parmswithcurlybraces, $superclasseswithcurlybraces, $soconstructor) = parseTokens($apio->lang(), $apio->sublang());
		# @_;
    my %parseTokens = %{parseTokens($apio->lang(), $apio->sublang())};

    my $eoDeclaration = 1;
    my $lastDeclaration = "";
    my $curDeclaration = "";
    my $sodec = $self;
    my $pendingHDcomment = "";

    my ($case_sensitive, $keywordhashref) = $apio->keywords();

    my $lastnode = undef;
    my $parserState = $self->parserState();
    if ($parserState) {
	print STDERR "PARSERSTATE\n" if ($localDebug);
	$lastnode = $parserState->{lastTreeNode};
	print STDERR "LASTNODE: $lastnode\n" if ($localDebug);
	if ($lastnode && $localDebug) { print STDERR "LASTNODE TEXT: \"".$lastnode->token()."\"\n"; }
    }

    my $enable_javadoc_comments = $HeaderDoc::parse_javadoc || ($apio->lang() eq "java");
    # print STDERR "EJC: $enable_javadoc_comments\n";

    if ($apio->isAPIOwner()) {
	print STDERR "Owner is APIOwner.  Using APIOprocessEmbeddedTagsRec for parse tree $self.\n" if ($localDebug);

	my $hashtreecur = undef;
	my $hashtreeroot = undef;

	$self->APIOprocessEmbeddedTagsRec($apiOwner, $case_sensitive, \%parseTokens, $lastnode, 0, 1, $enable_javadoc_comments, $xmlmode, $apio->lang(), $apio->sublang(), $hashtreecur, $hashtreeroot);
    } else {
	print STDERR "calling processEmbeddedTagsRec for $apio (".$apio->name().") for parse tree $self.\n" if ($localDebug);
	$self->processEmbeddedTagsRec($xmlmode, $eoDeclaration, \%parseTokens, $case_sensitive, $keywordhashref, $lastDeclaration, $curDeclaration, $pendingHDcomment,
		$apio, $apiolist, $sodec, $lastnode, $enable_javadoc_comments);
    }

print STDERR "PETDONE\n" if ($localDebug);

    $HeaderDoc::enable_cpp = $old_enable_cpp;

    return;
}

# /*!
#     @abstract
#         Gets the name and the corresponding comment
#         field type (e.g. <code>\@struct</code>) for a given
#         declaration
#     @discussion
#         This helper reduces the amount of code needed
#         to use the block parser in several places in
#         the parse tree code by consolidating this
#         logic in one place.
#     @param self
#         This parse tree object.
#     @param string
#         The declaration to parse.
#     @param apio
#         The object (may be API owner or not) to which
#         the resulting objects should be added.
#     @param typedefname
#         The keyword <code>typedef</code> or equivalent.
#     @param case_sensitive
#         Set to 0/1 depending on whether the current
#         programming language uses case-sensitive
#         token matching.
#     @param keywordhashref
#         A reference to the keyword hash for this
#         language (obtained with a call to the
#  {@link //apple_ref/perl/instm/HeaderDoc::HeaderElement/keywords//() keywords}
#         function).
#  */
sub getNameAndFieldTypeFromDeclaration
{
    my $self = shift;
    my $string = shift;
    my $apio = shift;
    my $typedefname = shift;
    my $case_sensitive = shift;
    my $keywordhashref = shift;

    my $localDebug = 0;
    my $inputCounter = 0;

    my $fullpath = $apio->fullpath();
    my $linenum = $apio->linenum();
    my $lang = $apio->lang();
    my $sublang = $apio->sublang();

    my $blockoffset = $linenum;
    my $argparse = 2;

    if ($string !~ /\S/) { return ("", ""); }

    # This never hurts just to make sure the parse terminates.
    # Be sure to add a newline before the semicolon in case
    # there's an inline comment at the end.
    $string .= "\n;\n";

    print STDERR "STRING WAS $string\n" if ($localDebug);

    cluck("getNameAndFieldTypeFromDeclaration backtrace\n") if ($localDebug);

    my @lines = split(/\n/, $string);

    # my ($sotemplate, $eotemplate, $operator, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $sofunction,
        # $parseTokens{soprocedure}, $parseTokens{sopreproc}, $parseTokens{lbrace}, $parseTokens{rbrace}, $unionname, $structname,
	# $enumname,
        # $typedefname, $varname, $constname, $structisbrace, $macronameref,
        # $classregexp, $classbraceregexp, $classclosebraceregexp, $accessregexp,
        # $requiredregexp, $propname, $objcdynamicname, $objcsynthesizename, $moduleregexp, $definename,
        # $functionisbrace, $classisbrace, $parseTokens{lbraceconditionalre}, $parseTokens{lbraceunconditionalre}, $assignmentwithcolon,
	# $labelregexp, $parmswithcurlybraces, $superclasseswithcurlybraces, $soconstructor) = parseTokens($lang, $sublang);

    # my @newlines = ();
    foreach my $line (@lines) {
	$line .= "\n";
	# push(@newlines, $line);
        # print STDERR "LINE: $line\n" if ($localDebug);
    }
    # @lines = @newlines;

    # my ($case_sensitive, $keywordhashref) = $apio->keywords();

    my $lastlang = $HeaderDoc::lang;
    my $lastsublang = $HeaderDoc::sublang;
    $HeaderDoc::lang = $apio->lang;
    $HeaderDoc::sublang = $apio->sublang;

    my ($inputCounter, $declaration, $typelist, $namelist, $posstypes, $value, $pplStackRef, $returntype, $privateDeclaration, $treeTop, $simpleTDcontents, $availability, $fileoffset, $conformsToList) = blockParse($fullpath, $blockoffset, \@lines, $inputCounter, $argparse, \%HeaderDoc::ignorePrefixes, \%HeaderDoc::perHeaderIgnorePrefixes, \%HeaderDoc::perHeaderIgnoreFuncMacros, $keywordhashref, $case_sensitive, $apio->lang, $apio->sublang);
# print "DEC WAS $string\n";
# print "DEC RETURNED WAS $declaration\n";

    $HeaderDoc::lang = $lastlang;
    $HeaderDoc::sublang = $lastsublang;

    # @@@ FIX UP TYPES HERE

    print STDERR "IC:$inputCounter DEC:$declaration TL:$typelist NL:$namelist PT:$posstypes VAL:$value PSR:$pplStackRef RT:$returntype PD:$privateDeclaration TT:$treeTop STC:$simpleTDcontents AV:$availability\n" if ($localDebug);

    $self->parsedParamCopy($pplStackRef, $lang, $sublang);

    my $name = $namelist;
    $name =~ s/^\s*//so; # ditch leading spaces
    $name =~ s/\s.*$//so; # ditch any additional names. (There shouldn't be any)
	# print STDERR "NAME WAS $name\n";
    my $typestring = $typelist . $posstypes;

print STDERR "TS: $typestring\n" if ($localDebug);

    my $type = "\@constant";
    if ($typestring =~ /^(function|method|ftmplt|operator|callback)/o) {
	$type = "\@$1";
	if ($typestring =~ /(ftmplt|operator)/) { $type = "\@function"; }
	# $type = "\@callback";
    } elsif ($typestring =~ /^(class|interface|module|category|protocol)/o) {
	$typestring =~ s/^ *//;
	$type = "\@$typestring";
	$type =~ s/ .*$//;
    } elsif ($typestring =~ /^(struct|union|record|enum|typedef)/o || (($typedefname ne "") && $typestring =~ /^$typedefname/)) {
	$type = "\@field";
    } elsif ($typestring =~ /(MACRO|#define)/o) {
	$type = "\@field";
	if ($apio eq "HeaderDoc::PDefine") {
		# The @defineblock case
		$type = "\@define";
	}
    } elsif ($typestring =~ /(constant)/o) {
	$type = "\@constant";
	print STDERR "VALUE: \"$value\"\n" if ($localDebug);
	if (($value eq "")) {
		# It's just a variable.
		if ($apio =~ /HeaderDoc::Enum/) {
			$type = "\@constant"; # Still a constant, albeit with an unknown value.
		} else {
			$type = "\@field";
		}
	}
    } else {
	warn "getNameAndFieldTypeFromDeclaration: UNKNOWN TYPE ($typestring) RETURNED BY BLOCKPARSE\n";
	print STDERR "STRING WAS $string\n" if ($localDebug);
    }

    if (!$name || ($name eq "")) {
	warn "COULD NOT GET NAME FROM DECLARATION.  DECLARATION WAS:\n$string\n";
	return ("", "");
    }
    print STDERR "TYPE $type, NAME $name\n" if ($localDebug);

    return ($name, $type);
}

# /*!
#     @abstract
#         Returns whether to process comments in the children of a node
#         with a given token.
#     @discussion
#         The tree walking code explicitly avoid things like strings
#     and regular expressions without the need to search for them by
#     disallowing children of any non-word, non-whitespace characters
#     other than parentheses, curly braces, and colons.
#
#     @param token
#         The token to check.
#     @param soc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param eoc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc_b
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param lbrace
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param case_sensitive
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::HeaderElement/keywords//() keywords}.
#  */
sub commentsNestedIn
{
    my $token = shift;
    my $soc = shift;
    my $eoc = shift;
    my $ilc = shift;
    my $ilc_b = shift;
    my $lbrace = shift;
    my $case_sensitive = shift;

    # if ($token eq $soc || $token eq $eoc || $token eq $ilc || $token eq $ilc_b) { return 1; }
    if ($token =~ /\W/o) {
	if ($token =~ /[{(}):]/o) { return 1; }
	if ($token =~ /^#/o) { return 2; }
	if (casecmp($token, $lbrace, $case_sensitive)) { return 1; }
	if ($token =~ /\s/o) { return 1; }
	return 0;
    }
    # if (casecmp($token, $lbrace, $case_sensitive)) { return 1; }
    return 1;
}

# /*!
#     @abstract
#         Processes embedded tags within an API owner (class).
#     @discussion
#         Unlike {@link processEmbeddedTagsRec}, this function
#     does significantly less work because the code in APIOwner.pm
#     handles most of the effort, thanks in part to the pots of
#     parser gold hidden in the parse tree's hollow.
#
#     This function also works differently because instead of
#     adding fields to a struct or whatever, it is adding
#     functions, data types, variables, and even classes
#     that are part of another class.
#  */
sub APIOprocessEmbeddedTagsRec($$$$$$$$)
{
    my $self = shift;
    my $apiOwner = shift; # my $apiolist = shift;
    my $case_sensitive = shift;
    my $parseTokensRef = shift;
    my $lastTreeNode = shift;
    my $skipchildren = shift;
    my $isroot = shift; # Don't add parse tree to the parent object twice.
    my $enable_javadoc_comments = shift;
    my $xmlmode = shift;
    my $lang = shift;
    my $sublang = shift;
    my $hashtreecur = shift;
    my $hashtreeroot = shift;

    my %parseTokens = %{$parseTokensRef};

    # print STDERR "EJC: $enable_javadoc_comments\n";

    my $localDebug = 0;

    if (1 && $localDebug) {
	my $apio = $self->apiOwner();
	if ($apio) {
		# if ($apio->name() eq "OSObject") {
		print STDERR "DUMPING TREE.\n"; $self->dbprint();
		# }
	}
    }

    my $continue = 1;
    if ($self == $lastTreeNode) {
	$continue = 0;
	print STDERR "CONTINUE -> 0\n" if ($localDebug);
	# return $continue; # Added this to try to fix a bug, but wasn't needed.
    }

    my $token = $self->token();
    my $firstchild = $self->firstchild();
    my $next = $self->next();

    print STDERR "APIOprocessEmbeddedTagsRec: TOKEN IS \"".$self->token()."\"\n" if ($localDebug);
    my $handled = 0;

    print STDERR "SOC: \"$parseTokens{soc}\" ILC: \"$parseTokens{ilc}\" ILC_B: \"$parseTokens{ilc_b}\"\n" if ($localDebug);

    if ((length($parseTokens{soc}) && ($token eq $parseTokens{soc})) ||
	(length($parseTokens{ilc}) && ($token eq $parseTokens{ilc})) ||
	(length($parseTokens{ilc_b}) && ($token eq $parseTokens{ilc_b}))) {

	if ((($token eq $parseTokens{soc} && $parseTokens{soc}) || ($token eq $parseTokens{ilc} && $parseTokens{ilc})) && $firstchild) {
		print STDERR "COMMENT CHECK\n" if ($localDebug);
		my $ntoken = $firstchild->token();
		my $nntoken = "";
		if ($firstchild->next()) {
			$nntoken = $firstchild->next()->token();
		}

		# The parser changed behavior.  The folowing is probably the only relevant behaviour,
		# but the previous behavior is preserved for partial inter-module backwards compatibility.
		if ($ntoken eq "" && $firstchild->next()) {
			$ntoken = $firstchild->next()->token();
			if ($firstchild->next()->next()) {
				$nntoken = $firstchild->next()->next()->token();
			} else {
				$nntoken = "";
			}
		}
		# print STDERR "FC: ".$firstchild->token()."\n";
		# print STDERR "FCN: ".$firstchild->next()->token()."\n";
		# print STDERR "FCNN: ".$firstchild->next()->next()->token()."\n";
		if ($lang eq "ruby" || $lang eq "python") {
			my $node = $firstchild;
			while ($node && ($node->token() !~ /\S/)) {
				$node = $node->next();
			}
			if ($node) { $nntoken = $node->token(); }
			# print STDERR "HERE: N = \"$ntoken\" NN = \"$nntoken\"\n";
			# if ($ntoken eq "\n" && $nntoken eq "!" && $firstchild->next()->next()->next() &&
			    # $firstchild->next()->next()->next()->next() &&
			    # $firstchild->next()->next()->next()->next()->next()) {
				# my $nnntoken = $firstchild->next()->next()->next()->token();
				# my $nnnntoken = $firstchild->next()->next()->next()->next()->token();
				# print STDERR "HERE: NNN = \"$nnntoken\" NNNN = \"$nnnntoken\"\n";
				# if ($nnntoken =~ /headerdoc/i && $nnnntoken eq "!") {
					# $ntoken = "!";
				# } else {
					# $ntoken = "";
				# }
			# } else {
					# $ntoken = "";
			# }
			if ($nntoken =~ /\s*!headerdoc!/i) {
				$ntoken = "!";
			} else {
				$ntoken = "";
			}
		}
		print STDERR "NTOKEN: $ntoken\n" if ($localDebug);
		print STDERR "NNTOKEN: $nntoken\n" if ($localDebug);
		if ((($lang ne "tcl" && ($ntoken eq "!" || ($enable_javadoc_comments && $ntoken eq "*" && $nntoken !~ /^\*/))) ||
		     ($lang eq "tcl" && ($ntoken eq "/*" && $nntoken eq "!" && $firstchild->next()->next()->next()))) &&
		    !$self->hidden()) {
			print STDERR "NODECHECK: $self\n" if ($localDebug);
			# HDCOMMENT
			$self->dbprint() if ($localDebug);
			my $string = $token.$firstchild->textTree();
			# print STDERR "STRING IS $string\n";
			if ($lang eq "ruby" || $lang eq "python") {
				$string = $firstchild->next()->next()->textTree();
				$string =~ s/^\s*!headerdoc!\s*//si;
				$string = "/*!".$string;
				$token = $parseTokens{soc};
				# print STDERR "STRING IS $string\n";
			} elsif ($lang eq "tcl") {
				# Scripting languages work differently.

				$string = "/*!".$firstchild->next()->next()->next()->textTree();

				# $self->dbprint();
				# warn("PreString: $string\n");

				my $foundend = 0;
				if ($string =~ /\*\//) { $foundend = 1; }
				my $pos = $self->next();
				# warn("NEXT TEXT: ".$pos->textTree()."\n");
				while ((!$foundend) && $pos) {
					print "MOVING TO NEXT.  POS IS $pos TOKEN IS ".$pos->token()."\n" if ($localDebug);
					my $temp = $pos->firstchild();
					if ($temp) {
						my $tempstr = $temp->textTree();
						if ($pos->token() eq "#") {
							$string .= $tempstr;
						} elsif ($tempstr =~ /\S/) {
							$foundend;
						} else {
							$string .= "\n";
						}
					} else {
						$string .= "\n";
					}
					if ($string =~ /\*\//) { $foundend = 1; }
					$pos = $pos->next();
				}
				# die("PostString: $string\n");
			} elsif ($lang eq "applescript") {
				$string =~ s/^\s*\(\*/\/\*/s;
			}
			if ($ntoken eq "*") { $string =~ s/^\s*\/\*\*/\/\*\!/s; }
			print STDERR "FOUND HDCOMMENT:\n$string\nEND HDCOMMENT\n" if ($localDebug);
			# $string =~ s/^\/[\*\/]\!//s;
			# $string =~ s/^\s*//s;
			if ($token eq $parseTokens{soc} && $parseTokens{soc}) {
				my $eoc = $parseTokens{eoc};
				$string =~ s/\Q$eoc\E\s*$//s;
			}

			print STDERR "STRING IS $string\n" if ($localDebug);

			# @@@ DAG CHECK ME
			# $string =~ s/^\s*\*\s*//mg;
			my $fieldref = stringToFields($string, $self->fullpath, $self->linenum, $xmlmode, $lang, $sublang);
			# print STDERR "APIOLIST AT INSERT IS $apiolist\n" if ($localDebug);
			# foreach my $owner (@{$apiolist}) {
			    # print STDERR "X POSSOWNER of $self: $owner\n" if ($localDebug);
			# }
			# foreach my $owner (@{$apiolist}) {
			    print STDERR "POSSOWNER of $self: $apiOwner\n" if ($localDebug);
			    if ($apiOwner && $apiOwner->isAPIOwner()) {
				print STDERR "ADDING[1] TO $apiOwner.\n" if ($localDebug);
				my $ncurly;
				($ncurly, $hashtreecur, $hashtreeroot) = $apiOwner->processComment($fieldref, 1, $self->nextTokenNoComments($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, 1, $enable_javadoc_comments), $parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $self, $hashtreecur, $hashtreeroot);

				# We have found the correct level.  Anything
				# nested deeper than this is bogus (unless we hit a curly brace).
				print STDERR "skipochildren -> 1 [1]" if ($localDebug);
				$skipchildren = 1;
				$next = $next->skipcurly($parseTokens{lbrace}, $ncurly); # nextTokenNoComments($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, 0, $enable_javadoc_comments);
				if ($localDebug) {
					print STDERR "NEXT IS $next (";
					if ($next) {print STDERR $next->token(); }
					print STDERR ")\n";
				}
			    }
			# }
			$handled = 1;
		}
	} elsif ($firstchild && $firstchild->next() && $firstchild->next()->next()) {
			my $pos = $firstchild->next();
			my $fcntoken = $pos->token();

			while ($fcntoken =~ /\s/ && $pos) {
				$pos = $pos->next;
				$fcntoken = $pos->token();
			}
			if (($fcntoken eq $parseTokens{soc} && $parseTokens{soc}) || ($fcntoken eq $parseTokens{ilc} && $parseTokens{ilc})) {
				my $fcnntoken = $firstchild->next()->next()->token();
				my $fcnnntoken = "";
				if ($firstchild->next()->next()->next()) {
					$fcnnntoken = $firstchild->next()->next()->next()->token();
				}
				if ($fcnntoken eq "!" || ($enable_javadoc_comments && $fcnntoken eq "*" && $fcnnntoken !~ /^\*/)) {
					# HDCOMMENT
					my $string = $fcntoken.$firstchild->textTree();
					if ($fcnntoken eq "*") { $string =~ s/^\s*\/\*\*/\/\*\!/s; }
					print STDERR "FOUND HDCOMMENT:\n$string\nEND HDCOMMENT\n" if ($localDebug);
					# $string =~ s/^\Q$fcntoken\E//s;
					# $string =~ s/^\s*\/[\*\/]\!//s;
					# $string =~ s/^\s*//s;
					if ($fcntoken eq $parseTokens{soc} && $parseTokens{soc}) {
						my $eoc = $parseTokens{eoc};
						$string =~ s/\Q$eoc\E\s*$//s;
					}
					# @@@ DAG CHECKME LEADING STARS
					# $string =~ s/^\s*\*\s*//mg;
					my $fieldref = stringToFields($string, $self->fullpath, $self->linenum, $xmlmode, $lang, $sublang);
					# foreach my $owner (@{$apiolist}) {
					    print STDERR "POSSOWNER of $self: $apiOwner\n" if ($localDebug);
					    if ($apiOwner && $apiOwner->isAPIOwner()) {
						print STDERR "ADDING[2] TO $apiOwner.\n" if ($localDebug);
						my $ncurly;
						($ncurly, $hashtreecur, $hashtreeroot) = $apiOwner->processComment($fieldref, 1, $self->nextTokenNoComments($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, 1, $enable_javadoc_comments), $parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $self, $hashtreecur, $hashtreeroot);
						print STDERR "skipochildren -> 1 [2]" if ($localDebug);
						$skipchildren = 1;
						# skip the current declaration before
						# processing anything else to avoid
						# bogus warnings from nested
						# HeaderDoc comments.
						$next = $next->skipcurly($parseTokens{lbrace}, $ncurly); # nextTokenNoComments($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, 0, $enable_javadoc_comments);
						if ($localDebug) {
							print STDERR "NEXT IS $next (";
							if ($next) {print STDERR $next->token(); }
							print STDERR ")\n";
						}
					    }
					# }
					$handled = 1;
				}
			}
	}
    }
    if (!$handled && $self->parserState() && !$self->parserState()->{APIODONE} && $HeaderDoc::process_everything && !$isroot) {
	print STDERR "Declaration without markup\n" if ($localDebug);
	# foreach my $owner (@{$apiolist}) {
	    # print STDERR "X POSSOWNER of $self: $apiOwner\n" if ($localDebug);
	# }
	# foreach my $apiOwner (@{$apiolist}) {
	    print STDERR "POSSOWNER of $self: $apiOwner\n" if ($localDebug);
	    # Found an embedded declaration that is not tagged.
	    my @fields = ();
	    if ($apiOwner && $apiOwner->isAPIOwner()) {
		my $ncurly;

		($ncurly, $hashtreecur, $hashtreeroot) = $apiOwner->processComment(\@fields, 1, $self, $parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $self, $hashtreecur, $hashtreeroot);

		$handled = 1;
	    } else {
		warn "$apiOwner is not API Owner\n";
	    }
	# }
    }
    if ($handled && $localDebug) {
	print STDERR "ADDED TREE TO $apiOwner (".$apiOwner->name().")\n";
	$self->dbprint();
	print STDERR "DUMPING API OWNER:\n";
	$apiOwner->dbprint();
	print STDERR "END DUMP.\n";
    }
    # print STDERR "PS: ".$self->parserState()."\n";
    # If we get here, we weren't a skipped brace, so we can start nesting again.
    if (length($parseTokens{lbrace}) && $token eq $parseTokens{lbrace}) {
	print STDERR "skipochildren -> 0 [3]" if ($localDebug);
	$skipchildren = 0;
    }

    if ($firstchild && !$skipchildren) {
	print STDERR "APIOprocessEmbeddedTagsRec: MAYBE GOING TO CHILDREN\n" if ($localDebug);
	my $nestallowed = commentsNestedIn($token, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $parseTokens{lbrace}, $case_sensitive);

        if ($nestallowed) {
		print STDERR "APIOprocessEmbeddedTagsRec: YUP.  CHILDREN.\n" if ($localDebug);
		my $newcontinue;
		($newcontinue, $hashtreecur, $hashtreeroot) = $firstchild->APIOprocessEmbeddedTagsRec($apiOwner, $case_sensitive, $parseTokensRef, $lastTreeNode, $skipchildren, 0, $enable_javadoc_comments, $xmlmode, $lang, $sublang, $hashtreecur, $hashtreeroot);
		if ($continue) { $continue = $newcontinue; }
		print STDERR "Back from Child\n" if ($localDebug);
		print STDERR "skipochildren -> $skipchildren [RECURSEOUT]" if ($localDebug);
	}
    }
    if ($next && $continue) {
	print STDERR "APIOprocessEmbeddedTagsRec: GOING TO NEXT\n" if ($localDebug);
	($continue, $hashtreecur, $hashtreeroot) = $next->APIOprocessEmbeddedTagsRec($apiOwner, $case_sensitive, $parseTokensRef, $lastTreeNode, $skipchildren, 0, $enable_javadoc_comments, $xmlmode, $lang, $sublang, $hashtreecur, $hashtreeroot);
	print STDERR "Back from Next\n" if ($localDebug);
    }
    print STDERR "HTC: $hashtreecur, HTR: $hashtreeroot\n" if ($localDebug);
print STDERR "SN: ".$self->next()." (".($self->next() ? $self->next()->token() : "").")\n" if ($localDebug);
print STDERR "RECURSEOUT (CONTINUE is $continue)\n" if ($localDebug);
    return ($continue, $hashtreecur, $hashtreeroot);
}

my $extDebug = 0;

# /*!
#     @abstract
#         Recursively walks the parse tree looking for HeaderDoc markup
#         and processes it.
#     @discussion
#         This subroutine processes the parse tree recursively looking for
#     (and subsequently processing) embedded headerdoc markup, then parses
#     the associated declarations.  This is a helper function that does
#     the actual work for processEmbeddedTags.
#  */
sub processEmbeddedTagsRec
{
    my $self = shift;
    my $xmlmode = shift;
    my $eoDeclaration = shift;
    my $parseTokensRef = shift;
    my $case_sensitive = shift;
    my $keywordhashref = shift;
    my $lastDeclaration = shift;
    my $curDeclaration = shift;
    my $pendingHDcomment = shift;
    my $apio = shift;
    my $apiolist = shift;
    my $sodec = shift;
    my $lastTreeNode = shift;
    my $enable_javadoc_comments = shift;

    my %parseTokens = %{$parseTokensRef};

    my $localDebug = 0 || $extDebug;
    my $fieldTypeDebug = 0;
    my $oldCurDeclaration = $curDeclaration;
    my $oldsodec = $sodec;
    my $ntoken = $self->nextpeeknc($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b});
    my $skipchildren = 0;
    my $oldPHD = $pendingHDcomment;
    my $do_process = 0;

    my $continue = 1;
    my $inBlockDefine = 0;
    my $dropinvalid = 0;
    my $lastsodec = $sodec;
    my $nextsodec = $sodec;

    my $hashtreecur = undef;
    my $hashtreeroot = undef;

print STDERR "PETREC\n" if ($localDebug);
print STDERR "PENDING COMMENT: $pendingHDcomment\n" if ($localDebug);
print STDERR "LAST DECLARATION: $lastDeclaration\n" if ($localDebug);

    if (!$self) { return ($eoDeclaration, $pendingHDcomment, $continue); }

    my $apioclass = ref($apio) || $apio;
    if ($apioclass =~ /HeaderDoc::PDefine/) {
	# print STDERR "HDPDEF: ISBLOCK: ".$apio->isBlock()." inDefineBlock: ".$apio->inDefineBlock().".\n";
        if ($apio->inDefineBlock() || $apio->isBlock()) {
	    $inBlockDefine = 1;
	    my $x = pop(@{$apiolist});
	    $do_process = $x;
	    if ($x) {
		push(@{$apiolist}, $x);
	    }
	}
    }

    if ($self == $lastTreeNode) {
	print STDERR "CONTINUE -> 0\n" if ($localDebug);
	$continue = 0;
    }

    # print STDERR "lastDec: $lastDeclaration\ncurDec: $curDeclaration\neoDec: $eoDeclaration\n" if ($localDebug);

    # Walk the tree.
    my $token = $self->token();
    $curDeclaration .= $token;

    # Used to turn on debugging at a given symbol.
    # if ($token eq "kDNSServiceFlagsSuppressUnusable") { $localDebug = 1; $extDebug = 1; }

    print STDERR "TOKEN: $token\n" if ($localDebug);

# if ($token !~ /\s/o) { print STDERR "TOKEN: \"$token\" SOC: \"$parseTokens{soc}\" ILC: \"$parseTokens{ilc}\" ILC_B: \"$parseTokens{ilc_b}\".\n"; }

    if (($token eq $parseTokens{soc} && $parseTokens{soc}) || ($token eq $parseTokens{ilc} && $parseTokens{ilc}) || ($token eq $parseTokens{ilc_b} && $parseTokens{ilc_b})) {
	my $firstchild = $self->firstchild();
	print STDERR "Comment start\n" if ($localDebug);

	if ($firstchild) {
	print STDERR "FCT: ".$firstchild->token()."\n" if ($localDebug);
	  my $nextchild = $firstchild->next();
	  my $nntoken = "";
	  if ($nextchild && $nextchild->next()) { $nntoken = $nextchild->next()->token(); }

	  if ($nextchild && ($nextchild->token eq "!" || ($enable_javadoc_comments && $nextchild->token eq "*" && $nntoken !~ /^\*/))) {
	      print STDERR "Found embedded HeaderDoc markup\n" if ($localDebug);
	      print STDERR "NCT: ".$nextchild->token()."\n" if ($localDebug);
		# print STDERR "NCT TREE:\n"; $self->printTree(); print STDERR "NCT ENDTREE\n";

		print STDERR "WILL SET SODEC.  SEARCHING IN:\n" if ($localDebug);
		$self->dbprint() if ($localDebug);
	      $sodec = $self->nextTokenNoComments($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, 0, $enable_javadoc_comments);
		# print STDERR "NCT SODECTREE:\n"; $sodec->printTree(); print STDERR "NCT ENDTREE\n";
		print STDERR "SODEC SET TO $sodec\n" if ($localDebug);
		if ($sodec) {
			$sodec->dbprint() if ($localDebug);
		}

	      my $string = $firstchild->textTree();
	      my $fullpath = $apio->fullpath();
	      my $linenum = $apio->linenum();
	      if ($token eq $parseTokens{soc} && $parseTokens{soc}) {
		my $eoc = $parseTokens{eoc};
		$string =~ s/\Q$eoc\E\s*$//s;
	      }
	      if ($string =~ /^\s*\!/o) {
		      $string =~ s/^\s*\!//so;
		      my $tagstring = $string;
		      $tagstring =~ s/^\s*\@//so;

		      print STDERR "EOD $eoDeclaration NT $ntoken STR $string\n" if ($localDebug);;

		      if ((($eoDeclaration && $lastDeclaration =~ /[a-zA-Z]/) || !$ntoken ||
			   $ntoken =~ /[)}]/o || casecmp($ntoken, $parseTokens{rbrace}, $case_sensitive)) &&
			  ($string !~ /^\s*\@/o || !validTag($tagstring, 0))) {

			if (!$inBlockDefine) {
				$string =~ s/\@abstract/ /sg;
				$string =~ s/\@discussion/ /sg;
			}

			# If we're at the end of a declaration (prior to the
			# following newline) and the declaration starts with
			# a string of text (JavaDoc-style markup), we need to
			# figure out the name of the previous declaration and
			# insert it.

			print STDERR "Using previous declaration because:\n" if ($localDebug);
			print STDERR "EODEC: $eoDeclaration NTOKEN: \"$ntoken\"\n" if ($localDebug);
			print STDERR "RBRACE: \"$parseTokens{rbrace}\" STRING: \"$string\"\n" if ($localDebug);

			if (!$eoDeclaration) {
				print STDERR "LASTDITCH PROCESSING\n" if ($localDebug);
			} else {
				print STDERR "EOD PROCESSING\n" if ($localDebug);
			}

			# Roll back to the previous start of declaration.
			# This comment is at the end of a line or whatever.
			$nextsodec = $sodec;
			$sodec = $lastsodec;

			$string =~ s/^\s*//so;
	
			print STDERR "COMMENTSTRING WAS: $string\n" if ($localDebug);
			print STDERR "PRE1\n" if ($localDebug);

			print STDERR "LAST DECLARATION: $lastDeclaration\n" if ($localDebug);

			print STDERR "calling getNameAndFieldTypeFromDeclaration [1]\n" if ($localDebug || $fieldTypeDebug);
			my ($name, $type) = $self->getNameAndFieldTypeFromDeclaration($lastDeclaration, $apio, $parseTokens{typedefname}, $case_sensitive, $keywordhashref);
	
			if ($name) {
				$string = "$type $name\n$string";
				print STDERR "COMMENTSTRING NOW: $string\n" if ($localDebug);
			}
		      } elsif (!$eoDeclaration && (!$ntoken ||
                           $ntoken =~ /[)]/o || casecmp($ntoken, $parseTokens{rbrace}, $case_sensitive)) &&
                           $string =~ /^\s*\@/o) {
                           # We have found an embedded headerdoc comment embedded that is
			   # right before a close parenthesis, but which starts with an @ sign.
				my $nlstring = $string;
				$nlstring =~ s/[\n\r]/ /sg;
				my $emptyok = emptyHDok($nlstring);
				if (!$emptyok) {
					warn "$fullpath:$linenum: warning: Found invalid headerdoc markup: $nlstring\n";
					$dropinvalid = 1;
				} elsif ($emptyok == 1) {
					warn "$fullpath:$linenum: warning Found headerdoc markup where none expected: $nlstring\n";
				} else {
					print STDERR "Found always-empty headerdoc markup at end of block: $nlstring\n" if ($localDebug);
					$dropinvalid = 2;
				}
		      }
		      $string =~ s/^\s*//so;
		      my $tagstring = $string;
		      $tagstring =~ s/^\s*\@//so;
		      # warn("STR: $string TAGSTR: $tagstring\n");
		      # warn(validTag("", 0)." != ".validTag($tagstring, 0));
		      if ($string =~ /^\s*\@/o && validTag($tagstring, 0)) { # $string =~ /^\s*\@/o
			print STDERR "COMMENTSTRING: $string\n" if ($localDebug);
	
			my $fieldref = stringToFields($string, $fullpath, $linenum, $xmlmode, $apio->lang(), $apio->sublang());
		# print STDERR "APIO: $apio\n";
			foreach my $owner (@{$apiolist}) {
			    my $copy = $fieldref;
			print STDERR "OWNER[1]: $owner\n" if ($localDebug);
			    if ($owner) {
				if (!$inBlockDefine || $do_process == $owner) {
					my @copyarray = @{$copy};
					# print STDERR "COPY[1]: ".$copyarray[1]."\n";
					if ($inBlockDefine && !length($copyarray[0])) { $copyarray[1] =~ s/^field .*?\n/discussion /s; $copy = \@copyarray; }
					# print STDERR "COPY[1]: ".$copyarray[1]."\n";
					if ($dropinvalid != 2) {
						$owner->processComment($copy, 1, $sodec, $parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $self, $hashtreecur, $hashtreeroot);
					}
				}
			    }
			}
# print STDERR "APIO: $apio\n";
			$apio->{APIREFSETUPDONE} = 0;
		      } else {
			if (!$dropinvalid) {
				if (!$inBlockDefine) {
					$string =~ s/\@abstract/ /sg;
					$string =~ s/\@discussion/ /sg;
				}
				$pendingHDcomment = $string;
			# } else {
				# warn("DROPPING $string\n");
			}
		      }
		      if (!$HeaderDoc::dumb_as_dirt) {
			# Drop this comment from the output.
			if ($xmlmode) {
				# We were doing this for HTML when we needed to
				# be able to reparse the tree after copying
				# it to a cloned data type.  This is no longer
				# needed, and the old method (above) is slightly
				# faster.
				$self->hidden(1); $skipchildren = 1;
			} else {
				$self->{TOKEN} = "";
				$self->{FIRSTCHILD} = undef;
				print STDERR "HIDING $self\n" if ($localDebug);
			}
			print STDERR "DROP\n" if ($localDebug);
			$curDeclaration = $oldCurDeclaration;
		      }
	      }
	   }
	}
    } elsif ($token =~ /[;,}]/o) {
	print STDERR "Punctuation\n" if ($localDebug);
	print STDERR "SETTING LASTDEC TO $curDeclaration\n" if ($localDebug);
	$lastDeclaration = "$curDeclaration\n";
	if ($pendingHDcomment) {
                # If we're at the end of a declaration (prior to the
                # following newline) and the declaration starts with
                # a string of text (JavaDoc-style markup), we need to
                # figure out the name of the previous declaration and
                # insert it.

			print STDERR "PRE2\n" if ($localDebug);
		print STDERR "calling getNameAndFieldTypeFromDeclaration [2]\n" if ($localDebug || $fieldTypeDebug);
                my ($name, $type) = $self->getNameAndFieldTypeFromDeclaration($lastDeclaration, $apio, $parseTokens{typedefname}, $case_sensitive, $keywordhashref);
		if ($name) {
                	my $string = "$type $name\n$pendingHDcomment";
			my $fullpath = $apio->fullpath();
			my $linenum = $apio->linenum();

			my $fieldref = stringToFields($string, $fullpath, $linenum, $xmlmode, $apio->lang(), $apio->sublang());
			print STDERR "COMMENTSTRING: $string\n" if ($localDebug);
			print STDERR "OWNERSTART\n" if ($localDebug);
			foreach my $owner (@{$apiolist}) {
				my $copy = $fieldref;
				print STDERR "OWNER[2]: $owner\n" if ($localDebug);
				if ($owner) {
				    if (!$inBlockDefine || $do_process == $owner) {
					my @copyarray = @{$copy};
					# print STDERR "COPY[1]: ".$copyarray[1]."\n";
					if ($inBlockDefine && !length($copyarray[0])) { $copyarray[1] =~ s/^field .*?\n/discussion /s; $copy = \@copyarray; }
					# print STDERR "COPY[1]: ".$copyarray[1]."\n";

					$owner->processComment($copy, 1, $sodec, $parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $self, $hashtreecur, $hashtreeroot);
				    }
				}
			}
			print STDERR "OWNEREND\n" if ($localDebug);
		}
# print STDERR "APIO: $apio\n";
		$apio->{APIREFSETUPDONE} = 0;

		$pendingHDcomment = "";
	} else {
		$eoDeclaration = 1;
	}
	$curDeclaration = "";
    } elsif ($token =~ /[\r\n]/o && $lastDeclaration) {
	print STDERR "NLCR\n" if ($localDebug);
	$lastDeclaration = "";
    } elsif ($token !~ /\s/o) {
	print STDERR "Whitespace\n" if ($localDebug);
	$eoDeclaration = 0;
    }

    $sodec = $nextsodec;

    my $firstchild = $self->firstchild();
    my $next = $self->next();

    if ($firstchild && !$skipchildren) {
	my $nestallowed = commentsNestedIn($token, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $parseTokens{lbrace}, $case_sensitive);
	if ($nestallowed == 1) {
		my $newcontinue;
		($eoDeclaration, $pendingHDcomment, $newcontinue) = $firstchild->processEmbeddedTagsRec($xmlmode, $eoDeclaration, $parseTokensRef, $case_sensitive, $keywordhashref, "", "", "", $apio, $apiolist, $sodec, $lastTreeNode, $enable_javadoc_comments);
		if ($continue) { $continue = $newcontinue; }
	} else {
		my $newcontinue;
		($eoDeclaration, $pendingHDcomment, $newcontinue) = $firstchild->processEmbeddedTagsRec($xmlmode, $eoDeclaration, $parseTokensRef, $case_sensitive, $keywordhashref, "", "$curDeclaration", "", $apio, $apiolist, $sodec, $lastTreeNode, $enable_javadoc_comments);
		if ($continue) { $continue = $newcontinue; }
	}
	$curDeclaration .= textTree($firstchild);
    } elsif ($firstchild && !$skipchildren) {
	$curDeclaration .= textTree($firstchild);
    }

    if ($ntoken) {
	print STDERR "NTOKEN: $ntoken\n" if ($localDebug);
    } else {
	print STDERR "NTOKEN: (null)\n" if ($localDebug);
    }

    if (!$ntoken || $ntoken =~ /[)]/o || casecmp($ntoken, $parseTokens{rbrace}, $case_sensitive)) {
	# Last-ditch chance to process pending comment.
	# This takes care of the edge case where some languages
	# do not require the last item in a struct/record to be
	# terminated by a semicolon or comma.

	if ($ntoken =~ /[)}]/o || casecmp($ntoken, $parseTokens{rbrace}, $case_sensitive)) {
		if ($oldCurDeclaration =~ /\S/) {
			# Don't use curDeclaration here.  It may have been cleared already.
			$lastDeclaration = $oldCurDeclaration.$token;
			print STDERR "CLOSEBRACE LASTDITCH: SETTING LASTDEC TO $lastDeclaration\n" if ($localDebug);
		}
	} else {
		if ($oldCurDeclaration =~ /\S/) {
			print STDERR "NONCLOSEBRACE LASTDITCH: SETTING LASTDEC TO $curDeclaration\n" if ($localDebug);
			$lastDeclaration = $curDeclaration;
		}
	}
	if ($pendingHDcomment) {
		print STDERR "LASTDITCH\n" if ($localDebug);

		print STDERR "LAST DECLARATION:\n$lastDeclaration\nEND OF LAST DECLARATION\n" if ($localDebug);

                # If we're at the end of a declaration (prior to the
                # following newline) and the declaration starts with
                # a string of text (JavaDoc-style markup), we need to
                # figure out the name of the previous declaration and
                # insert it.

			print STDERR "PRE3\n" if ($localDebug);

		if (!$lastDeclaration) { $lastDeclaration = $curDeclaration; };

		print STDERR "calling getNameAndFieldTypeFromDeclaration [3]\n" if ($localDebug || $fieldTypeDebug);
                my ($name, $type) = $self->getNameAndFieldTypeFromDeclaration($lastDeclaration, $apio, $parseTokens{typedefname}, $case_sensitive, $keywordhashref);
		if ($name) {
                	my $string = "$type $name\n$pendingHDcomment";
			my $fullpath = $apio->fullpath();
			my $linenum = $apio->linenum();

			my $fieldref = stringToFields($string, $fullpath, $linenum, $xmlmode, $apio->lang(), $apio->sublang());
			print STDERR "COMMENTSTRING: $string\n" if ($localDebug);
			foreach my $owner (@{$apiolist}) {
				my $copy = $fieldref;
				print STDERR "OWNER[3]: $owner\n" if ($localDebug);
				if ($owner) {
			    	if (!$inBlockDefine || $do_process == $owner) {
					my @copyarray = @{$copy};
					# print STDERR "COPY[1]: ".$copyarray[1]."\n";
					if ($inBlockDefine && !length($copyarray[0])) { $copyarray[1] =~ s/^field .*?\n/discussion /s; $copy = \@copyarray; }
					# print STDERR "COPY[1]: ".$copyarray[1]."\n";

					$owner->processComment($copy, 1, $sodec, $parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $self, $hashtreecur, $hashtreeroot);
				    }
				}
			}
		}
# print STDERR "APIO: $apio\n";
		$apio->{APIREFSETUPDONE} = 0;

		$pendingHDcomment = "";
	}
    }
			# $sodec = $oldsodec;
    if ($next && $continue) {
	($eoDeclaration, $pendingHDcomment, $continue) = $next->processEmbeddedTagsRec($xmlmode, $eoDeclaration, $parseTokensRef, $case_sensitive, $keywordhashref, $lastDeclaration, $curDeclaration, $pendingHDcomment, $apio, $apiolist, $sodec, $lastTreeNode, $enable_javadoc_comments);
    }

    return ($eoDeclaration, $pendingHDcomment, $continue);
}

# THIS CODE USED TO PROCESS COMMENTS WHENEVER IT IS TIME.
	      # my $fieldref = stringToFields($string, $fullpath, $linenum, $xmlmode, $lang, $sublang);
	      # $apio->processComment($fieldref, 1, $self, $parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $self, $hashtreecur, $hashtreeroot);
		# $apio->{APIREFSETUPDONE} = 0;

# /*!
#     @abstract
#         Gets/sets the next parse tree node (sibling) for this one.
#     @param self
#         This parse tree object.
#     @param NEXT
#         The new next node. (Optional.)
#  */
sub next {
    my $self = shift;

    if (@_) {
	my $node = shift;
        $self->{NEXT} = $node;
    }
    return $self->{NEXT};
}

# /*!
#     @abstract
#         Gets/sets the first child parse tree node for this one.
#     @param self
#         This parse tree object.
#     @param NEXT
#         The new child node. (Optional.)
#  */
sub firstchild {
    my $self = shift;

    if (@_) {
	my $node = shift;
        $self->{FIRSTCHILD} = $node;
    }
    return $self->{FIRSTCHILD};
}


# /*!
#     @abstract
#         Gets/sets the parent parse tree node for this one.
#     @param self
#         This parse tree object.
#     @param NEXT
#         The new parent node. (Optional.)
#  */
sub parent {
    my $self = shift;

    if (@_) {
	my $node = shift;
        $self->{PARENT} = $node;
	# weaken($self->{PARENT});
    }
    return $self->{PARENT};
}


# /*!
#     @abstract
#         Prints the text of the parse tree to standard output (for debugging).
#     @param self
#         The top of the tree/subtree to print
#     @seealso //apple_ref/perl/instm/HeaderDoc::ParseTree/dbprint//() dbprint
#  */
sub printTree {
    my $self = shift;

    print STDERR "BEGINPRINTTREE\n";
    print STDERR $self->textTree();
    print STDERR "ENDPRINTTREE\n";
}

# /*!
#     @abstract
#         Prints a text representation of the parse tree as a string.
#     @param self
#         The top of the tree/subtree to dump.
#  */
sub textTree {
    my $self = shift;
    my $nohidden = 0;
    if (@_) {
	$nohidden = shift;
    }
    my $parserState = $self->parserState();
    my $lastnode = undef;

    my $localDebug = 0;

    if ($parserState) {
	$lastnode = $parserState->{lastTreeNode};
    }

    print STDERR "TEXTTREE: LASTNODE: $lastnode\n" if ($localDebug);
    if ($lastnode && $localDebug) { print STDERR "LASTNODE TEXT: \"".$lastnode->token()."\"\n"; }

    my ($string, $continue) = $self->textTreeSub(0, $nohidden, "", "", "", $lastnode);
    return $string;
}

# /*!
#     @abstract
#         Prints a text representation of the parse tree as a string with
#         all comments omitted.
#     @param self
#         The top of the tree/subtree to dump.
#     @param lang
#         The current language.
#     @param sublang
#         The current language dialect (e.g. <code>cpp</code> for C++).
#  */
sub textTreeNC {
    my $self = shift;
    my $lang = shift;
    my $sublang = shift;
    # my ($sotemplate, $eotemplate, $operator, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $sofunction,
        # $parseTokens{soprocedure}, $parseTokens{sopreproc}, $parseTokens{lbrace}, $parseTokens{rbrace}, $unionname, $structname,
	# $enumname,
        # $typedefname, $varname, $constname, $structisbrace, $macronamesref,
        # $classregexp, $classbraceregexp, $classclosebraceregexp, $accessregexp,
	# $requiredregexp, $propname, $objcdynamicname, $objcsynthesizename, $moduleregexp, $definename,
	# $functionisbrace, $classisbrace, $parseTokens{lbraceconditionalre}, $parseTokens{lbraceunconditionalre}, $assignmentwithcolon,
	# $labelregexp, $parmswithcurlybraces, $superclasseswithcurlybraces, $parseTokens{soc}) = parseTokens($lang, $sublang);

    my $nohidden = 0;
    if (@_) {
	$nohidden = shift;
    }
    my %parseTokens = %{parseTokens($lang, $sublang)};

    my ($string, $continue) = $self->textTreeSub(1, $nohidden, $parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b});
    return $string;
}

# /*!
#     @abstract
#         The recursive portion of {@link textTree} and
#         {@link textTreeNC}.
#     @param self
#         The top of the tree/subtree to dump.
#     @param nc
#         Set to 1 if comments should be dropped, else 0.
#     @param nh
#         Set to 1 if hidden tokens should be dropped, else 0.
#     @param soc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc_b
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param lastnode
#         The last node to include.  This node and children of this node are
#         included, but not siblings of this node.
#  */
sub textTreeSub
{
    my $self = shift;
    my $nc = shift;
    my $nh = shift;
    my $soc = shift;
    my $ilc = shift;
    my $ilc_b = shift;
    my $lastnode = shift;

    my $localDebug = 0;
    my $continue = 1;

    print STDERR "TTSUB: LN: $lastnode SELF: $self TOK: ".$self->token()."\n" if ($localDebug);

    if ($lastnode == $self) {
	print STDERR "TTSUB: CONTINUE -> 0\n" if ($localDebug);
	$continue = 0;
    }

    my $string = "";
    my $skip = 0;
    my $token = $self->token();
    if ($nc) {
	if ($localDebug) {
		print STDERR "NC\n";
		print STDERR "SOC: $soc\n";
		print STDERR "ILC: $ilc\n";
		print STDERR "ILC_B: $ilc_b\n";
		print STDERR "TOK: $token\n";
	}
	if (($token eq "$soc") || ($token eq "$ilc") || ($token eq "$ilc_b")) {
		$skip = 1;
	}
    }
    if ($nh) {
	if ($self->hidden()) {
		$skip = 1;
	}
    }

    if (!$skip) {
	$string .= $token;
	# if (!$continue) {
		# return ($string, $continue);
	# }
	if ($self->{FIRSTCHILD}) {
		my $node = $self->{FIRSTCHILD};
		bless($node, "HeaderDoc::ParseTree");
		my ($newstring, $newcontinue) = $node->textTreeSub($nc, $nh, $soc, $ilc, $ilc_b, $lastnode);
		if ($continue) { $continue = $newcontinue; }
		$string .= $newstring;
	}
    }
    if (!$continue) {
	return ($string, $continue);
    }
    if ($self->{NEXT}) {
	my $node = $self->{NEXT};
	bless($node, "HeaderDoc::ParseTree");
	my ($newstring, $newcontinue) = $node->textTreeSub($nc, $nh, $soc, $ilc, $ilc_b, $lastnode);
	$continue = $newcontinue;
	$string .= $newstring;
    }

    return ($string, $continue);
}


# /*!
#     @abstract
#         Returns an XML representation of a parse tree/subtree.
#     @param self
#         The top of the tree/subtree to print.
#     @param keep_whitespace
#         Set to 1 if whitespace should be preserved in the
#         output, else 0.
#     @param drop_pdefine_contents
#         Set to 1 if the contents of <code>#define</code> macros should be
#         dropped from the output, else 0.
#  */
sub xmlTree {
    my $self = shift;
    my $keep_whitespace = shift;
    my $drop_pdefine_contents = shift;
    # my $apiOwner = shift;
    # my $apio = $self->apiOwner();

    if ($self->{XMLTREE}) { return $self->{XMLTREE}; }

    # $self->printTree();
    # $self->dbprint();

    my $apiOwner = undef;
    my $lang = undef;
    my $sublang = undef;
    my $occmethod = 0;
    my $localDebug = 0;

    my $debugName = ""; # "TypedefdStructWithCallbacksAndStructs";

    if ($self->apiOwner()) {
	$apiOwner = $self->apiOwner();
	bless($apiOwner, "HeaderDoc::HeaderElement");
	bless($apiOwner, $apiOwner->class());
	$lang = $apiOwner->lang();
	$sublang = $apiOwner->sublang();

	if (($debugName ne "") && ($apiOwner->name() eq $debugName)) {
		$colorDebug = 1;
	} else {
		$colorDebug = 0;
		print STDERR $apiOwner->name()."\n" if ($localDebug);
	}

	if ($apiOwner->class() eq "HeaderDoc::Method") {
		$occmethod = 1;
	} else {
		$occmethod = 0;
	}

	# print STDERR "APIOWNERS was type $apiOwner\n";
    } else {
	warn("WARNING: Could not find API Owner for parse tree $self.\nFaking data structures.  Please file a bug.\n");
	$lang = "C"; # $HeaderDoc::lang; # Default to C/C?
	$sublang = "C"; # $HeaderDoc::sublang;
	$apiOwner = HeaderDoc::HeaderElement->new("LANG" => $lang, "SUBLANG" => $sublang);
	$apiOwner->lang($lang);
	$apiOwner->sublang($sublang);
	$occmethod = 0; # guess
    }
    # colorizer goes here


    # my ($sotemplate, $eotemplate, $operator, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $sofunction,
        # $parseTokens{soprocedure}, $parseTokens{sopreproc}, $parseTokens{lbrace}, $parseTokens{rbrace}, $unionname, $structname,
	# $enumname,
        # $typedefname, $varname, $constname, $structisbrace, $parseTokens{macronames},
	# $classregexp, $classbraceregexp, $classclosebraceregexp, $accessregexp,
        # $requiredregexp, $propname, $objcdynamicname, $objcsynthesizename, $moduleregexp, $definename,
	# $functionisbrace, $classisbrace, $parseTokens{lbraceconditionalre}, $parseTokens{lbraceunconditionalre}, $assignmentwithcolon,
	# $labelregexp, $parmswithcurlybraces, $superclasseswithcurlybraces, $parseTokens{soc}) = parseTokens($apiOwner->lang(), $apiOwner->sublang());
    my %parseTokens = %{parseTokens($apiOwner->lang(), $apiOwner->sublang())};

    $self->processEmbeddedTags(1, $apiOwner);
	# , $sotemplate, $eotemplate, $operator, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $sofunction,
        # $parseTokens{soprocedure}, $parseTokens{sopreproc}, $parseTokens{lbrace}, $parseTokens{rbrace}, $unionname, $structname,
	# $enumname,
        # $typedefname, $varname, $constname, $structisbrace, $parseTokens{macronames});

    my $lastnode = undef;
    my $parserState = $self->parserState();
    if ($parserState) {
	$lastnode = $parserState->{lastTreeNode};
    }

    # my ($sotemplate, $eotemplate, $operator, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $sofunction,
        # $parseTokens{soprocedure}, $parseTokens{sopreproc}, $parseTokens{lbrace}, $parseTokens{rbrace}, $unionname, $structname,
	# $enumname,
        # $typedefname, $varname, $constname, $structisbrace, $parseTokens{macronames},
	# $classregexp, $classbraceregexp, $classclosebraceregexp, $accessregexp,
        # $requiredregexp, $propname, $objcdynamicname, $objcsynthesizename, $moduleregexp, $definename,
	# $functionisbrace, $classisbrace, $parseTokens{lbraceconditionalre}, $parseTokens{lbraceunconditionalre}, $assignmentwithcolon,
	# $labelregexp, $parmswithcurlybraces, $superclasseswithcurlybraces, $parseTokens{soc}) = parseTokens($lang, $sublang);
    my %parseTokens = %{parseTokens($lang, $sublang)};

    # my %pt = ();
    # $pt{sotemplate} = $sotemplate;
    # $pt{soconstructor} = $sotemplate;

    my $ctstate = newCTState( undef, keep_whitespace => $keep_whitespace, apio => $apiOwner, type => "",
    	depth => 0, inComment => 0, inQuote => 0, inRubyQuote => 0, inObjCMethod => $occmethod,
	lastBrace => "", parseTokensRef => \%parseTokens, prespace => "", lang => $lang, sublang => $sublang,
	xmlmode => 1, newlen => 0, breakable => 0, inMacro => 0, inEnum => 0, seenEquals => 0,
	lastKeyword => "", lastnstoken => "", lastTreeNode => $lastnode, lastTokenType => "",
	spaceSinceLastToken => 0, inAttribute => 0, inRaises => 0, inTypeOf => 0,
	drop_pdefine_contents => $drop_pdefine_contents, ASinName => 0);

    my ($retvalref, $junka, $junkb, $junkc, $junkd, $junke, $lastTokenType, $spaceSinceLastToken) = $self->colorTreeSub($ctstate);
    my $retval = ${$retvalref};

    # my $retval = "";
    # $retval = $self->textTree();
    # $self->printTree();

    $self->{XMLTREE} = $retval;

    return $retval;
}

# /*!
#     @abstract
#         Returns an HTML representation of a parse tree/subtree.
#     @param self
#         The top of the tree/subtree to print.
#     @param keep_whitespace
#         Set to 1 if whitespace should be preserved in the
#         output, else 0.
#     @param drop_pdefine_contents
#         Set to 1 if the contents of <code>#define</code> macros should be
#         dropped from the output, else 0.
#  */
sub htmlTree {
    my $self = shift;
    my $keep_whitespace = shift;
    my $drop_pdefine_contents = shift;
    # my $apiOwner = shift;

    # print STDERR "TREE\n";
    # $self->printTree();
    # $self->dbprint();
    # print STDERR "ENDTREE\n";
    my $apiOwner = undef;
    my $lang = undef;
    my $sublang = undef;
    my $occmethod = 0;
    my $localDebug = 0;

    my $debugName = ""; # "TypedefdStructWithCallbacksAndStructs";

    # $self->dbprint();

    if ($self->{HTMLTREE}) {
	 print STDERR "SHORTCUT\n" if ($localDebug);
	 return $self->{HTMLTREE};
    }

    if ($self->apiOwner()) {
	$apiOwner = $self->apiOwner();
	bless($apiOwner, "HeaderDoc::HeaderElement");
	bless($apiOwner, $apiOwner->class());
	$lang = $apiOwner->lang();
	$sublang = $apiOwner->sublang();

	# print STDERR "LANG: $lang APIO: $apiOwner\n";

	if (($debugName ne "") && ($apiOwner->name() eq $debugName)) {
		$colorDebug = 1;
	} else {
		$colorDebug = 0;
		print STDERR $apiOwner->name()."\n" if ($localDebug);
	}

	if ($apiOwner->class() eq "HeaderDoc::Method") {
		$occmethod = 1;
	} else {
		$occmethod = 0;
	}

	print STDERR "APIOWNERS was type $apiOwner\n" if ($localDebug);
    } else {
	warn("WARNING: Could not find API Owner for parse tree $self.\nFaking data structures.  Please file a bug.\n");
	$lang = "C"; # $HeaderDoc::lang;
	$sublang = "C"; # $HeaderDoc::sublang;
	$apiOwner = HeaderDoc::HeaderElement->new("LANG" => $lang, "SUBLANG" => $sublang);
	cluck("BT\n");
	$apiOwner->apiOwner($HeaderDoc::headerObject);
	$apiOwner->lang($lang);
	$apiOwner->sublang($sublang);
	$occmethod = 0; # guess
    }
    # colorizer goes here

    my $lastnode = undef;
    my $parserState = $self->parserState();
    if ($parserState) {
	$lastnode = $parserState->{lastTreeNode};
    }

    # if ($lang eq "shell") {
	# $keep_whitespace = 1;
    # }

    # my ($sotemplate, $eotemplate, $operator, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $sofunction,
        # $parseTokens{soprocedure}, $parseTokens{sopreproc}, $parseTokens{lbrace}, $parseTokens{rbrace}, $unionname, $structname,
	# $enumname,
        # $typedefname, $varname, $constname, $structisbrace, $parseTokens{macronames},
	# $classregexp, $classbraceregexp, $classclosebraceregexp, $accessregexp,
        # $requiredregexp, $propname, $objcdynamicname, $objcsynthesizename, $moduleregexp, $definename,
	# $functionisbrace, $classisbrace, $parseTokens{lbraceconditionalre}, $parseTokens{lbraceunconditionalre}, $assignmentwithcolon,
	# $labelregexp, $parmswithcurlybraces, $superclasseswithcurlybraces, $soconstructor) = parseTokens($lang, $sublang);
    my %parseTokens = %{parseTokens($lang, $sublang)};

    $self->processEmbeddedTags(0, $apiOwner); 
	# , $sotemplate, $eotemplate, $operator, $parseTokens{soc}, $parseTokens{eoc}, $parseTokens{ilc}, $parseTokens{ilc_b}, $sofunction,
        # $parseTokens{soprocedure}, $parseTokens{sopreproc}, $parseTokens{lbrace}, $parseTokens{rbrace}, $unionname, $structname,
	# $enumname,
        # $typedefname, $varname, $constname, $structisbrace, $parseTokens{macronames});

    # my %pt = ();
    # $pt{sotemplate} = $sotemplate;
    # $pt{soconstructor} = $sotemplate;

    my $ctstate = newCTState( undef, keep_whitespace => $keep_whitespace, apio => $apiOwner, type => "",
    	depth => 0, inComment => 0, inQuote => 0, inRubyQuote => 0, inObjCMethod => $occmethod,
	lastBrace => "", parseTokensRef => \%parseTokens, prespace => "", lang => $lang, sublang => $sublang,
	xmlmode => 0, newlen => 0, breakable => 0, inMacro => 0, inEnum => 0, seenEquals => 0,
	lastKeyword => "", lastnstoken => "", lastTreeNode => $lastnode, lastTokenType => "",
	spaceSinceLastToken => 0, inAttribute => 0, inRaises => 0, inTypeOf => 0,
	drop_pdefine_contents => $drop_pdefine_contents, ASinName => 0);

    my ($retvalref, $junka, $junkb, $junkc, $junkd, $junke, $lastTokenType, $spaceSinceLastToken) = $self->colorTreeSub($ctstate);
    my $retval = ${$retvalref};

    # my $retval = "";
    # $retval = $self->textTree();
    # $self->printTree();

    if ($HeaderDoc::align_columns) {
	my @retarr = split(/(\n)/s, $retval);
	my $newret = "";
	foreach my $line (@retarr) {
		my $first = "";
		# print STDERR "LINE: $line\n";
		if ($line =~ s/^<tr><td nowrap=\"nowrap\"><nowrap>//s) {
			$first = "<tr><td nowrap=\"nowrap\"><nowrap>";
			# print STDERR "FIRST (line = \"$line\")\n";
		}
		if ($line =~ s/^( +)//) {
			my $spaces = $1;
			my $count = ($spaces =~ tr/^ //);
			while ($count--) { $line = "&nbsp;$line"; }
			$newret .= "$first$line";
		} else {
			$newret .= "$first$line";
		}
	}
	$retval = $newret;
	$retval = "<table><tr><td nowrap=\"nowrap\"><nowrap>$retval</nowrap></td></tr></table>";
    }

    $self->{HTMLTREE} = $retval;

    return $retval;
}

# /*!
#     @abstract
#         Recursively walks the descendants of this node (and their
#         siblngs/descendants) and returns the first non-space node
#         that is not in a comment.
#     @param self
#         The node whose children you want to search.
#     @param soc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc_b
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#  */
sub childpeeknc
{
    my $self = shift;
    my $soc = shift;
    my $ilc = shift;
    my $ilc_b = shift;
    my $cache = $self->{CPNC};
    if ($cache) { return $cache; }

    my $node = $self->{FIRSTCHILD};

    if (!$node) { return ""; }

    bless($node, "HeaderDoc::ParseTree");

    if (!$node->token()) { return $node->childpeeknc($soc, $ilc, $ilc_b) || return $node->nextpeeknc($soc, $ilc, $ilc_b); }
    if ($node->token() =~ /\s/o) { return $node->childpeeknc($soc, $ilc, $ilc_b) || return $node->nextpeeknc($soc, $ilc, $ilc_b); }
    if ($node->token() eq $soc) { return $node->childpeeknc($soc, $ilc, $ilc_b) || return $node->nextpeeknc($soc, $ilc, $ilc_b); }
    if ($node->token() eq $ilc || $node->token() eq $ilc_b) { return $node->childpeeknc($soc, $ilc, $ilc_b) || return $node->nextpeeknc($soc, $ilc, $ilc_b); }

    $cache = $node->token();
    $self->{CPNC} = $cache;

    return $cache;
}

# /*!
#     @abstract
#         Recursively walks the siblings of this node (and their
#         descendants) and returns the first non-space node.
#     @param self
#         The node whose siblings you want to search.
#  */
sub nextpeek
{
    my $self = shift;

    # This cache appears to be slowing things down.
    # if ($self->{NPCACHE}) { return $self->{NPCACHE}; }

    my $node = undef;
    if ($self->firstchild()) {
	$node = $self->firstchild();
	$node = $node->next;
    } else {
	$node = $self->next();
    }

    if (!$node) {
	# $self->{NPCACHE} = "";
	return "";
    }

    my $token = $node->token();
    if ($token =~ /\s/o && $token !~ /[\r\n]/o) {
	my $ret = $node->nextpeek();
	# $self->{NPCACHE} = $ret;
	return $ret;
    }
    if ($node->hidden()) {
	my $ret = $node->nextpeek();
	# $self->{NPCACHE} = $ret;
	return $ret;
    }

    # $self->{NPCACHE} = $node->token();
    return $node->token();

}

# /*!
#     @abstract
#         Recursively walks the siblings of this node (and their
#         descendants) and returns the first non-space node that
#         is not in a comment.
#     @param self
#         The node whose siblings you want to search.
#     @param soc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc_b
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#  */
sub nextpeeknc
{
    my $self = shift;
    my $soc = shift;
    my $ilc = shift;
    my $ilc_b = shift;

    my $node = $self->nextTokenNoComments($soc, $ilc, $ilc_b, 0, 0);
    if (!$node) { return ""; }

    return $node->token();

}

# /*!
#     @abstract
#         Recursively walks the siblings of this node (and their
#         descendants) and returns the second non-space node that
#         is not in a comment.
#     @param self
#         The node whose siblings you want to search.
#     @param soc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc_b
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#  */
sub nextnextpeeknc
{
    my $self = shift;
    my $soc = shift;
    my $ilc = shift;
    my $ilc_b = shift;

    my $node = $self->nextTokenNoComments($soc, $ilc, $ilc_b, 0, 0);
    if (!$node) { return ""; }

    my $nodeafter = $node->nextTokenNoComments($soc, $ilc, $ilc_b, 0, 0);
    if (!$nodeafter) { return ""; }

    return $nodeafter->token();

}

# /*!
#     @abstract
#         Recursively walks the siblings of this node (and their
#         descendants) and returns the first node (including spaces) that
#         is not in a comment.
#     @param self
#         The node whose siblings you want to search.
#     @param soc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param ilc_b
#         The start-of-comment token for the current programming language.
#         Obtained by a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#  */
sub nextTokenNoComments($$$$$)
{
    my $self = shift;
    my $soc = shift;
    my $ilc = shift;
    my $ilc_b = shift;
    my $failOnHDComments = shift;
    my $enable_javadoc_comments = shift;

    my $localDebug = 0;

    my $cache = $self->{NTNC};
    if ($cache) { return $cache; }

    my $node = $self->{NEXT};

    if (!$node) { return undef }

    bless($node, "HeaderDoc::ParseTree");
# print STDERR "SOC: $soc ILC: $ilc ILC_B: $ilc_b\n" if ($colorDebug);

    # print STDERR "MAYBE ".$node->token()."\n";

    if ($failOnHDComments) {
	# print STDERR "FOHDC\n";
	# print STDERR "FC: ".$node->firstchild()."\n";
	if ($node->firstchild() && $node->firstchild()->next()) {
	    # print STDERR "POINT 1\n";
	    # first child always empty.
	    my $testnode = $node->firstchild()->next();
	    if (($node->token() eq $ilc) || ($node->token() eq $ilc_b)) {
	    # print STDERR "ILC\n";
		if ($node->token() eq $ilc) {
			my $ntoken = "";
			if ($testnode->next()) { $ntoken = $testnode->next()->token(); }
			if ($testnode->token() eq "!" || ($enable_javadoc_comments && $testnode->token eq "*" && $ntoken !~ /^\*/)) {
				print STDERR "Unexpected HD Comment\n" if ($localDebug);
				return undef;
			}
		} else {
			if ($testnode->token() eq $ilc) {
				my $nntoken = "";
				if ($testnode->next() && $testnode->next()->next()) { $nntoken = $testnode->next()->next()->token(); }
				if ($testnode->next() && (($testnode->next()->token() eq "!") || ($enable_javadoc_comments && $testnode->next()->token eq "*" && $nntoken !~ /^\*/))) {
					print STDERR "Unexpected HD Comment\n" if ($localDebug);
					return undef;
				}
			}
		}
	    } elsif ($node->token() eq $soc) {
	    # print STDERR "SOC\n";
		my $ntoken = "";
		if ($testnode->next()) { $ntoken = $testnode->next()->token(); }
		if (($testnode->token() eq "!") || ($enable_javadoc_comments && $testnode->token eq "*" && $ntoken !~ /^\*/)) {
			print STDERR "Unexpected HD Comment\n" if ($localDebug);
			return undef;
		}
	    # } else {
		# print STDERR "TOKEN: ".$node->token()."\n";
	    }
	}
    }

    if (!$node->token()) { return $self->{NTNC} = $node->nextTokenNoComments($soc, $ilc, $ilc_b, $failOnHDComments, $enable_javadoc_comments); }
    if ($node->token() =~ /\s/o) { return $self->{NTNC} = $node->nextTokenNoComments($soc, $ilc, $ilc_b, $failOnHDComments, $enable_javadoc_comments); }
    if ($node->token() eq $soc) { return $self->{NTNC} = $node->nextTokenNoComments($soc, $ilc, $ilc_b, $failOnHDComments, $enable_javadoc_comments); }
    if ($node->token() eq $ilc || $node->token() eq $ilc_b) { return $self->{NTNC} = $node->nextTokenNoComments($soc, $ilc, $ilc_b, $failOnHDComments, $enable_javadoc_comments); }

    $self->{NTNC} = $node;
    # weaken($self->{NTNC});
    return $node;
}

# /*!
#     @abstract
#         Walks up the parent tree until it reaches a node
#         that is a sibling of matchingnode, then returns
#         the node after that one.
#     @param self
#         The starting node.
#     @param matchingnode
#         The node whose level you want ot match.
#     @param fullpath
#         The path of the file containing this parse
#         tree (used for debugging messages).
#  */
sub nextAtLevelOf
{
    my $self = shift;
    my $matchingnode = shift;
    my $fullpath = shift;
    my $nextnode = $self;

    while ($nextnode && !$nextnode->isAfter($matchingnode)) {
	$nextnode = $nextnode->parent();
    }
    if ($nextnode) {
	$nextnode = $nextnode->next();
    } else {
	my $tt = $self;
	while ($tt->parent()) { $tt = $tt->parent(); }
	my $apio = $tt->apiOwner();
	cluck("TT: $tt\n");
	# my $fullpath = $apio->fullpath();
	warn("$fullpath:0:Ran off top of stack looking for next node.\n");
	# $nextnode = $matchingnode->next();
	$nextnode = undef;
    }
    return $nextnode;
}

# /*!
#     @abstract
#         Trivially returns whether this token should be
#         treated a the start of a macro.
#     @param self
#         This parse tree instance (unused).
#     @param token
#         The token to check.
#     @param lang
#         The current language.
#     @param sublang
#         The current language dialect (e.g. <code>cpp</code> for C++).
#  */
sub isMacro
{
    my $self = shift;
    my $token = shift;
    my $lang = shift;
    my $sublang = shift;

    if ($lang ne "C" && $lang ne "Csource") { return 0; }

    if ($token =~ /^\#\w+/o) { return 1; }

    return 0;
}

# /*!
#     @abstract
#         Returns an HTML or XML representation of a parse tree/subtree.
#     @param self
#         The top of the tree/subtree to print.
#     @param ctstate
#          The colortree state object.  It should contain the parameters
#          listed below.
#     @param keep_whitespace
#         Set to 1 if whitespace should be preserved in the
#         output, else 0.
#     @param apio
#         The owner of the parse tree.  This may be an API owner or
#         may jut be an ordinary type class instance (e.g. Struct).
#
#         This is mostly used for using methods in the HeaderElement
#         class and obtaining filenames/line numbers for debugging.
#     @param type
#         Indicates information about what we're currently parsing.
#         Possible values are:
#
#         <ul>
#             <li><code>typedef</code></li>
#             <li><code>struct</code></li>
#             <li><code>record</code></li>
#             <li><code>union</code></li>
#             <li><code>enum</code></li>
#             <li><code>funcptr</code></li>
#             <li><code>hexnumber</code></li>
#             <li><code>pastd</code> - pascal type declaration</li>
#             <li><code>pasvar</code> - pascal variable declaration</li>
#             <li><code>pasrec</code> - pascal record declaration</li>
#         </ul>
#     @param depth
#         The nesting depth within the parse tree.  Used for leading
#         space calculations.
#     @param inComment
#         On initial call, 0.  On recursion, possible values are:
#         <ul>
#             <li>0 &mdash; Not in a comment.</li>
#             <li>1 &mdash; In a multi-line (soc) comment.</li>
#             <li>2 &mdash; In a single-line (ilc) comment.</li>
#         </ul>
# 
#     @param inQuote
#         On initial call, 0.  On recursion, possible values are:
#         <ul>
#             <li>0 &mdash; Not in quote.</li>
#             <li>1 &mdash; Inside double quotes.</li>
#             <li>2 &mdash; Inside single quote.</li>
#             <li>3 &mdash; In a Python triple-quote or a Ruby quote.
#                 See <code>inRubyQuote</code> to determine which.</li>
#         </ul>
#     @param inRubyQuote
#         On initial call, 0.  On recursion, possible values are:

#         <ul>
#             <li>0 &mdash; Not in a Ruby quote.</li>
#             <li>1 &mdash; Inside either <code>%{</code> or <code>%/</code></li>
#             <li>2 &mdash; Inside <code>&lt;&lt;</code>.  Prevents coloring
#                 of the initial token.</li>
#         </ul>
#     @param inObjCMethod
#         Pass in 1 if this is a parse tree for an Objective-C method,
#         else pass in 0.
#     @param lastBrace
#         The last brace or parentheses character seen,
#         or +/- in Objective-C methods), or special tokens
#         that are treated like braces.
#     @param parseTokensRef
#         A reference to the parse tokens hash returned by
#         a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param prespace
#         The leading spaces that should be inserted
#         to the left of this token if it wraps to the start of
#         a line.
#     @param lang
#         The current language.
#     @param sublang
#         The current language dialect (e.g. <code>cpp</code> for C++).
#     @param xmlmode
#         Set to 1 if you want XML output, 0 for HTML.
#     @param newlen
#         The length of the current line.  Set to 0 on first call.
#     @param breakable
#         Indicates whether the formatter can break before this token.
#         <ul>
#             <li>0 &mdash; Not a good spot to break.</li>
#             <li>1 &mdash; Breakable before this token.</li>
#             <li>2 &mdash; Breakable <b>after</b> this token.</li>
#             <li>3 &mdash; This token <b>must</b> be at the start of a line.</li>
#         </ul>
#     @param inMacro
#         Set to 0 on first call.  During tree walk, this goes high when
#         the tree walk code enters a C preprocessor macro and remains high
#         until the end of the macro.
#     @param inEnum
#         Set to 0 on first call.  Goes high when inside an enumeration.
#     @param seenEquals
#         Set to 0 on first call.  Goes high after the equals sign in a
#         variable declaration.
#     @param lastKeyword
#         Contains the last keyword encountered.  Set to an empty string
#         on first call.  Used for guessing types in generating link
#         requests.
#     @param lastnstoken
#         The last non-space token encountered.  Set to an empty string
#         on first call.
#     @param lastTreeNode
#         The tree node at which output stops.  This node and its
#         descendants are included in output, but not this node's siblings.
#     @param lastTokenType
#         Contains the type of the last token emitted.  One of:
#         <code>star</code>, <code>string</code>, <code>char</code>,
#         <code>comment</code>, <code>preprocessor</code>,
#         <code>number</code>, <code>keyword</code>,
#         <code>function</code>, <code>var</code>,
#         <code>template</code>, <code>type</code>,
#         <code>param</code>, or <code>ignore</code>.
#
#         On initial call, an empty string. 
#     @param spaceSinceLastToken
#         Indicates that whitespace has been seen since the last non-space
#         token.  This ensures that required space does not get deleted.
#
#         On initial call, 0.
#     @param inAttribute
#         On initial call, 0.  On recursion, set to 1 after
#         the <code>attribute</code> keyword (or equivalent) is encountered.
#
#         Causes the next word token to be interpreted as ordinary text.
#
#         Also prevents the normal line break after the open parenthesis.
#     @param inRaises
#         On initial call, 0.  On recursion, set to 1 after
#         the <code>raises</code> keyword (or equivalent) is encountered.
#
#         Causes the next word token to be interpreted as a variable (an
#         exception) instead of a type.
#
#         Also prevents the normal line break after the open parenthesis.
#     @param inTypeOf
#         On initial call, 0.  On recursion, set to 1 after
#         the <code>typeof</code> keyword (or equivalent) is encountered.
#
#         Causes the next word token to be interpreted as a variable instead
#         of a type.
#
#         Also prevents the normal line break after the open parenthesis.
#     @param drop_pdefine_contents
#         Set to 1 if the contents of <code>#define</code> macros should be
#         dropped from the output, else 0.
#     @param ASinName
#         On initial call, set to 0.  During the recursive walk of the
#         parse tree, this gets set to 1 when the current tree node in
#         an AppleScript name.
#
#     @var tokenType
#         Contains the type of the last token emitted.  One of:
#         <code>star</code>, <code>string</code>, <code>char</code>,
#         <code>comment</code>, <code>preprocessor</code>,
#         <code>number</code>, <code>keyword</code>,
#         <code>function</code>, <code>var</code>,
#         <code>template</code>, <code>type</code>,
#         <code>param</code>, or <code>ignore</code>.
#
#     @discussion
#         This is a helper function shared by {@link htmlTree} and
#         {@link xmlTree}.  You should not call this directly.
#  */
sub colorTreeSub
{
    my $self = shift;
    my $ctstateref = shift;

    my $localDebug = 0;
    my $psDebug = 0;
    my $treeDebug = 0;
    my $dropDebug = 0;
    my $tokenDebug = 0;
    my $codePathDebug = 0;
    my $rubyDebug = 0;

    print STDERR "***** TOKEN: ".$self->{TOKEN}." *****\n" if ($codePathDebug || $localDebug);

# print STDERR "IN COLORTREESUB\n";

    my $continue = 1;
    my %ctstate = %{$ctstateref};

    if ($self == $ctstate{lastTreeNode}) {
	print STDERR "Node is last node in tree.  Ending after this node.\n" if ($localDebug || $codePathDebug);
	$continue = 0;
    }

    my %parseTokens = %{$ctstate{parseTokensRef}};
    my %macroList = %{$parseTokens{macronames}};
    my $oldLastBrace = $ctstate{lastBrace};
    my $oldDepth = $ctstate{depth};
    my $oldInMacro = $ctstate{inMacro};
    my $oldInQuote = $ctstate{inQuote};
    my $oldLastKeyword = $ctstate{lastKeyword};
    my $oldInComment = $ctstate{inComment} ;
    my $oldInAttribute = $ctstate{inAttribute};
    my $oldInRaises = $ctstate{inRaises};
    my $oldInTypeOf = $ctstate{inTypeOf};
    my $dropFP = 0;

    # This cache slows things down now that it works....
    # if ($self->{CTSUB}) { return (\$self->{CTSTRING}, $self->{CTSUB}); }

    my $keep_all_newlines = 0;

    print STDERR "DPC: ".$ctstate{drop_pdefine_contents}."\n" if ($localDebug);

    if ($ctstate{lang} eq "shell" || $ctstate{sublang} eq "javascript") {
	$keep_all_newlines = 1;
    } elsif ($ctstate{lang} eq "python") {
	$keep_all_newlines = 1;
	$ctstate{keep_whitespace} = 1;
    }

    if ($ctstate{xmlmode} && $localDebug) {
	print STDERR "XMLMODE.\n";
    }

    # foreach my $listitem (keys %macroList) { print STDERR "ML: $listitem\n"; }

    print STDERR "IM: ".$ctstate{inMacro}."\n" if ($localDebug);
    print STDERR "IAtt: ".$ctstate{inAttribute}."\n" if ($localDebug);
    print STDERR "IRai: ".$ctstate{inRaises}."\n" if ($localDebug);

    my $mustbreak = 0;
    my $nextprespace = "";
    my $string = "";
    my $tailstring = "";
    my $token = $self->{TOKEN};
    my $escapetoken = "";
    my ($case_sensitive, $keywordhashref) = $ctstate{apio}->keywords();
    my $tokennl = 0;
    if ($token =~ /^[\r\n]/o) { $tokennl = 1; }
    if (($token eq "\t" || $token =~ /^ +$/) && (!$ctstate{keep_whitespace})) { $token = " "; }
    if ($ctstate{inQuote} == 3) {
	$keep_all_newlines = 1;
	$ctstate{keep_whitespace} = 1;
    }
    if ($ctstate{keep_whitespace}) {
	$ctstate{prespace} = "";
	$nextprespace = "";
    }
    my $tokenIsKeyword = isKeyword($token, $keywordhashref, $case_sensitive);

print STDERR "TIK: $tokenIsKeyword\n" if ($localDebug);

    # my $ctoken = $self->childpeek($soc, $parseTokens{ilc}, $parseTokens{ilc_b});

    print STDERR "TK $token\n" if ($colorDebug || $rubyDebug || $colorDebug);
    print STDERR "IRQ is ".$ctstate{inRubyQuote}."\n" if ($rubyDebug || $colorDebug);
    print STDERR "IQ is ".$ctstate{inQuote}."\n" if ($rubyDebug || $colorDebug);

    my $ctoken = $self->childpeeknc($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b});
    my $ntoken = $self->nextpeek();
    my $ntokennc = $self->nextpeeknc($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b});
    my $nntokennc = $self->nextnextpeeknc($parseTokens{soc}, $parseTokens{ilc}, $parseTokens{ilc_b});
    my $tokenType = undef;
    my $drop = 0;
    my $firstCommentToken = 0;
    my $leavingComment = 0;
    my $hidden = ($self->hidden() && !$ctstate{xmlmode});

    my $isTypeStar = 0;

    my $begintr = "";
    my $endtr = "";
    my $newtd = "";
    if (!$ctstate{xmlmode} && $HeaderDoc::align_columns) {
	$begintr = "<tr><td nowrap=\"nowrap\"><nowrap>";
	$endtr = "</nowrap></td></tr>";
	$newtd = "</nowrap></td><td nowrap=\"nowrap\"><nowrap>";
    }

    if ($ntoken eq "::" && $ctstate{lang} eq "perl") {
	$ctstate{tokenAccum} .= $token.$ntoken;
	$ctstate{skipToken} = 2;
	# print STDERR "SKIPTOKEN -> 2 ($token $ntoken)\n";
    } elsif ($ctstate{tokenAccum} && (!$ctstate{skipToken})) {
	# print STDERR "ACCUM ($token prepended with ".$ctstate{tokenAccum}.")\n";
	$token = $ctstate{tokenAccum}.$token;
	$ctstate{tokenAccum} = "";
    }

    if ($treeDebug || $localDebug || $codePathDebug) {
    	my $tokenname = $token;
    	if ($token eq "\n") { $tokenname = "[NEWLINE]"; }
    	elsif ($token eq "\r") { $tokenname = "[CARRIAGE RETURN]"; }

    	my $ntokenname = $ntoken;
    	if ($ntoken eq "\n") { $ntokenname = "[NEWLINE]"; }
    	elsif ($ntoken eq "\r") { $ntokenname = "[CARRIAGE RETURN]"; }

    	my $lastnstokenname = $ctstate{lastnstoken};
    	if ($ctstate{lastnstoken} eq "\n") { $lastnstokenname = "[NEWLINE]"; }
    	elsif ($ctstate{lastnstoken} eq "\r") { $lastnstokenname = "[CARRIAGE RETURN]"; }

	print STDERR "TOKEN: $tokenname NTOKEN: $ntokenname LASTNSTOKEN: $lastnstokenname IC: ".$ctstate{inComment}."\n" if ($treeDebug || $localDebug || $codePathDebug);
    }
    print STDERR "OCC: ".$ctstate{inObjCMethod}."\n" if ($colorDebug || $localDebug);
    print STDERR "HIDDEN: $hidden\n" if ($localDebug);

    # last one in each chain prior to a "," or at end of chain is "var"
    # or "parm" (functions)
    print STDERR "TK $token NT $ntoken NTNC $ntokennc NNTNC $nntokennc LB: ".$ctstate{lastBrace}." PS: ".length($ctstate{prespace})."\n" if ($colorDebug);

    my $nospaceafter = 0;
    my $nextbreakable = 0;
    if ($ctstate{breakable} == 2) {
	$ctstate{breakable} = 0;
	$nextbreakable = 1;
    } elsif ($ctstate{breakable} == 3) {
	$mustbreak = 1;
	$ctstate{breakable} = 1;
	$nextbreakable = 0;
    }

    print STDERR "POST_CHECK MUSTBREAK: $mustbreak BREAKABLE: ".$ctstate{breakable}." NEXTBREAKABLE: $nextbreakable\n" if ($localDebug);

    if (($ctstate{lang} eq "C" || $ctstate{lang} eq "Csource") && $token eq $parseTokens{enumname}) {
	my $curname = $ctstate{apio}->name();
	print STDERR "NAME: $curname\n" if ($localDebug);
	print STDERR "NOW ENUM\n" if ($localDebug);
	$ctstate{inEnum} = 1;
    }

    if ($parseTokens{propname} && $token eq $parseTokens{propname}) {
	$ctstate{inObjCProperty} = 1;
    }

    ## Enable for certain testing.
    ## if ($self->parserState()) {
	## # reset for new declaration.
	## $ctstate{lastBrace} = "";
    ## }

    if ($ctstate{inObjCMethod} && $token =~ /^[+-]/o && ($ctstate{lastBrace} eq "")) {
	$ctstate{lastBrace} = $token;
    }

    my $MIG = 0;
    if ($ctstate{lang} eq "C" && $ctstate{sublang} eq "MIG") { $MIG = 1; }

    my $splitre = "";
    if ($ctstate{type} =~ /^(typedef|struct|record|union)/o) {
		$splitre = ";";
    } elsif ($ctstate{type} =~ /^(enum|funcptr)/o) {
		$splitre = ",";
    } elsif ($ctstate{lastBrace} eq "(") {
		$splitre = ",";
		if ($MIG) { $splitre = ";"; }
    } elsif ($ctstate{lastBrace} eq $parseTokens{lbrace}) {
		if ($ctstate{lang} eq "tcl") {
			$splitre = '\s';
		} elsif ($ctstate{inEnum}) {
			$splitre = ",";
		} else {
			$splitre = ";";
		}
    } elsif (($ctstate{lastBrace} eq $parseTokens{structname}) && $parseTokens{structisbrace}) {
		$splitre = ";";
    }
print STDERR "SPLITRE IS $splitre\n" if ($localDebug);
    if ($splitre && ($token =~ $splitre)) { # && ($ntoken !~ /^[\r\n]/o)) {
	print STDERR "WILL SPLIT AFTER \"$token\" AND BEFORE \"$ntoken\".\n" if ($localDebug);
	$nextbreakable = 3;
    }

print STDERR "SOC: \"$parseTokens{soc}\"\nEOC: \"$parseTokens{eoc}\"\nILC: \"$parseTokens{ilc}\"\nILC_B: \"$parseTokens{ilc_b}\"\nLBRACE: \"$parseTokens{lbrace}\"\nRBRACE: \"$parseTokens{rbrace}\"\nSOPROC: \"$parseTokens{soprocedure}\"\nSOFUNC: \"$parseTokens{sofunction}\"\nVAR: \"$parseTokens{varname}\"\nSTRUCTNAME: \"$parseTokens{structname}\"\nTYPEDEFNAME: \"$parseTokens{typedefname}\"\n" if ($tokenDebug);

print STDERR "inQuote: ".$ctstate{inQuote}."\noldInQuote: $oldInQuote\ninComment: ".$ctstate{inComment}."\ninMacro: ".$ctstate{inMacro}."\ninEnum: ".$ctstate{inEnum}."\n" if ($localDebug);
print STDERR "oldInMacro: $oldInMacro\noldInComment: $oldInComment\n" if ($localDebug);

    # print STDERR "TOKEN: $token\n" if ($localDebug);
    print STDERR "TOKEN: $token ASINNAME: ".$ctstate{ASinName}."\n" if ($localDebug);

    if ($ctstate{inEnum}) {
	# If we see this, anything nested below here is clearly not a union.
	if (casecmp($token, $parseTokens{unionname}, $case_sensitive)) { $ctstate{inEnum} = 0; };
	if (casecmp($token, $parseTokens{structname}, $case_sensitive)) { $ctstate{inEnum} = 0; };
	if (casecmp($token, $parseTokens{typedefname}, $case_sensitive)) { $ctstate{inEnum} = 0; };
    }

    my $nonword = 0; my $pascal = 0; my $ruby = 0;
    if ($token =~ /\W/) {
	$nonword = 1;
	if ($ctstate{ASinName} && (!$ctstate{inQuote}) && (!$ctstate{inComment}) && ($token !~ /\s/)) {
		$ctstate{ASinName} = -1;
		print STDERR "ASinName -> -1 [1]\n" if ($localDebug);
	}
    }
    if ($ctstate{lang} eq "pascal") { $pascal = 1; }
    if ($ctstate{lang} eq "ruby") { $ruby = 1; }

    if ($ctstate{lang} eq "applescript" && $token =~ /(given|of|in)/) {
	print STDERR "ASinName -> 3 [0a]\n" if ($localDebug);
	$ctstate{ASinName} = 3;
	print STDERR "AS lastBrace -> \"(\"\n" if ($localDebug);
	$ctstate{lastBrace} = "(";
    } elsif ($parseTokens{labelregexp} && $token =~ $parseTokens{labelregexp}) {
	print STDERR "ASinName -> 3 [0b]\n" if ($localDebug);
	$ctstate{ASinName} = 3;
	print STDERR "AS lastBrace -> \"(\"\n" if ($localDebug);
	$ctstate{lastBrace} = "(";
    }

    my $untypedLanguage = 0;
    if ($ctstate{sublang} eq "javascript" ||
	$ctstate{lang} eq "php" || $ctstate{lang} eq "perl" || $ctstate{lang} eq "python" ||
	$ctstate{lang} eq "shell" || $ctstate{lang} eq "applescript") {
		$untypedLanguage = 1;
    } elsif ($ruby || $ctstate{lang} eq "tcl") {
		$untypedLanguage = 2;
    }


    if ($ctstate{skipToken}) {
	$ctstate{skipToken}--;
	$drop = 1;
	$token = "";
    }
    if ($ctstate{lang} eq "C" || $ctstate{lang} eq "java" || $pascal || $ctstate{sublang} eq "javascript" ||
		$ctstate{lang} eq "php" || $ctstate{lang} eq "perl" || $ruby || $ctstate{lang} eq "python" ||
		$ctstate{lang} eq "Csource" || $ctstate{lang} eq "shell" || $ctstate{lang} eq "applescript" ||
	        $ctstate{lang} eq "tcl") {
	if ($ctstate{inQuote} == 1) {
		print STDERR "    STRING\n" if ($localDebug || $codePathDebug);
		$tokenType = "string";
	} elsif ($ctstate{inQuote} == 2) {
		print STDERR "    CHAR\n" if ($localDebug || $codePathDebug);
		$tokenType = "char";
	} elsif ($ctstate{inQuote} == 3 && $ctstate{inRubyQuote} != 2 && !$tokennl) {
		# print STDERR "preserve newlines.  TOKEN: \"$token\"\n" if ($rubyDebug || $colorDebug);
		$tokenType = "string";
	} elsif ($nonword && $token eq $parseTokens{soc} && $parseTokens{soc} ne "") {
	    if (!$hidden) {
		$tokenType = "comment";
		if ($ctstate{ASinName} == -1) {
			$ctstate{ASinName} = 1;
			print STDERR "ASinName -> 1 [2]\n" if ($localDebug);
		}
		print STDERR "    COMMENT [1]\n" if ($localDebug || $codePathDebug);
		if (!$ctstate{inComment}) {
			$ctstate{inComment} = 1;
			$firstCommentToken = 1;
			if ($ctstate{xmlmode}) {
				$string .= "<declaration_comment>";
			} else {
				$string .= "<span class=\"comment\">";
			}
		} else {
			print STDERR "    nested comment\n" if ($localDebug || $codePathDebug);
		}
	    } else {
		print STDERR "    COMMENT [1a]: HIDDEN\n" if ($localDebug || $codePathDebug);
	    }
	} elsif ($nonword && ((($token eq $parseTokens{ilc}) && ($parseTokens{ilc} ne "")) || (($token eq $parseTokens{ilc_b}) && ($parseTokens{ilc_b} ne "")))) {
	    if (!$hidden) {
		print STDERR "    ILCOMMENT [1]\n" if ($localDebug || $codePathDebug);
		$tokenType = "comment";

		if (!$ctstate{inComment}) {
			if ($ctstate{ASinName} == -1) {
				$ctstate{ASinName} = 1;
				print STDERR "ASinName -> 1 [3]\n" if ($localDebug);
			}
			print STDERR "    REALILCOMMENT\n" if ($localDebug || $codePathDebug);
			$ctstate{inComment} = 2;
			$firstCommentToken = 1;
			if ($ctstate{xmlmode}) {
				$string .= "<declaration_comment>";
			} else {
				$string .= "<span class=\"comment\">";
			}
		} else {
			print STDERR "    nested comment\n" if ($localDebug || $codePathDebug);
		}
	    } else {
		print STDERR "    ILCOMMENT [1a]: HIDDEN\n" if ($localDebug || $codePathDebug);
	    }
	} elsif ($nonword && $token eq $parseTokens{eoc} && $parseTokens{eoc} ne "") {
		print STDERR "    EOCOMMENT [1]\n" if ($localDebug || $codePathDebug);
		$tokenType = "comment";
		if ($ctstate{xmlmode}) {
			$tailstring .= "</declaration_comment>";
		} else {
			$tailstring = "</span>";
		}
		$leavingComment = 1;
		$ctstate{inComment} = 0;
	} elsif ($tokennl && ($ntoken !~ /^[\r\n]/o || $ctstate{keep_whitespace} || $keep_all_newlines)) {
		if ($ctstate{ASinName}) {
			$ctstate{ASinName} = 0;
			print STDERR "ASinName -> 0 [4]\n" if ($localDebug);
		}
		if ($ctstate{inComment} == 2) {
			print STDERR "    EOL INCOMMENT: END ILCOMMENT [1]\n" if ($localDebug || $codePathDebug);
			$tokenType = "comment";
			if ($ctstate{xmlmode}) {
				$string .= "</declaration_comment>";
			} else {
				$string .= "</span>";
			}
			$ctstate{inComment} = 0;
			$ctstate{newlen} = 0;
			$mustbreak = 1;
			# $token = "";
			$drop = 1;
		} elsif ($ctstate{inMacro} || $keep_all_newlines) {
			print STDERR "    EOL INMACRO\n" if ($localDebug || $codePathDebug);
			$mustbreak = 1;
			$ctstate{newlen} = 0;
		} elsif ($ctstate{inComment}) {
			print STDERR "    EOL INCOMMENT\n" if ($localDebug || $codePathDebug);
			$mustbreak = 1;
			$ctstate{newlen} = 0;
			# $token = "";
			$drop = 1;
		}
		$ctstate{breakable} = 0;
		$nextbreakable = 0;
		# $nextprespace = nspaces(4 * $ctstate{depth});
		$ctstate{newlen} = 0;
	# } elsif ($ntoken =~ /^[\r\n]/o) {
		# print STDERR "    NEXT TOKEN IS NLCR\n" if ($localDebug || $codePathDebug);
		# $ctstate{breakable} = 0;
		# $nextbreakable = 0;
	} elsif ($ctstate{inComment}) {
		print STDERR "    COMMENT [2:".$ctstate{inComment}."]\n" if ($localDebug || $codePathDebug);
		$tokenType = "comment";
		if ($ctstate{inComment} == 1) {
			if (($token =~ /^\s/o && !$tokennl && $ntoken !~ /^\s/o) && (!$ctstate{keep_whitespace})) {
				# Only allow wrapping of multi-line comments.
				# Don't blow in extra newlines at existing ones.
				$ctstate{breakable} = 1;
			}
		}
	} elsif ($ctstate{inMacro}) {
		print STDERR "    MACRO [IN]\n" if ($localDebug || $codePathDebug);
		$tokenType = "preprocessor";
	} elsif ($token eq "=") {
		print STDERR "    EQUALS\n" if ($localDebug || $codePathDebug);
		$nextbreakable = 1;
		if ($ctstate{type} eq "pastd") {
			$ctstate{type} = "";
			print STDERR "    END OF VAR\n" if ($localDebug || $codePathDebug);
		}
		if ($pascal) { $ctstate{seenEquals} = 1; }
	} elsif ($token eq "-") {
		print STDERR "    MINUS\n" if ($localDebug || $codePathDebug);
		if ($ntoken =~ /^\d/o) {
			$tokenType = "number";
			print STDERR "    NUMBER [1]\n" if ($localDebug || $codePathDebug);
		} else {
			print STDERR "    TEXT [1]\n" if ($localDebug || $codePathDebug);
			$tokenType = "";
		}
	} elsif ($token =~ /^\d+$/o || $token =~ /^0x[\dabcdef]+$/io) {
		$tokenType = "number";
		$ctstate{type} = "hexnumber";
		print STDERR "    \nNUMBER [2]: $token\n" if ($localDebug || $codePathDebug);
	} elsif (!$nonword && (casecmp($token, $parseTokens{sofunction}, $case_sensitive) || casecmp($token, $parseTokens{soprocedure}, $case_sensitive) || casecmp($token, $parseTokens{soconstructor}, $case_sensitive))) {
		$tokenType = "keyword";
		$ctstate{lastKeyword} = $token;
		print STDERR "    SOFUNC/SOPROC\n" if ($localDebug || $codePathDebug);
		$ctstate{type} = "funcproc";
		$ctstate{lastBrace} = "(";
		$oldLastBrace = "(";
		if ($ctstate{lang} eq "applescript") {
			$ctstate{ASinName} = 1;
			print STDERR "ASinName -> 1 [5]\n" if ($localDebug);
		}
	} elsif (!$nonword && $ctstate{type} eq "funcproc") {
		if ($token =~ /^\;/o) {
			$ctstate{type} = "";
			$nextbreakable = 3;
		}
		print STDERR "    FUNC/PROC NAME\n" if ($localDebug || $codePathDebug);
		$tokenType = "function";
	} elsif (!$nonword && casecmp($token, $parseTokens{constname}, $case_sensitive) && $ctstate{lang} eq "pascal") {
		$tokenType = "keyword";
		print STDERR "    VAR\n" if ($localDebug || $codePathDebug);
		$ctstate{type} = "pasvar";
	} elsif (!$nonword && casecmp($token, $parseTokens{varname}, $case_sensitive)) {
		$tokenType = "keyword";
		print STDERR "    VAR\n" if ($localDebug || $codePathDebug);
		$ctstate{type} = "pasvar";
	} elsif ($nonword && ($ctstate{type} eq "pasvar" || $ctstate{type} eq "pastd") &&
		 ($token =~ /^[\;\:\=]/o)) {
			# NOTE: '=' is handled elsewhere,
			# but it is included above for clarity.
			$ctstate{type} = "";
			print STDERR "    END OF VAR\n" if ($localDebug || $codePathDebug);
	} elsif ($ctstate{type} eq "pasvar" || $ctstate{type} eq "pastd") {
		print STDERR "    VAR NAME\n" if ($localDebug || $codePathDebug);
		$tokenType = "var";
	} elsif (!$nonword && ($pascal) && casecmp($token, $parseTokens{typedefname}, $case_sensitive)) {
		# TYPE: This is the start of a pascal type
		print STDERR "    TYPE\n" if ($localDebug || $codePathDebug);
		$tokenType = "keyword";
		$ctstate{type} = "pastd";
	} elsif (!$nonword && ($pascal) && casecmp($token, $parseTokens{structname}, $case_sensitive)) {
		# RECORD: This is the start of a pascal record
		print STDERR "    RECORD/STRUCT\n" if ($localDebug || $codePathDebug);
		$ctstate{lastBrace} = $token;
		$tokenType = "keyword";
		$ctstate{type} = "pasrec";
	} elsif (!$nonword && $tokenIsKeyword) {
		# This is a keyword in any language.
		$tokenType = "keyword";

		if ($ctstate{ASinName} != 1 && $ctstate{ASinName} != 3) {
			$ctstate{ASinName} = 0;
			print STDERR "ASinName -> 0 [6]\n" if ($localDebug);
		}

		if ($tokenIsKeyword == 2) {
			# This formatting change applies only to children of this node.
			$ctstate{inAttribute} = 1;
		} elsif ($tokenIsKeyword == 5) {
			# This formatting change applies only to children of this node.
			$ctstate{inRaises} = 1;
		} elsif ($tokenIsKeyword == 6) {
			$ctstate{inTypeOf} = 1;
		}


		# NOTE: If anybody ever wants "class" to show up colored
		# as a keyword within a template, the next block should be
		# made conditional on a command-line option.  Personally,
		# I find it distracting, hence the addition of these lines.

		if ($ctstate{lastBrace} eq $parseTokens{sotemplate} && $parseTokens{sotemplate} ne "") {
			$tokenType = "template";
		}

		print STDERR "    KEYWORD\n" if ($localDebug || $codePathDebug);
		# $ctstate{inMacro} = $self->isMacro($token, $ctstate{lang}, $ctstate{sublang});
		# We could have keywords in a macro, so don't set this
		# to zero.  It will get zeroed when we pop a level
		# anyway.  Just set it to 1 if needed.
		if ($case_sensitive) {
			if ($macroList{$token}) {
				print STDERR "    IN MACRO\n" if ($localDebug || $codePathDebug);
				$ctstate{inMacro} = 1;
			}
		} else {
		    foreach my $cmpToken (keys %macroList) {
			if (casecmp($token, $cmpToken, $case_sensitive)) {
				$ctstate{inMacro} = 1;
			}
		    }
		}
		print STDERR "    TOKEN IS $token, IM is now ".$ctstate{inMacro}."\n" if ($localDebug || $codePathDebug);
		if (casecmp($token, $parseTokens{rbrace}, $case_sensitive)) {
			print STDERR "PS: ".length($ctstate{prespace})." -> " if ($psDebug);
			# $ctstate{prespace} = nspaces(4 * ($ctstate{depth}-1));
			$mustbreak = 2;
			print STDERR length($ctstate{prespace})."\n" if ($psDebug);
		}
	} elsif (!$ctstate{inQuote} && !$ctstate{inComment} && isKnownMacroToken($token, \%macroList, $case_sensitive)) {
		# This is a preprocessor directive
				print STDERR "    IN MACRO\n" if ($localDebug || $codePathDebug);
				$ctstate{inMacro} = 1;
	} elsif (($token eq "*") && ($ctstate{depth} == 1) && ($ctstate{lastTokenType} eq "type" || $ctstate{lastTokenType} eq "star")) {
		print STDERR "    ASTERISK\n" if ($localDebug || $codePathDebug);
		# spacing fix for '*' characters
		if (!$ctstate{spaceSinceLastToken} && (!$ctstate{keep_whitespace}) && $ctstate{lastTokenType} ne "star") {
			if ($ctstate{prespace} == "") { $ctstate{prespace} = " "; }
		}
		$tokenType = "type";
		$isTypeStar = 1;
		$nospaceafter = 1;
	} elsif ($ntokennc eq ":" && $ctstate{inObjCMethod}) {
		# Detecting of objective-C method separators
		print STDERR "    COLON (FUNCTION [1])\n" if ($localDebug || $codePathDebug);
		$tokenType = "function";
	} elsif ($token eq ":" && $ctstate{inObjCMethod}) {
		# Detecting of objective-C method separators
		print STDERR "    COLON (FUNCTION [2])\n" if ($localDebug || $codePathDebug);
		$tokenType = "function";
	} elsif ($token eq ":" && $ctoken) {
		# Don't indent Objective-C method parts so far.
		$ctstate{depth} = $ctstate{depth} - 1; # We'll change it back before the next token.
	} elsif ($ntokennc eq "(" && !$ctstate{seenEquals} && !$ctstate{inAttribute} && !$ctstate{inRaises} && !$ctstate{inTypeOf}) {
		# Upcoming parenthesis handling
		$tokenType = "function";
		print STDERR "    LPAREN (FUNCTION [3])\n" if ($localDebug || $codePathDebug);
		if ($nntokennc eq "(" && !$untypedLanguage) {
			$tokenType = "type";
			$ctstate{type} = "funcptr";
		}
		if ($ctstate{inObjCMethod}) {
			$tokenType = ""; # shouldn't happen 
		}
		if ($token eq "(") { $dropFP = 1; }
	} elsif ((!$untypedLanguage) && $ntokennc eq $parseTokens{lbrace} && $parseTokens{lbrace} ne "") {
		# Upcoming brace handling
		$tokenType = "type";
		print STDERR "    LBRACE (TYPE [1])\n" if ($localDebug || $codePathDebug);
	} elsif (($ctstate{inAttribute} || $ctstate{inRaises} || $ctstate{inTypeOf}) && $token eq "(") {
		print STDERR "    LPAREN (ATTRIBUTE)\n" if ($localDebug || $codePathDebug);
		# Parenthesis handling for attributes
		$nextbreakable = 0;
	} elsif ($token eq "(") {
		print STDERR "    LPAREN (GENERAL)\n" if ($localDebug || $codePathDebug);
		# Parenthesis handling
		$ctstate{type} = "";
		if ($ctstate{inObjCProperty}) {
			$nextbreakable = 0;
			$oldLastBrace = "";
		} elsif ($ctstate{inObjCMethod} && $ctstate{lastBrace} =~ /^[+-]/o) {
			$nextbreakable = 0;
			$oldLastBrace = "";
		} elsif ($ctoken ne ")") {
			$nextbreakable = 3;
		}
		$ctstate{lastBrace} = $token;
		#if (!$ctstate{depth}) {
			#$nospaceafter = 2;
		#}
	} elsif ($token eq $parseTokens{sotemplate} && $parseTokens{sotemplate} ne "") {
		print STDERR "    TEMPLATE\n" if ($localDebug || $codePathDebug);
		# This is the word "template" or similar.
		$ctstate{lastBrace} = $token;
		$nextbreakable = 0;
		$ctstate{breakable} = 0;
	} elsif (casecmp($token, $parseTokens{lbrace}, $case_sensitive)) {
		print STDERR "    LBRACE (GENERAL)\n" if ($localDebug || $codePathDebug);
		# Brace handling.
		$ctstate{type} = "";
		$ctstate{lastBrace} = $token;
		$nextbreakable = 3;
		if (!casecmp($ctoken, $parseTokens{rbrace}, $case_sensitive)) {
			$nextbreakable = 3;
		}
	} elsif (($ctstate{lang} eq "python") && ($token eq "\"\"\"")) {
		if ($ctstate{inQuote}) { $ctstate{inQuote} = 0; }
		else { $ctstate{inQuote} = 3; }
	} elsif ($ruby && ($token eq "%{" || $token eq "%Q{")) {
		$ctstate{inQuote} = 3; $ctstate{inRubyQuote} = 1;
	} elsif ($ruby && ($token eq "%/")) {
		$ctstate{inQuote} = 3; $ctstate{inRubyQuote} = 1;
	} elsif ($ruby && ($token eq "<<")) {
		$ctstate{inQuote} = 3;
		$ctstate{inRubyQuote} = 2; # quote coloring starts after next token.
		print STDERR "IRQ -> 2\n" if ($rubyDebug || $colorDebug);
	} elsif ($ruby && ($ctstate{inRubyQuote} == 2)) {
		$ctstate{inRubyQuote} = 1;
		print STDERR "IRQ 2 -> 1\n" if ($rubyDebug || $colorDebug);
	} elsif ($token =~ /^\"/o && !$ctstate{inRubyQuote}) {
		print STDERR "    DQUOTE\n" if ($localDebug || $codePathDebug);
		# Double quote handling
		$ctstate{inQuote} = 1;
	} elsif ($token =~ /^\'/o && !$ctstate{inRubyQuote}) {
		print STDERR "    SQUOTE\n" if ($localDebug || $codePathDebug);
		# Single quote handling
		$ctstate{inQuote} = 2;
	} elsif ($ntokennc =~ /^(\)|\,|\;)/o || casecmp($ntokennc, $parseTokens{rbrace}, $case_sensitive)) {
		# Detection of the last token before the end of a part of a declaration.
		# last token
		print STDERR "    LASTTOKEN\n" if ($localDebug || $codePathDebug);
		if ($nextbreakable != 3) {
			$nextbreakable = 2;
		}
		if ($ctstate{lastBrace} eq $parseTokens{sotemplate} && $parseTokens{sotemplate} ne "") {
			$nextbreakable = 0;
		}
		if ($ctstate{lastBrace} eq "(" || ($ctstate{lastBrace} eq "{" && $parseTokens{parmswithcurlybraces})) {
			if ($MIG || $pascal) {
				$tokenType = "type";
				print STDERR "    TYPE [2]\n" if ($localDebug || $codePathDebug);
			} else {
				$tokenType = "param";
				print STDERR "    PARAM [1]\n" if ($localDebug || $codePathDebug);
			}
		} elsif ($ctstate{lastBrace} eq $parseTokens{sotemplate} && $parseTokens{sotemplate} ne "") {
			print STDERR "    TEMPLATE[1]\n" if ($localDebug || $codePathDebug);
			$tokenType = "template";
		} elsif ($ctstate{type} eq "funcptr") {
			$tokenType = "function";
			print STDERR "    FUNCTION [1]\n" if ($localDebug || $codePathDebug);
			$ctstate{breakable} = 0;
			$nextbreakable = 0;
		} else {
			if ($MIG || $pascal) {
				$tokenType = "type";
				print STDERR "    TYPE [2a]\n" if ($localDebug || $codePathDebug);
			} else {
				$tokenType = "var";
				print STDERR "    VAR [1] (LB: ".$ctstate{lastBrace}.")\n" if ($localDebug || $codePathDebug);
			}
		}
		if (casecmp($ntokennc, $parseTokens{rbrace}, $case_sensitive) && $ctstate{type} eq "pasrec") {
			$ctstate{type} = "";
		}
		if ($ntokennc eq ")") {
			$nextbreakable = 0;
			if ((!$untypedLanguage) && ($ctstate{inObjCMethod} || ($token eq "*"))) {
				print STDERR "    TYPE [3]\n" if ($localDebug || $codePathDebug);
				$tokenType = "type";
			}
		}
	} elsif ($ctstate{prespace} ne "" && ($token =~ /^\)/o || casecmp($token, $parseTokens{rbrace}, $case_sensitive))) {
		print STDERR "PS: ".length($ctstate{prespace})." -> " if ($psDebug);
		if (!$ctstate{keep_whitespace}) { $ctstate{prespace} = nspaces(4 * ($ctstate{depth}-1)); }
		print STDERR length($ctstate{prespace})."\n" if ($psDebug);
		$mustbreak = 2;
	} elsif (casecmp($token, $parseTokens{rbrace}, $case_sensitive)) {
		if (!$ctstate{keep_whitespace}) { $ctstate{prespace} = nspaces(4 * ($ctstate{depth}-1)); }
		print STDERR length($ctstate{prespace})."\n" if ($psDebug);
		$mustbreak = 2;
	} else {
		if ($ctstate{inObjCMethod}) {
			if ($ctstate{lastBrace} eq "(") {
				print STDERR "    TYPE [4]\n" if ($localDebug || $codePathDebug);
				$tokenType = "type";
			} else { 
				print STDERR "    PARAM [2]\n" if ($localDebug || $codePathDebug);
				$tokenType = "param";
			}
		} elsif ($MIG || $pascal) {
			if ($ctstate{lastBrace} eq "(") {
				print STDERR "    PARAM [3]\n" if ($localDebug || $codePathDebug);
				$tokenType = "param";
			}
		} else {
			if ($ctstate{lastBrace} eq $parseTokens{sotemplate} && ($parseTokens{sotemplate} ne "")) {
				print STDERR "    TEMPLATE [5]\n" if ($localDebug || $codePathDebug);
				$tokenType = "template";
			} elsif ($ctstate{inEnum}) {
				# Constants are a special case of variable
				print STDERR "    TYPE [5]\n" if ($localDebug || $codePathDebug);
				$tokenType = "var";
			} elsif ($untypedLanguage && $ctstate{lastBrace}) {
				print STDERR "    UNTYPED: PARAM [5]\n" if ($localDebug || $codePathDebug);
				$tokenType = "param";
			} elsif (($untypedLanguage == 1) ||
			         ($untypedLanguage == 2 && $ctstate{lastnstoken} !~ $parseTokens{classregexp})) {
				print STDERR "    UNTYPED: VAR [5]\n" if ($localDebug || $codePathDebug);
				$tokenType = "var";
			} else {
				print STDERR "    TYPE [5]\n" if ($localDebug || $codePathDebug);
				$tokenType = "type";
			}
		}
	}
    } else {
	my $fullpath = $ctstate{apio}->fullpath;
	my $linenum = $ctstate{apio}->linenum;
	warn "$fullpath:$linenum: warning: Unknown language ".$ctstate{lang}.". Not coloring. Please file a bug.\n";
    }
    if ($ctstate{inRaises} && $tokenType && ($tokenType ne "keyword")) {
	$tokenType = "type";
    }
    if ($ctstate{inTypeOf} && $tokenType && ($tokenType ne "keyword")) {
	$tokenType = "param";
    }

    if ((!$hidden) && $self->isAvailabilityMacro()) {
	if ($ctstate{xmlmode}) {
		$string .= "<declaration_availabilitymacro>";
		$tailstring = "</declaration_availabilitymacro>".$tailstring;
	} else {
		$string .= "<span class=\"availabilitymacro\">";
		$tailstring = "</span>".$tailstring;
	}
    }

# print STDERR "TOKEN: $token TYPE: $tokenType\n";

    if ($ctstate{ASinName} == 1) {
	$ctstate{ASinName} = 2;
	print STDERR "ASinName -> 2 [end]\n" if ($localDebug);
    } elsif ($ctstate{ASinName} == 2) {
	$tokenType = "function";
    } elsif ($ctstate{ASinName} == 3) {
	$ctstate{ASinName} = 4;
	print STDERR "ASinName -> 4 [end]\n" if ($localDebug);
    } elsif ($ctstate{ASinName} == 4) {
	$tokenType = "param";
    } elsif ($ctstate{ASinName} == -1) {
	$ctstate{ASinName} = 0;
	print STDERR "ASinName -> 0 [end]\n" if ($localDebug);
    }
    if ($hidden) {
	$tokenType = "ignore";
	if ($mustbreak) {
		$nextbreakable = 3;
	} else {
		$nextbreakable = 0;
	}
	$mustbreak = 0;
	$ctstate{breakable} = 0;
    }
    if (($ntoken =~ /[,;]/) && ($token =~ /[ \t]/) && !$ctstate{inComment} && !$ctstate{inMacro} && !$ctstate{inQuote}) {
	# print STDERR "DROP\n";
	$hidden = 1;
	$tokenType = "ignore";
	$nextbreakable = 0;
	$mustbreak = 0;
	$ctstate{breakable} = 0;
    }
    if ($MIG || $pascal) {
	if ($ctstate{lastnstoken} =~ /:/ && $ctstate{lastTokenType} eq "var") {
		$string .= $newtd;
	}
    } else {
	if (($ctstate{lastTokenType} eq "type") && !$hidden && ($token =~ /[\w\*]/) &&
		($tokenType eq "var" || $tokenType eq "param" ||
		 $tokenType eq "function" || $token eq "*") && ($ctstate{lastnstoken} =~ /\w/)) {
			$string .= $newtd;
	}
    }

    if (((($parseTokens{ilc} ne "") && ($ntoken eq $parseTokens{ilc})) || (($parseTokens{ilc_b} ne "") && ($ntoken eq $parseTokens{ilc_b}))) && !$ctstate{inComment}) {
	$ctstate{breakable} = 0; $nextbreakable = 0;
    } elsif (($parseTokens{soc} ne "") && $ntoken eq $parseTokens{soc} && !$ctstate{inComment}) {
	$ctstate{breakable} = 0; $nextbreakable = 0;
    }
print STDERR "NB: $nextbreakable\n" if ($localDebug);
print STDERR "TESTTYPE: $tokenType\n" if ($localDebug);

    if ($ctstate{inObjCMethod}) {
	$nextbreakable = 0;
	$ctstate{breakable} = 0;
	$mustbreak = 0;
	if ($ntoken eq ":" && $tokenType eq "function") {
		$ctstate{breakable} = 1;
	}
    }

    if ($ctstate{type} eq "pasrec" && $tokenType eq "") {
	print STDERR "PASREC VAR\n" if ($localDebug);
	$tokenType = "var";
    }
    else { print STDERR "TYPE: ".$ctstate{type}." TT: ".$tokenType."\n" if ($localDebug); }
    print STDERR "IM: ".$ctstate{inMacro}."\n" if ($localDebug);

    if (!$ctstate{inComment} && $token =~ /^\s/o && !$tokennl && ($mustbreak || !$ctstate{newlen}) && (!$ctstate{keep_whitespace})) {
	print STDERR "CASEA\n" if ($localDebug);
	print STDERR "NL: ".$ctstate{newlen}." TOK: \"$token\" PS: \"".$ctstate{prespace}."\" NPS: \"$nextprespace\"\n" if ($localDebug);
	print STDERR "dropping leading white space\n" if ($localDebug);
	$drop = 1;
    } elsif (!$ctstate{inComment} && $tokennl && (!$ctstate{keep_whitespace})) {
	print STDERR "CASEB\n" if ($localDebug);
	if ($ctstate{lastnstoken} ne $parseTokens{eoc}) {
		# Insert a space instead.

		print STDERR "dropping newline\n" if ($localDebug);
		$drop = 1;
		$string .= " ";
	} else {
		$mustbreak = 1;
	}
    } elsif ($ctstate{inComment} || $token =~ /^\s/o || ($token =~ /^\W/o && $token ne "*") || !$tokenType) {
	print STDERR "CASEC\n" if ($localDebug);
	my $macroTail = "";
	$escapetoken = $ctstate{apio}->textToXML($token);
	print STDERR "OLDPS: \"".$ctstate{prespace}."\" ET=\"$escapetoken\" DROP=$drop\n" if ($localDebug);
	if ($ctstate{inComment} && $ctstate{prespace} ne "" && !$hidden) {
		if ($ctstate{xmlmode}) {
			$string .= "</declaration_comment>\n".$ctstate{prespace}."<declaration_comment>";
		} else {
			$string .= "</span>\n$endtr".$ctstate{prespace}."$begintr<span class=\"comment\">";
		}
	} elsif ($ctstate{inMacro} && $token =~ /^\S/) {
		# Could be the initial keyword, which contains a '#'
		if (!$tokenType) {
			$tokenType = "preprocessor";
		}
		if ($ctstate{xmlmode}) {
			$string .= $ctstate{prespace}."<declaration_$tokenType>";
			$macroTail = "</declaration_$tokenType>";
		} else {
			$string .= $ctstate{prespace}."<span class=\"$tokenType\">";
			$macroTail = "</span>";
		}
	} elsif (!$hidden) {
		$string .= $ctstate{prespace};
	}
	if ($drop) { $escapetoken = ""; }
	if ($tokenType eq "ignore") {
	    if (!$HeaderDoc::dumb_as_dirt) {
		# Drop token.
		print STDERR "HD: DROPPING IGNORED TOKEN $escapetoken\n" if ($dropDebug);
		$escapetoken = "";
	    } else {
		print STDERR "HD BASIC: KEEPING IGNORED TOKEN $escapetoken\n" if ($dropDebug);
	    }
	}

	$string .= "$escapetoken$macroTail";
	print STDERR "comment: $token\n" if ($localDebug);
    } else {
	print STDERR "CASED\n" if ($localDebug);

	# my $add_link_requests = $HeaderDoc::add_link_requests;
	$escapetoken = $ctstate{apio}->textToXML($token);

	if (($tokenType ne "") && ($token ne "") && token !~ /^\s/o) {
		my $fontToken = "";
		if ($ctstate{xmlmode}) {
			$fontToken = "<declaration_$tokenType>$escapetoken</declaration_$tokenType>";
		} else {
		    if ($tokenType ne "ignore") {
			$fontToken = "<span class=\"$tokenType\">$escapetoken</span>";
		    } elsif (!$HeaderDoc::dumb_as_dirt) {
			# Drop token.
			print STDERR "HD: DROPPING IGNORED TOKEN $escapetoken\n" if ($dropDebug);
			$fontToken = "";
		    } else {
			print STDERR "HD BASIC: KEEPING IGNORED TOKEN $escapetoken\n" if ($dropDebug);
			$fontToken = $escapetoken;
		    }
		}
		my $linkTokenType = $tokenType;
		if ($ctstate{inTypeOf} && ($tokenType ne "keyword")) {
			$linkTokenType = "";
		}

		# print STDERR "HERE: \"$token\" TYPE: $tokenType\n";

		# Don't add noisy link requests in XML.
		if ((!$drop) && ($HeaderDoc::add_link_requests && ($tokenType =~ /^(function|var|type|preprocessor)/o || $ctstate{inTypeOf}) && !$ctstate{xmlmode})) {
			my $refToken = $ctstate{apio}->genRefFromDeclaration($ctstate{lastKeyword}, $escapetoken, $fontToken, $linkTokenType);
			# print "TOKEN: $token RT: $refToken\n";
			$string .= $ctstate{prespace}."$refToken";
		} else {
			$string .= $ctstate{prespace}."$fontToken";
		}
	} else {
		$escapetoken = $ctstate{apio}->textToXML($token);
		if ($tokenType eq "ignore") {
		    if (!$HeaderDoc::dumb_as_dirt) {
			# Drop token.
			print STDERR "HD: DROPPING IGNORED TOKEN $escapetoken\n" if ($dropDebug);
			$escapetoken = "";
		    } else {
			print STDERR "HD BASIC: KEEPING IGNORED TOKEN $escapetoken\n" if ($dropDebug);
		    }
		}
		$string .= $ctstate{prespace}."$escapetoken";
	}
	print STDERR "$tokenType: $token\n" if ($localDebug);
    }
    $ctstate{prespace} = $nextprespace;

    if (!$drop) {
	$ctstate{newlen} += length($token);
    }

    print STDERR "NL ".$ctstate{newlen}." MDL $HeaderDoc::maxDecLen BK ".$ctstate{breakable}." IM ".$ctstate{inMacro}."\n" if ($localDebug);
    if ((!$ctstate{keep_whitespace}) && ($mustbreak ||
		(($ctstate{newlen} > $HeaderDoc::maxDecLen) &&
		    $ctstate{breakable} && !$ctstate{inMacro} && !$hidden))) {
	print STDERR "MUSTBREAK CASE\n" if ($localDebug);
	if (($token =~ /^\s/o || $token eq "") && (!$ctstate{keep_whitespace})) {
		$nextprespace = nspaces(4 * ($ctstate{depth}+(1-$mustbreak)));
		print STDERR "PS WILL BE \"$nextprespace\"\n" if ($localDebug);
		$nextbreakable = 3;
	} else {
		print STDERR "NEWLEN: ".$ctstate{newlen}."\n" if ($localDebug);
		$ctstate{newlen} = length($token);
		print STDERR "NEWLEN [2]: ".$ctstate{newlen}."\n" if ($localDebug);
		print STDERR "MB: $mustbreak, DP: ".$ctstate{depth}."\n" if ($localDebug);
		my $ps = nspaces(4 * ($ctstate{depth}+(1-$mustbreak)));
		if (($ctstate{inComment} == 1 && !$firstCommentToken) || $leavingComment) {
		    if ($ctstate{xmlmode}) {
			$string = "</declaration_comment>\n$ps<declaration_comment>$string";
		    } else {
			$string = "</span>$endtr\n$begintr$ps<span class=\"comment\">$string";
		    }
		} else {
			$string = "$endtr\n$begintr$ps$string";
		}
		print STDERR "PS WAS \"$ps\"\n" if ($localDebug);
	}
    }

    if ($token !~ /^\s/o) { $ctstate{lastnstoken} = $token; }

    if ($token !~ /\s/) {
	if ($isTypeStar) {
		$ctstate{lastTokenType} = "star";
	} else {
    		$ctstate{lastTokenType} = $tokenType;
	}
	$ctstate{spaceSinceLastToken} = 0;
    } else {
	$ctstate{spaceSinceLastToken} = 1;
    }


    my $newstring = "";
    my $node = $self->{FIRSTCHILD};
    my $newstringref = undef;
    if ($node && $continue) {
	if ($nospaceafter == 1) { $nospaceafter = 0; }
	print STDERR "BEGIN CHILDREN\n" if ($localDebug || $colorDebug || $treeDebug);
	bless($node, "HeaderDoc::ParseTree");

	my $childctstate = newCTState(\%ctstate, depth => $ctstate{depth} + 1, breakable => $nextbreakable);

	($newstringref, $ctstate{newlen}, $nextbreakable, $ctstate{prespace}, $ctstate{lastnstoken}, $continue, $ctstate{lastTokenType}, $ctstate{spaceSinceLastToken}) = $node->colorTreeSub($childctstate);

	$newstring = ${$newstringref};
	print STDERR "END CHILDREN\n" if ($localDebug || $colorDebug || $treeDebug);
    }
    $string .= $newstring; $newstring = "";
    print STDERR "SET STRING TO $string\n" if ($localDebug);

    if (($ctstate{prespace} ne "")) {
	# if we inherit a need for prespace from a descendant, it means
	# that the descendant ended with a newline.  We don't want to
	# propagate the extra indentation to the next node, though, so
	# we'll regenerate the value of prespace here.
	$ctstate{prespace} = nspaces(4 * $ctstate{depth});
    }
    print STDERR "HMLT: ".$self->{HIDEMACROLASTTOKEN}."\n" if ($localDebug);

    if ($self->{HIDEMACROLASTTOKEN} && $ctstate{drop_pdefine_contents}) {
	$continue = 0;
    }

    $string .= $tailstring;
    $tailstring = "";
    print STDERR "LB ".$ctstate{lastBrace}." -> $oldLastBrace\n" if ($colorDebug || $localDebug);
    $ctstate{lastBrace} = $oldLastBrace;
    $ctstate{depth} = $oldDepth;

    print STDERR "Resetting inMacro (".$ctstate{inMacro}.") to previous value ($oldInMacro).\n" if ($localDebug);
    $ctstate{inMacro} = $oldInMacro;
    $ctstate{lastKeyword} = $oldLastKeyword;
    $ctstate{inComment} = $oldInComment;
    $ctstate{inQuote} = $oldInQuote;
    $ctstate{inAttribute} = $oldInAttribute;
    $ctstate{inRaises} = $oldInRaises;
    $ctstate{inTypeOf} = $oldInTypeOf;
    # if ($ctstate{inComment} && !$oldInComment) {
	# $ctstate{inComment} = $oldInComment;
	# if ($ctstate{xmlmode}) {
		# $string .= "</declaration_comment>";
	# } else {
		# $string .= "</span>";
	# }
    # }

    if ($dropFP) { $ctstate{type} = $ctstate{apio}->class(); }

    $node = $self->{NEXT};
    if ($node && $continue) {
	bless($node, "HeaderDoc::ParseTree");

	if ($nospaceafter) {
		while ($node && ($node->token =~ /[ \t]/)) {
			print STDERR "SKIPPED NODE (\"".$node->token."\").\n" if ($localDebug);
			$node = $node->next;
			bless($node, "HeaderDoc::ParseTree");
		}
		print STDERR "STOPPED SKIPPING AT NODE \"".$node->token."\".\n" if ($localDebug);
	}
	print STDERR "CONTINUING TO NODE \"".$node->token."\".\n" if ($localDebug);
	if ($node) {
		my $nextctstate = newCTState(\%ctstate, breakable => $nextbreakable);
		($newstringref, $ctstate{newlen}, $nextbreakable, $ctstate{prespace}, $ctstate{lastnstoken}, $continue, $ctstate{lastTokenType}, $ctstate{spaceSinceLastToken}) = $node->colorTreeSub($nextctstate);
		$newstring = ${$newstringref};
	}
    }
    $string .= $newstring;
    print STDERR "SET STRING TO $string\n" if ($localDebug);

    # $self->{CTSTRING} = $string;
    # $self->{CTSUB} = ($ctstate{newlen}, $nextbreakable, $ctstate{prespace}, $ctstate{lastnstoken});
    return (\$string, $ctstate{newlen}, $nextbreakable, $ctstate{prespace}, $ctstate{lastnstoken}, $continue, $ctstate{lastTokenType}, $ctstate{spaceSinceLastToken});
}

# /*!
#     @abstract
#         The recursive part of {@link test_output_dump}.
#     @param self
#         The parse tree/subtree to dump.
#     @param depth
#         The number of nesting levels above this token.
#     @param lastnode
#         The last node to emit.  This node and its children
#         appear in the output, but not its siblings.
#  */
sub test_output_dump_rec
{
    my $self = shift;
    my $depth = shift;
    my $lastnode = shift;
    my $ret = "";

    my $parserState = $self->parserState();
    if ($parserState && !$lastnode) {
        $lastnode = $parserState->{lastTreeNode};
    }

    if ($self->token ne "") {
      my $i = $depth-1;
      while ($i > 0) {
	$ret .= "|   ";
	$i--;
      }
      my $HYPHEN = "-";
      my $psString = "";
      if ($self->parserState()) {
	$HYPHEN = "*";
	$psString = " (HAS STATE)";
      }
      if ($depth) {
	$ret .= "+-$HYPHEN-";
      }
      if ($self->token =~ /\n$/) {
	$ret .= "[ NEWLINE ]$psString\n";
      } else {
	$ret .= $self->token()."$psString\n";
	# if ($self->token !~ /\n$/) { $ret .= "\n"; }
      }
    }

    if ($self == $lastnode) {
	$ret .= "-=-=-=-=-=-=- EODEC -=-=-=-=-=-=-\n";
    }

    if ($self->firstchild()) {
	$ret .= $self->firstchild()->test_output_dump_rec($depth+1, $lastnode);
    }
    if ($self->next()) {
	$ret .= $self->next()->test_output_dump_rec($depth, $lastnode);
    }

    return $ret;
}

# /*!
#     @abstract
#         The recursive part of
#  {@link //apple_ref/perl/instm/HeaderDoc::ParseTree/dbprint//() dbprint}.
#     @param self
#         The parse tree/subtree to dump.
#     @param depth
#         The number of nesting levels above this token.
#     @param lastnode
#         The last node to emit.  This node and its children
#         appear in the output, but not its siblings.
#  */
sub dbprintrec
{
    my $self = shift;
    my $depth = shift;
    my $lastnode = shift;
    my $include_noise = shift;

# print STDERR "IN: $include_noise\n";

    my $localDebug = 0;

# cluck("here\n");

    my $parserState = $self->parserState();
    if ($parserState && !$lastnode) {
        $lastnode = $parserState->{lastTreeNode};
    }

    if ($self->token ne "") {
      my $i = $depth-1;
      while ($i > 0) {
	print STDERR "|   ";
	$i--;
      }
      my $HYPHEN = "-";
      my $psString = "";
      my $reStateString = "";
      if ($self->{RE_STATE}) { $reStateString = " ".$self->{RE_STATE}; }
      if ($self->parserState()) {
	$HYPHEN = "*";
	$psString = " (TOKENID: ".$self.", PSID: ".$self->parserState().$reStateString.")";
      } else {
	$psString = " (TOKENID: ".$self.$reStateString.")";
      }
      if ($depth) {
	print STDERR "+-$HYPHEN-";
      }
      if ($self->token =~ /\n$/) {
	print STDERR "[ NEWLINE ]$psString\n";
      } else {
	print STDERR $self->token()."$psString\n";
	# if ($self->token !~ /\n$/) { print STDERR "\n"; }
      }
    }
    if ($include_noise) {
	printHash(%{$self});
	print STDERR "API OWNERS:\n";
	foreach my $apio (@{$self->{APIOWNERS}} ) {
		print "    ".$apio."\n";
	}
	print STDERR "END OF API OWNERS\n";
    }

    if ($self == $lastnode) {
	my $deb = "";
	if ($localDebug) { $deb = $lastnode; }
	print STDERR "-=-=-=-=-=-=- EODEC $deb-=-=-=-=-=-=-\n";
    }

    if ($self->firstchild()) {
	$self->firstchild()->dbprintrec($depth+1, $lastnode, $include_noise);
    }
    if ($self->next()) {
	$self->next()->dbprintrec($depth, $lastnode, $include_noise);
    }
}

# /*!
#     @abstract
#         Dumps a parse tree in a manner formatted for use by the
#         {@link //apple_ref/perl/cl/HeaderDoc::Test Test} class
#         (used for running HeaderDoc self tests).
#     @param self
#         The parse tree/subtree to dump.
#     @param depth
#         The number of nesting levels above this token.
#     @param lastnode
#         The last node to emit.  This node and its children
#         appear in the output, but not its siblings.
#     @discussion
#          This is similar to dbprint in many ways, but it
#          omits object IDs that change from run to run.
#  */
sub test_output_dump
{
    my $self = shift;

    return $self->test_output_dump_rec(1);
}

# /*!
#     @abstract
#         Prints a parse tree in a format designed for debugging.
#     @param self
#         The parse tree object to print.
#  */
sub dbprint
{
    my $self = shift;
    my $enable_noise = shift;
    $self->dbprintrec(1, undef, $enable_noise);
}

# /*!
#     @abstract
#         Gets/sets the filename associated with this tree node.
#     @param self
#         This tree node.
#     @param FILENAME
#         The new filename value to set. (Optional.)
#  */
sub filename
{
    my $self = shift;

    if (@_) {
	$self->{FILENAME} = shift;
    }

    return $self->{FILENAME};
}

# /*!
#     @abstract
#         Gets/sets the path associated with this tree node.
#     @param self
#         This tree node.
#     @param FILENAME
#         The new filename value to set. (Optional.)
#     @discussion
#         This contains the (relative or absolute) path
#         for the owning header, as passed in on the
#         command line.
#  */
sub fullpath
{
    my $self = shift;

    if (@_) {
	$self->{FULLPATH} = shift;
    }

    return $self->{FULLPATH};
}

# /*!
#     @abstract
#         Gets/sets the line number associated with this tree node.
#     @param self
#         This tree node.
#     @param LINENUM
#         The new line number value to set. (Optional.)
#     @discussion
#         This value indicates where this tree node came fron
#         within the enclosing header.
#  */
sub linenum
{
    my $self = shift;

    if (@_) {
	$self->{LINENUM} = shift;
    }
    return $self->{LINENUM};
}


# /*!
#     @abstract
#         Prints this object for debugging purposes.
#     @param self
#         This object.
#  */
sub printObject {
    my $self = shift;
 
    print STDERR "----- ParseTree Object ------\n";
    print STDERR "token: $self->{TOKEN}\n";
    print STDERR "next: $self->{NEXT}\n";
    print STDERR "firstchild: $self->{FIRSTCHILD}\n";
    print STDERR "\n";
}

# /*!
#     @abstract
#         Adds the parsed parameter strings from an
#         array (reference) to this node's parsed
#         parameters list.
#     @param self
#         This parse tree node.
#     @param pplref
#         A reference to a parsed parameter list.
#     @seealso rawParsedParams rawParsedParams
#  */
sub addRawParsedParams
{
    my $self = shift;
    my $pplref = shift;

    my @array = @{$pplref};

    foreach my $param (@array) {
	push(@{$self->{RAWPARSEDPARAMETERS}}, $pplref);
    }

    return $self->{RAWPARSEDPARAMETERS};
}


# /*!
#     @abstract
#         Returns a reference to an array of parsed parameter
#         strings associated with this node.
#     @param self
#         This parse tree node.
#     @seealso addRawParsedParams
#  */
sub rawParsedParams
{
    my $self = shift;

    return $self->{RAWPARSEDPARAMETERS};
}

# /*!
#     @abstract
#         Gets/sets the parser state object associated with
#         this tree node.
#     @param self
#         This tree node.
#     @param PARSERSTATE
#         The new parser state value to set. (Optional.)
#     @discussion
#         Parser state objects are inserted into the parse
#         tree to make it easier to interpret declarations
#         within classes.
#  */
sub parserState
{
    my $self = shift;
    my $localDebug = 0;

    if (@_) {
	my $state = shift;
	print STDERR "Setting parser state for $self\n" if ($localDebug);
	print STDERR "Last token (raw) is $state->{lastTreeNode}\n" if ($localDebug);
	print STDERR "Last token (text) is ".$state->{lastTreeNode}->token()."\n" if ($localDebug);
	$self->{PARSERSTATE} = $state;
    }

    return $self->{PARSERSTATE};
}

# /*!
#     @abstract
#         Uses gcc to calculate the value for a
#         string containing a complex <code>#define</code>
#         macro expression.
#     @param self
#         The parse tree object.  (Unused.)
#     @param rawvalue
#         The expression to interpret.
#     @discussion
#         Used by {@link getPTvalue}.
#
#         Although it should be possible for the
#         new code in MacroFilter.pm to be easily
#         adapted to do this job, at the time, that
#         code did not exist.
#  */
sub trygcc
{
    my $self = shift;
    my $rawvalue = shift;
    my $success = 0;
    my $value = 0;
    my $timestamp = time();
    my $localDebug = 0;

    # print STDERR "RV: $rawvalue\n";

    if (open(GCCFILE, ">/tmp/headerdoc-gcctemp-$timestamp.c")) {
	print GCCFILE "#include <inttypes.h>\nmain(){printf(\"%d\\n\", $rawvalue);}\n";
	close(GCCFILE);

	if (open(GCCPIPE, $HeaderDoc::c_compiler." /tmp/headerdoc-gcctemp-$timestamp.c -o /tmp/headerdoc-gcctemp-$timestamp > /dev/null 2> /dev/null |")) {
		my $junkstring = <GCCPIPE>;
		close(GCCPIPE);
		if ($?) {
			$success = 0;
		} else {
			$success = 1;
		}

		if ($success) {
			if (open(EXECPIPE, "/tmp/headerdoc-gcctemp-$timestamp |")) {
				my $retstring = <EXECPIPE>;
				$value = $retstring;
				$value =~ s/\n//sg;
				print STDERR "VALUE: $value\nSUCCESS: $success\n" if ($localDebug);
			} else {
				$success = 0;
			}
		}
	}
	unlink("/tmp/headerdoc-gcctemp-$timestamp.c");
	unlink("/tmp/headerdoc-gcctemp-$timestamp");
    }

    print STDERR "RET $success, $value\n" if ($localDebug);
    return ($success, $value);
}

# /*!
#     @abstract
#         Uses gcc to calculate the value for a
#         parse tree containing a complex <code>#define</code>
#         macro expression
#     @param self
#         The parse tree object.
#     @discussion
#         Although it should be possible for the
#         new code in MacroFilter.pm to be easily
#         adapted to do this job, at the time, that
#         code did not exist.
#  */
sub getPTvalue
{
    my $self = shift;
    my $success = 0;
    my $value = 0;
    my $localDebug = 0;

    my $pos = $self;

    while ($pos && ($pos->token() ne "#define")) {
	$pos = $pos->next();
    }
    if (!$pos) {
	return($success, $value);
    }
    $pos = $pos->firstchild();

    while ($pos && ($pos->hidden != 3) && ($pos->{HIDEMACROLASTTOKEN} != 2)) {
	$pos = $pos->next();
    }

    if ($pos) {
	my $rawvalue = $pos->textTree();
	print STDERR "getPTvalue: WE HAVE A WINNER.\n" if ($localDebug);
	print STDERR "RAWVALUE IS: $rawvalue\n" if ($localDebug);

	($success, $value) = $self->trygcc($rawvalue);
    }

    return($success, $value);
}

# /*!
#     @abstract
#         Releases references in an attempt to allow Perl to
#         garbage collect this parse tree.
#     @param self
#         The root of the parse tree to release.
#  */
sub dispose
{
    my $self = shift;

    # Decrement the reference count.

    # cluck("Dispose called on parse tree\n");

    if ($self->{REFCOUNT} > 0) {
	if ($HeaderDoc::debugAllocations) {
		print STDERR "Parse tree reference count is ".$self->{REFCOUNT}.".  Not disposing.\n";
		print STDERR "SELF: ".$self."\n";
		print STDERR "FC: ".$self->{FIRSTCHILD}."\n";
		print STDERR "NEXT: ".$self->{NEXT}."\n";
		print STDERR "PARENT: ".$self->{PARENT}."\n";
		print STDERR "APIOLIST: ".$self->{APIOWNERS}."\n";
		foreach my $obj (@{$self->{APIOWNERS}}) {
			print STDERR "    OWNER: $obj\n";
		}
		print STDERR "END OF LIST\n";
	}
	return;
    }

    print STDERR "Disposing of tree\n" if ($HeaderDoc::debugAllocations);

    $self->{PARENT} = undef;
    if ($self->{FIRSTCHILD}) {
	$self->{FIRSTCHILD}->dispose();
	$self->{FIRSTCHILD} = undef;
    }
    if ($self->{NEXT}) {
	$self->{NEXT}->dispose();
	$self->{NEXT} = undef;
    }
    $self->{NTNC} = undef;
    if ($self->{APIOWNERS}) { $self->{APIOWNERS} = (); }
    if ($self->{PARSERSTATE}) { $self->{PARSERSTATE} = undef; }
    $self->{PARSEDPARAMS} = ();
    $self->{RAWPARSEDPARAMETERS} = ();
    $self->{TOKEN} = undef;

    $self->dbprint() if ($HeaderDoc::debugAllocations);

}


# /*!
#     @abstract
#         Called by Perl when a node is garbage collected.
#     @param self
#         The root of the parse tree to release.
#     @discussion
#         Used for debugging memory behavior.
#  */
sub DESTROY
{
    my $self = shift;

    print STDERR "Destroying $self\n" if ($HeaderDoc::debugAllocations);
}

# /*!
#     @abstract
#         Counts the number of curly braces at this level in the parse tree.
#     @param self
#         This parse tree node.
#     @param lbrace
#         The left brace token to look for.
#     @param last
#         The last node to include.
#     @discussion
#         Because this function checks at a given level,
#         it updates the last node to be the ancestor
#         of the last node at the same level as the
#         starting node.  (Why doesn't this use
#         {@link nextAtLevelOf}?)
#  */
sub curlycount
{
    my $self = shift;
    my $lbrace = shift;
    my $last = shift;
    my $pos = $self;
    my $count = 0;

    while ($last && !$last->isAfter($self)) {
	$last = $last->parent();
    }
    if ($last) { $last = $last->next(); } # first node after this declaration

    while ($pos && $pos != $last) {
	if ($pos->token eq "$lbrace") {
		$count++;
	}
	$pos = $pos->next();
    }
    return $count;
}

# /*!
#     @abstract
#         Walks this parse tree node's sibling chain and
#         returns the node after the nth left brace.
#     @param self
#         The starting parse tree node.
#     @param lbrace
#         The left brace token to look for.
#     @param count
#         The number of braces to skip past.
#  */
sub skipcurly
{
    my $self = shift;
    my $lbrace = shift;
    my $count = shift;

    my $localDebug = 0;
    my $pos = $self;

    print STDERR "SKIPPING $count curly braces (lbrace = '$lbrace') at POS=$pos\n" if ($localDebug);
    if (!$count) { return $self; }

    while ($pos) {
	my $tok = $pos->token();
	print STDERR "TOKEN: '$tok'\n" if ($localDebug);
	if ($tok eq "$lbrace") {
		print STDERR "MATCH\n" if ($localDebug);
		if (!--$count) {
			my $next = $pos->next;
			if ($localDebug) {
				print STDERR "FOUND ONE.  Next tree is:\n";
				if ($next) {
					$next->dbprint();
				} else {
					print STDERR "UNDEFINED!\n";
				}
			}
			return $next;
		}
	}
	$pos = $pos->next();
    }
    warn "Yikes!  Ran out of open braces!\n";
    return $pos;
}

# /*!
#     @abstract
#         Returns whether this token is a start token
#         for a C preprocessor macro.
#     @param token
#         The token to check.
#     @param macroListRef
#         A reference to the macro hash returned by a
#         call to
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/parseTokens//() parseTokens}.
#     @param case_sensitive
#         Set to 0/1 depending on whether the language uses
#         case-sensitive token matching.  This value is
#         generally obtained with a call to
#  {@link //apple_ref/perl/instm/HeaderDoc::HeaderElement/keywords//() keywords}.
#  */
sub isKnownMacroToken
{
    my $token = shift;
    my $macroListRef = shift;
    my $case_sensitive = shift;

    my %macroList = %{$macroListRef};
    if ($case_sensitive) {
	if ($macroList{$token}) { return 1; }
	return 0;
    }

    foreach my $cmpToken (keys %macroList) {
	if (casecmp($token, $cmpToken, $case_sensitive)) {
		return 1;
	}
    }
    return 0;
}

# /*!
#     @abstract
#         Scrapes out the children of a new-style availability
#         macro and returns a sanitized array.
#     @param self
#         The parse tree node, taken from
#  {@link //apple_ref/perl/data/HeaderDoc::ParserState/availabilityNodesArray availabilityNodesArray}
#         in the
#  {@link //apple_ref/perl/cl/HeaderDoc::ParserState ParserState} object.
#     @discussion
#         Most of the hard work is done by
#  {@link //apple_ref/perl/instm/HeaderDoc::Utilities/complexAvailabilityToArray//() complexAvailabilityToArray}.
#
#         This function is exclusively used by
#  {@link //apple_ref/perl/instm/HeaderDoc::BlockParse/mergeComplexAvailability//() mergeComplexAvailability}.
#  */
sub parseComplexAvailability
{
    my $self = shift;
    my $localDebug = 0;

    print STDERR "parseComplexAvailability: dumping tree for $self.\n" if ($localDebug);

    $self->dbprint() if ($localDebug);

    my $token = $self->token();
    my $availstring = "";

    my $pos = $self->firstchild();
    while ($pos && ($pos->token() ne "(")) {
	$pos = $pos->next();
    }
    if (!$pos) { my @arr = (); return \@arr; }

    $pos = $pos->next();
    while ($pos && ($pos->token() ne ")")) {
	$availstring .= $pos->token();
	$pos = $pos->next();
    }

    print STDERR "TOKEN: $token\nSTRING: $availstring\n" if ($localDebug);
    return complexAvailabilityToArray($token, $availstring);
}

# /*!
#     @abstract
#         Gets/sets whether this token is an availability macro.
#     @param self
#         The parse tree node.
#     @param newvalue
#         The new value to set. (Optional.)
#  */
sub isAvailabilityMacro
{
    my $self = shift;

    if (@_) {
        $self->{ISAVAILABILITYMACRO} = shift;
    }

    if ($self->{ISAVAILABILITYMACRO}) {
	return $self->{ISAVAILABILITYMACRO};
    }

    # my $parent = $self->parent();
    # if ($parent) { return $parent->isAvailabilityMacro(); }

    return 0;
}

# /*!
#     @abstract
#         Partially translates Perl code into C
#  */
sub translateTree {
    my $self = shift;
    my %state = ();

    $self->translateTreeRec(\%state);
}

# /*!
#     @abstract
#         Returns the parse tree node containing the
#         next non-space token at the current level.
#     @param self
#         The initial parse tree node.
#  */
sub nextNSToken
{
    my $self = shift;
    my $next = $self->{NEXT};
    while ($next && $next->{TOKEN} !~ /\S/s) {
	$next = $next->{NEXT};
    }
    return $next;
}

# /*! @abstract
#         Does the real work of translating a Perl script (partially)
#         to C.
#     @param self
#         The tree to translate.
#     @param self
#         A translation state object (initially empty).
#  */
sub translateTreeRec {
    my $self = shift;
    my $stateref = shift;
    my %state = %{$stateref};
    my $localDebug = 0;

    my $enteringComment = 0;
    my $enteringString = 0;
    my $enteringSingle = 0;
    my $leavingHDComment = 0;

    print STDERR "TOK: ".$self->{TOKEN}."\n" if ($localDebug);

    if ((!$state{seenCode}) && $self->{TOKEN} && $self->{TOKEN} =~ /\S/s && $self->{TOKEN} ne "if" && $self->{TOKEN} ne "#" && $self->{TOKEN} ne "{" && $self->{TOKEN} ne "}") {
	print STDERR "SEENCODE -> 1\n" if ($localDebug);
	$state{seenCode} = 1;
	$state{insertIfBeforeNode} = $self;
    }
    print STDERR "CHECK: INSTRING: ".$state{inString}." INCOMMENT: ".$state{inComment}." INREGEXP: ".$state{inRegExp}." INSINGLE: ".$state{inSingle}."\n" if ($localDebug);
    if (!($state{inString} || $state{inComment} || $state{inRegExp} || $state{inSingle})) {
	# Don't mess with the guts of these things, for the most part.
	print STDERR "FIXUP OKAY\n" if ($localDebug);

	if ($self->{TOKEN} eq "\"") {
		$state{inString} = 1;
		$enteringString = 1;
	} elsif ($self->{TOKEN} eq "'") {
		$state{inSingle} = 1;
		$enteringSingle = 1;
	} elsif ($self->{TOKEN} eq "#") {
		print STDERR "ILC\n" if ($localDebug);
		$state{inComment} = 1;
		$enteringComment = 1;
		$state{socNode} = $self;
		if ($state{inHDComment}) {
			$self->{TOKEN} = " ";
		} else {
			$self->{TOKEN} = "//";
		}
		if ($self->{FIRSTCHILD}->textTree() =~ /\*\//) {
			$leavingHDComment = 1;
		}
	} elsif ($self->{TOKEN} eq "\$") { # Merge with next token.
		my $next = $self->{NEXT};
		if ($next) {
			$self->{TOKEN} = "";
			$self->{NEXT}->{TOKEN} = "\$".$self->{NEXT}->{TOKEN};
		}
	} elsif ($self->{TOKEN} eq ":") {
		if ($self->{NEXT} && $self->{NEXT}->{TOKEN} eq ":") {
			$self->{TOKEN} = "_";
			$self->{NEXT}->{TOKEN} = "_";
		}
	} elsif ($self->{TOKEN} eq "elsif") {
		$self->{TOKEN} = "else if";
	} elsif ($self->{TOKEN} eq "STDERR") {
		$self->{TOKEN} = "stderr";
	} elsif ($self->{TOKEN} eq "STDOUT") {
		$self->{TOKEN} = "stdout";
	} elsif ($self->{TOKEN} eq "STDIN") {
		$self->{TOKEN} = "stdin";
	} elsif ($self->{TOKEN} eq "<" && $self->{FIRSTCHILD} &&
		$self->{FIRSTCHILD}->{NEXT} &&
		$self->{FIRSTCHILD}->{NEXT}->{TOKEN} =~ /^[a-zA-Z]+$/ &&
		$self->{FIRSTCHILD}->{NEXT}->{NEXT} &&
		$self->{FIRSTCHILD}->{NEXT}->{NEXT}->{TOKEN} eq ">") {
			# print STDERR "FC: ".$self->{FIRSTCHILD}->{TOKEN}." NC ".$self->{FIRSTCHILD}->{NEXT}->{TOKEN}."\n";
			print STDERR "Read from file.\n" if ($localDebug);
			$self->{TOKEN} = "stringFromFP(";
			$self->{FIRSTCHILD}->{NEXT}->{NEXT}->{TOKEN} = ")";
	} elsif ($self->{TOKEN} eq "print" || $self->{TOKEN} eq "warn") {
		my $nextnstoken = $self->nextNSToken();
		if ($self->{TOKEN} eq "print") {
			my $ns = $self->nextNSToken();
			if ($ns && ($ns->{TOKEN} eq "\"" || $ns->{TOKEN} eq "\$")) {
				$self->{TOKEN} = "printf";
			} else {
				$self->{TOKEN} = "fprintf";
				if ($ns->{TOKEN} eq "STDERR") {
					$ns->{TOKEN} = "stderr";
				} elsif ($ns->{TOKEN} eq "STDOUT") {
					$ns->{TOKEN} = "stdout";
				}
				$ns->{TOKEN} .= ",";
			}
		}
		if ($nextnstoken->{TOKEN} ne "(") {
			$self->{TOKEN} .= "(";
			$state{addParenAtSemi} = 1;
			$self->{NEXT} = $nextnstoken;
		}
	} elsif ($self->{TOKEN} eq "split") {
		$self->{TOKEN} = "regexpSplit";
		$state{inSplit} = 1;
		print STDERR "inSplit -> 1\n" if ($localDebug);
	} elsif ($self->{TOKEN} eq "(" && $state{inSplit}) {
		$state{inSplit} = 2;
		print STDERR "inSplit -> 1\n" if ($localDebug);
	} elsif ($self->{TOKEN} eq "->") {
		print STDERR "->" if ($localDebug);
		my $pos = $self->{NEXT};
		while ($pos && $pos->{TOKEN} !~ /\S/s) {
			$pos = $pos->{NEXT};
		}
		if ($pos && $pos->{TOKEN} eq "{") {
			my $posb = $pos->{FIRSTCHILD};
			while ($posb && $posb->{TOKEN} ne "}") {
				$posb = $posb->{NEXT};
			}
			if ($pos && $posb) {
				$pos->{TOKEN} = "";
				$posb->{TOKEN} = "";
			}
		}
	} elsif ($self->{TOKEN} eq "eq" || $self->{TOKEN} eq "ne") {
		my $prev = $self->slowprev();
		while ($prev && ($prev->token !~ /\S/s)) {
			$prev = $prev->slowprev();
		}
		while ($prev && $prev->{TOKEN} !~ /\&/ && $prev->{TOKEN} !~/\|/) {
			print STDERR "PREV: $prev\n" if ($localDebug);
			print STDERR "TOK: ".$prev->{TOKEN}."\n" if ($localDebug);
			$prev = $prev->slowprev();
		}
		# })

		# If we run off the beginning of the chain, we hit an open parenthesis,
		# so we know we can always insert after the first (always-empty)
		# child.

		my $addspace = "";
		if (!$prev) {
			$prev = $self->parent()->firstchild();
		} else {
			# Add a space after || or &&
			$addspace = " ";
		}

		my $next = $self->{NEXT};
		while ($next && ($next->token !~ /\S/s)) {
			$next = $next->{NEXT};
		}
		print STDERR "NEXT IS \"".$next->{TOKEN}."\"\n" if ($localDebug);
		while ($next && $next->{NEXT} && $next->{NEXT}->{TOKEN} !~ /\&/ && $next->{NEXT}->{TOKEN} !~ /\|/ &&
				$next->{NEXT}->{TOKEN} !~ /\)/) {
			$next = $next->{NEXT};
			print STDERR "INLOOP NEXT IS ".$next->{TOKEN}."\n" if ($localDebug);
		}
		if ($prev && $next) {
			my $newnode = HeaderDoc::ParseTree->new();
			if ($self->{TOKEN} eq "eq") {
				$newnode->{TOKEN} = $addspace."!strcmp(";
			} else {
				$newnode->{TOKEN} = $addspace."strcmp(";
			}
			$newnode->{NEXT} = $prev->{NEXT};
			$prev->{NEXT} = $newnode;

			if ($next->{TOKEN} eq "\$") {
				$next->{TOKEN} = "";
				$next->{NEXT}->{TOKEN} = "\$".$next->{NEXT}->{TOKEN};
				$next = $next->{NEXT};
			}

			$newnode = HeaderDoc::ParseTree->new();
			$newnode->{TOKEN} = ")";
			$newnode->{NEXT} = $next->{NEXT};
			$next->{NEXT} = $newnode;

			$self->{TOKEN} = ",";
			$prev = $self->slowprev();
			while ($prev && ($prev->{TOKEN} =~ /\s/s)) {
				$prev->{TOKEN} = "";
				$prev = $prev->slowprev();
			}
		} else {
			warn("Could not fix \"eq\" because previous/next token not found (PREV: $prev NEXT: $next).\n");
		}
	} elsif ($state{seenCode} && ($self->{TOKEN} eq ";" || $self->{TOKEN} eq "{")) {
		$state{seenCode} = 0;
		if ($state{addParenAtSemi}) {
			$state{addParenAtSemi} = 0;
			$self->{TOKEN} = ")".$self->{TOKEN};
		}
		print STDERR "SEENCODE -> 0\n" if ($localDebug);
		if ($state{captureIf}) {
			my $lastcapture = $self->slowprev();

			my $beforenode = $state{insertIfBeforeNode};
			my $prev = $beforenode->slowprev();
			if (!$prev) {
				my $newnode = HeaderDoc::ParseTree->new();
				$newnode->{TOKEN} = $beforenode->{TOKEN};
				$newnode->{NEXT} = $beforenode->{NEXT};
				$beforenode->{NEXT} = $newnode;
				$beforenode->{TOKEN} = "";
				$prev = $beforenode;
				$beforenode = $newnode;
			}

			my $trailingspace = $self->slowprev();
			if (!$trailingspace) {
				$trailingspace = $self->parent();
			}
			while ($trailingspace &&
				((!$trailingspace->{TOKEN}) ||
				 ($trailingspace->{TOKEN} =~ /\s/s))) {

				# print STDERR "TS: $trailingspace\n" if ($localDebug);
				# print STDERR "PARENT: ".$trailingspace->parent()."\n" if ($localDebug);

				$trailingspace->{TOKEN} = "";
				my $temp = $trailingspace->slowprev();
				if ($temp) { $trailingspace = $temp; }
				else { $trailingspace = $trailingspace->parent(); }
			}

			my $copiedDataNode = HeaderDoc::ParseTree->new();
			$copiedDataNode->{NEXT} = $prev->{NEXT};
			$prev->{NEXT} = $copiedDataNode;

			$copiedDataNode->{TOKEN} = $state{capturedText}." ";
			print STDERR "ACCUM: ".$state{capturedText}."\n" if ($localDebug);

			$state{captureIf} = 0;
			print STDERR "captureIf -> 0\n" if ($localDebug);
		}
	} elsif ($self->{TOKEN} eq "if" && $state{seenCode}) {
		$state{capturedText} = "";
		$state{captureIf} = 1;
		print STDERR "captureIf -> 1\n" if ($localDebug);
	}
    } elsif ($state{inComment} == 1 && ($self->{TOKEN} eq "/*!" || $self->{TOKEN} eq "/**")) {
	$state{socNode}->{TOKEN} = " ";
	$state{inHDComment} = 1;
    } elsif ($state{inComment} && $self->{TOKEN} =~ /\S/) {
	$state{inComment} = 2;
    }

    print STDERR "RE_STATE: ".$self->{RE_STATE}."\n" if ($localDebug);
    if ($self->{RE_STATE} eq "RE_PREFIX") {
	print STDERR "Regexp prefix found.\n" if ($localDebug);
	if ($self->{TOKEN} eq "s") {
		$state{inRegExp} = 3;
		$state{rePrefix} = $self->{TOKEN};
		$self->{TOKEN} = "";
	} elsif ($self->{TOKEN} eq "tr") {
		$state{inRegExp} = 4;
		$state{rePrefix} = $self->{TOKEN};
		$self->{TOKEN} = "";
	} elsif ($self->{TOKEN} eq "m") {
		$self->{TOKEN} = "";
		print STDERR "Dropping prefix because 'm' is equivalent to no prefix.\n" if ($localDebug);
	} else {
		print STDERR "Unsupported prefix.  Ignoring this expression.\n" if ($localDebug);
		$state{inRegExp} = -1; # No 'tr' support yet.
	}
    } elsif ($self->{RE_STATE} eq "RE_PARTSEP") {
	if ($self->{NEXT}->{RE_STATE} eq "RE_PARTSEP") {
		$self->{NEXT}->{TOKEN} = "";
		$self->{NEXT}->{RE_STATE} = "";
	}
	$self->{TOKEN} = "\", \"";
    } elsif ($self->{RE_STATE} eq "RE_START" && ((!$state{inRegExp}) || $state{inRegExp} == 3 || $state{inRegExp} == 4)) {
	print STDERR "Regexp start found.  INSPLIT: ".$state{inSplit}."\n" if ($localDebug);
	$state{regexpName} = $state{prevTokenNode}->{TOKEN};

	print STDERR "PTN: ".$state{prevTokenNode}." (".$state{prevTokenNode}->{TOKEN}.") REGEXPNAME: ".$state{regexpName}."\n" if ($localDebug);

	if ($state{inSplit}) {
		$self->{TOKEN} = "\"";
		$state{inRegExp} = 2;
	} elsif ($state{regexpName} =~ /^\$[0-9A-Za-z_]+$/) {
		if (!$state{inRegExp}) {
			$state{inRegExp} = 1;
			$state{prevTokenNode}->{TOKEN} = "regexpMatch(".$state{regexpName}.", ";
		} elsif ($state{inRegExp} == 3) {
			$state{prevTokenNode}->{TOKEN} = "regexpReplace(".$state{regexpName}.", \@\@\@DEST\@\@\@, ";
		} elsif ($state{inRegExp} == 4) {
			$state{prevTokenNode}->{TOKEN} = "regexptr(".$state{regexpName}.", \@\@\@DEST\@\@\@, ";
		}
		my $pos = $state{prevTokenNode}->{NEXT};
		print STDERR "Deleting the =~ .\n" if ($localDebug);
		while ($pos && !$pos->{RE_STATE}) {
			print STDERR "Deleting \"".$pos->{TOKEN}."\" ($pos)\n" if ($localDebug);
			# Nuke the "=~"
			if ($pos->{TOKEN} eq "!") {
				$state{prevTokenNode}->{TOKEN} = "!".$state{prevTokenNode}->{TOKEN};
			}
			$pos->{TOKEN} = "";
			$pos = $pos->{NEXT}
		}
		if ($state{rePrefix} eq "s") {
			$self->{TOKEN} = "\"";
		# } elsif ($state{rePrefix} eq "tr") {
			# $self->{TOKEN} = "\"[";
		} else {
			$self->{TOKEN} = "\"";
		}
	} else {
		$state{inRegExp} = -1;
	}
    } elsif ($self->{RE_STATE} eq "RE_END") {
	print STDERR "End of regexp found.\n" if ($localDebug);

	# Defaults.
	my $multiline = 0;
	my $insensitive = 0;
	my $global = 0;
	my $complement = 0;
	my $delete = 0;
	my $nondestructive = 0;
	my $squash = 0;

	my $next = $self->{NEXT};
	if ($next && $next->{TOKEN} =~ /^[siogmcdr]*$/) {
		my $re_trailer = $next->{TOKEN};
		if ($re_trailer =~ /s/) {
			$multiline = 0;  # for 's'
			$squash = 1;     # for 'tr'
		} elsif ($re_trailer =~ /m/) {
			$multiline = 1;
		}
		if ($re_trailer =~ /g/) {
			$global = 1;
		}
		if ($re_trailer =~ /i/) {
			$insensitive = 1;
		}
		if ($re_trailer =~ /c/) {
			$complement = 1;
		}
		if ($re_trailer =~ /d/) {
			$delete = 1;
		}
		if ($re_trailer =~ /r/) {
			$nondestructive = 1;
		}
		$next->{TOKEN} = "";
	}
	if ($state{inRegExp} == 1) {
		$self->{TOKEN} = "\", $multiline, $insensitive, NULL)";
	} elsif ($state{inRegExp} == 2) {
		# Leave parentheses open.
		$self->{TOKEN} = "\"";

		my $next = $self->nextNSToken();
		print STDERR "THISNEXT: ".$next->{TOKEN}."\n" if ($localDebug);
		if (!$next) {
			die("Malformed split\n");
		}
		# $next should be the comma after the regular expression.

		$next = $next->nextNSToken();
		if ($next->{TOKEN} eq "\$") {
			$next->{TOKEN} = "";
			$next->{NEXT}->{TOKEN} = "\$".$next->{NEXT}->{TOKEN};
			$next = $next->{NEXT};
		}
		# $next should be the data to split.

		my $argNode = HeaderDoc::ParseTree->new();
		$argNode->{NEXT} = $next->{NEXT};
		$next->{NEXT} = $argNode;
		$argNode->{TOKEN} = ", $multiline, $insensitive";

		my $afternext = $argNode->nextNSToken();
		print STDERR "AN: ".$afternext->{TOKEN}."\n" if ($localDebug);
		if (!$afternext || ($afternext->{TOKEN} ne ",")) {
			$argNode->{TOKEN} .= ", 0";
		}
	} elsif ($state{inRegExp} == 3) {
		if ($state{rePrefix} eq "s") {
			$self->{TOKEN} = "\", $multiline, $insensitive, $global)";
		# } elsif ($state{rePrefix} eq "tr") {
			# $self->{TOKEN} = "]\"";
		} else {
			die("Unknown RE prefix \"".$state{rePrefix}."\" [1]\n");
		}
	} elsif ($state{inRegExp} == 4) {
		if ($state{rePrefix} eq "tr") {
			# Allowed: "s", but not "m", not "i", not "g"
			$self->{TOKEN} = "\", $squash, $complement, $delete)";

			if ($nondestructive) {
				$state{prevTokenNode}->{TOKEN} = "regexptr(".$state{regexpName}.", NULL, ";
			}
		} else {
			die("Unknown RE prefix \"".$state{rePrefix}."\" [2]\n");
		}
	}
	$state{inRegExp} = 0;
	$state{rePrefix} = "";
    } elsif ($state{inRegExp}) {
	$self->{TOKEN} =~ s/\\/\\\\/g;
    }

    if ($state{inComment} && $self->{TOKEN} =~ /[\n\r]/) {
	print STDERR "Leaving comment.\n" if ($localDebug);
	$state{inComment} = 0;
    }

    print STDERR "Checking token.\n" if ($localDebug);
    if ($self->{TOKEN} =~ /^\$[a-zA-Z0-9_]+$/) {
	print STDERR "setting PTN\n" if ($localDebug);
	$state{prevTokenNode} = $self;
    }

    if ($state{captureIf}) {
	print STDERR "APPEND ".$self->{TOKEN}."\n" if ($localDebug);
	$state{capturedText} .= $self->{TOKEN};
	$self->{TOKEN} = "";
    }

    my $resetSplit = 0;
    if ($state{inSplit} == 2) {
	$resetSplit = 1;
	$state{inSplit} = 3;
    }

    if ($self->{FIRSTCHILD}) {
	print STDERR "Processing children.\n" if ($localDebug);
	my $tempstateref = $self->{FIRSTCHILD}->translateTreeRec(\%state);
	my %tempstate = %{$tempstateref};
	if ($state{inComment} && $tempstate{inHDComment}) {
		$state{inHDComment} = 1;
	}
	if ($state{captureIf}) {
		$state{capturedText} = $tempstate{capturedText};
	}
	$state{seenCode} = $tempstate{seenCode};
	$state{prevTokenNode} = $tempstate{prevtoken};
    }
    if ($resetSplit) { $state{inSplit} = 0; print STDERR "inSplit -> 0\n" if ($localDebug) };
    if ($enteringComment) { $state{inComment} = 0; $state{seenCode} = 0; }
    if ($enteringString) { $state{inString} = 0; }
    if ($enteringSingle) { $state{inSingle} = 0; }
    if ($leavingHDComment) { $state{inHDComment} = 0; $state{seenCode} = 0; }
    if ($self->{NEXT}) {
	print STDERR "Processing next.\n" if ($localDebug);
	$stateref = $self->{NEXT}->translateTreeRec(\%state);
	%state = %{$stateref};
    }
    print STDERR "Returning.\n" if ($localDebug);
    return \%state;
}
1;