--- moin--main--1.2--patch-364/MoinMoin/parser/wiki.py	2005-01-16 16:07:50.096079352 +0100
+++ moin--main--1.3--patch-546/MoinMoin/parser/wiki.py	2005-01-16 16:08:19.182657520 +0100
@@ -6,17 +6,11 @@
     @license: GNU GPL, see COPYING for details.
 """
 
-# Imports
 import os, re
 from MoinMoin import config, wikimacro, wikiutil
 from MoinMoin.Page import Page
 from MoinMoin.util import web
 
-
-#############################################################################
-### MoinMoin Wiki Markup Parser
-#############################################################################
-
 class Parser:
     """
         Object that turns Wiki markup into HTML.
@@ -34,29 +28,29 @@
     # some common strings
     PARENT_PREFIX = wikiutil.PARENT_PREFIX
     attachment_schemas = ["attachment", "inline", "drawing"]
-    punct_pattern = re.escape('''"\'}]|:,.)?!''')
-    url_pattern = ('http|https|ftp|nntp|news|mailto|telnet|wiki|file|' +
-            '|'.join(attachment_schemas) + 
-            (config.url_schemas and '|' + '|'.join(config.url_schemas) or ''))
+    punct_pattern = re.escape(u'''"\'}]|:,.)?!''')
+    url_pattern = (u'http|https|ftp|nntp|news|mailto|telnet|wiki|file|' +
+            u'|'.join(attachment_schemas) + 
+            (config.url_schemas and u'|' + u'|'.join(config.url_schemas) or ''))
 
     # some common rules
-    word_rule = r'(?:(?<![%(l)s])|^)%(parent)s(?:%(subpages)s(?:[%(u)s][%(l)s]+){2,})+(?![%(u)s%(l)s]+)' % {
-        'u': config.upperletters,
-        'l': config.lowerletters,
+    word_rule = ur'(?:(?<![%(l)s])|^)%(parent)s(?:%(subpages)s(?:[%(u)s][%(l)s]+){2,})+(?![%(u)s%(l)s]+)' % {
+        'u': config.chars_upper,
+        'l': config.chars_lower,
         'subpages': config.allow_subpages and (wikiutil.CHILD_PREFIX + '?') or '',
-        'parent': config.allow_subpages and (r'(?:%s)?' % re.escape(PARENT_PREFIX)) or '',
+        'parent': config.allow_subpages and (ur'(?:%s)?' % re.escape(PARENT_PREFIX)) or '',
     }
-    url_rule = r'%(url_guard)s(%(url)s)\:([^\s\<%(punct)s]|([%(punct)s][^\s\<%(punct)s]))+' % {
-        'url_guard': '(^|(?<!\w))',
+    url_rule = ur'%(url_guard)s(%(url)s)\:([^\s\<%(punct)s]|([%(punct)s][^\s\<%(punct)s]))+' % {
+        'url_guard': u'(^|(?<!\w))',
         'url': url_pattern,
         'punct': punct_pattern,
     }
 
-    ol_rule = r"^\s+(?:[0-9]+|[aAiI])\.(?:#\d+)?\s"
-    dl_rule = r"^\s+.*?::\s"
+    ol_rule = ur"^\s+(?:[0-9]+|[aAiI])\.(?:#\d+)?\s"
+    dl_rule = ur"^\s+.*?::\s"
 
     # the big, fat, ugly one ;)
-    formatting_rules = r"""(?:(?P<emph_ibb>'''''(?=[^']+'''))
+    formatting_rules = ur"""(?:(?P<emph_ibb>'''''(?=[^']+'''))
 (?P<emph_ibi>'''''(?=[^']+''))
 (?P<emph_ib_or_bi>'{5}(?=[^']))
 (?P<emph>'{2,3})
@@ -66,37 +60,44 @@
 (?P<tt>\{\{\{.*?\}\}\})
 (?P<processor>(\{\{\{(#!.*|\s*$)))
 (?P<pre>(\{\{\{ ?|\}\}\}))
+(?P<small>(\~- ?|-\~))
+(?P<big>(\~\+ ?|\+\~))
 (?P<rule>-{4,})
 (?P<comment>^\#\#.*$)
-(?P<macro>\[\[(%(macronames)s)(?:\(.*?\))?\]\]))
-(?P<li>^\s+\*)
+(?P<macro>\[\[(%%(macronames)s)(?:\(.*?\))?\]\]))
 (?P<ol>%(ol_rule)s)
 (?P<dl>%(dl_rule)s)
+(?P<li>^\s+\*?)
 (?P<tableZ>\|\| $)
-(?P<table>(?:\|\|)+(?:<[^>]*?>)?(?=.))
+(?P<table>(?:\|\|)+(?:<[^>]*?>)?(?!\|? $))
 (?P<heading>^\s*(?P<hmarker>=+)\s.*\s(?P=hmarker) $)
 (?P<interwiki>[A-Z][a-zA-Z]+\:[^\s'\"\:\<\|]([^\s%(punct)s]|([%(punct)s][^\s%(punct)s]))+)
 (?P<word>%(word_rule)s)
 (?P<url_bracket>\[((%(url)s)\:|#|\:)[^\s\]]+(\s[^\]]+)?\])
 (?P<url>%(url_rule)s)
-(?P<email>[-\w._+]+\@[\w-]+\.[\w.-]+)
+(?P<email>[-\w._+]+\@[\w-]+(\.[\w-]+)+)
 (?P<smiley>(?<=\s)(%(smiley)s)(?=\s))
 (?P<smileyA>^(%(smiley)s)(?=\s))
 (?P<ent>[<>&])"""  % {
         'url': url_pattern,
         'punct': punct_pattern,
-        'macronames': '|'.join(wikimacro.names),
         'ol_rule': ol_rule,
         'dl_rule': dl_rule,
         'url_rule': url_rule,
         'word_rule': word_rule,
-        'smiley': '|'.join(map(re.escape, config.smileys.keys()))}
+        'smiley': u'|'.join(map(re.escape, config.smileys.keys()))}
+
+    # Don't start p before these 
+    no_new_p_before = ("heading rule table tableZ tr td ul ol dl dt dd li "
+                       "processor macro pre")
+    no_new_p_before = dict(zip(no_new_p_before.split(), [1] * len(no_new_p_before)))
 
     def __init__(self, raw, request, **kw):
         self.raw = raw
         self.request = request
         self.form = request.form
         self._ = request.getText
+        self.cfg = request.cfg
 
         self.macro = None
 
@@ -108,22 +109,31 @@
         self.in_dd = 0
         self.in_pre = 0
         self.in_table = 0
+        self.is_big = False
+        self.is_small = False
         self.inhibit_p = 0 # if set, do not auto-create a <p>aragraph
-        self.titles = {}
+        self.titles = request._page_headings
 
         # holds the nesting level (in chars) of open lists
         self.list_indents = []
         self.list_types = []
+        
+        self.formatting_rules = self.formatting_rules % {'macronames': u'|'.join(wikimacro.getNames(self.cfg))}
 
     def _close_item(self, result):
         #result.append("<!-- close item begin -->\n")
-        if self.formatter.in_p:
-            result.append(self.formatter.paragraph(0))
+        if self.in_table:
+            result.append(self.formatter.table(0))
+            self.in_table = 0
         if self.in_li:
             self.in_li = 0
+            if self.formatter.in_p:
+                result.append(self.formatter.paragraph(0))
             result.append(self.formatter.listitem(0))
         if self.in_dd:
             self.in_dd = 0
+            if self.formatter.in_p:
+                result.append(self.formatter.paragraph(0))
             result.append(self.formatter.definition_desc(0))
         #result.append("<!-- close item end -->\n")
 
@@ -147,12 +157,11 @@
         elif config.allow_subpages and url[0] == wikiutil.CHILD_PREFIX:
             # fancy link to subpage [wiki:/SubPage text]
             return self._word_repl(url, text)
-        elif Page(url).exists():
+        elif Page(self.request, url).exists():
             # fancy link to local page [wiki:LocalPage text]
             return self._word_repl(url, text)
 
         wikitag, wikiurl, wikitail, wikitag_bad = wikiutil.resolve_wiki(self.request, url)
-        wikiurl = wikiutil.mapURL(wikiurl)
         href = wikiutil.join_wiki(wikiurl, wikitail)
 
         # check for image URL, and possibly return IMG tag
@@ -163,19 +172,9 @@
         if wikitag is None:
             return self._word_repl(wikitail)
               
-        # return InterWiki hyperlink
-        if wikitag_bad:
-            html_class = 'badinterwiki'
-        else:
-            html_class = 'interwiki'
-        text = self.highlight_text(text) # also cgi.escapes if necessary
-
-        icon = ''
-        if self.request.user.show_fancy_links:
-            icon = self.request.theme.make_icon('interwiki', {'wikitag': wikitag}) 
-        return self.formatter.url(href, icon + text,
-            title=wikitag, unescaped=1, pretty_url=kw.get('pretty_url', 0), css = html_class)
-
+        return (self.formatter.interwikilink(1, wikitag, wikitail) + 
+                self.formatter.text(text) +
+                self.formatter.interwikilink(0))
 
     def attachment(self, url_and_text, **kw):
         """ This gets called on attachment URLs.
@@ -209,15 +208,15 @@
             fname = fname + ".png"
             url = url + ".png"
             # fallback for old gif drawings (1.1 -> 1.2)
-            fpath = os.path.join(AttachFile.getAttachDir(pagename), fname)
+            fpath = AttachFile.getFilename(self.request, pagename, fname)
             if not os.path.exists(fpath):
                 gfname = fname[:-4] + ".gif"
                 gurl = url[:-4] + ".gif"
-                gfpath = os.path.join(AttachFile.getAttachDir(pagename), gfname)
+                gfpath = AttachFile.getFilename(self.request, pagename, gfname)
                 if os.path.exists(gfpath):
                     fname, url, fpath = gfname, gurl, gfpath
         else:
-            fpath = os.path.join(AttachFile.getAttachDir(pagename), fname)
+            fpath = AttachFile.getFilename(self.request, pagename, fname)
 
         # check whether attachment exists, possibly point to upload form
         if not os.path.exists(fpath):
@@ -226,19 +225,19 @@
             else:
                 linktext = _('Upload new attachment "%(filename)s"')
             return wikiutil.link_tag(self.request,
-                '%s?action=AttachFile&amp;rename=%s%s' % (
-                    wikiutil.quoteWikiname(pagename),
-                    urllib.quote_plus(fname),
-                    drawing and ('&amp;drawing=%s' % urllib.quote(drawing)) or ''),
-                linktext % {'filename': fname})
+                self.formatter.text('%s?action=AttachFile&rename=%s%s' % (
+                    wikiutil.quoteWikinameURL(pagename),
+                    urllib.quote_plus(fname.encode(config.charset)),
+                    drawing and ('&drawing=%s' % urllib.quote(drawing.encode(config.charset))) or '')),
+                linktext % {'filename': self.formatter.text(fname)})
 
         # check for image URL, and possibly return IMG tag
         # (images are always inlined, just like for other URLs)
         if not kw.get('pretty_url', 0) and wikiutil.isPicture(url):
             if drawing:
                 # check for map file
-                mappath = os.path.join(AttachFile.getAttachDir(pagename), drawing + '.map')
-                edit_link = '%s?action=AttachFile&amp;rename=%s&amp;drawing=%s' % (wikiutil.quoteWikiname(pagename), urllib.quote_plus(fname), urllib.quote(drawing))
+                mappath = AttachFile.getFilename(self.request, pagename, drawing + '.map')
+                edit_link = self.formatter.text('%s?action=AttachFile&rename=%s&drawing=%s' % (wikiutil.quoteWikinameURL(pagename), urllib.quote_plus(fname.encode(config.charset)), urllib.quote(drawing.encode(config.charset))))
                 if os.path.exists(mappath):
                     # we have a image map. inline it and add a map ref
                     # to the img tag
@@ -255,7 +254,7 @@
                         # add alt and title tags to areas
                         map = re.sub('href\s*=\s*"((?!%TWIKIDRAW%).+?)"',r'href="\1" alt="\1" title="\1"',map)
                         # add in edit links plus alt and title attributes
-                        map = map.replace('%TWIKIDRAW%"', edit_link + '" alt="' + _('Edit drawing %(filename)s') % {'filename': fname} + '" title="' + _('Edit drawing %(filename)s') % {'filename': fname} + '"')
+                        map = map.replace('%TWIKIDRAW%"', edit_link + '" alt="' + _('Edit drawing %(filename)s') % {'filename': self.formatter.text(fname)} + '" title="' + _('Edit drawing %(filename)s') % {'filename': self.formatter.text(fname)} + '"')
                         # unxml, because 4.01 concrete will not validate />
                         map = map.replace('/>','>')
                         return map + self.formatter.image(alt=drawing,
@@ -265,36 +264,51 @@
                         edit_link,
                         self.formatter.image(alt=url,
                             src=AttachFile.getAttachUrl(pagename, url, self.request, addts=1), html_class="drawing"),
-                        attrs='title="%s"' % (_('Edit drawing %(filename)s') % {'filename': fname}))
+                        attrs='title="%s"' % (_('Edit drawing %(filename)s') % {'filename': self.formatter.text(fname)}))
             else:
                 return self.formatter.image(alt=url,
                     src=AttachFile.getAttachUrl(pagename, url, self.request, addts=1))
 
-        # try to inline the attachment (we only accept a list
-        # of known extensions)
+        # try to inline the attachment (parser know what they
+        # can handle)
         base, ext = os.path.splitext(url)
-        if inline and ext in ['.py']:
-            if ext == '.py':
-                import cStringIO
-                from MoinMoin.parser import python
-
-                buff = cStringIO.StringIO()
-                colorizer = python.Parser(open(fpath, 'r').read(), self.request, out = buff)
+        if inline:
+            Parser = wikiutil.getParserForExtension(self.cfg, ext)
+            if Parser is not None:
+                content = file(fpath, 'r').read()
+                # Try to decode text. It might return junk, but we don't
+                # have enough information with attachments.
+                content = wikiutil.decodeUnknownInput(content)
+                colorizer = Parser(content, self.request)
                 colorizer.format(self.formatter)
-                return self.formatter.preformatted(1) + \
-                    self.formatter.rawHTML(buff.getvalue()) + \
-                    self.formatter.preformatted(0)
-
-        return self.formatter.url(
-            AttachFile.getAttachUrl(pagename, url, self.request),
-            text, pretty_url=kw.get('pretty_url', 0))
 
+        url = AttachFile.getAttachUrl(pagename, url, self.request)
+
+        if kw.get('pretty_url', 0) and wikiutil.isPicture(url):
+            return self.formatter.image(src=url)
+        else:
+            return (self.formatter.url(1, url) +
+                    self.formatter.text(text) +
+                    self.formatter.url(0))
 
     def _u_repl(self, word):
         """Handle underline."""
         self.is_u = not self.is_u
         return self.formatter.underline(self.is_u)
 
+    def _small_repl(self, word):
+        """Handle small."""
+        if word.strip() == '~-' and self.is_small: return word
+        if word.strip() == '-~' and not self.is_small: return word
+        self.is_small = not self.is_small
+        return self.formatter.small(self.is_small)
+
+    def _big_repl(self, word):
+        """Handle big."""
+        if word.strip() == '~+' and self.is_big: return word
+        if word.strip() == '+~' and not self.is_big: return word
+        self.is_big = not self.is_big
+        return self.formatter.big(self.is_big)
 
     def _emph_repl(self, word):
         """Handle emphasis, i.e. '' and '''."""
@@ -337,25 +351,27 @@
     def _sup_repl(self, word):
         """Handle superscript."""
         return self.formatter.sup(1) + \
-            self.highlight_text(word[1:-1]) + \
+            self.formatter.text(word[1:-1]) + \
             self.formatter.sup(0)
 
 
     def _sub_repl(self, word):
         """Handle subscript."""
         return self.formatter.sub(1) + \
-            self.highlight_text(word[2:-2]) + \
+            self.formatter.text(word[2:-2]) + \
             self.formatter.sub(0)
 
 
     def _rule_repl(self, word):
         """Handle sequences of dashes."""
-        self.inhibit_p = 1
-        result = self._undent()
+        ##self.inhibit_p = 1
+        result = self._undent() + self._closeP()
         if len(word) <= 4:
             result = result + self.formatter.rule()
         else:
-            result = result + self.formatter.rule(min(len(word), 10) - 2)
+            # Create variable rule size 1 - 6. Actual size defined in css.
+            size = min(len(word), 10) - 4
+            result = result + self.formatter.rule(size)
         return result
 
 
@@ -372,20 +388,17 @@
         if not text:
             # if a simple, self-referencing link, emit it as plain text
             if word == self.formatter.page.page_name:
-                return word
+                return self.formatter.text(word)
             text = word
         if config.allow_subpages and word.startswith(wikiutil.CHILD_PREFIX):
             word = self.formatter.page.page_name + word
-        text = self.highlight_text(text)
-        if word == text:
-            return self.formatter.pagelink(word)
-        else:
-            return self.formatter.pagelink(word, text)
+        return (self.formatter.pagelink(1, word) +
+                self.formatter.text(text) +
+                self.formatter.pagelink(0, word))
 
     def _notword_repl(self, word):
         """Handle !NotWikiNames."""
-        return self.highlight_text(word[1:])
-
+        return self.formatter.text(word[1:])
 
     def _interwiki_repl(self, word):
         """Handle InterWiki links."""
@@ -400,7 +413,15 @@
         if scheme in self.attachment_schemas:
             return self.attachment([word])
 
-        return self.formatter.url(word, text=self.highlight_text(word))
+        if wikiutil.isPicture(word):
+            # Get image name http://here.com/dir/image.gif -> image
+            name = word.split('/')[-1]
+            name = ''.join(name.split('.')[:-1])
+            return self.formatter.image(src=word, alt=name)
+        else:
+            return (self.formatter.url(1, word, type='www') +
+                    self.formatter.text(word) +
+                    self.formatter.url(0))
 
 
     def _wikiname_bracket_repl(self, word):
@@ -427,7 +448,9 @@
 
         if words[0][0] == '#':
             # anchor link
-            return self.formatter.url(words[0], self.highlight_text(words[1]))
+            return (self.formatter.url(1, words[0]) +
+                    self.formatter.text(words[1]) +
+                    self.formatter.url(0))
 
         scheme = words[0].split(":", 1)[0]
         if scheme == "wiki": return self.interwiki(words, pretty_url=1)
@@ -435,16 +458,21 @@
             return self.attachment(words, pretty_url=1)
 
         if wikiutil.isPicture(words[1]) and re.match(self.url_rule, words[1]):
-            text = self.formatter.image(title=words[0], alt=words[0], src=words[1])
+            return (self.formatter.url(1, words[0], 'external', unescaped=1) +
+                    self.formatter.image(title=words[0], alt=words[0], src=words[1]) +
+                    self.formatter.url(0))
         else:
-            text = web.getLinkIcon(self.request, self.formatter, scheme)
-            text += self.highlight_text(words[1])
-        return self.formatter.url(words[0], text, 'external', pretty_url=1, unescaped=1)
+            return (self.formatter.url(1, words[0], 'external',
+                                       type='www', unescaped=1) +
+                    self.formatter.text(words[1]) +
+                    self.formatter.url(0))
 
 
     def _email_repl(self, word):
         """Handle email addresses (without a leading mailto:)."""
-        return self.formatter.url("mailto:" + word, self.highlight_text(word))
+        return (self.formatter.url(1, "mailto:" + word, type='mailto') +
+                self.formatter.text(word) +
+                self.formatter.url(0))
 
 
     def _ent_repl(self, word):
@@ -463,15 +491,22 @@
     def _li_repl(self, match):
         """Handle bullet lists."""
         result = []
+        indented_only = (match == (" " * len(match)))
+        if indented_only and self.in_li: return ''
+            
         self._close_item(result)
-        self.inhibit_p = 1
+        #self.inhibit_p = 1
         self.in_li = 1
         css_class = ''
         if self.line_was_empty and not self.first_list_item:
             css_class = 'gap'
-        result.append(" "*4*self._indent_level())
-        result.append(self.formatter.listitem(1, css_class=css_class))
-        result.append(self.formatter.paragraph(1))
+        if indented_only:
+            result.append(self.formatter.listitem(1, css_class=css_class,
+                                             style="list-style-type:none"))
+        else:
+            result.append(self.formatter.listitem(1, css_class=css_class))
+        # Suspected p!
+        ## result.append(self.formatter.paragraph(1))
         return ''.join(result)
 
 
@@ -484,15 +519,15 @@
         """Handle definition lists."""
         result = []
         self._close_item(result)
-        self.inhibit_p = 1
+        #self.inhibit_p = 1
         self.in_dd = 1
         result.extend([
-            " "*4*self._indent_level(),
             self.formatter.definition_term(1),
-            self.formatter.text(match[:-3]),
+            self.formatter.text(match[1:-3]),
             self.formatter.definition_term(0),
             self.formatter.definition_desc(1),
-            self.formatter.paragraph(1)
+            ## CHANGE: no automatic paragraph
+            ##self.formatter.paragraph(1)
         ])
         return ''.join(result)
 
@@ -507,73 +542,72 @@
         open = []   # don't make one out of these two statements!
         close = []
 
-        # Close open paragraphs and list items
-        if self._indent_level() != new_level:
-            self._close_item(close)
-        else:
-            if not self.line_was_empty:                                                                                                       
-                self.inhibit_p = 1                                                                                                            
+
+        if self._indent_level() != new_level and self.in_table:
+            close.append(self.formatter.table(0))
+            self.in_table = 0
+        #    #self._close_item(close)
+        #else:
+        #    if not self.line_was_empty:
+        #        self.inhibit_p = 1
     
         # Close lists while char-wise indent is greater than the current one
-        while self._indent_level() > new_level:
-            indentstr = " "*4*self._indent_level()
+        while ((self._indent_level() > new_level) or
+               ( new_level and
+                (self._indent_level() == new_level) and
+                (self.list_types[-1]) != list_type)):
+            self._close_item(close)
             if self.list_types[-1] == 'ol':
                 tag = self.formatter.number_list(0)
             elif self.list_types[-1] == 'dl':
                 tag = self.formatter.definition_list(0)
             else:
                 tag = self.formatter.bullet_list(0)
-            close.append("\n%s%s\n" % (indentstr, tag))
+            close.append(tag)
 
             del(self.list_indents[-1])
             del(self.list_types[-1])
             
-            if new_level:
-                self.inhibit_p = 1
-            else:
-                self.inhibit_p = 0
+            #if new_level:
+            #    self.inhibit_p = 1
+            #else:
+            #    self.inhibit_p = 0
+
+            if self.list_types: # we are still in a list
+                if self.list_types[-1] == 'dl':
+                    self.in_dd = 1
+                else:
+                    self.in_li = 1
                 
-            # XXX This would give valid, but silly looking html.
-            # the right way is that inner list has to be CONTAINED in outer li -
-            # but in the one before, not a new one, like this code does:
-            #if self.list_types: # we are still in a list, bracket with li /li
-            #    if self.list_types[-1] in ['ol', 'ul']:
-            #        open.append(" "*4*new_level)
-            #        open.append(self.formatter.listitem(0))
-            #    elif self.list_types[-1] == 'dl':
-            #        open.append(" "*4*new_level)
-            #        open.append(self.formatter.definition_desc(0))
-
         # Open new list, if necessary
         if self._indent_level() < new_level:
-            # XXX see comment 10 lines above
-            #if self.list_types: # we already are in a list, bracket with li /li
-            #    if self.list_types[-1] in ['ol', 'ul']:
-            #        open.append(" "*4*new_level)
-            #        open.append(self.formatter.listitem(1))
-            #    elif self.list_types[-1] == 'dl':
-            #        open.append(" "*4*new_level)
-            #        open.append(self.formatter.definition_desc(1))
                     
             self.list_indents.append(new_level)
             self.list_types.append(list_type)
+
+            if self.formatter.in_p:
+                close.append(self.formatter.paragraph(0))
             
-            indentstr = " "*4*new_level
             if list_type == 'ol':
                 tag = self.formatter.number_list(1, numtype, numstart)
             elif list_type == 'dl':
                 tag = self.formatter.definition_list(1)
             else:
                 tag = self.formatter.bullet_list(1)
-            open.append("\n%s%s\n" % (indentstr, tag))
+            open.append(tag)
             
             self.first_list_item = 1
-            self.inhibit_p = 1
-            
+            ## Maybe this prevent p creation in lists?
+            ##self.inhibit_p = 1
+            self.in_li = 0
+            self.in_dd = 0
         # If list level changes, close an open table
         if self.in_table and (open or close):
             close[0:0] = [self.formatter.table(0)]
             self.in_table = 0
+        
+        ## Maybe this prevent p creation in lists?
+        ##self.inhibit_p = bool(self.list_types)
 
         return ''.join(close) + ''.join(open)
 
@@ -599,7 +633,7 @@
     def _tt_repl(self, word):
         """Handle inline code."""
         return self.formatter.code(1) + \
-            self.highlight_text(word[3:-3]) + \
+            self.formatter.text(word[3:-3]) + \
             self.formatter.code(0)
 
 
@@ -607,7 +641,7 @@
         """Handle backticked inline code."""
         if len(word) == 2: return ""
         return self.formatter.code(1) + \
-            self.highlight_text(word[1:-1]) + \
+            self.formatter.text(word[1:-1]) + \
             self.formatter.code(0)
 
 
@@ -689,22 +723,32 @@
     def _tableZ_repl(self, word):
         """Handle table row end."""
         if self.in_table:
-            return self.formatter.table_cell(0) + self.formatter.table_row(0)
+            result = ''
+            # REMOVED: check for self.in_li, p should always close
+            if self.formatter.in_p:
+                result = self.formatter.paragraph(0)
+            result += self.formatter.table_cell(0) + self.formatter.table_row(0)
+            return result
         else:
             return word
 
     def _table_repl(self, word):
         """Handle table cell separator."""
         if self.in_table:
+            result = []
             # check for attributes
             attrs, attrerr = self._getTableAttrs(word)
 
             # start the table row?
             if self.table_rowstart:
                 self.table_rowstart = 0
-                leader = self.formatter.table_row(1, attrs)
+                result.append(self.formatter.table_row(1, attrs))
             else:
-                leader = self.formatter.table_cell(0)
+                # Close table cell, first closing open p
+                # REMOVED check for self.in_li, paragraph should close always!
+                if self.formatter.in_p:
+                    result.append(self.formatter.paragraph(0))
+                result.append(self.formatter.table_cell(0))
 
             # check for adjacent cell markers
             if word.count("|") > 2:
@@ -713,8 +757,9 @@
                 if not attrs.has_key('colspan'):
                     attrs['colspan'] = '"%d"' % (word.count("|")/2)
 
-            # return the complete cell markup           
-            return leader + self.formatter.table_cell(1, attrs) + attrerr
+            # return the complete cell markup
+            result.append(self.formatter.table_cell(1, attrs) + attrerr)         
+            return ''.join(result) 
         else:
             return word
 
@@ -723,13 +768,7 @@
         """Handle section headings."""
         import sha
 
-        self.inhibit_p = 1
-        icons = ''
-        if self.request.user.show_topbottom:
-            bottom = self.request.theme.make_icon('bottom')
-            icons = icons + self.formatter.url("#bottom", bottom, unescaped=1)
-            top = self.request.theme.make_icon('top')
-            icons = icons + self.formatter.url("#top", top, unescaped=1)
+        ##self.inhibit_p = 1
 
         h = word.strip()
         level = 1
@@ -737,46 +776,57 @@
             level = level+1
         depth = min(5,level)
 
+        # this is needed for Included pages
+        # TODO but it might still result in unpredictable results
+        # when included the same page multiple times
         title_text = h[level:-level].strip()
-        self.titles.setdefault(title_text, 0)
-        self.titles[title_text] += 1
+        pntt = self.formatter.page.page_name + title_text
+        self.titles.setdefault(pntt, 0)
+        self.titles[pntt] += 1
 
         unique_id = ''
-        if self.titles[title_text] > 1:
-            unique_id = '-%d' % self.titles[title_text]
-
-        return self.formatter.heading(depth, self.highlight_text(title_text), icons=icons, id="head-"+sha.new(title_text).hexdigest()+unique_id)
-
-
+        if self.titles[pntt] > 1:
+            unique_id = '-%d' % self.titles[pntt]
+        result = self._closeP()
+        result += self.formatter.heading(1, depth, id="head-"+sha.new(pntt.encode(config.charset)).hexdigest()+unique_id)
+                                     
+        return (result + self.formatter.text(title_text) +
+                self.formatter.heading(0, depth))
+    
     def _processor_repl(self, word):
         """Handle processed code displays."""
         if word[:3] == '{{{': word = word[3:]
 
         self.processor = None
         self.processor_name = None
+        self.processor_is_parser = 0
         s_word = word.strip()
         if s_word == '#!':
             # empty bang paths lead to a normal code display
             # can be used to escape real, non-empty bang paths
             word = ''
             self.in_pre = 3
-            return  self.formatter.preformatted(1)
+            return self._closeP() + self.formatter.preformatted(1)
         elif s_word[:2] == '#!':
+            # first try to find a processor for this (will go away in 1.4)
             processor_name = s_word[2:].split()[0]
-            self.processor = wikiutil.importPlugin("processor", processor_name, "process")
-            if not self.processor and s_word.find('python') > 0:
-                from MoinMoin.processor.Colorize import process
-                self.processor = process
-                self.processor_name = "Colorize"
+            self.processor = wikiutil.importPlugin(
+                self.request.cfg, "processor", processor_name, "process")
+            # now look for a parser with that name
+            if self.processor is None:
+                self.processor = wikiutil.importPlugin(
+                    self.request.cfg, "parser", processor_name, "Parser")
+                if self.processor:
+                    self.processor_is_parser = 1
 
         if self.processor:
             self.processor_name = processor_name
             self.in_pre = 2
             self.colorize_lines = [word]
             return ''
-        elif  s_word:
+        elif s_word:
             self.in_pre = 3
-            return self.formatter.preformatted(1) + \
+            return self._closeP() + self.formatter.preformatted(1) + \
                    self.formatter.text(s_word + ' (-)')
         else:
             self.in_pre = 1
@@ -787,17 +837,18 @@
         word = word.strip()
         if word == '{{{' and not self.in_pre:
             self.in_pre = 3
-            return self.formatter.preformatted(self.in_pre)
+            ##self.inhibit_p = 1
+            return self._closeP() + self.formatter.preformatted(self.in_pre)
         elif word == '}}}' and self.in_pre:
             self.in_pre = 0
-            self.inhibit_p = 1
+            self.inhibit_p = 0
             return self.formatter.preformatted(self.in_pre)
         return word
 
 
     def _smiley_repl(self, word):
         """Handle smileys."""
-        return wikiutil.getSmiley(word, self.formatter)
+        return self.formatter.smiley(word)
 
     _smileyA_repl = _smiley_repl
 
@@ -805,7 +856,11 @@
     def _comment_repl(self, word):
         return ''
 
-
+    def _closeP(self):
+        if self.formatter.in_p:
+            return self.formatter.paragraph(0)
+        return ''
+        
     def _macro_repl(self, word):
         """Handle macros ([[macroname]])."""
         macro_name = word[2:-2]
@@ -820,71 +875,63 @@
         # create macro instance
         if self.macro is None:
             self.macro = wikimacro.Macro(self)
-
-        # call the macro
         return self.formatter.macro(self.macro, macro_name, args)
 
-
-    def highlight_text(self, text, **kw):
-        if not self.hilite_re: return self.formatter.text(text)
-        
-        # work around for dom/xml formatter
-        # if not self.hilite_re: return text
-        # XXX bad idea: this allowed `<b>raw html</b>` to get through!
-        
-        result = []
-        lastpos = 0
-        match = self.hilite_re.search(text)
-        while match and lastpos < len(text):
-            # add the match we found
-            result.append(self.formatter.text(text[lastpos:match.start()]))
-            result.append(self.formatter.highlight(1))
-            result.append(self.formatter.text(match.group(0)))
-            result.append(self.formatter.highlight(0))
-
-            # search for the next one
-            lastpos = match.end() + (match.end() == lastpos)
-            match = self.hilite_re.search(text, lastpos)
-
-        result.append(self.formatter.text(text[lastpos:]))
-        return ''.join(result)
-
     def scan(self, scan_re, line):
-        """ scans the line for wiki syntax and replaces the
-            found regular expressions
-            calls highlight_text if self.hilite_re is set
+        """ Scans one line
+        
+        Append text before match, invoke replace() with match, and 
+        add text after match.
         """
         result = []
         lastpos = 0
-        match = scan_re.search(line)
-        while match and lastpos < len(line):
-            # add the match we found
-            if self.hilite_re:
-                result.append(self.highlight_text(line[lastpos:match.start()]))
-            else:
+
+        ###result.append(u'<span class="info">[scan: <tt>"%s"</tt>]</span>' % line)
+      
+        for match in scan_re.finditer(line):
+            # Add text before the match
+            if lastpos < match.start():
+                
+                ###result.append(u'<span class="info">[add text before match: <tt>"%s"</tt>]</span>' % line[lastpos:match.start()])
+                
+                if not (self.inhibit_p or self.in_pre or self.formatter.in_p):
+                    result.append(self.formatter.paragraph(1))
                 result.append(self.formatter.text(line[lastpos:match.start()]))
+            
+            # Replace match with markup
             result.append(self.replace(match))
-
-            # search for the next one
-            lastpos = match.end() + (match.end() == lastpos)
-            match = scan_re.search(line, lastpos)
-
-        if self.hilite_re:
-            result.append(self.highlight_text(line[lastpos:]))
-        else:
-            result.append(self.formatter.text(line[lastpos:]))
-        return ''.join(result)
+            lastpos = match.end()
+        
+        ###result.append('<span class="info">[no match, add rest: <tt>"%s"<tt>]</span>' % line[lastpos:])
+        
+        # No match: Add paragraph with the text of the line
+        if not (self.in_pre or self.inhibit_p or
+                self.formatter.in_p) and lastpos < len(line):
+            result.append(self.formatter.paragraph(1))
+        result.append(self.formatter.text(line[lastpos:]))
+        return u''.join(result)
 
     def replace(self, match):
-        #hit = filter(lambda g: g[1], match.groupdict().items())
+        """ Replace match using type name """
+        result = []
         for type, hit in match.groupdict().items():
             if hit is not None and type != "hmarker":
-                ##print "###", cgi.escape(`type`), cgi.escape(`hit`), "###"
+                
+                ###result.append(u'<span class="info">[replace: %s: "%s"]</span>' % (type, hit))
                 if self.in_pre and type not in ['pre', 'ent']:
-                    return self.highlight_text(hit)
+                    return self.formatter.text(hit) 
                 else:
-                    return getattr(self, '_' + type + '_repl')(hit)
+                    # Open p for certain types
+                    if not (self.inhibit_p or self.formatter.in_p
+                            or self.in_pre or (type in self.no_new_p_before)):
+                        result.append(self.formatter.paragraph(1))
+                    
+                    # Get replace method and replece hit
+                    replace = getattr(self, '_' + type + '_repl')
+                    result.append(replace(hit))
+                    return ''.join(result)
         else:
+            # We should never get here
             import pprint
             raise Exception("Can't handle match " + `match`
                 + "\n" + pprint.pformat(match.groupdict())
@@ -892,7 +939,6 @@
 
         return ""
 
-
     def format(self, formatter):
         """ For each line, scan through looking for magic
             strings, outputting verbatim any intervening text.
@@ -902,23 +948,25 @@
 
         # prepare regex patterns
         rules = self.formatting_rules.replace('\n', '|')
-        if config.allow_extended_names:
-            rules = rules + r'|(?P<wikiname_bracket>\[".*?"\])'
-        if config.bang_meta:
-            rules = r'(?P<notword>!%(word_rule)s)|%(rules)s' % {
+        if self.cfg.allow_extended_names:
+            rules = rules + ur'|(?P<wikiname_bracket>\[".*?"\])'
+        if self.cfg.bang_meta:
+            rules = ur'(?P<notword>!%(word_rule)s)|%(rules)s' % {
                 'word_rule': self.word_rule,
                 'rules': rules,
             }
-        if config.backtick_meta:
-            rules = rules + r'|(?P<tt_bt>`.*?`)'
-        if config.allow_numeric_entities:
-            rules = r'(?P<ent_numeric>&#\d{1,5};)|' + rules
-
-        scan_re = re.compile(rules)
-        number_re = re.compile(self.ol_rule)
-        term_re = re.compile(self.dl_rule)
-        indent_re = re.compile("^\s*")
-        eol_re = re.compile(r'\r?\n')
+        if self.cfg.backtick_meta:
+            rules = rules + ur'|(?P<tt_bt>`.*?`)'
+        if self.cfg.allow_numeric_entities:
+            rules = ur'(?P<ent_numeric>&#\d{1,5};)|' + rules
+
+        self.request.clock.start('compile_huge_and_ugly')        
+        scan_re = re.compile(rules, re.UNICODE)
+        number_re = re.compile(self.ol_rule, re.UNICODE)
+        term_re = re.compile(self.dl_rule, re.UNICODE)
+        indent_re = re.compile("^\s*", re.UNICODE)
+        eol_re = re.compile(r'\r?\n', re.UNICODE)
+        self.request.clock.stop('compile_huge_and_ugly')        
 
         # get text and replace TABs
         rawtext = self.raw.expandtabs()
@@ -928,6 +976,7 @@
         self.lines = eol_re.split(rawtext)
         self.line_is_empty = 0
 
+        # Main loop
         for line in self.lines:
             self.lineno = self.lineno + 1
             self.table_rowstart = 1
@@ -937,25 +986,32 @@
             self.inhibit_p = 0
 
             if self.in_pre:
+                # TODO: move this into function
                 # still looking for processing instructions
+                # TODO: use strings for pre state, not numbers
                 if self.in_pre == 1:
                     self.processor = None
+                    self.processor_is_parser = 0
                     processor_name = ''
                     if (line.strip()[:2] == "#!"):
-                        from MoinMoin.processor import processors
                         processor_name = line.strip()[2:].split()[0]
-                        self.processor = wikiutil.importPlugin("processor", processor_name, "process")
-                        if not self.processor and (line.find('python') > 0):
-                            from MoinMoin.processor.Colorize import process
-                            self.processor = process
-                            processor_name = "Colorize"
+                        self.processor = wikiutil.importPlugin(
+                            self.request.cfg, "processor", processor_name, "process")
+                                                               
+                        # now look for a parser with that name
+                        if self.processor is None:
+                            self.processor = wikiutil.importPlugin(
+                                self.request.cfg, "parser", processor_name, "Parser") 
+                            if self.processor:
+                                self.processor_is_parser = 1
                     if self.processor:
                         self.in_pre = 2
                         self.colorize_lines = [line]
                         self.processor_name = processor_name
                         continue
                     else:
-                        self.request.write(self.formatter.preformatted(1))
+                        self.request.write(self._closeP() +
+                                           self.formatter.preformatted(1))
                         self.in_pre = 3
                 if self.in_pre == 2:
                     # processing mode
@@ -965,8 +1021,14 @@
                         continue
                     if line[:endpos]:
                         self.colorize_lines.append(line[:endpos])
-                    self.request.write(
-                        self.formatter.processor(self.processor_name, self.colorize_lines))
+                    
+                    # Close p before calling processor
+                    # TODO: do we really need this?
+                    self.request.write(self._closeP())
+                    res = self.formatter.processor(self.processor_name,
+                                                   self.colorize_lines, 
+                                                   self.processor_is_parser)
+                    self.request.write(res)
                     del self.colorize_lines
                     self.in_pre = 0
                     self.processor = None
@@ -974,19 +1036,23 @@
                     # send rest of line through regex machinery
                     line = line[endpos+3:]                    
             else:
-                # paragraph break on empty lines
+                # we don't have \n as whitespace any more
+                # This is the space between lines we join to one paragraph
+                line = line + ' '
+                
+                # Paragraph break on empty lines
                 if not line.strip():
-                    #self.request.write("<!-- empty line start -->\n")
-                    if self.formatter.in_p:
-                        self.request.write(self.formatter.paragraph(0))
                     if self.in_table:
                         self.request.write(self.formatter.table(0))
                         self.in_table = 0
+                    # CHANGE: removed check for not self.list_types
+                    # p should close on every empty line
+                    if (self.formatter.in_p):
+                        self.request.write(self.formatter.paragraph(0))
                     self.line_is_empty = 1
-                    #self.request.write("<!-- empty line end -->\n")
                     continue
 
-                # check indent level
+                # Check indent level
                 indent = indent_re.match(line)
                 indlen = len(indent.group(0))
                 indtype = "ul"
@@ -1010,45 +1076,50 @@
                             indtype = "dl"
 
                 # output proper indentation tags
-                #self.request.write("<!-- inhibit_p==%d -->\n" % self.inhibit_p)
-                #self.request.write("<!-- #%d calling _indent_to -->\n" % self.lineno)
-                self.request.write(self._indent_to(indlen, indtype, numtype, numstart))
-                #self.request.write("<!-- #%d after calling _indent_to -->\n" % self.lineno)
-                #self.request.write("<!-- inhibit_p==%d -->\n" % self.inhibit_p)
+                self.request.write(self._indent_to(indlen, indtype, numtype,
+                                                   numstart))
 
-                # start or end table mode
-                if not self.in_table and line[indlen:indlen+2] == "||" and line[-2:] == "||":
+                # Table mode
+                # TODO: move into function?                
+                if (not self.in_table and line[indlen:indlen + 2] == "||"
+                    and line[-3:] == "|| " and len(line) >= 5 + indlen):
+                    # Start table
+                    if self.list_types and not self.in_li:
+                        self.request.write(self.formatter.listitem
+                                           (1, style="list-style-type:none"))
+                        ## CHANGE: no automatic p on li
+                        ##self.request.write(self.formatter.paragraph(1))
+                        self.in_li = 1
+                        
+                    # CHANGE: removed check for self.in_li
+                    # paragraph should end before table, always!
+                    if self.formatter.in_p:
+                        self.request.write(self.formatter.paragraph(0))
                     attrs, attrerr = self._getTableAttrs(line[indlen+2:])
                     self.request.write(self.formatter.table(1, attrs) + attrerr)
-                    self.in_table = self.lineno
-                elif self.in_table and not(line[:2]=="##" or # intra-table comments should not break a table 
-                    line[indlen:indlen+2] == "||" and line[-2:] == "||"):
+                    self.in_table = True # self.lineno
+                elif (self.in_table and not
+                      # intra-table comments should not break a table
+                      (line[:2]=="##" or  
+                       line[indlen:indlen + 2] == "||" and
+                       line[-3:] == "|| " and
+                       len(line) >= 5 + indlen)):
+                    
+                    # Close table
                     self.request.write(self.formatter.table(0))
                     self.in_table = 0
-
-            # convert line from wiki markup to HTML and print it
-            if not self.in_pre:   # we don't want to have trailing blanks in pre
-                line = line + " " # we don't have \n as whitespace any more
-
-            formatted_line = self.scan(scan_re, line) # this also sets self.inhibit_p as side effect!
-            
-            #self.request.write("<!-- inhibit_p==%d -->\n" % self.inhibit_p)
-            if not (self.inhibit_p or self.in_pre or self.in_table or self.formatter.in_p):
-                self.request.write(self.formatter.paragraph(1))
-
-            #self.request.write("<!-- %s\n     start -->\n" % line)
+                                            
+            # Scan line, format and write
+            formatted_line = self.scan(scan_re, line)
             self.request.write(formatted_line)
-            #self.request.write("<!-- end -->\n")
 
             if self.in_pre:
                 self.request.write(self.formatter.linebreak())
-            #if self.in_li:
-            #    self.in_li = 0
-            #    self.request.write(self.formatter.listitem(0))
 
-        # close code displays, paragraphs, tables and open lists
+        # Close code displays, paragraphs, tables and open lists
+        self.request.write(self._undent())
         if self.in_pre: self.request.write(self.formatter.preformatted(0))
         if self.formatter.in_p: self.request.write(self.formatter.paragraph(0))
         if self.in_table: self.request.write(self.formatter.table(0))
-        self.request.write(self._undent())
+