diff -u -r -P /home/wr/install/moin-1.3/MoinMoin/action/AttachFile.py ./action/AttachFile.py --- /home/wr/install/moin-1.3/MoinMoin/action/AttachFile.py 2004-08-18 07:00:26.000000000 +0200 +++ ./action/AttachFile.py 2004-08-25 15:59:38.727164243 +0200 @@ -51,19 +51,33 @@ """ Get directory where attachments for page `pagename` are stored. """ if htdocs_access(request): + print "a" # direct file access via webserver, from public htdocs area pagename = wikiutil.quoteWikinameFS(pagename) attach_dir = os.path.join(request.cfg.attachments['dir'], pagename, "attachments") else: + print "b" # send file via CGI, from page storage area attach_dir = wikiutil.getPagePath(request, pagename, "attachments", check_create=create) if create and not os.path.isdir(attach_dir): filesys.makeDirs(attach_dir) + print "attach_dir=",attach_dir return attach_dir +def getTextVersionDir(request, pagename, create=0): + """ Get directory where the converted text version is stored. + It is for PageName: data/cache/AttachSearch/PageName/filename + """ + txt_dir = os.path.join(request.cfg.data_dir, "cache", "AttachSearch", pagename) + + if create and not os.path.isdir(txt_dir): + filesys.makeDirs(txt_dir) + return txt_dir + + def getAttachUrl(pagename, filename, request, addts=0): """ Get URL that points to attachment `filename` of page `pagename`. diff -u -r -P /home/wr/install/moin-1.3/MoinMoin/action/fullsearch.py ./action/fullsearch.py --- /home/wr/install/moin-1.3/MoinMoin/action/fullsearch.py 2004-08-18 07:00:31.000000000 +0200 +++ ./action/fullsearch.py 2004-08-23 16:33:20.000000000 +0200 @@ -38,7 +38,8 @@ query = search.QueryParser(literal=request.form.has_key('literal'), case=case).parse_query(needle) - hits = search.searchPages(request, query) + # hits = search.searchPages(request, query) + hits = search.searchEverything(request, query) search.sort_by_weight(hits) formatter = Formatter(request) diff -u -r -P /home/wr/install/moin-1.3/MoinMoin/attach2txt/__init__.py ./attach2txt/__init__.py --- /home/wr/install/moin-1.3/MoinMoin/attach2txt/__init__.py 1970-01-01 01:00:00.000000000 +0100 +++ ./attach2txt/__init__.py 2004-08-24 10:49:38.000000000 +0200 @@ -0,0 +1,16 @@ +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - Attachment converter package + + @copyright: 2004 Willi Richert + @license: GNU GPL, see COPYING for details. +""" + + +from MoinMoin.util import pysupport +modules = pysupport.getPackageModules(__file__) +print "modules in init.py:", modules + +import pdf2txt + +converter_mapping = {"pdf":pdf2txt.convert} diff -u -r -P /home/wr/install/moin-1.3/MoinMoin/attach2txt/pdf2txt.py ./attach2txt/pdf2txt.py --- /home/wr/install/moin-1.3/MoinMoin/attach2txt/pdf2txt.py 1970-01-01 01:00:00.000000000 +0100 +++ ./attach2txt/pdf2txt.py 2004-08-25 16:17:40.954215485 +0200 @@ -0,0 +1,26 @@ +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - pdf to txt converter. + + You will need pdftotext (xpdf package) in the PATH. Works only on Linux. + + @copyright: 2004 by Willi Richert (w.richert@gmx.net) + @license: GNU GPL, see COPYING for details. +""" + +import os, mimetypes, time, urllib +from MoinMoin import config, user, util, wikiutil +from MoinMoin.Page import Page +from MoinMoin.util import MoinMoinNoFooter, filesys + +converter_name = __name__.split('.')[-1] + +def convert(att_fn, txt_fn): + print att_fn + print 'pdftotext "%s" "%s"'%(att_fn, txt_fn) + ret = os.system("pdftotext \"%s\" \"%s\""%(att_fn, txt_fn)) + if ret != 0: + open(txt_fn, "w").close() # empty the file + return 1 + else: + return 0 diff -u -r -P /home/wr/install/moin-1.3/MoinMoin/Attachment.py ./Attachment.py --- /home/wr/install/moin-1.3/MoinMoin/Attachment.py 1970-01-01 01:00:00.000000000 +0100 +++ ./Attachment.py 2004-08-25 16:14:50.309377429 +0200 @@ -0,0 +1,187 @@ +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - Attachment class. Derived from Page.py. + + @copyright: 2004 by Willi Richert + @license: GNU GPL, see COPYING for details. +""" + +# Imports +import os.path +from MoinMoin import config +#import MoinMoin.util.web +from MoinMoin.logfile import eventlog +from MoinMoin.action import AttachFile +from MoinMoin.attach2txt import converter_mapping +from MoinMoin.wikiutil import quoteWikinameFS + +class Attachment: + """Attachment - Manage an (immutable) attachment associated with a page. + For search, the attachments text versions are saved in + data/cache/AttachSearch/PageName/filename + """ + + def __init__(self, request, att_name, page, **keywords): + """ + Create attachment object. + + @param page_name: WikiName of the associated page + @keyword: ignored + """ + self.request = request + self.att_name = att_name + + self._assoc_page_name = page.page_name + + self.att_filename = os.path.join(AttachFile.getAttachDir(self.request, self._assoc_page_name), self.att_name) + txt_dir = AttachFile.getTextVersionDir(self.request, self._assoc_page_name, create=1) + self.txt_filename = os.path.join(txt_dir, self.att_name+".txt") + + self.suffix = os.path.splitext(self.att_name)[1][1:] # suffix without the dot + self._raw_body = None + self._raw_body_modified = 0 + self.hilite_re = None + + + + def exists(self): + """ + Does this page exist? + + @rtype: bool + @return: true, if page exists + """ + return os.path.exists(self.att_filename) + + + def size(self): + """ + Get Attachment size. + + @rtype: int + @return: attachment size, 0 for non-existent pages. + """ + if self._raw_body is not None: + return len(self._raw_body) + + try: + return os.path.getsize(self.att_filename) + except EnvironmentError, e: + import errno + if e.errno == errno.ENOENT: return 0 + raise + + def getTextVersion(self, request, create=0): + """ Returns the extracted textual content of the attachment, if possible. + @param att: att string without pagename: e.g. "file.pdf" + @rtype: string + @return: Textual content of the attachment. + """ + # if we've come so far, the attachment dir does exist together with the attachment + #att_dir = AttachFile.getAttachDir(request, self._assoc_page_name) + att_file = self.att_filename + txt_file = self.txt_filename + + alreadyConverted=1 + try: + print "considering:",att_file,txt_file + if not os.path.isfile(txt_file): + alreadyConverted=0 + print 1 + else: + att_ctime = os.path.getmtime(att_file) + txt_ctime = os.path.getmtime(txt_file) + + if att_ctime>txt_file: + alreadyConverted=0 + print 2 + print 3 + except os.error: + alreadyConverted=0 + print 4 + + if not alreadyConverted: + # we have to convert this attachment if the proper tools are available + print "We have to convert the attachment %s"%att_file + convertResult = converter_mapping[self.suffix](att_file, txt_file) + else: + convertResult = 0 + + if alreadyConverted or convertResult==0: + file = open(txt_file, "r") + content = file.read() + return content + else: + return "" + + + def get_raw_body(self): + """ + Load the raw textual version of the attachment. None if not convertable. + + @rtype: string + @return: raw text contents of this attachment + """ + + if self._raw_body is None: + att = self.getTextVersion(self.request, self.att_name) + self.set_raw_body(att) + + return self._raw_body + + + def set_raw_body(self, body, modified=0): + """ + Set the raw body text (prevents loading from disk). + + @param body: raw body text + @param modified: 1 means that we internally modified the raw text and + that it is not in sync with the page file on disk. + This is used e.g. by PageEditor when previewing the page. + """ + self._raw_body = body + self._raw_body_modified = modified + + def link_to(self, request, text=None, querystr=None, anchor=None, **kw): + """ + Return HTML markup that links to this attachment. + See wikiutil.link_tag() for possible keyword parameters. + + @param request: the request object + @param text: inner text of the link + @param querystr: the query string to add after a "?" after the url + @param anchor: if specified, make a link to this anchor + @keyword on: opening/closing tag only + @keyword attachment_indicator: if 1, add attachment indicator after link tag + @keyword css_class: css class to use + @rtype: string + @return: formatted link + """ + text = text or self.split_title(request) + fmt = getattr(self, 'formatter', None) + + url = wikiutil.quoteWikinameURL(self.page_name) + if querystr: + querystr = util.web.makeQueryString(querystr) + url = "%s?%s" % (url, querystr) + if anchor: url = "%s#%s" % (url, urllib.quote_plus(anchor.encode(config.charset))) + + # create a link to attachments if any exist + attach_link = '' + if kw.get('attachment_indicator', 0): + from MoinMoin.action import AttachFile + attach_link = AttachFile.getIndicator(request, self.page_name) + + if self.exists(): + return wikiutil.link_tag(request, url, text, formatter=fmt, **kw) + attach_link + else: + kw['css_class'] = 'nonexistent' + + if request.user.show_nonexist_qm: + return wikiutil.link_tag(request, url, + '?', formatter=fmt, **kw) + text + attach_link + else: + return wikiutil.link_tag(request, url, text, formatter=fmt, **kw) + attach_link + + + AttachFile.getAttachUrl(self._assoc_page_name, self.att_name, request, addts=0) diff -u -r -P /home/wr/install/moin-1.3/MoinMoin/formatter/text_html.py ./formatter/text_html.py --- /home/wr/install/moin-1.3/MoinMoin/formatter/text_html.py 2004-08-18 07:00:33.000000000 +0200 +++ ./formatter/text_html.py 2004-08-24 09:02:59.000000000 +0200 @@ -100,6 +100,14 @@ apply(FormatterBase.pagelink, (self, on, pagename), kw) return Page(self.request, pagename, formatter=self).link_to(self.request, on=on, **kw) + def attachlink(self, on, attname='', **kw): + """ Link to an attachment. + + See wikiutil.link_tag() for possible keyword parameters. + """ + apply(FormatterBase.pagelink, (self, on, attname), kw) + return Attachment(self.request, attname, formatter=self).link_to(self.request, on=on, **kw) + def interwikilink(self, on, interwiki='', pagename='', **kw): if not on: return '' diff -u -r -P /home/wr/install/moin-1.3/MoinMoin/search.py ./search.py --- /home/wr/install/moin-1.3/MoinMoin/search.py 2004-08-18 07:00:25.000000000 +0200 +++ ./search.py 2004-08-25 16:15:01.523658101 +0200 @@ -5,10 +5,12 @@ @license: GNU GPL, see COPYING for details """ -import re, time, sys, urllib +import re, time, sys, urllib, os #sys.path.append('..') from MoinMoin import wikiutil, config from MoinMoin.Page import Page +from MoinMoin.action import AttachFile +from Attachment import Attachment #try: # import xapian @@ -165,11 +167,11 @@ def highlight_re(self): return u"(%s)" % self.pattern - def search(self, page): - body = page.get_raw_body() + def search(self, obj): # obj is page or attachment + body = obj.get_raw_body() pos = 0 - fragments = self.titlesearch.search(page) + fragments = self.titlesearch.search(obj) if fragments is None: fragments = [] while 1: match = self.search_re.search(body, pos) @@ -205,8 +207,11 @@ def highlight_re(self): return u"(%s)" % self.pattern - def search(self, page): - match = self.search_re.search(page.page_name) + def search(self, obj): + if isinstance(obj, Page): + match = self.search_re.search(obj.page_name) + else: + match = self.search_re.search(obj.att_name) if ((self.negated and match) or (not self.negated and not match)): return None @@ -230,10 +235,9 @@ ### Results ############################################################################ - -class FoundPage: - def __init__(self, page_name, matches=[], page=None): - self.page_name = page_name +class FoundObject: + def __init__(self, name, matches=[], page=None): + self.page_name = name self.page = page self._matches = matches @@ -252,11 +256,20 @@ def get_matches(self): return self._matches[:] + +class FoundPage(FoundObject): + pass - -class FoundAttachment(FoundPage): +class FoundAttachment(FoundObject): + """ + The attachments text versions are saved in + data/cache/AttachSearch/PageName/filename + """ + # TODO: needs to be more attachment like pass + + class Match: def __init__(self, start=0, end=0): self.start = start @@ -382,6 +395,40 @@ return hits +def searchEverything(request, query, **kw): + """ + Search the text of all pages and their attachment's content for query. + @param query: the expression we want to search for + @rtype: list + @return: List of FoundPage objects + """ + from MoinMoin.Page import Page + + hits = [] + all_pages = wikiutil.getPageList(request.cfg.text_dir) + for page_name in all_pages: + page = Page(request, page_name) + if not request.user.may.read(page_name): + continue + result = query.search(page) + if result: + hits.append(FoundPage(page_name, result)) + + # search now in all attachments of this page + attach_dir = AttachFile.getAttachDir(request, page_name) + if os.path.exists(attach_dir): + att_list = os.listdir(attach_dir) + for att in att_list: + print "searchEverything: att=",att + + result = query.search(Attachment(request, att, page)) # TODO: argument hides as a page + if result: + # TODO: append not the page name but the attachment itself directly be means of + # FoundAttachment (see the class stub above) + hits.append(FoundAttachment(page_name, result)) + + return hits + ############################################################################## ### Sort results