root/calcium/trunk/lib/BeautifulSoup.py

"""Beautiful Soup
Elixir and Tonic
"The Screen-Scraper's Friend"
v2.1.1
http://www.crummy.com/software/BeautifulSoup/

Beautiful Soup parses arbitrarily invalid XML- or HTML-like substance
into a tree representation. It provides methods and Pythonic idioms
that make it easy to search and modify the tree.

A well-formed XML/HTML document will yield a well-formed data
structure. An ill-formed XML/HTML document will yield a
correspondingly ill-formed data structure. If your document is only
locally well-formed, you can use this library to find and process the
well-formed part of it. The BeautifulSoup class has heuristics for
obtaining a sensible parse tree in the face of common HTML errors.

Beautiful Soup has no external dependencies. It works with Python 2.2
and up.

Beautiful Soup defines classes for four different parsing strategies:

 * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific
   language that kind of looks like XML.

 * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid
   or invalid.

 * ICantBelieveItsBeautifulSoup, for parsing valid but bizarre HTML
   that trips up BeautifulSoup.

 * BeautifulSOAP, for making it easier to parse XML documents that use
   lots of subelements containing a single string, where you'd prefer
   they put that string into an attribute (such as SOAP messages).

You can subclass BeautifulStoneSoup or BeautifulSoup to create a
parsing strategy specific to an XML schema or a particular bizarre
HTML document. Typically your subclass would just override
SELF_CLOSING_TAGS and/or NESTABLE_TAGS.
"""
from __future__ import generators

__author__ = "Leonard Richardson (leonardr@segfault.org)"
__version__ = "2.1.1"
__date__ = "$Date: 2004/10/18 00:14:20 $"
__copyright__ = "Copyright (c) 2004-2005 Leonard Richardson"
__license__ = "PSF"

from sgmllib import SGMLParser, SGMLParseError
import types
import re
import sgmllib

#This code makes Beautiful Soup able to parse XML with namespaces
sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')

class NullType(object):

    """Similar to NoneType with a corresponding singleton instance
    'Null' that, unlike None, accepts any message and returns itself.

    Examples:
    >>> Null("send", "a", "message")("and one more",
    ...      "and what you get still") is Null
    True
    """

    def __new__(cls):                    return Null
    def __call__(self, *args, **kwargs): return Null
##    def __getstate__(self, *args):       return Null
    def __getattr__(self, attr):         return Null
    def __getitem__(self, item):         return Null
    def __setattr__(self, attr, value):  pass
    def __setitem__(self, item, value):  pass
    def __len__(self):                   return 0
    # FIXME: is this a python bug? otherwise ``for x in Null: pass``
    #        never terminates...
    def __iter__(self):                  return iter([])
    def __contains__(self, item):        return False
    def __repr__(self):                  return "Null"
Null = object.__new__(NullType)

class PageElement:
    """Contains the navigational information for some part of the page
    (either a tag or a piece of text)"""

    def setup(self, parent=Null, previous=Null):
        """Sets up the initial relations between this element and
        other elements."""
        self.parent = parent
        self.previous = previous
        self.next = Null
        self.previousSibling = Null
        self.nextSibling = Null
        if self.parent and self.parent.contents:
            self.previousSibling = self.parent.contents[-1]
            self.previousSibling.nextSibling = self

    def findNext(self, name=None, attrs={}, text=None):
        """Returns the first item that matches the given criteria and
        appears after this Tag in the document."""
        return self._first(self.fetchNext, name, attrs, text)
    firstNext = findNext

    def fetchNext(self, name=None, attrs={}, text=None, limit=None):
        """Returns all items that match the given criteria and appear
        before after Tag in the document."""
        return self._fetch(name, attrs, text, limit, self.nextGenerator)

    def findNextSibling(self, name=None, attrs={}, text=None):
        """Returns the closest sibling to this Tag that matches the
        given criteria and appears after this Tag in the document."""
        return self._first(self.fetchNextSiblings, name, attrs, text)
    firstNextSibling = findNextSibling

    def fetchNextSiblings(self, name=None, attrs={}, text=None, limit=None):
        """Returns the siblings of this Tag that match the given
        criteria and appear after this Tag in the document."""
        return self._fetch(name, attrs, text, limit, self.nextSiblingGenerator)

    def findPrevious(self, name=None, attrs={}, text=None):
        """Returns the first item that matches the given criteria and
        appears before this Tag in the document."""
        return self._first(self.fetchPrevious, name, attrs, text)

    def fetchPrevious(self, name=None, attrs={}, text=None, limit=None):
        """Returns all items that match the given criteria and appear
        before this Tag in the document."""
        return self._fetch(name, attrs, text, limit, self.previousGenerator)
    firstPrevious = findPrevious

    def findPreviousSibling(self, name=None, attrs={}, text=None):
        """Returns the closest sibling to this Tag that matches the
        given criteria and appears before this Tag in the document."""
        return self._first(self.fetchPreviousSiblings, name, attrs, text)
    firstPreviousSibling = findPreviousSibling

    def fetchPreviousSiblings(self, name=None, attrs={}, text=None,
                              limit=None):
        """Returns the siblings of this Tag that match the given
        criteria and appear before this Tag in the document."""
        return self._fetch(name, attrs, text, limit,
                           self.previousSiblingGenerator)

    def findParent(self, name=None, attrs={}):
        """Returns the closest parent of this Tag that matches the given
        criteria."""
        r = Null
        l = self.fetchParents(name, attrs, 1)
        if l:
            r = l[0]
        return r
    firstParent = findParent

    def fetchParents(self, name=None, attrs={}, limit=None):
        """Returns the parents of this Tag that match the given
        criteria."""
        return self._fetch(name, attrs, None, limit, self.parentGenerator)

    #These methods do the real heavy lifting.

    def _first(self, method, name, attrs, text):
        r = Null
        l = method(name, attrs, text, 1)
        if l:
            r = l[0]
        return r
    
    def _fetch(self, name, attrs, text, limit, generator):
        "Iterates over a generator looking for things that match."
        if not hasattr(attrs, 'items'):
            attrs = {'class' : attrs}

        results = []
        g = generator()
        while True:
            try:
                i = g.next()
            except StopIteration:
                break
            found = None
            if isinstance(i, Tag):
                if not text:
                    if not name or self._matches(i, name):
                        match = True
                        for attr, matchAgainst in attrs.items():
                            check = i.get(attr)
                            if not self._matches(check, matchAgainst):
                                match = False
                                break
                        if match:
                            found = i
            elif text:
                if self._matches(i, text):
                    found = i                    
            if found:
                results.append(found)
                if limit and len(results) >= limit:
                    break
        return results

    #Generators that can be used to navigate starting from both
    #NavigableTexts and Tags.                
    def nextGenerator(self):
        i = self
        while i:
            i = i.next
            yield i

    def nextSiblingGenerator(self):
        i = self
        while i:
            i = i.nextSibling
            yield i

    def previousGenerator(self):
        i = self
        while i:
            i = i.previous
            yield i

    def previousSiblingGenerator(self):
        i = self
        while i:
            i = i.previousSibling
            yield i

    def parentGenerator(self):
        i = self
        while i:
            i = i.parent
            yield i

    def _matches(self, chunk, howToMatch):
        #print 'looking for %s in %s' % (howToMatch, chunk)
        #
        # If given a list of items, return true if the list contains a
        # text element that matches.
        if isList(chunk) and not isinstance(chunk, Tag):
            for tag in chunk:
                if isinstance(tag, NavigableText) and self._matches(tag, howToMatch):
                    return True
            return False
        if callable(howToMatch):
            return howToMatch(chunk)
        if isinstance(chunk, Tag):
            #Custom match methods take the tag as an argument, but all other
            #ways of matching match the tag name as a string
            chunk = chunk.name
        #Now we know that chunk is a string
        if not isinstance(chunk, basestring):
            chunk = str(chunk)
        if hasattr(howToMatch, 'match'):
            # It's a regexp object.
            return howToMatch.search(chunk)
        if isList(howToMatch):
            return chunk in howToMatch
        if hasattr(howToMatch, 'items'):
            return howToMatch.has_key(chunk)
        #It's just a string
        return str(howToMatch) == chunk

class NavigableText(PageElement):

    def __getattr__(self, attr):
        "For backwards compatibility, text.string gives you text"
        if attr == 'string':
            return self
        else:
            raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)
        
class NavigableString(str, NavigableText):
    pass

class NavigableUnicodeString(unicode, NavigableText):
    pass

class Tag(PageElement):

    """Represents a found HTML tag with its attributes and contents."""

    def __init__(self, name, attrs=None, parent=Null, previous=Null):
        "Basic constructor."
        self.name = name
        if attrs == None:
            attrs = []
        self.attrs = attrs
        self.contents = []
        self.setup(parent, previous)
        self.hidden = False

    def get(self, key, default=None):
        """Returns the value of the 'key' attribute for the tag, or
        the value given for 'default' if it doesn't have that
        attribute."""
        return self._getAttrMap().get(key, default)    

    def __getitem__(self, key):
        """tag[key] returns the value of the 'key' attribute for the tag,
        and throws an exception if it's not there."""
        return self._getAttrMap()[key]

    def __iter__(self):
        "Iterating over a tag iterates over its contents."
        return iter(self.contents)

    def __len__(self):
        "The length of a tag is the length of its list of contents."
        return len(self.contents)

    def __contains__(self, x):
        return x in self.contents

    def __nonzero__(self):
        "A tag is non-None even if it has no contents."
        return True

    def __setitem__(self, key, value):        
        """Setting tag[key] sets the value of the 'key' attribute for the
        tag."""
        self._getAttrMap()
        self.attrMap[key] = value
        found = False
        for i in range(0, len(self.attrs)):
            if self.attrs[i][0] == key:
                self.attrs[i] = (key, value)
                found = True
        if not found:
            self.attrs.append((key, value))
        self._getAttrMap()[key] = value

    def __delitem__(self, key):
        "Deleting tag[key] deletes all 'key' attributes for the tag."
        for item in self.attrs:
            if item[0] == key:
                self.attrs.remove(item)
                #We don't break because bad HTML can define the same
                #attribute multiple times.
            self._getAttrMap()
            if self.attrMap.has_key(key):
                del self.attrMap[key]

    def __call__(self, *args, **kwargs):
        """Calling a tag like a function is the same as calling its
        fetch() method. Eg. tag('a') returns a list of all the A tags
        found within this tag."""
        return apply(self.fetch, args, kwargs)

    def __getattr__(self, tag):
        if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3:
            return self.first(tag[:-3])
        elif tag.find('__') != 0:
            return self.first(tag)

    def __eq__(self, other):
        """Returns true iff this tag has the same name, the same attributes,
        and the same contents (recursively) as the given tag.

        NOTE: right now this will return false if two tags have the
        same attributes in a different order. Should this be fixed?"""
        if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
            return False
        for i in range(0, len(self.contents)):
            if self.contents[i] != other.contents[i]:
                return False
        return True

    def __ne__(self, other):
        """Returns true iff this tag is not identical to the other tag,
        as defined in __eq__."""
        return not self == other

    def __repr__(self):
        """Renders this tag as a string."""
        return str(self)

    def __unicode__(self):
        return self.__str__(1)

    def __str__(self, needUnicode=None, showStructureIndent=None):
        """Returns a string or Unicode representation of this tag and
        its contents.

        NOTE: since Python's HTML parser consumes whitespace, this
        method is not certain to reproduce the whitespace present in
        the original string."""
        
        attrs = []
        if self.attrs:
            for key, val in self.attrs:
                attrs.append('%s="%s"' % (key, val))
        close = ''
        closeTag = ''
        if self.isSelfClosing():
            close = ' /'
        else:
            closeTag = '</%s>' % self.name
        indentIncrement = None        
        if showStructureIndent != None:
            indentIncrement = showStructureIndent
            if not self.hidden:
                indentIncrement += 1
        contents = self.renderContents(indentIncrement, needUnicode=needUnicode)        
        if showStructureIndent:
            space = '\n%s' % (' ' * showStructureIndent)
        if self.hidden:
            s = contents
        else:
            s = []
            attributeString = ''
            if attrs:
                attributeString = ' ' + ' '.join(attrs)            
            if showStructureIndent:
                s.append(space)
            s.append('<%s%s%s>' % (self.name, attributeString, close))
            s.append(contents)
            if closeTag and showStructureIndent != None:
                s.append(space)
            s.append(closeTag)
            s = ''.join(s)
        isUnicode = type(s) == types.UnicodeType
        if needUnicode and not isUnicode:
            s = unicode(s)
        elif isUnicode and needUnicode==False:
            s = str(s)
        return s

    def prettify(self, needUnicode=None):
        return self.__str__(needUnicode, showStructureIndent=True)

    def renderContents(self, showStructureIndent=None, needUnicode=None):
        """Renders the contents of this tag as a (possibly Unicode) 
        string."""
        s=[]
        for c in self:
            text = None
            if isinstance(c, NavigableUnicodeString) or type(c) == types.UnicodeType:
                text = unicode(c)
            elif isinstance(c, Tag):
                s.append(c.__str__(needUnicode, showStructureIndent))
            elif needUnicode:
                text = unicode(c)
            else:
                text = str(c)
            if text:
                if showStructureIndent != None:
                    if text[-1] == '\n':
                        text = text[:-1]
                s.append(text)
        return ''.join(s)    

    #Soup methods

    def firstText(self, text, recursive=True):
        """Convenience method to retrieve the first piece of text matching the
        given criteria. 'text' can be a string, a regular expression object,
        a callable that takes a string and returns whether or not the
        string 'matches', etc."""
        return self.first(recursive=recursive, text=text)

    def fetchText(self, text, recursive=True, limit=None):
        """Convenience method to retrieve all pieces of text matching the
        given criteria. 'text' can be a string, a regular expression object,
        a callable that takes a string and returns whether or not the
        string 'matches', etc."""
        return self.fetch(recursive=recursive, text=text, limit=limit)

    def first(self, name=None, attrs={}, recursive=True, text=None):
        """Return only the first child of this
        Tag matching the given criteria."""
        r = Null
        l = self.fetch(name, attrs, recursive, text, 1)
        if l:
            r = l[0]
        return r
    findChild = first

    def fetch(self, name=None, attrs={}, recursive=True, text=None,
              limit=None):
        """Extracts a list of Tag objects that match the given
        criteria.  You can specify the name of the Tag and any
        attributes you want the Tag to have.

        The value of a key-value pair in the 'attrs' map can be a
        string, a list of strings, a regular expression object, or a
        callable that takes a string and returns whether or not the
        string matches for some custom definition of 'matches'. The
        same is true of the tag name."""
        generator = self.recursiveChildGenerator
        if not recursive:
            generator = self.childGenerator
        return self._fetch(name, attrs, text, limit, generator)
    fetchChildren = fetch
    
    #Utility methods

    def isSelfClosing(self):
        """Returns true iff this is a self-closing tag as defined in the HTML
        standard.

        TODO: This is specific to BeautifulSoup and its subclasses, but it's
        used by __str__"""
        return self.name in BeautifulSoup.SELF_CLOSING_TAGS

    def append(self, tag):
        """Appends the given tag to the contents of this tag."""
        self.contents.append(tag)

    #Private methods

    def _getAttrMap(self):
        """Initializes a map representation of this tag's attributes,
        if not already initialized."""
        if not getattr(self, 'attrMap'):
            self.attrMap = {}
            for (key, value) in self.attrs:
                self.attrMap[key] = value 
        return self.attrMap

    #Generator methods
    def childGenerator(self):
        for i in range(0, len(self.contents)):
            yield self.contents[i]
        raise StopIteration
    
    def recursiveChildGenerator(self):
        stack = [(self, 0)]
        while stack:
            tag, start = stack.pop()
            if isinstance(tag, Tag):            
                for i in range(start, len(tag.contents)):
                    a = tag.contents[i]
                    yield a
                    if isinstance(a, Tag) and tag.contents:
                        if i < len(tag.contents) - 1:
                            stack.append((tag, i+1))
                        stack.append((a, 0))
                        break
        raise StopIteration


def isList(l):
    """Convenience method that works with all 2.x versions of Python
    to determine whether or not something is listlike."""
    return hasattr(l, '__iter__') \
           or (type(l) in (types.ListType, types.TupleType))

def buildTagMap(default, *args):
    """Turns a list of maps, lists, or scalars into a single map.
    Used to build the SELF_CLOSING_TAGS and NESTABLE_TAGS maps out
    of lists and partial maps."""
    built = {}
    for portion in args:
        if hasattr(portion, 'items'):
            #It's a map. Merge it.
            for k,v in portion.items():
                built[k] = v
        elif isList(portion):
            #It's a list. Map each item to the default.
            for k in portion:
                built[k] = default
        else:
            #It's a scalar. Map it to the default.
            built[portion] = default
    return built

class BeautifulStoneSoup(Tag, SGMLParser):

    """This class contains the basic parser and fetch code. It defines
    a parser that knows nothing about tag behavior except for the
    following:
   
      You can't close a tag without closing all the tags it encloses.
      That is, "<foo><bar></foo>" actually means
      "<foo><bar></bar></foo>".

    [Another possible explanation is "<foo><bar /></foo>", but since
    this class defines no SELF_CLOSING_TAGS, it will never use that
    explanation.]

    This class is useful for parsing XML or made-up markup languages,
    or when BeautifulSoup makes an assumption counter to what you were
    expecting."""

    SELF_CLOSING_TAGS = {}
    NESTABLE_TAGS = {}
    RESET_NESTING_TAGS = {}
    QUOTE_TAGS = {}

    #As a public service we will by default silently replace MS smart quotes
    #and similar characters with their HTML or ASCII equivalents.
    MS_CHARS = { '\x80' : '&euro;',
                 '\x81' : ' ',
                 '\x82' : '&sbquo;',
                 '\x83' : '&fnof;',
                 '\x84' : '&bdquo;',
                 '\x85' : '&hellip;',
                 '\x86' : '&dagger;',
                 '\x87' : '&Dagger;',
                 '\x88' : '&caret;',
                 '\x89' : '%',
                 '\x8A' : '&Scaron;',
                 '\x8B' : '&lt;',
                 '\x8C' : '&OElig;',
                 '\x8D' : '?',
                 '\x8E' : 'Z',
                 '\x8F' : '?',
                 '\x90' : '?',
                 '\x91' : '&lsquo;',
                 '\x92' : '&rsquo;',
                 '\x93' : '&ldquo;',
                 '\x94' : '&rdquo;',
                 '\x95' : '&bull;',
                 '\x96' : '&ndash;',
                 '\x97' : '&mdash;',
                 '\x98' : '&tilde;',
                 '\x99' : '&trade;',
                 '\x9a' : '&scaron;',
                 '\x9b' : '&gt;',
                 '\x9c' : '&oelig;',
                 '\x9d' : '?',
                 '\x9e' : 'z',
                 '\x9f' : '&Yuml;',}

    PARSER_MASSAGE = [(re.compile('(<[^<>]*)/>'),
                       lambda(x):x.group(1) + ' />'),
                      (re.compile('<!\s+([^<>]*)>'),
                       lambda(x):'<!' + x.group(1) + '>'),
                      (re.compile("([\x80-\x9f])"),
                       lambda(x): BeautifulStoneSoup.MS_CHARS.get(x.group(1)))
                      ]

    ROOT_TAG_NAME = '[document]'

    def __init__(self, text=None, avoidParserProblems=True,
                 initialTextIsEverything=True):
        """Initialize this as the 'root tag' and feed in any text to
        the parser.

        NOTE about avoidParserProblems: sgmllib will process most bad
        HTML, and BeautifulSoup has tricks for dealing with some HTML
        that kills sgmllib, but Beautiful Soup can nonetheless choke
        or lose data if your data uses self-closing tags or
        declarations incorrectly. By default, Beautiful Soup sanitizes
        its input to avoid the vast majority of these problems. The
        problems are relatively rare, even in bad HTML, so feel free
        to pass in False to avoidParserProblems if they don't apply to
        you, and you'll get better performance. The only reason I have
        this turned on by default is so I don't get so many tech
        support questions.

        The two most common instances of invalid HTML that will choke
        sgmllib are fixed by the default parser massage techniques:

         <br/> (No space between name of closing tag and tag close)
         <! --Comment--> (Extraneous whitespace in declaration)

        You can pass in a custom list of (RE object, replace method)
        tuples to get Beautiful Soup to scrub your input the way you
        want."""
        Tag.__init__(self, self.ROOT_TAG_NAME)
        if avoidParserProblems \
           and not isList(avoidParserProblems):
            avoidParserProblems = self.PARSER_MASSAGE            
        self.avoidParserProblems = avoidParserProblems
        SGMLParser.__init__(self)
        self.quoteStack = []
        self.hidden = 1
        self.reset()
        if hasattr(text, 'read'):
            #It's a file-type object.
            text = text.read()
        if text:
            self.feed(text)
        if initialTextIsEverything:
            self.done()

    def __getattr__(self, methodName):
        """This method routes method call requests to either the SGMLParser
        superclass or the Tag superclass, depending on the method name."""
        if methodName.find('start_') == 0 or methodName.find('end_') == 0 \
               or methodName.find('do_') == 0:
            return SGMLParser.__getattr__(self, methodName)
        elif methodName.find('__') != 0:
            return Tag.__getattr__(self, methodName)
        else:
            raise AttributeError

    def feed(self, text):
        if self.avoidParserProblems:
            for fix, m in self.avoidParserProblems:
                text = fix.sub(m, text)
        SGMLParser.feed(self, text)

    def done(self):
        """Called when you're done parsing, so that the unclosed tags can be
        correctly processed."""
        self.endData() #NEW
        while self.currentTag.name != self.ROOT_TAG_NAME:
            self.popTag()
            
    def reset(self):
        SGMLParser.reset(self)
        self.currentData = []
        self.currentTag = None
        self.tagStack = []
        self.pushTag(self)        
    
    def popTag(self):
        tag = self.tagStack.pop()
        # Tags with just one string-owning child get the child as a
        # 'string' property, so that soup.tag.string is shorthand for
        # soup.tag.contents[0]
        if len(self.currentTag.contents) == 1 and \
           isinstance(self.currentTag.contents[0], NavigableText):
            self.currentTag.string = self.currentTag.contents[0]

        #print "Pop", tag.name
        if self.tagStack:
            self.currentTag = self.tagStack[-1]
        return self.currentTag

    def pushTag(self, tag):