lxml.html.soupparser

1 """External interface to the BeautifulSoup HTML parser. 2 """ 3 4 __all__ = ["fromstring", "parse", "convert_tree"] 5 6 import re 7 from lxml import etree, html 8 9 try: 10 from bs4 import ( 11 BeautifulSoup, Tag, Comment, ProcessingInstruction, NavigableString, 12 Declaration, Doctype) 13 _DECLARATION_OR_DOCTYPE = (Declaration, Doctype) 14 except ImportError: 15 from BeautifulSoup import ( 16 BeautifulSoup, Tag, Comment, ProcessingInstruction, NavigableString, 17 Declaration) 18 _DECLARATION_OR_DOCTYPE = Declaration

19 20 21 -def fromstring(data, beautifulsoup=None, makeelement=None, **bsargs):

22 """Parse a string of HTML data into an Element tree using the 23 BeautifulSoup parser. 24 25 Returns the root ``<html>`` Element of the tree. 26 27 You can pass a different BeautifulSoup parser through the 28 `beautifulsoup` keyword, and a diffent Element factory function 29 through the `makeelement` keyword. By default, the standard 30 ``BeautifulSoup`` class and the default factory of `lxml.html` are 31 used. 32 """ 33 return _parse(data, beautifulsoup, makeelement, **bsargs)

34

35 36 -def parse(file, beautifulsoup=None, makeelement=None, **bsargs):

37 """Parse a file into an ElemenTree using the BeautifulSoup parser. 38 39 You can pass a different BeautifulSoup parser through the 40 `beautifulsoup` keyword, and a diffent Element factory function 41 through the `makeelement` keyword. By default, the standard 42 ``BeautifulSoup`` class and the default factory of `lxml.html` are 43 used. 44 """ 45 if not hasattr(file, 'read'): 46 file = open(file) 47 root = _parse(file, beautifulsoup, makeelement, **bsargs) 48 return etree.ElementTree(root)

49

50 51 -def convert_tree(beautiful_soup_tree, makeelement=None):

52 """Convert a BeautifulSoup tree to a list of Element trees. 53 54 Returns a list instead of a single root Element to support 55 HTML-like soup with more than one root element. 56 57 You can pass a different Element factory through the `makeelement` 58 keyword. 59 """ 60 root = _convert_tree(beautiful_soup_tree, makeelement) 61 children = root.getchildren() 62 for child in children: 63 root.remove(child) 64 return children

65

66 67 # helpers 68 69 -def _parse(source, beautifulsoup, makeelement, **bsargs):

70 if beautifulsoup is None: 71 beautifulsoup = BeautifulSoup 72 if hasattr(beautifulsoup, "HTML_ENTITIES"): # bs3 73 if 'convertEntities' not in bsargs: 74 bsargs['convertEntities'] = 'html' 75 if hasattr(beautifulsoup, "DEFAULT_BUILDER_FEATURES"): # bs4 76 if 'features' not in bsargs: 77 bsargs['features'] = 'html.parser' # use Python html parser 78 tree = beautifulsoup(source, **bsargs) 79 root = _convert_tree(tree, makeelement) 80 # from ET: wrap the document in a html root element, if necessary 81 if len(root) == 1 and root[0].tag == "html": 82 return root[0] 83 root.tag = "html" 84 return root

85 86 87 _parse_doctype_declaration = re.compile( 88 r'(?:\s|[<!])*DOCTYPE\s*HTML' 89 r'(?:\s+PUBLIC)?(?:\s+(\'[^\']*\'|"[^"]*"))?' 90 r'(?:\s+(\'[^\']*\'|"[^"]*"))?', 91 re.IGNORECASE).match

92 93 94 -class _PseudoTag:

95 # Minimal imitation of BeautifulSoup.Tag

96 - def __init__(self, contents):

97 self.name = 'html' 98 self.attrs = [] 99 self.contents = contents

100

101 - def __iter__(self):

102 return self.contents.__iter__()

103

104 105 -def _convert_tree(beautiful_soup_tree, makeelement):

106 if makeelement is None: 107 makeelement = html.html_parser.makeelement 108 109 # Split the tree into three parts: 110 # i) everything before the root element: document type 111 # declaration, comments, processing instructions, whitespace 112 # ii) the root(s), 113 # iii) everything after the root: comments, processing 114 # instructions, whitespace 115 first_element_idx = last_element_idx = None 116 html_root = declaration = None 117 for i, e in enumerate(beautiful_soup_tree): 118 if isinstance(e, Tag): 119 if first_element_idx is None: 120 first_element_idx = i 121 last_element_idx = i 122 if html_root is None and e.name and e.name.lower() == 'html': 123 html_root = e 124 elif declaration is None and isinstance(e, _DECLARATION_OR_DOCTYPE): 125 declaration = e 126 127 # For a nice, well-formatted document, the variable roots below is 128 # a list consisting of a single <html> element. However, the document 129 # may be a soup like '<meta><head><title>Hello</head><body>Hi 130 # all<\p>'. In this example roots is a list containing meta, head 131 # and body elements. 132 if first_element_idx is None: 133 pre_root = post_root = [] 134 roots = beautiful_soup_tree.contents 135 else: 136 pre_root = beautiful_soup_tree.contents[:first_element_idx] 137 roots = beautiful_soup_tree.contents[first_element_idx:last_element_idx+1] 138 post_root = beautiful_soup_tree.contents[last_element_idx+1:] 139 140 # Reorganize so that there is one <html> root... 141 if html_root is not None: 142 # ... use existing one if possible, ... 143 i = roots.index(html_root) 144 html_root.contents = roots[:i] + html_root.contents + roots[i+1:] 145 else: 146 # ... otherwise create a new one. 147 html_root = _PseudoTag(roots) 148 149 convert_node = _init_node_converters(makeelement) 150 151 # Process pre_root 152 res_root = convert_node(html_root) 153 prev = res_root 154 for e in reversed(pre_root): 155 converted = convert_node(e) 156 if converted is not None: 157 prev.addprevious(converted) 158 prev = converted 159 160 # ditto for post_root 161 prev = res_root 162 for e in post_root: 163 converted = convert_node(e) 164 if converted is not None: 165 prev.addnext(converted) 166 prev = converted 167 168 if declaration is not None: 169 try: 170 # bs4 provides full Doctype string 171 doctype_string = declaration.output_ready() 172 except AttributeError: 173 doctype_string = declaration.string 174 175 match = _parse_doctype_declaration(doctype_string) 176 if not match: 177 # Something is wrong if we end up in here. Since soupparser should 178 # tolerate errors, do not raise Exception, just let it pass. 179 pass 180 else: 181 external_id, sys_uri = match.groups() 182 docinfo = res_root.getroottree().docinfo 183 # strip quotes and update DOCTYPE values (any of None, '', '...') 184 docinfo.public_id = external_id and external_id[1:-1] 185 docinfo.system_url = sys_uri and sys_uri[1:-1] 186 187 return res_root

188

189 190 -def _init_node_converters(makeelement):

191 converters = {} 192 ordered_node_types = [] 193 194 def converter(*types): 195 def add(handler): 196 for t in types: 197 converters[t] = handler 198 ordered_node_types.append(t) 199 return handler

200 return add 201 202 def find_best_converter(node): 203 for t in ordered_node_types: 204 if isinstance(node, t): 205 return converters[t] 206 return None 207 208 def convert_node(bs_node, parent=None): 209 # duplicated in convert_tag() below 210 try: 211 handler = converters[type(bs_node)] 212 except KeyError: 213 handler = converters[type(bs_node)] = find_best_converter(bs_node) 214 if handler is None: 215 return None 216 return handler(bs_node, parent) 217 218 def map_attrs(bs_attrs): 219 if isinstance(bs_attrs, dict): # bs4 220 attribs = {} 221 for k, v in bs_attrs.items(): 222 if isinstance(v, list): 223 v = " ".join(v) 224 attribs[k] = unescape(v) 225 else: 226 attribs = dict((k, unescape(v)) for k, v in bs_attrs) 227 return attribs 228 229 def append_text(parent, text): 230 if len(parent) == 0: 231 parent.text = (parent.text or '') + text 232 else: 233 parent[-1].tail = (parent[-1].tail or '') + text 234 235 # converters are tried in order of their definition 236 237 @converter(Tag, _PseudoTag) 238 def convert_tag(bs_node, parent): 239 attrs = bs_node.attrs 240 if parent is not None: 241 attribs = map_attrs(attrs) if attrs else None 242 res = etree.SubElement(parent, bs_node.name, attrib=attribs) 243 else: 244 attribs = map_attrs(attrs) if attrs else {} 245 res = makeelement(bs_node.name, attrib=attribs) 246 247 for child in bs_node: 248 # avoid double recursion by inlining convert_node(), see above 249 try: 250 handler = converters[type(child)] 251 except KeyError: 252 pass 253 else: 254 if handler is not None: 255 handler(child, res) 256 continue 257 convert_node(child, res) 258 return res 259 260 @converter(Comment) 261 def convert_comment(bs_node, parent): 262 res = html.HtmlComment(bs_node) 263 if parent is not None: 264 parent.append(res) 265 return res 266 267 @converter(ProcessingInstruction) 268 def convert_pi(bs_node, parent): 269 if bs_node.endswith('?'): 270 # The PI is of XML style (<?as df?>) but BeautifulSoup 271 # interpreted it as being SGML style (<?as df>). Fix. 272 bs_node = bs_node[:-1] 273 res = etree.ProcessingInstruction(*bs_node.split(' ', 1)) 274 if parent is not None: 275 parent.append(res) 276 return res 277 278 @converter(NavigableString) 279 def convert_text(bs_node, parent): 280 if parent is not None: 281 append_text(parent, unescape(bs_node)) 282 return None 283 284 return convert_node 285 286 287 # copied from ET's ElementSoup 288 289 try: 290 from html.entities import name2codepoint # Python 3 291 except ImportError: 292 from htmlentitydefs import name2codepoint 293 294 295 handle_entities = re.compile(r"&(\w+);").sub 296 297 298 try: 299 unichr 300 except NameError: 301 # Python 3 302 unichr = chr

303 304 305 -def unescape(string):

306 if not string: 307 return '' 308 # work around oddities in BeautifulSoup's entity handling 309 def unescape_entity(m): 310 try: 311 return unichr(name2codepoint[m.group(1)]) 312 except KeyError: 313 return m.group(0) # use as is

314 return handle_entities(unescape_entity, string) 315

Source Code for Module lxml.html.soupparser