1 import threading
2 import re
3 import urlparse
4 import copy
5 from lxml import etree
6 from lxml.html import defs
7 from lxml import cssselect
8 from lxml.html.setmixin import SetMixin
9 try:
10 from UserDict import DictMixin
11 except ImportError:
12
13 from lxml.html._dictmixin import DictMixin
14 import sets
15
16 __all__ = [
17 'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring',
18 'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form',
19 'find_rel_links', 'find_class', 'make_links_absolute',
20 'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser']
21
22 _rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]")
23
24 _class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]")
25 _id_xpath = etree.XPath("descendant-or-self::*[@id=$id]")
26 _collect_string_content = etree.XPath("string()")
27 _css_url_re = re.compile(r'url\((.*?)\)', re.I)
28 _css_import_re = re.compile(r'@import "(.*?)"')
29 _label_xpath = etree.XPath("//label[@for=$id]")
30
32
34 """
35 Returns the base URL, given when the page was parsed.
36
37 Use with ``urlparse.urljoin(el.base_url, href)`` to get
38 absolute URLs.
39 """
40 return self.getroottree().docinfo.URL
41 base_url = property(base_url, doc=base_url.__doc__)
42
48 forms = property(forms, doc=forms.__doc__)
49
51 """
52 Return the <body> element. Can be called from a child element
53 to get the document's head.
54 """
55 return self.xpath('//body')[0]
56 body = property(body, doc=body.__doc__)
57
59 """
60 Returns the <head> element. Can be called from a child
61 element to get the document's head.
62 """
63 return self.xpath('//head')[0]
64 head = property(head, doc=head.__doc__)
65
67 """
68 Get or set any <label> element associated with this element.
69 """
70 id = self.get('id')
71 if not id:
72 return None
73 result = _label_xpath(self, id=id)
74 if not result:
75 return None
76 else:
77 return result[0]
79 id = self.get('id')
80 if not id:
81 raise TypeError(
82 "You cannot set a label for an element (%r) that has no id"
83 % self)
84 if not label.tag == 'label':
85 raise TypeError(
86 "You can only assign label to a label element (not %r)"
87 % label)
88 label.set('for', id)
93 label = property(label__get, label__set, label__del, doc=label__get.__doc__)
94
96 """
97 Removes this element from the tree, including its children and
98 text. The tail text is joined to the previous element or
99 parent.
100 """
101 parent = self.getparent()
102 assert parent is not None
103 if self.tail:
104 previous = self.getprevious()
105 if previous is None:
106 parent.text = (parent.text or '') + self.tail
107 else:
108 previous.tail = (previous.tail or '') + self.tail
109 parent.remove(self)
110
112 """
113 Remove the tag, but not its children or text. The children and text
114 are merged into the parent.
115
116 Example::
117
118 >>> h = fragment_fromstring('<div>Hello <b>World!</b></div>')
119 >>> h.find('//b').drop_tag()
120 >>> print tostring(h)
121 <div>Hello World!</div>
122 """
123 parent = self.getparent()
124 assert parent is not None
125 previous = self.getprevious()
126 if self.text and isinstance(self.tag, basestring):
127
128 if previous is None:
129 parent.text = (parent.text or '') + self.text
130 else:
131 previous.tail = (previous.tail or '') + self.text
132 if self.tail:
133 if len(self):
134 last = self[-1]
135 last.tail = (last.tail or '') + self.tail
136 elif previous is None:
137 parent.text = (parent.text or '') + self.tail
138 else:
139 previous.tail = (previous.tail or '') + self.tail
140 index = parent.index(self)
141 parent[index:index+1] = self[:]
142
144 """
145 Find any links like ``<a rel="{rel}">...</a>``; returns a list of elements.
146 """
147 rel = rel.lower()
148 return [el for el in _rel_links_xpath(self)
149 if el.get('rel').lower() == rel]
150
152 """
153 Find any elements with the given class name.
154 """
155 return _class_xpath(self, class_name=class_name)
156
158 """
159 Get the first element in a document with the given id. If none is
160 found, return the default argument if provided or raise KeyError
161 otherwise.
162
163 Note that there can be more than one element with the same id,
164 and this isn't uncommon in HTML documents found in the wild.
165 Browsers return only the first match, and this function does
166 the same.
167 """
168 try:
169
170
171 return _id_xpath(self, id=id)[0]
172 except IndexError:
173 if default:
174 return default[0]
175 else:
176 raise KeyError, id
177
178 - def text_content(self):
179 """
180 Return the text content of the tag (and the text in any children).
181 """
182 return _collect_string_content(self)
183
185 """
186 Run the CSS expression on this element and its children,
187 returning a list of the results.
188
189 Equivalent to lxml.cssselect.CSSSelect(expr)(self) -- note
190 that pre-compiling the expression can provide a substantial
191 speedup.
192 """
193 return cssselect.CSSSelect(expr)(self)
194
195
196
197
198
200 """
201 Make all links in the document absolute, given the
202 ``base_url`` for the document (the full URL where the document
203 came from), or if no ``base_url`` is given, then the ``.base_url`` of the document.
204
205 If ``resolve_base_href`` is true, then any ``<base href>``
206 tags in the document are used *and* removed from the document.
207 If it is false then any such tag is ignored.
208 """
209 if base_url is None:
210 base_url = self.base_url
211 if base_url is None:
212 raise TypeError(
213 "No base_url given, and the document has no base_url")
214 if resolve_base_href:
215 self.resolve_base_href()
216 def link_repl(href):
217 return urlparse.urljoin(base_url, href)
218 self.rewrite_links(link_repl)
219
221 """
222 Find any ``<base href>`` tag in the document, and apply its
223 values to all links found in the document. Also remove the
224 tag once it has been applied.
225 """
226 base_href = None
227 basetags = self.xpath('//base[@href]')
228 for b in basetags:
229 base_href = b.get('href')
230 b.drop_tree()
231 if not base_href:
232 return
233 self.make_links_absolute(base_href, resolve_base_href=False)
234
236 """
237 Yield (element, attribute, link, pos), where attribute may be None
238 (indicating the link is in the text). ``pos`` is the position
239 where the link occurs; often 0, but sometimes something else in
240 the case of links in stylesheets or style tags.
241
242 Note: <base href> is *not* taken into account in any way. The
243 link you get is exactly the link in the document.
244 """
245 link_attrs = defs.link_attrs
246 for el in self.getiterator():
247 attribs = el.attrib
248 for attrib in link_attrs:
249 if attrib in attribs:
250 yield (el, attrib, attribs[attrib], 0)
251 if el.tag == 'style' and el.text:
252 for match in _css_url_re.finditer(el.text):
253 yield (el, None, match.group(1), match.start(1))
254 for match in _css_import_re.finditer(el.text):
255 yield (el, None, match.group(1), match.start(1))
256 if 'style' in attribs:
257 for match in _css_url_re.finditer(attribs['style']):
258 yield (el, 'style', match.group(1), match.start(1))
259
260 - def rewrite_links(self, link_repl_func, resolve_base_href=True,
261 base_href=None):
262 """
263 Rewrite all the links in the document. For each link
264 ``link_repl_func(link)`` will be called, and the return value
265 will replace the old link.
266
267 Note that links may not be absolute (unless you first called
268 ``make_links_absolute()``), and may be internal (e.g.,
269 ``'#anchor'``). They can also be values like
270 ``'mailto:email'`` or ``'javascript:expr'``.
271
272 If you give ``base_href`` then all links passed to
273 ``link_repl_func()`` will take that into account.
274
275 If the ``link_repl_func`` returns None, the attribute or
276 tag text will be removed completely.
277 """
278 if base_href is not None:
279
280
281 self.make_links_absolute(base_href, resolve_base_href=resolve_base_href)
282 elif resolve_base_href:
283 self.resolve_base_href()
284 for el, attrib, link, pos in self.iterlinks():
285 new_link = link_repl_func(link)
286 if new_link == link:
287 continue
288 if new_link is None:
289
290 if attrib is None:
291 el.text = ''
292 else:
293 del el.attrib[attrib]
294 continue
295 if attrib is None:
296 new = el.text[:pos] + new_link + el.text[pos+len(link):]
297 el.text = new
298 else:
299 cur = el.attrib[attrib]
300 if not pos and len(cur) == len(link):
301
302 el.attrib[attrib] = new_link
303 else:
304 new = cur[:pos] + new_link + cur[pos+len(link):]
305 el.attrib[attrib] = new
306
307
309 """
310 An object that represents a method on an element as a function;
311 the function takes either an element or an HTML string. It
312 returns whatever the function normally returns, or if the function
313 works in-place (and so returns None) it returns a serialized form
314 of the resulting document.
315 """
321 if isinstance(doc, basestring):
322 if 'copy' in kw:
323 raise TypeError(
324 "The keyword 'copy' can only be used with element inputs to %s, not a string input" % self.name)
325 return_string = True
326 doc = fromstring(doc, **kw)
327 else:
328 if 'copy' in kw:
329 copy = kw.pop('copy')
330 else:
331 copy = self.copy
332 return_string = False
333 if copy:
334 doc = copy.deepcopy(doc)
335 meth = getattr(doc, self.name)
336 result = meth(*args, **kw)
337
338 if result is None:
339
340 if return_string:
341 return tostring(doc)
342 else:
343 return doc
344 else:
345 return result
346
347 find_rel_links = _MethodFunc('find_rel_links', copy=False)
348 find_class = _MethodFunc('find_class', copy=False)
349 make_links_absolute = _MethodFunc('make_links_absolute', copy=True)
350 resolve_base_href = _MethodFunc('resolve_base_href', copy=True)
351 iterlinks = _MethodFunc('iterlinks', copy=False)
352 rewrite_links = _MethodFunc('rewrite_links', copy=True)
353
356
359
362
365
366
368 """A lookup scheme for HTML Element classes.
369
370 To create a lookup instance with different Element classes, pass a tag
371 name mapping of Element classes in the ``classes`` keyword argument and/or
372 a tag name mapping of Mixin classes in the ``mixins`` keyword argument.
373 The special key '*' denotes a Mixin class that should be mixed into all
374 Element classes.
375 """
376 _default_element_classes = {}
377
378 - def __init__(self, classes=None, mixins=None):
395
396 - def lookup(self, node_type, document, namespace, name):
407
408
409
410
411
418
420 """
421 Parses several HTML elements, returning a list of elements.
422
423 The first item in the list may be a string (though leading
424 whitespace is removed). If no_leading_text is true, then it will
425 be an error if there is leading text, and it will always be a list
426 of only elements.
427 """
428
429 start = html[:20].lstrip().lower()
430 if not start.startswith('<html') and not start.startswith('<!doctype'):
431 html = '<html><body>%s</body></html>' % html
432 doc = document_fromstring(html, **kw)
433 assert doc.tag == 'html'
434 bodies = [e for e in doc if e.tag == 'body']
435 assert len(bodies) == 1, ("too many bodies: %r in %r" % (bodies, html))
436 body = bodies[0]
437 elements = []
438 if no_leading_text and body.text and body.text.strip():
439 raise etree.ParserError(
440 "There is leading text: %r" % body.text)
441 if body.text and body.text.strip():
442 elements.append(body.text)
443 elements.extend(body)
444
445
446 return elements
447
449 """
450 Parses a single HTML element; it is an error if there is more than
451 one element, or if anything but whitespace precedes or follows the
452 element.
453
454 If create_parent is true (or is a tag name) then a parent node
455 will be created to encapsulate the HTML in a single element.
456 """
457 if create_parent:
458 if not isinstance(create_parent, basestring):
459 create_parent = 'div'
460 return fragment_fromstring('<%s>%s</%s>' % (
461 create_parent, html, create_parent), **kw)
462 elements = fragments_fromstring(html, no_leading_text=True)
463 if not elements:
464 raise etree.ParserError(
465 "No elements found")
466 if len(elements) > 1:
467 raise etree.ParserError(
468 "Multiple elements found (%s)"
469 % ', '.join([_element_name(e) for e in elements]))
470 el = elements[0]
471 if el.tail and el.tail.strip():
472 raise etree.ParserError(
473 "Element followed by text: %r" % el.tail)
474 el.tail = None
475 return el
476
531
532 -def parse(filename, parser=None, **kw):
533 """
534 Parse a filename, URL, or file-like object into an HTML document.
535
536 You may pass the keyword argument ``base_url='http://...'`` to set
537 the base URL.
538 """
539 if parser is None:
540 parser = html_parser
541 return etree.parse(filename, parser, **kw)
542
550
552 if isinstance(el, etree.CommentBase):
553 return 'comment'
554 elif isinstance(el, basestring):
555 return 'string'
556 else:
557 return el.tag
558
559
560
561
562
663
664 HtmlElementClassLookup._default_element_classes['form'] = FormElement
665
698
700 import urllib
701
702 if method == 'GET':
703 if '?' in url:
704 url += '&'
705 else:
706 url += '?'
707 url += urllib.urlencode(values)
708 data = None
709 else:
710 data = urllib.urlencode(values)
711 return urllib.urlopen(url, data)
712
714
722 raise KeyError(
723 "You cannot remove keys from ElementDict")
727 return item in self.inputs
728
730 return '<%s for form %s>' % (
731 self.__class__.__name__,
732 self.inputs.form._name())
733
797
825
826 -class TextareaElement(InputMixin, HtmlElement):
827 """
828 ``<textarea>`` element. You can get the name with ``.name`` and
829 get/set the value with ``.value``
830 """
831
832 - def value__get(self):
833 """
834 Get/set the value (which is the contents of this element)
835 """
836 return self.text or ''
837 - def value__set(self, value):
839 - def value__del(self):
841 value = property(value__get, value__set, value__del, doc=value__get.__doc__)
842
843 HtmlElementClassLookup._default_element_classes['textarea'] = TextareaElement
844
846 """
847 ``<select>`` element. You can get the name with ``.name``.
848
849 ``.value`` will be the value of the selected option, unless this
850 is a multi-select element (``<select multiple>``), in which case
851 it will be a set-like object. In either case ``.value_options``
852 gives the possible values.
853
854 The boolean attribute ``.multiple`` shows if this is a
855 multi-select.
856 """
857
859 """
860 Get/set the value of this select (the selected option).
861
862 If this is a multi-select, this is a set-like object that
863 represents all the selected options.
864 """
865 if self.multiple:
866 return MultipleSelectOptions(self)
867 for el in self.getiterator('option'):
868 if 'selected' in el.attrib:
869 value = el.get('value')
870
871 return value
872 return None
873
875 if self.multiple:
876 if isinstance(value, basestring):
877 raise TypeError(
878 "You must pass in a sequence")
879 self.value.clear()
880 self.value.update(value)
881 return
882 if value is not None:
883 for el in self.getiterator('option'):
884
885 if el.get('value') == value:
886 checked_option = el
887 break
888 else:
889 raise ValueError(
890 "There is no option with the value of %r" % value)
891 for el in self.getiterator('option'):
892 if 'selected' in el.attrib:
893 del el.attrib['selected']
894 if value is not None:
895 checked_option.set('selected', '')
896
903
904 value = property(value__get, value__set, value__del, doc=value__get.__doc__)
905
907 """
908 All the possible values this select can have (the ``value``
909 attribute of all the ``<option>`` elements.
910 """
911 return [el.get('value') for el in self.getiterator('option')]
912 value_options = property(value_options, doc=value_options.__doc__)
913
915 """
916 Boolean attribute: is there a ``multiple`` attribute on this element.
917 """
918 return 'multiple' in self.attrib
920 if value:
921 self.set('multiple', '')
922 elif 'multiple' in self.attrib:
923 del self.attrib['multiple']
924 multiple = property(multiple__get, multiple__set, doc=multiple__get.__doc__)
925
926 HtmlElementClassLookup._default_element_classes['select'] = SelectElement
927
929 """
930 Represents all the selected options in a ``<select multiple>`` element.
931
932 You can add to this set-like option to select an option, or remove
933 to unselect the option.
934 """
935
938
940 """
941 Iterator of all the ``<option>`` elements.
942 """
943 return self.select.getiterator('option')
944 options = property(options)
945
947 for option in self.options:
948 yield option.get('value')
949
950 - def add(self, item):
951 for option in self.options:
952 if option.get('value') == item:
953 option.set('selected', '')
954 break
955 else:
956 raise ValueError(
957 "There is no option with the value %r" % item)
958
960 for option in self.options:
961 if option.get('value') == item:
962 if 'selected' in option.attrib:
963 del option.attrib['selected']
964 else:
965 raise ValueError(
966 "The option %r is not currently selected" % item)
967 break
968 else:
969 raise ValueError(
970 "There is not option with the value %r" % item)
971
973 return '<%s {%s} for select name=%r>' % (
974 self.__class__.__name__,
975 ', '.join([repr(v) for v in self]),
976 self.select.name)
977
979 """
980 This object represents several ``<input type=radio>`` elements
981 that have the same name.
982
983 You can use this like a list, but also use the property
984 ``.value`` to check/uncheck inputs. Also you can use
985 ``.value_options`` to get the possible values.
986 """
987
989 """
990 Get/set the value, which checks the radio with that value (and
991 unchecks any other value).
992 """
993 for el in self:
994 if 'checked' in el.attrib:
995 return el.get('value')
996 return None
997
999 if value is not None:
1000 for el in self:
1001 if el.get('value') == value:
1002 checked_option = el
1003 break
1004 else:
1005 raise ValueError(
1006 "There is no radio input with the value %r" % value)
1007 for el in self:
1008 if 'checked' in el.attrib:
1009 del el.attrib['checked']
1010 if value is not None:
1011 checked_option.set('checked', '')
1012
1015
1016 value = property(value__get, value__set, value__del, doc=value__get.__doc__)
1017
1019 """
1020 Returns a list of all the possible values.
1021 """
1022 return [el.get('value') for el in self]
1023 value_options = property(value_options, doc=value_options.__doc__)
1024
1026 return '%s(%s)' % (
1027 self.__class__.__name__,
1028 list.__repr__(self))
1029
1031 """
1032 Represents a group of checkboxes (``<input type=checkbox>``) that
1033 have the same name.
1034
1035 In addition to using this like a list, the ``.value`` attribute
1036 returns a set-like object that you can add to or remove from to
1037 check and uncheck checkboxes. You can also use ``.value_options``
1038 to get the possible values.
1039 """
1040
1042 """
1043 Return a set-like object that can be modified to check or
1044 uncheck individual checkboxes according to their value.
1045 """
1046 return CheckboxValues(self)
1056 value = property(value__get, value__set, value__del, doc=value__get.__doc__)
1057
1059 return '%s(%s)' % (
1060 self.__class__.__name__, list.__repr__(self))
1061
1063
1064 """
1065 Represents the values of the checked checkboxes in a group of
1066 checkboxes with the same name.
1067 """
1068
1071
1073 return iter([
1074 el.get('value')
1075 for el in self.group
1076 if 'checked' in el.attrib])
1077
1078 - def add(self, value):
1079 for el in self.group:
1080 if el.get('value') == value:
1081 el.set('checked', '')
1082 break
1083 else:
1084 raise KeyError("No checkbox with value %r" % value)
1085
1087 for el in self.group:
1088 if el.get('value') == value:
1089 if 'checked' in el.attrib:
1090 del el.attrib['checked']
1091 else:
1092 raise KeyError(
1093 "The checkbox with value %r was already unchecked" % value)
1094 break
1095 else:
1096 raise KeyError(
1097 "No checkbox with value %r" % value)
1098
1100 return '<%s {%s} for checkboxes name=%r>' % (
1101 self.__class__.__name__,
1102 ', '.join([repr(v) for v in self]),
1103 self.group.name)
1104
1188
1189 HtmlElementClassLookup._default_element_classes['input'] = InputElement
1190
1192 """
1193 Represents a ``<label>`` element.
1194
1195 Label elements are linked to other elements with their ``for``
1196 attribute. You can access this element with ``label.for_element``.
1197 """
1198
1200 """
1201 Get/set the element this label points to. Return None if it
1202 can't be found.
1203 """
1204 id = self.get('for')
1205 if not id:
1206 return None
1207 return self.body.get_element_by_id(id)
1209 id = other.get('id')
1210 if not id:
1211 raise TypeError(
1212 "Element %r has no id attribute" % other)
1213 self.set('for', id)
1217 for_element = property(for_element__get, for_element__set, for_element__del,
1218 doc=for_element__get.__doc__)
1219
1220 HtmlElementClassLookup._default_element_classes['label'] = LabelElement
1221
1222
1223
1224
1225
1226
1227
1228 __replace_meta_content_type = re.compile(
1229 r'<meta http-equiv="Content-Type".*?>').sub
1230
1231 -def tostring(doc, pretty_print=False, include_meta_content_type=False):
1232 """
1233 return HTML string representation of the document given
1234
1235 note: this will create a meta http-equiv="Content" tag in the head
1236 and may replace any that are present
1237 """
1238 assert doc is not None
1239 html = etree.tostring(doc, method="html", pretty_print=pretty_print)
1240 if not include_meta_content_type:
1241 html = __replace_meta_content_type('', html)
1242 return html
1243
1245 """
1246 Open the HTML document in a web browser (saving it to a temporary
1247 file to open it).
1248 """
1249 import os
1250 import webbrowser
1251 try:
1252 write_doc = doc.write
1253 except AttributeError:
1254 write_doc = etree.ElementTree(element=doc).write
1255 fn = os.tempnam() + '.html'
1256 write_doc(fn, method="html")
1257 url = 'file://' + fn.replace(os.path.sep, '/')
1258 print url
1259 webbrowser.open(url)
1260
1261
1262
1263
1264
1269
1273
1274 html_parser = HTMLParser()
1275