languagemodels.preprocess

 1from html import unescape
 2from html.parser import HTMLParser
 3
 4
 5def get_html_paragraphs(src: str):
 6    """
 7    Return plain text paragraphs from an HTML source
 8
 9    :param src: HTML document to convert to plain text paragraphs
10    :return: Plain text paragraphs of document
11
12    This function is designed to be quick rather than robust.
13
14    It follows a simple approach to extracting text:
15
16    1. Ignore all content inside the following elements listed in `ignore`.
17    2. Merge inline text content into paragraphs from `inlines` set.
18    3. Convert any newly merged text element with at least `min_length`
19    characters to a paragraph in the output text.
20
21    >>> get_html_paragraphs(open("test/wp.html", encoding="utf-8").read())
22    'Bolu Province (Turkish: Bolu ili) is a province...'
23
24    >>> get_html_paragraphs(open("test/npr.html", encoding="utf-8").read())
25    "First, the good news. Netflix reported a record ..."
26    """
27
28    class ParagraphExtractor(HTMLParser):
29        paras = [""]
30        ignoring = []
31        ignore = ("script", "style", "header", "footer")
32        ignore_attrs = {('hidden', 'hidden'), }
33        inlines = ("a", "b", "i", "span", "sup", "sub", "strong", "em")
34
35        def handle_starttag(self, tag, attrs):
36            if tag in self.ignore or self.ignore_attrs & set(attrs):
37                self.ignoring.append(tag)
38
39            if tag not in self.inlines and self.paras[-1]:
40                self.paras.append("")
41
42        def handle_endtag(self, tag):
43            if self.ignoring and self.ignoring[-1] == tag:
44                self.ignoring.pop()
45
46            if tag not in self.inlines and self.paras[-1]:
47                self.paras.append("")
48
49        def handle_data(self, data):
50            if not self.ignoring:
51                if self.paras and self.paras[-1]:
52                    self.paras[-1] += unescape(data)
53                else:
54                    self.paras.append(data)
55
56        def get_plain(self):
57            return "\n\n".join([p.rstrip() for p in self.paras if len(p.strip()) > 140])
58
59    extractor = ParagraphExtractor()
60    extractor.feed(src)
61    return extractor.get_plain()