Add the Fun section to the Learn tab with comic-friendly reader view (#3)
The Learn tab gets a Fun section below Learning with the last 3 days of Fun articles. Fun articles stay out of Today and the Briefing. New curated Fun sources: xkcd and The Daily WTF (on by default) and r/ProgrammerHumor top of the day (off). Reader view for Fun articles shows the picture whole in the body instead of as a 16:9 banner cropped to fit, and doesn't call a comic without text a teaser. When the page gives no picture (Reddit blocks reader view), the feed's image is shown. The extractor now keeps an image's title text as a caption (xkcd's last joke), images in blockquotes and behind in-page anchors (The Daily WTF's Error'd screenshots), drops image maps and license boxes, and on a text tie prefers the page with more blocks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a6b8d8b537
commit
8240a767ef
10 files changed
+387
-20
No files matched your search
@@ -49,9 +49,10 @@ class ArticleExtractor {
|
||||
/// Hard cap on what we send to the model per article (~10k tokens).
|
||||
static const maxChars = 40000;
|
||||
|
||||
/// Loads the article page and returns whichever is longer: the extracted
|
||||
/// page or the full content some feeds ship. Never throws; on failure
|
||||
/// (paywall, bot protection, CORS on web) it falls back to feed content.
|
||||
/// Loads the article page and returns whichever has more text (on a tie,
|
||||
/// more blocks): the extracted page or the full content some feeds ship.
|
||||
/// Never throws; on failure (paywall, bot protection, CORS on web) it falls
|
||||
/// back to feed content.
|
||||
///
|
||||
/// With [acceptConsentWalls], a redirect to a consent page the
|
||||
/// [ConsentService] knows is accepted once and the article fetched again.
|
||||
@@ -81,7 +82,12 @@ class ArticleExtractor {
|
||||
if (page.wallUrl != null) return ArticleContent(fromFeed.blocks, consentWall: true);
|
||||
if (page.status != 200) return fromFeed;
|
||||
final fromPage = parse(FeedService.decodeBody(page.bytes), base);
|
||||
return fromPage.plainText.length > fromFeed.plainText.length ? fromPage : fromFeed;
|
||||
final pageText = fromPage.plainText.length, feedText = fromFeed.plainText.length;
|
||||
// Without text either way (a comic), the one with the picture wins.
|
||||
return pageText > feedText ||
|
||||
(pageText == feedText && fromPage.blocks.length > fromFeed.blocks.length)
|
||||
? fromPage
|
||||
: fromFeed;
|
||||
} catch (_) {
|
||||
return fromFeed;
|
||||
}
|
||||
@@ -119,7 +125,8 @@ class ArticleExtractor {
|
||||
/// Class/id fragments of page furniture that sits inside article bodies.
|
||||
static final _junkMarker = RegExp(
|
||||
r'comment|related|newsletter|share|social|promo|advert|paywall|subscribe|'
|
||||
r'recommend|author-box|byline|breadcrumb|tags|toolbar|consent|cookie|card|native-ad',
|
||||
r'recommend|author-box|byline|breadcrumb|tags|toolbar|consent|cookie|card|native-ad|'
|
||||
r'license',
|
||||
caseSensitive: false,
|
||||
);
|
||||
|
||||
@@ -155,14 +162,15 @@ class ArticleExtractor {
|
||||
final blocks = <ReaderBlock>[];
|
||||
final seenImages = <String>{};
|
||||
var chars = 0;
|
||||
void add(BlockType type, String text) {
|
||||
if (chars > maxChars) return;
|
||||
bool add(BlockType type, String text) {
|
||||
if (chars > maxChars) return false;
|
||||
if (type == BlockType.image) {
|
||||
if (!seenImages.add(text)) return;
|
||||
if (!seenImages.add(text)) return false;
|
||||
} else {
|
||||
chars += text.length;
|
||||
}
|
||||
blocks.add(ReaderBlock(type, text));
|
||||
return true;
|
||||
}
|
||||
|
||||
void walk(dom.Element el) {
|
||||
@@ -190,6 +198,8 @@ class ArticleExtractor {
|
||||
case 'blockquote':
|
||||
final t = _clean(el.text);
|
||||
if (t.isNotEmpty) add(BlockType.quote, t);
|
||||
// Quoted screenshots, as in The Daily WTF's Error'd.
|
||||
el.querySelectorAll('img').forEach((img) => _addImage(img, base, add));
|
||||
return;
|
||||
case 'pre':
|
||||
final t = el.text.trimRight();
|
||||
@@ -221,7 +231,7 @@ class ArticleExtractor {
|
||||
static String _clean(String s) => s.replaceAll(RegExp(r'\s+'), ' ').trim();
|
||||
|
||||
static void _addImage(
|
||||
dom.Element img, Uri base, void Function(BlockType, String) add) {
|
||||
dom.Element img, Uri base, bool Function(BlockType, String) add) {
|
||||
// Lazy-loading sites keep the real URL in data attributes.
|
||||
final raw = img.attributes['data-src'] ??
|
||||
img.attributes['data-lazy-src'] ??
|
||||
@@ -232,19 +242,25 @@ class ArticleExtractor {
|
||||
}
|
||||
final width = int.tryParse(img.attributes['width'] ?? '');
|
||||
if (width != null && width < 120) return; // icons, avatars, tracking pixels
|
||||
if (img.attributes.containsKey('usemap')) return; // clickable navigation
|
||||
// Images linking to another page are author avatars or related-story
|
||||
// thumbnails; links to the full-size image file are fine.
|
||||
// thumbnails; links to the full-size image file or within the page are fine.
|
||||
for (var p = img.parent; p != null; p = p.parent) {
|
||||
if (p.localName == 'a') {
|
||||
final href = p.attributes['href'] ?? '';
|
||||
if (!RegExp(r'\.(jpe?g|png|webp|gif|avif)(\?|$)', caseSensitive: false)
|
||||
.hasMatch(href)) {
|
||||
if (!href.startsWith('#') &&
|
||||
!RegExp(r'\.(jpe?g|png|webp|gif|avif)(\?|$)', caseSensitive: false).hasMatch(href)) {
|
||||
return;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
final url = base.resolve(raw).toString();
|
||||
if (url.startsWith('http')) add(BlockType.image, url);
|
||||
if (!url.startsWith('http') || !add(BlockType.image, url)) return;
|
||||
// Comics such as xkcd put a last joke in the title text.
|
||||
final title = _clean(img.attributes['title'] ?? '');
|
||||
if (title.length >= 20 && title != _clean(img.attributes['alt'] ?? '')) {
|
||||
add(BlockType.caption, title);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user