Add the Fun section to the Learn tab with comic-friendly reader view (#3)

The Learn tab gets a Fun section below Learning with the last 3 days of Fun articles. Fun articles stay out of Today and the Briefing. New curated Fun sources: xkcd and The Daily WTF (on by default) and r/ProgrammerHumor top of the day (off). Reader view for Fun articles shows the picture whole in the body instead of as a 16:9 banner cropped to fit, and doesn't call a comic without text a teaser. When the page gives no picture (Reddit blocks reader view), the feed's image is shown. The extractor now keeps an image's title text as a caption (xkcd's last joke), images in blockquotes and behind in-page anchors (The Daily WTF's Error'd screenshots), drops image maps and license boxes, and on a text tie prefers the page with more blocks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 15:56:42 +02:00
1 parent a6b8d8b537
commit 8240a767ef
@@ -102,8 +102,86 @@ void main() {
        isFragment: true);
    expect(content.blocks, hasLength(2));
  });
+
+  test('an xkcd page gives the comic and its title text, without the page around it', () {
+    final content = ArticleExtractor.parse(xkcdPage, Uri.parse('https://xkcd.com/3312/'));
+
+    expect(content.blocks.map((b) => (b.type, b.text)), [
+      (BlockType.image, 'https://imgs.xkcd.com/comics/momentum_demonstration.png'),
+      (BlockType.caption, xkcdTitleText),
+    ]);
+  });
+
+  test("The Daily WTF's Error'd screenshots show between their stories", () {
+    final content = ArticleExtractor.parse(
+        errordPage, Uri.parse('https://thedailywtf.com/articles/scrollpocalypse'));
+
+    expect(content.blocks.map((b) => b.type), [
+      BlockType.paragraph,
+      BlockType.image,
+      BlockType.paragraph,
+      BlockType.image,
+    ]);
+    expect(content.blocks[1].text, 'https://cdn.example.com/errord/first.png');
+  });
 }

+const xkcdTitleText = '"I\'m doing really well on the Hertzsprung-Russell academic diagram, '
+    'which has spectral class on one axis and Rate My Professors rating on the other."';
+
+/// Trimmed from https://xkcd.com/3312/.
+const xkcdPage = '''
+<!DOCTYPE html>
+<html><head><title>xkcd: Momentum Demonstration</title></head>
+<body>
+<div id="topContainer">
+<div id="topLeft"><ul><li><a href="/archive">Archive</a></li><li><a href="/about">About</a></li></ul></div>
+<div id="topRight">
+<div id="masthead"><span><a href="/"><img src="/s/0b7742.png" alt="xkcd.com logo" height="83" width="185"/></a></span>
+<span id="slogan">A webcomic of romance,<br/> sarcasm, math, and language.</span></div>
+<div id="news"><div id="xkcdNews"><a href="https://store.xkcd.com/"><img src="https://imgs.xkcd.com/store_news/news_back.png" alt="The xkcd store is back, with shirts, stickers, and posters for sale." title="xkcd store news" /></a></div></div>
+</div>
+</div>
+<div id="middleContainer" class="box">
+<div id="ctitle">Momentum Demonstration</div>
+<ul class="comicNav"><li><a href="/1/">|&lt;</a></li><li><a rel="prev" href="/3311/">&lt; Prev</a></li></ul>
+<div id="comic">
+<img src="//imgs.xkcd.com/comics/momentum_demonstration.png" title="&quot;I&#39;m doing really well on the Hertzsprung-Russell academic diagram, which has spectral class on one axis and Rate My Professors rating on the other.&quot;" alt="Momentum Demonstration" srcset="//imgs.xkcd.com/comics/momentum_demonstration_2x.png 2x" style="image-orientation:none" />
+</div>
+<ul class="comicNav"><li><a href="/1/">|&lt;</a></li><li><a rel="prev" href="/3311/">&lt; Prev</a></li></ul>
+<br />
+Permanent link to this comic: <a href="https://xkcd.com/3312">https://xkcd.com/3312/</a><br />
+Image URL (for hotlinking/embedding): <a href= "https://imgs.xkcd.com/comics/momentum_demonstration.png">https://imgs.xkcd.com/comics/momentum_demonstration.png</a>
+<div id="transcript" style="display: none"></div>
+</div>
+<div id="bottom" class="box">
+<img src="//imgs.xkcd.com/s/a899e84.jpg" width="520" height="100" alt="Selected Comics" usemap="#comicmap"/>
+<map id="comicmap" name="comicmap"><area shape="rect" coords="0,0,100,100" href="/150/" alt="Grownups"/></map>
+<div id="comicLinks">Comics I enjoy:<br/><a href="http://threewordphrase.com/">Three Word Phrase</a></div>
+<div id="footnote" style="width:70%">xkcd.com is best viewed with Netscape Navigator 4.0 or below on a Pentium 3&plusmn;1 emulated in Javascript on an Apple IIGS</div>
+<div id="licenseText">
+<p>
+This work is licensed under a
+<a href="https://creativecommons.org/licenses/by-nc/2.5/">Creative Commons Attribution-NonCommercial 2.5 License</a>.
+</p><p>
+This means you're free to copy and share these comics (but not to sell them). <a rel="license" href="/license.html">More details</a>.</p>
+</div>
+</div>
+</body></html>''';
+
+/// Trimmed from https://thedailywtf.com/articles/scrollpocalypse.
+const errordPage = '''
+<html><body>
+<div itemscope itemtype="http://schema.org/Article" class="article-container errord">
+<div itemprop="articleBody" class="article-body">
+<p>Dr. Doom...scroller titled this one "Scrollbar-ception". Said he "Screenshot of the TikTok webpage."</p>
+<blockquote><p><a href="#d6711f9e"><img itemprop="image" border="0" alt="d6711f9e" src="https://cdn.example.com/errord/first.png"/></a></p><p> </p></blockquote>
+<p>An anonymous reader clarified "I was reorganizing the document by drag-and-dropping table cells."</p>
+<blockquote><p><a href="#01270f37"><img itemprop="image" border="0" alt="01270f37" src="https://cdn.example.com/errord/second.png"/></a></p><p> </p></blockquote>
+</div>
+</div>
+</body></html>''';
+
 /// Answers every request as if the server had redirected to [finalUrl].
 class _RedirectingClient extends http.BaseClient {
  _RedirectingClient(this.finalUrl, this.body);
@@ -0,0 +1,109 @@
+import 'dart:io';
+import 'dart:ui' as ui;
+
+import 'package:daily_tech_news/models/article.dart';
+import 'package:daily_tech_news/models/news_source.dart';
+import 'package:daily_tech_news/services/article_extractor.dart';
+import 'package:daily_tech_news/ui/article_screen.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:http/http.dart' as http;
+import 'package:http/testing.dart';
+import 'package:shared_preferences/shared_preferences.dart';
+
+import 'support/fake_app.dart';
+
+/// A plain PNG of the given size, standing in for a comic strip or meme.
+Future<List<int>> _png(int width, int height) async {
+  final recorder = ui.PictureRecorder();
+  Canvas(recorder).drawRect(
+      Rect.fromLTWH(0, 0, width.toDouble(), height.toDouble()), Paint()..color = Colors.black);
+  final image = await recorder.endRecording().toImage(width, height);
+  return (await image.toByteData(format: ui.ImageByteFormat.png))!.buffer.asUint8List();
+}
+
+/// Serves a [width] x [height] PNG from this machine, so the reader view can
+/// load and lay out a real picture. Returns its URL.
+Future<String> _serveImage(WidgetTester tester, int width, int height) async {
+  HttpOverrides.global = null;
+  final server = (await tester.runAsync(() => HttpServer.bind(InternetAddress.loopbackIPv4, 0)))!;
+  addTearDown(() => tester.runAsync(server.close));
+  final png = (await tester.runAsync(() => _png(width, height)))!;
+  server.listen((request) => request.response
+    // No idle keep-alive timer left behind in the test's fake time.
+    ..persistentConnection = false
+    ..headers.contentType = ContentType('image', 'png')
+    ..add(png)
+    ..close());
+  return 'http://127.0.0.1:${server.port}/picture.png';
+}
+
+/// Opens [article] in the reader, with the article page answered by [page],
+/// and waits until the picture at [imageUrl] has loaded. Returns its finder.
+Future<Finder> _openAndLoad(
+    WidgetTester tester, Article article, String imageUrl, http.Response page) async {
+  final state = (await tester.runAsync(() => launchApp([article.source],
+      extractor: ArticleExtractor(client: MockClient((_) async => page)))))!;
+  await tester.pumpWidget(MaterialApp(home: ArticleScreen(state: state, article: article)));
+  final picture = find.descendant(
+      of: find.byWidgetPredicate((w) =>
+          w is Image && w.image is NetworkImage && (w.image as NetworkImage).url == imageUrl),
+      matching: find.byType(RawImage));
+  for (var i = 0; i < 100; i++) {
+    await tester.runAsync(() => Future<void>.delayed(const Duration(milliseconds: 20)));
+    await tester.pump();
+    if (picture.evaluate().isNotEmpty && tester.widget<RawImage>(picture).image != null) break;
+  }
+  return picture;
+}
+
+double _aspect(WidgetTester tester, Finder picture) {
+  final size = tester.getSize(picture);
+  return size.height / size.width;
+}
+
+const _titleText = 'The title text is the last joke, hidden in the mouse-over.';
+
+void main() {
+  setUp(() => SharedPreferences.setMockInitialValues({}));
+
+  testWidgets('opening an xkcd comic shows the whole comic and its title text', (tester) async {
+    final comicUrl = await _serveImage(tester, 300, 100);
+    final page = '''
+<html><body>
+<div id="ctitle">Strip</div>
+<div id="comic"><img src="$comicUrl" title="$_titleText" alt="Strip"></div>
+<div id="licenseText"><p>This work is licensed under a Creative Commons Attribution-NonCommercial 2.5 License.</p></div>
+</body></html>''';
+    // What the xkcd feed gives: a title, a link and the comic as the image.
+    final article = Article(
+        title: 'Strip',
+        link: 'https://xkcd.com/9999/',
+        source: testSource('xkcd', SourceKind.fun),
+        imageUrl: comicUrl);
+
+    final comic = await _openAndLoad(tester, article, comicUrl,
+        http.Response(page, 200, headers: {'content-type': 'text/html; charset=utf-8'}));
+
+    expect(tester.widget<RawImage>(comic).image, isNotNull, reason: 'the comic loads');
+    expect(_aspect(tester, comic), closeTo(100 / 300, 0.01), reason: 'the comic is not cropped');
+    expect(find.text(_titleText), findsOneWidget);
+    expect(find.textContaining("couldn't be loaded"), findsNothing);
+  });
+
+  testWidgets('a meme whose page blocks the reader still shows the picture from the feed',
+      (tester) async {
+    final memeUrl = await _serveImage(tester, 100, 200);
+    final article = Article(
+        title: 'vibeCoding',
+        link: 'https://www.reddit.com/r/ProgrammerHumor/comments/1/vibecoding/',
+        source: testSource('programmerhumor', SourceKind.fun),
+        imageUrl: memeUrl);
+
+    final meme = await _openAndLoad(tester, article, memeUrl, http.Response('Blocked', 403));
+
+    expect(tester.widget<RawImage>(meme).image, isNotNull, reason: 'the meme loads');
+    expect(_aspect(tester, meme), closeTo(200 / 100, 0.01), reason: 'the meme is not cropped');
+    expect(find.textContaining("couldn't be loaded"), findsNothing);
+  });
+}
@@ -0,0 +1,102 @@
+import 'package:daily_tech_news/data/default_sources.dart';
+import 'package:daily_tech_news/models/article.dart';
+import 'package:daily_tech_news/models/briefing.dart';
+import 'package:daily_tech_news/models/news_source.dart';
+import 'package:daily_tech_news/services/summarizer.dart';
+import 'package:daily_tech_news/state/app_state.dart';
+import 'package:daily_tech_news/ui/feed_screen.dart';
+import 'package:daily_tech_news/ui/learn_screen.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:shared_preferences/shared_preferences.dart';
+
+import 'support/fake_app.dart';
+
+/// Remembers which articles a Briefing was asked for.
+class _RecordingSummarizer implements Summarizer {
+  List<Article>? briefingInput;
+
+  @override
+  String get name => 'test';
+
+  @override
+  Future<String> summarizeArticle(Article article, String text, SourceLanguage language) async =>
+      '';
+
+  @override
+  Future<DailyBriefing> createBriefing(List<Article> articles, SourceLanguage language) async {
+    briefingInput = articles;
+    return DailyBriefing(createdAt: DateTime.now(), intro: '', topics: const [], generatedBy: name);
+  }
+}
+
+void main() {
+  final verge = testSource('verge', SourceKind.news);
+  final bytebytego = testSource('bytebytego', SourceKind.learning);
+  final xkcd = testSource('xkcd', SourceKind.fun);
+  final feeds = {
+    verge.feedUrl: [('News from this morning', const Duration(hours: 2))],
+    bytebytego.feedUrl: [('Explainer from 2 days ago', const Duration(days: 2))],
+    xkcd.feedUrl: [('Comic from today', const Duration(hours: 5)),
+      ('Comic from 2 days ago', const Duration(days: 2)),
+      ('Comic from 4 days ago', const Duration(days: 4))],
+  };
+
+  setUp(() => SharedPreferences.setMockInitialValues({}));
+
+  Future<AppState> launch(WidgetTester tester, {Summarizer? summarizer}) async =>
+      (await tester.runAsync(() => launchApp([verge, bytebytego, xkcd],
+          feeds: feeds, summarizer: summarizer)))!;
+
+  double top(WidgetTester tester, String text) => tester.getTopLeft(find.text(text)).dy;
+
+  test('the curated Fun sources are xkcd and The Daily WTF on, r/ProgrammerHumor off', () {
+    final fun = {
+      for (final s in defaultSources)
+        if (s.kind == SourceKind.fun) s.name: (s.feedUrl, s.language, s.enabledByDefault),
+    };
+
+    expect(fun, {
+      'xkcd': ('https://xkcd.com/atom.xml', SourceLanguage.en, true),
+      'The Daily WTF': ('https://thedailywtf.com/rss', SourceLanguage.en, true),
+      'r/ProgrammerHumor': (
+        'https://www.reddit.com/r/ProgrammerHumor/top/.rss?t=day',
+        SourceLanguage.en,
+        false,
+      ),
+    });
+  });
+
+  testWidgets('the Learn tab has a Fun section below Learning with the last 3 days of Fun',
+      (tester) async {
+    final state = await launch(tester);
+
+    await tester.pumpWidget(MaterialApp(home: LearnScreen(state: state)));
+
+    expect(find.text('Fun'), findsOneWidget);
+    expect(find.text('2 articles · last 3 days'), findsOneWidget);
+    expect(top(tester, 'Learning'), lessThan(top(tester, 'Fun')));
+    expect(top(tester, 'Explainer from 2 days ago'), lessThan(top(tester, 'Fun')));
+    expect(top(tester, 'Fun'), lessThan(top(tester, 'Comic from today')));
+    expect(top(tester, 'Comic from today'), lessThan(top(tester, 'Comic from 2 days ago')));
+    expect(find.text('Comic from 4 days ago'), findsNothing);
+  });
+
+  testWidgets('Today leaves out Fun articles', (tester) async {
+    final state = await launch(tester);
+
+    await tester.pumpWidget(MaterialApp(home: FeedScreen(state: state)));
+
+    expect(find.text('News from this morning'), findsOneWidget);
+    expect(find.text('Comic from today'), findsNothing);
+  });
+
+  testWidgets('the Briefing leaves out Fun articles', (tester) async {
+    final summarizer = _RecordingSummarizer();
+    final state = await launch(tester, summarizer: summarizer);
+
+    await tester.runAsync(state.generateBriefing);
+
+    expect(summarizer.briefingInput!.map((a) => a.title), ['News from this morning']);
+  });
+}
@@ -88,6 +88,7 @@ void main() {
      'heise', 'verge', 'mit-tr', 'new-source',
      // Added to the catalog later, on by default.
      'techmeme', 'lobsters', 'the-decoder', 'simonwillison', 'bytebytego',
+      'xkcd', 'thedailywtf',
    });
  });

@@ -103,6 +104,7 @@ void main() {
    expect(state.enabledSourceIds, {
      'heise', 'golem', 't3n', 'verge',
      'techmeme', 'lobsters', 'the-decoder', 'simonwillison', 'bytebytego',
+      'xkcd', 'thedailywtf',
    });
    expect(state.enabledSources.map((s) => s.id), isNot(contains('wired')));
  });
@@ -1,5 +1,6 @@
 import 'package:daily_tech_news/models/article.dart';
 import 'package:daily_tech_news/models/news_source.dart';
+import 'package:daily_tech_news/services/article_extractor.dart';
 import 'package:daily_tech_news/services/feed_service.dart';
 import 'package:daily_tech_news/services/storage.dart';
 import 'package:daily_tech_news/services/summarizer.dart';
@@ -58,11 +59,13 @@ Future<AppState> launchApp(
  Map<String, List<(String, Duration)>> feeds = const {},
  FeedService? feedService,
  Summarizer? summarizer,
+  ArticleExtractor? extractor,
 }) async {
  final state = AppState(await Storage.open(),
      feedService: feedService ?? FakeFeeds(feeds),
      curatedSources: curatedSources,
-      summarizer: summarizer);
+      summarizer: summarizer,
+      extractor: extractor);
  await state.init();
  return state;
 }