From 84cb11e4dd80076294b6bad7a6f7c0864de579c7 Mon Sep 17 00:00:00 2001 From: Cedric Hornberger Date: Fri, 16 Oct 2026 15:57:48 +0200 Subject: [PATCH] Hide paid Substack newsletter posts with the paid filter (#7) Substack cuts paid-subscriber posts short in the feed and ends their content with a "Read more" link to the post itself. PaywallService.isPaid now recognizes that ending, so "hide paid articles" hides these posts in Today and the Learn tab and counts them as hidden, without a page request. Free posts that ask readers to subscribe or end with the subscribe widget are not affected. Tests use trimmed real items from the Interconnects and Ahead of AI feeds. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- lib/services/paywall_service.dart | 16 +++++- test/fixtures/ahead_of_ai_feed.xml | 15 ++++++ test/fixtures/interconnects_feed.xml | 13 +++++ test/paid_newsletter_posts_test.dart | 75 ++++++++++++++++++++++++++++ test/paywall_service_test.dart | 18 +++++++ 6 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/ahead_of_ai_feed.xml create mode 100644 test/fixtures/interconnects_feed.xml create mode 100644 test/paid_newsletter_posts_test.dart diff --git a/README.md b/README.md index 2b4b972..187a45a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,9 @@ or write a daily briefing. It's meant to replace the n8n → Discord workflow. - **Paid articles and ads hidden**: heise+ and Golem `(g+)` are recognized by their title; SPIEGEL+ and t3n Pro by a marker in the article page, which the app checks once per new article in the background (it reads only until the marker, - and remembers results for 14 days). Sponsored posts (`Anzeige:`) are hidden too. + and remembers results for 14 days). Paid-subscriber posts from Substack newsletters + (Interconnects, Ahead of AI, …) are recognized in the feed itself, where Substack + cuts them off with a "Read more" link. Sponsored posts (`Anzeige:`) are hidden too. The feed header shows how many were hidden; both filters can be turned off. - **Ad and cookie-banner blocking** in the original-page view: requests to ~80 ad/tracker hosts and the common consent managers (Sourcepoint, OneTrust, diff --git a/lib/services/paywall_service.dart b/lib/services/paywall_service.dart index c4d9e7d..fa0e1f0 100644 --- a/lib/services/paywall_service.dart +++ b/lib/services/paywall_service.dart @@ -9,6 +9,8 @@ import 'storage.dart'; /// Recognizes paid and sponsored articles so the feed can hide them. /// /// Most paid articles are marked in the title (heise+, Golem's "(g+)"). +/// Substack newsletters cut paid posts short in the feed and end them with a +/// "Read more" link to the post itself, so those are recognized from the feed. /// SPIEGEL and t3n only mark them in the page's schema.org data /// (`"isAccessibleForFree": false`), so for those sources each new article is /// checked once in the background and the result is remembered. @@ -33,6 +35,14 @@ class PaywallService { caseSensitive: false, ); + /// What Substack appends to a paid post's feed content in place of the rest: + /// `

Read more

`, as the very end. Free posts + /// end with their own text, even when they ask readers to subscribe. + static final _substackCut = RegExp( + r'

\s*\s*Read more\s*\s*

\s*$', + caseSensitive: false, + ); + static final _marker = RegExp(r'''"isAccessibleForFree"\s*:\s*"?(true|false)"?''', caseSensitive: false); @@ -41,7 +51,11 @@ class PaywallService { static const _concurrency = 6; static const _keepFor = Duration(days: 14); - bool isPaid(Article a) => paidTitle.hasMatch(a.title) || (_checks[a.link]?.paid ?? false); + bool isPaid(Article a) => + paidTitle.hasMatch(a.title) || _isCutSubstackPost(a) || (_checks[a.link]?.paid ?? false); + + static bool _isCutSubstackPost(Article a) => + _substackCut.firstMatch(a.contentHtml)?.group(1) == a.link; bool isSponsored(Article a) => sponsoredTitle.hasMatch(a.title); diff --git a/test/fixtures/ahead_of_ai_feed.xml b/test/fixtures/ahead_of_ai_feed.xml new file mode 100644 index 0000000..deee5a3 --- /dev/null +++ b/test/fixtures/ahead_of_ai_feed.xml @@ -0,0 +1,15 @@ +<![CDATA[Ahead of AI]]>https://magazine.sebastianraschka.comSubstack + + +<![CDATA[Building an AI Text Detector From Scratch]]>https://magazine.sebastianraschka.com/p/ai-detector-from-scratchhttps://magazine.sebastianraschka.com/p/ai-detector-from-scratchSat, 15 Aug 2026 11:54:24 GMTSubstack recently launched its AI detector feature in the UI, which is super interesting.

For this, we are going to fine-tune a DistilBERT classifier (similar to what I described in one of my early Substack articles, Finetuning Large Language Models), but more details on that later when we get to that stage.

+

+ + Read more + +

+ ]]> + +<![CDATA[Controlling Reasoning Effort in LLMs]]>https://magazine.sebastianraschka.com/p/controlling-reasoning-effort-in-llmshttps://magazine.sebastianraschka.com/p/controlling-reasoning-effort-in-llmsSat, 18 Jul 2026 11:16:09 GMTIt has been almost two years since OpenAI released o1, a model that popularized the idea of LLM-based reasoning models. DeepSeek-R1 followed about four months later, together with details of a reinforcement learning with verifiable rewards (RLVR) recipe to train such reasoning models.

If you liked my previous Build a Large Language Model (From Scratch) book, this is essentially a sequel implementing inference-time scaling techniques and reinforcement learning algorithms from scratch.

And if you want to support future long-form articles like this one, consider becoming a paid subscriber. It helps me keep writing these independent deep dives and sharing the accompanying code, figures, and experiments.

]]> + +<![CDATA[GPT-6 Astra, Looped Transformers, and Hidden Reasoning]]>https://magazine.sebastianraschka.com/p/gpt-6-astra-looped-transformers-andhttps://magazine.sebastianraschka.com/p/gpt-6-astra-looped-transformers-andWed, 09 Sep 2026 11:14:25 GMTA lot has happened in the last few weeks. I am sure that OpenAI’s GPT-6 Astra is top of mind for everyone right now. In particular, thoughts on its performance, the looped transformer/recurrent depth aspects, and rumors that Astra is “hiding” its reasoning trace (i.e., chain of thought).

]]> + diff --git a/test/fixtures/interconnects_feed.xml b/test/fixtures/interconnects_feed.xml new file mode 100644 index 0000000..a161bfb --- /dev/null +++ b/test/fixtures/interconnects_feed.xml @@ -0,0 +1,13 @@ +<![CDATA[Interconnects AI]]>https://www.interconnects.aiSubstack + + +<![CDATA[Lessons from the hacks]]>https://www.interconnects.ai/p/lessons-from-the-hackshttps://www.interconnects.ai/p/lessons-from-the-hacksSun, 09 Aug 2026 14:57:11 GMTThe recent run of cyberattacks by in-development frontier models has got me thinking a lot about how our current incentive systems are not well suited for such fast technological transitions.

Book sale

I’m also hosting a book launch where you can get a free signed copy tomorrow from 5-8PM in Seattle (Fremont/Ballard area) – we still have some extra space so I’m opening signups to paid subscribers below the paywall:

+

+ + Read more + +

+ ]]> + +<![CDATA[When will average people feel AI’s impact?]]>https://www.interconnects.ai/p/when-will-average-people-feel-aishttps://www.interconnects.ai/p/when-will-average-people-feel-aisWed, 09 Sep 2026 11:01:23 GMTHousekeeping: Paid subscribers to Interconnects now get a permanent 40% discount on my book when purchasing at Manning.com. Access the code at the Interconnects perks page.

Many AI optimists tend to compare what is happening in this AI boom to the industrial revolution, or to other periods of rapid technological advancement and diffusion into society.

]]> + diff --git a/test/paid_newsletter_posts_test.dart b/test/paid_newsletter_posts_test.dart new file mode 100644 index 0000000..665580d --- /dev/null +++ b/test/paid_newsletter_posts_test.dart @@ -0,0 +1,75 @@ +import 'dart:io'; + +import 'package:daily_tech_news/models/article.dart'; +import 'package:daily_tech_news/models/news_source.dart'; +import 'package:daily_tech_news/services/feed_parser.dart'; +import 'package:daily_tech_news/services/feed_service.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +import 'support/fake_app.dart'; + +/// Serves saved Substack feeds (trimmed real items) instead of the network, +/// dated an hour ago so every time window keeps them. +class _SavedFeeds extends FeedService { + _SavedFeeds(this.fixtureByFeedUrl); + + final Map fixtureByFeedUrl; + + @override + Future> fetchSource(NewsSource source, {String corsProxy = ''}) async { + final xml = File(fixtureByFeedUrl[source.feedUrl]!).readAsStringSync(); + final anHourAgo = DateTime.now().toUtc().subtract(const Duration(hours: 1)); + return [ + for (final a in FeedParser.parse(xml, source)) + Article( + title: a.title, + link: a.link, + source: a.source, + published: anHourAgo, + excerpt: a.excerpt, + contentHtml: a.contentHtml, + ), + ]; + } +} + +List _titles(List
articles) => articles.map((a) => a.title).toList(); + +void main() { + final interconnects = testSource('interconnects', SourceKind.learning); + final substackNews = testSource('substack-news', SourceKind.news); + + setUp(() => SharedPreferences.setMockInitialValues({})); + + _SavedFeeds savedFeeds() => _SavedFeeds({ + interconnects.feedUrl: 'test/fixtures/interconnects_feed.xml', + substackNews.feedUrl: 'test/fixtures/ahead_of_ai_feed.xml', + }); + + test('paid newsletter posts are hidden in the Learn tab and Today and counted as hidden', + () async { + final state = await launchApp([interconnects, substackNews], feedService: savedFeeds()); + + expect(_titles(state.articlesOf(SourceKind.learning)), + ['When will average people feel AI’s impact?']); + expect(state.hiddenCountOf(SourceKind.learning), 1); + expect(_titles(state.articlesOf(SourceKind.news)), [ + 'Controlling Reasoning Effort in LLMs', + 'GPT-6 Astra, Looped Transformers, and Hidden Reasoning', + ]); + expect(state.hiddenCountOf(SourceKind.news), 1); + }); + + test('with "hide paid articles" off, paid newsletter posts show as before', () async { + final state = await launchApp([interconnects, substackNews], feedService: savedFeeds()); + + await state.updateSettings(state.settings.copyWith(hidePaid: false)); + + expect(_titles(state.articlesOf(SourceKind.learning)).toSet(), + {'Lessons from the hacks', 'When will average people feel AI’s impact?'}); + expect(state.hiddenCountOf(SourceKind.learning), 0); + expect(_titles(state.articlesOf(SourceKind.news)), contains('Building an AI Text Detector From Scratch')); + expect(state.hiddenCountOf(SourceKind.news), 0); + }); +} diff --git a/test/paywall_service_test.dart b/test/paywall_service_test.dart index 64142c1..0d2ba19 100644 --- a/test/paywall_service_test.dart +++ b/test/paywall_service_test.dart @@ -1,5 +1,8 @@ +import 'dart:io'; + import 'package:daily_tech_news/models/article.dart'; import 'package:daily_tech_news/models/news_source.dart'; +import 'package:daily_tech_news/services/feed_parser.dart'; import 'package:daily_tech_news/services/paywall_service.dart'; import 'package:daily_tech_news/services/storage.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -38,6 +41,21 @@ void main() { expect(paywall.isSponsored(_article('Anzeigenmarkt bricht ein')), isFalse); }); + test('recognizes paid Substack posts by the cut-off content in the feed', () async { + final paywall = PaywallService(await _storage(), client: MockClient((_) async => http.Response('', 500))); + final posts = { + for (final a in FeedParser.parse( + File('test/fixtures/ahead_of_ai_feed.xml').readAsStringSync(), _source())) + a.title: a, + }; + + expect(paywall.isPaid(posts['Building an AI Text Detector From Scratch']!), isTrue); + expect(paywall.isPaid(posts['Controlling Reasoning Effort in LLMs']!), isFalse, + reason: 'a free post asking readers to become paid subscribers'); + expect(paywall.isPaid(posts['GPT-6 Astra, Looped Transformers, and Hidden Reasoning']!), isFalse, + reason: 'a free post ending with the subscribe widget'); + }); + test('checks pages only for flagged sources and remembers results', () async { final storage = await _storage(); final requested = [];