Hide paid Substack newsletter posts with the paid filter (#7)

Substack cuts paid-subscriber posts short in the feed and ends their
content with a "Read more" link to the post itself. PaywallService.isPaid
now recognizes that ending, so "hide paid articles" hides these posts in
Today and the Learn tab and counts them as hidden, without a page request.
Free posts that ask readers to subscribe or end with the subscribe widget
are not affected.

Tests use trimmed real items from the Interconnects and Ahead of AI feeds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -25,7 +25,9 @@ or write a daily briefing. It's meant to replace the n8n → Discord workflow.
 - **Paid articles and ads hidden**: heise+ and Golem `(g+)` are recognized by
   their title; SPIEGEL+ and t3n Pro by a marker in the article page, which the app
   checks once per new article in the background (it reads only until the marker,
-  and remembers results for 14 days). Sponsored posts (`Anzeige:`) are hidden too.
+  and remembers results for 14 days). Paid-subscriber posts from Substack newsletters
+  (Interconnects, Ahead of AI, …) are recognized in the feed itself, where Substack
+  cuts them off with a "Read more" link. Sponsored posts (`Anzeige:`) are hidden too.
   The feed header shows how many were hidden; both filters can be turned off.
 - **Ad and cookie-banner blocking** in the original-page view: requests to ~80
   ad/tracker hosts and the common consent managers (Sourcepoint, OneTrust,

lib/services/paywall_service.dart

@@ -9,6 +9,8 @@ import 'storage.dart';
 /// Recognizes paid and sponsored articles so the feed can hide them.
 ///
 /// Most paid articles are marked in the title (heise+, Golem's "(g+)").
+/// Substack newsletters cut paid posts short in the feed and end them with a
+/// "Read more" link to the post itself, so those are recognized from the feed.
 /// SPIEGEL and t3n only mark them in the page's schema.org data
 /// (`"isAccessibleForFree": false`), so for those sources each new article is
 /// checked once in the background and the result is remembered.
@@ -33,6 +35,14 @@ class PaywallService {
     caseSensitive: false,
   );
 
+  /// What Substack appends to a paid post's feed content in place of the rest:
+  /// `<p><a href="{post url}">Read more</a></p>`, as the very end. Free posts
+  /// end with their own text, even when they ask readers to subscribe.
+  static final _substackCut = RegExp(
+    r'<p>\s*<a href="([^"]+)">\s*Read more\s*</a>\s*</p>\s*$',
+    caseSensitive: false,
+  );
+
   static final _marker =
       RegExp(r'''"isAccessibleForFree"\s*:\s*"?(true|false)"?''', caseSensitive: false);
 
@@ -41,7 +51,11 @@ class PaywallService {
   static const _concurrency = 6;
   static const _keepFor = Duration(days: 14);
 
-  bool isPaid(Article a) => paidTitle.hasMatch(a.title) || (_checks[a.link]?.paid ?? false);
+  bool isPaid(Article a) =>
+      paidTitle.hasMatch(a.title) || _isCutSubstackPost(a) || (_checks[a.link]?.paid ?? false);
+
+  static bool _isCutSubstackPost(Article a) =>
+      _substackCut.firstMatch(a.contentHtml)?.group(1) == a.link;
 
   bool isSponsored(Article a) => sponsoredTitle.hasMatch(a.title);
 

test/fixtures/ahead_of_ai_feed.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0"><channel><title><![CDATA[Ahead of AI]]></title><link>https://magazine.sebastianraschka.com</link><generator>Substack</generator>
+<!-- Real items from https://magazine.sebastianraschka.com/feed (October 2026), content:encoded trimmed to its first paragraph and its end. -->
+<!-- Paid post (audience "only_paid" on its page): Substack cuts the content and ends it with a "Read more" link to the post. -->
+<item><title><![CDATA[Building an AI Text Detector From Scratch]]></title><description><![CDATA[An End-to-End Project With Dataset Construction, Model Training, Local Deployment, and RLVR]]></description><link>https://magazine.sebastianraschka.com/p/ai-detector-from-scratch</link><guid isPermaLink="false">https://magazine.sebastianraschka.com/p/ai-detector-from-scratch</guid><dc:creator><![CDATA[Sebastian Raschka, PhD]]></dc:creator><pubDate>Sat, 15 Aug 2026 11:54:24 GMT</pubDate><content:encoded><![CDATA[<p>Substack recently launched its AI detector feature in the UI, which is super interesting.</p><p>For this, we are going to fine-tune a DistilBERT classifier (similar to what I described in one of my early Substack articles, <a href="https://magazine.sebastianraschka.com/p/finetuning-large-language-models">Finetuning Large Language Models</a>), but more details on that later when we get to that stage.</p><p></p>
+      <p>
+          <a href="https://magazine.sebastianraschka.com/p/ai-detector-from-scratch">
+              Read more
+          </a>
+      </p>
+   ]]></content:encoded></item>
+<!-- Free post (audience "everyone") whose footer asks readers to become paid subscribers. -->
+<item><title><![CDATA[Controlling Reasoning Effort in LLMs]]></title><description><![CDATA[How LLMs Learn Low-, Medium-, and High-Effort Reasoning Modes]]></description><link>https://magazine.sebastianraschka.com/p/controlling-reasoning-effort-in-llms</link><guid isPermaLink="false">https://magazine.sebastianraschka.com/p/controlling-reasoning-effort-in-llms</guid><dc:creator><![CDATA[Sebastian Raschka, PhD]]></dc:creator><pubDate>Sat, 18 Jul 2026 11:16:09 GMT</pubDate><content:encoded><![CDATA[<p><span>It has been almost two years since OpenAI released o1, a model that popularized the idea of LLM-based reasoning models. DeepSeek-R1 followed about four months later, together with details of a reinforcement learning with verifiable rewards (RLVR) recipe to train such reasoning models.</span></p><p>If you liked my previous <a href="https://amzn.to/4fqvn0D">Build a Large Language Model (From Scratch)</a> book, this is essentially a sequel implementing inference-time scaling techniques and reinforcement learning algorithms from scratch.</p><p>And if you want to support future long-form articles like this one, consider <a href="https://magazine.sebastianraschka.com/subscribe">becoming a paid subscriber</a>. It helps me keep writing these independent deep dives and sharing the accompanying code, figures, and experiments.</p><p></p>]]></content:encoded></item>
+<!-- Free post (audience "everyone") that ends with Substack's subscribe widget. -->
+<item><title><![CDATA[GPT-6 Astra, Looped Transformers, and Hidden Reasoning]]></title><description><![CDATA[A Look at Recurrent Depth, Hidden Chains of Thought, and Recent Research on Looping Transformer Blocks]]></description><link>https://magazine.sebastianraschka.com/p/gpt-6-astra-looped-transformers-and</link><guid isPermaLink="false">https://magazine.sebastianraschka.com/p/gpt-6-astra-looped-transformers-and</guid><dc:creator><![CDATA[Sebastian Raschka, PhD]]></dc:creator><pubDate>Wed, 09 Sep 2026 11:14:25 GMT</pubDate><content:encoded><![CDATA[<p>A lot has happened in the last few weeks. I am sure that OpenAI&#8217;s GPT-6 Astra is top of mind for everyone right now. In particular, thoughts on its performance, the looped transformer/recurrent depth aspects, and rumors that Astra is &#8220;hiding&#8221; its reasoning trace (i.e., chain of thought).</p><div class="subscription-widget-wrap-editor" data-attrs="{&quot;url&quot;:&quot;https://magazine.sebastianraschka.com/subscribe?&quot;,&quot;text&quot;:&quot;Subscribe&quot;,&quot;language&quot;:&quot;en&quot;}" data-component-name="SubscribeWidgetToDOM"><div class="subscription-widget show-subscribe"><div class="preamble"><p class="cta-caption"></p></div><form class="subscription-widget-subscribe"><input type="email" class="email-input" name="email" placeholder="Type your email&#8230;" tabindex="-1"><input type="submit" class="button primary" value="Subscribe"><div class="fake-input-wrapper"><div class="fake-input"></div><div class="fake-button"></div></div></form></div></div>]]></content:encoded></item>
+</channel></rss>

test/fixtures/interconnects_feed.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0"><channel><title><![CDATA[Interconnects AI]]></title><link>https://www.interconnects.ai</link><generator>Substack</generator>
+<!-- Real items from https://www.interconnects.ai/feed (October 2026), content:encoded trimmed to its first paragraph and its end. -->
+<!-- Paid post (audience "only_paid" on its page). -->
+<item><title><![CDATA[Lessons from the hacks]]></title><description><![CDATA[Musings on model alignment, what determines safety, and where we go from here.]]></description><link>https://www.interconnects.ai/p/lessons-from-the-hacks</link><guid isPermaLink="false">https://www.interconnects.ai/p/lessons-from-the-hacks</guid><dc:creator><![CDATA[Nathan Lambert]]></dc:creator><pubDate>Sun, 09 Aug 2026 14:57:11 GMT</pubDate><content:encoded><![CDATA[<p><span>The recent run of cyberattacks by in-development frontier models has got me thinking a lot about how our current incentive systems are not well suited for such fast technological transitions.</span></p><div><hr></div><h4>Book sale</h4><p><span>I&#8217;m also hosting a book launch where you can get a free signed copy tomorrow from 5-8PM in Seattle (Fremont/Ballard area) &#8211; we still have some extra space so I&#8217;m opening signups to paid subscribers below the paywall:</span></p>
+      <p>
+          <a href="https://www.interconnects.ai/p/lessons-from-the-hacks">
+              Read more
+          </a>
+      </p>
+   ]]></content:encoded></item>
+<!-- Free post (audience "everyone") that mentions paid subscribers and ends with an embedded post's "Read more". -->
+<item><title><![CDATA[When will average people feel AI’s impact?]]></title><description><![CDATA[We&#8217;re <5 years into a compounding revolution which could take a century, and how the AI industry should manage this.]]></description><link>https://www.interconnects.ai/p/when-will-average-people-feel-ais</link><guid isPermaLink="false">https://www.interconnects.ai/p/when-will-average-people-feel-ais</guid><dc:creator><![CDATA[Nathan Lambert]]></dc:creator><pubDate>Wed, 09 Sep 2026 11:01:23 GMT</pubDate><content:encoded><![CDATA[<h5>Housekeeping: Paid subscribers to Interconnects now get a permanent 40% discount on my book when purchasing at Manning.com. Access the code at the Interconnects <a href="https://www.interconnects.ai/perks">perks page</a>.</h5><p>Many AI optimists tend to compare what is happening in this AI boom to the industrial revolution, or to other periods of rapid technological advancement and diffusion into society.</p><div><a class="embedded-post" native="true" href="https://www.fabricatedknowledge.com/p/mythos-and-engels-pause?utm_source=substack&amp;utm_campaign=post_embed&amp;utm_medium=web&amp;embedding_publication_id=48206"><div class="embedded-post-header"><span class="embedded-post-publication-name">Fabricated Knowledge</span></div><div class="embedded-post-title-wrapper"><div class="embedded-post-title">Engels' Pause and the Permanent Underclass </div></div><div class="embedded-post-cta-wrapper"><span class="embedded-post-cta">Read more</span></div><div class="embedded-post-meta">6 months ago &#183; 214 likes &#183; 5 comments &#183; Doug O'Laughlin</div></a></div>]]></content:encoded></item>
+</channel></rss>

test/paid_newsletter_posts_test.dart

@@ -0,0 +1,75 @@
+import 'dart:io';
+
+import 'package:daily_tech_news/models/article.dart';
+import 'package:daily_tech_news/models/news_source.dart';
+import 'package:daily_tech_news/services/feed_parser.dart';
+import 'package:daily_tech_news/services/feed_service.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:shared_preferences/shared_preferences.dart';
+
+import 'support/fake_app.dart';
+
+/// Serves saved Substack feeds (trimmed real items) instead of the network,
+/// dated an hour ago so every time window keeps them.
+class _SavedFeeds extends FeedService {
+  _SavedFeeds(this.fixtureByFeedUrl);
+
+  final Map<String, String> fixtureByFeedUrl;
+
+  @override
+  Future<List<Article>> fetchSource(NewsSource source, {String corsProxy = ''}) async {
+    final xml = File(fixtureByFeedUrl[source.feedUrl]!).readAsStringSync();
+    final anHourAgo = DateTime.now().toUtc().subtract(const Duration(hours: 1));
+    return [
+      for (final a in FeedParser.parse(xml, source))
+        Article(
+          title: a.title,
+          link: a.link,
+          source: a.source,
+          published: anHourAgo,
+          excerpt: a.excerpt,
+          contentHtml: a.contentHtml,
+        ),
+    ];
+  }
+}
+
+List<String> _titles(List<Article> articles) => articles.map((a) => a.title).toList();
+
+void main() {
+  final interconnects = testSource('interconnects', SourceKind.learning);
+  final substackNews = testSource('substack-news', SourceKind.news);
+
+  setUp(() => SharedPreferences.setMockInitialValues({}));
+
+  _SavedFeeds savedFeeds() => _SavedFeeds({
+        interconnects.feedUrl: 'test/fixtures/interconnects_feed.xml',
+        substackNews.feedUrl: 'test/fixtures/ahead_of_ai_feed.xml',
+      });
+
+  test('paid newsletter posts are hidden in the Learn tab and Today and counted as hidden',
+      () async {
+    final state = await launchApp([interconnects, substackNews], feedService: savedFeeds());
+
+    expect(_titles(state.articlesOf(SourceKind.learning)),
+        ['When will average people feel AI’s impact?']);
+    expect(state.hiddenCountOf(SourceKind.learning), 1);
+    expect(_titles(state.articlesOf(SourceKind.news)), [
+      'Controlling Reasoning Effort in LLMs',
+      'GPT-6 Astra, Looped Transformers, and Hidden Reasoning',
+    ]);
+    expect(state.hiddenCountOf(SourceKind.news), 1);
+  });
+
+  test('with "hide paid articles" off, paid newsletter posts show as before', () async {
+    final state = await launchApp([interconnects, substackNews], feedService: savedFeeds());
+
+    await state.updateSettings(state.settings.copyWith(hidePaid: false));
+
+    expect(_titles(state.articlesOf(SourceKind.learning)).toSet(),
+        {'Lessons from the hacks', 'When will average people feel AI’s impact?'});
+    expect(state.hiddenCountOf(SourceKind.learning), 0);
+    expect(_titles(state.articlesOf(SourceKind.news)), contains('Building an AI Text Detector From Scratch'));
+    expect(state.hiddenCountOf(SourceKind.news), 0);
+  });
+}

test/paywall_service_test.dart

@@ -1,5 +1,8 @@
+import 'dart:io';
+
 import 'package:daily_tech_news/models/article.dart';
 import 'package:daily_tech_news/models/news_source.dart';
+import 'package:daily_tech_news/services/feed_parser.dart';
 import 'package:daily_tech_news/services/paywall_service.dart';
 import 'package:daily_tech_news/services/storage.dart';
 import 'package:flutter_test/flutter_test.dart';
@@ -38,6 +41,21 @@ void main() {
     expect(paywall.isSponsored(_article('Anzeigenmarkt bricht ein')), isFalse);
   });
 
+  test('recognizes paid Substack posts by the cut-off content in the feed', () async {
+    final paywall = PaywallService(await _storage(), client: MockClient((_) async => http.Response('', 500)));
+    final posts = {
+      for (final a in FeedParser.parse(
+          File('test/fixtures/ahead_of_ai_feed.xml').readAsStringSync(), _source()))
+        a.title: a,
+    };
+
+    expect(paywall.isPaid(posts['Building an AI Text Detector From Scratch']!), isTrue);
+    expect(paywall.isPaid(posts['Controlling Reasoning Effort in LLMs']!), isFalse,
+        reason: 'a free post asking readers to become paid subscribers');
+    expect(paywall.isPaid(posts['GPT-6 Astra, Looped Transformers, and Hidden Reasoning']!), isFalse,
+        reason: 'a free post ending with the subscribe widget');
+  });
+
   test('checks pages only for flagged sources and remembers results', () async {
     final storage = await _storage();
     final requested = <String>[];