diff --git a/lib/services/paywall_service.dart b/lib/services/paywall_service.dart index fa0e1f0..93c0ba0 100644 --- a/lib/services/paywall_service.dart +++ b/lib/services/paywall_service.dart @@ -54,8 +54,24 @@ class PaywallService { bool isPaid(Article a) => paidTitle.hasMatch(a.title) || _isCutSubstackPost(a) || (_checks[a.link]?.paid ?? false); - static bool _isCutSubstackPost(Article a) => - _substackCut.firstMatch(a.contentHtml)?.group(1) == a.link; + static bool _isCutSubstackPost(Article a) { + final readMore = _substackCut.firstMatch(a.contentHtml)?.group(1); + return readMore != null && _samePost(readMore, a.link); + } + + /// Whether two links point at the same post. Only host and path count: + /// scheme, host case, a trailing slash, the query (tracking parameters) + /// and the fragment don't. + static bool _samePost(String a, String b) { + String? key(String link) { + final uri = Uri.tryParse(link.trim()); + if (uri == null || uri.host.isEmpty) return null; + return '${uri.host.toLowerCase()}${uri.path.replaceFirst(RegExp(r'/+$'), '')}'; + } + + final keyA = key(a); + return keyA != null && keyA == key(b); + } bool isSponsored(Article a) => sponsoredTitle.hasMatch(a.title); diff --git a/test/paywall_service_test.dart b/test/paywall_service_test.dart index 2f3ecf9..476c7b2 100644 --- a/test/paywall_service_test.dart +++ b/test/paywall_service_test.dart @@ -56,6 +56,31 @@ void main() { reason: 'a free post ending with the subscribe widget'); }); + test('matches the "Read more" link of a paid Substack post to the post despite URL noise', + () async { + final paywall = PaywallService(await _storage(), client: MockClient((_) async => http.Response('', 500))); + Article cutPost({required String link, required String readMore}) => Article( + title: 'A paid post', + link: link, + source: _source(), + contentHtml: '

The first paragraph.

Read more

', + ); + const post = 'https://example.substack.com/p/a-paid-post'; + + expect(paywall.isPaid(cutPost(link: post, readMore: '$post/')), isTrue, + reason: 'trailing slash on the "Read more" link'); + expect(paywall.isPaid(cutPost(link: '$post/', readMore: post)), isTrue, + reason: 'trailing slash on the article link'); + expect(paywall.isPaid(cutPost(link: post, readMore: '$post?utm_source=substack&utm_medium=email')), + isTrue, reason: 'tracking query on the "Read more" link'); + expect(paywall.isPaid(cutPost(link: '$post?utm_campaign=post#footnote-1', readMore: post)), isTrue, + reason: 'query and fragment on the article link'); + expect(paywall.isPaid(cutPost(link: post, readMore: 'http://Example.Substack.com/p/a-paid-post')), + isTrue, reason: 'other scheme and host case'); + expect(paywall.isPaid(cutPost(link: post, readMore: 'https://example.substack.com/p/another-post')), + isFalse, reason: 'a "Read more" link to a different post'); + }); + test('checks pages only for flagged sources and remembers results', () async { final storage = await _storage(); final requested = [];